yt_dlp/extractor/youtube.py

   1 # coding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import calendar
   6 import copy
   7 import datetime
   8 import functools
   9 import hashlib
  10 import itertools
  11 import json
  12 import math
  13 import os.path
  14 import random
  15 import re
  16 import sys
  17 import time
  18 import traceback
  19 import threading
  20
  21 from .common import InfoExtractor, SearchInfoExtractor
  22 from ..compat import (
  23     compat_chr,
  24     compat_HTTPError,
  25     compat_parse_qs,
  26     compat_str,
  27     compat_urllib_parse_unquote_plus,
  28     compat_urllib_parse_urlencode,
  29     compat_urllib_parse_urlparse,
  30     compat_urlparse,
  31 )
  32 from ..jsinterp import JSInterpreter
  33 from ..utils import (
  34     bug_reports_message,
  35     clean_html,
  36     datetime_from_str,
  37     dict_get,
  38     error_to_compat_str,
  39     ExtractorError,
  40     float_or_none,
  41     format_field,
  42     int_or_none,
  43     is_html,
  44     join_nonempty,
  45     mimetype2ext,
  46     network_exceptions,
  47     NO_DEFAULT,
  48     orderedSet,
  49     parse_codecs,
  50     parse_count,
  51     parse_duration,
  52     parse_iso8601,
  53     parse_qs,
  54     qualities,
  55     remove_end,
  56     remove_start,
  57     smuggle_url,
  58     str_or_none,
  59     str_to_int,
  60     strftime_or_none,
  61     traverse_obj,
  62     try_get,
  63     unescapeHTML,
  64     unified_strdate,
  65     unified_timestamp,
  66     unsmuggle_url,
  67     update_url_query,
  68     url_or_none,
  69     urljoin,
  70     variadic,
  71 )
  72
  73
  74 def get_first(obj, keys, **kwargs):
  75     return traverse_obj(obj, (..., *variadic(keys)), **kwargs, get_all=False)
  76
  77
  78 # any clients starting with _ cannot be explicity requested by the user
  79 INNERTUBE_CLIENTS = {
  80     'web': {
  81         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  82         'INNERTUBE_CONTEXT': {
  83             'client': {
  84                 'clientName': 'WEB',
  85                 'clientVersion': '2.20211221.00.00',
  86             }
  87         },
  88         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  89     },
  90     'web_embedded': {
  91         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  92         'INNERTUBE_CONTEXT': {
  93             'client': {
  94                 'clientName': 'WEB_EMBEDDED_PLAYER',
  95                 'clientVersion': '1.20211215.00.01',
  96             },
  97         },
  98         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  99     },
 100     'web_music': {
 101         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
 102         'INNERTUBE_HOST': 'music.youtube.com',
 103         'INNERTUBE_CONTEXT': {
 104             'client': {
 105                 'clientName': 'WEB_REMIX',
 106                 'clientVersion': '1.20211213.00.00',
 107             }
 108         },
 109         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 110     },
 111     'web_creator': {
 112         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 113         'INNERTUBE_CONTEXT': {
 114             'client': {
 115                 'clientName': 'WEB_CREATOR',
 116                 'clientVersion': '1.20211220.02.00',
 117             }
 118         },
 119         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 120     },
 121     'android': {
 122         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 123         'INNERTUBE_CONTEXT': {
 124             'client': {
 125                 'clientName': 'ANDROID',
 126                 'clientVersion': '16.49',
 127             }
 128         },
 129         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 130         'REQUIRE_JS_PLAYER': False
 131     },
 132     'android_embedded': {
 133         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 134         'INNERTUBE_CONTEXT': {
 135             'client': {
 136                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 137                 'clientVersion': '16.49',
 138             },
 139         },
 140         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 141         'REQUIRE_JS_PLAYER': False
 142     },
 143     'android_music': {
 144         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 145         'INNERTUBE_CONTEXT': {
 146             'client': {
 147                 'clientName': 'ANDROID_MUSIC',
 148                 'clientVersion': '4.57',
 149             }
 150         },
 151         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 152         'REQUIRE_JS_PLAYER': False
 153     },
 154     'android_creator': {
 155         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 156         'INNERTUBE_CONTEXT': {
 157             'client': {
 158                 'clientName': 'ANDROID_CREATOR',
 159                 'clientVersion': '21.47',
 160             },
 161         },
 162         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 163         'REQUIRE_JS_PLAYER': False
 164     },
 165     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 166     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 167     'ios': {
 168         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 169         'INNERTUBE_CONTEXT': {
 170             'client': {
 171                 'clientName': 'IOS',
 172                 'clientVersion': '16.46',
 173                 'deviceModel': 'iPhone14,3',
 174             }
 175         },
 176         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 177         'REQUIRE_JS_PLAYER': False
 178     },
 179     'ios_embedded': {
 180         'INNERTUBE_CONTEXT': {
 181             'client': {
 182                 'clientName': 'IOS_MESSAGES_EXTENSION',
 183                 'clientVersion': '16.46',
 184                 'deviceModel': 'iPhone14,3',
 185             },
 186         },
 187         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 188         'REQUIRE_JS_PLAYER': False
 189     },
 190     'ios_music': {
 191         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 192         'INNERTUBE_CONTEXT': {
 193             'client': {
 194                 'clientName': 'IOS_MUSIC',
 195                 'clientVersion': '4.57',
 196             },
 197         },
 198         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 199         'REQUIRE_JS_PLAYER': False
 200     },
 201     'ios_creator': {
 202         'INNERTUBE_CONTEXT': {
 203             'client': {
 204                 'clientName': 'IOS_CREATOR',
 205                 'clientVersion': '21.47',
 206             },
 207         },
 208         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 209         'REQUIRE_JS_PLAYER': False
 210     },
 211     # mweb has 'ultralow' formats
 212     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 213     'mweb': {
 214         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 215         'INNERTUBE_CONTEXT': {
 216             'client': {
 217                 'clientName': 'MWEB',
 218                 'clientVersion': '2.20211221.01.00',
 219             }
 220         },
 221         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 222     }
 223 }
 224
 225
 226 def build_innertube_clients():
 227     third_party = {
 228         'embedUrl': 'https://google.com',  # Can be any valid URL
 229     }
 230     base_clients = ('android', 'web', 'ios', 'mweb')
 231     priority = qualities(base_clients[::-1])
 232
 233     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 234         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 235         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 236         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 237         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 238         ytcfg['priority'] = 10 * priority(client.split('_', 1)[0])
 239
 240         if client in base_clients:
 241             INNERTUBE_CLIENTS[f'{client}_agegate'] = agegate_ytcfg = copy.deepcopy(ytcfg)
 242             agegate_ytcfg['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 243             agegate_ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = third_party
 244             agegate_ytcfg['priority'] -= 1
 245         elif client.endswith('_embedded'):
 246             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = third_party
 247             ytcfg['priority'] -= 2
 248         else:
 249             ytcfg['priority'] -= 3
 250
 251
 252 build_innertube_clients()
 253
 254
 255 class YoutubeBaseInfoExtractor(InfoExtractor):
 256     """Provide base functions for Youtube extractors"""
 257
 258     _RESERVED_NAMES = (
 259         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 260         r'shorts|movies|results|shared|hashtag|trending|explore|feed|feeds|'
 261         r'browse|oembed|get_video_info|iframe_api|s/player|'
 262         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 263
 264     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 265
 266     _NETRC_MACHINE = 'youtube'
 267
 268     # If True it will raise an error if no login info is provided
 269     _LOGIN_REQUIRED = False
 270
 271     _INVIDIOUS_SITES = (
 272         # invidious-redirect websites
 273         r'(?:www\.)?redirect\.invidious\.io',
 274         r'(?:(?:www|dev)\.)?invidio\.us',
 275         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
 276         r'(?:www\.)?invidious\.pussthecat\.org',
 277         r'(?:www\.)?invidious\.zee\.li',
 278         r'(?:www\.)?invidious\.ethibox\.fr',
 279         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 280         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 281         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 282         # youtube-dl invidious instances list
 283         r'(?:(?:www|no)\.)?invidiou\.sh',
 284         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 285         r'(?:www\.)?invidious\.kabi\.tk',
 286         r'(?:www\.)?invidious\.mastodon\.host',
 287         r'(?:www\.)?invidious\.zapashcanon\.fr',
 288         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 289         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 290         r'(?:www\.)?invidious\.himiko\.cloud',
 291         r'(?:www\.)?invidious\.reallyancient\.tech',
 292         r'(?:www\.)?invidious\.tube',
 293         r'(?:www\.)?invidiou\.site',
 294         r'(?:www\.)?invidious\.site',
 295         r'(?:www\.)?invidious\.xyz',
 296         r'(?:www\.)?invidious\.nixnet\.xyz',
 297         r'(?:www\.)?invidious\.048596\.xyz',
 298         r'(?:www\.)?invidious\.drycat\.fr',
 299         r'(?:www\.)?inv\.skyn3t\.in',
 300         r'(?:www\.)?tube\.poal\.co',
 301         r'(?:www\.)?tube\.connect\.cafe',
 302         r'(?:www\.)?vid\.wxzm\.sx',
 303         r'(?:www\.)?vid\.mint\.lgbt',
 304         r'(?:www\.)?vid\.puffyan\.us',
 305         r'(?:www\.)?yewtu\.be',
 306         r'(?:www\.)?yt\.elukerio\.org',
 307         r'(?:www\.)?yt\.lelux\.fi',
 308         r'(?:www\.)?invidious\.ggc-project\.de',
 309         r'(?:www\.)?yt\.maisputain\.ovh',
 310         r'(?:www\.)?ytprivate\.com',
 311         r'(?:www\.)?invidious\.13ad\.de',
 312         r'(?:www\.)?invidious\.toot\.koeln',
 313         r'(?:www\.)?invidious\.fdn\.fr',
 314         r'(?:www\.)?watch\.nettohikari\.com',
 315         r'(?:www\.)?invidious\.namazso\.eu',
 316         r'(?:www\.)?invidious\.silkky\.cloud',
 317         r'(?:www\.)?invidious\.exonip\.de',
 318         r'(?:www\.)?invidious\.riverside\.rocks',
 319         r'(?:www\.)?invidious\.blamefran\.net',
 320         r'(?:www\.)?invidious\.moomoo\.de',
 321         r'(?:www\.)?ytb\.trom\.tf',
 322         r'(?:www\.)?yt\.cyberhost\.uk',
 323         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 324         r'(?:www\.)?qklhadlycap4cnod\.onion',
 325         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 326         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 327         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 328         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 329         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 330         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 331         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 332         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 333         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 334         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 335     )
 336
 337     def _login(self):
 338         """
 339         Attempt to log in to YouTube.
 340         If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
 341         """
 342
 343         if (self._LOGIN_REQUIRED
 344                 and self.get_param('cookiefile') is None
 345                 and self.get_param('cookiesfrombrowser') is None):
 346             self.raise_login_required(
 347                 'Login details are needed to download this content', method='cookies')
 348         username, password = self._get_login_info()
 349         if username:
 350             self.report_warning(f'Cannot login to YouTube using username and password. {self._LOGIN_HINTS["cookies"]}')
 351
 352     def _initialize_consent(self):
 353         cookies = self._get_cookies('https://www.youtube.com/')
 354         if cookies.get('__Secure-3PSID'):
 355             return
 356         consent_id = None
 357         consent = cookies.get('CONSENT')
 358         if consent:
 359             if 'YES' in consent.value:
 360                 return
 361             consent_id = self._search_regex(
 362                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 363         if not consent_id:
 364             consent_id = random.randint(100, 999)
 365         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 366
 367     def _initialize_pref(self):
 368         cookies = self._get_cookies('https://www.youtube.com/')
 369         pref_cookie = cookies.get('PREF')
 370         pref = {}
 371         if pref_cookie:
 372             try:
 373                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 374             except ValueError:
 375                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 376         pref.update({'hl': 'en', 'tz': 'UTC'})
 377         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 378
 379     def _real_initialize(self):
 380         self._initialize_pref()
 381         self._initialize_consent()
 382         self._login()
 383
 384     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 385     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 386     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 387
 388     def _get_default_ytcfg(self, client='web'):
 389         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 390
 391     def _get_innertube_host(self, client='web'):
 392         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 393
 394     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 395         # try_get but with fallback to default ytcfg client values when present
 396         _func = lambda y: try_get(y, getter, expected_type)
 397         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 398
 399     def _extract_client_name(self, ytcfg, default_client='web'):
 400         return self._ytcfg_get_safe(
 401             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 402                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 403
 404     def _extract_client_version(self, ytcfg, default_client='web'):
 405         return self._ytcfg_get_safe(
 406             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 407                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 408
 409     def _extract_api_key(self, ytcfg=None, default_client='web'):
 410         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 411
 412     def _extract_context(self, ytcfg=None, default_client='web'):
 413         context = get_first(
 414             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 415         # Enforce language and tz for extraction
 416         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 417         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 418         return context
 419
 420     _SAPISID = None
 421
 422     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 423         time_now = round(time.time())
 424         if self._SAPISID is None:
 425             yt_cookies = self._get_cookies('https://www.youtube.com')
 426             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 427             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 428             sapisid_cookie = dict_get(
 429                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 430             if sapisid_cookie and sapisid_cookie.value:
 431                 self._SAPISID = sapisid_cookie.value
 432                 self.write_debug('Extracted SAPISID cookie')
 433                 # SAPISID cookie is required if not already present
 434                 if not yt_cookies.get('SAPISID'):
 435                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 436                     self._set_cookie(
 437                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 438             else:
 439                 self._SAPISID = False
 440         if not self._SAPISID:
 441             return None
 442         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 443         sapisidhash = hashlib.sha1(
 444             f'{time_now} {self._SAPISID} {origin}'.encode('utf-8')).hexdigest()
 445         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 446
 447     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 448                   note='Downloading API JSON', errnote='Unable to download API page',
 449                   context=None, api_key=None, api_hostname=None, default_client='web'):
 450
 451         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 452         data.update(query)
 453         real_headers = self.generate_api_headers(default_client=default_client)
 454         real_headers.update({'content-type': 'application/json'})
 455         if headers:
 456             real_headers.update(headers)
 457         return self._download_json(
 458             'https://%s/youtubei/v1/%s' % (api_hostname or self._get_innertube_host(default_client), ep),
 459             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 460             data=json.dumps(data).encode('utf8'), headers=real_headers,
 461             query={'key': api_key or self._extract_api_key()})
 462
 463     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 464         data = self._search_regex(
 465             (r'%s\s*%s' % (self._YT_INITIAL_DATA_RE, self._YT_INITIAL_BOUNDARY_RE),
 466              self._YT_INITIAL_DATA_RE), webpage, 'yt initial data', fatal=fatal)
 467         if data:
 468             return self._parse_json(data, item_id, fatal=fatal)
 469
 470     @staticmethod
 471     def _extract_session_index(*data):
 472         """
 473         Index of current account in account list.
 474         See: https://github.com/yt-dlp/yt-dlp/pull/519
 475         """
 476         for ytcfg in data:
 477             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 478             if session_index is not None:
 479                 return session_index
 480
 481     # Deprecated?
 482     def _extract_identity_token(self, ytcfg=None, webpage=None):
 483         if ytcfg:
 484             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 485             if token:
 486                 return token
 487         if webpage:
 488             return self._search_regex(
 489                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 490                 'identity token', default=None, fatal=False)
 491
 492     @staticmethod
 493     def _extract_account_syncid(*args):
 494         """
 495         Extract syncId required to download private playlists of secondary channels
 496         @params response and/or ytcfg
 497         """
 498         for data in args:
 499             # ytcfg includes channel_syncid if on secondary channel
 500             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 501             if delegated_sid:
 502                 return delegated_sid
 503             sync_ids = (try_get(
 504                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 505                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 506             if len(sync_ids) >= 2 and sync_ids[1]:
 507                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 508                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 509                 return sync_ids[0]
 510
 511     @staticmethod
 512     def _extract_visitor_data(*args):
 513         """
 514         Extracts visitorData from an API response or ytcfg
 515         Appears to be used to track session state
 516         """
 517         return get_first(
 518             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 519             expected_type=str)
 520
 521     @property
 522     def is_authenticated(self):
 523         return bool(self._generate_sapisidhash_header())
 524
 525     def extract_ytcfg(self, video_id, webpage):
 526         if not webpage:
 527             return {}
 528         return self._parse_json(
 529             self._search_regex(
 530                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 531                 default='{}'), video_id, fatal=False) or {}
 532
 533     def generate_api_headers(
 534             self, *, ytcfg=None, account_syncid=None, session_index=None,
 535             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 536
 537         origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
 538         headers = {
 539             'X-YouTube-Client-Name': compat_str(
 540                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 541             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 542             'Origin': origin,
 543             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 544             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 545             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 546         }
 547         if session_index is None:
 548             session_index = self._extract_session_index(ytcfg)
 549         if account_syncid or session_index is not None:
 550             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 551
 552         auth = self._generate_sapisidhash_header(origin)
 553         if auth is not None:
 554             headers['Authorization'] = auth
 555             headers['X-Origin'] = origin
 556         return {h: v for h, v in headers.items() if v is not None}
 557
 558     @staticmethod
 559     def _build_api_continuation_query(continuation, ctp=None):
 560         query = {
 561             'continuation': continuation
 562         }
 563         # TODO: Inconsistency with clickTrackingParams.
 564         # Currently we have a fixed ctp contained within context (from ytcfg)
 565         # and a ctp in root query for continuation.
 566         if ctp:
 567             query['clickTracking'] = {'clickTrackingParams': ctp}
 568         return query
 569
 570     @classmethod
 571     def _extract_next_continuation_data(cls, renderer):
 572         next_continuation = try_get(
 573             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 574                        lambda x: x['continuation']['reloadContinuationData']), dict)
 575         if not next_continuation:
 576             return
 577         continuation = next_continuation.get('continuation')
 578         if not continuation:
 579             return
 580         ctp = next_continuation.get('clickTrackingParams')
 581         return cls._build_api_continuation_query(continuation, ctp)
 582
 583     @classmethod
 584     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 585         if isinstance(continuation_ep, dict):
 586             continuation = try_get(
 587                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 588             if not continuation:
 589                 return
 590             ctp = continuation_ep.get('clickTrackingParams')
 591             return cls._build_api_continuation_query(continuation, ctp)
 592
 593     @classmethod
 594     def _extract_continuation(cls, renderer):
 595         next_continuation = cls._extract_next_continuation_data(renderer)
 596         if next_continuation:
 597             return next_continuation
 598
 599         contents = []
 600         for key in ('contents', 'items'):
 601             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 602
 603         for content in contents:
 604             if not isinstance(content, dict):
 605                 continue
 606             continuation_ep = try_get(
 607                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 608                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 609                 dict)
 610             continuation = cls._extract_continuation_ep_data(continuation_ep)
 611             if continuation:
 612                 return continuation
 613
 614     @classmethod
 615     def _extract_alerts(cls, data):
 616         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 617             if not isinstance(alert_dict, dict):
 618                 continue
 619             for alert in alert_dict.values():
 620                 alert_type = alert.get('type')
 621                 if not alert_type:
 622                     continue
 623                 message = cls._get_text(alert, 'text')
 624                 if message:
 625                     yield alert_type, message
 626
 627     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 628         errors = []
 629         warnings = []
 630         for alert_type, alert_message in alerts:
 631             if alert_type.lower() == 'error' and fatal:
 632                 errors.append([alert_type, alert_message])
 633             else:
 634                 warnings.append([alert_type, alert_message])
 635
 636         for alert_type, alert_message in (warnings + errors[:-1]):
 637             self.report_warning('YouTube said: %s - %s' % (alert_type, alert_message), only_once=only_once)
 638         if errors:
 639             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 640
 641     def _extract_and_report_alerts(self, data, *args, **kwargs):
 642         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 643
 644     def _extract_badges(self, renderer: dict):
 645         badges = set()
 646         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 647             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 648             if label:
 649                 badges.add(label.lower())
 650         return badges
 651
 652     @staticmethod
 653     def _get_text(data, *path_list, max_runs=None):
 654         for path in path_list or [None]:
 655             if path is None:
 656                 obj = [data]
 657             else:
 658                 obj = traverse_obj(data, path, default=[])
 659                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 660                     obj = [obj]
 661             for item in obj:
 662                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 663                 if text:
 664                     return text
 665                 runs = try_get(item, lambda x: x['runs'], list) or []
 666                 if not runs and isinstance(item, list):
 667                     runs = item
 668
 669                 runs = runs[:min(len(runs), max_runs or len(runs))]
 670                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 671                 if text:
 672                     return text
 673
 674     def _get_count(self, data, *path_list):
 675         count_text = self._get_text(data, *path_list) or ''
 676         count = parse_count(count_text)
 677         if count is None:
 678             count = str_to_int(
 679                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 680         return count
 681
 682     @staticmethod
 683     def _extract_thumbnails(data, *path_list):
 684         """
 685         Extract thumbnails from thumbnails dict
 686         @param path_list: path list to level that contains 'thumbnails' key
 687         """
 688         thumbnails = []
 689         for path in path_list or [()]:
 690             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 691                 thumbnail_url = url_or_none(thumbnail.get('url'))
 692                 if not thumbnail_url:
 693                     continue
 694                 # Sometimes youtube gives a wrong thumbnail URL. See:
 695                 # https://github.com/yt-dlp/yt-dlp/issues/233
 696                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 697                 if 'maxresdefault' in thumbnail_url:
 698                     thumbnail_url = thumbnail_url.split('?')[0]
 699                 thumbnails.append({
 700                     'url': thumbnail_url,
 701                     'height': int_or_none(thumbnail.get('height')),
 702                     'width': int_or_none(thumbnail.get('width')),
 703                 })
 704         return thumbnails
 705
 706     @staticmethod
 707     def extract_relative_time(relative_time_text):
 708         """
 709         Extracts a relative time from string and converts to dt object
 710         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 711         """
 712         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 713         if mobj:
 714             start = mobj.group('start')
 715             if start:
 716                 return datetime_from_str(start)
 717             try:
 718                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 719             except ValueError:
 720                 return None
 721
 722     def _extract_time_text(self, renderer, *path_list):
 723         text = self._get_text(renderer, *path_list) or ''
 724         dt = self.extract_relative_time(text)
 725         timestamp = None
 726         if isinstance(dt, datetime.datetime):
 727             timestamp = calendar.timegm(dt.timetuple())
 728
 729         if timestamp is None:
 730             timestamp = (
 731                 unified_timestamp(text) or unified_timestamp(
 732                     self._search_regex(
 733                         (r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*on)?\s*(.+\d)', r'\w+[\s,\.-]*\w+[\s,\.-]+20\d{2}'),
 734                         text.lower(), 'time text', default=None)))
 735
 736         if text and timestamp is None:
 737             self.report_warning('Cannot parse localized time text' + bug_reports_message(), only_once=True)
 738         return timestamp, text
 739
 740     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 741                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 742                           default_client='web'):
 743         response = None
 744         last_error = None
 745         count = -1
 746         retries = self.get_param('extractor_retries', 3)
 747         if check_get_keys is None:
 748             check_get_keys = []
 749         while count < retries:
 750             count += 1
 751             if last_error:
 752                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 753             try:
 754                 response = self._call_api(
 755                     ep=ep, fatal=True, headers=headers,
 756                     video_id=item_id, query=query,
 757                     context=self._extract_context(ytcfg, default_client),
 758                     api_key=self._extract_api_key(ytcfg, default_client),
 759                     api_hostname=api_hostname, default_client=default_client,
 760                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 761             except ExtractorError as e:
 762                 if isinstance(e.cause, network_exceptions):
 763                     if isinstance(e.cause, compat_HTTPError):
 764                         first_bytes = e.cause.read(512)
 765                         if not is_html(first_bytes):
 766                             yt_error = try_get(
 767                                 self._parse_json(
 768                                     self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 769                                 lambda x: x['error']['message'], compat_str)
 770                             if yt_error:
 771                                 self._report_alerts([('ERROR', yt_error)], fatal=False)
 772                     # Downloading page may result in intermittent 5xx HTTP error
 773                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 774                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 775                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 776                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 777                         last_error = error_to_compat_str(e.cause or e.msg)
 778                         if count < retries:
 779                             continue
 780                 if fatal:
 781                     raise
 782                 else:
 783                     self.report_warning(error_to_compat_str(e))
 784                     return
 785
 786             else:
 787                 try:
 788                     self._extract_and_report_alerts(response, only_once=True)
 789                 except ExtractorError as e:
 790                     # YouTube servers may return errors we want to retry on in a 200 OK response
 791                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 792                     if 'unknown error' in e.msg.lower():
 793                         last_error = e.msg
 794                         continue
 795                     if fatal:
 796                         raise
 797                     self.report_warning(error_to_compat_str(e))
 798                     return
 799                 if not check_get_keys or dict_get(response, check_get_keys):
 800                     break
 801                 # Youtube sometimes sends incomplete data
 802                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 803                 last_error = 'Incomplete data received'
 804                 if count >= retries:
 805                     if fatal:
 806                         raise ExtractorError(last_error)
 807                     else:
 808                         self.report_warning(last_error)
 809                         return
 810         return response
 811
 812     @staticmethod
 813     def is_music_url(url):
 814         return re.match(r'https?://music\.youtube\.com/', url) is not None
 815
 816     def _extract_video(self, renderer):
 817         video_id = renderer.get('videoId')
 818         title = self._get_text(renderer, 'title')
 819         description = self._get_text(renderer, 'descriptionSnippet')
 820         duration = parse_duration(self._get_text(
 821             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 822         view_count = self._get_count(renderer, 'viewCountText')
 823
 824         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 825         channel_id = traverse_obj(
 826             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'), expected_type=str, get_all=False)
 827         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 828         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 829         overlay_style = traverse_obj(
 830             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'), get_all=False, expected_type=str)
 831         badges = self._extract_badges(renderer)
 832         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 833
 834         return {
 835             '_type': 'url',
 836             'ie_key': YoutubeIE.ie_key(),
 837             'id': video_id,
 838             'url': f'https://www.youtube.com/watch?v={video_id}',
 839             'title': title,
 840             'description': description,
 841             'duration': duration,
 842             'view_count': view_count,
 843             'uploader': uploader,
 844             'channel_id': channel_id,
 845             'thumbnails': thumbnails,
 846             #  'upload_date': strftime_or_none(timestamp, '%Y%m%d'),
 847             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 848                             else 'was_live' if 'streamed' in time_text.lower()
 849                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 850                             else None),
 851             'release_timestamp': scheduled_timestamp,
 852             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 853         }
 854
 855
 856 class YoutubeIE(YoutubeBaseInfoExtractor):
 857     IE_DESC = 'YouTube'
 858     _VALID_URL = r"""(?x)^
 859                      (
 860                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 861                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 862                             (?:www\.)?deturl\.com/www\.youtube\.com|
 863                             (?:www\.)?pwnyoutube\.com|
 864                             (?:www\.)?hooktube\.com|
 865                             (?:www\.)?yourepeat\.com|
 866                             tube\.majestyc\.net|
 867                             %(invidious)s|
 868                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 869                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 870                          (?:                                                  # the various things that can precede the ID:
 871                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 872                              |(?:                                             # or the v= param in all its forms
 873                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 874                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 875                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 876                                  v=
 877                              )
 878                          ))
 879                          |(?:
 880                             youtu\.be|                                        # just youtu.be/xxxx
 881                             vid\.plus|                                        # or vid.plus/xxxx
 882                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 883                             %(invidious)s
 884                          )/
 885                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 886                          )
 887                      )?                                                       # all until now is optional -> you can pass the naked ID
 888                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 889                      (?(1).+)?                                                # if we found the ID, everything can follow
 890                      (?:\#|$)""" % {
 891         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 892     }
 893     _PLAYER_INFO_RE = (
 894         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 895         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 896         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 897     )
 898     _formats = {
 899         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 900         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 901         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 902         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 903         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 904         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 905         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 906         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 907         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 908         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 909         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 910         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 911         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 912         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 913         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 914         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 915         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 916         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 917
 918
 919         # 3D videos
 920         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 921         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 922         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 923         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 924         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 925         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 926         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 927
 928         # Apple HTTP Live Streaming
 929         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 930         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 931         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 932         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 933         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 934         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 935         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 936         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 937
 938         # DASH mp4 video
 939         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 940         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 941         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 942         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 943         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 944         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 945         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 946         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 947         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 948         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 949         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 950         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 951
 952         # Dash mp4 audio
 953         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 954         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 955         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 956         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 957         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 958         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 959         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 960
 961         # Dash webm
 962         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 963         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 964         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 965         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 966         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 967         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 968         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
 969         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 970         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 971         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 972         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 973         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 974         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 975         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 976         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 977         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
 978         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 979         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 980         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 981         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 982         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 983         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 984
 985         # Dash webm audio
 986         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
 987         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
 988
 989         # Dash webm audio with opus inside
 990         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
 991         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
 992         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
 993
 994         # RTMP (unnamed)
 995         '_rtmp': {'protocol': 'rtmp'},
 996
 997         # av01 video only formats sometimes served with "unknown" codecs
 998         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
 999         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1000         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1001         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1002         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1003         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1004         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1005         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1006     }
1007     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1008
1009     _GEO_BYPASS = False
1010
1011     IE_NAME = 'youtube'
1012     _TESTS = [
1013         {
1014             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1015             'info_dict': {
1016                 'id': 'BaW_jenozKc',
1017                 'ext': 'mp4',
1018                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1019                 'uploader': 'Philipp Hagemeister',
1020                 'uploader_id': 'phihag',
1021                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1022                 'channel': 'Philipp Hagemeister',
1023                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1024                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1025                 'upload_date': '20121002',
1026                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1027                 'categories': ['Science & Technology'],
1028                 'tags': ['youtube-dl'],
1029                 'duration': 10,
1030                 'view_count': int,
1031                 'like_count': int,
1032                 'availability': 'public',
1033                 'playable_in_embed': True,
1034                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1035                 'live_status': 'not_live',
1036                 'age_limit': 0,
1037                 'start_time': 1,
1038                 'end_time': 9,
1039                 'channel_follower_count': int
1040             }
1041         },
1042         {
1043             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1044             'note': 'Embed-only video (#1746)',
1045             'info_dict': {
1046                 'id': 'yZIXLfi8CZQ',
1047                 'ext': 'mp4',
1048                 'upload_date': '20120608',
1049                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1050                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1051                 'uploader': 'SET India',
1052                 'uploader_id': 'setindia',
1053                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1054                 'age_limit': 18,
1055             },
1056             'skip': 'Private video',
1057         },
1058         {
1059             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1060             'note': 'Use the first video ID in the URL',
1061             'info_dict': {
1062                 'id': 'BaW_jenozKc',
1063                 'ext': 'mp4',
1064                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1065                 'uploader': 'Philipp Hagemeister',
1066                 'uploader_id': 'phihag',
1067                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1068                 'channel': 'Philipp Hagemeister',
1069                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1070                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1071                 'upload_date': '20121002',
1072                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1073                 'categories': ['Science & Technology'],
1074                 'tags': ['youtube-dl'],
1075                 'duration': 10,
1076                 'view_count': int,
1077                 'like_count': int,
1078                 'availability': 'public',
1079                 'playable_in_embed': True,
1080                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1081                 'live_status': 'not_live',
1082                 'age_limit': 0,
1083                 'channel_follower_count': int
1084             },
1085             'params': {
1086                 'skip_download': True,
1087             },
1088         },
1089         {
1090             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1091             'note': '256k DASH audio (format 141) via DASH manifest',
1092             'info_dict': {
1093                 'id': 'a9LDPn-MO4I',
1094                 'ext': 'm4a',
1095                 'upload_date': '20121002',
1096                 'uploader_id': '8KVIDEO',
1097                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1098                 'description': '',
1099                 'uploader': '8KVIDEO',
1100                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1101             },
1102             'params': {
1103                 'youtube_include_dash_manifest': True,
1104                 'format': '141',
1105             },
1106             'skip': 'format 141 not served anymore',
1107         },
1108         # DASH manifest with encrypted signature
1109         {
1110             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1111             'info_dict': {
1112                 'id': 'IB3lcPjvWLA',
1113                 'ext': 'm4a',
1114                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1115                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1116                 'duration': 244,
1117                 'uploader': 'AfrojackVEVO',
1118                 'uploader_id': 'AfrojackVEVO',
1119                 'upload_date': '20131011',
1120                 'abr': 129.495,
1121                 'like_count': int,
1122                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1123                 'playable_in_embed': True,
1124                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1125                 'view_count': int,
1126                 'track': 'The Spark',
1127                 'live_status': 'not_live',
1128                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1129                 'channel': 'Afrojack',
1130                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1131                 'tags': 'count:19',
1132                 'availability': 'public',
1133                 'categories': ['Music'],
1134                 'age_limit': 0,
1135                 'alt_title': 'The Spark',
1136                 'channel_follower_count': int
1137             },
1138             'params': {
1139                 'youtube_include_dash_manifest': True,
1140                 'format': '141/bestaudio[ext=m4a]',
1141             },
1142         },
1143         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1144         {
1145             'note': 'Embed allowed age-gate video',
1146             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1147             'info_dict': {
1148                 'id': 'HtVdAasjOgU',
1149                 'ext': 'mp4',
1150                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1151                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1152                 'duration': 142,
1153                 'uploader': 'The Witcher',
1154                 'uploader_id': 'WitcherGame',
1155                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1156                 'upload_date': '20140605',
1157                 'age_limit': 18,
1158                 'categories': ['Gaming'],
1159                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1160                 'availability': 'needs_auth',
1161                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1162                 'like_count': int,
1163                 'channel': 'The Witcher',
1164                 'live_status': 'not_live',
1165                 'tags': 'count:17',
1166                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1167                 'playable_in_embed': True,
1168                 'view_count': int,
1169                 'channel_follower_count': int
1170             },
1171         },
1172         {
1173             'note': 'Age-gate video with embed allowed in public site',
1174             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1175             'info_dict': {
1176                 'id': 'HsUATh_Nc2U',
1177                 'ext': 'mp4',
1178                 'title': 'Godzilla 2 (Official Video)',
1179                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1180                 'upload_date': '20200408',
1181                 'uploader_id': 'FlyingKitty900',
1182                 'uploader': 'FlyingKitty',
1183                 'age_limit': 18,
1184                 'availability': 'needs_auth',
1185                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1186                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1187                 'channel': 'FlyingKitty',
1188                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1189                 'view_count': int,
1190                 'categories': ['Entertainment'],
1191                 'live_status': 'not_live',
1192                 'tags': ['Flyingkitty', 'godzilla 2'],
1193                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1194                 'like_count': int,
1195                 'duration': 177,
1196                 'playable_in_embed': True,
1197                 'channel_follower_count': int
1198             },
1199         },
1200         {
1201             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1202             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1203             'info_dict': {
1204                 'id': 'Tq92D6wQ1mg',
1205                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1206                 'ext': 'mp4',
1207                 'upload_date': '20191227',
1208                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1209                 'uploader': 'Projekt Melody',
1210                 'description': 'md5:17eccca93a786d51bc67646756894066',
1211                 'age_limit': 18,
1212                 'like_count': int,
1213                 'availability': 'needs_auth',
1214                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1215                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1216                 'view_count': int,
1217                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1218                 'channel': 'Projekt Melody',
1219                 'live_status': 'not_live',
1220                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1221                 'playable_in_embed': True,
1222                 'categories': ['Entertainment'],
1223                 'duration': 106,
1224                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1225                 'channel_follower_count': int
1226             },
1227         },
1228         {
1229             'note': 'Non-Agegated non-embeddable video',
1230             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1231             'info_dict': {
1232                 'id': 'MeJVWBSsPAY',
1233                 'ext': 'mp4',
1234                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1235                 'uploader': 'Herr Lurik',
1236                 'uploader_id': 'st3in234',
1237                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1238                 'upload_date': '20130730',
1239                 'track': 'Such mich find mich',
1240                 'age_limit': 0,
1241                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1242                 'like_count': int,
1243                 'playable_in_embed': False,
1244                 'creator': 'OOMPH!',
1245                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1246                 'view_count': int,
1247                 'alt_title': 'Such mich find mich',
1248                 'duration': 210,
1249                 'channel': 'Herr Lurik',
1250                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1251                 'categories': ['Music'],
1252                 'availability': 'public',
1253                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1254                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1255                 'live_status': 'not_live',
1256                 'artist': 'OOMPH!',
1257                 'channel_follower_count': int
1258             },
1259         },
1260         {
1261             'note': 'Non-bypassable age-gated video',
1262             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1263             'only_matching': True,
1264         },
1265         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1266         # YouTube Red ad is not captured for creator
1267         {
1268             'url': '__2ABJjxzNo',
1269             'info_dict': {
1270                 'id': '__2ABJjxzNo',
1271                 'ext': 'mp4',
1272                 'duration': 266,
1273                 'upload_date': '20100430',
1274                 'uploader_id': 'deadmau5',
1275                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1276                 'creator': 'deadmau5',
1277                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1278                 'uploader': 'deadmau5',
1279                 'title': 'Deadmau5 - Some Chords (HD)',
1280                 'alt_title': 'Some Chords',
1281                 'availability': 'public',
1282                 'tags': 'count:14',
1283                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1284                 'view_count': int,
1285                 'live_status': 'not_live',
1286                 'channel': 'deadmau5',
1287                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1288                 'like_count': int,
1289                 'track': 'Some Chords',
1290                 'artist': 'deadmau5',
1291                 'playable_in_embed': True,
1292                 'age_limit': 0,
1293                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1294                 'categories': ['Music'],
1295                 'album': 'Some Chords',
1296                 'channel_follower_count': int
1297             },
1298             'expected_warnings': [
1299                 'DASH manifest missing',
1300             ]
1301         },
1302         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1303         {
1304             'url': 'lqQg6PlCWgI',
1305             'info_dict': {
1306                 'id': 'lqQg6PlCWgI',
1307                 'ext': 'mp4',
1308                 'duration': 6085,
1309                 'upload_date': '20150827',
1310                 'uploader_id': 'olympic',
1311                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1312                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1313                 'uploader': 'Olympics',
1314                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1315                 'like_count': int,
1316                 'release_timestamp': 1343767800,
1317                 'playable_in_embed': True,
1318                 'categories': ['Sports'],
1319                 'release_date': '20120731',
1320                 'channel': 'Olympics',
1321                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1322                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1323                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1324                 'age_limit': 0,
1325                 'availability': 'public',
1326                 'live_status': 'was_live',
1327                 'view_count': int,
1328                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1329                 'channel_follower_count': int
1330             },
1331             'params': {
1332                 'skip_download': 'requires avconv',
1333             }
1334         },
1335         # Non-square pixels
1336         {
1337             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1338             'info_dict': {
1339                 'id': '_b-2C3KPAM0',
1340                 'ext': 'mp4',
1341                 'stretched_ratio': 16 / 9.,
1342                 'duration': 85,
1343                 'upload_date': '20110310',
1344                 'uploader_id': 'AllenMeow',
1345                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1346                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1347                 'uploader': '孫ᄋᄅ',
1348                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1349                 'playable_in_embed': True,
1350                 'channel': '孫ᄋᄅ',
1351                 'age_limit': 0,
1352                 'tags': 'count:11',
1353                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1354                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1355                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1356                 'view_count': int,
1357                 'categories': ['People & Blogs'],
1358                 'like_count': int,
1359                 'live_status': 'not_live',
1360                 'availability': 'unlisted',
1361                 'channel_follower_count': int
1362             },
1363         },
1364         # url_encoded_fmt_stream_map is empty string
1365         {
1366             'url': 'qEJwOuvDf7I',
1367             'info_dict': {
1368                 'id': 'qEJwOuvDf7I',
1369                 'ext': 'webm',
1370                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1371                 'description': '',
1372                 'upload_date': '20150404',
1373                 'uploader_id': 'spbelect',
1374                 'uploader': 'Наблюдатели Петербурга',
1375             },
1376             'params': {
1377                 'skip_download': 'requires avconv',
1378             },
1379             'skip': 'This live event has ended.',
1380         },
1381         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1382         {
1383             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1384             'info_dict': {
1385                 'id': 'FIl7x6_3R5Y',
1386                 'ext': 'webm',
1387                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1388                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1389                 'duration': 220,
1390                 'upload_date': '20150625',
1391                 'uploader_id': 'dorappi2000',
1392                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1393                 'uploader': 'dorappi2000',
1394                 'formats': 'mincount:31',
1395             },
1396             'skip': 'not actual anymore',
1397         },
1398         # DASH manifest with segment_list
1399         {
1400             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1401             'md5': '8ce563a1d667b599d21064e982ab9e31',
1402             'info_dict': {
1403                 'id': 'CsmdDsKjzN8',
1404                 'ext': 'mp4',
1405                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1406                 'uploader': 'Airtek',
1407                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1408                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1409                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1410             },
1411             'params': {
1412                 'youtube_include_dash_manifest': True,
1413                 'format': '135',  # bestvideo
1414             },
1415             'skip': 'This live event has ended.',
1416         },
1417         {
1418             # Multifeed videos (multiple cameras), URL is for Main Camera
1419             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1420             'info_dict': {
1421                 'id': 'jvGDaLqkpTg',
1422                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1423                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1424             },
1425             'playlist': [{
1426                 'info_dict': {
1427                     'id': 'jvGDaLqkpTg',
1428                     'ext': 'mp4',
1429                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1430                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1431                     'duration': 10643,
1432                     'upload_date': '20161111',
1433                     'uploader': 'Team PGP',
1434                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1435                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1436                 },
1437             }, {
1438                 'info_dict': {
1439                     'id': '3AKt1R1aDnw',
1440                     'ext': 'mp4',
1441                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1442                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1443                     'duration': 10991,
1444                     'upload_date': '20161111',
1445                     'uploader': 'Team PGP',
1446                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1447                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1448                 },
1449             }, {
1450                 'info_dict': {
1451                     'id': 'RtAMM00gpVc',
1452                     'ext': 'mp4',
1453                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1454                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1455                     'duration': 10995,
1456                     'upload_date': '20161111',
1457                     'uploader': 'Team PGP',
1458                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1459                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1460                 },
1461             }, {
1462                 'info_dict': {
1463                     'id': '6N2fdlP3C5U',
1464                     'ext': 'mp4',
1465                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1466                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1467                     'duration': 10990,
1468                     'upload_date': '20161111',
1469                     'uploader': 'Team PGP',
1470                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1471                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1472                 },
1473             }],
1474             'params': {
1475                 'skip_download': True,
1476             },
1477             'skip': 'Not multifeed anymore',
1478         },
1479         {
1480             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1481             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1482             'info_dict': {
1483                 'id': 'gVfLd0zydlo',
1484                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1485             },
1486             'playlist_count': 2,
1487             'skip': 'Not multifeed anymore',
1488         },
1489         {
1490             'url': 'https://vid.plus/FlRa-iH7PGw',
1491             'only_matching': True,
1492         },
1493         {
1494             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1495             'only_matching': True,
1496         },
1497         {
1498             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1499             # Also tests cut-off URL expansion in video description (see
1500             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1501             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1502             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1503             'info_dict': {
1504                 'id': 'lsguqyKfVQg',
1505                 'ext': 'mp4',
1506                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1507                 'alt_title': 'Dark Walk',
1508                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1509                 'duration': 133,
1510                 'upload_date': '20151119',
1511                 'uploader_id': 'IronSoulElf',
1512                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1513                 'uploader': 'IronSoulElf',
1514                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1515                 'track': 'Dark Walk',
1516                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1517                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1518                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1519                 'categories': ['Film & Animation'],
1520                 'view_count': int,
1521                 'live_status': 'not_live',
1522                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1523                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1524                 'tags': 'count:13',
1525                 'availability': 'public',
1526                 'channel': 'IronSoulElf',
1527                 'playable_in_embed': True,
1528                 'like_count': int,
1529                 'age_limit': 0,
1530                 'channel_follower_count': int
1531             },
1532             'params': {
1533                 'skip_download': True,
1534             },
1535         },
1536         {
1537             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1538             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1539             'only_matching': True,
1540         },
1541         {
1542             # Video with yt:stretch=17:0
1543             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1544             'info_dict': {
1545                 'id': 'Q39EVAstoRM',
1546                 'ext': 'mp4',
1547                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1548                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1549                 'upload_date': '20151107',
1550                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1551                 'uploader': 'CH GAMER DROID',
1552             },
1553             'params': {
1554                 'skip_download': True,
1555             },
1556             'skip': 'This video does not exist.',
1557         },
1558         {
1559             # Video with incomplete 'yt:stretch=16:'
1560             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1561             'only_matching': True,
1562         },
1563         {
1564             # Video licensed under Creative Commons
1565             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1566             'info_dict': {
1567                 'id': 'M4gD1WSo5mA',
1568                 'ext': 'mp4',
1569                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1570                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1571                 'duration': 721,
1572                 'upload_date': '20150127',
1573                 'uploader_id': 'BerkmanCenter',
1574                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1575                 'uploader': 'The Berkman Klein Center for Internet & Society',
1576                 'license': 'Creative Commons Attribution license (reuse allowed)',
1577                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1578                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1579                 'like_count': int,
1580                 'age_limit': 0,
1581                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1582                 'channel': 'The Berkman Klein Center for Internet & Society',
1583                 'availability': 'public',
1584                 'view_count': int,
1585                 'categories': ['Education'],
1586                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1587                 'live_status': 'not_live',
1588                 'playable_in_embed': True,
1589                 'channel_follower_count': int
1590             },
1591             'params': {
1592                 'skip_download': True,
1593             },
1594         },
1595         {
1596             # Channel-like uploader_url
1597             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1598             'info_dict': {
1599                 'id': 'eQcmzGIKrzg',
1600                 'ext': 'mp4',
1601                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1602                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1603                 'duration': 4060,
1604                 'upload_date': '20151119',
1605                 'uploader': 'Bernie Sanders',
1606                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1607                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1608                 'license': 'Creative Commons Attribution license (reuse allowed)',
1609                 'playable_in_embed': True,
1610                 'tags': 'count:12',
1611                 'like_count': int,
1612                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1613                 'age_limit': 0,
1614                 'availability': 'public',
1615                 'categories': ['News & Politics'],
1616                 'channel': 'Bernie Sanders',
1617                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1618                 'view_count': int,
1619                 'live_status': 'not_live',
1620                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1621                 'channel_follower_count': int
1622             },
1623             'params': {
1624                 'skip_download': True,
1625             },
1626         },
1627         {
1628             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1629             'only_matching': True,
1630         },
1631         {
1632             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1633             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1634             'only_matching': True,
1635         },
1636         {
1637             # Rental video preview
1638             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1639             'info_dict': {
1640                 'id': 'uGpuVWrhIzE',
1641                 'ext': 'mp4',
1642                 'title': 'Piku - Trailer',
1643                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1644                 'upload_date': '20150811',
1645                 'uploader': 'FlixMatrix',
1646                 'uploader_id': 'FlixMatrixKaravan',
1647                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1648                 'license': 'Standard YouTube License',
1649             },
1650             'params': {
1651                 'skip_download': True,
1652             },
1653             'skip': 'This video is not available.',
1654         },
1655         {
1656             # YouTube Red video with episode data
1657             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1658             'info_dict': {
1659                 'id': 'iqKdEhx-dD4',
1660                 'ext': 'mp4',
1661                 'title': 'Isolation - Mind Field (Ep 1)',
1662                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1663                 'duration': 2085,
1664                 'upload_date': '20170118',
1665                 'uploader': 'Vsauce',
1666                 'uploader_id': 'Vsauce',
1667                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1668                 'series': 'Mind Field',
1669                 'season_number': 1,
1670                 'episode_number': 1,
1671                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1672                 'tags': 'count:12',
1673                 'view_count': int,
1674                 'availability': 'public',
1675                 'age_limit': 0,
1676                 'channel': 'Vsauce',
1677                 'episode': 'Episode 1',
1678                 'categories': ['Entertainment'],
1679                 'season': 'Season 1',
1680                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1681                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1682                 'like_count': int,
1683                 'playable_in_embed': True,
1684                 'live_status': 'not_live',
1685                 'channel_follower_count': int
1686             },
1687             'params': {
1688                 'skip_download': True,
1689             },
1690             'expected_warnings': [
1691                 'Skipping DASH manifest',
1692             ],
1693         },
1694         {
1695             # The following content has been identified by the YouTube community
1696             # as inappropriate or offensive to some audiences.
1697             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1698             'info_dict': {
1699                 'id': '6SJNVb0GnPI',
1700                 'ext': 'mp4',
1701                 'title': 'Race Differences in Intelligence',
1702                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1703                 'duration': 965,
1704                 'upload_date': '20140124',
1705                 'uploader': 'New Century Foundation',
1706                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1707                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1708             },
1709             'params': {
1710                 'skip_download': True,
1711             },
1712             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1713         },
1714         {
1715             # itag 212
1716             'url': '1t24XAntNCY',
1717             'only_matching': True,
1718         },
1719         {
1720             # geo restricted to JP
1721             'url': 'sJL6WA-aGkQ',
1722             'only_matching': True,
1723         },
1724         {
1725             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1726             'only_matching': True,
1727         },
1728         {
1729             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1730             'only_matching': True,
1731         },
1732         {
1733             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1734             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1735             'only_matching': True,
1736         },
1737         {
1738             # DRM protected
1739             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1740             'only_matching': True,
1741         },
1742         {
1743             # Video with unsupported adaptive stream type formats
1744             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1745             'info_dict': {
1746                 'id': 'Z4Vy8R84T1U',
1747                 'ext': 'mp4',
1748                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1749                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1750                 'duration': 433,
1751                 'upload_date': '20130923',
1752                 'uploader': 'Amelia Putri Harwita',
1753                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1754                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1755                 'formats': 'maxcount:10',
1756             },
1757             'params': {
1758                 'skip_download': True,
1759                 'youtube_include_dash_manifest': False,
1760             },
1761             'skip': 'not actual anymore',
1762         },
1763         {
1764             # Youtube Music Auto-generated description
1765             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1766             'info_dict': {
1767                 'id': 'MgNrAu2pzNs',
1768                 'ext': 'mp4',
1769                 'title': 'Voyeur Girl',
1770                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1771                 'upload_date': '20190312',
1772                 'uploader': 'Stephen - Topic',
1773                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1774                 'artist': 'Stephen',
1775                 'track': 'Voyeur Girl',
1776                 'album': 'it\'s too much love to know my dear',
1777                 'release_date': '20190313',
1778                 'release_year': 2019,
1779                 'alt_title': 'Voyeur Girl',
1780                 'view_count': int,
1781                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1782                 'playable_in_embed': True,
1783                 'like_count': int,
1784                 'categories': ['Music'],
1785                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1786                 'channel': 'Stephen',
1787                 'availability': 'public',
1788                 'creator': 'Stephen',
1789                 'duration': 169,
1790                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1791                 'age_limit': 0,
1792                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1793                 'tags': 'count:11',
1794                 'live_status': 'not_live',
1795                 'channel_follower_count': int
1796             },
1797             'params': {
1798                 'skip_download': True,
1799             },
1800         },
1801         {
1802             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1803             'only_matching': True,
1804         },
1805         {
1806             # invalid -> valid video id redirection
1807             'url': 'DJztXj2GPfl',
1808             'info_dict': {
1809                 'id': 'DJztXj2GPfk',
1810                 'ext': 'mp4',
1811                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1812                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1813                 'upload_date': '20090125',
1814                 'uploader': 'Prochorowka',
1815                 'uploader_id': 'Prochorowka',
1816                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1817                 'artist': 'Panjabi MC',
1818                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1819                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1820             },
1821             'params': {
1822                 'skip_download': True,
1823             },
1824             'skip': 'Video unavailable',
1825         },
1826         {
1827             # empty description results in an empty string
1828             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1829             'info_dict': {
1830                 'id': 'x41yOUIvK2k',
1831                 'ext': 'mp4',
1832                 'title': 'IMG 3456',
1833                 'description': '',
1834                 'upload_date': '20170613',
1835                 'uploader_id': 'ElevageOrVert',
1836                 'uploader': 'ElevageOrVert',
1837                 'view_count': int,
1838                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1839                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1840                 'like_count': int,
1841                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1842                 'tags': [],
1843                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1844                 'availability': 'public',
1845                 'age_limit': 0,
1846                 'categories': ['Pets & Animals'],
1847                 'duration': 7,
1848                 'playable_in_embed': True,
1849                 'live_status': 'not_live',
1850                 'channel': 'ElevageOrVert',
1851                 'channel_follower_count': int
1852             },
1853             'params': {
1854                 'skip_download': True,
1855             },
1856         },
1857         {
1858             # with '};' inside yt initial data (see [1])
1859             # see [2] for an example with '};' inside ytInitialPlayerResponse
1860             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1861             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1862             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1863             'info_dict': {
1864                 'id': 'CHqg6qOn4no',
1865                 'ext': 'mp4',
1866                 'title': 'Part 77   Sort a list of simple types in c#',
1867                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1868                 'upload_date': '20130831',
1869                 'uploader_id': 'kudvenkat',
1870                 'uploader': 'kudvenkat',
1871                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1872                 'like_count': int,
1873                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1874                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1875                 'live_status': 'not_live',
1876                 'categories': ['Education'],
1877                 'availability': 'public',
1878                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1879                 'tags': 'count:12',
1880                 'playable_in_embed': True,
1881                 'age_limit': 0,
1882                 'view_count': int,
1883                 'duration': 522,
1884                 'channel': 'kudvenkat',
1885                 'channel_follower_count': int
1886             },
1887             'params': {
1888                 'skip_download': True,
1889             },
1890         },
1891         {
1892             # another example of '};' in ytInitialData
1893             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1894             'only_matching': True,
1895         },
1896         {
1897             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1898             'only_matching': True,
1899         },
1900         {
1901             # https://github.com/ytdl-org/youtube-dl/pull/28094
1902             'url': 'OtqTfy26tG0',
1903             'info_dict': {
1904                 'id': 'OtqTfy26tG0',
1905                 'ext': 'mp4',
1906                 'title': 'Burn Out',
1907                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1908                 'upload_date': '20141120',
1909                 'uploader': 'The Cinematic Orchestra - Topic',
1910                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1911                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1912                 'artist': 'The Cinematic Orchestra',
1913                 'track': 'Burn Out',
1914                 'album': 'Every Day',
1915                 'like_count': int,
1916                 'live_status': 'not_live',
1917                 'alt_title': 'Burn Out',
1918                 'duration': 614,
1919                 'age_limit': 0,
1920                 'view_count': int,
1921                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1922                 'creator': 'The Cinematic Orchestra',
1923                 'channel': 'The Cinematic Orchestra',
1924                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1925                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1926                 'availability': 'public',
1927                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1928                 'categories': ['Music'],
1929                 'playable_in_embed': True,
1930                 'channel_follower_count': int
1931             },
1932             'params': {
1933                 'skip_download': True,
1934             },
1935         },
1936         {
1937             # controversial video, only works with bpctr when authenticated with cookies
1938             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1939             'only_matching': True,
1940         },
1941         {
1942             # controversial video, requires bpctr/contentCheckOk
1943             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1944             'info_dict': {
1945                 'id': 'SZJvDhaSDnc',
1946                 'ext': 'mp4',
1947                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1948                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1949                 'uploader': 'CBS Mornings',
1950                 'uploader_id': 'CBSThisMorning',
1951                 'upload_date': '20140716',
1952                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1953                 'duration': 170,
1954                 'categories': ['News & Politics'],
1955                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
1956                 'view_count': int,
1957                 'channel': 'CBS Mornings',
1958                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
1959                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
1960                 'age_limit': 18,
1961                 'availability': 'needs_auth',
1962                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
1963                 'like_count': int,
1964                 'live_status': 'not_live',
1965                 'playable_in_embed': True,
1966                 'channel_follower_count': int
1967             }
1968         },
1969         {
1970             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
1971             'url': 'cBvYw8_A0vQ',
1972             'info_dict': {
1973                 'id': 'cBvYw8_A0vQ',
1974                 'ext': 'mp4',
1975                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
1976                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
1977                 'upload_date': '20201120',
1978                 'uploader': 'Walk around Japan',
1979                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
1980                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
1981                 'duration': 1456,
1982                 'categories': ['Travel & Events'],
1983                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
1984                 'view_count': int,
1985                 'channel': 'Walk around Japan',
1986                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
1987                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
1988                 'age_limit': 0,
1989                 'availability': 'public',
1990                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
1991                 'live_status': 'not_live',
1992                 'playable_in_embed': True,
1993                 'channel_follower_count': int
1994             },
1995             'params': {
1996                 'skip_download': True,
1997             },
1998         }, {
1999             # Has multiple audio streams
2000             'url': 'WaOKSUlf4TM',
2001             'only_matching': True
2002         }, {
2003             # Requires Premium: has format 141 when requested using YTM url
2004             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2005             'only_matching': True
2006         }, {
2007             # multiple subtitles with same lang_code
2008             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2009             'only_matching': True,
2010         }, {
2011             # Force use android client fallback
2012             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2013             'info_dict': {
2014                 'id': 'YOelRv7fMxY',
2015                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2016                 'ext': '3gp',
2017                 'upload_date': '20210624',
2018                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2019                 'uploader': 'colinfurze',
2020                 'uploader_id': 'colinfurze',
2021                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2022                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2023                 'duration': 596,
2024                 'categories': ['Entertainment'],
2025                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2026                 'view_count': int,
2027                 'channel': 'colinfurze',
2028                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2029                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2030                 'age_limit': 0,
2031                 'availability': 'public',
2032                 'like_count': int,
2033                 'live_status': 'not_live',
2034                 'playable_in_embed': True,
2035                 'channel_follower_count': int
2036             },
2037             'params': {
2038                 'format': '17',  # 3gp format available on android
2039                 'extractor_args': {'youtube': {'player_client': ['android']}},
2040             },
2041         },
2042         {
2043             # Skip download of additional client configs (remix client config in this case)
2044             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2045             'only_matching': True,
2046             'params': {
2047                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2048             },
2049         }, {
2050             # shorts
2051             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2052             'only_matching': True,
2053         }, {
2054             'note': 'Storyboards',
2055             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2056             'info_dict': {
2057                 'id': '5KLPxDtMqe8',
2058                 'ext': 'mhtml',
2059                 'format_id': 'sb0',
2060                 'title': 'Your Brain is Plastic',
2061                 'uploader_id': 'scishow',
2062                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2063                 'upload_date': '20140324',
2064                 'uploader': 'SciShow',
2065                 'like_count': int,
2066                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2067                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2068                 'view_count': int,
2069                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2070                 'playable_in_embed': True,
2071                 'tags': 'count:12',
2072                 'uploader_url': 'http://www.youtube.com/user/scishow',
2073                 'availability': 'public',
2074                 'channel': 'SciShow',
2075                 'live_status': 'not_live',
2076                 'duration': 248,
2077                 'categories': ['Education'],
2078                 'age_limit': 0,
2079                 'channel_follower_count': int
2080             }, 'params': {'format': 'mhtml', 'skip_download': True}
2081         }
2082     ]
2083
2084     @classmethod
2085     def suitable(cls, url):
2086         from ..utils import parse_qs
2087
2088         qs = parse_qs(url)
2089         if qs.get('list', [None])[0]:
2090             return False
2091         return super(YoutubeIE, cls).suitable(url)
2092
2093     def __init__(self, *args, **kwargs):
2094         super(YoutubeIE, self).__init__(*args, **kwargs)
2095         self._code_cache = {}
2096         self._player_cache = {}
2097
2098     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2099         lock = threading.Lock()
2100
2101         is_live = True
2102         start_time = time.time()
2103         formats = [f for f in formats if f.get('is_from_start')]
2104
2105         def refetch_manifest(format_id, delay):
2106             nonlocal formats, start_time, is_live
2107             if time.time() <= start_time + delay:
2108                 return
2109
2110             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2111             video_details = traverse_obj(
2112                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2113             microformats = traverse_obj(
2114                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2115                 expected_type=dict, default=[])
2116             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2117             start_time = time.time()
2118
2119         def mpd_feed(format_id, delay):
2120             """
2121             @returns (manifest_url, manifest_stream_number, is_live) or None
2122             """
2123             with lock:
2124                 refetch_manifest(format_id, delay)
2125
2126             f = next((f for f in formats if f['format_id'] == format_id), None)
2127             if not f:
2128                 if not is_live:
2129                     self.to_screen(f'{video_id}: Video is no longer live')
2130                 else:
2131                     self.report_warning(
2132                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2133                 return None
2134             return f['manifest_url'], f['manifest_stream_number'], is_live
2135
2136         for f in formats:
2137             f['protocol'] = 'http_dash_segments_generator'
2138             f['fragments'] = functools.partial(
2139                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2140
2141     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2142         FETCH_SPAN, MAX_DURATION = 5, 432000
2143
2144         mpd_url, stream_number, is_live = None, None, True
2145
2146         begin_index = 0
2147         download_start_time = ctx.get('start') or time.time()
2148
2149         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2150         if lack_early_segments:
2151             self.report_warning(bug_reports_message(
2152                 'Starting download from the last 120 hours of the live stream since '
2153                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2154             lack_early_segments = True
2155
2156         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2157         fragments, fragment_base_url = None, None
2158
2159         def _extract_sequence_from_mpd(refresh_sequence):
2160             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2161             # Obtain from MPD's maximum seq value
2162             old_mpd_url = mpd_url
2163             last_error = ctx.pop('last_error', None)
2164             expire_fast = last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2165             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2166                                                or (mpd_url, stream_number, False))
2167             if not refresh_sequence:
2168                 if expire_fast and not is_live:
2169                     return False, last_seq
2170                 elif old_mpd_url == mpd_url:
2171                     return True, last_seq
2172             try:
2173                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2174                     mpd_url, None, note=False, errnote=False, fatal=False)
2175             except ExtractorError:
2176                 fmts = None
2177             if not fmts:
2178                 no_fragment_score += 1
2179                 return False, last_seq
2180             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2181             fragments = fmt_info['fragments']
2182             fragment_base_url = fmt_info['fragment_base_url']
2183             assert fragment_base_url
2184
2185             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2186             return True, _last_seq
2187
2188         while is_live:
2189             fetch_time = time.time()
2190             if no_fragment_score > 30:
2191                 return
2192             if last_segment_url:
2193                 # Obtain from "X-Head-Seqnum" header value from each segment
2194                 try:
2195                     urlh = self._request_webpage(
2196                         last_segment_url, None, note=False, errnote=False, fatal=False)
2197                 except ExtractorError:
2198                     urlh = None
2199                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2200                 if last_seq is None:
2201                     no_fragment_score += 1
2202                     last_segment_url = None
2203                     continue
2204             else:
2205                 should_continue, last_seq = _extract_sequence_from_mpd(True)
2206                 if not should_continue:
2207                     continue
2208
2209             if known_idx > last_seq:
2210                 last_segment_url = None
2211                 continue
2212
2213             last_seq += 1
2214
2215             if begin_index < 0 and known_idx < 0:
2216                 # skip from the start when it's negative value
2217                 known_idx = last_seq + begin_index
2218             if lack_early_segments:
2219                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2220             try:
2221                 for idx in range(known_idx, last_seq):
2222                     # do not update sequence here or you'll get skipped some part of it
2223                     should_continue, _ = _extract_sequence_from_mpd(False)
2224                     if not should_continue:
2225                         known_idx = idx - 1
2226                         raise ExtractorError('breaking out of outer loop')
2227                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2228                     yield {
2229                         'url': last_segment_url,
2230                     }
2231                 if known_idx == last_seq:
2232                     no_fragment_score += 5
2233                 else:
2234                     no_fragment_score = 0
2235                 known_idx = last_seq
2236             except ExtractorError:
2237                 continue
2238
2239             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2240
2241     def _extract_player_url(self, *ytcfgs, webpage=None):
2242         player_url = traverse_obj(
2243             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2244             get_all=False, expected_type=compat_str)
2245         if not player_url:
2246             return
2247         if player_url.startswith('//'):
2248             player_url = 'https:' + player_url
2249         elif not re.match(r'https?://', player_url):
2250             player_url = compat_urlparse.urljoin(
2251                 'https://www.youtube.com', player_url)
2252         return player_url
2253
2254     def _download_player_url(self, video_id, fatal=False):
2255         res = self._download_webpage(
2256             'https://www.youtube.com/iframe_api',
2257             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2258         if res:
2259             player_version = self._search_regex(
2260                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2261             if player_version:
2262                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2263
2264     def _signature_cache_id(self, example_sig):
2265         """ Return a string representation of a signature """
2266         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2267
2268     @classmethod
2269     def _extract_player_info(cls, player_url):
2270         for player_re in cls._PLAYER_INFO_RE:
2271             id_m = re.search(player_re, player_url)
2272             if id_m:
2273                 break
2274         else:
2275             raise ExtractorError('Cannot identify player %r' % player_url)
2276         return id_m.group('id')
2277
2278     def _load_player(self, video_id, player_url, fatal=True):
2279         player_id = self._extract_player_info(player_url)
2280         if player_id not in self._code_cache:
2281             code = self._download_webpage(
2282                 player_url, video_id, fatal=fatal,
2283                 note='Downloading player ' + player_id,
2284                 errnote='Download of %s failed' % player_url)
2285             if code:
2286                 self._code_cache[player_id] = code
2287         return self._code_cache.get(player_id)
2288
2289     def _extract_signature_function(self, video_id, player_url, example_sig):
2290         player_id = self._extract_player_info(player_url)
2291
2292         # Read from filesystem cache
2293         func_id = 'js_%s_%s' % (
2294             player_id, self._signature_cache_id(example_sig))
2295         assert os.path.basename(func_id) == func_id
2296
2297         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
2298         if cache_spec is not None:
2299             return lambda s: ''.join(s[i] for i in cache_spec)
2300
2301         code = self._load_player(video_id, player_url)
2302         if code:
2303             res = self._parse_sig_js(code)
2304
2305             test_string = ''.join(map(compat_chr, range(len(example_sig))))
2306             cache_res = res(test_string)
2307             cache_spec = [ord(c) for c in cache_res]
2308
2309             self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
2310             return res
2311
2312     def _print_sig_code(self, func, example_sig):
2313         if not self.get_param('youtube_print_sig_code'):
2314             return
2315
2316         def gen_sig_code(idxs):
2317             def _genslice(start, end, step):
2318                 starts = '' if start == 0 else str(start)
2319                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2320                 steps = '' if step == 1 else (':%d' % step)
2321                 return 's[%s%s%s]' % (starts, ends, steps)
2322
2323             step = None
2324             # Quelch pyflakes warnings - start will be set when step is set
2325             start = '(Never used)'
2326             for i, prev in zip(idxs[1:], idxs[:-1]):
2327                 if step is not None:
2328                     if i - prev == step:
2329                         continue
2330                     yield _genslice(start, prev, step)
2331                     step = None
2332                     continue
2333                 if i - prev in [-1, 1]:
2334                     step = i - prev
2335                     start = prev
2336                     continue
2337                 else:
2338                     yield 's[%d]' % prev
2339             if step is None:
2340                 yield 's[%d]' % i
2341             else:
2342                 yield _genslice(start, i, step)
2343
2344         test_string = ''.join(map(compat_chr, range(len(example_sig))))
2345         cache_res = func(test_string)
2346         cache_spec = [ord(c) for c in cache_res]
2347         expr_code = ' + '.join(gen_sig_code(cache_spec))
2348         signature_id_tuple = '(%s)' % (
2349             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2350         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2351                 '    return %s\n') % (signature_id_tuple, expr_code)
2352         self.to_screen('Extracted signature function:\n' + code)
2353
2354     def _parse_sig_js(self, jscode):
2355         funcname = self._search_regex(
2356             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2357              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2358              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2359              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2360              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2361              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2362              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2363              # Obsolete patterns
2364              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2365              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2366              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2367              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2368              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2369              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2370              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2371              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2372             jscode, 'Initial JS player signature function name', group='sig')
2373
2374         jsi = JSInterpreter(jscode)
2375         initial_function = jsi.extract_function(funcname)
2376         return lambda s: initial_function([s])
2377
2378     def _decrypt_signature(self, s, video_id, player_url):
2379         """Turn the encrypted s field into a working signature"""
2380
2381         if player_url is None:
2382             raise ExtractorError('Cannot decrypt signature without player_url')
2383
2384         try:
2385             player_id = (player_url, self._signature_cache_id(s))
2386             if player_id not in self._player_cache:
2387                 func = self._extract_signature_function(
2388                     video_id, player_url, s
2389                 )
2390                 self._player_cache[player_id] = func
2391             func = self._player_cache[player_id]
2392             self._print_sig_code(func, s)
2393             return func(s)
2394         except Exception as e:
2395             raise ExtractorError('Signature extraction failed: ' + traceback.format_exc(), cause=e)
2396
2397     def _decrypt_nsig(self, s, video_id, player_url):
2398         """Turn the encrypted n field into a working signature"""
2399         if player_url is None:
2400             raise ExtractorError('Cannot decrypt nsig without player_url')
2401         if player_url.startswith('//'):
2402             player_url = 'https:' + player_url
2403         elif not re.match(r'https?://', player_url):
2404             player_url = compat_urlparse.urljoin(
2405                 'https://www.youtube.com', player_url)
2406
2407         sig_id = ('nsig_value', s)
2408         if sig_id in self._player_cache:
2409             return self._player_cache[sig_id]
2410
2411         try:
2412             player_id = ('nsig', player_url)
2413             if player_id not in self._player_cache:
2414                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2415             func = self._player_cache[player_id]
2416             self._player_cache[sig_id] = func(s)
2417             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2418             return self._player_cache[sig_id]
2419         except Exception as e:
2420             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2421
2422     def _extract_n_function_name(self, jscode):
2423         return self._search_regex(
2424             (r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]{3})\([a-zA-Z0-9]\)',),
2425             jscode, 'Initial JS player n function name', group='nfunc')
2426
2427     def _extract_n_function(self, video_id, player_url):
2428         player_id = self._extract_player_info(player_url)
2429         func_code = self._downloader.cache.load('youtube-nsig', player_id)
2430
2431         if func_code:
2432             jsi = JSInterpreter(func_code)
2433         else:
2434             jscode = self._load_player(video_id, player_url)
2435             funcname = self._extract_n_function_name(jscode)
2436             jsi = JSInterpreter(jscode)
2437             func_code = jsi.extract_function_code(funcname)
2438             self._downloader.cache.store('youtube-nsig', player_id, func_code)
2439
2440         if self.get_param('youtube_print_sig_code'):
2441             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2442
2443         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2444
2445     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2446         """
2447         Extract signatureTimestamp (sts)
2448         Required to tell API what sig/player version is in use.
2449         """
2450         sts = None
2451         if isinstance(ytcfg, dict):
2452             sts = int_or_none(ytcfg.get('STS'))
2453
2454         if not sts:
2455             # Attempt to extract from player
2456             if player_url is None:
2457                 error_msg = 'Cannot extract signature timestamp without player_url.'
2458                 if fatal:
2459                     raise ExtractorError(error_msg)
2460                 self.report_warning(error_msg)
2461                 return
2462             code = self._load_player(video_id, player_url, fatal=fatal)
2463             if code:
2464                 sts = int_or_none(self._search_regex(
2465                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2466                     'JS player signature timestamp', group='sts', fatal=fatal))
2467         return sts
2468
2469     def _mark_watched(self, video_id, player_responses):
2470         playback_url = get_first(
2471             player_responses, ('playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
2472             expected_type=url_or_none)
2473         if not playback_url:
2474             self.report_warning('Unable to mark watched')
2475             return
2476         parsed_playback_url = compat_urlparse.urlparse(playback_url)
2477         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
2478
2479         # cpn generation algorithm is reverse engineered from base.js.
2480         # In fact it works even with dummy cpn.
2481         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2482         cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
2483
2484         qs.update({
2485             'ver': ['2'],
2486             'cpn': [cpn],
2487         })
2488         playback_url = compat_urlparse.urlunparse(
2489             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2490
2491         self._download_webpage(
2492             playback_url, video_id, 'Marking watched',
2493             'Unable to mark watched', fatal=False)
2494
2495     @staticmethod
2496     def _extract_urls(webpage):
2497         # Embedded YouTube player
2498         entries = [
2499             unescapeHTML(mobj.group('url'))
2500             for mobj in re.finditer(r'''(?x)
2501             (?:
2502                 <iframe[^>]+?src=|
2503                 data-video-url=|
2504                 <embed[^>]+?src=|
2505                 embedSWF\(?:\s*|
2506                 <object[^>]+data=|
2507                 new\s+SWFObject\(
2508             )
2509             (["\'])
2510                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2511                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2512             \1''', webpage)]
2513
2514         # lazyYT YouTube embed
2515         entries.extend(list(map(
2516             unescapeHTML,
2517             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2518
2519         # Wordpress "YouTube Video Importer" plugin
2520         matches = re.findall(r'''(?x)<div[^>]+
2521             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2522             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2523         entries.extend(m[-1] for m in matches)
2524
2525         return entries
2526
2527     @staticmethod
2528     def _extract_url(webpage):
2529         urls = YoutubeIE._extract_urls(webpage)
2530         return urls[0] if urls else None
2531
2532     @classmethod
2533     def extract_id(cls, url):
2534         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2535         if mobj is None:
2536             raise ExtractorError('Invalid URL: %s' % url)
2537         return mobj.group('id')
2538
2539     def _extract_chapters_from_json(self, data, duration):
2540         chapter_list = traverse_obj(
2541             data, (
2542                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2543                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2544             ), expected_type=list)
2545
2546         return self._extract_chapters(
2547             chapter_list,
2548             chapter_time=lambda chapter: float_or_none(
2549                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2550             chapter_title=lambda chapter: traverse_obj(
2551                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2552             duration=duration)
2553
2554     def _extract_chapters_from_engagement_panel(self, data, duration):
2555         content_list = traverse_obj(
2556             data,
2557             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2558             expected_type=list, default=[])
2559         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2560         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2561
2562         return next((
2563             filter(None, (
2564                 self._extract_chapters(
2565                     traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2566                     chapter_time, chapter_title, duration)
2567                 for contents in content_list
2568             ))), [])
2569
2570     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration):
2571         chapters = []
2572         last_chapter = {'start_time': 0}
2573         for idx, chapter in enumerate(chapter_list or []):
2574             title = chapter_title(chapter)
2575             start_time = chapter_time(chapter)
2576             if start_time is None:
2577                 continue
2578             last_chapter['end_time'] = start_time
2579             if start_time < last_chapter['start_time']:
2580                 if idx == 1:
2581                     chapters.pop()
2582                     self.report_warning('Invalid start time for chapter "%s"' % last_chapter['title'])
2583                 else:
2584                     self.report_warning(f'Invalid start time for chapter "{title}"')
2585                     continue
2586             last_chapter = {'start_time': start_time, 'title': title}
2587             chapters.append(last_chapter)
2588         last_chapter['end_time'] = duration
2589         return chapters
2590
2591     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
2592         return self._parse_json(self._search_regex(
2593             (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
2594              regex), webpage, name, default='{}'), video_id, fatal=False)
2595
2596     def _extract_comment(self, comment_renderer, parent=None):
2597         comment_id = comment_renderer.get('commentId')
2598         if not comment_id:
2599             return
2600
2601         text = self._get_text(comment_renderer, 'contentText')
2602
2603         # note: timestamp is an estimate calculated from the current time and time_text
2604         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2605         author = self._get_text(comment_renderer, 'authorText')
2606         author_id = try_get(comment_renderer,
2607                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2608
2609         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2610                                                        lambda x: x['likeCount']), compat_str)) or 0
2611         author_thumbnail = try_get(comment_renderer,
2612                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2613
2614         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2615         is_favorited = 'creatorHeart' in (try_get(
2616             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2617         return {
2618             'id': comment_id,
2619             'text': text,
2620             'timestamp': timestamp,
2621             'time_text': time_text,
2622             'like_count': votes,
2623             'is_favorited': is_favorited,
2624             'author': author,
2625             'author_id': author_id,
2626             'author_thumbnail': author_thumbnail,
2627             'author_is_uploader': author_is_uploader,
2628             'parent': parent or 'root'
2629         }
2630
2631     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2632
2633         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2634
2635         def extract_header(contents):
2636             _continuation = None
2637             for content in contents:
2638                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2639                 expected_comment_count = self._get_count(
2640                     comments_header_renderer, 'countText', 'commentsCount')
2641
2642                 if expected_comment_count:
2643                     tracker['est_total'] = expected_comment_count
2644                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2645                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2646
2647                 sort_menu_item = try_get(
2648                     comments_header_renderer,
2649                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2650                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2651
2652                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2653                 if not _continuation:
2654                     continue
2655
2656                 sort_text = str_or_none(sort_menu_item.get('title'))
2657                 if not sort_text:
2658                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2659                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2660                 break
2661             return _continuation
2662
2663         def extract_thread(contents):
2664             if not parent:
2665                 tracker['current_page_thread'] = 0
2666             for content in contents:
2667                 if not parent and tracker['total_parent_comments'] >= max_parents:
2668                     yield
2669                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2670                 comment_renderer = get_first(
2671                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2672                     expected_type=dict, default={})
2673
2674                 comment = self._extract_comment(comment_renderer, parent)
2675                 if not comment:
2676                     continue
2677
2678                 tracker['running_total'] += 1
2679                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2680                 yield comment
2681
2682                 # Attempt to get the replies
2683                 comment_replies_renderer = try_get(
2684                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2685
2686                 if comment_replies_renderer:
2687                     tracker['current_page_thread'] += 1
2688                     comment_entries_iter = self._comment_entries(
2689                         comment_replies_renderer, ytcfg, video_id,
2690                         parent=comment.get('id'), tracker=tracker)
2691                     for reply_comment in itertools.islice(comment_entries_iter, min(max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments']))):
2692                         yield reply_comment
2693
2694         # Keeps track of counts across recursive calls
2695         if not tracker:
2696             tracker = dict(
2697                 running_total=0,
2698                 est_total=0,
2699                 current_page_thread=0,
2700                 total_parent_comments=0,
2701                 total_reply_comments=0)
2702
2703         # TODO: Deprecated
2704         # YouTube comments have a max depth of 2
2705         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2706         if max_depth:
2707             self._downloader.deprecation_warning(
2708                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2709         if max_depth == 1 and parent:
2710             return
2711
2712         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2713             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2714
2715         continuation = self._extract_continuation(root_continuation_data)
2716         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2717         if message and not parent:
2718             self.report_warning(message, video_id=video_id)
2719
2720         response = None
2721         is_first_continuation = parent is None
2722
2723         for page_num in itertools.count(0):
2724             if not continuation:
2725                 break
2726             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2727             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2728             if page_num == 0:
2729                 if is_first_continuation:
2730                     note_prefix = 'Downloading comment section API JSON'
2731                 else:
2732                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2733                         tracker['current_page_thread'], comment_prog_str)
2734             else:
2735                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2736                     '       ' if parent else '', ' replies' if parent else '',
2737                     page_num, comment_prog_str)
2738
2739             response = self._extract_response(
2740                 item_id=None, query=continuation,
2741                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2742                 check_get_keys='onResponseReceivedEndpoints')
2743
2744             continuation_contents = traverse_obj(
2745                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2746
2747             continuation = None
2748             for continuation_section in continuation_contents:
2749                 continuation_items = traverse_obj(
2750                     continuation_section,
2751                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2752                     get_all=False, expected_type=list) or []
2753                 if is_first_continuation:
2754                     continuation = extract_header(continuation_items)
2755                     is_first_continuation = False
2756                     if continuation:
2757                         break
2758                     continue
2759
2760                 for entry in extract_thread(continuation_items):
2761                     if not entry:
2762                         return
2763                     yield entry
2764                 continuation = self._extract_continuation({'contents': continuation_items})
2765                 if continuation:
2766                     break
2767
2768     def _get_comments(self, ytcfg, video_id, contents, webpage):
2769         """Entry for comment extraction"""
2770         def _real_comment_extract(contents):
2771             renderer = next((
2772                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2773                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2774             yield from self._comment_entries(renderer, ytcfg, video_id)
2775
2776         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2777         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2778
2779     @staticmethod
2780     def _get_checkok_params():
2781         return {'contentCheckOk': True, 'racyCheckOk': True}
2782
2783     @classmethod
2784     def _generate_player_context(cls, sts=None):
2785         context = {
2786             'html5Preference': 'HTML5_PREF_WANTS',
2787         }
2788         if sts is not None:
2789             context['signatureTimestamp'] = sts
2790         return {
2791             'playbackContext': {
2792                 'contentPlaybackContext': context
2793             },
2794             **cls._get_checkok_params()
2795         }
2796
2797     @staticmethod
2798     def _is_agegated(player_response):
2799         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
2800             return True
2801
2802         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
2803         AGE_GATE_REASONS = (
2804             'confirm your age', 'age-restricted', 'inappropriate',  # reason
2805             'age_verification_required', 'age_check_required',  # status
2806         )
2807         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
2808
2809     @staticmethod
2810     def _is_unplayable(player_response):
2811         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
2812
2813     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
2814
2815         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
2816         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
2817         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
2818         headers = self.generate_api_headers(
2819             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
2820
2821         yt_query = {'videoId': video_id}
2822         yt_query.update(self._generate_player_context(sts))
2823         return self._extract_response(
2824             item_id=video_id, ep='player', query=yt_query,
2825             ytcfg=player_ytcfg, headers=headers, fatal=True,
2826             default_client=client,
2827             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
2828         ) or None
2829
2830     def _get_requested_clients(self, url, smuggled_data):
2831         requested_clients = []
2832         default = ['android', 'web']
2833         allowed_clients = sorted(
2834             [client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'],
2835             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
2836         for client in self._configuration_arg('player_client'):
2837             if client in allowed_clients:
2838                 requested_clients.append(client)
2839             elif client == 'default':
2840                 requested_clients.extend(default)
2841             elif client == 'all':
2842                 requested_clients.extend(allowed_clients)
2843             else:
2844                 self.report_warning(f'Skipping unsupported client {client}')
2845         if not requested_clients:
2846             requested_clients = default
2847
2848         if smuggled_data.get('is_music_url') or self.is_music_url(url):
2849             requested_clients.extend(
2850                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
2851
2852         return orderedSet(requested_clients)
2853
2854     def _extract_player_ytcfg(self, client, video_id):
2855         url = {
2856             'web_music': 'https://music.youtube.com',
2857             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
2858         }.get(client)
2859         if not url:
2860             return {}
2861         webpage = self._download_webpage(url, video_id, fatal=False, note='Downloading %s config' % client.replace('_', ' ').strip())
2862         return self.extract_ytcfg(video_id, webpage) or {}
2863
2864     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
2865         initial_pr = None
2866         if webpage:
2867             initial_pr = self._extract_yt_initial_variable(
2868                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
2869                 video_id, 'initial player response')
2870
2871         original_clients = clients
2872         clients = clients[::-1]
2873         prs = []
2874
2875         def append_client(client_name):
2876             if client_name in INNERTUBE_CLIENTS and client_name not in original_clients:
2877                 clients.append(client_name)
2878
2879         # Android player_response does not have microFormats which are needed for
2880         # extraction of some data. So we return the initial_pr with formats
2881         # stripped out even if not requested by the user
2882         # See: https://github.com/yt-dlp/yt-dlp/issues/501
2883         if initial_pr:
2884             pr = dict(initial_pr)
2885             pr['streamingData'] = None
2886             prs.append(pr)
2887
2888         last_error = None
2889         tried_iframe_fallback = False
2890         player_url = None
2891         while clients:
2892             client = clients.pop()
2893             player_ytcfg = master_ytcfg if client == 'web' else {}
2894             if 'configs' not in self._configuration_arg('player_skip'):
2895                 player_ytcfg = self._extract_player_ytcfg(client, video_id) or player_ytcfg
2896
2897             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
2898             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
2899             if 'js' in self._configuration_arg('player_skip'):
2900                 require_js_player = False
2901                 player_url = None
2902
2903             if not player_url and not tried_iframe_fallback and require_js_player:
2904                 player_url = self._download_player_url(video_id)
2905                 tried_iframe_fallback = True
2906
2907             try:
2908                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
2909                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
2910             except ExtractorError as e:
2911                 if last_error:
2912                     self.report_warning(last_error)
2913                 last_error = e
2914                 continue
2915
2916             if pr:
2917                 prs.append(pr)
2918
2919             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
2920             if client.endswith('_agegate') and self._is_unplayable(pr) and self.is_authenticated:
2921                 append_client(client.replace('_agegate', '_creator'))
2922             elif self._is_agegated(pr):
2923                 append_client(f'{client}_agegate')
2924
2925         if last_error:
2926             if not len(prs):
2927                 raise last_error
2928             self.report_warning(last_error)
2929         return prs, player_url
2930
2931     def _extract_formats(self, streaming_data, video_id, player_url, is_live):
2932         itags, stream_ids = {}, []
2933         itag_qualities, res_qualities = {}, {}
2934         q = qualities([
2935             # Normally tiny is the smallest video-only formats. But
2936             # audio-only formats with unknown quality may get tagged as tiny
2937             'tiny',
2938             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
2939             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
2940         ])
2941         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
2942
2943         for fmt in streaming_formats:
2944             if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
2945                 continue
2946
2947             itag = str_or_none(fmt.get('itag'))
2948             audio_track = fmt.get('audioTrack') or {}
2949             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
2950             if stream_id in stream_ids:
2951                 continue
2952
2953             quality = fmt.get('quality')
2954             height = int_or_none(fmt.get('height'))
2955             if quality == 'tiny' or not quality:
2956                 quality = fmt.get('audioQuality', '').lower() or quality
2957             # The 3gp format (17) in android client has a quality of "small",
2958             # but is actually worse than other formats
2959             if itag == '17':
2960                 quality = 'tiny'
2961             if quality:
2962                 if itag:
2963                     itag_qualities[itag] = quality
2964                 if height:
2965                     res_qualities[height] = quality
2966             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
2967             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
2968             # number of fragment that would subsequently requested with (`&sq=N`)
2969             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
2970                 continue
2971
2972             fmt_url = fmt.get('url')
2973             if not fmt_url:
2974                 sc = compat_parse_qs(fmt.get('signatureCipher'))
2975                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
2976                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
2977                 if not (sc and fmt_url and encrypted_sig):
2978                     continue
2979                 if not player_url:
2980                     continue
2981                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
2982                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
2983                 fmt_url += '&' + sp + '=' + signature
2984
2985             query = parse_qs(fmt_url)
2986             throttled = False
2987             if query.get('n'):
2988                 try:
2989                     fmt_url = update_url_query(fmt_url, {
2990                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
2991                 except ExtractorError as e:
2992                     self.report_warning(
2993                         f'nsig extraction failed: You may experience throttling for some formats\n'
2994                         f'n = {query["n"][0]} ; player = {player_url}\n{e}', only_once=True)
2995                     throttled = True
2996
2997             if itag:
2998                 itags[itag] = 'https'
2999                 stream_ids.append(stream_id)
3000
3001             tbr = float_or_none(
3002                 fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3003             dct = {
3004                 'asr': int_or_none(fmt.get('audioSampleRate')),
3005                 'filesize': int_or_none(fmt.get('contentLength')),
3006                 'format_id': itag,
3007                 'format_note': join_nonempty(
3008                     '%s%s' % (audio_track.get('displayName') or '',
3009                               ' (default)' if audio_track.get('audioIsDefault') else ''),
3010                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3011                     throttled and 'THROTTLED', delim=', '),
3012                 'source_preference': -10 if throttled else -1,
3013                 'fps': int_or_none(fmt.get('fps')) or None,
3014                 'height': height,
3015                 'quality': q(quality),
3016                 'tbr': tbr,
3017                 'url': fmt_url,
3018                 'width': int_or_none(fmt.get('width')),
3019                 'language': audio_track.get('id', '').split('.')[0],
3020                 'language_preference': 1 if audio_track.get('audioIsDefault') else -1,
3021             }
3022             mime_mobj = re.match(
3023                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3024             if mime_mobj:
3025                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3026                 dct.update(parse_codecs(mime_mobj.group(2)))
3027             no_audio = dct.get('acodec') == 'none'
3028             no_video = dct.get('vcodec') == 'none'
3029             if no_audio:
3030                 dct['vbr'] = tbr
3031             if no_video:
3032                 dct['abr'] = tbr
3033             if no_audio or no_video:
3034                 dct['downloader_options'] = {
3035                     # Youtube throttles chunks >~10M
3036                     'http_chunk_size': 10485760,
3037                 }
3038                 if dct.get('ext'):
3039                     dct['container'] = dct['ext'] + '_dash'
3040             yield dct
3041
3042         live_from_start = is_live and self.get_param('live_from_start')
3043         skip_manifests = self._configuration_arg('skip')
3044         if not self.get_param('youtube_include_hls_manifest', True):
3045             skip_manifests.append('hls')
3046         get_dash = 'dash' not in skip_manifests and (
3047             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3048         get_hls = not live_from_start and 'hls' not in skip_manifests
3049
3050         def process_manifest_format(f, proto, itag):
3051             if itag in itags:
3052                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3053                     return False
3054                 itag = f'{itag}-{proto}'
3055             if itag:
3056                 f['format_id'] = itag
3057                 itags[itag] = proto
3058
3059             f['quality'] = next((
3060                 q(qdict[val])
3061                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3062                 if val in qdict), -1)
3063             return True
3064
3065         for sd in streaming_data:
3066             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3067             if hls_manifest_url:
3068                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3069                     if process_manifest_format(f, 'hls', self._search_regex(
3070                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3071                         yield f
3072
3073             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3074             if dash_manifest_url:
3075                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3076                     if process_manifest_format(f, 'dash', f['format_id']):
3077                         f['filesize'] = int_or_none(self._search_regex(
3078                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3079                         if live_from_start:
3080                             f['is_from_start'] = True
3081
3082                         yield f
3083
3084     def _extract_storyboard(self, player_responses, duration):
3085         spec = get_first(
3086             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3087         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3088         if not base_url:
3089             return
3090         L = len(spec) - 1
3091         for i, args in enumerate(spec):
3092             args = args.split('#')
3093             counts = list(map(int_or_none, args[:5]))
3094             if len(args) != 8 or not all(counts):
3095                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3096                 continue
3097             width, height, frame_count, cols, rows = counts
3098             N, sigh = args[6:]
3099
3100             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3101             fragment_count = frame_count / (cols * rows)
3102             fragment_duration = duration / fragment_count
3103             yield {
3104                 'format_id': f'sb{i}',
3105                 'format_note': 'storyboard',
3106                 'ext': 'mhtml',
3107                 'protocol': 'mhtml',
3108                 'acodec': 'none',
3109                 'vcodec': 'none',
3110                 'url': url,
3111                 'width': width,
3112                 'height': height,
3113                 'fragments': [{
3114                     'path': url.replace('$M', str(j)),
3115                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3116                 } for j in range(math.ceil(fragment_count))],
3117             }
3118
3119     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3120         webpage = None
3121         if 'webpage' not in self._configuration_arg('player_skip'):
3122             webpage = self._download_webpage(
3123                 webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
3124
3125         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3126
3127         player_responses, player_url = self._extract_player_responses(
3128             self._get_requested_clients(url, smuggled_data),
3129             video_id, webpage, master_ytcfg)
3130
3131         return webpage, master_ytcfg, player_responses, player_url
3132
3133     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url):
3134         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3135         is_live = get_first(video_details, 'isLive')
3136         if is_live is None:
3137             is_live = get_first(live_broadcast_details, 'isLiveNow')
3138
3139         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3140         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live))
3141
3142         return live_broadcast_details, is_live, streaming_data, formats
3143
3144     def _real_extract(self, url):
3145         url, smuggled_data = unsmuggle_url(url, {})
3146         video_id = self._match_id(url)
3147
3148         base_url = self.http_scheme() + '//www.youtube.com/'
3149         webpage_url = base_url + 'watch?v=' + video_id
3150
3151         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3152
3153         playability_statuses = traverse_obj(
3154             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3155
3156         trailer_video_id = get_first(
3157             playability_statuses,
3158             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3159             expected_type=str)
3160         if trailer_video_id:
3161             return self.url_result(
3162                 trailer_video_id, self.ie_key(), trailer_video_id)
3163
3164         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3165                        if webpage else (lambda x: None))
3166
3167         video_details = traverse_obj(
3168             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3169         microformats = traverse_obj(
3170             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3171             expected_type=dict, default=[])
3172         video_title = (
3173             get_first(video_details, 'title')
3174             or self._get_text(microformats, (..., 'title'))
3175             or search_meta(['og:title', 'twitter:title', 'title']))
3176         video_description = get_first(video_details, 'shortDescription')
3177
3178         multifeed_metadata_list = get_first(
3179             player_responses,
3180             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3181             expected_type=str)
3182         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3183             if self.get_param('noplaylist'):
3184                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3185             else:
3186                 entries = []
3187                 feed_ids = []
3188                 for feed in multifeed_metadata_list.split(','):
3189                     # Unquote should take place before split on comma (,) since textual
3190                     # fields may contain comma as well (see
3191                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3192                     feed_data = compat_parse_qs(
3193                         compat_urllib_parse_unquote_plus(feed))
3194
3195                     def feed_entry(name):
3196                         return try_get(
3197                             feed_data, lambda x: x[name][0], compat_str)
3198
3199                     feed_id = feed_entry('id')
3200                     if not feed_id:
3201                         continue
3202                     feed_title = feed_entry('title')
3203                     title = video_title
3204                     if feed_title:
3205                         title += ' (%s)' % feed_title
3206                     entries.append({
3207                         '_type': 'url_transparent',
3208                         'ie_key': 'Youtube',
3209                         'url': smuggle_url(
3210                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3211                             {'force_singlefeed': True}),
3212                         'title': title,
3213                     })
3214                     feed_ids.append(feed_id)
3215                 self.to_screen(
3216                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3217                     % (', '.join(feed_ids), video_id))
3218                 return self.playlist_result(
3219                     entries, video_id, video_title, video_description)
3220
3221         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(video_id, microformats, video_details, player_responses, player_url)
3222
3223         if not formats:
3224             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3225                 self.report_drm(video_id)
3226             pemr = get_first(
3227                 playability_statuses,
3228                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3229             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3230             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3231             if subreason:
3232                 if subreason == 'The uploader has not made this video available in your country.':
3233                     countries = get_first(microformats, 'availableCountries')
3234                     if not countries:
3235                         regions_allowed = search_meta('regionsAllowed')
3236                         countries = regions_allowed.split(',') if regions_allowed else None
3237                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3238                 reason += f'. {subreason}'
3239             if reason:
3240                 self.raise_no_formats(reason, expected=True)
3241
3242         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3243         if not keywords and webpage:
3244             keywords = [
3245                 unescapeHTML(m.group('content'))
3246                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3247         for keyword in keywords:
3248             if keyword.startswith('yt:stretch='):
3249                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3250                 if mobj:
3251                     # NB: float is intentional for forcing float division
3252                     w, h = (float(v) for v in mobj.groups())
3253                     if w > 0 and h > 0:
3254                         ratio = w / h
3255                         for f in formats:
3256                             if f.get('vcodec') != 'none':
3257                                 f['stretched_ratio'] = ratio
3258                         break
3259         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3260         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3261         if thumbnail_url:
3262             thumbnails.append({
3263                 'url': thumbnail_url,
3264             })
3265         original_thumbnails = thumbnails.copy()
3266
3267         # The best resolution thumbnails sometimes does not appear in the webpage
3268         # See: https://github.com/ytdl-org/youtube-dl/issues/29049, https://github.com/yt-dlp/yt-dlp/issues/340
3269         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3270         thumbnail_names = [
3271             'maxresdefault', 'hq720', 'sddefault', 'sd1', 'sd2', 'sd3',
3272             'hqdefault', 'hq1', 'hq2', 'hq3', '0',
3273             'mqdefault', 'mq1', 'mq2', 'mq3',
3274             'default', '1', '2', '3'
3275         ]
3276         n_thumbnail_names = len(thumbnail_names)
3277         thumbnails.extend({
3278             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3279                 video_id=video_id, name=name, ext=ext,
3280                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3281         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3282         for thumb in thumbnails:
3283             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3284             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3285         self._remove_duplicate_formats(thumbnails)
3286         self._downloader._sort_thumbnails(original_thumbnails)
3287
3288         category = get_first(microformats, 'category') or search_meta('genre')
3289         channel_id = str_or_none(
3290             get_first(video_details, 'channelId')
3291             or get_first(microformats, 'externalChannelId')
3292             or search_meta('channelId'))
3293         duration = int_or_none(
3294             get_first(video_details, 'lengthSeconds')
3295             or get_first(microformats, 'lengthSeconds')
3296             or parse_duration(search_meta('duration'))) or None
3297         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3298
3299         live_content = get_first(video_details, 'isLiveContent')
3300         is_upcoming = get_first(video_details, 'isUpcoming')
3301         if is_live is None:
3302             if is_upcoming or live_content is False:
3303                 is_live = False
3304         if is_upcoming is None and (live_content or is_live):
3305             is_upcoming = False
3306         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3307         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3308         if not duration and live_end_time and live_start_time:
3309             duration = live_end_time - live_start_time
3310
3311         if is_live and self.get_param('live_from_start'):
3312             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3313
3314         formats.extend(self._extract_storyboard(player_responses, duration))
3315
3316         # Source is given priority since formats that throttle are given lower source_preference
3317         # When throttling issue is fully fixed, remove this
3318         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3319
3320         info = {
3321             'id': video_id,
3322             'title': video_title,
3323             'formats': formats,
3324             'thumbnails': thumbnails,
3325             # The best thumbnail that we are sure exists. Prevents unnecessary
3326             # URL checking if user don't care about getting the best possible thumbnail
3327             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3328             'description': video_description,
3329             'upload_date': unified_strdate(
3330                 get_first(microformats, 'uploadDate')
3331                 or search_meta('uploadDate')),
3332             'uploader': get_first(video_details, 'author'),
3333             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3334             'uploader_url': owner_profile_url,
3335             'channel_id': channel_id,
3336             'channel_url': format_field(channel_id, template='https://www.youtube.com/channel/%s'),
3337             'duration': duration,
3338             'view_count': int_or_none(
3339                 get_first((video_details, microformats), (..., 'viewCount'))
3340                 or search_meta('interactionCount')),
3341             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3342             'age_limit': 18 if (
3343                 get_first(microformats, 'isFamilySafe') is False
3344                 or search_meta('isFamilyFriendly') == 'false'
3345                 or search_meta('og:restrictions:age') == '18+') else 0,
3346             'webpage_url': webpage_url,
3347             'categories': [category] if category else None,
3348             'tags': keywords,
3349             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3350             'is_live': is_live,
3351             'was_live': (False if is_live or is_upcoming or live_content is False
3352                          else None if is_live is None or is_upcoming is None
3353                          else live_content),
3354             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3355             'release_timestamp': live_start_time,
3356         }
3357
3358         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3359         if pctr:
3360             def get_lang_code(track):
3361                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3362                         or track.get('languageCode'))
3363
3364             # Converted into dicts to remove duplicates
3365             captions = {
3366                 get_lang_code(sub): sub
3367                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3368             translation_languages = {
3369                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3370                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3371
3372             def process_language(container, base_url, lang_code, sub_name, query):
3373                 lang_subs = container.setdefault(lang_code, [])
3374                 for fmt in self._SUBTITLE_FORMATS:
3375                     query.update({
3376                         'fmt': fmt,
3377                     })
3378                     lang_subs.append({
3379                         'ext': fmt,
3380                         'url': update_url_query(base_url, query),
3381                         'name': sub_name,
3382                     })
3383
3384             subtitles, automatic_captions = {}, {}
3385             for lang_code, caption_track in captions.items():
3386                 base_url = caption_track.get('baseUrl')
3387                 if not base_url:
3388                     continue
3389                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3390                 if caption_track.get('kind') != 'asr':
3391                     if not lang_code:
3392                         continue
3393                     process_language(
3394                         subtitles, base_url, lang_code, lang_name, {})
3395                     if not caption_track.get('isTranslatable'):
3396                         continue
3397                 for trans_code, trans_name in translation_languages.items():
3398                     if not trans_code:
3399                         continue
3400                     if caption_track.get('kind') != 'asr':
3401                         trans_code += f'-{lang_code}'
3402                         trans_name += format_field(lang_name, template=' from %s')
3403                     process_language(
3404                         automatic_captions, base_url, trans_code, trans_name, {'tlang': trans_code})
3405             info['automatic_captions'] = automatic_captions
3406             info['subtitles'] = subtitles
3407
3408         parsed_url = compat_urllib_parse_urlparse(url)
3409         for component in [parsed_url.fragment, parsed_url.query]:
3410             query = compat_parse_qs(component)
3411             for k, v in query.items():
3412                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3413                     d_k += '_time'
3414                     if d_k not in info and k in s_ks:
3415                         info[d_k] = parse_duration(query[k][0])
3416
3417         # Youtube Music Auto-generated description
3418         if video_description:
3419             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
3420             if mobj:
3421                 release_year = mobj.group('release_year')
3422                 release_date = mobj.group('release_date')
3423                 if release_date:
3424                     release_date = release_date.replace('-', '')
3425                     if not release_year:
3426                         release_year = release_date[:4]
3427                 info.update({
3428                     'album': mobj.group('album'.strip()),
3429                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3430                     'track': mobj.group('track').strip(),
3431                     'release_date': release_date,
3432                     'release_year': int_or_none(release_year),
3433                 })
3434
3435         initial_data = None
3436         if webpage:
3437             initial_data = self._extract_yt_initial_variable(
3438                 webpage, self._YT_INITIAL_DATA_RE, video_id,
3439                 'yt initial data')
3440         if not initial_data:
3441             query = {'videoId': video_id}
3442             query.update(self._get_checkok_params())
3443             initial_data = self._extract_response(
3444                 item_id=video_id, ep='next', fatal=False,
3445                 ytcfg=master_ytcfg, query=query,
3446                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3447                 note='Downloading initial data API JSON')
3448
3449         try:
3450             # This will error if there is no livechat
3451             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3452             info.setdefault('subtitles', {})['live_chat'] = [{
3453                 'url': 'https://www.youtube.com/watch?v=%s' % video_id,  # url is needed to set cookies
3454                 'video_id': video_id,
3455                 'ext': 'json',
3456                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3457             }]
3458         except (KeyError, IndexError, TypeError):
3459             pass
3460
3461         if initial_data:
3462             info['chapters'] = (
3463                 self._extract_chapters_from_json(initial_data, duration)
3464                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3465                 or None)
3466
3467             contents = try_get(
3468                 initial_data,
3469                 lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'],
3470                 list) or []
3471             for content in contents:
3472                 vpir = content.get('videoPrimaryInfoRenderer')
3473                 if vpir:
3474                     stl = vpir.get('superTitleLink')
3475                     if stl:
3476                         stl = self._get_text(stl)
3477                         if try_get(
3478                                 vpir,
3479                                 lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3480                             info['location'] = stl
3481                         else:
3482                             mobj = re.search(r'(.+?)\s*S(\d+)\s*•\s*E(\d+)', stl)
3483                             if mobj:
3484                                 info.update({
3485                                     'series': mobj.group(1),
3486                                     'season_number': int(mobj.group(2)),
3487                                     'episode_number': int(mobj.group(3)),
3488                                 })
3489                     for tlb in (try_get(
3490                             vpir,
3491                             lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3492                             list) or []):
3493                         tbr = tlb.get('toggleButtonRenderer') or {}
3494                         for getter, regex in [(
3495                                 lambda x: x['defaultText']['accessibility']['accessibilityData'],
3496                                 r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3497                                     lambda x: x['accessibility'],
3498                                     lambda x: x['accessibilityData']['accessibilityData'],
3499                                 ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3500                             label = (try_get(tbr, getter, dict) or {}).get('label')
3501                             if label:
3502                                 mobj = re.match(regex, label)
3503                                 if mobj:
3504                                     info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3505                                     break
3506                     sbr_tooltip = try_get(
3507                         vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3508                     if sbr_tooltip:
3509                         like_count, dislike_count = sbr_tooltip.split(' / ')
3510                         info.update({
3511                             'like_count': str_to_int(like_count),
3512                             'dislike_count': str_to_int(dislike_count),
3513                         })
3514                 vsir = content.get('videoSecondaryInfoRenderer')
3515                 if vsir:
3516                     vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3517                     info.update({
3518                         'channel': self._get_text(vor, 'title'),
3519                         'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3520
3521                     rows = try_get(
3522                         vsir,
3523                         lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3524                         list) or []
3525                     multiple_songs = False
3526                     for row in rows:
3527                         if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3528                             multiple_songs = True
3529                             break
3530                     for row in rows:
3531                         mrr = row.get('metadataRowRenderer') or {}
3532                         mrr_title = mrr.get('title')
3533                         if not mrr_title:
3534                             continue
3535                         mrr_title = self._get_text(mrr, 'title')
3536                         mrr_contents_text = self._get_text(mrr, ('contents', 0))
3537                         if mrr_title == 'License':
3538                             info['license'] = mrr_contents_text
3539                         elif not multiple_songs:
3540                             if mrr_title == 'Album':
3541                                 info['album'] = mrr_contents_text
3542                             elif mrr_title == 'Artist':
3543                                 info['artist'] = mrr_contents_text
3544                             elif mrr_title == 'Song':
3545                                 info['track'] = mrr_contents_text
3546
3547         fallbacks = {
3548             'channel': 'uploader',
3549             'channel_id': 'uploader_id',
3550             'channel_url': 'uploader_url',
3551         }
3552         for to, frm in fallbacks.items():
3553             if not info.get(to):
3554                 info[to] = info.get(frm)
3555
3556         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3557             v = info.get(s_k)
3558             if v:
3559                 info[d_k] = v
3560
3561         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3562         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3563         is_membersonly = None
3564         is_premium = None
3565         if initial_data and is_private is not None:
3566             is_membersonly = False
3567             is_premium = False
3568             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3569             badge_labels = set()
3570             for content in contents:
3571                 if not isinstance(content, dict):
3572                     continue
3573                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3574             for badge_label in badge_labels:
3575                 if badge_label.lower() == 'members only':
3576                     is_membersonly = True
3577                 elif badge_label.lower() == 'premium':
3578                     is_premium = True
3579                 elif badge_label.lower() == 'unlisted':
3580                     is_unlisted = True
3581
3582         info['availability'] = self._availability(
3583             is_private=is_private,
3584             needs_premium=is_premium,
3585             needs_subscription=is_membersonly,
3586             needs_auth=info['age_limit'] >= 18,
3587             is_unlisted=None if is_private is None else is_unlisted)
3588
3589         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3590
3591         self.mark_watched(video_id, player_responses)
3592
3593         return info
3594
3595
3596 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3597
3598     def _extract_channel_id(self, webpage):
3599         channel_id = self._html_search_meta(
3600             'channelId', webpage, 'channel id', default=None)
3601         if channel_id:
3602             return channel_id
3603         channel_url = self._html_search_meta(
3604             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3605              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3606              'twitter:app:url:googleplay'), webpage, 'channel url')
3607         return self._search_regex(
3608             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3609             channel_url, 'channel id')
3610
3611     @staticmethod
3612     def _extract_basic_item_renderer(item):
3613         # Modified from _extract_grid_item_renderer
3614         known_basic_renderers = (
3615             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer'
3616         )
3617         for key, renderer in item.items():
3618             if not isinstance(renderer, dict):
3619                 continue
3620             elif key in known_basic_renderers:
3621                 return renderer
3622             elif key.startswith('grid') and key.endswith('Renderer'):
3623                 return renderer
3624
3625     def _grid_entries(self, grid_renderer):
3626         for item in grid_renderer['items']:
3627             if not isinstance(item, dict):
3628                 continue
3629             renderer = self._extract_basic_item_renderer(item)
3630             if not isinstance(renderer, dict):
3631                 continue
3632             title = self._get_text(renderer, 'title')
3633
3634             # playlist
3635             playlist_id = renderer.get('playlistId')
3636             if playlist_id:
3637                 yield self.url_result(
3638                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3639                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3640                     video_title=title)
3641                 continue
3642             # video
3643             video_id = renderer.get('videoId')
3644             if video_id:
3645                 yield self._extract_video(renderer)
3646                 continue
3647             # channel
3648             channel_id = renderer.get('channelId')
3649             if channel_id:
3650                 yield self.url_result(
3651                     'https://www.youtube.com/channel/%s' % channel_id,
3652                     ie=YoutubeTabIE.ie_key(), video_title=title)
3653                 continue
3654             # generic endpoint URL support
3655             ep_url = urljoin('https://www.youtube.com/', try_get(
3656                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3657                 compat_str))
3658             if ep_url:
3659                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3660                     if ie.suitable(ep_url):
3661                         yield self.url_result(
3662                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3663                         break
3664
3665     def _shelf_entries_from_content(self, shelf_renderer):
3666         content = shelf_renderer.get('content')
3667         if not isinstance(content, dict):
3668             return
3669         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3670         if renderer:
3671             # TODO: add support for nested playlists so each shelf is processed
3672             # as separate playlist
3673             # TODO: this includes only first N items
3674             for entry in self._grid_entries(renderer):
3675                 yield entry
3676         renderer = content.get('horizontalListRenderer')
3677         if renderer:
3678             # TODO
3679             pass
3680
3681     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3682         ep = try_get(
3683             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3684             compat_str)
3685         shelf_url = urljoin('https://www.youtube.com', ep)
3686         if shelf_url:
3687             # Skipping links to another channels, note that checking for
3688             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3689             # will not work
3690             if skip_channels and '/channels?' in shelf_url:
3691                 return
3692             title = self._get_text(shelf_renderer, 'title')
3693             yield self.url_result(shelf_url, video_title=title)
3694         # Shelf may not contain shelf URL, fallback to extraction from content
3695         for entry in self._shelf_entries_from_content(shelf_renderer):
3696             yield entry
3697
3698     def _playlist_entries(self, video_list_renderer):
3699         for content in video_list_renderer['contents']:
3700             if not isinstance(content, dict):
3701                 continue
3702             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
3703             if not isinstance(renderer, dict):
3704                 continue
3705             video_id = renderer.get('videoId')
3706             if not video_id:
3707                 continue
3708             yield self._extract_video(renderer)
3709
3710     def _rich_entries(self, rich_grid_renderer):
3711         renderer = try_get(
3712             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
3713         video_id = renderer.get('videoId')
3714         if not video_id:
3715             return
3716         yield self._extract_video(renderer)
3717
3718     def _video_entry(self, video_renderer):
3719         video_id = video_renderer.get('videoId')
3720         if video_id:
3721             return self._extract_video(video_renderer)
3722
3723     def _post_thread_entries(self, post_thread_renderer):
3724         post_renderer = try_get(
3725             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
3726         if not post_renderer:
3727             return
3728         # video attachment
3729         video_renderer = try_get(
3730             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
3731         video_id = video_renderer.get('videoId')
3732         if video_id:
3733             entry = self._extract_video(video_renderer)
3734             if entry:
3735                 yield entry
3736         # playlist attachment
3737         playlist_id = try_get(
3738             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
3739         if playlist_id:
3740             yield self.url_result(
3741                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
3742                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3743         # inline video links
3744         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
3745         for run in runs:
3746             if not isinstance(run, dict):
3747                 continue
3748             ep_url = try_get(
3749                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
3750             if not ep_url:
3751                 continue
3752             if not YoutubeIE.suitable(ep_url):
3753                 continue
3754             ep_video_id = YoutubeIE._match_id(ep_url)
3755             if video_id == ep_video_id:
3756                 continue
3757             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
3758
3759     def _post_thread_continuation_entries(self, post_thread_continuation):
3760         contents = post_thread_continuation.get('contents')
3761         if not isinstance(contents, list):
3762             return
3763         for content in contents:
3764             renderer = content.get('backstagePostThreadRenderer')
3765             if not isinstance(renderer, dict):
3766                 continue
3767             for entry in self._post_thread_entries(renderer):
3768                 yield entry
3769
3770     r''' # unused
3771     def _rich_grid_entries(self, contents):
3772         for content in contents:
3773             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
3774             if video_renderer:
3775                 entry = self._video_entry(video_renderer)
3776                 if entry:
3777                     yield entry
3778     '''
3779     def _extract_entries(self, parent_renderer, continuation_list):
3780         # continuation_list is modified in-place with continuation_list = [continuation_token]
3781         continuation_list[:] = [None]
3782         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
3783         for content in contents:
3784             if not isinstance(content, dict):
3785                 continue
3786             is_renderer = try_get(content, lambda x: x['itemSectionRenderer'], dict)
3787             if not is_renderer:
3788                 renderer = content.get('richItemRenderer')
3789                 if renderer:
3790                     for entry in self._rich_entries(renderer):
3791                         yield entry
3792                     continuation_list[0] = self._extract_continuation(parent_renderer)
3793                 continue
3794             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
3795             for isr_content in isr_contents:
3796                 if not isinstance(isr_content, dict):
3797                     continue
3798
3799                 known_renderers = {
3800                     'playlistVideoListRenderer': self._playlist_entries,
3801                     'gridRenderer': self._grid_entries,
3802                     'shelfRenderer': lambda x: self._shelf_entries(x),
3803                     'backstagePostThreadRenderer': self._post_thread_entries,
3804                     'videoRenderer': lambda x: [self._video_entry(x)],
3805                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
3806                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
3807                 }
3808                 for key, renderer in isr_content.items():
3809                     if key not in known_renderers:
3810                         continue
3811                     for entry in known_renderers[key](renderer):
3812                         if entry:
3813                             yield entry
3814                     continuation_list[0] = self._extract_continuation(renderer)
3815                     break
3816
3817             if not continuation_list[0]:
3818                 continuation_list[0] = self._extract_continuation(is_renderer)
3819
3820         if not continuation_list[0]:
3821             continuation_list[0] = self._extract_continuation(parent_renderer)
3822
3823     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
3824         continuation_list = [None]
3825         extract_entries = lambda x: self._extract_entries(x, continuation_list)
3826         tab_content = try_get(tab, lambda x: x['content'], dict)
3827         if not tab_content:
3828             return
3829         parent_renderer = (
3830             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
3831             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
3832         for entry in extract_entries(parent_renderer):
3833             yield entry
3834         continuation = continuation_list[0]
3835
3836         for page_num in itertools.count(1):
3837             if not continuation:
3838                 break
3839             headers = self.generate_api_headers(
3840                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
3841             response = self._extract_response(
3842                 item_id='%s page %s' % (item_id, page_num),
3843                 query=continuation, headers=headers, ytcfg=ytcfg,
3844                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
3845
3846             if not response:
3847                 break
3848             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
3849             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
3850             visitor_data = self._extract_visitor_data(response) or visitor_data
3851
3852             known_continuation_renderers = {
3853                 'playlistVideoListContinuation': self._playlist_entries,
3854                 'gridContinuation': self._grid_entries,
3855                 'itemSectionContinuation': self._post_thread_continuation_entries,
3856                 'sectionListContinuation': extract_entries,  # for feeds
3857             }
3858             continuation_contents = try_get(
3859                 response, lambda x: x['continuationContents'], dict) or {}
3860             continuation_renderer = None
3861             for key, value in continuation_contents.items():
3862                 if key not in known_continuation_renderers:
3863                     continue
3864                 continuation_renderer = value
3865                 continuation_list = [None]
3866                 for entry in known_continuation_renderers[key](continuation_renderer):
3867                     yield entry
3868                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
3869                 break
3870             if continuation_renderer:
3871                 continue
3872
3873             known_renderers = {
3874                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
3875                 'gridVideoRenderer': (self._grid_entries, 'items'),
3876                 'gridChannelRenderer': (self._grid_entries, 'items'),
3877                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
3878                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
3879                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
3880                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
3881             }
3882             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
3883             continuation_items = try_get(
3884                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
3885             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
3886             video_items_renderer = None
3887             for key, value in continuation_item.items():
3888                 if key not in known_renderers:
3889                     continue
3890                 video_items_renderer = {known_renderers[key][1]: continuation_items}
3891                 continuation_list = [None]
3892                 for entry in known_renderers[key][0](video_items_renderer):
3893                     yield entry
3894                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
3895                 break
3896             if video_items_renderer:
3897                 continue
3898             break
3899
3900     @staticmethod
3901     def _extract_selected_tab(tabs):
3902         for tab in tabs:
3903             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
3904             if renderer.get('selected') is True:
3905                 return renderer
3906         else:
3907             raise ExtractorError('Unable to find selected tab')
3908
3909     @classmethod
3910     def _extract_uploader(cls, data):
3911         uploader = {}
3912         renderer = cls._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
3913         owner = try_get(
3914             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
3915         if owner:
3916             uploader['uploader'] = owner.get('text')
3917             uploader['uploader_id'] = try_get(
3918                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
3919             uploader['uploader_url'] = urljoin(
3920                 'https://www.youtube.com/',
3921                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
3922         return {k: v for k, v in uploader.items() if v is not None}
3923
3924     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
3925         playlist_id = title = description = channel_url = channel_name = channel_id = None
3926         tags = []
3927
3928         selected_tab = self._extract_selected_tab(tabs)
3929         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
3930         renderer = try_get(
3931             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
3932         if renderer:
3933             channel_name = renderer.get('title')
3934             channel_url = renderer.get('channelUrl')
3935             channel_id = renderer.get('externalId')
3936         else:
3937             renderer = try_get(
3938                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
3939
3940         if renderer:
3941             title = renderer.get('title')
3942             description = renderer.get('description', '')
3943             playlist_id = channel_id
3944             tags = renderer.get('keywords', '').split()
3945
3946         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
3947         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
3948         def _get_uncropped(url):
3949             return url_or_none((url or '').split('=')[0] + '=s0')
3950
3951         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
3952         if avatar_thumbnails:
3953             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
3954             if uncropped_avatar:
3955                 avatar_thumbnails.append({
3956                     'url': uncropped_avatar,
3957                     'id': 'avatar_uncropped',
3958                     'preference': 1
3959                 })
3960
3961         channel_banners = self._extract_thumbnails(
3962             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
3963         for banner in channel_banners:
3964             banner['preference'] = -10
3965
3966         if channel_banners:
3967             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
3968             if uncropped_banner:
3969                 channel_banners.append({
3970                     'url': uncropped_banner,
3971                     'id': 'banner_uncropped',
3972                     'preference': -5
3973                 })
3974
3975         primary_thumbnails = self._extract_thumbnails(
3976             primary_sidebar_renderer, ('thumbnailRenderer', 'playlistVideoThumbnailRenderer', 'thumbnail'))
3977
3978         if playlist_id is None:
3979             playlist_id = item_id
3980
3981         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
3982         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
3983         if title is None:
3984             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
3985         title += format_field(selected_tab, 'title', ' - %s')
3986         title += format_field(selected_tab, 'expandedText', ' - %s')
3987
3988         metadata = {
3989             'playlist_id': playlist_id,
3990             'playlist_title': title,
3991             'playlist_description': description,
3992             'uploader': channel_name,
3993             'uploader_id': channel_id,
3994             'uploader_url': channel_url,
3995             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
3996             'tags': tags,
3997             'view_count': self._get_count(playlist_stats, 1),
3998             'availability': self._extract_availability(data),
3999             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
4000             'playlist_count': self._get_count(playlist_stats, 0),
4001             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4002         }
4003         if not channel_id:
4004             metadata.update(self._extract_uploader(data))
4005         metadata.update({
4006             'channel': metadata['uploader'],
4007             'channel_id': metadata['uploader_id'],
4008             'channel_url': metadata['uploader_url']})
4009         return self.playlist_result(
4010             self._entries(
4011                 selected_tab, playlist_id, ytcfg,
4012                 self._extract_account_syncid(ytcfg, data),
4013                 self._extract_visitor_data(data, ytcfg)),
4014             **metadata)
4015
4016     def _extract_mix_playlist(self, playlist, playlist_id, data, ytcfg):
4017         first_id = last_id = response = None
4018         for page_num in itertools.count(1):
4019             videos = list(self._playlist_entries(playlist))
4020             if not videos:
4021                 return
4022             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4023             if start >= len(videos):
4024                 return
4025             for video in videos[start:]:
4026                 if video['id'] == first_id:
4027                     self.to_screen('First video %s found again; Assuming end of Mix' % first_id)
4028                     return
4029                 yield video
4030             first_id = first_id or videos[0]['id']
4031             last_id = videos[-1]['id']
4032             watch_endpoint = try_get(
4033                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4034             headers = self.generate_api_headers(
4035                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4036                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4037             query = {
4038                 'playlistId': playlist_id,
4039                 'videoId': watch_endpoint.get('videoId') or last_id,
4040                 'index': watch_endpoint.get('index') or len(videos),
4041                 'params': watch_endpoint.get('params') or 'OAE%3D'
4042             }
4043             response = self._extract_response(
4044                 item_id='%s page %d' % (playlist_id, page_num),
4045                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4046                 check_get_keys='contents'
4047             )
4048             playlist = try_get(
4049                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4050
4051     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4052         title = playlist.get('title') or try_get(
4053             data, lambda x: x['titleText']['simpleText'], compat_str)
4054         playlist_id = playlist.get('playlistId') or item_id
4055
4056         # Delegating everything except mix playlists to regular tab-based playlist URL
4057         playlist_url = urljoin(url, try_get(
4058             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4059             compat_str))
4060         if playlist_url and playlist_url != url:
4061             return self.url_result(
4062                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4063                 video_title=title)
4064
4065         return self.playlist_result(
4066             self._extract_mix_playlist(playlist, playlist_id, data, ytcfg),
4067             playlist_id=playlist_id, playlist_title=title)
4068
4069     def _extract_availability(self, data):
4070         """
4071         Gets the availability of a given playlist/tab.
4072         Note: Unless YouTube tells us explicitly, we do not assume it is public
4073         @param data: response
4074         """
4075         is_private = is_unlisted = None
4076         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4077         badge_labels = self._extract_badges(renderer)
4078
4079         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4080         privacy_dropdown_entries = try_get(
4081             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4082         for renderer_dict in privacy_dropdown_entries:
4083             is_selected = try_get(
4084                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4085             if not is_selected:
4086                 continue
4087             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4088             if label:
4089                 badge_labels.add(label.lower())
4090                 break
4091
4092         for badge_label in badge_labels:
4093             if badge_label == 'unlisted':
4094                 is_unlisted = True
4095             elif badge_label == 'private':
4096                 is_private = True
4097             elif badge_label == 'public':
4098                 is_unlisted = is_private = False
4099         return self._availability(is_private, False, False, False, is_unlisted)
4100
4101     @staticmethod
4102     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4103         sidebar_renderer = try_get(
4104             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4105         for item in sidebar_renderer:
4106             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4107             if renderer:
4108                 return renderer
4109
4110     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4111         """
4112         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4113         """
4114         browse_id = params = None
4115         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4116         if not renderer:
4117             return
4118         menu_renderer = try_get(
4119             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4120         for menu_item in menu_renderer:
4121             if not isinstance(menu_item, dict):
4122                 continue
4123             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4124             text = try_get(
4125                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4126             if not text or text.lower() != 'show unavailable videos':
4127                 continue
4128             browse_endpoint = try_get(
4129                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4130             browse_id = browse_endpoint.get('browseId')
4131             params = browse_endpoint.get('params')
4132             break
4133
4134         headers = self.generate_api_headers(
4135             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4136             visitor_data=self._extract_visitor_data(data, ytcfg))
4137         query = {
4138             'params': params or 'wgYCCAA=',
4139             'browseId': browse_id or 'VL%s' % item_id
4140         }
4141         return self._extract_response(
4142             item_id=item_id, headers=headers, query=query,
4143             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4144             note='Downloading API JSON with unavailable videos')
4145
4146     def _extract_webpage(self, url, item_id, fatal=True):
4147         retries = self.get_param('extractor_retries', 3)
4148         count = -1
4149         webpage = data = last_error = None
4150         while count < retries:
4151             count += 1
4152             # Sometimes youtube returns a webpage with incomplete ytInitialData
4153             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4154             if last_error:
4155                 self.report_warning('%s. Retrying ...' % last_error)
4156             try:
4157                 webpage = self._download_webpage(
4158                     url, item_id,
4159                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4160                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4161             except ExtractorError as e:
4162                 if isinstance(e.cause, network_exceptions):
4163                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4164                         last_error = error_to_compat_str(e.cause or e.msg)
4165                         if count < retries:
4166                             continue
4167                 if fatal:
4168                     raise
4169                 self.report_warning(error_to_compat_str(e))
4170                 break
4171             else:
4172                 try:
4173                     self._extract_and_report_alerts(data)
4174                 except ExtractorError as e:
4175                     if fatal:
4176                         raise
4177                     self.report_warning(error_to_compat_str(e))
4178                     break
4179
4180                 if dict_get(data, ('contents', 'currentVideoEndpoint')):
4181                     break
4182
4183                 last_error = 'Incomplete yt initial data received'
4184                 if count >= retries:
4185                     if fatal:
4186                         raise ExtractorError(last_error)
4187                     self.report_warning(last_error)
4188                     break
4189
4190         return webpage, data
4191
4192     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4193         data = None
4194         if 'webpage' not in self._configuration_arg('skip'):
4195             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4196             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4197         if not data:
4198             if not ytcfg and self.is_authenticated:
4199                 msg = 'Playlists that require authentication may not extract correctly without a successful webpage download.'
4200                 if 'authcheck' not in self._configuration_arg('skip') and fatal:
4201                     raise ExtractorError(
4202                         msg + ' If you are not downloading private content, or your cookies are only for the first account and channel,'
4203                               ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4204                         expected=True)
4205                 self.report_warning(msg, only_once=True)
4206             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4207         return data, ytcfg
4208
4209     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4210         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4211         resolve_response = self._extract_response(
4212             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4213             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4214         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4215         for ep_key, ep in endpoints.items():
4216             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4217             if params:
4218                 return self._extract_response(
4219                     item_id=item_id, query=params, ep=ep, headers=headers,
4220                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4221                     check_get_keys=('contents', 'currentVideoEndpoint'))
4222         err_note = 'Failed to resolve url (does the playlist exist?)'
4223         if fatal:
4224             raise ExtractorError(err_note, expected=True)
4225         self.report_warning(err_note, item_id)
4226
4227     @staticmethod
4228     def _smuggle_data(entries, data):
4229         for entry in entries:
4230             if data:
4231                 entry['url'] = smuggle_url(entry['url'], data)
4232             yield entry
4233
4234     _SEARCH_PARAMS = None
4235
4236     def _search_results(self, query, params=NO_DEFAULT):
4237         data = {'query': query}
4238         if params is NO_DEFAULT:
4239             params = self._SEARCH_PARAMS
4240         if params:
4241             data['params'] = params
4242         continuation_list = [None]
4243         for page_num in itertools.count(1):
4244             data.update(continuation_list[0] or {})
4245             search = self._extract_response(
4246                 item_id='query "%s" page %s' % (query, page_num), ep='search', query=data,
4247                 check_get_keys=('contents', 'onResponseReceivedCommands'))
4248             slr_contents = try_get(
4249                 search,
4250                 (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
4251                  lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
4252                 list)
4253             yield from self._extract_entries({'contents': slr_contents}, continuation_list)
4254             if not continuation_list[0]:
4255                 break
4256
4257
4258 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4259     IE_DESC = 'YouTube Tabs'
4260     _VALID_URL = r'''(?x:
4261         https?://
4262             (?:\w+\.)?
4263             (?:
4264                 youtube(?:kids)?\.com|
4265                 %(invidious)s
4266             )/
4267             (?:
4268                 (?P<channel_type>channel|c|user|browse)/|
4269                 (?P<not_channel>
4270                     feed/|hashtag/|
4271                     (?:playlist|watch)\?.*?\blist=
4272                 )|
4273                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4274             )
4275             (?P<id>[^/?\#&]+)
4276     )''' % {
4277         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4278         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4279     }
4280     IE_NAME = 'youtube:tab'
4281
4282     _TESTS = [{
4283         'note': 'playlists, multipage',
4284         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4285         'playlist_mincount': 94,
4286         'info_dict': {
4287             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4288             'title': 'Igor Kleiner - Playlists',
4289             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4290             'uploader': 'Igor Kleiner',
4291             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4292             'channel': 'Igor Kleiner',
4293             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4294             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4295             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4296             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4297             'channel_follower_count': int
4298         },
4299     }, {
4300         'note': 'playlists, multipage, different order',
4301         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4302         'playlist_mincount': 94,
4303         'info_dict': {
4304             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4305             'title': 'Igor Kleiner - Playlists',
4306             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4307             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4308             'uploader': 'Igor Kleiner',
4309             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4310             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4311             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4312             'channel': 'Igor Kleiner',
4313             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4314             'channel_follower_count': int
4315         },
4316     }, {
4317         'note': 'playlists, series',
4318         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4319         'playlist_mincount': 5,
4320         'info_dict': {
4321             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4322             'title': '3Blue1Brown - Playlists',
4323             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4324             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4325             'uploader': '3Blue1Brown',
4326             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4327             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4328             'channel': '3Blue1Brown',
4329             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4330             'tags': ['Mathematics'],
4331             'channel_follower_count': int
4332         },
4333     }, {
4334         'note': 'playlists, singlepage',
4335         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4336         'playlist_mincount': 4,
4337         'info_dict': {
4338             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4339             'title': 'ThirstForScience - Playlists',
4340             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4341             'uploader': 'ThirstForScience',
4342             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4343             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4344             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4345             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4346             'tags': 'count:13',
4347             'channel': 'ThirstForScience',
4348             'channel_follower_count': int
4349         }
4350     }, {
4351         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4352         'only_matching': True,
4353     }, {
4354         'note': 'basic, single video playlist',
4355         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4356         'info_dict': {
4357             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4358             'uploader': 'Sergey M.',
4359             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4360             'title': 'youtube-dl public playlist',
4361             'description': '',
4362             'tags': [],
4363             'view_count': int,
4364             'modified_date': '20201130',
4365             'channel': 'Sergey M.',
4366             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4367             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4368             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4369         },
4370         'playlist_count': 1,
4371     }, {
4372         'note': 'empty playlist',
4373         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4374         'info_dict': {
4375             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4376             'uploader': 'Sergey M.',
4377             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4378             'title': 'youtube-dl empty playlist',
4379             'tags': [],
4380             'channel': 'Sergey M.',
4381             'description': '',
4382             'modified_date': '20160902',
4383             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4384             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4385             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4386         },
4387         'playlist_count': 0,
4388     }, {
4389         'note': 'Home tab',
4390         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4391         'info_dict': {
4392             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4393             'title': 'lex will - Home',
4394             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4395             'uploader': 'lex will',
4396             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4397             'channel': 'lex will',
4398             'tags': ['bible', 'history', 'prophesy'],
4399             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4400             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4401             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4402             'channel_follower_count': int
4403         },
4404         'playlist_mincount': 2,
4405     }, {
4406         'note': 'Videos tab',
4407         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4408         'info_dict': {
4409             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4410             'title': 'lex will - Videos',
4411             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4412             'uploader': 'lex will',
4413             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4414             'tags': ['bible', 'history', 'prophesy'],
4415             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4416             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4417             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4418             'channel': 'lex will',
4419             'channel_follower_count': int
4420         },
4421         'playlist_mincount': 975,
4422     }, {
4423         'note': 'Videos tab, sorted by popular',
4424         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4425         'info_dict': {
4426             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4427             'title': 'lex will - Videos',
4428             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4429             'uploader': 'lex will',
4430             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4431             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4432             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4433             'channel': 'lex will',
4434             'tags': ['bible', 'history', 'prophesy'],
4435             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4436             'channel_follower_count': int
4437         },
4438         'playlist_mincount': 199,
4439     }, {
4440         'note': 'Playlists tab',
4441         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4442         'info_dict': {
4443             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4444             'title': 'lex will - Playlists',
4445             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4446             'uploader': 'lex will',
4447             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4448             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4449             'channel': 'lex will',
4450             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4451             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4452             'tags': ['bible', 'history', 'prophesy'],
4453             'channel_follower_count': int
4454         },
4455         'playlist_mincount': 17,
4456     }, {
4457         'note': 'Community tab',
4458         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4459         'info_dict': {
4460             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4461             'title': 'lex will - Community',
4462             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4463             'uploader': 'lex will',
4464             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4465             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4466             'channel': 'lex will',
4467             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4468             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4469             'tags': ['bible', 'history', 'prophesy'],
4470             'channel_follower_count': int
4471         },
4472         'playlist_mincount': 18,
4473     }, {
4474         'note': 'Channels tab',
4475         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4476         'info_dict': {
4477             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4478             'title': 'lex will - Channels',
4479             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4480             'uploader': 'lex will',
4481             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4482             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4483             'channel': 'lex will',
4484             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4485             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4486             'tags': ['bible', 'history', 'prophesy'],
4487             'channel_follower_count': int
4488         },
4489         'playlist_mincount': 12,
4490     }, {
4491         'note': 'Search tab',
4492         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4493         'playlist_mincount': 40,
4494         'info_dict': {
4495             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4496             'title': '3Blue1Brown - Search - linear algebra',
4497             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4498             'uploader': '3Blue1Brown',
4499             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4500             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4501             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4502             'tags': ['Mathematics'],
4503             'channel': '3Blue1Brown',
4504             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4505             'channel_follower_count': int
4506         },
4507     }, {
4508         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4509         'only_matching': True,
4510     }, {
4511         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4512         'only_matching': True,
4513     }, {
4514         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4515         'only_matching': True,
4516     }, {
4517         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4518         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4519         'info_dict': {
4520             'title': '29C3: Not my department',
4521             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4522             'uploader': 'Christiaan008',
4523             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4524             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4525             'tags': [],
4526             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4527             'view_count': int,
4528             'modified_date': '20150605',
4529             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4530             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4531             'channel': 'Christiaan008',
4532         },
4533         'playlist_count': 96,
4534     }, {
4535         'note': 'Large playlist',
4536         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4537         'info_dict': {
4538             'title': 'Uploads from Cauchemar',
4539             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4540             'uploader': 'Cauchemar',
4541             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4542             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4543             'tags': [],
4544             'modified_date': r're:\d{8}',
4545             'channel': 'Cauchemar',
4546             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4547             'view_count': int,
4548             'description': '',
4549             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4550         },
4551         'playlist_mincount': 1123,
4552         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4553     }, {
4554         'note': 'even larger playlist, 8832 videos',
4555         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4556         'only_matching': True,
4557     }, {
4558         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4559         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4560         'info_dict': {
4561             'title': 'Uploads from Interstellar Movie',
4562             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4563             'uploader': 'Interstellar Movie',
4564             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4565             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4566             'tags': [],
4567             'view_count': int,
4568             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4569             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4570             'channel': 'Interstellar Movie',
4571             'description': '',
4572             'modified_date': r're:\d{8}',
4573         },
4574         'playlist_mincount': 21,
4575     }, {
4576         'note': 'Playlist with "show unavailable videos" button',
4577         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4578         'info_dict': {
4579             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4580             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4581             'uploader': 'Phim Siêu Nhân Nhật Bản',
4582             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4583             'view_count': int,
4584             'channel': 'Phim Siêu Nhân Nhật Bản',
4585             'tags': [],
4586             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4587             'description': '',
4588             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4589             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4590             'modified_date': r're:\d{8}',
4591         },
4592         'playlist_mincount': 200,
4593         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4594     }, {
4595         'note': 'Playlist with unavailable videos in page 7',
4596         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4597         'info_dict': {
4598             'title': 'Uploads from BlankTV',
4599             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4600             'uploader': 'BlankTV',
4601             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4602             'channel': 'BlankTV',
4603             'channel_url': 'https://www.youtube.com/c/blanktv',
4604             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4605             'view_count': int,
4606             'tags': [],
4607             'uploader_url': 'https://www.youtube.com/c/blanktv',
4608             'modified_date': r're:\d{8}',
4609             'description': '',
4610         },
4611         'playlist_mincount': 1000,
4612         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4613     }, {
4614         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4615         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4616         'info_dict': {
4617             'title': 'Data Analysis with Dr Mike Pound',
4618             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4619             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4620             'uploader': 'Computerphile',
4621             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4622             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4623             'tags': [],
4624             'view_count': int,
4625             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4626             'channel_url': 'https://www.youtube.com/user/Computerphile',
4627             'channel': 'Computerphile',
4628         },
4629         'playlist_mincount': 11,
4630     }, {
4631         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4632         'only_matching': True,
4633     }, {
4634         'note': 'Playlist URL that does not actually serve a playlist',
4635         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4636         'info_dict': {
4637             'id': 'FqZTN594JQw',
4638             'ext': 'webm',
4639             'title': "Smiley's People 01 detective, Adventure Series, Action",
4640             'uploader': 'STREEM',
4641             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4642             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4643             'upload_date': '20150526',
4644             'license': 'Standard YouTube License',
4645             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4646             'categories': ['People & Blogs'],
4647             'tags': list,
4648             'view_count': int,
4649             'like_count': int,
4650         },
4651         'params': {
4652             'skip_download': True,
4653         },
4654         'skip': 'This video is not available.',
4655         'add_ie': [YoutubeIE.ie_key()],
4656     }, {
4657         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
4658         'only_matching': True,
4659     }, {
4660         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
4661         'only_matching': True,
4662     }, {
4663         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
4664         'info_dict': {
4665             'id': 'GgL890LIznQ',  # This will keep changing
4666             'ext': 'mp4',
4667             'title': str,
4668             'uploader': 'Sky News',
4669             'uploader_id': 'skynews',
4670             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
4671             'upload_date': r're:\d{8}',
4672             'description': str,
4673             'categories': ['News & Politics'],
4674             'tags': list,
4675             'like_count': int,
4676             'release_timestamp': 1642502819,
4677             'channel': 'Sky News',
4678             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
4679             'age_limit': 0,
4680             'view_count': int,
4681             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
4682             'playable_in_embed': True,
4683             'release_date': '20220118',
4684             'availability': 'public',
4685             'live_status': 'is_live',
4686             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
4687             'channel_follower_count': int
4688         },
4689         'params': {
4690             'skip_download': True,
4691         },
4692         'expected_warnings': ['Ignoring subtitle tracks found in '],
4693     }, {
4694         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
4695         'info_dict': {
4696             'id': 'a48o2S1cPoo',
4697             'ext': 'mp4',
4698             'title': 'The Young Turks - Live Main Show',
4699             'uploader': 'The Young Turks',
4700             'uploader_id': 'TheYoungTurks',
4701             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
4702             'upload_date': '20150715',
4703             'license': 'Standard YouTube License',
4704             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
4705             'categories': ['News & Politics'],
4706             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
4707             'like_count': int,
4708         },
4709         'params': {
4710             'skip_download': True,
4711         },
4712         'only_matching': True,
4713     }, {
4714         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
4715         'only_matching': True,
4716     }, {
4717         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
4718         'only_matching': True,
4719     }, {
4720         'note': 'A channel that is not live. Should raise error',
4721         'url': 'https://www.youtube.com/user/numberphile/live',
4722         'only_matching': True,
4723     }, {
4724         'url': 'https://www.youtube.com/feed/trending',
4725         'only_matching': True,
4726     }, {
4727         'url': 'https://www.youtube.com/feed/library',
4728         'only_matching': True,
4729     }, {
4730         'url': 'https://www.youtube.com/feed/history',
4731         'only_matching': True,
4732     }, {
4733         'url': 'https://www.youtube.com/feed/subscriptions',
4734         'only_matching': True,
4735     }, {
4736         'url': 'https://www.youtube.com/feed/watch_later',
4737         'only_matching': True,
4738     }, {
4739         'note': 'Recommended - redirects to home page.',
4740         'url': 'https://www.youtube.com/feed/recommended',
4741         'only_matching': True,
4742     }, {
4743         'note': 'inline playlist with not always working continuations',
4744         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
4745         'only_matching': True,
4746     }, {
4747         'url': 'https://www.youtube.com/course',
4748         'only_matching': True,
4749     }, {
4750         'url': 'https://www.youtube.com/zsecurity',
4751         'only_matching': True,
4752     }, {
4753         'url': 'http://www.youtube.com/NASAgovVideo/videos',
4754         'only_matching': True,
4755     }, {
4756         'url': 'https://www.youtube.com/TheYoungTurks/live',
4757         'only_matching': True,
4758     }, {
4759         'url': 'https://www.youtube.com/hashtag/cctv9',
4760         'info_dict': {
4761             'id': 'cctv9',
4762             'title': '#cctv9',
4763             'tags': [],
4764         },
4765         'playlist_mincount': 350,
4766     }, {
4767         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
4768         'only_matching': True,
4769     }, {
4770         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
4771         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
4772         'only_matching': True
4773     }, {
4774         'note': '/browse/ should redirect to /channel/',
4775         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
4776         'only_matching': True
4777     }, {
4778         'note': 'VLPL, should redirect to playlist?list=PL...',
4779         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
4780         'info_dict': {
4781             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
4782             'uploader': 'NoCopyrightSounds',
4783             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
4784             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
4785             'title': 'NCS Releases',
4786             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
4787             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
4788             'modified_date': r're:\d{8}',
4789             'view_count': int,
4790             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
4791             'tags': [],
4792             'channel': 'NoCopyrightSounds',
4793         },
4794         'playlist_mincount': 166,
4795         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4796     }, {
4797         'note': 'Topic, should redirect to playlist?list=UU...',
4798         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
4799         'info_dict': {
4800             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
4801             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
4802             'title': 'Uploads from Royalty Free Music - Topic',
4803             'uploader': 'Royalty Free Music - Topic',
4804             'tags': [],
4805             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
4806             'channel': 'Royalty Free Music - Topic',
4807             'view_count': int,
4808             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4809             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4810             'modified_date': r're:\d{8}',
4811             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4812             'description': '',
4813         },
4814         'expected_warnings': [
4815             'The URL does not have a videos tab',
4816             r'[Uu]navailable videos (are|will be) hidden',
4817         ],
4818         'playlist_mincount': 101,
4819     }, {
4820         'note': 'Topic without a UU playlist',
4821         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
4822         'info_dict': {
4823             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
4824             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
4825             'tags': [],
4826         },
4827         'expected_warnings': [
4828             'the playlist redirect gave error',
4829         ],
4830         'playlist_mincount': 9,
4831     }, {
4832         'note': 'Youtube music Album',
4833         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
4834         'info_dict': {
4835             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
4836             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
4837             'tags': [],
4838             'view_count': int,
4839             'description': '',
4840             'availability': 'unlisted',
4841             'modified_date': r're:\d{8}',
4842         },
4843         'playlist_count': 50,
4844     }, {
4845         'note': 'unlisted single video playlist',
4846         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
4847         'info_dict': {
4848             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
4849             'uploader': 'colethedj',
4850             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
4851             'title': 'yt-dlp unlisted playlist test',
4852             'availability': 'unlisted',
4853             'tags': [],
4854             'modified_date': '20211208',
4855             'channel': 'colethedj',
4856             'view_count': int,
4857             'description': '',
4858             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
4859             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
4860             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
4861         },
4862         'playlist_count': 1,
4863     }, {
4864         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
4865         'url': 'https://www.youtube.com/feed/recommended',
4866         'info_dict': {
4867             'id': 'recommended',
4868             'title': 'recommended',
4869             'tags': [],
4870         },
4871         'playlist_mincount': 50,
4872         'params': {
4873             'skip_download': True,
4874             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
4875         },
4876     }, {
4877         'note': 'API Fallback: /videos tab, sorted by oldest first',
4878         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
4879         'info_dict': {
4880             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
4881             'title': 'Cody\'sLab - Videos',
4882             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
4883             'uploader': 'Cody\'sLab',
4884             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
4885             'channel': 'Cody\'sLab',
4886             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
4887             'tags': [],
4888             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
4889             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
4890             'channel_follower_count': int
4891         },
4892         'playlist_mincount': 650,
4893         'params': {
4894             'skip_download': True,
4895             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
4896         },
4897     }, {
4898         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
4899         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
4900         'info_dict': {
4901             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
4902             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
4903             'title': 'Uploads from Royalty Free Music - Topic',
4904             'uploader': 'Royalty Free Music - Topic',
4905             'modified_date': r're:\d{8}',
4906             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
4907             'description': '',
4908             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4909             'tags': [],
4910             'channel': 'Royalty Free Music - Topic',
4911             'view_count': int,
4912             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4913         },
4914         'expected_warnings': [
4915             'does not have a videos tab',
4916             r'[Uu]navailable videos (are|will be) hidden',
4917         ],
4918         'playlist_mincount': 101,
4919         'params': {
4920             'skip_download': True,
4921             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
4922         },
4923     }]
4924
4925     @classmethod
4926     def suitable(cls, url):
4927         return False if YoutubeIE.suitable(url) else super(
4928             YoutubeTabIE, cls).suitable(url)
4929
4930     def _real_extract(self, url):
4931         url, smuggled_data = unsmuggle_url(url, {})
4932         if self.is_music_url(url):
4933             smuggled_data['is_music_url'] = True
4934         info_dict = self.__real_extract(url, smuggled_data)
4935         if info_dict.get('entries'):
4936             info_dict['entries'] = self._smuggle_data(info_dict['entries'], smuggled_data)
4937         return info_dict
4938
4939     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
4940
4941     def __real_extract(self, url, smuggled_data):
4942         item_id = self._match_id(url)
4943         url = compat_urlparse.urlunparse(
4944             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
4945         compat_opts = self.get_param('compat_opts', [])
4946
4947         def get_mobj(url):
4948             mobj = self._URL_RE.match(url).groupdict()
4949             mobj.update((k, '') for k, v in mobj.items() if v is None)
4950             return mobj
4951
4952         mobj, redirect_warning = get_mobj(url), None
4953         # Youtube returns incomplete data if tabname is not lower case
4954         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
4955         if is_channel:
4956             if smuggled_data.get('is_music_url'):
4957                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
4958                     item_id = item_id[2:]
4959                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
4960                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
4961                     mdata = self._extract_tab_endpoint(
4962                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
4963                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
4964                                         get_all=False, expected_type=compat_str)
4965                     if not murl:
4966                         raise ExtractorError('Failed to resolve album to playlist')
4967                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
4968                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
4969                     pre = f'https://www.youtube.com/channel/{item_id}'
4970
4971         original_tab_name = tab
4972         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
4973             # Home URLs should redirect to /videos/
4974             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
4975                                 'To download only the videos in the home page, add a "/featured" to the URL')
4976             tab = '/videos'
4977
4978         url = ''.join((pre, tab, post))
4979         mobj = get_mobj(url)
4980
4981         # Handle both video/playlist URLs
4982         qs = parse_qs(url)
4983         video_id, playlist_id = [qs.get(key, [None])[0] for key in ('v', 'list')]
4984
4985         if not video_id and mobj['not_channel'].startswith('watch'):
4986             if not playlist_id:
4987                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
4988                 raise ExtractorError('Unable to recognize tab page')
4989             # Common mistake: https://www.youtube.com/watch?list=playlist_id
4990             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
4991             url = f'https://www.youtube.com/playlist?list={playlist_id}'
4992             mobj = get_mobj(url)
4993
4994         if video_id and playlist_id:
4995             if self.get_param('noplaylist'):
4996                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
4997                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
4998                                        ie=YoutubeIE.ie_key(), video_id=video_id)
4999             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
5000
5001         data, ytcfg = self._extract_data(url, item_id)
5002
5003         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5004         if tabs:
5005             selected_tab = self._extract_selected_tab(tabs)
5006             selected_tab_name = selected_tab.get('title', '').lower()
5007             if selected_tab_name == 'home':
5008                 selected_tab_name = 'featured'
5009             requested_tab_name = mobj['tab'][1:]
5010             if 'no-youtube-channel-redirect' not in compat_opts:
5011                 if requested_tab_name == 'live':
5012                     # Live tab should have redirected to the video
5013                     raise ExtractorError('The channel is not currently live', expected=True)
5014                 if requested_tab_name not in ('', selected_tab_name):
5015                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5016                     if not original_tab_name:
5017                         if item_id[:2] == 'UC':
5018                             # Topic channels don't have /videos. Use the equivalent playlist instead
5019                             pl_id = f'UU{item_id[2:]}'
5020                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5021                             try:
5022                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5023                             except ExtractorError:
5024                                 redirect_warning += ' and the playlist redirect gave error'
5025                             else:
5026                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5027                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5028                         if selected_tab_name and selected_tab_name != requested_tab_name:
5029                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5030                     else:
5031                         raise ExtractorError(redirect_warning, expected=True)
5032
5033         if redirect_warning:
5034             self.to_screen(redirect_warning)
5035         self.write_debug(f'Final URL: {url}')
5036
5037         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5038         if 'no-youtube-unavailable-videos' not in compat_opts:
5039             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5040         self._extract_and_report_alerts(data, only_once=True)
5041         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5042         if tabs:
5043             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5044
5045         playlist = traverse_obj(
5046             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5047         if playlist:
5048             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5049
5050         video_id = traverse_obj(
5051             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5052         if video_id:
5053             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5054                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5055             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5056                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5057
5058         raise ExtractorError('Unable to recognize tab page')
5059
5060
5061 class YoutubePlaylistIE(InfoExtractor):
5062     IE_DESC = 'YouTube playlists'
5063     _VALID_URL = r'''(?x)(?:
5064                         (?:https?://)?
5065                         (?:\w+\.)?
5066                         (?:
5067                             (?:
5068                                 youtube(?:kids)?\.com|
5069                                 %(invidious)s
5070                             )
5071                             /.*?\?.*?\blist=
5072                         )?
5073                         (?P<id>%(playlist_id)s)
5074                      )''' % {
5075         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5076         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5077     }
5078     IE_NAME = 'youtube:playlist'
5079     _TESTS = [{
5080         'note': 'issue #673',
5081         'url': 'PLBB231211A4F62143',
5082         'info_dict': {
5083             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5084             'id': 'PLBB231211A4F62143',
5085             'uploader': 'Wickman',
5086             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5087             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5088             'view_count': int,
5089             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5090             'modified_date': r're:\d{8}',
5091             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5092             'channel': 'Wickman',
5093             'tags': [],
5094             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5095         },
5096         'playlist_mincount': 29,
5097     }, {
5098         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5099         'info_dict': {
5100             'title': 'YDL_safe_search',
5101             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5102         },
5103         'playlist_count': 2,
5104         'skip': 'This playlist is private',
5105     }, {
5106         'note': 'embedded',
5107         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5108         'playlist_count': 4,
5109         'info_dict': {
5110             'title': 'JODA15',
5111             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5112             'uploader': 'milan',
5113             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5114             'description': '',
5115             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5116             'tags': [],
5117             'modified_date': '20140919',
5118             'view_count': int,
5119             'channel': 'milan',
5120             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5121             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5122         },
5123         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5124     }, {
5125         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5126         'playlist_mincount': 654,
5127         'info_dict': {
5128             'title': '2018 Chinese New Singles (11/6 updated)',
5129             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5130             'uploader': 'LBK',
5131             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5132             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5133             'channel': 'LBK',
5134             'view_count': int,
5135             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5136             'tags': [],
5137             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5138             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5139             'modified_date': r're:\d{8}',
5140         },
5141         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5142     }, {
5143         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5144         'only_matching': True,
5145     }, {
5146         # music album playlist
5147         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5148         'only_matching': True,
5149     }]
5150
5151     @classmethod
5152     def suitable(cls, url):
5153         if YoutubeTabIE.suitable(url):
5154             return False
5155         from ..utils import parse_qs
5156         qs = parse_qs(url)
5157         if qs.get('v', [None])[0]:
5158             return False
5159         return super(YoutubePlaylistIE, cls).suitable(url)
5160
5161     def _real_extract(self, url):
5162         playlist_id = self._match_id(url)
5163         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5164         url = update_url_query(
5165             'https://www.youtube.com/playlist',
5166             parse_qs(url) or {'list': playlist_id})
5167         if is_music_url:
5168             url = smuggle_url(url, {'is_music_url': True})
5169         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5170
5171
5172 class YoutubeYtBeIE(InfoExtractor):
5173     IE_DESC = 'youtu.be'
5174     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5175     _TESTS = [{
5176         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5177         'info_dict': {
5178             'id': 'yeWKywCrFtk',
5179             'ext': 'mp4',
5180             'title': 'Small Scale Baler and Braiding Rugs',
5181             'uploader': 'Backus-Page House Museum',
5182             'uploader_id': 'backuspagemuseum',
5183             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5184             'upload_date': '20161008',
5185             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5186             'categories': ['Nonprofits & Activism'],
5187             'tags': list,
5188             'like_count': int,
5189             'age_limit': 0,
5190             'playable_in_embed': True,
5191             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5192             'channel': 'Backus-Page House Museum',
5193             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5194             'live_status': 'not_live',
5195             'view_count': int,
5196             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5197             'availability': 'public',
5198             'duration': 59,
5199         },
5200         'params': {
5201             'noplaylist': True,
5202             'skip_download': True,
5203         },
5204     }, {
5205         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5206         'only_matching': True,
5207     }]
5208
5209     def _real_extract(self, url):
5210         mobj = self._match_valid_url(url)
5211         video_id = mobj.group('id')
5212         playlist_id = mobj.group('playlist_id')
5213         return self.url_result(
5214             update_url_query('https://www.youtube.com/watch', {
5215                 'v': video_id,
5216                 'list': playlist_id,
5217                 'feature': 'youtu.be',
5218             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5219
5220
5221 class YoutubeLivestreamEmbedIE(InfoExtractor):
5222     IE_DESC = 'YouTube livestream embeds'
5223     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5224     _TESTS = [{
5225         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5226         'only_matching': True,
5227     }]
5228
5229     def _real_extract(self, url):
5230         channel_id = self._match_id(url)
5231         return self.url_result(
5232             f'https://www.youtube.com/channel/{channel_id}/live',
5233             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5234
5235
5236 class YoutubeYtUserIE(InfoExtractor):
5237     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5238     IE_NAME = 'youtube:user'
5239     _VALID_URL = r'ytuser:(?P<id>.+)'
5240     _TESTS = [{
5241         'url': 'ytuser:phihag',
5242         'only_matching': True,
5243     }]
5244
5245     def _real_extract(self, url):
5246         user_id = self._match_id(url)
5247         return self.url_result(
5248             'https://www.youtube.com/user/%s/videos' % user_id,
5249             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5250
5251
5252 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5253     IE_NAME = 'youtube:favorites'
5254     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5255     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5256     _LOGIN_REQUIRED = True
5257     _TESTS = [{
5258         'url': ':ytfav',
5259         'only_matching': True,
5260     }, {
5261         'url': ':ytfavorites',
5262         'only_matching': True,
5263     }]
5264
5265     def _real_extract(self, url):
5266         return self.url_result(
5267             'https://www.youtube.com/playlist?list=LL',
5268             ie=YoutubeTabIE.ie_key())
5269
5270
5271 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5272     IE_DESC = 'YouTube search'
5273     IE_NAME = 'youtube:search'
5274     _SEARCH_KEY = 'ytsearch'
5275     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5276     _TESTS = []
5277
5278
5279 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5280     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5281     _SEARCH_KEY = 'ytsearchdate'
5282     IE_DESC = 'YouTube search, newest videos first'
5283     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5284
5285
5286 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5287     IE_DESC = 'YouTube search URLs with sorting and filter support'
5288     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5289     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?:[^&]+)(?:[&]|$)'
5290     _TESTS = [{
5291         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5292         'playlist_mincount': 5,
5293         'info_dict': {
5294             'id': 'youtube-dl test video',
5295             'title': 'youtube-dl test video',
5296         }
5297     }, {
5298         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5299         'playlist_mincount': 5,
5300         'info_dict': {
5301             'id': 'python',
5302             'title': 'python',
5303         }
5304
5305     }, {
5306         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5307         'only_matching': True,
5308     }]
5309
5310     def _real_extract(self, url):
5311         qs = parse_qs(url)
5312         query = (qs.get('search_query') or qs.get('q'))[0]
5313         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5314
5315
5316 class YoutubeFeedsInfoExtractor(YoutubeTabIE):
5317     """
5318     Base class for feed extractors
5319     Subclasses must define the _FEED_NAME property.
5320     """
5321     _LOGIN_REQUIRED = True
5322     _TESTS = []
5323
5324     @property
5325     def IE_NAME(self):
5326         return 'youtube:%s' % self._FEED_NAME
5327
5328     def _real_extract(self, url):
5329         return self.url_result(
5330             'https://www.youtube.com/feed/%s' % self._FEED_NAME,
5331             ie=YoutubeTabIE.ie_key())
5332
5333
5334 class YoutubeWatchLaterIE(InfoExtractor):
5335     IE_NAME = 'youtube:watchlater'
5336     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5337     _VALID_URL = r':ytwatchlater'
5338     _TESTS = [{
5339         'url': ':ytwatchlater',
5340         'only_matching': True,
5341     }]
5342
5343     def _real_extract(self, url):
5344         return self.url_result(
5345             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5346
5347
5348 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5349     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5350     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5351     _FEED_NAME = 'recommended'
5352     _LOGIN_REQUIRED = False
5353     _TESTS = [{
5354         'url': ':ytrec',
5355         'only_matching': True,
5356     }, {
5357         'url': ':ytrecommended',
5358         'only_matching': True,
5359     }, {
5360         'url': 'https://youtube.com',
5361         'only_matching': True,
5362     }]
5363
5364
5365 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5366     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5367     _VALID_URL = r':ytsub(?:scription)?s?'
5368     _FEED_NAME = 'subscriptions'
5369     _TESTS = [{
5370         'url': ':ytsubs',
5371         'only_matching': True,
5372     }, {
5373         'url': ':ytsubscriptions',
5374         'only_matching': True,
5375     }]
5376
5377
5378 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5379     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5380     _VALID_URL = r':ythis(?:tory)?'
5381     _FEED_NAME = 'history'
5382     _TESTS = [{
5383         'url': ':ythistory',
5384         'only_matching': True,
5385     }]
5386
5387
5388 class YoutubeTruncatedURLIE(InfoExtractor):
5389     IE_NAME = 'youtube:truncated_url'
5390     IE_DESC = False  # Do not list
5391     _VALID_URL = r'''(?x)
5392         (?:https?://)?
5393         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5394         (?:watch\?(?:
5395             feature=[a-z_]+|
5396             annotation_id=annotation_[^&]+|
5397             x-yt-cl=[0-9]+|
5398             hl=[^&]*|
5399             t=[0-9]+
5400         )?
5401         |
5402             attribution_link\?a=[^&]+
5403         )
5404         $
5405     '''
5406
5407     _TESTS = [{
5408         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5409         'only_matching': True,
5410     }, {
5411         'url': 'https://www.youtube.com/watch?',
5412         'only_matching': True,
5413     }, {
5414         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5415         'only_matching': True,
5416     }, {
5417         'url': 'https://www.youtube.com/watch?feature=foo',
5418         'only_matching': True,
5419     }, {
5420         'url': 'https://www.youtube.com/watch?hl=en-GB',
5421         'only_matching': True,
5422     }, {
5423         'url': 'https://www.youtube.com/watch?t=2372',
5424         'only_matching': True,
5425     }]
5426
5427     def _real_extract(self, url):
5428         raise ExtractorError(
5429             'Did you forget to quote the URL? Remember that & is a meta '
5430             'character in most shells, so you want to put the URL in quotes, '
5431             'like  youtube-dl '
5432             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5433             ' or simply  youtube-dl BaW_jenozKc  .',
5434             expected=True)
5435
5436
5437 class YoutubeClipIE(InfoExtractor):
5438     IE_NAME = 'youtube:clip'
5439     IE_DESC = False  # Do not list
5440     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/'
5441
5442     def _real_extract(self, url):
5443         self.report_warning('YouTube clips are not currently supported. The entire video will be downloaded instead')
5444         return self.url_result(url, 'Generic')
5445
5446
5447 class YoutubeTruncatedIDIE(InfoExtractor):
5448     IE_NAME = 'youtube:truncated_id'
5449     IE_DESC = False  # Do not list
5450     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
5451
5452     _TESTS = [{
5453         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
5454         'only_matching': True,
5455     }]
5456
5457     def _real_extract(self, url):
5458         video_id = self._match_id(url)
5459         raise ExtractorError(
5460             'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
5461             expected=True)