yt_dlp/extractor/youtube.py

   1 # coding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import calendar
   6 import copy
   7 import datetime
   8 import functools
   9 import hashlib
  10 import itertools
  11 import json
  12 import math
  13 import os.path
  14 import random
  15 import re
  16 import sys
  17 import time
  18 import traceback
  19 import threading
  20
  21 from .common import InfoExtractor, SearchInfoExtractor
  22 from ..compat import (
  23     compat_chr,
  24     compat_HTTPError,
  25     compat_parse_qs,
  26     compat_str,
  27     compat_urllib_parse_unquote_plus,
  28     compat_urllib_parse_urlencode,
  29     compat_urllib_parse_urlparse,
  30     compat_urlparse,
  31 )
  32 from ..jsinterp import JSInterpreter
  33 from ..utils import (
  34     bug_reports_message,
  35     clean_html,
  36     datetime_from_str,
  37     dict_get,
  38     error_to_compat_str,
  39     ExtractorError,
  40     float_or_none,
  41     format_field,
  42     int_or_none,
  43     is_html,
  44     join_nonempty,
  45     mimetype2ext,
  46     network_exceptions,
  47     NO_DEFAULT,
  48     orderedSet,
  49     parse_codecs,
  50     parse_count,
  51     parse_duration,
  52     parse_iso8601,
  53     parse_qs,
  54     qualities,
  55     remove_end,
  56     remove_start,
  57     smuggle_url,
  58     str_or_none,
  59     str_to_int,
  60     strftime_or_none,
  61     traverse_obj,
  62     try_get,
  63     unescapeHTML,
  64     unified_strdate,
  65     unified_timestamp,
  66     unsmuggle_url,
  67     update_url_query,
  68     url_or_none,
  69     urljoin,
  70     variadic,
  71 )
  72
  73
  74 def get_first(obj, keys, **kwargs):
  75     return traverse_obj(obj, (..., *variadic(keys)), **kwargs, get_all=False)
  76
  77
  78 # any clients starting with _ cannot be explicity requested by the user
  79 INNERTUBE_CLIENTS = {
  80     'web': {
  81         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  82         'INNERTUBE_CONTEXT': {
  83             'client': {
  84                 'clientName': 'WEB',
  85                 'clientVersion': '2.20211221.00.00',
  86             }
  87         },
  88         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  89     },
  90     'web_embedded': {
  91         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  92         'INNERTUBE_CONTEXT': {
  93             'client': {
  94                 'clientName': 'WEB_EMBEDDED_PLAYER',
  95                 'clientVersion': '1.20211215.00.01',
  96             },
  97         },
  98         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  99     },
 100     'web_music': {
 101         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
 102         'INNERTUBE_HOST': 'music.youtube.com',
 103         'INNERTUBE_CONTEXT': {
 104             'client': {
 105                 'clientName': 'WEB_REMIX',
 106                 'clientVersion': '1.20211213.00.00',
 107             }
 108         },
 109         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 110     },
 111     'web_creator': {
 112         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 113         'INNERTUBE_CONTEXT': {
 114             'client': {
 115                 'clientName': 'WEB_CREATOR',
 116                 'clientVersion': '1.20211220.02.00',
 117             }
 118         },
 119         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 120     },
 121     'android': {
 122         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 123         'INNERTUBE_CONTEXT': {
 124             'client': {
 125                 'clientName': 'ANDROID',
 126                 'clientVersion': '16.49',
 127             }
 128         },
 129         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 130         'REQUIRE_JS_PLAYER': False
 131     },
 132     'android_embedded': {
 133         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 134         'INNERTUBE_CONTEXT': {
 135             'client': {
 136                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 137                 'clientVersion': '16.49',
 138             },
 139         },
 140         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 141         'REQUIRE_JS_PLAYER': False
 142     },
 143     'android_music': {
 144         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 145         'INNERTUBE_CONTEXT': {
 146             'client': {
 147                 'clientName': 'ANDROID_MUSIC',
 148                 'clientVersion': '4.57',
 149             }
 150         },
 151         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 152         'REQUIRE_JS_PLAYER': False
 153     },
 154     'android_creator': {
 155         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 156         'INNERTUBE_CONTEXT': {
 157             'client': {
 158                 'clientName': 'ANDROID_CREATOR',
 159                 'clientVersion': '21.47',
 160             },
 161         },
 162         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 163         'REQUIRE_JS_PLAYER': False
 164     },
 165     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 166     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 167     'ios': {
 168         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 169         'INNERTUBE_CONTEXT': {
 170             'client': {
 171                 'clientName': 'IOS',
 172                 'clientVersion': '16.46',
 173                 'deviceModel': 'iPhone14,3',
 174             }
 175         },
 176         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 177         'REQUIRE_JS_PLAYER': False
 178     },
 179     'ios_embedded': {
 180         'INNERTUBE_CONTEXT': {
 181             'client': {
 182                 'clientName': 'IOS_MESSAGES_EXTENSION',
 183                 'clientVersion': '16.46',
 184                 'deviceModel': 'iPhone14,3',
 185             },
 186         },
 187         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 188         'REQUIRE_JS_PLAYER': False
 189     },
 190     'ios_music': {
 191         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 192         'INNERTUBE_CONTEXT': {
 193             'client': {
 194                 'clientName': 'IOS_MUSIC',
 195                 'clientVersion': '4.57',
 196             },
 197         },
 198         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 199         'REQUIRE_JS_PLAYER': False
 200     },
 201     'ios_creator': {
 202         'INNERTUBE_CONTEXT': {
 203             'client': {
 204                 'clientName': 'IOS_CREATOR',
 205                 'clientVersion': '21.47',
 206             },
 207         },
 208         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 209         'REQUIRE_JS_PLAYER': False
 210     },
 211     # mweb has 'ultralow' formats
 212     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 213     'mweb': {
 214         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 215         'INNERTUBE_CONTEXT': {
 216             'client': {
 217                 'clientName': 'MWEB',
 218                 'clientVersion': '2.20211221.01.00',
 219             }
 220         },
 221         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 222     }
 223 }
 224
 225
 226 def build_innertube_clients():
 227     third_party = {
 228         'embedUrl': 'https://google.com',  # Can be any valid URL
 229     }
 230     base_clients = ('android', 'web', 'ios', 'mweb')
 231     priority = qualities(base_clients[::-1])
 232
 233     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 234         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 235         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 236         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 237         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 238         ytcfg['priority'] = 10 * priority(client.split('_', 1)[0])
 239
 240         if client in base_clients:
 241             INNERTUBE_CLIENTS[f'{client}_agegate'] = agegate_ytcfg = copy.deepcopy(ytcfg)
 242             agegate_ytcfg['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 243             agegate_ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = third_party
 244             agegate_ytcfg['priority'] -= 1
 245         elif client.endswith('_embedded'):
 246             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = third_party
 247             ytcfg['priority'] -= 2
 248         else:
 249             ytcfg['priority'] -= 3
 250
 251
 252 build_innertube_clients()
 253
 254
 255 class YoutubeBaseInfoExtractor(InfoExtractor):
 256     """Provide base functions for Youtube extractors"""
 257
 258     _RESERVED_NAMES = (
 259         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|clip|'
 260         r'shorts|movies|results|shared|hashtag|trending|explore|feed|feeds|'
 261         r'browse|oembed|get_video_info|iframe_api|s/player|'
 262         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 263
 264     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 265
 266     _NETRC_MACHINE = 'youtube'
 267
 268     # If True it will raise an error if no login info is provided
 269     _LOGIN_REQUIRED = False
 270
 271     _INVIDIOUS_SITES = (
 272         # invidious-redirect websites
 273         r'(?:www\.)?redirect\.invidious\.io',
 274         r'(?:(?:www|dev)\.)?invidio\.us',
 275         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
 276         r'(?:www\.)?invidious\.pussthecat\.org',
 277         r'(?:www\.)?invidious\.zee\.li',
 278         r'(?:www\.)?invidious\.ethibox\.fr',
 279         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 280         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 281         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 282         # youtube-dl invidious instances list
 283         r'(?:(?:www|no)\.)?invidiou\.sh',
 284         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 285         r'(?:www\.)?invidious\.kabi\.tk',
 286         r'(?:www\.)?invidious\.mastodon\.host',
 287         r'(?:www\.)?invidious\.zapashcanon\.fr',
 288         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 289         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 290         r'(?:www\.)?invidious\.himiko\.cloud',
 291         r'(?:www\.)?invidious\.reallyancient\.tech',
 292         r'(?:www\.)?invidious\.tube',
 293         r'(?:www\.)?invidiou\.site',
 294         r'(?:www\.)?invidious\.site',
 295         r'(?:www\.)?invidious\.xyz',
 296         r'(?:www\.)?invidious\.nixnet\.xyz',
 297         r'(?:www\.)?invidious\.048596\.xyz',
 298         r'(?:www\.)?invidious\.drycat\.fr',
 299         r'(?:www\.)?inv\.skyn3t\.in',
 300         r'(?:www\.)?tube\.poal\.co',
 301         r'(?:www\.)?tube\.connect\.cafe',
 302         r'(?:www\.)?vid\.wxzm\.sx',
 303         r'(?:www\.)?vid\.mint\.lgbt',
 304         r'(?:www\.)?vid\.puffyan\.us',
 305         r'(?:www\.)?yewtu\.be',
 306         r'(?:www\.)?yt\.elukerio\.org',
 307         r'(?:www\.)?yt\.lelux\.fi',
 308         r'(?:www\.)?invidious\.ggc-project\.de',
 309         r'(?:www\.)?yt\.maisputain\.ovh',
 310         r'(?:www\.)?ytprivate\.com',
 311         r'(?:www\.)?invidious\.13ad\.de',
 312         r'(?:www\.)?invidious\.toot\.koeln',
 313         r'(?:www\.)?invidious\.fdn\.fr',
 314         r'(?:www\.)?watch\.nettohikari\.com',
 315         r'(?:www\.)?invidious\.namazso\.eu',
 316         r'(?:www\.)?invidious\.silkky\.cloud',
 317         r'(?:www\.)?invidious\.exonip\.de',
 318         r'(?:www\.)?invidious\.riverside\.rocks',
 319         r'(?:www\.)?invidious\.blamefran\.net',
 320         r'(?:www\.)?invidious\.moomoo\.de',
 321         r'(?:www\.)?ytb\.trom\.tf',
 322         r'(?:www\.)?yt\.cyberhost\.uk',
 323         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 324         r'(?:www\.)?qklhadlycap4cnod\.onion',
 325         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 326         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 327         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 328         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 329         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 330         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 331         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 332         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 333         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 334         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 335     )
 336
 337     def _login(self):
 338         """
 339         Attempt to log in to YouTube.
 340         If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
 341         """
 342
 343         if (self._LOGIN_REQUIRED
 344                 and self.get_param('cookiefile') is None
 345                 and self.get_param('cookiesfrombrowser') is None):
 346             self.raise_login_required(
 347                 'Login details are needed to download this content', method='cookies')
 348         username, password = self._get_login_info()
 349         if username:
 350             self.report_warning(f'Cannot login to YouTube using username and password. {self._LOGIN_HINTS["cookies"]}')
 351
 352     def _initialize_consent(self):
 353         cookies = self._get_cookies('https://www.youtube.com/')
 354         if cookies.get('__Secure-3PSID'):
 355             return
 356         consent_id = None
 357         consent = cookies.get('CONSENT')
 358         if consent:
 359             if 'YES' in consent.value:
 360                 return
 361             consent_id = self._search_regex(
 362                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 363         if not consent_id:
 364             consent_id = random.randint(100, 999)
 365         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 366
 367     def _initialize_pref(self):
 368         cookies = self._get_cookies('https://www.youtube.com/')
 369         pref_cookie = cookies.get('PREF')
 370         pref = {}
 371         if pref_cookie:
 372             try:
 373                 pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
 374             except ValueError:
 375                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 376         pref.update({'hl': 'en', 'tz': 'UTC'})
 377         self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
 378
 379     def _real_initialize(self):
 380         self._initialize_pref()
 381         self._initialize_consent()
 382         self._login()
 383
 384     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 385     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 386     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 387
 388     def _get_default_ytcfg(self, client='web'):
 389         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 390
 391     def _get_innertube_host(self, client='web'):
 392         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 393
 394     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 395         # try_get but with fallback to default ytcfg client values when present
 396         _func = lambda y: try_get(y, getter, expected_type)
 397         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 398
 399     def _extract_client_name(self, ytcfg, default_client='web'):
 400         return self._ytcfg_get_safe(
 401             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 402                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 403
 404     def _extract_client_version(self, ytcfg, default_client='web'):
 405         return self._ytcfg_get_safe(
 406             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 407                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 408
 409     def _extract_api_key(self, ytcfg=None, default_client='web'):
 410         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 411
 412     def _extract_context(self, ytcfg=None, default_client='web'):
 413         context = get_first(
 414             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 415         # Enforce language and tz for extraction
 416         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 417         client_context.update({'hl': 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 418         return context
 419
 420     _SAPISID = None
 421
 422     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 423         time_now = round(time.time())
 424         if self._SAPISID is None:
 425             yt_cookies = self._get_cookies('https://www.youtube.com')
 426             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 427             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 428             sapisid_cookie = dict_get(
 429                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 430             if sapisid_cookie and sapisid_cookie.value:
 431                 self._SAPISID = sapisid_cookie.value
 432                 self.write_debug('Extracted SAPISID cookie')
 433                 # SAPISID cookie is required if not already present
 434                 if not yt_cookies.get('SAPISID'):
 435                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 436                     self._set_cookie(
 437                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 438             else:
 439                 self._SAPISID = False
 440         if not self._SAPISID:
 441             return None
 442         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 443         sapisidhash = hashlib.sha1(
 444             f'{time_now} {self._SAPISID} {origin}'.encode('utf-8')).hexdigest()
 445         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 446
 447     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 448                   note='Downloading API JSON', errnote='Unable to download API page',
 449                   context=None, api_key=None, api_hostname=None, default_client='web'):
 450
 451         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 452         data.update(query)
 453         real_headers = self.generate_api_headers(default_client=default_client)
 454         real_headers.update({'content-type': 'application/json'})
 455         if headers:
 456             real_headers.update(headers)
 457         return self._download_json(
 458             'https://%s/youtubei/v1/%s' % (api_hostname or self._get_innertube_host(default_client), ep),
 459             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 460             data=json.dumps(data).encode('utf8'), headers=real_headers,
 461             query={'key': api_key or self._extract_api_key()})
 462
 463     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 464         data = self._search_regex(
 465             (r'%s\s*%s' % (self._YT_INITIAL_DATA_RE, self._YT_INITIAL_BOUNDARY_RE),
 466              self._YT_INITIAL_DATA_RE), webpage, 'yt initial data', fatal=fatal)
 467         if data:
 468             return self._parse_json(data, item_id, fatal=fatal)
 469
 470     @staticmethod
 471     def _extract_session_index(*data):
 472         """
 473         Index of current account in account list.
 474         See: https://github.com/yt-dlp/yt-dlp/pull/519
 475         """
 476         for ytcfg in data:
 477             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 478             if session_index is not None:
 479                 return session_index
 480
 481     # Deprecated?
 482     def _extract_identity_token(self, ytcfg=None, webpage=None):
 483         if ytcfg:
 484             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 485             if token:
 486                 return token
 487         if webpage:
 488             return self._search_regex(
 489                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 490                 'identity token', default=None, fatal=False)
 491
 492     @staticmethod
 493     def _extract_account_syncid(*args):
 494         """
 495         Extract syncId required to download private playlists of secondary channels
 496         @params response and/or ytcfg
 497         """
 498         for data in args:
 499             # ytcfg includes channel_syncid if on secondary channel
 500             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 501             if delegated_sid:
 502                 return delegated_sid
 503             sync_ids = (try_get(
 504                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 505                        lambda x: x['DATASYNC_ID']), compat_str) or '').split('||')
 506             if len(sync_ids) >= 2 and sync_ids[1]:
 507                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 508                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 509                 return sync_ids[0]
 510
 511     @staticmethod
 512     def _extract_visitor_data(*args):
 513         """
 514         Extracts visitorData from an API response or ytcfg
 515         Appears to be used to track session state
 516         """
 517         return get_first(
 518             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 519             expected_type=str)
 520
 521     @property
 522     def is_authenticated(self):
 523         return bool(self._generate_sapisidhash_header())
 524
 525     def extract_ytcfg(self, video_id, webpage):
 526         if not webpage:
 527             return {}
 528         return self._parse_json(
 529             self._search_regex(
 530                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 531                 default='{}'), video_id, fatal=False) or {}
 532
 533     def generate_api_headers(
 534             self, *, ytcfg=None, account_syncid=None, session_index=None,
 535             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 536
 537         origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
 538         headers = {
 539             'X-YouTube-Client-Name': compat_str(
 540                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 541             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 542             'Origin': origin,
 543             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 544             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 545             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg)
 546         }
 547         if session_index is None:
 548             session_index = self._extract_session_index(ytcfg)
 549         if account_syncid or session_index is not None:
 550             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 551
 552         auth = self._generate_sapisidhash_header(origin)
 553         if auth is not None:
 554             headers['Authorization'] = auth
 555             headers['X-Origin'] = origin
 556         return {h: v for h, v in headers.items() if v is not None}
 557
 558     @staticmethod
 559     def _build_api_continuation_query(continuation, ctp=None):
 560         query = {
 561             'continuation': continuation
 562         }
 563         # TODO: Inconsistency with clickTrackingParams.
 564         # Currently we have a fixed ctp contained within context (from ytcfg)
 565         # and a ctp in root query for continuation.
 566         if ctp:
 567             query['clickTracking'] = {'clickTrackingParams': ctp}
 568         return query
 569
 570     @classmethod
 571     def _extract_next_continuation_data(cls, renderer):
 572         next_continuation = try_get(
 573             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 574                        lambda x: x['continuation']['reloadContinuationData']), dict)
 575         if not next_continuation:
 576             return
 577         continuation = next_continuation.get('continuation')
 578         if not continuation:
 579             return
 580         ctp = next_continuation.get('clickTrackingParams')
 581         return cls._build_api_continuation_query(continuation, ctp)
 582
 583     @classmethod
 584     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 585         if isinstance(continuation_ep, dict):
 586             continuation = try_get(
 587                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 588             if not continuation:
 589                 return
 590             ctp = continuation_ep.get('clickTrackingParams')
 591             return cls._build_api_continuation_query(continuation, ctp)
 592
 593     @classmethod
 594     def _extract_continuation(cls, renderer):
 595         next_continuation = cls._extract_next_continuation_data(renderer)
 596         if next_continuation:
 597             return next_continuation
 598
 599         contents = []
 600         for key in ('contents', 'items'):
 601             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 602
 603         for content in contents:
 604             if not isinstance(content, dict):
 605                 continue
 606             continuation_ep = try_get(
 607                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 608                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 609                 dict)
 610             continuation = cls._extract_continuation_ep_data(continuation_ep)
 611             if continuation:
 612                 return continuation
 613
 614     @classmethod
 615     def _extract_alerts(cls, data):
 616         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 617             if not isinstance(alert_dict, dict):
 618                 continue
 619             for alert in alert_dict.values():
 620                 alert_type = alert.get('type')
 621                 if not alert_type:
 622                     continue
 623                 message = cls._get_text(alert, 'text')
 624                 if message:
 625                     yield alert_type, message
 626
 627     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 628         errors = []
 629         warnings = []
 630         for alert_type, alert_message in alerts:
 631             if alert_type.lower() == 'error' and fatal:
 632                 errors.append([alert_type, alert_message])
 633             else:
 634                 warnings.append([alert_type, alert_message])
 635
 636         for alert_type, alert_message in (warnings + errors[:-1]):
 637             self.report_warning('YouTube said: %s - %s' % (alert_type, alert_message), only_once=only_once)
 638         if errors:
 639             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 640
 641     def _extract_and_report_alerts(self, data, *args, **kwargs):
 642         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 643
 644     def _extract_badges(self, renderer: dict):
 645         badges = set()
 646         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 647             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 648             if label:
 649                 badges.add(label.lower())
 650         return badges
 651
 652     @staticmethod
 653     def _get_text(data, *path_list, max_runs=None):
 654         for path in path_list or [None]:
 655             if path is None:
 656                 obj = [data]
 657             else:
 658                 obj = traverse_obj(data, path, default=[])
 659                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 660                     obj = [obj]
 661             for item in obj:
 662                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 663                 if text:
 664                     return text
 665                 runs = try_get(item, lambda x: x['runs'], list) or []
 666                 if not runs and isinstance(item, list):
 667                     runs = item
 668
 669                 runs = runs[:min(len(runs), max_runs or len(runs))]
 670                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 671                 if text:
 672                     return text
 673
 674     def _get_count(self, data, *path_list):
 675         count_text = self._get_text(data, *path_list) or ''
 676         count = parse_count(count_text)
 677         if count is None:
 678             count = str_to_int(
 679                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 680         return count
 681
 682     @staticmethod
 683     def _extract_thumbnails(data, *path_list):
 684         """
 685         Extract thumbnails from thumbnails dict
 686         @param path_list: path list to level that contains 'thumbnails' key
 687         """
 688         thumbnails = []
 689         for path in path_list or [()]:
 690             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...), default=[]):
 691                 thumbnail_url = url_or_none(thumbnail.get('url'))
 692                 if not thumbnail_url:
 693                     continue
 694                 # Sometimes youtube gives a wrong thumbnail URL. See:
 695                 # https://github.com/yt-dlp/yt-dlp/issues/233
 696                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 697                 if 'maxresdefault' in thumbnail_url:
 698                     thumbnail_url = thumbnail_url.split('?')[0]
 699                 thumbnails.append({
 700                     'url': thumbnail_url,
 701                     'height': int_or_none(thumbnail.get('height')),
 702                     'width': int_or_none(thumbnail.get('width')),
 703                 })
 704         return thumbnails
 705
 706     @staticmethod
 707     def extract_relative_time(relative_time_text):
 708         """
 709         Extracts a relative time from string and converts to dt object
 710         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 711         """
 712         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 713         if mobj:
 714             start = mobj.group('start')
 715             if start:
 716                 return datetime_from_str(start)
 717             try:
 718                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 719             except ValueError:
 720                 return None
 721
 722     def _extract_time_text(self, renderer, *path_list):
 723         text = self._get_text(renderer, *path_list) or ''
 724         dt = self.extract_relative_time(text)
 725         timestamp = None
 726         if isinstance(dt, datetime.datetime):
 727             timestamp = calendar.timegm(dt.timetuple())
 728
 729         if timestamp is None:
 730             timestamp = (
 731                 unified_timestamp(text) or unified_timestamp(
 732                     self._search_regex(
 733                         (r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*on)?\s*(.+\d)', r'\w+[\s,\.-]*\w+[\s,\.-]+20\d{2}'),
 734                         text.lower(), 'time text', default=None)))
 735
 736         if text and timestamp is None:
 737             self.report_warning('Cannot parse localized time text' + bug_reports_message(), only_once=True)
 738         return timestamp, text
 739
 740     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 741                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 742                           default_client='web'):
 743         response = None
 744         last_error = None
 745         count = -1
 746         retries = self.get_param('extractor_retries', 3)
 747         if check_get_keys is None:
 748             check_get_keys = []
 749         while count < retries:
 750             count += 1
 751             if last_error:
 752                 self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'))
 753             try:
 754                 response = self._call_api(
 755                     ep=ep, fatal=True, headers=headers,
 756                     video_id=item_id, query=query,
 757                     context=self._extract_context(ytcfg, default_client),
 758                     api_key=self._extract_api_key(ytcfg, default_client),
 759                     api_hostname=api_hostname, default_client=default_client,
 760                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 761             except ExtractorError as e:
 762                 if isinstance(e.cause, network_exceptions):
 763                     if isinstance(e.cause, compat_HTTPError) and not is_html(e.cause.read(512)):
 764                         e.cause.seek(0)
 765                         yt_error = try_get(
 766                             self._parse_json(e.cause.read().decode(), item_id, fatal=False),
 767                             lambda x: x['error']['message'], compat_str)
 768                         if yt_error:
 769                             self._report_alerts([('ERROR', yt_error)], fatal=False)
 770                     # Downloading page may result in intermittent 5xx HTTP error
 771                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 772                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 773                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 774                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 775                         last_error = error_to_compat_str(e.cause or e.msg)
 776                         if count < retries:
 777                             continue
 778                 if fatal:
 779                     raise
 780                 else:
 781                     self.report_warning(error_to_compat_str(e))
 782                     return
 783
 784             else:
 785                 try:
 786                     self._extract_and_report_alerts(response, only_once=True)
 787                 except ExtractorError as e:
 788                     # YouTube servers may return errors we want to retry on in a 200 OK response
 789                     # See: https://github.com/yt-dlp/yt-dlp/issues/839
 790                     if 'unknown error' in e.msg.lower():
 791                         last_error = e.msg
 792                         continue
 793                     if fatal:
 794                         raise
 795                     self.report_warning(error_to_compat_str(e))
 796                     return
 797                 if not check_get_keys or dict_get(response, check_get_keys):
 798                     break
 799                 # Youtube sometimes sends incomplete data
 800                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 801                 last_error = 'Incomplete data received'
 802                 if count >= retries:
 803                     if fatal:
 804                         raise ExtractorError(last_error)
 805                     else:
 806                         self.report_warning(last_error)
 807                         return
 808         return response
 809
 810     @staticmethod
 811     def is_music_url(url):
 812         return re.match(r'https?://music\.youtube\.com/', url) is not None
 813
 814     def _extract_video(self, renderer):
 815         video_id = renderer.get('videoId')
 816         title = self._get_text(renderer, 'title')
 817         description = self._get_text(renderer, 'descriptionSnippet')
 818         duration = parse_duration(self._get_text(
 819             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 820         view_count = self._get_count(renderer, 'viewCountText')
 821
 822         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 823         channel_id = traverse_obj(
 824             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'), expected_type=str, get_all=False)
 825         timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
 826         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
 827         overlay_style = traverse_obj(
 828             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'), get_all=False, expected_type=str)
 829         badges = self._extract_badges(renderer)
 830         thumbnails = self._extract_thumbnails(renderer, 'thumbnail')
 831
 832         return {
 833             '_type': 'url',
 834             'ie_key': YoutubeIE.ie_key(),
 835             'id': video_id,
 836             'url': f'https://www.youtube.com/watch?v={video_id}',
 837             'title': title,
 838             'description': description,
 839             'duration': duration,
 840             'view_count': view_count,
 841             'uploader': uploader,
 842             'channel_id': channel_id,
 843             'thumbnails': thumbnails,
 844             #  'upload_date': strftime_or_none(timestamp, '%Y%m%d'),
 845             'live_status': ('is_upcoming' if scheduled_timestamp is not None
 846                             else 'was_live' if 'streamed' in time_text.lower()
 847                             else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
 848                             else None),
 849             'release_timestamp': scheduled_timestamp,
 850             'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
 851         }
 852
 853
 854 class YoutubeIE(YoutubeBaseInfoExtractor):
 855     IE_DESC = 'YouTube'
 856     _VALID_URL = r"""(?x)^
 857                      (
 858                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 859                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 860                             (?:www\.)?deturl\.com/www\.youtube\.com|
 861                             (?:www\.)?pwnyoutube\.com|
 862                             (?:www\.)?hooktube\.com|
 863                             (?:www\.)?yourepeat\.com|
 864                             tube\.majestyc\.net|
 865                             %(invidious)s|
 866                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 867                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 868                          (?:                                                  # the various things that can precede the ID:
 869                              (?:(?:v|embed|e|shorts)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
 870                              |(?:                                             # or the v= param in all its forms
 871                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 872                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 873                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 874                                  v=
 875                              )
 876                          ))
 877                          |(?:
 878                             youtu\.be|                                        # just youtu.be/xxxx
 879                             vid\.plus|                                        # or vid.plus/xxxx
 880                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 881                             %(invidious)s
 882                          )/
 883                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 884                          )
 885                      )?                                                       # all until now is optional -> you can pass the naked ID
 886                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 887                      (?(1).+)?                                                # if we found the ID, everything can follow
 888                      (?:\#|$)""" % {
 889         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
 890     }
 891     _PLAYER_INFO_RE = (
 892         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 893         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 894         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 895     )
 896     _formats = {
 897         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 898         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 899         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 900         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 901         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 902         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 903         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 904         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 905         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 906         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 907         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 908         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 909         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 910         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 911         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 912         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 913         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 914         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 915
 916
 917         # 3D videos
 918         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 919         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 920         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 921         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 922         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 923         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 924         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 925
 926         # Apple HTTP Live Streaming
 927         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 928         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 929         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 930         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 931         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 932         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 933         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 934         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 935
 936         # DASH mp4 video
 937         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 938         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 939         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 940         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 941         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 942         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 943         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 944         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 945         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 946         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 947         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 948         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 949
 950         # Dash mp4 audio
 951         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 952         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 953         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 954         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 955         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 956         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 957         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 958
 959         # Dash webm
 960         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 961         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 962         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 963         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 964         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 965         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 966         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
 967         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 968         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 969         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 970         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 971         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 972         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 973         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 974         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 975         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
 976         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 977         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 978         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 979         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 980         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 981         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 982
 983         # Dash webm audio
 984         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
 985         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
 986
 987         # Dash webm audio with opus inside
 988         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
 989         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
 990         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
 991
 992         # RTMP (unnamed)
 993         '_rtmp': {'protocol': 'rtmp'},
 994
 995         # av01 video only formats sometimes served with "unknown" codecs
 996         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
 997         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
 998         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
 999         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1000         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1001         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1002         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1003         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1004     }
1005     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1006
1007     _GEO_BYPASS = False
1008
1009     IE_NAME = 'youtube'
1010     _TESTS = [
1011         {
1012             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1013             'info_dict': {
1014                 'id': 'BaW_jenozKc',
1015                 'ext': 'mp4',
1016                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1017                 'uploader': 'Philipp Hagemeister',
1018                 'uploader_id': 'phihag',
1019                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1020                 'channel': 'Philipp Hagemeister',
1021                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1022                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1023                 'upload_date': '20121002',
1024                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1025                 'categories': ['Science & Technology'],
1026                 'tags': ['youtube-dl'],
1027                 'duration': 10,
1028                 'view_count': int,
1029                 'like_count': int,
1030                 'availability': 'public',
1031                 'playable_in_embed': True,
1032                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1033                 'live_status': 'not_live',
1034                 'age_limit': 0,
1035                 'start_time': 1,
1036                 'end_time': 9,
1037                 'channel_follower_count': int
1038             }
1039         },
1040         {
1041             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1042             'note': 'Embed-only video (#1746)',
1043             'info_dict': {
1044                 'id': 'yZIXLfi8CZQ',
1045                 'ext': 'mp4',
1046                 'upload_date': '20120608',
1047                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1048                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1049                 'uploader': 'SET India',
1050                 'uploader_id': 'setindia',
1051                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1052                 'age_limit': 18,
1053             },
1054             'skip': 'Private video',
1055         },
1056         {
1057             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1058             'note': 'Use the first video ID in the URL',
1059             'info_dict': {
1060                 'id': 'BaW_jenozKc',
1061                 'ext': 'mp4',
1062                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1063                 'uploader': 'Philipp Hagemeister',
1064                 'uploader_id': 'phihag',
1065                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1066                 'channel': 'Philipp Hagemeister',
1067                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1068                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1069                 'upload_date': '20121002',
1070                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1071                 'categories': ['Science & Technology'],
1072                 'tags': ['youtube-dl'],
1073                 'duration': 10,
1074                 'view_count': int,
1075                 'like_count': int,
1076                 'availability': 'public',
1077                 'playable_in_embed': True,
1078                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1079                 'live_status': 'not_live',
1080                 'age_limit': 0,
1081                 'channel_follower_count': int
1082             },
1083             'params': {
1084                 'skip_download': True,
1085             },
1086         },
1087         {
1088             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1089             'note': '256k DASH audio (format 141) via DASH manifest',
1090             'info_dict': {
1091                 'id': 'a9LDPn-MO4I',
1092                 'ext': 'm4a',
1093                 'upload_date': '20121002',
1094                 'uploader_id': '8KVIDEO',
1095                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1096                 'description': '',
1097                 'uploader': '8KVIDEO',
1098                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1099             },
1100             'params': {
1101                 'youtube_include_dash_manifest': True,
1102                 'format': '141',
1103             },
1104             'skip': 'format 141 not served anymore',
1105         },
1106         # DASH manifest with encrypted signature
1107         {
1108             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1109             'info_dict': {
1110                 'id': 'IB3lcPjvWLA',
1111                 'ext': 'm4a',
1112                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1113                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1114                 'duration': 244,
1115                 'uploader': 'AfrojackVEVO',
1116                 'uploader_id': 'AfrojackVEVO',
1117                 'upload_date': '20131011',
1118                 'abr': 129.495,
1119                 'like_count': int,
1120                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1121                 'playable_in_embed': True,
1122                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1123                 'view_count': int,
1124                 'track': 'The Spark',
1125                 'live_status': 'not_live',
1126                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1127                 'channel': 'Afrojack',
1128                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1129                 'tags': 'count:19',
1130                 'availability': 'public',
1131                 'categories': ['Music'],
1132                 'age_limit': 0,
1133                 'alt_title': 'The Spark',
1134                 'channel_follower_count': int
1135             },
1136             'params': {
1137                 'youtube_include_dash_manifest': True,
1138                 'format': '141/bestaudio[ext=m4a]',
1139             },
1140         },
1141         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1142         {
1143             'note': 'Embed allowed age-gate video',
1144             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1145             'info_dict': {
1146                 'id': 'HtVdAasjOgU',
1147                 'ext': 'mp4',
1148                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1149                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1150                 'duration': 142,
1151                 'uploader': 'The Witcher',
1152                 'uploader_id': 'WitcherGame',
1153                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1154                 'upload_date': '20140605',
1155                 'age_limit': 18,
1156                 'categories': ['Gaming'],
1157                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1158                 'availability': 'needs_auth',
1159                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1160                 'like_count': int,
1161                 'channel': 'The Witcher',
1162                 'live_status': 'not_live',
1163                 'tags': 'count:17',
1164                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1165                 'playable_in_embed': True,
1166                 'view_count': int,
1167                 'channel_follower_count': int
1168             },
1169         },
1170         {
1171             'note': 'Age-gate video with embed allowed in public site',
1172             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1173             'info_dict': {
1174                 'id': 'HsUATh_Nc2U',
1175                 'ext': 'mp4',
1176                 'title': 'Godzilla 2 (Official Video)',
1177                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1178                 'upload_date': '20200408',
1179                 'uploader_id': 'FlyingKitty900',
1180                 'uploader': 'FlyingKitty',
1181                 'age_limit': 18,
1182                 'availability': 'needs_auth',
1183                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1184                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1185                 'channel': 'FlyingKitty',
1186                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1187                 'view_count': int,
1188                 'categories': ['Entertainment'],
1189                 'live_status': 'not_live',
1190                 'tags': ['Flyingkitty', 'godzilla 2'],
1191                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1192                 'like_count': int,
1193                 'duration': 177,
1194                 'playable_in_embed': True,
1195                 'channel_follower_count': int
1196             },
1197         },
1198         {
1199             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1200             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1201             'info_dict': {
1202                 'id': 'Tq92D6wQ1mg',
1203                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1204                 'ext': 'mp4',
1205                 'upload_date': '20191227',
1206                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1207                 'uploader': 'Projekt Melody',
1208                 'description': 'md5:17eccca93a786d51bc67646756894066',
1209                 'age_limit': 18,
1210                 'like_count': int,
1211                 'availability': 'needs_auth',
1212                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1213                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1214                 'view_count': int,
1215                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1216                 'channel': 'Projekt Melody',
1217                 'live_status': 'not_live',
1218                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1219                 'playable_in_embed': True,
1220                 'categories': ['Entertainment'],
1221                 'duration': 106,
1222                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1223                 'channel_follower_count': int
1224             },
1225         },
1226         {
1227             'note': 'Non-Agegated non-embeddable video',
1228             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1229             'info_dict': {
1230                 'id': 'MeJVWBSsPAY',
1231                 'ext': 'mp4',
1232                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1233                 'uploader': 'Herr Lurik',
1234                 'uploader_id': 'st3in234',
1235                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1236                 'upload_date': '20130730',
1237                 'track': 'Such mich find mich',
1238                 'age_limit': 0,
1239                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1240                 'like_count': int,
1241                 'playable_in_embed': False,
1242                 'creator': 'OOMPH!',
1243                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1244                 'view_count': int,
1245                 'alt_title': 'Such mich find mich',
1246                 'duration': 210,
1247                 'channel': 'Herr Lurik',
1248                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1249                 'categories': ['Music'],
1250                 'availability': 'public',
1251                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1252                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1253                 'live_status': 'not_live',
1254                 'artist': 'OOMPH!',
1255                 'channel_follower_count': int
1256             },
1257         },
1258         {
1259             'note': 'Non-bypassable age-gated video',
1260             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1261             'only_matching': True,
1262         },
1263         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1264         # YouTube Red ad is not captured for creator
1265         {
1266             'url': '__2ABJjxzNo',
1267             'info_dict': {
1268                 'id': '__2ABJjxzNo',
1269                 'ext': 'mp4',
1270                 'duration': 266,
1271                 'upload_date': '20100430',
1272                 'uploader_id': 'deadmau5',
1273                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1274                 'creator': 'deadmau5',
1275                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1276                 'uploader': 'deadmau5',
1277                 'title': 'Deadmau5 - Some Chords (HD)',
1278                 'alt_title': 'Some Chords',
1279                 'availability': 'public',
1280                 'tags': 'count:14',
1281                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1282                 'view_count': int,
1283                 'live_status': 'not_live',
1284                 'channel': 'deadmau5',
1285                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1286                 'like_count': int,
1287                 'track': 'Some Chords',
1288                 'artist': 'deadmau5',
1289                 'playable_in_embed': True,
1290                 'age_limit': 0,
1291                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1292                 'categories': ['Music'],
1293                 'album': 'Some Chords',
1294                 'channel_follower_count': int
1295             },
1296             'expected_warnings': [
1297                 'DASH manifest missing',
1298             ]
1299         },
1300         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1301         {
1302             'url': 'lqQg6PlCWgI',
1303             'info_dict': {
1304                 'id': 'lqQg6PlCWgI',
1305                 'ext': 'mp4',
1306                 'duration': 6085,
1307                 'upload_date': '20150827',
1308                 'uploader_id': 'olympic',
1309                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1310                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1311                 'uploader': 'Olympics',
1312                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1313                 'like_count': int,
1314                 'release_timestamp': 1343767800,
1315                 'playable_in_embed': True,
1316                 'categories': ['Sports'],
1317                 'release_date': '20120731',
1318                 'channel': 'Olympics',
1319                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1320                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1321                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1322                 'age_limit': 0,
1323                 'availability': 'public',
1324                 'live_status': 'was_live',
1325                 'view_count': int,
1326                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1327                 'channel_follower_count': int
1328             },
1329             'params': {
1330                 'skip_download': 'requires avconv',
1331             }
1332         },
1333         # Non-square pixels
1334         {
1335             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1336             'info_dict': {
1337                 'id': '_b-2C3KPAM0',
1338                 'ext': 'mp4',
1339                 'stretched_ratio': 16 / 9.,
1340                 'duration': 85,
1341                 'upload_date': '20110310',
1342                 'uploader_id': 'AllenMeow',
1343                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1344                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1345                 'uploader': '孫ᄋᄅ',
1346                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1347                 'playable_in_embed': True,
1348                 'channel': '孫ᄋᄅ',
1349                 'age_limit': 0,
1350                 'tags': 'count:11',
1351                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1352                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1353                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1354                 'view_count': int,
1355                 'categories': ['People & Blogs'],
1356                 'like_count': int,
1357                 'live_status': 'not_live',
1358                 'availability': 'unlisted',
1359                 'channel_follower_count': int
1360             },
1361         },
1362         # url_encoded_fmt_stream_map is empty string
1363         {
1364             'url': 'qEJwOuvDf7I',
1365             'info_dict': {
1366                 'id': 'qEJwOuvDf7I',
1367                 'ext': 'webm',
1368                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1369                 'description': '',
1370                 'upload_date': '20150404',
1371                 'uploader_id': 'spbelect',
1372                 'uploader': 'Наблюдатели Петербурга',
1373             },
1374             'params': {
1375                 'skip_download': 'requires avconv',
1376             },
1377             'skip': 'This live event has ended.',
1378         },
1379         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1380         {
1381             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1382             'info_dict': {
1383                 'id': 'FIl7x6_3R5Y',
1384                 'ext': 'webm',
1385                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1386                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1387                 'duration': 220,
1388                 'upload_date': '20150625',
1389                 'uploader_id': 'dorappi2000',
1390                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1391                 'uploader': 'dorappi2000',
1392                 'formats': 'mincount:31',
1393             },
1394             'skip': 'not actual anymore',
1395         },
1396         # DASH manifest with segment_list
1397         {
1398             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1399             'md5': '8ce563a1d667b599d21064e982ab9e31',
1400             'info_dict': {
1401                 'id': 'CsmdDsKjzN8',
1402                 'ext': 'mp4',
1403                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1404                 'uploader': 'Airtek',
1405                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1406                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1407                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1408             },
1409             'params': {
1410                 'youtube_include_dash_manifest': True,
1411                 'format': '135',  # bestvideo
1412             },
1413             'skip': 'This live event has ended.',
1414         },
1415         {
1416             # Multifeed videos (multiple cameras), URL is for Main Camera
1417             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1418             'info_dict': {
1419                 'id': 'jvGDaLqkpTg',
1420                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1421                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1422             },
1423             'playlist': [{
1424                 'info_dict': {
1425                     'id': 'jvGDaLqkpTg',
1426                     'ext': 'mp4',
1427                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1428                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1429                     'duration': 10643,
1430                     'upload_date': '20161111',
1431                     'uploader': 'Team PGP',
1432                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1433                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1434                 },
1435             }, {
1436                 'info_dict': {
1437                     'id': '3AKt1R1aDnw',
1438                     'ext': 'mp4',
1439                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1440                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1441                     'duration': 10991,
1442                     'upload_date': '20161111',
1443                     'uploader': 'Team PGP',
1444                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1445                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1446                 },
1447             }, {
1448                 'info_dict': {
1449                     'id': 'RtAMM00gpVc',
1450                     'ext': 'mp4',
1451                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1452                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1453                     'duration': 10995,
1454                     'upload_date': '20161111',
1455                     'uploader': 'Team PGP',
1456                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1457                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1458                 },
1459             }, {
1460                 'info_dict': {
1461                     'id': '6N2fdlP3C5U',
1462                     'ext': 'mp4',
1463                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1464                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1465                     'duration': 10990,
1466                     'upload_date': '20161111',
1467                     'uploader': 'Team PGP',
1468                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1469                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1470                 },
1471             }],
1472             'params': {
1473                 'skip_download': True,
1474             },
1475             'skip': 'Not multifeed anymore',
1476         },
1477         {
1478             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1479             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1480             'info_dict': {
1481                 'id': 'gVfLd0zydlo',
1482                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1483             },
1484             'playlist_count': 2,
1485             'skip': 'Not multifeed anymore',
1486         },
1487         {
1488             'url': 'https://vid.plus/FlRa-iH7PGw',
1489             'only_matching': True,
1490         },
1491         {
1492             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1493             'only_matching': True,
1494         },
1495         {
1496             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1497             # Also tests cut-off URL expansion in video description (see
1498             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1499             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1500             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1501             'info_dict': {
1502                 'id': 'lsguqyKfVQg',
1503                 'ext': 'mp4',
1504                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1505                 'alt_title': 'Dark Walk',
1506                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1507                 'duration': 133,
1508                 'upload_date': '20151119',
1509                 'uploader_id': 'IronSoulElf',
1510                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1511                 'uploader': 'IronSoulElf',
1512                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1513                 'track': 'Dark Walk',
1514                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1515                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1516                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1517                 'categories': ['Film & Animation'],
1518                 'view_count': int,
1519                 'live_status': 'not_live',
1520                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1521                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1522                 'tags': 'count:13',
1523                 'availability': 'public',
1524                 'channel': 'IronSoulElf',
1525                 'playable_in_embed': True,
1526                 'like_count': int,
1527                 'age_limit': 0,
1528                 'channel_follower_count': int
1529             },
1530             'params': {
1531                 'skip_download': True,
1532             },
1533         },
1534         {
1535             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1536             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1537             'only_matching': True,
1538         },
1539         {
1540             # Video with yt:stretch=17:0
1541             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1542             'info_dict': {
1543                 'id': 'Q39EVAstoRM',
1544                 'ext': 'mp4',
1545                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1546                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1547                 'upload_date': '20151107',
1548                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1549                 'uploader': 'CH GAMER DROID',
1550             },
1551             'params': {
1552                 'skip_download': True,
1553             },
1554             'skip': 'This video does not exist.',
1555         },
1556         {
1557             # Video with incomplete 'yt:stretch=16:'
1558             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1559             'only_matching': True,
1560         },
1561         {
1562             # Video licensed under Creative Commons
1563             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1564             'info_dict': {
1565                 'id': 'M4gD1WSo5mA',
1566                 'ext': 'mp4',
1567                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1568                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1569                 'duration': 721,
1570                 'upload_date': '20150127',
1571                 'uploader_id': 'BerkmanCenter',
1572                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1573                 'uploader': 'The Berkman Klein Center for Internet & Society',
1574                 'license': 'Creative Commons Attribution license (reuse allowed)',
1575                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1576                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1577                 'like_count': int,
1578                 'age_limit': 0,
1579                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1580                 'channel': 'The Berkman Klein Center for Internet & Society',
1581                 'availability': 'public',
1582                 'view_count': int,
1583                 'categories': ['Education'],
1584                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1585                 'live_status': 'not_live',
1586                 'playable_in_embed': True,
1587                 'channel_follower_count': int
1588             },
1589             'params': {
1590                 'skip_download': True,
1591             },
1592         },
1593         {
1594             # Channel-like uploader_url
1595             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1596             'info_dict': {
1597                 'id': 'eQcmzGIKrzg',
1598                 'ext': 'mp4',
1599                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1600                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1601                 'duration': 4060,
1602                 'upload_date': '20151119',
1603                 'uploader': 'Bernie Sanders',
1604                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1605                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1606                 'license': 'Creative Commons Attribution license (reuse allowed)',
1607                 'playable_in_embed': True,
1608                 'tags': 'count:12',
1609                 'like_count': int,
1610                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1611                 'age_limit': 0,
1612                 'availability': 'public',
1613                 'categories': ['News & Politics'],
1614                 'channel': 'Bernie Sanders',
1615                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1616                 'view_count': int,
1617                 'live_status': 'not_live',
1618                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1619                 'channel_follower_count': int
1620             },
1621             'params': {
1622                 'skip_download': True,
1623             },
1624         },
1625         {
1626             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1627             'only_matching': True,
1628         },
1629         {
1630             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1631             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1632             'only_matching': True,
1633         },
1634         {
1635             # Rental video preview
1636             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1637             'info_dict': {
1638                 'id': 'uGpuVWrhIzE',
1639                 'ext': 'mp4',
1640                 'title': 'Piku - Trailer',
1641                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1642                 'upload_date': '20150811',
1643                 'uploader': 'FlixMatrix',
1644                 'uploader_id': 'FlixMatrixKaravan',
1645                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1646                 'license': 'Standard YouTube License',
1647             },
1648             'params': {
1649                 'skip_download': True,
1650             },
1651             'skip': 'This video is not available.',
1652         },
1653         {
1654             # YouTube Red video with episode data
1655             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1656             'info_dict': {
1657                 'id': 'iqKdEhx-dD4',
1658                 'ext': 'mp4',
1659                 'title': 'Isolation - Mind Field (Ep 1)',
1660                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1661                 'duration': 2085,
1662                 'upload_date': '20170118',
1663                 'uploader': 'Vsauce',
1664                 'uploader_id': 'Vsauce',
1665                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1666                 'series': 'Mind Field',
1667                 'season_number': 1,
1668                 'episode_number': 1,
1669                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1670                 'tags': 'count:12',
1671                 'view_count': int,
1672                 'availability': 'public',
1673                 'age_limit': 0,
1674                 'channel': 'Vsauce',
1675                 'episode': 'Episode 1',
1676                 'categories': ['Entertainment'],
1677                 'season': 'Season 1',
1678                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1679                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1680                 'like_count': int,
1681                 'playable_in_embed': True,
1682                 'live_status': 'not_live',
1683                 'channel_follower_count': int
1684             },
1685             'params': {
1686                 'skip_download': True,
1687             },
1688             'expected_warnings': [
1689                 'Skipping DASH manifest',
1690             ],
1691         },
1692         {
1693             # The following content has been identified by the YouTube community
1694             # as inappropriate or offensive to some audiences.
1695             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1696             'info_dict': {
1697                 'id': '6SJNVb0GnPI',
1698                 'ext': 'mp4',
1699                 'title': 'Race Differences in Intelligence',
1700                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1701                 'duration': 965,
1702                 'upload_date': '20140124',
1703                 'uploader': 'New Century Foundation',
1704                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1705                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1706             },
1707             'params': {
1708                 'skip_download': True,
1709             },
1710             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1711         },
1712         {
1713             # itag 212
1714             'url': '1t24XAntNCY',
1715             'only_matching': True,
1716         },
1717         {
1718             # geo restricted to JP
1719             'url': 'sJL6WA-aGkQ',
1720             'only_matching': True,
1721         },
1722         {
1723             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1724             'only_matching': True,
1725         },
1726         {
1727             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1728             'only_matching': True,
1729         },
1730         {
1731             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1732             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1733             'only_matching': True,
1734         },
1735         {
1736             # DRM protected
1737             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1738             'only_matching': True,
1739         },
1740         {
1741             # Video with unsupported adaptive stream type formats
1742             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1743             'info_dict': {
1744                 'id': 'Z4Vy8R84T1U',
1745                 'ext': 'mp4',
1746                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1747                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1748                 'duration': 433,
1749                 'upload_date': '20130923',
1750                 'uploader': 'Amelia Putri Harwita',
1751                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1752                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1753                 'formats': 'maxcount:10',
1754             },
1755             'params': {
1756                 'skip_download': True,
1757                 'youtube_include_dash_manifest': False,
1758             },
1759             'skip': 'not actual anymore',
1760         },
1761         {
1762             # Youtube Music Auto-generated description
1763             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1764             'info_dict': {
1765                 'id': 'MgNrAu2pzNs',
1766                 'ext': 'mp4',
1767                 'title': 'Voyeur Girl',
1768                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1769                 'upload_date': '20190312',
1770                 'uploader': 'Stephen - Topic',
1771                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1772                 'artist': 'Stephen',
1773                 'track': 'Voyeur Girl',
1774                 'album': 'it\'s too much love to know my dear',
1775                 'release_date': '20190313',
1776                 'release_year': 2019,
1777                 'alt_title': 'Voyeur Girl',
1778                 'view_count': int,
1779                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1780                 'playable_in_embed': True,
1781                 'like_count': int,
1782                 'categories': ['Music'],
1783                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
1784                 'channel': 'Stephen',
1785                 'availability': 'public',
1786                 'creator': 'Stephen',
1787                 'duration': 169,
1788                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
1789                 'age_limit': 0,
1790                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1791                 'tags': 'count:11',
1792                 'live_status': 'not_live',
1793                 'channel_follower_count': int
1794             },
1795             'params': {
1796                 'skip_download': True,
1797             },
1798         },
1799         {
1800             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1801             'only_matching': True,
1802         },
1803         {
1804             # invalid -> valid video id redirection
1805             'url': 'DJztXj2GPfl',
1806             'info_dict': {
1807                 'id': 'DJztXj2GPfk',
1808                 'ext': 'mp4',
1809                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1810                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1811                 'upload_date': '20090125',
1812                 'uploader': 'Prochorowka',
1813                 'uploader_id': 'Prochorowka',
1814                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1815                 'artist': 'Panjabi MC',
1816                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1817                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1818             },
1819             'params': {
1820                 'skip_download': True,
1821             },
1822             'skip': 'Video unavailable',
1823         },
1824         {
1825             # empty description results in an empty string
1826             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1827             'info_dict': {
1828                 'id': 'x41yOUIvK2k',
1829                 'ext': 'mp4',
1830                 'title': 'IMG 3456',
1831                 'description': '',
1832                 'upload_date': '20170613',
1833                 'uploader_id': 'ElevageOrVert',
1834                 'uploader': 'ElevageOrVert',
1835                 'view_count': int,
1836                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
1837                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
1838                 'like_count': int,
1839                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
1840                 'tags': [],
1841                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
1842                 'availability': 'public',
1843                 'age_limit': 0,
1844                 'categories': ['Pets & Animals'],
1845                 'duration': 7,
1846                 'playable_in_embed': True,
1847                 'live_status': 'not_live',
1848                 'channel': 'ElevageOrVert',
1849                 'channel_follower_count': int
1850             },
1851             'params': {
1852                 'skip_download': True,
1853             },
1854         },
1855         {
1856             # with '};' inside yt initial data (see [1])
1857             # see [2] for an example with '};' inside ytInitialPlayerResponse
1858             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1859             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1860             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1861             'info_dict': {
1862                 'id': 'CHqg6qOn4no',
1863                 'ext': 'mp4',
1864                 'title': 'Part 77   Sort a list of simple types in c#',
1865                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1866                 'upload_date': '20130831',
1867                 'uploader_id': 'kudvenkat',
1868                 'uploader': 'kudvenkat',
1869                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
1870                 'like_count': int,
1871                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
1872                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
1873                 'live_status': 'not_live',
1874                 'categories': ['Education'],
1875                 'availability': 'public',
1876                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
1877                 'tags': 'count:12',
1878                 'playable_in_embed': True,
1879                 'age_limit': 0,
1880                 'view_count': int,
1881                 'duration': 522,
1882                 'channel': 'kudvenkat',
1883                 'channel_follower_count': int
1884             },
1885             'params': {
1886                 'skip_download': True,
1887             },
1888         },
1889         {
1890             # another example of '};' in ytInitialData
1891             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1892             'only_matching': True,
1893         },
1894         {
1895             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1896             'only_matching': True,
1897         },
1898         {
1899             # https://github.com/ytdl-org/youtube-dl/pull/28094
1900             'url': 'OtqTfy26tG0',
1901             'info_dict': {
1902                 'id': 'OtqTfy26tG0',
1903                 'ext': 'mp4',
1904                 'title': 'Burn Out',
1905                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1906                 'upload_date': '20141120',
1907                 'uploader': 'The Cinematic Orchestra - Topic',
1908                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1909                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1910                 'artist': 'The Cinematic Orchestra',
1911                 'track': 'Burn Out',
1912                 'album': 'Every Day',
1913                 'like_count': int,
1914                 'live_status': 'not_live',
1915                 'alt_title': 'Burn Out',
1916                 'duration': 614,
1917                 'age_limit': 0,
1918                 'view_count': int,
1919                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1920                 'creator': 'The Cinematic Orchestra',
1921                 'channel': 'The Cinematic Orchestra',
1922                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
1923                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1924                 'availability': 'public',
1925                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
1926                 'categories': ['Music'],
1927                 'playable_in_embed': True,
1928                 'channel_follower_count': int
1929             },
1930             'params': {
1931                 'skip_download': True,
1932             },
1933         },
1934         {
1935             # controversial video, only works with bpctr when authenticated with cookies
1936             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1937             'only_matching': True,
1938         },
1939         {
1940             # controversial video, requires bpctr/contentCheckOk
1941             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1942             'info_dict': {
1943                 'id': 'SZJvDhaSDnc',
1944                 'ext': 'mp4',
1945                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1946                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1947                 'uploader': 'CBS Mornings',
1948                 'uploader_id': 'CBSThisMorning',
1949                 'upload_date': '20140716',
1950                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
1951                 'duration': 170,
1952                 'categories': ['News & Politics'],
1953                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
1954                 'view_count': int,
1955                 'channel': 'CBS Mornings',
1956                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
1957                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
1958                 'age_limit': 18,
1959                 'availability': 'needs_auth',
1960                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
1961                 'like_count': int,
1962                 'live_status': 'not_live',
1963                 'playable_in_embed': True,
1964                 'channel_follower_count': int
1965             }
1966         },
1967         {
1968             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
1969             'url': 'cBvYw8_A0vQ',
1970             'info_dict': {
1971                 'id': 'cBvYw8_A0vQ',
1972                 'ext': 'mp4',
1973                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
1974                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
1975                 'upload_date': '20201120',
1976                 'uploader': 'Walk around Japan',
1977                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
1978                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
1979                 'duration': 1456,
1980                 'categories': ['Travel & Events'],
1981                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
1982                 'view_count': int,
1983                 'channel': 'Walk around Japan',
1984                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
1985                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
1986                 'age_limit': 0,
1987                 'availability': 'public',
1988                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
1989                 'live_status': 'not_live',
1990                 'playable_in_embed': True,
1991                 'channel_follower_count': int
1992             },
1993             'params': {
1994                 'skip_download': True,
1995             },
1996         }, {
1997             # Has multiple audio streams
1998             'url': 'WaOKSUlf4TM',
1999             'only_matching': True
2000         }, {
2001             # Requires Premium: has format 141 when requested using YTM url
2002             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2003             'only_matching': True
2004         }, {
2005             # multiple subtitles with same lang_code
2006             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2007             'only_matching': True,
2008         }, {
2009             # Force use android client fallback
2010             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2011             'info_dict': {
2012                 'id': 'YOelRv7fMxY',
2013                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2014                 'ext': '3gp',
2015                 'upload_date': '20210624',
2016                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2017                 'uploader': 'colinfurze',
2018                 'uploader_id': 'colinfurze',
2019                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2020                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2021                 'duration': 596,
2022                 'categories': ['Entertainment'],
2023                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2024                 'view_count': int,
2025                 'channel': 'colinfurze',
2026                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2027                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2028                 'age_limit': 0,
2029                 'availability': 'public',
2030                 'like_count': int,
2031                 'live_status': 'not_live',
2032                 'playable_in_embed': True,
2033                 'channel_follower_count': int
2034             },
2035             'params': {
2036                 'format': '17',  # 3gp format available on android
2037                 'extractor_args': {'youtube': {'player_client': ['android']}},
2038             },
2039         },
2040         {
2041             # Skip download of additional client configs (remix client config in this case)
2042             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2043             'only_matching': True,
2044             'params': {
2045                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2046             },
2047         }, {
2048             # shorts
2049             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2050             'only_matching': True,
2051         }, {
2052             'note': 'Storyboards',
2053             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2054             'info_dict': {
2055                 'id': '5KLPxDtMqe8',
2056                 'ext': 'mhtml',
2057                 'format_id': 'sb0',
2058                 'title': 'Your Brain is Plastic',
2059                 'uploader_id': 'scishow',
2060                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2061                 'upload_date': '20140324',
2062                 'uploader': 'SciShow',
2063                 'like_count': int,
2064                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2065                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2066                 'view_count': int,
2067                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2068                 'playable_in_embed': True,
2069                 'tags': 'count:12',
2070                 'uploader_url': 'http://www.youtube.com/user/scishow',
2071                 'availability': 'public',
2072                 'channel': 'SciShow',
2073                 'live_status': 'not_live',
2074                 'duration': 248,
2075                 'categories': ['Education'],
2076                 'age_limit': 0,
2077                 'channel_follower_count': int
2078             }, 'params': {'format': 'mhtml', 'skip_download': True}
2079         }
2080     ]
2081
2082     @classmethod
2083     def suitable(cls, url):
2084         from ..utils import parse_qs
2085
2086         qs = parse_qs(url)
2087         if qs.get('list', [None])[0]:
2088             return False
2089         return super(YoutubeIE, cls).suitable(url)
2090
2091     def __init__(self, *args, **kwargs):
2092         super(YoutubeIE, self).__init__(*args, **kwargs)
2093         self._code_cache = {}
2094         self._player_cache = {}
2095
2096     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
2097         lock = threading.Lock()
2098
2099         is_live = True
2100         start_time = time.time()
2101         formats = [f for f in formats if f.get('is_from_start')]
2102
2103         def refetch_manifest(format_id, delay):
2104             nonlocal formats, start_time, is_live
2105             if time.time() <= start_time + delay:
2106                 return
2107
2108             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2109             video_details = traverse_obj(
2110                 prs, (..., 'videoDetails'), expected_type=dict, default=[])
2111             microformats = traverse_obj(
2112                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2113                 expected_type=dict, default=[])
2114             _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
2115             start_time = time.time()
2116
2117         def mpd_feed(format_id, delay):
2118             """
2119             @returns (manifest_url, manifest_stream_number, is_live) or None
2120             """
2121             with lock:
2122                 refetch_manifest(format_id, delay)
2123
2124             f = next((f for f in formats if f['format_id'] == format_id), None)
2125             if not f:
2126                 if not is_live:
2127                     self.to_screen(f'{video_id}: Video is no longer live')
2128                 else:
2129                     self.report_warning(
2130                         f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
2131                 return None
2132             return f['manifest_url'], f['manifest_stream_number'], is_live
2133
2134         for f in formats:
2135             f['protocol'] = 'http_dash_segments_generator'
2136             f['fragments'] = functools.partial(
2137                 self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
2138
2139     def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
2140         FETCH_SPAN, MAX_DURATION = 5, 432000
2141
2142         mpd_url, stream_number, is_live = None, None, True
2143
2144         begin_index = 0
2145         download_start_time = ctx.get('start') or time.time()
2146
2147         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2148         if lack_early_segments:
2149             self.report_warning(bug_reports_message(
2150                 'Starting download from the last 120 hours of the live stream since '
2151                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2152             lack_early_segments = True
2153
2154         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2155         fragments, fragment_base_url = None, None
2156
2157         def _extract_sequence_from_mpd(refresh_sequence):
2158             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2159             # Obtain from MPD's maximum seq value
2160             old_mpd_url = mpd_url
2161             last_error = ctx.pop('last_error', None)
2162             expire_fast = last_error and isinstance(last_error, compat_HTTPError) and last_error.code == 403
2163             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2164                                                or (mpd_url, stream_number, False))
2165             if not refresh_sequence:
2166                 if expire_fast and not is_live:
2167                     return False, last_seq
2168                 elif old_mpd_url == mpd_url:
2169                     return True, last_seq
2170             try:
2171                 fmts, _ = self._extract_mpd_formats_and_subtitles(
2172                     mpd_url, None, note=False, errnote=False, fatal=False)
2173             except ExtractorError:
2174                 fmts = None
2175             if not fmts:
2176                 no_fragment_score += 1
2177                 return False, last_seq
2178             fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2179             fragments = fmt_info['fragments']
2180             fragment_base_url = fmt_info['fragment_base_url']
2181             assert fragment_base_url
2182
2183             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2184             return True, _last_seq
2185
2186         while is_live:
2187             fetch_time = time.time()
2188             if no_fragment_score > 30:
2189                 return
2190             if last_segment_url:
2191                 # Obtain from "X-Head-Seqnum" header value from each segment
2192                 try:
2193                     urlh = self._request_webpage(
2194                         last_segment_url, None, note=False, errnote=False, fatal=False)
2195                 except ExtractorError:
2196                     urlh = None
2197                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2198                 if last_seq is None:
2199                     no_fragment_score += 1
2200                     last_segment_url = None
2201                     continue
2202             else:
2203                 should_continue, last_seq = _extract_sequence_from_mpd(True)
2204                 if not should_continue:
2205                     continue
2206
2207             if known_idx > last_seq:
2208                 last_segment_url = None
2209                 continue
2210
2211             last_seq += 1
2212
2213             if begin_index < 0 and known_idx < 0:
2214                 # skip from the start when it's negative value
2215                 known_idx = last_seq + begin_index
2216             if lack_early_segments:
2217                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2218             try:
2219                 for idx in range(known_idx, last_seq):
2220                     # do not update sequence here or you'll get skipped some part of it
2221                     should_continue, _ = _extract_sequence_from_mpd(False)
2222                     if not should_continue:
2223                         known_idx = idx - 1
2224                         raise ExtractorError('breaking out of outer loop')
2225                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2226                     yield {
2227                         'url': last_segment_url,
2228                     }
2229                 if known_idx == last_seq:
2230                     no_fragment_score += 5
2231                 else:
2232                     no_fragment_score = 0
2233                 known_idx = last_seq
2234             except ExtractorError:
2235                 continue
2236
2237             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2238
2239     def _extract_player_url(self, *ytcfgs, webpage=None):
2240         player_url = traverse_obj(
2241             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2242             get_all=False, expected_type=compat_str)
2243         if not player_url:
2244             return
2245         if player_url.startswith('//'):
2246             player_url = 'https:' + player_url
2247         elif not re.match(r'https?://', player_url):
2248             player_url = compat_urlparse.urljoin(
2249                 'https://www.youtube.com', player_url)
2250         return player_url
2251
2252     def _download_player_url(self, video_id, fatal=False):
2253         res = self._download_webpage(
2254             'https://www.youtube.com/iframe_api',
2255             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2256         if res:
2257             player_version = self._search_regex(
2258                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2259             if player_version:
2260                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2261
2262     def _signature_cache_id(self, example_sig):
2263         """ Return a string representation of a signature """
2264         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
2265
2266     @classmethod
2267     def _extract_player_info(cls, player_url):
2268         for player_re in cls._PLAYER_INFO_RE:
2269             id_m = re.search(player_re, player_url)
2270             if id_m:
2271                 break
2272         else:
2273             raise ExtractorError('Cannot identify player %r' % player_url)
2274         return id_m.group('id')
2275
2276     def _load_player(self, video_id, player_url, fatal=True):
2277         player_id = self._extract_player_info(player_url)
2278         if player_id not in self._code_cache:
2279             code = self._download_webpage(
2280                 player_url, video_id, fatal=fatal,
2281                 note='Downloading player ' + player_id,
2282                 errnote='Download of %s failed' % player_url)
2283             if code:
2284                 self._code_cache[player_id] = code
2285         return self._code_cache.get(player_id)
2286
2287     def _extract_signature_function(self, video_id, player_url, example_sig):
2288         player_id = self._extract_player_info(player_url)
2289
2290         # Read from filesystem cache
2291         func_id = 'js_%s_%s' % (
2292             player_id, self._signature_cache_id(example_sig))
2293         assert os.path.basename(func_id) == func_id
2294
2295         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
2296         if cache_spec is not None:
2297             return lambda s: ''.join(s[i] for i in cache_spec)
2298
2299         code = self._load_player(video_id, player_url)
2300         if code:
2301             res = self._parse_sig_js(code)
2302
2303             test_string = ''.join(map(compat_chr, range(len(example_sig))))
2304             cache_res = res(test_string)
2305             cache_spec = [ord(c) for c in cache_res]
2306
2307             self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
2308             return res
2309
2310     def _print_sig_code(self, func, example_sig):
2311         if not self.get_param('youtube_print_sig_code'):
2312             return
2313
2314         def gen_sig_code(idxs):
2315             def _genslice(start, end, step):
2316                 starts = '' if start == 0 else str(start)
2317                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2318                 steps = '' if step == 1 else (':%d' % step)
2319                 return 's[%s%s%s]' % (starts, ends, steps)
2320
2321             step = None
2322             # Quelch pyflakes warnings - start will be set when step is set
2323             start = '(Never used)'
2324             for i, prev in zip(idxs[1:], idxs[:-1]):
2325                 if step is not None:
2326                     if i - prev == step:
2327                         continue
2328                     yield _genslice(start, prev, step)
2329                     step = None
2330                     continue
2331                 if i - prev in [-1, 1]:
2332                     step = i - prev
2333                     start = prev
2334                     continue
2335                 else:
2336                     yield 's[%d]' % prev
2337             if step is None:
2338                 yield 's[%d]' % i
2339             else:
2340                 yield _genslice(start, i, step)
2341
2342         test_string = ''.join(map(compat_chr, range(len(example_sig))))
2343         cache_res = func(test_string)
2344         cache_spec = [ord(c) for c in cache_res]
2345         expr_code = ' + '.join(gen_sig_code(cache_spec))
2346         signature_id_tuple = '(%s)' % (
2347             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
2348         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2349                 '    return %s\n') % (signature_id_tuple, expr_code)
2350         self.to_screen('Extracted signature function:\n' + code)
2351
2352     def _parse_sig_js(self, jscode):
2353         funcname = self._search_regex(
2354             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2355              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2356              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2357              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2358              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2359              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2360              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2361              # Obsolete patterns
2362              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2363              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2364              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2365              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2366              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2367              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2368              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2369              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2370             jscode, 'Initial JS player signature function name', group='sig')
2371
2372         jsi = JSInterpreter(jscode)
2373         initial_function = jsi.extract_function(funcname)
2374         return lambda s: initial_function([s])
2375
2376     def _decrypt_signature(self, s, video_id, player_url):
2377         """Turn the encrypted s field into a working signature"""
2378
2379         if player_url is None:
2380             raise ExtractorError('Cannot decrypt signature without player_url')
2381
2382         try:
2383             player_id = (player_url, self._signature_cache_id(s))
2384             if player_id not in self._player_cache:
2385                 func = self._extract_signature_function(
2386                     video_id, player_url, s
2387                 )
2388                 self._player_cache[player_id] = func
2389             func = self._player_cache[player_id]
2390             self._print_sig_code(func, s)
2391             return func(s)
2392         except Exception as e:
2393             raise ExtractorError('Signature extraction failed: ' + traceback.format_exc(), cause=e)
2394
2395     def _decrypt_nsig(self, s, video_id, player_url):
2396         """Turn the encrypted n field into a working signature"""
2397         if player_url is None:
2398             raise ExtractorError('Cannot decrypt nsig without player_url')
2399         if player_url.startswith('//'):
2400             player_url = 'https:' + player_url
2401         elif not re.match(r'https?://', player_url):
2402             player_url = compat_urlparse.urljoin(
2403                 'https://www.youtube.com', player_url)
2404
2405         sig_id = ('nsig_value', s)
2406         if sig_id in self._player_cache:
2407             return self._player_cache[sig_id]
2408
2409         try:
2410             player_id = ('nsig', player_url)
2411             if player_id not in self._player_cache:
2412                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
2413             func = self._player_cache[player_id]
2414             self._player_cache[sig_id] = func(s)
2415             self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
2416             return self._player_cache[sig_id]
2417         except Exception as e:
2418             raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
2419
2420     def _extract_n_function_name(self, jscode):
2421         return self._search_regex(
2422             (r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]{3})\([a-zA-Z0-9]\)',),
2423             jscode, 'Initial JS player n function name', group='nfunc')
2424
2425     def _extract_n_function(self, video_id, player_url):
2426         player_id = self._extract_player_info(player_url)
2427         func_code = self._downloader.cache.load('youtube-nsig', player_id)
2428
2429         if func_code:
2430             jsi = JSInterpreter(func_code)
2431         else:
2432             jscode = self._load_player(video_id, player_url)
2433             funcname = self._extract_n_function_name(jscode)
2434             jsi = JSInterpreter(jscode)
2435             func_code = jsi.extract_function_code(funcname)
2436             self._downloader.cache.store('youtube-nsig', player_id, func_code)
2437
2438         if self.get_param('youtube_print_sig_code'):
2439             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
2440
2441         return lambda s: jsi.extract_function_from_code(*func_code)([s])
2442
2443     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
2444         """
2445         Extract signatureTimestamp (sts)
2446         Required to tell API what sig/player version is in use.
2447         """
2448         sts = None
2449         if isinstance(ytcfg, dict):
2450             sts = int_or_none(ytcfg.get('STS'))
2451
2452         if not sts:
2453             # Attempt to extract from player
2454             if player_url is None:
2455                 error_msg = 'Cannot extract signature timestamp without player_url.'
2456                 if fatal:
2457                     raise ExtractorError(error_msg)
2458                 self.report_warning(error_msg)
2459                 return
2460             code = self._load_player(video_id, player_url, fatal=fatal)
2461             if code:
2462                 sts = int_or_none(self._search_regex(
2463                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2464                     'JS player signature timestamp', group='sts', fatal=fatal))
2465         return sts
2466
2467     def _mark_watched(self, video_id, player_responses):
2468         playback_url = get_first(
2469             player_responses, ('playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
2470             expected_type=url_or_none)
2471         if not playback_url:
2472             self.report_warning('Unable to mark watched')
2473             return
2474         parsed_playback_url = compat_urlparse.urlparse(playback_url)
2475         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
2476
2477         # cpn generation algorithm is reverse engineered from base.js.
2478         # In fact it works even with dummy cpn.
2479         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2480         cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
2481
2482         qs.update({
2483             'ver': ['2'],
2484             'cpn': [cpn],
2485         })
2486         playback_url = compat_urlparse.urlunparse(
2487             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2488
2489         self._download_webpage(
2490             playback_url, video_id, 'Marking watched',
2491             'Unable to mark watched', fatal=False)
2492
2493     @staticmethod
2494     def _extract_urls(webpage):
2495         # Embedded YouTube player
2496         entries = [
2497             unescapeHTML(mobj.group('url'))
2498             for mobj in re.finditer(r'''(?x)
2499             (?:
2500                 <iframe[^>]+?src=|
2501                 data-video-url=|
2502                 <embed[^>]+?src=|
2503                 embedSWF\(?:\s*|
2504                 <object[^>]+data=|
2505                 new\s+SWFObject\(
2506             )
2507             (["\'])
2508                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2509                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2510             \1''', webpage)]
2511
2512         # lazyYT YouTube embed
2513         entries.extend(list(map(
2514             unescapeHTML,
2515             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2516
2517         # Wordpress "YouTube Video Importer" plugin
2518         matches = re.findall(r'''(?x)<div[^>]+
2519             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2520             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2521         entries.extend(m[-1] for m in matches)
2522
2523         return entries
2524
2525     @staticmethod
2526     def _extract_url(webpage):
2527         urls = YoutubeIE._extract_urls(webpage)
2528         return urls[0] if urls else None
2529
2530     @classmethod
2531     def extract_id(cls, url):
2532         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2533         if mobj is None:
2534             raise ExtractorError('Invalid URL: %s' % url)
2535         return mobj.group('id')
2536
2537     def _extract_chapters_from_json(self, data, duration):
2538         chapter_list = traverse_obj(
2539             data, (
2540                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2541                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2542             ), expected_type=list)
2543
2544         return self._extract_chapters(
2545             chapter_list,
2546             chapter_time=lambda chapter: float_or_none(
2547                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2548             chapter_title=lambda chapter: traverse_obj(
2549                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2550             duration=duration)
2551
2552     def _extract_chapters_from_engagement_panel(self, data, duration):
2553         content_list = traverse_obj(
2554             data,
2555             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2556             expected_type=list, default=[])
2557         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2558         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2559
2560         return next((
2561             filter(None, (
2562                 self._extract_chapters(
2563                     traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2564                     chapter_time, chapter_title, duration)
2565                 for contents in content_list
2566             ))), [])
2567
2568     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration):
2569         chapters = []
2570         last_chapter = {'start_time': 0}
2571         for idx, chapter in enumerate(chapter_list or []):
2572             title = chapter_title(chapter)
2573             start_time = chapter_time(chapter)
2574             if start_time is None:
2575                 continue
2576             last_chapter['end_time'] = start_time
2577             if start_time < last_chapter['start_time']:
2578                 if idx == 1:
2579                     chapters.pop()
2580                     self.report_warning('Invalid start time for chapter "%s"' % last_chapter['title'])
2581                 else:
2582                     self.report_warning(f'Invalid start time for chapter "{title}"')
2583                     continue
2584             last_chapter = {'start_time': start_time, 'title': title}
2585             chapters.append(last_chapter)
2586         last_chapter['end_time'] = duration
2587         return chapters
2588
2589     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
2590         return self._parse_json(self._search_regex(
2591             (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
2592              regex), webpage, name, default='{}'), video_id, fatal=False)
2593
2594     def _extract_comment(self, comment_renderer, parent=None):
2595         comment_id = comment_renderer.get('commentId')
2596         if not comment_id:
2597             return
2598
2599         text = self._get_text(comment_renderer, 'contentText')
2600
2601         # note: timestamp is an estimate calculated from the current time and time_text
2602         timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
2603         author = self._get_text(comment_renderer, 'authorText')
2604         author_id = try_get(comment_renderer,
2605                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2606
2607         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2608                                                        lambda x: x['likeCount']), compat_str)) or 0
2609         author_thumbnail = try_get(comment_renderer,
2610                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2611
2612         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2613         is_favorited = 'creatorHeart' in (try_get(
2614             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2615         return {
2616             'id': comment_id,
2617             'text': text,
2618             'timestamp': timestamp,
2619             'time_text': time_text,
2620             'like_count': votes,
2621             'is_favorited': is_favorited,
2622             'author': author,
2623             'author_id': author_id,
2624             'author_thumbnail': author_thumbnail,
2625             'author_is_uploader': author_is_uploader,
2626             'parent': parent or 'root'
2627         }
2628
2629     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
2630
2631         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
2632
2633         def extract_header(contents):
2634             _continuation = None
2635             for content in contents:
2636                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
2637                 expected_comment_count = self._get_count(
2638                     comments_header_renderer, 'countText', 'commentsCount')
2639
2640                 if expected_comment_count:
2641                     tracker['est_total'] = expected_comment_count
2642                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
2643                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
2644
2645                 sort_menu_item = try_get(
2646                     comments_header_renderer,
2647                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2648                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2649
2650                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2651                 if not _continuation:
2652                     continue
2653
2654                 sort_text = str_or_none(sort_menu_item.get('title'))
2655                 if not sort_text:
2656                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2657                 self.to_screen('Sorting comments by %s' % sort_text.lower())
2658                 break
2659             return _continuation
2660
2661         def extract_thread(contents):
2662             if not parent:
2663                 tracker['current_page_thread'] = 0
2664             for content in contents:
2665                 if not parent and tracker['total_parent_comments'] >= max_parents:
2666                     yield
2667                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2668                 comment_renderer = get_first(
2669                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
2670                     expected_type=dict, default={})
2671
2672                 comment = self._extract_comment(comment_renderer, parent)
2673                 if not comment:
2674                     continue
2675
2676                 tracker['running_total'] += 1
2677                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
2678                 yield comment
2679
2680                 # Attempt to get the replies
2681                 comment_replies_renderer = try_get(
2682                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2683
2684                 if comment_replies_renderer:
2685                     tracker['current_page_thread'] += 1
2686                     comment_entries_iter = self._comment_entries(
2687                         comment_replies_renderer, ytcfg, video_id,
2688                         parent=comment.get('id'), tracker=tracker)
2689                     for reply_comment in itertools.islice(comment_entries_iter, min(max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments']))):
2690                         yield reply_comment
2691
2692         # Keeps track of counts across recursive calls
2693         if not tracker:
2694             tracker = dict(
2695                 running_total=0,
2696                 est_total=0,
2697                 current_page_thread=0,
2698                 total_parent_comments=0,
2699                 total_reply_comments=0)
2700
2701         # TODO: Deprecated
2702         # YouTube comments have a max depth of 2
2703         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
2704         if max_depth:
2705             self._downloader.deprecation_warning(
2706                 '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
2707         if max_depth == 1 and parent:
2708             return
2709
2710         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
2711             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
2712
2713         continuation = self._extract_continuation(root_continuation_data)
2714         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
2715         if message and not parent:
2716             self.report_warning(message, video_id=video_id)
2717
2718         response = None
2719         is_first_continuation = parent is None
2720
2721         for page_num in itertools.count(0):
2722             if not continuation:
2723                 break
2724             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
2725             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
2726             if page_num == 0:
2727                 if is_first_continuation:
2728                     note_prefix = 'Downloading comment section API JSON'
2729                 else:
2730                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2731                         tracker['current_page_thread'], comment_prog_str)
2732             else:
2733                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2734                     '       ' if parent else '', ' replies' if parent else '',
2735                     page_num, comment_prog_str)
2736
2737             response = self._extract_response(
2738                 item_id=None, query=continuation,
2739                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2740                 check_get_keys='onResponseReceivedEndpoints')
2741
2742             continuation_contents = traverse_obj(
2743                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
2744
2745             continuation = None
2746             for continuation_section in continuation_contents:
2747                 continuation_items = traverse_obj(
2748                     continuation_section,
2749                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
2750                     get_all=False, expected_type=list) or []
2751                 if is_first_continuation:
2752                     continuation = extract_header(continuation_items)
2753                     is_first_continuation = False
2754                     if continuation:
2755                         break
2756                     continue
2757
2758                 for entry in extract_thread(continuation_items):
2759                     if not entry:
2760                         return
2761                     yield entry
2762                 continuation = self._extract_continuation({'contents': continuation_items})
2763                 if continuation:
2764                     break
2765
2766     def _get_comments(self, ytcfg, video_id, contents, webpage):
2767         """Entry for comment extraction"""
2768         def _real_comment_extract(contents):
2769             renderer = next((
2770                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
2771                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
2772             yield from self._comment_entries(renderer, ytcfg, video_id)
2773
2774         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
2775         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
2776
2777     @staticmethod
2778     def _get_checkok_params():
2779         return {'contentCheckOk': True, 'racyCheckOk': True}
2780
2781     @classmethod
2782     def _generate_player_context(cls, sts=None):
2783         context = {
2784             'html5Preference': 'HTML5_PREF_WANTS',
2785         }
2786         if sts is not None:
2787             context['signatureTimestamp'] = sts
2788         return {
2789             'playbackContext': {
2790                 'contentPlaybackContext': context
2791             },
2792             **cls._get_checkok_params()
2793         }
2794
2795     @staticmethod
2796     def _is_agegated(player_response):
2797         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
2798             return True
2799
2800         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
2801         AGE_GATE_REASONS = (
2802             'confirm your age', 'age-restricted', 'inappropriate',  # reason
2803             'age_verification_required', 'age_check_required',  # status
2804         )
2805         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
2806
2807     @staticmethod
2808     def _is_unplayable(player_response):
2809         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
2810
2811     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr):
2812
2813         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
2814         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
2815         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
2816         headers = self.generate_api_headers(
2817             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
2818
2819         yt_query = {'videoId': video_id}
2820         yt_query.update(self._generate_player_context(sts))
2821         return self._extract_response(
2822             item_id=video_id, ep='player', query=yt_query,
2823             ytcfg=player_ytcfg, headers=headers, fatal=True,
2824             default_client=client,
2825             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
2826         ) or None
2827
2828     def _get_requested_clients(self, url, smuggled_data):
2829         requested_clients = []
2830         default = ['android', 'web']
2831         allowed_clients = sorted(
2832             [client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'],
2833             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
2834         for client in self._configuration_arg('player_client'):
2835             if client in allowed_clients:
2836                 requested_clients.append(client)
2837             elif client == 'default':
2838                 requested_clients.extend(default)
2839             elif client == 'all':
2840                 requested_clients.extend(allowed_clients)
2841             else:
2842                 self.report_warning(f'Skipping unsupported client {client}')
2843         if not requested_clients:
2844             requested_clients = default
2845
2846         if smuggled_data.get('is_music_url') or self.is_music_url(url):
2847             requested_clients.extend(
2848                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
2849
2850         return orderedSet(requested_clients)
2851
2852     def _extract_player_ytcfg(self, client, video_id):
2853         url = {
2854             'web_music': 'https://music.youtube.com',
2855             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
2856         }.get(client)
2857         if not url:
2858             return {}
2859         webpage = self._download_webpage(url, video_id, fatal=False, note='Downloading %s config' % client.replace('_', ' ').strip())
2860         return self.extract_ytcfg(video_id, webpage) or {}
2861
2862     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg):
2863         initial_pr = None
2864         if webpage:
2865             initial_pr = self._extract_yt_initial_variable(
2866                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
2867                 video_id, 'initial player response')
2868
2869         original_clients = clients
2870         clients = clients[::-1]
2871         prs = []
2872
2873         def append_client(client_name):
2874             if client_name in INNERTUBE_CLIENTS and client_name not in original_clients:
2875                 clients.append(client_name)
2876
2877         # Android player_response does not have microFormats which are needed for
2878         # extraction of some data. So we return the initial_pr with formats
2879         # stripped out even if not requested by the user
2880         # See: https://github.com/yt-dlp/yt-dlp/issues/501
2881         if initial_pr:
2882             pr = dict(initial_pr)
2883             pr['streamingData'] = None
2884             prs.append(pr)
2885
2886         last_error = None
2887         tried_iframe_fallback = False
2888         player_url = None
2889         while clients:
2890             client = clients.pop()
2891             player_ytcfg = master_ytcfg if client == 'web' else {}
2892             if 'configs' not in self._configuration_arg('player_skip'):
2893                 player_ytcfg = self._extract_player_ytcfg(client, video_id) or player_ytcfg
2894
2895             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
2896             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
2897             if 'js' in self._configuration_arg('player_skip'):
2898                 require_js_player = False
2899                 player_url = None
2900
2901             if not player_url and not tried_iframe_fallback and require_js_player:
2902                 player_url = self._download_player_url(video_id)
2903                 tried_iframe_fallback = True
2904
2905             try:
2906                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
2907                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr)
2908             except ExtractorError as e:
2909                 if last_error:
2910                     self.report_warning(last_error)
2911                 last_error = e
2912                 continue
2913
2914             if pr:
2915                 prs.append(pr)
2916
2917             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
2918             if client.endswith('_agegate') and self._is_unplayable(pr) and self.is_authenticated:
2919                 append_client(client.replace('_agegate', '_creator'))
2920             elif self._is_agegated(pr):
2921                 append_client(f'{client}_agegate')
2922
2923         if last_error:
2924             if not len(prs):
2925                 raise last_error
2926             self.report_warning(last_error)
2927         return prs, player_url
2928
2929     def _extract_formats(self, streaming_data, video_id, player_url, is_live):
2930         itags, stream_ids = {}, []
2931         itag_qualities, res_qualities = {}, {}
2932         q = qualities([
2933             # Normally tiny is the smallest video-only formats. But
2934             # audio-only formats with unknown quality may get tagged as tiny
2935             'tiny',
2936             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
2937             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
2938         ])
2939         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
2940
2941         for fmt in streaming_formats:
2942             if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
2943                 continue
2944
2945             itag = str_or_none(fmt.get('itag'))
2946             audio_track = fmt.get('audioTrack') or {}
2947             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
2948             if stream_id in stream_ids:
2949                 continue
2950
2951             quality = fmt.get('quality')
2952             height = int_or_none(fmt.get('height'))
2953             if quality == 'tiny' or not quality:
2954                 quality = fmt.get('audioQuality', '').lower() or quality
2955             # The 3gp format (17) in android client has a quality of "small",
2956             # but is actually worse than other formats
2957             if itag == '17':
2958                 quality = 'tiny'
2959             if quality:
2960                 if itag:
2961                     itag_qualities[itag] = quality
2962                 if height:
2963                     res_qualities[height] = quality
2964             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
2965             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
2966             # number of fragment that would subsequently requested with (`&sq=N`)
2967             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
2968                 continue
2969
2970             fmt_url = fmt.get('url')
2971             if not fmt_url:
2972                 sc = compat_parse_qs(fmt.get('signatureCipher'))
2973                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
2974                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
2975                 if not (sc and fmt_url and encrypted_sig):
2976                     continue
2977                 if not player_url:
2978                     continue
2979                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
2980                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
2981                 fmt_url += '&' + sp + '=' + signature
2982
2983             query = parse_qs(fmt_url)
2984             throttled = False
2985             if query.get('n'):
2986                 try:
2987                     fmt_url = update_url_query(fmt_url, {
2988                         'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
2989                 except ExtractorError as e:
2990                     self.report_warning(
2991                         f'nsig extraction failed: You may experience throttling for some formats\n'
2992                         f'n = {query["n"][0]} ; player = {player_url}\n{e}', only_once=True)
2993                     throttled = True
2994
2995             if itag:
2996                 itags[itag] = 'https'
2997                 stream_ids.append(stream_id)
2998
2999             tbr = float_or_none(
3000                 fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3001             dct = {
3002                 'asr': int_or_none(fmt.get('audioSampleRate')),
3003                 'filesize': int_or_none(fmt.get('contentLength')),
3004                 'format_id': itag,
3005                 'format_note': join_nonempty(
3006                     '%s%s' % (audio_track.get('displayName') or '',
3007                               ' (default)' if audio_track.get('audioIsDefault') else ''),
3008                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3009                     throttled and 'THROTTLED', delim=', '),
3010                 'source_preference': -10 if throttled else -1,
3011                 'fps': int_or_none(fmt.get('fps')) or None,
3012                 'height': height,
3013                 'quality': q(quality),
3014                 'tbr': tbr,
3015                 'url': fmt_url,
3016                 'width': int_or_none(fmt.get('width')),
3017                 'language': audio_track.get('id', '').split('.')[0],
3018                 'language_preference': 1 if audio_track.get('audioIsDefault') else -1,
3019             }
3020             mime_mobj = re.match(
3021                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3022             if mime_mobj:
3023                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3024                 dct.update(parse_codecs(mime_mobj.group(2)))
3025             no_audio = dct.get('acodec') == 'none'
3026             no_video = dct.get('vcodec') == 'none'
3027             if no_audio:
3028                 dct['vbr'] = tbr
3029             if no_video:
3030                 dct['abr'] = tbr
3031             if no_audio or no_video:
3032                 dct['downloader_options'] = {
3033                     # Youtube throttles chunks >~10M
3034                     'http_chunk_size': 10485760,
3035                 }
3036                 if dct.get('ext'):
3037                     dct['container'] = dct['ext'] + '_dash'
3038             yield dct
3039
3040         live_from_start = is_live and self.get_param('live_from_start')
3041         skip_manifests = self._configuration_arg('skip')
3042         if not self.get_param('youtube_include_hls_manifest', True):
3043             skip_manifests.append('hls')
3044         get_dash = 'dash' not in skip_manifests and (
3045             not is_live or live_from_start or self._configuration_arg('include_live_dash'))
3046         get_hls = not live_from_start and 'hls' not in skip_manifests
3047
3048         def process_manifest_format(f, proto, itag):
3049             if itag in itags:
3050                 if itags[itag] == proto or f'{itag}-{proto}' in itags:
3051                     return False
3052                 itag = f'{itag}-{proto}'
3053             if itag:
3054                 f['format_id'] = itag
3055                 itags[itag] = proto
3056
3057             f['quality'] = next((
3058                 q(qdict[val])
3059                 for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
3060                 if val in qdict), -1)
3061             return True
3062
3063         for sd in streaming_data:
3064             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
3065             if hls_manifest_url:
3066                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
3067                     if process_manifest_format(f, 'hls', self._search_regex(
3068                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3069                         yield f
3070
3071             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
3072             if dash_manifest_url:
3073                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
3074                     if process_manifest_format(f, 'dash', f['format_id']):
3075                         f['filesize'] = int_or_none(self._search_regex(
3076                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3077                         if live_from_start:
3078                             f['is_from_start'] = True
3079
3080                         yield f
3081
3082     def _extract_storyboard(self, player_responses, duration):
3083         spec = get_first(
3084             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3085         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3086         if not base_url:
3087             return
3088         L = len(spec) - 1
3089         for i, args in enumerate(spec):
3090             args = args.split('#')
3091             counts = list(map(int_or_none, args[:5]))
3092             if len(args) != 8 or not all(counts):
3093                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3094                 continue
3095             width, height, frame_count, cols, rows = counts
3096             N, sigh = args[6:]
3097
3098             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3099             fragment_count = frame_count / (cols * rows)
3100             fragment_duration = duration / fragment_count
3101             yield {
3102                 'format_id': f'sb{i}',
3103                 'format_note': 'storyboard',
3104                 'ext': 'mhtml',
3105                 'protocol': 'mhtml',
3106                 'acodec': 'none',
3107                 'vcodec': 'none',
3108                 'url': url,
3109                 'width': width,
3110                 'height': height,
3111                 'fragments': [{
3112                     'path': url.replace('$M', str(j)),
3113                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3114                 } for j in range(math.ceil(fragment_count))],
3115             }
3116
3117     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3118         webpage = None
3119         if 'webpage' not in self._configuration_arg('player_skip'):
3120             webpage = self._download_webpage(
3121                 webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
3122
3123         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3124
3125         player_responses, player_url = self._extract_player_responses(
3126             self._get_requested_clients(url, smuggled_data),
3127             video_id, webpage, master_ytcfg)
3128
3129         return webpage, master_ytcfg, player_responses, player_url
3130
3131     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url):
3132         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3133         is_live = get_first(video_details, 'isLive')
3134         if is_live is None:
3135             is_live = get_first(live_broadcast_details, 'isLiveNow')
3136
3137         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
3138         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live))
3139
3140         return live_broadcast_details, is_live, streaming_data, formats
3141
3142     def _real_extract(self, url):
3143         url, smuggled_data = unsmuggle_url(url, {})
3144         video_id = self._match_id(url)
3145
3146         base_url = self.http_scheme() + '//www.youtube.com/'
3147         webpage_url = base_url + 'watch?v=' + video_id
3148
3149         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3150
3151         playability_statuses = traverse_obj(
3152             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
3153
3154         trailer_video_id = get_first(
3155             playability_statuses,
3156             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3157             expected_type=str)
3158         if trailer_video_id:
3159             return self.url_result(
3160                 trailer_video_id, self.ie_key(), trailer_video_id)
3161
3162         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3163                        if webpage else (lambda x: None))
3164
3165         video_details = traverse_obj(
3166             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
3167         microformats = traverse_obj(
3168             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3169             expected_type=dict, default=[])
3170         video_title = (
3171             get_first(video_details, 'title')
3172             or self._get_text(microformats, (..., 'title'))
3173             or search_meta(['og:title', 'twitter:title', 'title']))
3174         video_description = get_first(video_details, 'shortDescription')
3175
3176         multifeed_metadata_list = get_first(
3177             player_responses,
3178             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3179             expected_type=str)
3180         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3181             if self.get_param('noplaylist'):
3182                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3183             else:
3184                 entries = []
3185                 feed_ids = []
3186                 for feed in multifeed_metadata_list.split(','):
3187                     # Unquote should take place before split on comma (,) since textual
3188                     # fields may contain comma as well (see
3189                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3190                     feed_data = compat_parse_qs(
3191                         compat_urllib_parse_unquote_plus(feed))
3192
3193                     def feed_entry(name):
3194                         return try_get(
3195                             feed_data, lambda x: x[name][0], compat_str)
3196
3197                     feed_id = feed_entry('id')
3198                     if not feed_id:
3199                         continue
3200                     feed_title = feed_entry('title')
3201                     title = video_title
3202                     if feed_title:
3203                         title += ' (%s)' % feed_title
3204                     entries.append({
3205                         '_type': 'url_transparent',
3206                         'ie_key': 'Youtube',
3207                         'url': smuggle_url(
3208                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3209                             {'force_singlefeed': True}),
3210                         'title': title,
3211                     })
3212                     feed_ids.append(feed_id)
3213                 self.to_screen(
3214                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
3215                     % (', '.join(feed_ids), video_id))
3216                 return self.playlist_result(
3217                     entries, video_id, video_title, video_description)
3218
3219         live_broadcast_details, is_live, streaming_data, formats = self._list_formats(video_id, microformats, video_details, player_responses, player_url)
3220
3221         if not formats:
3222             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
3223                 self.report_drm(video_id)
3224             pemr = get_first(
3225                 playability_statuses,
3226                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
3227             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
3228             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
3229             if subreason:
3230                 if subreason == 'The uploader has not made this video available in your country.':
3231                     countries = get_first(microformats, 'availableCountries')
3232                     if not countries:
3233                         regions_allowed = search_meta('regionsAllowed')
3234                         countries = regions_allowed.split(',') if regions_allowed else None
3235                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
3236                 reason += f'. {subreason}'
3237             if reason:
3238                 self.raise_no_formats(reason, expected=True)
3239
3240         keywords = get_first(video_details, 'keywords', expected_type=list) or []
3241         if not keywords and webpage:
3242             keywords = [
3243                 unescapeHTML(m.group('content'))
3244                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
3245         for keyword in keywords:
3246             if keyword.startswith('yt:stretch='):
3247                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
3248                 if mobj:
3249                     # NB: float is intentional for forcing float division
3250                     w, h = (float(v) for v in mobj.groups())
3251                     if w > 0 and h > 0:
3252                         ratio = w / h
3253                         for f in formats:
3254                             if f.get('vcodec') != 'none':
3255                                 f['stretched_ratio'] = ratio
3256                         break
3257         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
3258         thumbnail_url = search_meta(['og:image', 'twitter:image'])
3259         if thumbnail_url:
3260             thumbnails.append({
3261                 'url': thumbnail_url,
3262             })
3263         original_thumbnails = thumbnails.copy()
3264
3265         # The best resolution thumbnails sometimes does not appear in the webpage
3266         # See: https://github.com/ytdl-org/youtube-dl/issues/29049, https://github.com/yt-dlp/yt-dlp/issues/340
3267         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
3268         thumbnail_names = [
3269             'maxresdefault', 'hq720', 'sddefault', 'sd1', 'sd2', 'sd3',
3270             'hqdefault', 'hq1', 'hq2', 'hq3', '0',
3271             'mqdefault', 'mq1', 'mq2', 'mq3',
3272             'default', '1', '2', '3'
3273         ]
3274         n_thumbnail_names = len(thumbnail_names)
3275         thumbnails.extend({
3276             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
3277                 video_id=video_id, name=name, ext=ext,
3278                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
3279         } for name in thumbnail_names for ext in ('webp', 'jpg'))
3280         for thumb in thumbnails:
3281             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
3282             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
3283         self._remove_duplicate_formats(thumbnails)
3284         self._downloader._sort_thumbnails(original_thumbnails)
3285
3286         category = get_first(microformats, 'category') or search_meta('genre')
3287         channel_id = str_or_none(
3288             get_first(video_details, 'channelId')
3289             or get_first(microformats, 'externalChannelId')
3290             or search_meta('channelId'))
3291         duration = int_or_none(
3292             get_first(video_details, 'lengthSeconds')
3293             or get_first(microformats, 'lengthSeconds')
3294             or parse_duration(search_meta('duration'))) or None
3295         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
3296
3297         live_content = get_first(video_details, 'isLiveContent')
3298         is_upcoming = get_first(video_details, 'isUpcoming')
3299         if is_live is None:
3300             if is_upcoming or live_content is False:
3301                 is_live = False
3302         if is_upcoming is None and (live_content or is_live):
3303             is_upcoming = False
3304         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
3305         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
3306         if not duration and live_end_time and live_start_time:
3307             duration = live_end_time - live_start_time
3308
3309         if is_live and self.get_param('live_from_start'):
3310             self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
3311
3312         formats.extend(self._extract_storyboard(player_responses, duration))
3313
3314         # Source is given priority since formats that throttle are given lower source_preference
3315         # When throttling issue is fully fixed, remove this
3316         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
3317
3318         info = {
3319             'id': video_id,
3320             'title': video_title,
3321             'formats': formats,
3322             'thumbnails': thumbnails,
3323             # The best thumbnail that we are sure exists. Prevents unnecessary
3324             # URL checking if user don't care about getting the best possible thumbnail
3325             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
3326             'description': video_description,
3327             'upload_date': unified_strdate(
3328                 get_first(microformats, 'uploadDate')
3329                 or search_meta('uploadDate')),
3330             'uploader': get_first(video_details, 'author'),
3331             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
3332             'uploader_url': owner_profile_url,
3333             'channel_id': channel_id,
3334             'channel_url': format_field(channel_id, template='https://www.youtube.com/channel/%s'),
3335             'duration': duration,
3336             'view_count': int_or_none(
3337                 get_first((video_details, microformats), (..., 'viewCount'))
3338                 or search_meta('interactionCount')),
3339             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
3340             'age_limit': 18 if (
3341                 get_first(microformats, 'isFamilySafe') is False
3342                 or search_meta('isFamilyFriendly') == 'false'
3343                 or search_meta('og:restrictions:age') == '18+') else 0,
3344             'webpage_url': webpage_url,
3345             'categories': [category] if category else None,
3346             'tags': keywords,
3347             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
3348             'is_live': is_live,
3349             'was_live': (False if is_live or is_upcoming or live_content is False
3350                          else None if is_live is None or is_upcoming is None
3351                          else live_content),
3352             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
3353             'release_timestamp': live_start_time,
3354         }
3355
3356         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
3357         if pctr:
3358             def get_lang_code(track):
3359                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
3360                         or track.get('languageCode'))
3361
3362             # Converted into dicts to remove duplicates
3363             captions = {
3364                 get_lang_code(sub): sub
3365                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
3366             translation_languages = {
3367                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
3368                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
3369
3370             def process_language(container, base_url, lang_code, sub_name, query):
3371                 lang_subs = container.setdefault(lang_code, [])
3372                 for fmt in self._SUBTITLE_FORMATS:
3373                     query.update({
3374                         'fmt': fmt,
3375                     })
3376                     lang_subs.append({
3377                         'ext': fmt,
3378                         'url': update_url_query(base_url, query),
3379                         'name': sub_name,
3380                     })
3381
3382             subtitles, automatic_captions = {}, {}
3383             for lang_code, caption_track in captions.items():
3384                 base_url = caption_track.get('baseUrl')
3385                 if not base_url:
3386                     continue
3387                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
3388                 if caption_track.get('kind') != 'asr':
3389                     if not lang_code:
3390                         continue
3391                     process_language(
3392                         subtitles, base_url, lang_code, lang_name, {})
3393                     if not caption_track.get('isTranslatable'):
3394                         continue
3395                 for trans_code, trans_name in translation_languages.items():
3396                     if not trans_code:
3397                         continue
3398                     if caption_track.get('kind') != 'asr':
3399                         trans_code += f'-{lang_code}'
3400                         trans_name += format_field(lang_name, template=' from %s')
3401                     process_language(
3402                         automatic_captions, base_url, trans_code, trans_name, {'tlang': trans_code})
3403             info['automatic_captions'] = automatic_captions
3404             info['subtitles'] = subtitles
3405
3406         parsed_url = compat_urllib_parse_urlparse(url)
3407         for component in [parsed_url.fragment, parsed_url.query]:
3408             query = compat_parse_qs(component)
3409             for k, v in query.items():
3410                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
3411                     d_k += '_time'
3412                     if d_k not in info and k in s_ks:
3413                         info[d_k] = parse_duration(query[k][0])
3414
3415         # Youtube Music Auto-generated description
3416         if video_description:
3417             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
3418             if mobj:
3419                 release_year = mobj.group('release_year')
3420                 release_date = mobj.group('release_date')
3421                 if release_date:
3422                     release_date = release_date.replace('-', '')
3423                     if not release_year:
3424                         release_year = release_date[:4]
3425                 info.update({
3426                     'album': mobj.group('album'.strip()),
3427                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3428                     'track': mobj.group('track').strip(),
3429                     'release_date': release_date,
3430                     'release_year': int_or_none(release_year),
3431                 })
3432
3433         initial_data = None
3434         if webpage:
3435             initial_data = self._extract_yt_initial_variable(
3436                 webpage, self._YT_INITIAL_DATA_RE, video_id,
3437                 'yt initial data')
3438         if not initial_data:
3439             query = {'videoId': video_id}
3440             query.update(self._get_checkok_params())
3441             initial_data = self._extract_response(
3442                 item_id=video_id, ep='next', fatal=False,
3443                 ytcfg=master_ytcfg, query=query,
3444                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
3445                 note='Downloading initial data API JSON')
3446
3447         try:
3448             # This will error if there is no livechat
3449             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3450             info.setdefault('subtitles', {})['live_chat'] = [{
3451                 'url': 'https://www.youtube.com/watch?v=%s' % video_id,  # url is needed to set cookies
3452                 'video_id': video_id,
3453                 'ext': 'json',
3454                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3455             }]
3456         except (KeyError, IndexError, TypeError):
3457             pass
3458
3459         if initial_data:
3460             info['chapters'] = (
3461                 self._extract_chapters_from_json(initial_data, duration)
3462                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3463                 or None)
3464
3465             contents = try_get(
3466                 initial_data,
3467                 lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'],
3468                 list) or []
3469             for content in contents:
3470                 vpir = content.get('videoPrimaryInfoRenderer')
3471                 if vpir:
3472                     stl = vpir.get('superTitleLink')
3473                     if stl:
3474                         stl = self._get_text(stl)
3475                         if try_get(
3476                                 vpir,
3477                                 lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3478                             info['location'] = stl
3479                         else:
3480                             mobj = re.search(r'(.+?)\s*S(\d+)\s*•\s*E(\d+)', stl)
3481                             if mobj:
3482                                 info.update({
3483                                     'series': mobj.group(1),
3484                                     'season_number': int(mobj.group(2)),
3485                                     'episode_number': int(mobj.group(3)),
3486                                 })
3487                     for tlb in (try_get(
3488                             vpir,
3489                             lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3490                             list) or []):
3491                         tbr = tlb.get('toggleButtonRenderer') or {}
3492                         for getter, regex in [(
3493                                 lambda x: x['defaultText']['accessibility']['accessibilityData'],
3494                                 r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3495                                     lambda x: x['accessibility'],
3496                                     lambda x: x['accessibilityData']['accessibilityData'],
3497                                 ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3498                             label = (try_get(tbr, getter, dict) or {}).get('label')
3499                             if label:
3500                                 mobj = re.match(regex, label)
3501                                 if mobj:
3502                                     info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3503                                     break
3504                     sbr_tooltip = try_get(
3505                         vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3506                     if sbr_tooltip:
3507                         like_count, dislike_count = sbr_tooltip.split(' / ')
3508                         info.update({
3509                             'like_count': str_to_int(like_count),
3510                             'dislike_count': str_to_int(dislike_count),
3511                         })
3512                 vsir = content.get('videoSecondaryInfoRenderer')
3513                 if vsir:
3514                     vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
3515                     info.update({
3516                         'channel': self._get_text(vor, 'title'),
3517                         'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
3518
3519                     rows = try_get(
3520                         vsir,
3521                         lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3522                         list) or []
3523                     multiple_songs = False
3524                     for row in rows:
3525                         if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3526                             multiple_songs = True
3527                             break
3528                     for row in rows:
3529                         mrr = row.get('metadataRowRenderer') or {}
3530                         mrr_title = mrr.get('title')
3531                         if not mrr_title:
3532                             continue
3533                         mrr_title = self._get_text(mrr, 'title')
3534                         mrr_contents_text = self._get_text(mrr, ('contents', 0))
3535                         if mrr_title == 'License':
3536                             info['license'] = mrr_contents_text
3537                         elif not multiple_songs:
3538                             if mrr_title == 'Album':
3539                                 info['album'] = mrr_contents_text
3540                             elif mrr_title == 'Artist':
3541                                 info['artist'] = mrr_contents_text
3542                             elif mrr_title == 'Song':
3543                                 info['track'] = mrr_contents_text
3544
3545         fallbacks = {
3546             'channel': 'uploader',
3547             'channel_id': 'uploader_id',
3548             'channel_url': 'uploader_url',
3549         }
3550         for to, frm in fallbacks.items():
3551             if not info.get(to):
3552                 info[to] = info.get(frm)
3553
3554         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3555             v = info.get(s_k)
3556             if v:
3557                 info[d_k] = v
3558
3559         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3560         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3561         is_membersonly = None
3562         is_premium = None
3563         if initial_data and is_private is not None:
3564             is_membersonly = False
3565             is_premium = False
3566             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3567             badge_labels = set()
3568             for content in contents:
3569                 if not isinstance(content, dict):
3570                     continue
3571                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3572             for badge_label in badge_labels:
3573                 if badge_label.lower() == 'members only':
3574                     is_membersonly = True
3575                 elif badge_label.lower() == 'premium':
3576                     is_premium = True
3577                 elif badge_label.lower() == 'unlisted':
3578                     is_unlisted = True
3579
3580         info['availability'] = self._availability(
3581             is_private=is_private,
3582             needs_premium=is_premium,
3583             needs_subscription=is_membersonly,
3584             needs_auth=info['age_limit'] >= 18,
3585             is_unlisted=None if is_private is None else is_unlisted)
3586
3587         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
3588
3589         self.mark_watched(video_id, player_responses)
3590
3591         return info
3592
3593
3594 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
3595
3596     def _extract_channel_id(self, webpage):
3597         channel_id = self._html_search_meta(
3598             'channelId', webpage, 'channel id', default=None)
3599         if channel_id:
3600             return channel_id
3601         channel_url = self._html_search_meta(
3602             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3603              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3604              'twitter:app:url:googleplay'), webpage, 'channel url')
3605         return self._search_regex(
3606             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3607             channel_url, 'channel id')
3608
3609     @staticmethod
3610     def _extract_basic_item_renderer(item):
3611         # Modified from _extract_grid_item_renderer
3612         known_basic_renderers = (
3613             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer'
3614         )
3615         for key, renderer in item.items():
3616             if not isinstance(renderer, dict):
3617                 continue
3618             elif key in known_basic_renderers:
3619                 return renderer
3620             elif key.startswith('grid') and key.endswith('Renderer'):
3621                 return renderer
3622
3623     def _grid_entries(self, grid_renderer):
3624         for item in grid_renderer['items']:
3625             if not isinstance(item, dict):
3626                 continue
3627             renderer = self._extract_basic_item_renderer(item)
3628             if not isinstance(renderer, dict):
3629                 continue
3630             title = self._get_text(renderer, 'title')
3631
3632             # playlist
3633             playlist_id = renderer.get('playlistId')
3634             if playlist_id:
3635                 yield self.url_result(
3636                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3637                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3638                     video_title=title)
3639                 continue
3640             # video
3641             video_id = renderer.get('videoId')
3642             if video_id:
3643                 yield self._extract_video(renderer)
3644                 continue
3645             # channel
3646             channel_id = renderer.get('channelId')
3647             if channel_id:
3648                 yield self.url_result(
3649                     'https://www.youtube.com/channel/%s' % channel_id,
3650                     ie=YoutubeTabIE.ie_key(), video_title=title)
3651                 continue
3652             # generic endpoint URL support
3653             ep_url = urljoin('https://www.youtube.com/', try_get(
3654                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3655                 compat_str))
3656             if ep_url:
3657                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3658                     if ie.suitable(ep_url):
3659                         yield self.url_result(
3660                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3661                         break
3662
3663     def _shelf_entries_from_content(self, shelf_renderer):
3664         content = shelf_renderer.get('content')
3665         if not isinstance(content, dict):
3666             return
3667         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3668         if renderer:
3669             # TODO: add support for nested playlists so each shelf is processed
3670             # as separate playlist
3671             # TODO: this includes only first N items
3672             for entry in self._grid_entries(renderer):
3673                 yield entry
3674         renderer = content.get('horizontalListRenderer')
3675         if renderer:
3676             # TODO
3677             pass
3678
3679     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3680         ep = try_get(
3681             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3682             compat_str)
3683         shelf_url = urljoin('https://www.youtube.com', ep)
3684         if shelf_url:
3685             # Skipping links to another channels, note that checking for
3686             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3687             # will not work
3688             if skip_channels and '/channels?' in shelf_url:
3689                 return
3690             title = self._get_text(shelf_renderer, 'title')
3691             yield self.url_result(shelf_url, video_title=title)
3692         # Shelf may not contain shelf URL, fallback to extraction from content
3693         for entry in self._shelf_entries_from_content(shelf_renderer):
3694             yield entry
3695
3696     def _playlist_entries(self, video_list_renderer):
3697         for content in video_list_renderer['contents']:
3698             if not isinstance(content, dict):
3699                 continue
3700             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
3701             if not isinstance(renderer, dict):
3702                 continue
3703             video_id = renderer.get('videoId')
3704             if not video_id:
3705                 continue
3706             yield self._extract_video(renderer)
3707
3708     def _rich_entries(self, rich_grid_renderer):
3709         renderer = try_get(
3710             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
3711         video_id = renderer.get('videoId')
3712         if not video_id:
3713             return
3714         yield self._extract_video(renderer)
3715
3716     def _video_entry(self, video_renderer):
3717         video_id = video_renderer.get('videoId')
3718         if video_id:
3719             return self._extract_video(video_renderer)
3720
3721     def _post_thread_entries(self, post_thread_renderer):
3722         post_renderer = try_get(
3723             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
3724         if not post_renderer:
3725             return
3726         # video attachment
3727         video_renderer = try_get(
3728             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
3729         video_id = video_renderer.get('videoId')
3730         if video_id:
3731             entry = self._extract_video(video_renderer)
3732             if entry:
3733                 yield entry
3734         # playlist attachment
3735         playlist_id = try_get(
3736             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
3737         if playlist_id:
3738             yield self.url_result(
3739                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
3740                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3741         # inline video links
3742         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
3743         for run in runs:
3744             if not isinstance(run, dict):
3745                 continue
3746             ep_url = try_get(
3747                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
3748             if not ep_url:
3749                 continue
3750             if not YoutubeIE.suitable(ep_url):
3751                 continue
3752             ep_video_id = YoutubeIE._match_id(ep_url)
3753             if video_id == ep_video_id:
3754                 continue
3755             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
3756
3757     def _post_thread_continuation_entries(self, post_thread_continuation):
3758         contents = post_thread_continuation.get('contents')
3759         if not isinstance(contents, list):
3760             return
3761         for content in contents:
3762             renderer = content.get('backstagePostThreadRenderer')
3763             if not isinstance(renderer, dict):
3764                 continue
3765             for entry in self._post_thread_entries(renderer):
3766                 yield entry
3767
3768     r''' # unused
3769     def _rich_grid_entries(self, contents):
3770         for content in contents:
3771             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
3772             if video_renderer:
3773                 entry = self._video_entry(video_renderer)
3774                 if entry:
3775                     yield entry
3776     '''
3777     def _extract_entries(self, parent_renderer, continuation_list):
3778         # continuation_list is modified in-place with continuation_list = [continuation_token]
3779         continuation_list[:] = [None]
3780         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
3781         for content in contents:
3782             if not isinstance(content, dict):
3783                 continue
3784             is_renderer = try_get(content, lambda x: x['itemSectionRenderer'], dict)
3785             if not is_renderer:
3786                 renderer = content.get('richItemRenderer')
3787                 if renderer:
3788                     for entry in self._rich_entries(renderer):
3789                         yield entry
3790                     continuation_list[0] = self._extract_continuation(parent_renderer)
3791                 continue
3792             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
3793             for isr_content in isr_contents:
3794                 if not isinstance(isr_content, dict):
3795                     continue
3796
3797                 known_renderers = {
3798                     'playlistVideoListRenderer': self._playlist_entries,
3799                     'gridRenderer': self._grid_entries,
3800                     'shelfRenderer': lambda x: self._shelf_entries(x),
3801                     'backstagePostThreadRenderer': self._post_thread_entries,
3802                     'videoRenderer': lambda x: [self._video_entry(x)],
3803                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
3804                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
3805                 }
3806                 for key, renderer in isr_content.items():
3807                     if key not in known_renderers:
3808                         continue
3809                     for entry in known_renderers[key](renderer):
3810                         if entry:
3811                             yield entry
3812                     continuation_list[0] = self._extract_continuation(renderer)
3813                     break
3814
3815             if not continuation_list[0]:
3816                 continuation_list[0] = self._extract_continuation(is_renderer)
3817
3818         if not continuation_list[0]:
3819             continuation_list[0] = self._extract_continuation(parent_renderer)
3820
3821     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
3822         continuation_list = [None]
3823         extract_entries = lambda x: self._extract_entries(x, continuation_list)
3824         tab_content = try_get(tab, lambda x: x['content'], dict)
3825         if not tab_content:
3826             return
3827         parent_renderer = (
3828             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
3829             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
3830         for entry in extract_entries(parent_renderer):
3831             yield entry
3832         continuation = continuation_list[0]
3833
3834         for page_num in itertools.count(1):
3835             if not continuation:
3836                 break
3837             headers = self.generate_api_headers(
3838                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
3839             response = self._extract_response(
3840                 item_id='%s page %s' % (item_id, page_num),
3841                 query=continuation, headers=headers, ytcfg=ytcfg,
3842                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
3843
3844             if not response:
3845                 break
3846             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
3847             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
3848             visitor_data = self._extract_visitor_data(response) or visitor_data
3849
3850             known_continuation_renderers = {
3851                 'playlistVideoListContinuation': self._playlist_entries,
3852                 'gridContinuation': self._grid_entries,
3853                 'itemSectionContinuation': self._post_thread_continuation_entries,
3854                 'sectionListContinuation': extract_entries,  # for feeds
3855             }
3856             continuation_contents = try_get(
3857                 response, lambda x: x['continuationContents'], dict) or {}
3858             continuation_renderer = None
3859             for key, value in continuation_contents.items():
3860                 if key not in known_continuation_renderers:
3861                     continue
3862                 continuation_renderer = value
3863                 continuation_list = [None]
3864                 for entry in known_continuation_renderers[key](continuation_renderer):
3865                     yield entry
3866                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
3867                 break
3868             if continuation_renderer:
3869                 continue
3870
3871             known_renderers = {
3872                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
3873                 'gridVideoRenderer': (self._grid_entries, 'items'),
3874                 'gridChannelRenderer': (self._grid_entries, 'items'),
3875                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
3876                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
3877                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
3878                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
3879             }
3880             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
3881             continuation_items = try_get(
3882                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
3883             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
3884             video_items_renderer = None
3885             for key, value in continuation_item.items():
3886                 if key not in known_renderers:
3887                     continue
3888                 video_items_renderer = {known_renderers[key][1]: continuation_items}
3889                 continuation_list = [None]
3890                 for entry in known_renderers[key][0](video_items_renderer):
3891                     yield entry
3892                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
3893                 break
3894             if video_items_renderer:
3895                 continue
3896             break
3897
3898     @staticmethod
3899     def _extract_selected_tab(tabs):
3900         for tab in tabs:
3901             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
3902             if renderer.get('selected') is True:
3903                 return renderer
3904         else:
3905             raise ExtractorError('Unable to find selected tab')
3906
3907     @classmethod
3908     def _extract_uploader(cls, data):
3909         uploader = {}
3910         renderer = cls._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
3911         owner = try_get(
3912             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
3913         if owner:
3914             uploader['uploader'] = owner.get('text')
3915             uploader['uploader_id'] = try_get(
3916                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
3917             uploader['uploader_url'] = urljoin(
3918                 'https://www.youtube.com/',
3919                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
3920         return {k: v for k, v in uploader.items() if v is not None}
3921
3922     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
3923         playlist_id = title = description = channel_url = channel_name = channel_id = None
3924         tags = []
3925
3926         selected_tab = self._extract_selected_tab(tabs)
3927         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
3928         renderer = try_get(
3929             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
3930         if renderer:
3931             channel_name = renderer.get('title')
3932             channel_url = renderer.get('channelUrl')
3933             channel_id = renderer.get('externalId')
3934         else:
3935             renderer = try_get(
3936                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
3937
3938         if renderer:
3939             title = renderer.get('title')
3940             description = renderer.get('description', '')
3941             playlist_id = channel_id
3942             tags = renderer.get('keywords', '').split()
3943
3944         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
3945         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
3946         def _get_uncropped(url):
3947             return url_or_none((url or '').split('=')[0] + '=s0')
3948
3949         avatar_thumbnails = self._extract_thumbnails(renderer, 'avatar')
3950         if avatar_thumbnails:
3951             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
3952             if uncropped_avatar:
3953                 avatar_thumbnails.append({
3954                     'url': uncropped_avatar,
3955                     'id': 'avatar_uncropped',
3956                     'preference': 1
3957                 })
3958
3959         channel_banners = self._extract_thumbnails(
3960             data, ('header', ..., ['banner', 'mobileBanner', 'tvBanner']))
3961         for banner in channel_banners:
3962             banner['preference'] = -10
3963
3964         if channel_banners:
3965             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
3966             if uncropped_banner:
3967                 channel_banners.append({
3968                     'url': uncropped_banner,
3969                     'id': 'banner_uncropped',
3970                     'preference': -5
3971                 })
3972
3973         primary_thumbnails = self._extract_thumbnails(
3974             primary_sidebar_renderer, ('thumbnailRenderer', 'playlistVideoThumbnailRenderer', 'thumbnail'))
3975
3976         if playlist_id is None:
3977             playlist_id = item_id
3978
3979         playlist_stats = traverse_obj(primary_sidebar_renderer, 'stats')
3980         last_updated_unix, _ = self._extract_time_text(playlist_stats, 2)
3981         if title is None:
3982             title = self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag')) or playlist_id
3983         title += format_field(selected_tab, 'title', ' - %s')
3984         title += format_field(selected_tab, 'expandedText', ' - %s')
3985
3986         metadata = {
3987             'playlist_id': playlist_id,
3988             'playlist_title': title,
3989             'playlist_description': description,
3990             'uploader': channel_name,
3991             'uploader_id': channel_id,
3992             'uploader_url': channel_url,
3993             'thumbnails': primary_thumbnails + avatar_thumbnails + channel_banners,
3994             'tags': tags,
3995             'view_count': self._get_count(playlist_stats, 1),
3996             'availability': self._extract_availability(data),
3997             'modified_date': strftime_or_none(last_updated_unix, '%Y%m%d'),
3998             'playlist_count': self._get_count(playlist_stats, 0),
3999             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4000         }
4001         if not channel_id:
4002             metadata.update(self._extract_uploader(data))
4003         metadata.update({
4004             'channel': metadata['uploader'],
4005             'channel_id': metadata['uploader_id'],
4006             'channel_url': metadata['uploader_url']})
4007         return self.playlist_result(
4008             self._entries(
4009                 selected_tab, playlist_id, ytcfg,
4010                 self._extract_account_syncid(ytcfg, data),
4011                 self._extract_visitor_data(data, ytcfg)),
4012             **metadata)
4013
4014     def _extract_mix_playlist(self, playlist, playlist_id, data, ytcfg):
4015         first_id = last_id = response = None
4016         for page_num in itertools.count(1):
4017             videos = list(self._playlist_entries(playlist))
4018             if not videos:
4019                 return
4020             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4021             if start >= len(videos):
4022                 return
4023             for video in videos[start:]:
4024                 if video['id'] == first_id:
4025                     self.to_screen('First video %s found again; Assuming end of Mix' % first_id)
4026                     return
4027                 yield video
4028             first_id = first_id or videos[0]['id']
4029             last_id = videos[-1]['id']
4030             watch_endpoint = try_get(
4031                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4032             headers = self.generate_api_headers(
4033                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4034                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4035             query = {
4036                 'playlistId': playlist_id,
4037                 'videoId': watch_endpoint.get('videoId') or last_id,
4038                 'index': watch_endpoint.get('index') or len(videos),
4039                 'params': watch_endpoint.get('params') or 'OAE%3D'
4040             }
4041             response = self._extract_response(
4042                 item_id='%s page %d' % (playlist_id, page_num),
4043                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4044                 check_get_keys='contents'
4045             )
4046             playlist = try_get(
4047                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4048
4049     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4050         title = playlist.get('title') or try_get(
4051             data, lambda x: x['titleText']['simpleText'], compat_str)
4052         playlist_id = playlist.get('playlistId') or item_id
4053
4054         # Delegating everything except mix playlists to regular tab-based playlist URL
4055         playlist_url = urljoin(url, try_get(
4056             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4057             compat_str))
4058         if playlist_url and playlist_url != url:
4059             return self.url_result(
4060                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4061                 video_title=title)
4062
4063         return self.playlist_result(
4064             self._extract_mix_playlist(playlist, playlist_id, data, ytcfg),
4065             playlist_id=playlist_id, playlist_title=title)
4066
4067     def _extract_availability(self, data):
4068         """
4069         Gets the availability of a given playlist/tab.
4070         Note: Unless YouTube tells us explicitly, we do not assume it is public
4071         @param data: response
4072         """
4073         is_private = is_unlisted = None
4074         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4075         badge_labels = self._extract_badges(renderer)
4076
4077         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4078         privacy_dropdown_entries = try_get(
4079             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4080         for renderer_dict in privacy_dropdown_entries:
4081             is_selected = try_get(
4082                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4083             if not is_selected:
4084                 continue
4085             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4086             if label:
4087                 badge_labels.add(label.lower())
4088                 break
4089
4090         for badge_label in badge_labels:
4091             if badge_label == 'unlisted':
4092                 is_unlisted = True
4093             elif badge_label == 'private':
4094                 is_private = True
4095             elif badge_label == 'public':
4096                 is_unlisted = is_private = False
4097         return self._availability(is_private, False, False, False, is_unlisted)
4098
4099     @staticmethod
4100     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4101         sidebar_renderer = try_get(
4102             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4103         for item in sidebar_renderer:
4104             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4105             if renderer:
4106                 return renderer
4107
4108     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
4109         """
4110         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4111         """
4112         browse_id = params = None
4113         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4114         if not renderer:
4115             return
4116         menu_renderer = try_get(
4117             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4118         for menu_item in menu_renderer:
4119             if not isinstance(menu_item, dict):
4120                 continue
4121             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4122             text = try_get(
4123                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4124             if not text or text.lower() != 'show unavailable videos':
4125                 continue
4126             browse_endpoint = try_get(
4127                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4128             browse_id = browse_endpoint.get('browseId')
4129             params = browse_endpoint.get('params')
4130             break
4131
4132         headers = self.generate_api_headers(
4133             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4134             visitor_data=self._extract_visitor_data(data, ytcfg))
4135         query = {
4136             'params': params or 'wgYCCAA=',
4137             'browseId': browse_id or 'VL%s' % item_id
4138         }
4139         return self._extract_response(
4140             item_id=item_id, headers=headers, query=query,
4141             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4142             note='Downloading API JSON with unavailable videos')
4143
4144     def _extract_webpage(self, url, item_id, fatal=True):
4145         retries = self.get_param('extractor_retries', 3)
4146         count = -1
4147         webpage = data = last_error = None
4148         while count < retries:
4149             count += 1
4150             # Sometimes youtube returns a webpage with incomplete ytInitialData
4151             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4152             if last_error:
4153                 self.report_warning('%s. Retrying ...' % last_error)
4154             try:
4155                 webpage = self._download_webpage(
4156                     url, item_id,
4157                     note='Downloading webpage%s' % (' (retry #%d)' % count if count else '',))
4158                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
4159             except ExtractorError as e:
4160                 if isinstance(e.cause, network_exceptions):
4161                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
4162                         last_error = error_to_compat_str(e.cause or e.msg)
4163                         if count < retries:
4164                             continue
4165                 if fatal:
4166                     raise
4167                 self.report_warning(error_to_compat_str(e))
4168                 break
4169             else:
4170                 try:
4171                     self._extract_and_report_alerts(data)
4172                 except ExtractorError as e:
4173                     if fatal:
4174                         raise
4175                     self.report_warning(error_to_compat_str(e))
4176                     break
4177
4178                 if dict_get(data, ('contents', 'currentVideoEndpoint')):
4179                     break
4180
4181                 last_error = 'Incomplete yt initial data received'
4182                 if count >= retries:
4183                     if fatal:
4184                         raise ExtractorError(last_error)
4185                     self.report_warning(last_error)
4186                     break
4187
4188         return webpage, data
4189
4190     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
4191         data = None
4192         if 'webpage' not in self._configuration_arg('skip'):
4193             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
4194             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
4195         if not data:
4196             if not ytcfg and self.is_authenticated:
4197                 msg = 'Playlists that require authentication may not extract correctly without a successful webpage download.'
4198                 if 'authcheck' not in self._configuration_arg('skip') and fatal:
4199                     raise ExtractorError(
4200                         msg + ' If you are not downloading private content, or your cookies are only for the first account and channel,'
4201                               ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
4202                         expected=True)
4203                 self.report_warning(msg, only_once=True)
4204             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
4205         return data, ytcfg
4206
4207     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
4208         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
4209         resolve_response = self._extract_response(
4210             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
4211             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
4212         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
4213         for ep_key, ep in endpoints.items():
4214             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
4215             if params:
4216                 return self._extract_response(
4217                     item_id=item_id, query=params, ep=ep, headers=headers,
4218                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
4219                     check_get_keys=('contents', 'currentVideoEndpoint'))
4220         err_note = 'Failed to resolve url (does the playlist exist?)'
4221         if fatal:
4222             raise ExtractorError(err_note, expected=True)
4223         self.report_warning(err_note, item_id)
4224
4225     @staticmethod
4226     def _smuggle_data(entries, data):
4227         for entry in entries:
4228             if data:
4229                 entry['url'] = smuggle_url(entry['url'], data)
4230             yield entry
4231
4232     _SEARCH_PARAMS = None
4233
4234     def _search_results(self, query, params=NO_DEFAULT):
4235         data = {'query': query}
4236         if params is NO_DEFAULT:
4237             params = self._SEARCH_PARAMS
4238         if params:
4239             data['params'] = params
4240         continuation_list = [None]
4241         for page_num in itertools.count(1):
4242             data.update(continuation_list[0] or {})
4243             search = self._extract_response(
4244                 item_id='query "%s" page %s' % (query, page_num), ep='search', query=data,
4245                 check_get_keys=('contents', 'onResponseReceivedCommands'))
4246             slr_contents = try_get(
4247                 search,
4248                 (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
4249                  lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
4250                 list)
4251             yield from self._extract_entries({'contents': slr_contents}, continuation_list)
4252             if not continuation_list[0]:
4253                 break
4254
4255
4256 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
4257     IE_DESC = 'YouTube Tabs'
4258     _VALID_URL = r'''(?x:
4259         https?://
4260             (?:\w+\.)?
4261             (?:
4262                 youtube(?:kids)?\.com|
4263                 %(invidious)s
4264             )/
4265             (?:
4266                 (?P<channel_type>channel|c|user|browse)/|
4267                 (?P<not_channel>
4268                     feed/|hashtag/|
4269                     (?:playlist|watch)\?.*?\blist=
4270                 )|
4271                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
4272             )
4273             (?P<id>[^/?\#&]+)
4274     )''' % {
4275         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
4276         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
4277     }
4278     IE_NAME = 'youtube:tab'
4279
4280     _TESTS = [{
4281         'note': 'playlists, multipage',
4282         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
4283         'playlist_mincount': 94,
4284         'info_dict': {
4285             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4286             'title': 'Igor Kleiner - Playlists',
4287             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4288             'uploader': 'Igor Kleiner',
4289             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4290             'channel': 'Igor Kleiner',
4291             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4292             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4293             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4294             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4295             'channel_follower_count': int
4296         },
4297     }, {
4298         'note': 'playlists, multipage, different order',
4299         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
4300         'playlist_mincount': 94,
4301         'info_dict': {
4302             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
4303             'title': 'Igor Kleiner - Playlists',
4304             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
4305             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4306             'uploader': 'Igor Kleiner',
4307             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4308             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
4309             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
4310             'channel': 'Igor Kleiner',
4311             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
4312             'channel_follower_count': int
4313         },
4314     }, {
4315         'note': 'playlists, series',
4316         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
4317         'playlist_mincount': 5,
4318         'info_dict': {
4319             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4320             'title': '3Blue1Brown - Playlists',
4321             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4322             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4323             'uploader': '3Blue1Brown',
4324             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4325             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4326             'channel': '3Blue1Brown',
4327             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4328             'tags': ['Mathematics'],
4329             'channel_follower_count': int
4330         },
4331     }, {
4332         'note': 'playlists, singlepage',
4333         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
4334         'playlist_mincount': 4,
4335         'info_dict': {
4336             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4337             'title': 'ThirstForScience - Playlists',
4338             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
4339             'uploader': 'ThirstForScience',
4340             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4341             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4342             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
4343             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
4344             'tags': 'count:13',
4345             'channel': 'ThirstForScience',
4346             'channel_follower_count': int
4347         }
4348     }, {
4349         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
4350         'only_matching': True,
4351     }, {
4352         'note': 'basic, single video playlist',
4353         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4354         'info_dict': {
4355             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4356             'uploader': 'Sergey M.',
4357             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4358             'title': 'youtube-dl public playlist',
4359             'description': '',
4360             'tags': [],
4361             'view_count': int,
4362             'modified_date': '20201130',
4363             'channel': 'Sergey M.',
4364             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4365             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4366             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4367         },
4368         'playlist_count': 1,
4369     }, {
4370         'note': 'empty playlist',
4371         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4372         'info_dict': {
4373             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4374             'uploader': 'Sergey M.',
4375             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
4376             'title': 'youtube-dl empty playlist',
4377             'tags': [],
4378             'channel': 'Sergey M.',
4379             'description': '',
4380             'modified_date': '20160902',
4381             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
4382             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4383             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4384         },
4385         'playlist_count': 0,
4386     }, {
4387         'note': 'Home tab',
4388         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
4389         'info_dict': {
4390             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4391             'title': 'lex will - Home',
4392             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4393             'uploader': 'lex will',
4394             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4395             'channel': 'lex will',
4396             'tags': ['bible', 'history', 'prophesy'],
4397             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4398             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4399             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4400             'channel_follower_count': int
4401         },
4402         'playlist_mincount': 2,
4403     }, {
4404         'note': 'Videos tab',
4405         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
4406         'info_dict': {
4407             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4408             'title': 'lex will - Videos',
4409             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4410             'uploader': 'lex will',
4411             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4412             'tags': ['bible', 'history', 'prophesy'],
4413             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4414             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4415             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4416             'channel': 'lex will',
4417             'channel_follower_count': int
4418         },
4419         'playlist_mincount': 975,
4420     }, {
4421         'note': 'Videos tab, sorted by popular',
4422         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
4423         'info_dict': {
4424             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4425             'title': 'lex will - Videos',
4426             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4427             'uploader': 'lex will',
4428             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4429             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4430             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4431             'channel': 'lex will',
4432             'tags': ['bible', 'history', 'prophesy'],
4433             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4434             'channel_follower_count': int
4435         },
4436         'playlist_mincount': 199,
4437     }, {
4438         'note': 'Playlists tab',
4439         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
4440         'info_dict': {
4441             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4442             'title': 'lex will - Playlists',
4443             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4444             'uploader': 'lex will',
4445             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4446             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4447             'channel': 'lex will',
4448             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4449             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4450             'tags': ['bible', 'history', 'prophesy'],
4451             'channel_follower_count': int
4452         },
4453         'playlist_mincount': 17,
4454     }, {
4455         'note': 'Community tab',
4456         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
4457         'info_dict': {
4458             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4459             'title': 'lex will - Community',
4460             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4461             'uploader': 'lex will',
4462             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4463             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4464             'channel': 'lex will',
4465             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4466             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4467             'tags': ['bible', 'history', 'prophesy'],
4468             'channel_follower_count': int
4469         },
4470         'playlist_mincount': 18,
4471     }, {
4472         'note': 'Channels tab',
4473         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
4474         'info_dict': {
4475             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4476             'title': 'lex will - Channels',
4477             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
4478             'uploader': 'lex will',
4479             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4480             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4481             'channel': 'lex will',
4482             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
4483             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
4484             'tags': ['bible', 'history', 'prophesy'],
4485             'channel_follower_count': int
4486         },
4487         'playlist_mincount': 12,
4488     }, {
4489         'note': 'Search tab',
4490         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
4491         'playlist_mincount': 40,
4492         'info_dict': {
4493             'id': 'UCYO_jab_esuFRV4b17AJtAw',
4494             'title': '3Blue1Brown - Search - linear algebra',
4495             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
4496             'uploader': '3Blue1Brown',
4497             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
4498             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4499             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
4500             'tags': ['Mathematics'],
4501             'channel': '3Blue1Brown',
4502             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
4503             'channel_follower_count': int
4504         },
4505     }, {
4506         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4507         'only_matching': True,
4508     }, {
4509         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4510         'only_matching': True,
4511     }, {
4512         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
4513         'only_matching': True,
4514     }, {
4515         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
4516         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4517         'info_dict': {
4518             'title': '29C3: Not my department',
4519             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
4520             'uploader': 'Christiaan008',
4521             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4522             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
4523             'tags': [],
4524             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4525             'view_count': int,
4526             'modified_date': '20150605',
4527             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
4528             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
4529             'channel': 'Christiaan008',
4530         },
4531         'playlist_count': 96,
4532     }, {
4533         'note': 'Large playlist',
4534         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
4535         'info_dict': {
4536             'title': 'Uploads from Cauchemar',
4537             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
4538             'uploader': 'Cauchemar',
4539             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4540             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
4541             'tags': [],
4542             'modified_date': r're:\d{8}',
4543             'channel': 'Cauchemar',
4544             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
4545             'view_count': int,
4546             'description': '',
4547             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
4548         },
4549         'playlist_mincount': 1123,
4550         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4551     }, {
4552         'note': 'even larger playlist, 8832 videos',
4553         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
4554         'only_matching': True,
4555     }, {
4556         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
4557         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
4558         'info_dict': {
4559             'title': 'Uploads from Interstellar Movie',
4560             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
4561             'uploader': 'Interstellar Movie',
4562             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4563             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
4564             'tags': [],
4565             'view_count': int,
4566             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
4567             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
4568             'channel': 'Interstellar Movie',
4569             'description': '',
4570             'modified_date': r're:\d{8}',
4571         },
4572         'playlist_mincount': 21,
4573     }, {
4574         'note': 'Playlist with "show unavailable videos" button',
4575         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
4576         'info_dict': {
4577             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
4578             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
4579             'uploader': 'Phim Siêu Nhân Nhật Bản',
4580             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4581             'view_count': int,
4582             'channel': 'Phim Siêu Nhân Nhật Bản',
4583             'tags': [],
4584             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4585             'description': '',
4586             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
4587             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
4588             'modified_date': r're:\d{8}',
4589         },
4590         'playlist_mincount': 200,
4591         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4592     }, {
4593         'note': 'Playlist with unavailable videos in page 7',
4594         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
4595         'info_dict': {
4596             'title': 'Uploads from BlankTV',
4597             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
4598             'uploader': 'BlankTV',
4599             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4600             'channel': 'BlankTV',
4601             'channel_url': 'https://www.youtube.com/c/blanktv',
4602             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
4603             'view_count': int,
4604             'tags': [],
4605             'uploader_url': 'https://www.youtube.com/c/blanktv',
4606             'modified_date': r're:\d{8}',
4607             'description': '',
4608         },
4609         'playlist_mincount': 1000,
4610         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4611     }, {
4612         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
4613         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4614         'info_dict': {
4615             'title': 'Data Analysis with Dr Mike Pound',
4616             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
4617             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4618             'uploader': 'Computerphile',
4619             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
4620             'uploader_url': 'https://www.youtube.com/user/Computerphile',
4621             'tags': [],
4622             'view_count': int,
4623             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
4624             'channel_url': 'https://www.youtube.com/user/Computerphile',
4625             'channel': 'Computerphile',
4626         },
4627         'playlist_mincount': 11,
4628     }, {
4629         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
4630         'only_matching': True,
4631     }, {
4632         'note': 'Playlist URL that does not actually serve a playlist',
4633         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
4634         'info_dict': {
4635             'id': 'FqZTN594JQw',
4636             'ext': 'webm',
4637             'title': "Smiley's People 01 detective, Adventure Series, Action",
4638             'uploader': 'STREEM',
4639             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
4640             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
4641             'upload_date': '20150526',
4642             'license': 'Standard YouTube License',
4643             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
4644             'categories': ['People & Blogs'],
4645             'tags': list,
4646             'view_count': int,
4647             'like_count': int,
4648         },
4649         'params': {
4650             'skip_download': True,
4651         },
4652         'skip': 'This video is not available.',
4653         'add_ie': [YoutubeIE.ie_key()],
4654     }, {
4655         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
4656         'only_matching': True,
4657     }, {
4658         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
4659         'only_matching': True,
4660     }, {
4661         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
4662         'info_dict': {
4663             'id': 'GgL890LIznQ',  # This will keep changing
4664             'ext': 'mp4',
4665             'title': str,
4666             'uploader': 'Sky News',
4667             'uploader_id': 'skynews',
4668             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
4669             'upload_date': r're:\d{8}',
4670             'description': str,
4671             'categories': ['News & Politics'],
4672             'tags': list,
4673             'like_count': int,
4674             'release_timestamp': 1642502819,
4675             'channel': 'Sky News',
4676             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
4677             'age_limit': 0,
4678             'view_count': int,
4679             'thumbnail': 'https://i.ytimg.com/vi/GgL890LIznQ/maxresdefault_live.jpg',
4680             'playable_in_embed': True,
4681             'release_date': '20220118',
4682             'availability': 'public',
4683             'live_status': 'is_live',
4684             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
4685             'channel_follower_count': int
4686         },
4687         'params': {
4688             'skip_download': True,
4689         },
4690         'expected_warnings': ['Ignoring subtitle tracks found in '],
4691     }, {
4692         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
4693         'info_dict': {
4694             'id': 'a48o2S1cPoo',
4695             'ext': 'mp4',
4696             'title': 'The Young Turks - Live Main Show',
4697             'uploader': 'The Young Turks',
4698             'uploader_id': 'TheYoungTurks',
4699             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
4700             'upload_date': '20150715',
4701             'license': 'Standard YouTube License',
4702             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
4703             'categories': ['News & Politics'],
4704             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
4705             'like_count': int,
4706         },
4707         'params': {
4708             'skip_download': True,
4709         },
4710         'only_matching': True,
4711     }, {
4712         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
4713         'only_matching': True,
4714     }, {
4715         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
4716         'only_matching': True,
4717     }, {
4718         'note': 'A channel that is not live. Should raise error',
4719         'url': 'https://www.youtube.com/user/numberphile/live',
4720         'only_matching': True,
4721     }, {
4722         'url': 'https://www.youtube.com/feed/trending',
4723         'only_matching': True,
4724     }, {
4725         'url': 'https://www.youtube.com/feed/library',
4726         'only_matching': True,
4727     }, {
4728         'url': 'https://www.youtube.com/feed/history',
4729         'only_matching': True,
4730     }, {
4731         'url': 'https://www.youtube.com/feed/subscriptions',
4732         'only_matching': True,
4733     }, {
4734         'url': 'https://www.youtube.com/feed/watch_later',
4735         'only_matching': True,
4736     }, {
4737         'note': 'Recommended - redirects to home page.',
4738         'url': 'https://www.youtube.com/feed/recommended',
4739         'only_matching': True,
4740     }, {
4741         'note': 'inline playlist with not always working continuations',
4742         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
4743         'only_matching': True,
4744     }, {
4745         'url': 'https://www.youtube.com/course',
4746         'only_matching': True,
4747     }, {
4748         'url': 'https://www.youtube.com/zsecurity',
4749         'only_matching': True,
4750     }, {
4751         'url': 'http://www.youtube.com/NASAgovVideo/videos',
4752         'only_matching': True,
4753     }, {
4754         'url': 'https://www.youtube.com/TheYoungTurks/live',
4755         'only_matching': True,
4756     }, {
4757         'url': 'https://www.youtube.com/hashtag/cctv9',
4758         'info_dict': {
4759             'id': 'cctv9',
4760             'title': '#cctv9',
4761             'tags': [],
4762         },
4763         'playlist_mincount': 350,
4764     }, {
4765         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
4766         'only_matching': True,
4767     }, {
4768         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
4769         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
4770         'only_matching': True
4771     }, {
4772         'note': '/browse/ should redirect to /channel/',
4773         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
4774         'only_matching': True
4775     }, {
4776         'note': 'VLPL, should redirect to playlist?list=PL...',
4777         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
4778         'info_dict': {
4779             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
4780             'uploader': 'NoCopyrightSounds',
4781             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
4782             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
4783             'title': 'NCS Releases',
4784             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
4785             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
4786             'modified_date': r're:\d{8}',
4787             'view_count': int,
4788             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
4789             'tags': [],
4790             'channel': 'NoCopyrightSounds',
4791         },
4792         'playlist_mincount': 166,
4793         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
4794     }, {
4795         'note': 'Topic, should redirect to playlist?list=UU...',
4796         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
4797         'info_dict': {
4798             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
4799             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
4800             'title': 'Uploads from Royalty Free Music - Topic',
4801             'uploader': 'Royalty Free Music - Topic',
4802             'tags': [],
4803             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
4804             'channel': 'Royalty Free Music - Topic',
4805             'view_count': int,
4806             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4807             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4808             'modified_date': r're:\d{8}',
4809             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4810             'description': '',
4811         },
4812         'expected_warnings': [
4813             'The URL does not have a videos tab',
4814             r'[Uu]navailable videos (are|will be) hidden',
4815         ],
4816         'playlist_mincount': 101,
4817     }, {
4818         'note': 'Topic without a UU playlist',
4819         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
4820         'info_dict': {
4821             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
4822             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
4823             'tags': [],
4824         },
4825         'expected_warnings': [
4826             'the playlist redirect gave error',
4827         ],
4828         'playlist_mincount': 9,
4829     }, {
4830         'note': 'Youtube music Album',
4831         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
4832         'info_dict': {
4833             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
4834             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
4835             'tags': [],
4836             'view_count': int,
4837             'description': '',
4838             'availability': 'unlisted',
4839             'modified_date': r're:\d{8}',
4840         },
4841         'playlist_count': 50,
4842     }, {
4843         'note': 'unlisted single video playlist',
4844         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
4845         'info_dict': {
4846             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
4847             'uploader': 'colethedj',
4848             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
4849             'title': 'yt-dlp unlisted playlist test',
4850             'availability': 'unlisted',
4851             'tags': [],
4852             'modified_date': '20211208',
4853             'channel': 'colethedj',
4854             'view_count': int,
4855             'description': '',
4856             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
4857             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
4858             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
4859         },
4860         'playlist_count': 1,
4861     }, {
4862         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
4863         'url': 'https://www.youtube.com/feed/recommended',
4864         'info_dict': {
4865             'id': 'recommended',
4866             'title': 'recommended',
4867             'tags': [],
4868         },
4869         'playlist_mincount': 50,
4870         'params': {
4871             'skip_download': True,
4872             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
4873         },
4874     }, {
4875         'note': 'API Fallback: /videos tab, sorted by oldest first',
4876         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
4877         'info_dict': {
4878             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
4879             'title': 'Cody\'sLab - Videos',
4880             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
4881             'uploader': 'Cody\'sLab',
4882             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
4883             'channel': 'Cody\'sLab',
4884             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
4885             'tags': [],
4886             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
4887             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
4888             'channel_follower_count': int
4889         },
4890         'playlist_mincount': 650,
4891         'params': {
4892             'skip_download': True,
4893             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
4894         },
4895     }, {
4896         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
4897         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
4898         'info_dict': {
4899             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
4900             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
4901             'title': 'Uploads from Royalty Free Music - Topic',
4902             'uploader': 'Royalty Free Music - Topic',
4903             'modified_date': r're:\d{8}',
4904             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
4905             'description': '',
4906             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4907             'tags': [],
4908             'channel': 'Royalty Free Music - Topic',
4909             'view_count': int,
4910             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
4911         },
4912         'expected_warnings': [
4913             'does not have a videos tab',
4914             r'[Uu]navailable videos (are|will be) hidden',
4915         ],
4916         'playlist_mincount': 101,
4917         'params': {
4918             'skip_download': True,
4919             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
4920         },
4921     }]
4922
4923     @classmethod
4924     def suitable(cls, url):
4925         return False if YoutubeIE.suitable(url) else super(
4926             YoutubeTabIE, cls).suitable(url)
4927
4928     def _real_extract(self, url):
4929         url, smuggled_data = unsmuggle_url(url, {})
4930         if self.is_music_url(url):
4931             smuggled_data['is_music_url'] = True
4932         info_dict = self.__real_extract(url, smuggled_data)
4933         if info_dict.get('entries'):
4934             info_dict['entries'] = self._smuggle_data(info_dict['entries'], smuggled_data)
4935         return info_dict
4936
4937     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/\w+))?(?P<post>.*)$')
4938
4939     def __real_extract(self, url, smuggled_data):
4940         item_id = self._match_id(url)
4941         url = compat_urlparse.urlunparse(
4942             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
4943         compat_opts = self.get_param('compat_opts', [])
4944
4945         def get_mobj(url):
4946             mobj = self._URL_RE.match(url).groupdict()
4947             mobj.update((k, '') for k, v in mobj.items() if v is None)
4948             return mobj
4949
4950         mobj, redirect_warning = get_mobj(url), None
4951         # Youtube returns incomplete data if tabname is not lower case
4952         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
4953         if is_channel:
4954             if smuggled_data.get('is_music_url'):
4955                 if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
4956                     item_id = item_id[2:]
4957                     pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
4958                 elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
4959                     mdata = self._extract_tab_endpoint(
4960                         f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
4961                     murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
4962                                         get_all=False, expected_type=compat_str)
4963                     if not murl:
4964                         raise ExtractorError('Failed to resolve album to playlist')
4965                     return self.url_result(murl, ie=YoutubeTabIE.ie_key())
4966                 elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
4967                     pre = f'https://www.youtube.com/channel/{item_id}'
4968
4969         original_tab_name = tab
4970         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
4971             # Home URLs should redirect to /videos/
4972             redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
4973                                 'To download only the videos in the home page, add a "/featured" to the URL')
4974             tab = '/videos'
4975
4976         url = ''.join((pre, tab, post))
4977         mobj = get_mobj(url)
4978
4979         # Handle both video/playlist URLs
4980         qs = parse_qs(url)
4981         video_id, playlist_id = [qs.get(key, [None])[0] for key in ('v', 'list')]
4982
4983         if not video_id and mobj['not_channel'].startswith('watch'):
4984             if not playlist_id:
4985                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
4986                 raise ExtractorError('Unable to recognize tab page')
4987             # Common mistake: https://www.youtube.com/watch?list=playlist_id
4988             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
4989             url = f'https://www.youtube.com/playlist?list={playlist_id}'
4990             mobj = get_mobj(url)
4991
4992         if video_id and playlist_id:
4993             if self.get_param('noplaylist'):
4994                 self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
4995                 return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
4996                                        ie=YoutubeIE.ie_key(), video_id=video_id)
4997             self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
4998
4999         data, ytcfg = self._extract_data(url, item_id)
5000
5001         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5002         if tabs:
5003             selected_tab = self._extract_selected_tab(tabs)
5004             selected_tab_name = selected_tab.get('title', '').lower()
5005             if selected_tab_name == 'home':
5006                 selected_tab_name = 'featured'
5007             requested_tab_name = mobj['tab'][1:]
5008             if 'no-youtube-channel-redirect' not in compat_opts:
5009                 if requested_tab_name == 'live':
5010                     # Live tab should have redirected to the video
5011                     raise ExtractorError('The channel is not currently live', expected=True)
5012                 if requested_tab_name not in ('', selected_tab_name):
5013                     redirect_warning = f'The channel does not have a {requested_tab_name} tab'
5014                     if not original_tab_name:
5015                         if item_id[:2] == 'UC':
5016                             # Topic channels don't have /videos. Use the equivalent playlist instead
5017                             pl_id = f'UU{item_id[2:]}'
5018                             pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
5019                             try:
5020                                 data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
5021                             except ExtractorError:
5022                                 redirect_warning += ' and the playlist redirect gave error'
5023                             else:
5024                                 item_id, url, selected_tab_name = pl_id, pl_url, requested_tab_name
5025                                 redirect_warning += f'. Redirecting to playlist {pl_id} instead'
5026                         if selected_tab_name and selected_tab_name != requested_tab_name:
5027                             redirect_warning += f'. {selected_tab_name} tab is being downloaded instead'
5028                     else:
5029                         raise ExtractorError(redirect_warning, expected=True)
5030
5031         if redirect_warning:
5032             self.to_screen(redirect_warning)
5033         self.write_debug(f'Final URL: {url}')
5034
5035         # YouTube sometimes provides a button to reload playlist with unavailable videos.
5036         if 'no-youtube-unavailable-videos' not in compat_opts:
5037             data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
5038         self._extract_and_report_alerts(data, only_once=True)
5039         tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
5040         if tabs:
5041             return self._extract_from_tabs(item_id, ytcfg, data, tabs)
5042
5043         playlist = traverse_obj(
5044             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
5045         if playlist:
5046             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
5047
5048         video_id = traverse_obj(
5049             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
5050         if video_id:
5051             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
5052                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
5053             return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
5054                                    ie=YoutubeIE.ie_key(), video_id=video_id)
5055
5056         raise ExtractorError('Unable to recognize tab page')
5057
5058
5059 class YoutubePlaylistIE(InfoExtractor):
5060     IE_DESC = 'YouTube playlists'
5061     _VALID_URL = r'''(?x)(?:
5062                         (?:https?://)?
5063                         (?:\w+\.)?
5064                         (?:
5065                             (?:
5066                                 youtube(?:kids)?\.com|
5067                                 %(invidious)s
5068                             )
5069                             /.*?\?.*?\blist=
5070                         )?
5071                         (?P<id>%(playlist_id)s)
5072                      )''' % {
5073         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
5074         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5075     }
5076     IE_NAME = 'youtube:playlist'
5077     _TESTS = [{
5078         'note': 'issue #673',
5079         'url': 'PLBB231211A4F62143',
5080         'info_dict': {
5081             'title': '[OLD]Team Fortress 2 (Class-based LP)',
5082             'id': 'PLBB231211A4F62143',
5083             'uploader': 'Wickman',
5084             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5085             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
5086             'view_count': int,
5087             'uploader_url': 'https://www.youtube.com/user/Wickydoo',
5088             'modified_date': r're:\d{8}',
5089             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
5090             'channel': 'Wickman',
5091             'tags': [],
5092             'channel_url': 'https://www.youtube.com/user/Wickydoo',
5093         },
5094         'playlist_mincount': 29,
5095     }, {
5096         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5097         'info_dict': {
5098             'title': 'YDL_safe_search',
5099             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
5100         },
5101         'playlist_count': 2,
5102         'skip': 'This playlist is private',
5103     }, {
5104         'note': 'embedded',
5105         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5106         'playlist_count': 4,
5107         'info_dict': {
5108             'title': 'JODA15',
5109             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
5110             'uploader': 'milan',
5111             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5112             'description': '',
5113             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5114             'tags': [],
5115             'modified_date': '20140919',
5116             'view_count': int,
5117             'channel': 'milan',
5118             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
5119             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
5120         },
5121         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5122     }, {
5123         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5124         'playlist_mincount': 654,
5125         'info_dict': {
5126             'title': '2018 Chinese New Singles (11/6 updated)',
5127             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
5128             'uploader': 'LBK',
5129             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5130             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
5131             'channel': 'LBK',
5132             'view_count': int,
5133             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
5134             'tags': [],
5135             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
5136             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
5137             'modified_date': r're:\d{8}',
5138         },
5139         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5140     }, {
5141         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
5142         'only_matching': True,
5143     }, {
5144         # music album playlist
5145         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
5146         'only_matching': True,
5147     }]
5148
5149     @classmethod
5150     def suitable(cls, url):
5151         if YoutubeTabIE.suitable(url):
5152             return False
5153         from ..utils import parse_qs
5154         qs = parse_qs(url)
5155         if qs.get('v', [None])[0]:
5156             return False
5157         return super(YoutubePlaylistIE, cls).suitable(url)
5158
5159     def _real_extract(self, url):
5160         playlist_id = self._match_id(url)
5161         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
5162         url = update_url_query(
5163             'https://www.youtube.com/playlist',
5164             parse_qs(url) or {'list': playlist_id})
5165         if is_music_url:
5166             url = smuggle_url(url, {'is_music_url': True})
5167         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5168
5169
5170 class YoutubeYtBeIE(InfoExtractor):
5171     IE_DESC = 'youtu.be'
5172     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
5173     _TESTS = [{
5174         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
5175         'info_dict': {
5176             'id': 'yeWKywCrFtk',
5177             'ext': 'mp4',
5178             'title': 'Small Scale Baler and Braiding Rugs',
5179             'uploader': 'Backus-Page House Museum',
5180             'uploader_id': 'backuspagemuseum',
5181             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
5182             'upload_date': '20161008',
5183             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
5184             'categories': ['Nonprofits & Activism'],
5185             'tags': list,
5186             'like_count': int,
5187             'age_limit': 0,
5188             'playable_in_embed': True,
5189             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
5190             'channel': 'Backus-Page House Museum',
5191             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
5192             'live_status': 'not_live',
5193             'view_count': int,
5194             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
5195             'availability': 'public',
5196             'duration': 59,
5197         },
5198         'params': {
5199             'noplaylist': True,
5200             'skip_download': True,
5201         },
5202     }, {
5203         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
5204         'only_matching': True,
5205     }]
5206
5207     def _real_extract(self, url):
5208         mobj = self._match_valid_url(url)
5209         video_id = mobj.group('id')
5210         playlist_id = mobj.group('playlist_id')
5211         return self.url_result(
5212             update_url_query('https://www.youtube.com/watch', {
5213                 'v': video_id,
5214                 'list': playlist_id,
5215                 'feature': 'youtu.be',
5216             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
5217
5218
5219 class YoutubeLivestreamEmbedIE(InfoExtractor):
5220     IE_DESC = 'YouTube livestream embeds'
5221     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
5222     _TESTS = [{
5223         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
5224         'only_matching': True,
5225     }]
5226
5227     def _real_extract(self, url):
5228         channel_id = self._match_id(url)
5229         return self.url_result(
5230             f'https://www.youtube.com/channel/{channel_id}/live',
5231             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
5232
5233
5234 class YoutubeYtUserIE(InfoExtractor):
5235     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
5236     IE_NAME = 'youtube:user'
5237     _VALID_URL = r'ytuser:(?P<id>.+)'
5238     _TESTS = [{
5239         'url': 'ytuser:phihag',
5240         'only_matching': True,
5241     }]
5242
5243     def _real_extract(self, url):
5244         user_id = self._match_id(url)
5245         return self.url_result(
5246             'https://www.youtube.com/user/%s/videos' % user_id,
5247             ie=YoutubeTabIE.ie_key(), video_id=user_id)
5248
5249
5250 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
5251     IE_NAME = 'youtube:favorites'
5252     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
5253     _VALID_URL = r':ytfav(?:ou?rite)?s?'
5254     _LOGIN_REQUIRED = True
5255     _TESTS = [{
5256         'url': ':ytfav',
5257         'only_matching': True,
5258     }, {
5259         'url': ':ytfavorites',
5260         'only_matching': True,
5261     }]
5262
5263     def _real_extract(self, url):
5264         return self.url_result(
5265             'https://www.youtube.com/playlist?list=LL',
5266             ie=YoutubeTabIE.ie_key())
5267
5268
5269 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5270     IE_DESC = 'YouTube search'
5271     IE_NAME = 'youtube:search'
5272     _SEARCH_KEY = 'ytsearch'
5273     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
5274     _TESTS = []
5275
5276
5277 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
5278     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
5279     _SEARCH_KEY = 'ytsearchdate'
5280     IE_DESC = 'YouTube search, newest videos first'
5281     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
5282
5283
5284 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
5285     IE_DESC = 'YouTube search URLs with sorting and filter support'
5286     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
5287     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?:[^&]+)(?:[&]|$)'
5288     _TESTS = [{
5289         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
5290         'playlist_mincount': 5,
5291         'info_dict': {
5292             'id': 'youtube-dl test video',
5293             'title': 'youtube-dl test video',
5294         }
5295     }, {
5296         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
5297         'playlist_mincount': 5,
5298         'info_dict': {
5299             'id': 'python',
5300             'title': 'python',
5301         }
5302
5303     }, {
5304         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
5305         'only_matching': True,
5306     }]
5307
5308     def _real_extract(self, url):
5309         qs = parse_qs(url)
5310         query = (qs.get('search_query') or qs.get('q'))[0]
5311         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
5312
5313
5314 class YoutubeFeedsInfoExtractor(YoutubeTabIE):
5315     """
5316     Base class for feed extractors
5317     Subclasses must define the _FEED_NAME property.
5318     """
5319     _LOGIN_REQUIRED = True
5320     _TESTS = []
5321
5322     @property
5323     def IE_NAME(self):
5324         return 'youtube:%s' % self._FEED_NAME
5325
5326     def _real_extract(self, url):
5327         return self.url_result(
5328             'https://www.youtube.com/feed/%s' % self._FEED_NAME,
5329             ie=YoutubeTabIE.ie_key())
5330
5331
5332 class YoutubeWatchLaterIE(InfoExtractor):
5333     IE_NAME = 'youtube:watchlater'
5334     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
5335     _VALID_URL = r':ytwatchlater'
5336     _TESTS = [{
5337         'url': ':ytwatchlater',
5338         'only_matching': True,
5339     }]
5340
5341     def _real_extract(self, url):
5342         return self.url_result(
5343             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
5344
5345
5346 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
5347     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
5348     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
5349     _FEED_NAME = 'recommended'
5350     _LOGIN_REQUIRED = False
5351     _TESTS = [{
5352         'url': ':ytrec',
5353         'only_matching': True,
5354     }, {
5355         'url': ':ytrecommended',
5356         'only_matching': True,
5357     }, {
5358         'url': 'https://youtube.com',
5359         'only_matching': True,
5360     }]
5361
5362
5363 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
5364     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
5365     _VALID_URL = r':ytsub(?:scription)?s?'
5366     _FEED_NAME = 'subscriptions'
5367     _TESTS = [{
5368         'url': ':ytsubs',
5369         'only_matching': True,
5370     }, {
5371         'url': ':ytsubscriptions',
5372         'only_matching': True,
5373     }]
5374
5375
5376 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
5377     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
5378     _VALID_URL = r':ythis(?:tory)?'
5379     _FEED_NAME = 'history'
5380     _TESTS = [{
5381         'url': ':ythistory',
5382         'only_matching': True,
5383     }]
5384
5385
5386 class YoutubeTruncatedURLIE(InfoExtractor):
5387     IE_NAME = 'youtube:truncated_url'
5388     IE_DESC = False  # Do not list
5389     _VALID_URL = r'''(?x)
5390         (?:https?://)?
5391         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
5392         (?:watch\?(?:
5393             feature=[a-z_]+|
5394             annotation_id=annotation_[^&]+|
5395             x-yt-cl=[0-9]+|
5396             hl=[^&]*|
5397             t=[0-9]+
5398         )?
5399         |
5400             attribution_link\?a=[^&]+
5401         )
5402         $
5403     '''
5404
5405     _TESTS = [{
5406         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
5407         'only_matching': True,
5408     }, {
5409         'url': 'https://www.youtube.com/watch?',
5410         'only_matching': True,
5411     }, {
5412         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
5413         'only_matching': True,
5414     }, {
5415         'url': 'https://www.youtube.com/watch?feature=foo',
5416         'only_matching': True,
5417     }, {
5418         'url': 'https://www.youtube.com/watch?hl=en-GB',
5419         'only_matching': True,
5420     }, {
5421         'url': 'https://www.youtube.com/watch?t=2372',
5422         'only_matching': True,
5423     }]
5424
5425     def _real_extract(self, url):
5426         raise ExtractorError(
5427             'Did you forget to quote the URL? Remember that & is a meta '
5428             'character in most shells, so you want to put the URL in quotes, '
5429             'like  youtube-dl '
5430             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
5431             ' or simply  youtube-dl BaW_jenozKc  .',
5432             expected=True)
5433
5434
5435 class YoutubeClipIE(InfoExtractor):
5436     IE_NAME = 'youtube:clip'
5437     IE_DESC = False  # Do not list
5438     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/'
5439
5440     def _real_extract(self, url):
5441         self.report_warning('YouTube clips are not currently supported. The entire video will be downloaded instead')
5442         return self.url_result(url, 'Generic')
5443
5444
5445 class YoutubeTruncatedIDIE(InfoExtractor):
5446     IE_NAME = 'youtube:truncated_id'
5447     IE_DESC = False  # Do not list
5448     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
5449
5450     _TESTS = [{
5451         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
5452         'only_matching': True,
5453     }]
5454
5455     def _real_extract(self, url):
5456         video_id = self._match_id(url)
5457         raise ExtractorError(
5458             'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
5459             expected=True)