yt_dlp/extractor/youtube.py

   1 # coding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import base64
   6 import calendar
   7 import copy
   8 import datetime
   9 import hashlib
  10 import itertools
  11 import json
  12 import os.path
  13 import random
  14 import re
  15 import time
  16 import traceback
  17
  18 from .common import InfoExtractor, SearchInfoExtractor
  19 from ..compat import (
  20     compat_chr,
  21     compat_HTTPError,
  22     compat_parse_qs,
  23     compat_str,
  24     compat_urllib_parse_unquote_plus,
  25     compat_urllib_parse_urlencode,
  26     compat_urllib_parse_urlparse,
  27     compat_urlparse,
  28 )
  29 from ..jsinterp import JSInterpreter
  30 from ..utils import (
  31     bytes_to_intlist,
  32     clean_html,
  33     datetime_from_str,
  34     dict_get,
  35     error_to_compat_str,
  36     ExtractorError,
  37     float_or_none,
  38     format_field,
  39     int_or_none,
  40     intlist_to_bytes,
  41     mimetype2ext,
  42     network_exceptions,
  43     orderedSet,
  44     parse_codecs,
  45     parse_count,
  46     parse_duration,
  47     parse_iso8601,
  48     qualities,
  49     remove_start,
  50     smuggle_url,
  51     str_or_none,
  52     str_to_int,
  53     traverse_obj,
  54     try_get,
  55     unescapeHTML,
  56     unified_strdate,
  57     unsmuggle_url,
  58     update_url_query,
  59     url_or_none,
  60     urlencode_postdata,
  61     urljoin,
  62     variadic,
  63 )
  64
  65
  66 def parse_qs(url):
  67     return compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
  68
  69
  70 # any clients starting with _ cannot be explicity requested by the user
  71 INNERTUBE_CLIENTS = {
  72     'web': {
  73         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  74         'INNERTUBE_CONTEXT': {
  75             'client': {
  76                 'clientName': 'WEB',
  77                 'clientVersion': '2.20210622.10.00',
  78             }
  79         },
  80         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  81     },
  82     'web_embedded': {
  83         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  84         'INNERTUBE_CONTEXT': {
  85             'client': {
  86                 'clientName': 'WEB_EMBEDDED_PLAYER',
  87                 'clientVersion': '1.20210620.0.1',
  88             },
  89         },
  90         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  91     },
  92     'web_music': {
  93         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  94         'INNERTUBE_HOST': 'music.youtube.com',
  95         'INNERTUBE_CONTEXT': {
  96             'client': {
  97                 'clientName': 'WEB_REMIX',
  98                 'clientVersion': '1.20210621.00.00',
  99             }
 100         },
 101         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 102     },
 103     'web_creator': {
 104         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 105         'INNERTUBE_CONTEXT': {
 106             'client': {
 107                 'clientName': 'WEB_CREATOR',
 108                 'clientVersion': '1.20210621.00.00',
 109             }
 110         },
 111         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 112     },
 113     'android': {
 114         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 115         'INNERTUBE_CONTEXT': {
 116             'client': {
 117                 'clientName': 'ANDROID',
 118                 'clientVersion': '16.20',
 119             }
 120         },
 121         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 122     },
 123     'android_embedded': {
 124         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 125         'INNERTUBE_CONTEXT': {
 126             'client': {
 127                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 128                 'clientVersion': '16.20',
 129             },
 130         },
 131         'INNERTUBE_CONTEXT_CLIENT_NAME': 55
 132     },
 133     'android_music': {
 134         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
 135         'INNERTUBE_HOST': 'music.youtube.com',
 136         'INNERTUBE_CONTEXT': {
 137             'client': {
 138                 'clientName': 'ANDROID_MUSIC',
 139                 'clientVersion': '4.32',
 140             }
 141         },
 142         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 143     },
 144     'android_creator': {
 145         'INNERTUBE_CONTEXT': {
 146             'client': {
 147                 'clientName': 'ANDROID_CREATOR',
 148                 'clientVersion': '21.24.100',
 149             },
 150         },
 151         'INNERTUBE_CONTEXT_CLIENT_NAME': 14
 152     },
 153     # ios has HLS live streams
 154     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680
 155     'ios': {
 156         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 157         'INNERTUBE_CONTEXT': {
 158             'client': {
 159                 'clientName': 'IOS',
 160                 'clientVersion': '16.20',
 161             }
 162         },
 163         'INNERTUBE_CONTEXT_CLIENT_NAME': 5
 164     },
 165     'ios_embedded': {
 166         'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
 167         'INNERTUBE_CONTEXT': {
 168             'client': {
 169                 'clientName': 'IOS_MESSAGES_EXTENSION',
 170                 'clientVersion': '16.20',
 171             },
 172         },
 173         'INNERTUBE_CONTEXT_CLIENT_NAME': 66
 174     },
 175     'ios_music': {
 176         'INNERTUBE_API_KEY': 'AIzaSyDK3iBpDP9nHVTk2qL73FLJICfOC3c51Og',
 177         'INNERTUBE_HOST': 'music.youtube.com',
 178         'INNERTUBE_CONTEXT': {
 179             'client': {
 180                 'clientName': 'IOS_MUSIC',
 181                 'clientVersion': '4.32',
 182             },
 183         },
 184         'INNERTUBE_CONTEXT_CLIENT_NAME': 26
 185     },
 186     'ios_creator': {
 187         'INNERTUBE_CONTEXT': {
 188             'client': {
 189                 'clientName': 'IOS_CREATOR',
 190                 'clientVersion': '21.24.100',
 191             },
 192         },
 193         'INNERTUBE_CONTEXT_CLIENT_NAME': 15
 194     },
 195     # mweb has 'ultralow' formats
 196     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 197     'mweb': {
 198         'INNERTUBE_API_KEY': 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8',
 199         'INNERTUBE_CONTEXT': {
 200             'client': {
 201                 'clientName': 'MWEB',
 202                 'clientVersion': '2.20210721.07.00',
 203             }
 204         },
 205         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 206     },
 207 }
 208
 209
 210 def build_innertube_clients():
 211     third_party = {
 212         'embedUrl': 'https://google.com',  # Can be any valid URL
 213     }
 214     base_clients = ('android', 'web', 'ios', 'mweb')
 215     priority = qualities(base_clients[::-1])
 216
 217     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 218         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 219         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 220         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 221         ytcfg['priority'] = 10 * priority(client.split('_', 1)[0])
 222
 223         if client in base_clients:
 224             INNERTUBE_CLIENTS[f'{client}_agegate'] = agegate_ytcfg = copy.deepcopy(ytcfg)
 225             agegate_ytcfg['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 226             agegate_ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = third_party
 227             agegate_ytcfg['priority'] -= 1
 228         elif client.endswith('_embedded'):
 229             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = third_party
 230             ytcfg['priority'] -= 2
 231         else:
 232             ytcfg['priority'] -= 3
 233
 234
 235 build_innertube_clients()
 236
 237
 238 class YoutubeBaseInfoExtractor(InfoExtractor):
 239     """Provide base functions for Youtube extractors"""
 240
 241     _RESERVED_NAMES = (
 242         r'channel|c|user|playlist|watch|w|v|embed|e|watch_popup|'
 243         r'shorts|movies|results|shared|hashtag|trending|feed|feeds|'
 244         r'browse|oembed|get_video_info|iframe_api|s/player|'
 245         r'storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout')
 246
 247     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 248
 249     _NETRC_MACHINE = 'youtube'
 250
 251     # If True it will raise an error if no login info is provided
 252     _LOGIN_REQUIRED = False
 253
 254     r'''  # Unused since login is broken
 255     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
 256     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
 257
 258     _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
 259     _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
 260     _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
 261     '''
 262
 263     def _login(self):
 264         """
 265         Attempt to log in to YouTube.
 266         True is returned if successful or skipped.
 267         False is returned if login failed.
 268
 269         If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
 270         """
 271
 272         def warn(message):
 273             self.report_warning(message)
 274
 275         # username+password login is broken
 276         if (self._LOGIN_REQUIRED
 277                 and self.get_param('cookiefile') is None
 278                 and self.get_param('cookiesfrombrowser') is None):
 279             self.raise_login_required(
 280                 'Login details are needed to download this content', method='cookies')
 281         username, password = self._get_login_info()
 282         if username:
 283             warn('Logging in using username and password is broken. %s' % self._LOGIN_HINTS['cookies'])
 284         return
 285
 286         # Everything below this is broken!
 287         r'''
 288         # No authentication to be performed
 289         if username is None:
 290             if self._LOGIN_REQUIRED and self.get_param('cookiefile') is None:
 291                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
 292             # if self.get_param('cookiefile'):  # TODO remove 'and False' later - too many people using outdated cookies and open issues, remind them.
 293             #     self.to_screen('[Cookies] Reminder - Make sure to always use up to date cookies!')
 294             return True
 295
 296         login_page = self._download_webpage(
 297             self._LOGIN_URL, None,
 298             note='Downloading login page',
 299             errnote='unable to fetch login page', fatal=False)
 300         if login_page is False:
 301             return
 302
 303         login_form = self._hidden_inputs(login_page)
 304
 305         def req(url, f_req, note, errnote):
 306             data = login_form.copy()
 307             data.update({
 308                 'pstMsg': 1,
 309                 'checkConnection': 'youtube',
 310                 'checkedDomains': 'youtube',
 311                 'hl': 'en',
 312                 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
 313                 'f.req': json.dumps(f_req),
 314                 'flowName': 'GlifWebSignIn',
 315                 'flowEntry': 'ServiceLogin',
 316                 # TODO: reverse actual botguard identifier generation algo
 317                 'bgRequest': '["identifier",""]',
 318             })
 319             return self._download_json(
 320                 url, None, note=note, errnote=errnote,
 321                 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
 322                 fatal=False,
 323                 data=urlencode_postdata(data), headers={
 324                     'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
 325                     'Google-Accounts-XSRF': 1,
 326                 })
 327
 328         lookup_req = [
 329             username,
 330             None, [], None, 'US', None, None, 2, False, True,
 331             [
 332                 None, None,
 333                 [2, 1, None, 1,
 334                  'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
 335                  None, [], 4],
 336                 1, [None, None, []], None, None, None, True
 337             ],
 338             username,
 339         ]
 340
 341         lookup_results = req(
 342             self._LOOKUP_URL, lookup_req,
 343             'Looking up account info', 'Unable to look up account info')
 344
 345         if lookup_results is False:
 346             return False
 347
 348         user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
 349         if not user_hash:
 350             warn('Unable to extract user hash')
 351             return False
 352
 353         challenge_req = [
 354             user_hash,
 355             None, 1, None, [1, None, None, None, [password, None, True]],
 356             [
 357                 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
 358                 1, [None, None, []], None, None, None, True
 359             ]]
 360
 361         challenge_results = req(
 362             self._CHALLENGE_URL, challenge_req,
 363             'Logging in', 'Unable to log in')
 364
 365         if challenge_results is False:
 366             return
 367
 368         login_res = try_get(challenge_results, lambda x: x[0][5], list)
 369         if login_res:
 370             login_msg = try_get(login_res, lambda x: x[5], compat_str)
 371             warn(
 372                 'Unable to login: %s' % 'Invalid password'
 373                 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
 374             return False
 375
 376         res = try_get(challenge_results, lambda x: x[0][-1], list)
 377         if not res:
 378             warn('Unable to extract result entry')
 379             return False
 380
 381         login_challenge = try_get(res, lambda x: x[0][0], list)
 382         if login_challenge:
 383             challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
 384             if challenge_str == 'TWO_STEP_VERIFICATION':
 385                 # SEND_SUCCESS - TFA code has been successfully sent to phone
 386                 # QUOTA_EXCEEDED - reached the limit of TFA codes
 387                 status = try_get(login_challenge, lambda x: x[5], compat_str)
 388                 if status == 'QUOTA_EXCEEDED':
 389                     warn('Exceeded the limit of TFA codes, try later')
 390                     return False
 391
 392                 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
 393                 if not tl:
 394                     warn('Unable to extract TL')
 395                     return False
 396
 397                 tfa_code = self._get_tfa_info('2-step verification code')
 398
 399                 if not tfa_code:
 400                     warn(
 401                         'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
 402                         '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
 403                     return False
 404
 405                 tfa_code = remove_start(tfa_code, 'G-')
 406
 407                 tfa_req = [
 408                     user_hash, None, 2, None,
 409                     [
 410                         9, None, None, None, None, None, None, None,
 411                         [None, tfa_code, True, 2]
 412                     ]]
 413
 414                 tfa_results = req(
 415                     self._TFA_URL.format(tl), tfa_req,
 416                     'Submitting TFA code', 'Unable to submit TFA code')
 417
 418                 if tfa_results is False:
 419                     return False
 420
 421                 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
 422                 if tfa_res:
 423                     tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
 424                     warn(
 425                         'Unable to finish TFA: %s' % 'Invalid TFA code'
 426                         if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
 427                     return False
 428
 429                 check_cookie_url = try_get(
 430                     tfa_results, lambda x: x[0][-1][2], compat_str)
 431             else:
 432                 CHALLENGES = {
 433                     'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
 434                     'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
 435                     'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
 436                 }
 437                 challenge = CHALLENGES.get(
 438                     challenge_str,
 439                     '%s returned error %s.' % (self.IE_NAME, challenge_str))
 440                 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
 441                 return False
 442         else:
 443             check_cookie_url = try_get(res, lambda x: x[2], compat_str)
 444
 445         if not check_cookie_url:
 446             warn('Unable to extract CheckCookie URL')
 447             return False
 448
 449         check_cookie_results = self._download_webpage(
 450             check_cookie_url, None, 'Checking cookie', fatal=False)
 451
 452         if check_cookie_results is False:
 453             return False
 454
 455         if 'https://myaccount.google.com/' not in check_cookie_results:
 456             warn('Unable to log in')
 457             return False
 458
 459         return True
 460         '''
 461
 462     def _initialize_consent(self):
 463         cookies = self._get_cookies('https://www.youtube.com/')
 464         if cookies.get('__Secure-3PSID'):
 465             return
 466         consent_id = None
 467         consent = cookies.get('CONSENT')
 468         if consent:
 469             if 'YES' in consent.value:
 470                 return
 471             consent_id = self._search_regex(
 472                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 473         if not consent_id:
 474             consent_id = random.randint(100, 999)
 475         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 476
 477     def _real_initialize(self):
 478         self._initialize_consent()
 479         if self._downloader is None:
 480             return
 481         if not self._login():
 482             return
 483
 484     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 485     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 486     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 487
 488     def _get_default_ytcfg(self, client='web'):
 489         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 490
 491     def _get_innertube_host(self, client='web'):
 492         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 493
 494     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 495         # try_get but with fallback to default ytcfg client values when present
 496         _func = lambda y: try_get(y, getter, expected_type)
 497         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 498
 499     def _extract_client_name(self, ytcfg, default_client='web'):
 500         return self._ytcfg_get_safe(
 501             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 502                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), compat_str, default_client)
 503
 504     @staticmethod
 505     def _extract_session_index(*data):
 506         for ytcfg in data:
 507             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 508             if session_index is not None:
 509                 return session_index
 510
 511     def _extract_client_version(self, ytcfg, default_client='web'):
 512         return self._ytcfg_get_safe(
 513             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 514                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), compat_str, default_client)
 515
 516     def _extract_api_key(self, ytcfg=None, default_client='web'):
 517         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
 518
 519     def _extract_context(self, ytcfg=None, default_client='web'):
 520         _get_context = lambda y: try_get(y, lambda x: x['INNERTUBE_CONTEXT'], dict)
 521         context = _get_context(ytcfg)
 522         if context:
 523             return context
 524
 525         context = _get_context(self._get_default_ytcfg(default_client))
 526         if not ytcfg:
 527             return context
 528
 529         # Recreate the client context (required)
 530         context['client'].update({
 531             'clientVersion': self._extract_client_version(ytcfg, default_client),
 532             'clientName': self._extract_client_name(ytcfg, default_client),
 533         })
 534         visitor_data = try_get(ytcfg, lambda x: x['VISITOR_DATA'], compat_str)
 535         if visitor_data:
 536             context['client']['visitorData'] = visitor_data
 537         return context
 538
 539     _SAPISID = None
 540
 541     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 542         time_now = round(time.time())
 543         if self._SAPISID is None:
 544             yt_cookies = self._get_cookies('https://www.youtube.com')
 545             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 546             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 547             sapisid_cookie = dict_get(
 548                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 549             if sapisid_cookie and sapisid_cookie.value:
 550                 self._SAPISID = sapisid_cookie.value
 551                 self.write_debug('Extracted SAPISID cookie')
 552                 # SAPISID cookie is required if not already present
 553                 if not yt_cookies.get('SAPISID'):
 554                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 555                     self._set_cookie(
 556                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 557             else:
 558                 self._SAPISID = False
 559         if not self._SAPISID:
 560             return None
 561         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 562         sapisidhash = hashlib.sha1(
 563             f'{time_now} {self._SAPISID} {origin}'.encode('utf-8')).hexdigest()
 564         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 565
 566     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 567                   note='Downloading API JSON', errnote='Unable to download API page',
 568                   context=None, api_key=None, api_hostname=None, default_client='web'):
 569
 570         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 571         data.update(query)
 572         real_headers = self.generate_api_headers(default_client=default_client)
 573         real_headers.update({'content-type': 'application/json'})
 574         if headers:
 575             real_headers.update(headers)
 576         return self._download_json(
 577             'https://%s/youtubei/v1/%s' % (api_hostname or self._get_innertube_host(default_client), ep),
 578             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 579             data=json.dumps(data).encode('utf8'), headers=real_headers,
 580             query={'key': api_key or self._extract_api_key()})
 581
 582     def extract_yt_initial_data(self, video_id, webpage):
 583         return self._parse_json(
 584             self._search_regex(
 585                 (r'%s\s*%s' % (self._YT_INITIAL_DATA_RE, self._YT_INITIAL_BOUNDARY_RE),
 586                  self._YT_INITIAL_DATA_RE), webpage, 'yt initial data'),
 587             video_id)
 588
 589     def _extract_identity_token(self, webpage, item_id):
 590         if not webpage:
 591             return None
 592         ytcfg = self.extract_ytcfg(item_id, webpage)
 593         if ytcfg:
 594             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
 595             if token:
 596                 return token
 597         return self._search_regex(
 598             r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 599             'identity token', default=None)
 600
 601     @staticmethod
 602     def _extract_account_syncid(*args):
 603         """
 604         Extract syncId required to download private playlists of secondary channels
 605         @params response and/or ytcfg
 606         """
 607         for data in args:
 608             # ytcfg includes channel_syncid if on secondary channel
 609             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], compat_str)
 610             if delegated_sid:
 611                 return delegated_sid
 612             sync_ids = (try_get(
 613                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 614                        lambda x: x['DATASYNC_ID']), compat_str) or '').split("||")
 615             if len(sync_ids) >= 2 and sync_ids[1]:
 616                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 617                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 618                 return sync_ids[0]
 619
 620     def extract_ytcfg(self, video_id, webpage):
 621         if not webpage:
 622             return {}
 623         return self._parse_json(
 624             self._search_regex(
 625                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 626                 default='{}'), video_id, fatal=False) or {}
 627
 628     def generate_api_headers(
 629             self, ytcfg=None, identity_token=None, account_syncid=None,
 630             visitor_data=None, api_hostname=None, default_client='web', session_index=None):
 631         origin = 'https://' + (api_hostname if api_hostname else self._get_innertube_host(default_client))
 632         headers = {
 633             'X-YouTube-Client-Name': compat_str(
 634                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 635             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 636             'Origin': origin
 637         }
 638         if not visitor_data and ytcfg:
 639             visitor_data = try_get(
 640                 self._extract_context(ytcfg, default_client), lambda x: x['client']['visitorData'], compat_str)
 641         if identity_token:
 642             headers['X-Youtube-Identity-Token'] = identity_token
 643         if account_syncid:
 644             headers['X-Goog-PageId'] = account_syncid
 645         if session_index is None and ytcfg:
 646             session_index = self._extract_session_index(ytcfg)
 647         if account_syncid or session_index is not None:
 648             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 649         if visitor_data:
 650             headers['X-Goog-Visitor-Id'] = visitor_data
 651         auth = self._generate_sapisidhash_header(origin)
 652         if auth is not None:
 653             headers['Authorization'] = auth
 654             headers['X-Origin'] = origin
 655         return headers
 656
 657     @staticmethod
 658     def _build_api_continuation_query(continuation, ctp=None):
 659         query = {
 660             'continuation': continuation
 661         }
 662         # TODO: Inconsistency with clickTrackingParams.
 663         # Currently we have a fixed ctp contained within context (from ytcfg)
 664         # and a ctp in root query for continuation.
 665         if ctp:
 666             query['clickTracking'] = {'clickTrackingParams': ctp}
 667         return query
 668
 669     @classmethod
 670     def _extract_next_continuation_data(cls, renderer):
 671         next_continuation = try_get(
 672             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 673                        lambda x: x['continuation']['reloadContinuationData']), dict)
 674         if not next_continuation:
 675             return
 676         continuation = next_continuation.get('continuation')
 677         if not continuation:
 678             return
 679         ctp = next_continuation.get('clickTrackingParams')
 680         return cls._build_api_continuation_query(continuation, ctp)
 681
 682     @classmethod
 683     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 684         if isinstance(continuation_ep, dict):
 685             continuation = try_get(
 686                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
 687             if not continuation:
 688                 return
 689             ctp = continuation_ep.get('clickTrackingParams')
 690             return cls._build_api_continuation_query(continuation, ctp)
 691
 692     @classmethod
 693     def _extract_continuation(cls, renderer):
 694         next_continuation = cls._extract_next_continuation_data(renderer)
 695         if next_continuation:
 696             return next_continuation
 697
 698         contents = []
 699         for key in ('contents', 'items'):
 700             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
 701
 702         for content in contents:
 703             if not isinstance(content, dict):
 704                 continue
 705             continuation_ep = try_get(
 706                 content, (lambda x: x['continuationItemRenderer']['continuationEndpoint'],
 707                           lambda x: x['continuationItemRenderer']['button']['buttonRenderer']['command']),
 708                 dict)
 709             continuation = cls._extract_continuation_ep_data(continuation_ep)
 710             if continuation:
 711                 return continuation
 712
 713     @classmethod
 714     def _extract_alerts(cls, data):
 715         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 716             if not isinstance(alert_dict, dict):
 717                 continue
 718             for alert in alert_dict.values():
 719                 alert_type = alert.get('type')
 720                 if not alert_type:
 721                     continue
 722                 message = cls._get_text(alert, 'text')
 723                 if message:
 724                     yield alert_type, message
 725
 726     def _report_alerts(self, alerts, expected=True):
 727         errors = []
 728         warnings = []
 729         for alert_type, alert_message in alerts:
 730             if alert_type.lower() == 'error':
 731                 errors.append([alert_type, alert_message])
 732             else:
 733                 warnings.append([alert_type, alert_message])
 734
 735         for alert_type, alert_message in (warnings + errors[:-1]):
 736             self.report_warning('YouTube said: %s - %s' % (alert_type, alert_message))
 737         if errors:
 738             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 739
 740     def _extract_and_report_alerts(self, data, *args, **kwargs):
 741         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 742
 743     def _extract_badges(self, renderer: dict):
 744         badges = set()
 745         for badge in try_get(renderer, lambda x: x['badges'], list) or []:
 746             label = try_get(badge, lambda x: x['metadataBadgeRenderer']['label'], compat_str)
 747             if label:
 748                 badges.add(label.lower())
 749         return badges
 750
 751     @staticmethod
 752     def _get_text(data, *path_list, max_runs=None):
 753         for path in path_list or [None]:
 754             if path is None:
 755                 obj = [data]
 756             else:
 757                 obj = traverse_obj(data, path, default=[])
 758                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 759                     obj = [obj]
 760             for item in obj:
 761                 text = try_get(item, lambda x: x['simpleText'], compat_str)
 762                 if text:
 763                     return text
 764                 runs = try_get(item, lambda x: x['runs'], list) or []
 765                 if not runs and isinstance(item, list):
 766                     runs = item
 767
 768                 runs = runs[:min(len(runs), max_runs or len(runs))]
 769                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str, default=[]))
 770                 if text:
 771                     return text
 772
 773     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 774                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 775                           default_client='web'):
 776         response = None
 777         last_error = None
 778         count = -1
 779         retries = self.get_param('extractor_retries', 3)
 780         if check_get_keys is None:
 781             check_get_keys = []
 782         while count < retries:
 783             count += 1
 784             if last_error:
 785                 self.report_warning('%s. Retrying ...' % last_error)
 786             try:
 787                 response = self._call_api(
 788                     ep=ep, fatal=True, headers=headers,
 789                     video_id=item_id, query=query,
 790                     context=self._extract_context(ytcfg, default_client),
 791                     api_key=self._extract_api_key(ytcfg, default_client),
 792                     api_hostname=api_hostname, default_client=default_client,
 793                     note='%s%s' % (note, ' (retry #%d)' % count if count else ''))
 794             except ExtractorError as e:
 795                 if isinstance(e.cause, network_exceptions):
 796                     # Downloading page may result in intermittent 5xx HTTP error
 797                     # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 798                     # We also want to catch all other network exceptions since errors in later pages can be troublesome
 799                     # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 800                     if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (403, 429):
 801                         last_error = error_to_compat_str(e.cause or e)
 802                         if count < retries:
 803                             continue
 804                 if fatal:
 805                     raise
 806                 else:
 807                     self.report_warning(error_to_compat_str(e))
 808                     return
 809
 810             else:
 811                 # Youtube may send alerts if there was an issue with the continuation page
 812                 try:
 813                     self._extract_and_report_alerts(response, expected=False)
 814                 except ExtractorError as e:
 815                     if fatal:
 816                         raise
 817                     self.report_warning(error_to_compat_str(e))
 818                     return
 819                 if not check_get_keys or dict_get(response, check_get_keys):
 820                     break
 821                 # Youtube sometimes sends incomplete data
 822                 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 823                 last_error = 'Incomplete data received'
 824                 if count >= retries:
 825                     if fatal:
 826                         raise ExtractorError(last_error)
 827                     else:
 828                         self.report_warning(last_error)
 829                         return
 830         return response
 831
 832     @staticmethod
 833     def is_music_url(url):
 834         return re.match(r'https?://music\.youtube\.com/', url) is not None
 835
 836     def _extract_video(self, renderer):
 837         video_id = renderer.get('videoId')
 838         title = self._get_text(renderer, 'title')
 839         description = self._get_text(renderer, 'descriptionSnippet')
 840         duration = parse_duration(self._get_text(
 841             renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 842         view_count_text = self._get_text(renderer, 'viewCountText') or ''
 843         view_count = str_to_int(self._search_regex(
 844             r'^([\d,]+)', re.sub(r'\s', '', view_count_text),
 845             'view count', default=None))
 846
 847         uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
 848
 849         return {
 850             '_type': 'url',
 851             'ie_key': YoutubeIE.ie_key(),
 852             'id': video_id,
 853             'url': video_id,
 854             'title': title,
 855             'description': description,
 856             'duration': duration,
 857             'view_count': view_count,
 858             'uploader': uploader,
 859         }
 860
 861
 862 class YoutubeIE(YoutubeBaseInfoExtractor):
 863     IE_DESC = 'YouTube.com'
 864     _INVIDIOUS_SITES = (
 865         # invidious-redirect websites
 866         r'(?:www\.)?redirect\.invidious\.io',
 867         r'(?:(?:www|dev)\.)?invidio\.us',
 868         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
 869         r'(?:www\.)?invidious\.pussthecat\.org',
 870         r'(?:www\.)?invidious\.zee\.li',
 871         r'(?:www\.)?invidious\.ethibox\.fr',
 872         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 873         # youtube-dl invidious instances list
 874         r'(?:(?:www|no)\.)?invidiou\.sh',
 875         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 876         r'(?:www\.)?invidious\.kabi\.tk',
 877         r'(?:www\.)?invidious\.mastodon\.host',
 878         r'(?:www\.)?invidious\.zapashcanon\.fr',
 879         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 880         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 881         r'(?:www\.)?invidious\.himiko\.cloud',
 882         r'(?:www\.)?invidious\.reallyancient\.tech',
 883         r'(?:www\.)?invidious\.tube',
 884         r'(?:www\.)?invidiou\.site',
 885         r'(?:www\.)?invidious\.site',
 886         r'(?:www\.)?invidious\.xyz',
 887         r'(?:www\.)?invidious\.nixnet\.xyz',
 888         r'(?:www\.)?invidious\.048596\.xyz',
 889         r'(?:www\.)?invidious\.drycat\.fr',
 890         r'(?:www\.)?inv\.skyn3t\.in',
 891         r'(?:www\.)?tube\.poal\.co',
 892         r'(?:www\.)?tube\.connect\.cafe',
 893         r'(?:www\.)?vid\.wxzm\.sx',
 894         r'(?:www\.)?vid\.mint\.lgbt',
 895         r'(?:www\.)?vid\.puffyan\.us',
 896         r'(?:www\.)?yewtu\.be',
 897         r'(?:www\.)?yt\.elukerio\.org',
 898         r'(?:www\.)?yt\.lelux\.fi',
 899         r'(?:www\.)?invidious\.ggc-project\.de',
 900         r'(?:www\.)?yt\.maisputain\.ovh',
 901         r'(?:www\.)?ytprivate\.com',
 902         r'(?:www\.)?invidious\.13ad\.de',
 903         r'(?:www\.)?invidious\.toot\.koeln',
 904         r'(?:www\.)?invidious\.fdn\.fr',
 905         r'(?:www\.)?watch\.nettohikari\.com',
 906         r'(?:www\.)?invidious\.namazso\.eu',
 907         r'(?:www\.)?invidious\.silkky\.cloud',
 908         r'(?:www\.)?invidious\.exonip\.de',
 909         r'(?:www\.)?invidious\.riverside\.rocks',
 910         r'(?:www\.)?invidious\.blamefran\.net',
 911         r'(?:www\.)?invidious\.moomoo\.de',
 912         r'(?:www\.)?ytb\.trom\.tf',
 913         r'(?:www\.)?yt\.cyberhost\.uk',
 914         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 915         r'(?:www\.)?qklhadlycap4cnod\.onion',
 916         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 917         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 918         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 919         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 920         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 921         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 922         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 923         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 924         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 925         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 926     )
 927     _VALID_URL = r"""(?x)^
 928                      (
 929                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 930                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 931                             (?:www\.)?deturl\.com/www\.youtube\.com|
 932                             (?:www\.)?pwnyoutube\.com|
 933                             (?:www\.)?hooktube\.com|
 934                             (?:www\.)?yourepeat\.com|
 935                             tube\.majestyc\.net|
 936                             %(invidious)s|
 937                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 938                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 939                          (?:                                                  # the various things that can precede the ID:
 940                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
 941                              |(?:                                             # or the v= param in all its forms
 942                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 943                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 944                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 945                                  v=
 946                              )
 947                          ))
 948                          |(?:
 949                             youtu\.be|                                        # just youtu.be/xxxx
 950                             vid\.plus|                                        # or vid.plus/xxxx
 951                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 952                             %(invidious)s
 953                          )/
 954                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 955                          )
 956                      )?                                                       # all until now is optional -> you can pass the naked ID
 957                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 958                      (?(1).+)?                                                # if we found the ID, everything can follow
 959                      (?:\#|$)""" % {
 960         'invidious': '|'.join(_INVIDIOUS_SITES),
 961     }
 962     _PLAYER_INFO_RE = (
 963         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 964         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 965         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 966     )
 967     _formats = {
 968         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 969         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 970         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 971         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 972         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 973         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 974         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 975         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 976         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 977         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 978         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 979         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 980         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 981         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 982         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 983         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 984         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 985         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 986
 987
 988         # 3D videos
 989         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 990         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 991         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 992         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 993         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 994         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 995         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 996
 997         # Apple HTTP Live Streaming
 998         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 999         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1000         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1001         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1002         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1003         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1004         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1005         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1006
1007         # DASH mp4 video
1008         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1009         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1010         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1011         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1012         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1013         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1014         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1015         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1016         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1017         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1018         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1019         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1020
1021         # Dash mp4 audio
1022         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1023         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1024         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1025         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1026         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1027         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1028         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1029
1030         # Dash webm
1031         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1032         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1033         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1034         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1035         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1036         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1037         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1038         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1039         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1040         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1041         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1042         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1043         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1044         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1045         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1046         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1047         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1048         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1049         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1050         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1051         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1052         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1053
1054         # Dash webm audio
1055         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1056         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1057
1058         # Dash webm audio with opus inside
1059         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1060         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1061         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1062
1063         # RTMP (unnamed)
1064         '_rtmp': {'protocol': 'rtmp'},
1065
1066         # av01 video only formats sometimes served with "unknown" codecs
1067         '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1068         '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1069         '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1070         '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1071     }
1072     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1073
1074     _GEO_BYPASS = False
1075
1076     IE_NAME = 'youtube'
1077     _TESTS = [
1078         {
1079             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1080             'info_dict': {
1081                 'id': 'BaW_jenozKc',
1082                 'ext': 'mp4',
1083                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1084                 'uploader': 'Philipp Hagemeister',
1085                 'uploader_id': 'phihag',
1086                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1087                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1088                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1089                 'upload_date': '20121002',
1090                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
1091                 'categories': ['Science & Technology'],
1092                 'tags': ['youtube-dl'],
1093                 'duration': 10,
1094                 'view_count': int,
1095                 'like_count': int,
1096                 'dislike_count': int,
1097                 'start_time': 1,
1098                 'end_time': 9,
1099             }
1100         },
1101         {
1102             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1103             'note': 'Embed-only video (#1746)',
1104             'info_dict': {
1105                 'id': 'yZIXLfi8CZQ',
1106                 'ext': 'mp4',
1107                 'upload_date': '20120608',
1108                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1109                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1110                 'uploader': 'SET India',
1111                 'uploader_id': 'setindia',
1112                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1113                 'age_limit': 18,
1114             },
1115             'skip': 'Private video',
1116         },
1117         {
1118             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1119             'note': 'Use the first video ID in the URL',
1120             'info_dict': {
1121                 'id': 'BaW_jenozKc',
1122                 'ext': 'mp4',
1123                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1124                 'uploader': 'Philipp Hagemeister',
1125                 'uploader_id': 'phihag',
1126                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1127                 'upload_date': '20121002',
1128                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
1129                 'categories': ['Science & Technology'],
1130                 'tags': ['youtube-dl'],
1131                 'duration': 10,
1132                 'view_count': int,
1133                 'like_count': int,
1134                 'dislike_count': int,
1135             },
1136             'params': {
1137                 'skip_download': True,
1138             },
1139         },
1140         {
1141             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1142             'note': '256k DASH audio (format 141) via DASH manifest',
1143             'info_dict': {
1144                 'id': 'a9LDPn-MO4I',
1145                 'ext': 'm4a',
1146                 'upload_date': '20121002',
1147                 'uploader_id': '8KVIDEO',
1148                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1149                 'description': '',
1150                 'uploader': '8KVIDEO',
1151                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1152             },
1153             'params': {
1154                 'youtube_include_dash_manifest': True,
1155                 'format': '141',
1156             },
1157             'skip': 'format 141 not served anymore',
1158         },
1159         # DASH manifest with encrypted signature
1160         {
1161             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1162             'info_dict': {
1163                 'id': 'IB3lcPjvWLA',
1164                 'ext': 'm4a',
1165                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1166                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1167                 'duration': 244,
1168                 'uploader': 'AfrojackVEVO',
1169                 'uploader_id': 'AfrojackVEVO',
1170                 'upload_date': '20131011',
1171                 'abr': 129.495,
1172             },
1173             'params': {
1174                 'youtube_include_dash_manifest': True,
1175                 'format': '141/bestaudio[ext=m4a]',
1176             },
1177         },
1178         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1179         {
1180             'note': 'Embed allowed age-gate video',
1181             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1182             'info_dict': {
1183                 'id': 'HtVdAasjOgU',
1184                 'ext': 'mp4',
1185                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1186                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1187                 'duration': 142,
1188                 'uploader': 'The Witcher',
1189                 'uploader_id': 'WitcherGame',
1190                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1191                 'upload_date': '20140605',
1192                 'age_limit': 18,
1193             },
1194         },
1195         {
1196             'note': 'Age-gate video with embed allowed in public site',
1197             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1198             'info_dict': {
1199                 'id': 'HsUATh_Nc2U',
1200                 'ext': 'mp4',
1201                 'title': 'Godzilla 2 (Official Video)',
1202                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1203                 'upload_date': '20200408',
1204                 'uploader_id': 'FlyingKitty900',
1205                 'uploader': 'FlyingKitty',
1206                 'age_limit': 18,
1207             },
1208         },
1209         {
1210             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1211             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1212             'info_dict': {
1213                 'id': 'Tq92D6wQ1mg',
1214                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1215                 'ext': 'mp4',
1216                 'upload_date': '20191227',
1217                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1218                 'uploader': 'Projekt Melody',
1219                 'description': 'md5:17eccca93a786d51bc67646756894066',
1220                 'age_limit': 18,
1221             },
1222         },
1223         {
1224             'note': 'Non-Agegated non-embeddable video',
1225             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1226             'info_dict': {
1227                 'id': 'MeJVWBSsPAY',
1228                 'ext': 'mp4',
1229                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1230                 'uploader': 'Herr Lurik',
1231                 'uploader_id': 'st3in234',
1232                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1233                 'upload_date': '20130730',
1234             },
1235         },
1236         {
1237             'note': 'Non-bypassable age-gated video',
1238             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1239             'only_matching': True,
1240         },
1241         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1242         # YouTube Red ad is not captured for creator
1243         {
1244             'url': '__2ABJjxzNo',
1245             'info_dict': {
1246                 'id': '__2ABJjxzNo',
1247                 'ext': 'mp4',
1248                 'duration': 266,
1249                 'upload_date': '20100430',
1250                 'uploader_id': 'deadmau5',
1251                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1252                 'creator': 'deadmau5',
1253                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1254                 'uploader': 'deadmau5',
1255                 'title': 'Deadmau5 - Some Chords (HD)',
1256                 'alt_title': 'Some Chords',
1257             },
1258             'expected_warnings': [
1259                 'DASH manifest missing',
1260             ]
1261         },
1262         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1263         {
1264             'url': 'lqQg6PlCWgI',
1265             'info_dict': {
1266                 'id': 'lqQg6PlCWgI',
1267                 'ext': 'mp4',
1268                 'duration': 6085,
1269                 'upload_date': '20150827',
1270                 'uploader_id': 'olympic',
1271                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1272                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
1273                 'uploader': 'Olympics',
1274                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1275             },
1276             'params': {
1277                 'skip_download': 'requires avconv',
1278             }
1279         },
1280         # Non-square pixels
1281         {
1282             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1283             'info_dict': {
1284                 'id': '_b-2C3KPAM0',
1285                 'ext': 'mp4',
1286                 'stretched_ratio': 16 / 9.,
1287                 'duration': 85,
1288                 'upload_date': '20110310',
1289                 'uploader_id': 'AllenMeow',
1290                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1291                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1292                 'uploader': '孫ᄋᄅ',
1293                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1294             },
1295         },
1296         # url_encoded_fmt_stream_map is empty string
1297         {
1298             'url': 'qEJwOuvDf7I',
1299             'info_dict': {
1300                 'id': 'qEJwOuvDf7I',
1301                 'ext': 'webm',
1302                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1303                 'description': '',
1304                 'upload_date': '20150404',
1305                 'uploader_id': 'spbelect',
1306                 'uploader': 'Наблюдатели Петербурга',
1307             },
1308             'params': {
1309                 'skip_download': 'requires avconv',
1310             },
1311             'skip': 'This live event has ended.',
1312         },
1313         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1314         {
1315             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1316             'info_dict': {
1317                 'id': 'FIl7x6_3R5Y',
1318                 'ext': 'webm',
1319                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1320                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1321                 'duration': 220,
1322                 'upload_date': '20150625',
1323                 'uploader_id': 'dorappi2000',
1324                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1325                 'uploader': 'dorappi2000',
1326                 'formats': 'mincount:31',
1327             },
1328             'skip': 'not actual anymore',
1329         },
1330         # DASH manifest with segment_list
1331         {
1332             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1333             'md5': '8ce563a1d667b599d21064e982ab9e31',
1334             'info_dict': {
1335                 'id': 'CsmdDsKjzN8',
1336                 'ext': 'mp4',
1337                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1338                 'uploader': 'Airtek',
1339                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1340                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1341                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1342             },
1343             'params': {
1344                 'youtube_include_dash_manifest': True,
1345                 'format': '135',  # bestvideo
1346             },
1347             'skip': 'This live event has ended.',
1348         },
1349         {
1350             # Multifeed videos (multiple cameras), URL is for Main Camera
1351             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
1352             'info_dict': {
1353                 'id': 'jvGDaLqkpTg',
1354                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
1355                 'description': 'md5:e03b909557865076822aa169218d6a5d',
1356             },
1357             'playlist': [{
1358                 'info_dict': {
1359                     'id': 'jvGDaLqkpTg',
1360                     'ext': 'mp4',
1361                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
1362                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1363                     'duration': 10643,
1364                     'upload_date': '20161111',
1365                     'uploader': 'Team PGP',
1366                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1367                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1368                 },
1369             }, {
1370                 'info_dict': {
1371                     'id': '3AKt1R1aDnw',
1372                     'ext': 'mp4',
1373                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
1374                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1375                     'duration': 10991,
1376                     'upload_date': '20161111',
1377                     'uploader': 'Team PGP',
1378                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1379                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1380                 },
1381             }, {
1382                 'info_dict': {
1383                     'id': 'RtAMM00gpVc',
1384                     'ext': 'mp4',
1385                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
1386                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1387                     'duration': 10995,
1388                     'upload_date': '20161111',
1389                     'uploader': 'Team PGP',
1390                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1391                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1392                 },
1393             }, {
1394                 'info_dict': {
1395                     'id': '6N2fdlP3C5U',
1396                     'ext': 'mp4',
1397                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
1398                     'description': 'md5:e03b909557865076822aa169218d6a5d',
1399                     'duration': 10990,
1400                     'upload_date': '20161111',
1401                     'uploader': 'Team PGP',
1402                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
1403                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
1404                 },
1405             }],
1406             'params': {
1407                 'skip_download': True,
1408             },
1409             'skip': 'Not multifeed anymore',
1410         },
1411         {
1412             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1413             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1414             'info_dict': {
1415                 'id': 'gVfLd0zydlo',
1416                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1417             },
1418             'playlist_count': 2,
1419             'skip': 'Not multifeed anymore',
1420         },
1421         {
1422             'url': 'https://vid.plus/FlRa-iH7PGw',
1423             'only_matching': True,
1424         },
1425         {
1426             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1427             'only_matching': True,
1428         },
1429         {
1430             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1431             # Also tests cut-off URL expansion in video description (see
1432             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1433             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1434             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1435             'info_dict': {
1436                 'id': 'lsguqyKfVQg',
1437                 'ext': 'mp4',
1438                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1439                 'alt_title': 'Dark Walk',
1440                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1441                 'duration': 133,
1442                 'upload_date': '20151119',
1443                 'uploader_id': 'IronSoulElf',
1444                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1445                 'uploader': 'IronSoulElf',
1446                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1447                 'track': 'Dark Walk',
1448                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1449                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1450             },
1451             'params': {
1452                 'skip_download': True,
1453             },
1454         },
1455         {
1456             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1457             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1458             'only_matching': True,
1459         },
1460         {
1461             # Video with yt:stretch=17:0
1462             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1463             'info_dict': {
1464                 'id': 'Q39EVAstoRM',
1465                 'ext': 'mp4',
1466                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1467                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1468                 'upload_date': '20151107',
1469                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1470                 'uploader': 'CH GAMER DROID',
1471             },
1472             'params': {
1473                 'skip_download': True,
1474             },
1475             'skip': 'This video does not exist.',
1476         },
1477         {
1478             # Video with incomplete 'yt:stretch=16:'
1479             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1480             'only_matching': True,
1481         },
1482         {
1483             # Video licensed under Creative Commons
1484             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1485             'info_dict': {
1486                 'id': 'M4gD1WSo5mA',
1487                 'ext': 'mp4',
1488                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1489                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1490                 'duration': 721,
1491                 'upload_date': '20150127',
1492                 'uploader_id': 'BerkmanCenter',
1493                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1494                 'uploader': 'The Berkman Klein Center for Internet & Society',
1495                 'license': 'Creative Commons Attribution license (reuse allowed)',
1496             },
1497             'params': {
1498                 'skip_download': True,
1499             },
1500         },
1501         {
1502             # Channel-like uploader_url
1503             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1504             'info_dict': {
1505                 'id': 'eQcmzGIKrzg',
1506                 'ext': 'mp4',
1507                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1508                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1509                 'duration': 4060,
1510                 'upload_date': '20151119',
1511                 'uploader': 'Bernie Sanders',
1512                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1513                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1514                 'license': 'Creative Commons Attribution license (reuse allowed)',
1515             },
1516             'params': {
1517                 'skip_download': True,
1518             },
1519         },
1520         {
1521             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1522             'only_matching': True,
1523         },
1524         {
1525             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1526             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1527             'only_matching': True,
1528         },
1529         {
1530             # Rental video preview
1531             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1532             'info_dict': {
1533                 'id': 'uGpuVWrhIzE',
1534                 'ext': 'mp4',
1535                 'title': 'Piku - Trailer',
1536                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1537                 'upload_date': '20150811',
1538                 'uploader': 'FlixMatrix',
1539                 'uploader_id': 'FlixMatrixKaravan',
1540                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1541                 'license': 'Standard YouTube License',
1542             },
1543             'params': {
1544                 'skip_download': True,
1545             },
1546             'skip': 'This video is not available.',
1547         },
1548         {
1549             # YouTube Red video with episode data
1550             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1551             'info_dict': {
1552                 'id': 'iqKdEhx-dD4',
1553                 'ext': 'mp4',
1554                 'title': 'Isolation - Mind Field (Ep 1)',
1555                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1556                 'duration': 2085,
1557                 'upload_date': '20170118',
1558                 'uploader': 'Vsauce',
1559                 'uploader_id': 'Vsauce',
1560                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1561                 'series': 'Mind Field',
1562                 'season_number': 1,
1563                 'episode_number': 1,
1564             },
1565             'params': {
1566                 'skip_download': True,
1567             },
1568             'expected_warnings': [
1569                 'Skipping DASH manifest',
1570             ],
1571         },
1572         {
1573             # The following content has been identified by the YouTube community
1574             # as inappropriate or offensive to some audiences.
1575             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1576             'info_dict': {
1577                 'id': '6SJNVb0GnPI',
1578                 'ext': 'mp4',
1579                 'title': 'Race Differences in Intelligence',
1580                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1581                 'duration': 965,
1582                 'upload_date': '20140124',
1583                 'uploader': 'New Century Foundation',
1584                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1585                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1586             },
1587             'params': {
1588                 'skip_download': True,
1589             },
1590             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1591         },
1592         {
1593             # itag 212
1594             'url': '1t24XAntNCY',
1595             'only_matching': True,
1596         },
1597         {
1598             # geo restricted to JP
1599             'url': 'sJL6WA-aGkQ',
1600             'only_matching': True,
1601         },
1602         {
1603             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1604             'only_matching': True,
1605         },
1606         {
1607             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1608             'only_matching': True,
1609         },
1610         {
1611             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1612             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1613             'only_matching': True,
1614         },
1615         {
1616             # DRM protected
1617             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1618             'only_matching': True,
1619         },
1620         {
1621             # Video with unsupported adaptive stream type formats
1622             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1623             'info_dict': {
1624                 'id': 'Z4Vy8R84T1U',
1625                 'ext': 'mp4',
1626                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1627                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1628                 'duration': 433,
1629                 'upload_date': '20130923',
1630                 'uploader': 'Amelia Putri Harwita',
1631                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1632                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1633                 'formats': 'maxcount:10',
1634             },
1635             'params': {
1636                 'skip_download': True,
1637                 'youtube_include_dash_manifest': False,
1638             },
1639             'skip': 'not actual anymore',
1640         },
1641         {
1642             # Youtube Music Auto-generated description
1643             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1644             'info_dict': {
1645                 'id': 'MgNrAu2pzNs',
1646                 'ext': 'mp4',
1647                 'title': 'Voyeur Girl',
1648                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1649                 'upload_date': '20190312',
1650                 'uploader': 'Stephen - Topic',
1651                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1652                 'artist': 'Stephen',
1653                 'track': 'Voyeur Girl',
1654                 'album': 'it\'s too much love to know my dear',
1655                 'release_date': '20190313',
1656                 'release_year': 2019,
1657             },
1658             'params': {
1659                 'skip_download': True,
1660             },
1661         },
1662         {
1663             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1664             'only_matching': True,
1665         },
1666         {
1667             # invalid -> valid video id redirection
1668             'url': 'DJztXj2GPfl',
1669             'info_dict': {
1670                 'id': 'DJztXj2GPfk',
1671                 'ext': 'mp4',
1672                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1673                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1674                 'upload_date': '20090125',
1675                 'uploader': 'Prochorowka',
1676                 'uploader_id': 'Prochorowka',
1677                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1678                 'artist': 'Panjabi MC',
1679                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1680                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1681             },
1682             'params': {
1683                 'skip_download': True,
1684             },
1685             'skip': 'Video unavailable',
1686         },
1687         {
1688             # empty description results in an empty string
1689             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1690             'info_dict': {
1691                 'id': 'x41yOUIvK2k',
1692                 'ext': 'mp4',
1693                 'title': 'IMG 3456',
1694                 'description': '',
1695                 'upload_date': '20170613',
1696                 'uploader_id': 'ElevageOrVert',
1697                 'uploader': 'ElevageOrVert',
1698             },
1699             'params': {
1700                 'skip_download': True,
1701             },
1702         },
1703         {
1704             # with '};' inside yt initial data (see [1])
1705             # see [2] for an example with '};' inside ytInitialPlayerResponse
1706             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1707             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1708             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1709             'info_dict': {
1710                 'id': 'CHqg6qOn4no',
1711                 'ext': 'mp4',
1712                 'title': 'Part 77   Sort a list of simple types in c#',
1713                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1714                 'upload_date': '20130831',
1715                 'uploader_id': 'kudvenkat',
1716                 'uploader': 'kudvenkat',
1717             },
1718             'params': {
1719                 'skip_download': True,
1720             },
1721         },
1722         {
1723             # another example of '};' in ytInitialData
1724             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1725             'only_matching': True,
1726         },
1727         {
1728             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1729             'only_matching': True,
1730         },
1731         {
1732             # https://github.com/ytdl-org/youtube-dl/pull/28094
1733             'url': 'OtqTfy26tG0',
1734             'info_dict': {
1735                 'id': 'OtqTfy26tG0',
1736                 'ext': 'mp4',
1737                 'title': 'Burn Out',
1738                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1739                 'upload_date': '20141120',
1740                 'uploader': 'The Cinematic Orchestra - Topic',
1741                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1742                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1743                 'artist': 'The Cinematic Orchestra',
1744                 'track': 'Burn Out',
1745                 'album': 'Every Day',
1746                 'release_data': None,
1747                 'release_year': None,
1748             },
1749             'params': {
1750                 'skip_download': True,
1751             },
1752         },
1753         {
1754             # controversial video, only works with bpctr when authenticated with cookies
1755             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1756             'only_matching': True,
1757         },
1758         {
1759             # controversial video, requires bpctr/contentCheckOk
1760             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
1761             'info_dict': {
1762                 'id': 'SZJvDhaSDnc',
1763                 'ext': 'mp4',
1764                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
1765                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
1766                 'uploader': 'CBS This Morning',
1767                 'uploader_id': 'CBSThisMorning',
1768                 'upload_date': '20140716',
1769                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7'
1770             }
1771         },
1772         {
1773             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
1774             'url': 'cBvYw8_A0vQ',
1775             'info_dict': {
1776                 'id': 'cBvYw8_A0vQ',
1777                 'ext': 'mp4',
1778                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
1779                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
1780                 'upload_date': '20201120',
1781                 'uploader': 'Walk around Japan',
1782                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
1783                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
1784             },
1785             'params': {
1786                 'skip_download': True,
1787             },
1788         }, {
1789             # Has multiple audio streams
1790             'url': 'WaOKSUlf4TM',
1791             'only_matching': True
1792         }, {
1793             # Requires Premium: has format 141 when requested using YTM url
1794             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
1795             'only_matching': True
1796         }, {
1797             # multiple subtitles with same lang_code
1798             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
1799             'only_matching': True,
1800         }, {
1801             # Force use android client fallback
1802             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
1803             'info_dict': {
1804                 'id': 'YOelRv7fMxY',
1805                 'title': 'DIGGING A SECRET TUNNEL Part 1',
1806                 'ext': '3gp',
1807                 'upload_date': '20210624',
1808                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
1809                 'uploader': 'colinfurze',
1810                 'uploader_id': 'colinfurze',
1811                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
1812                 'description': 'md5:b5096f56af7ccd7a555c84db81738b22'
1813             },
1814             'params': {
1815                 'format': '17',  # 3gp format available on android
1816                 'extractor_args': {'youtube': {'player_client': ['android']}},
1817             },
1818         },
1819         {
1820             # Skip download of additional client configs (remix client config in this case)
1821             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1822             'only_matching': True,
1823             'params': {
1824                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
1825             },
1826         }
1827     ]
1828
1829     @classmethod
1830     def suitable(cls, url):
1831         # Hack for lazy extractors until more generic solution is implemented
1832         # (see #28780)
1833         from .youtube import parse_qs
1834         qs = parse_qs(url)
1835         if qs.get('list', [None])[0]:
1836             return False
1837         return super(YoutubeIE, cls).suitable(url)
1838
1839     def __init__(self, *args, **kwargs):
1840         super(YoutubeIE, self).__init__(*args, **kwargs)
1841         self._code_cache = {}
1842         self._player_cache = {}
1843
1844     def _extract_player_url(self, ytcfg=None, webpage=None):
1845         player_url = try_get(ytcfg, (lambda x: x['PLAYER_JS_URL']), str)
1846         if not player_url and webpage:
1847             player_url = self._search_regex(
1848                 r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
1849                 webpage, 'player URL', fatal=False)
1850         if not player_url:
1851             return None
1852         if player_url.startswith('//'):
1853             player_url = 'https:' + player_url
1854         elif not re.match(r'https?://', player_url):
1855             player_url = compat_urlparse.urljoin(
1856                 'https://www.youtube.com', player_url)
1857         return player_url
1858
1859     def _signature_cache_id(self, example_sig):
1860         """ Return a string representation of a signature """
1861         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
1862
1863     @classmethod
1864     def _extract_player_info(cls, player_url):
1865         for player_re in cls._PLAYER_INFO_RE:
1866             id_m = re.search(player_re, player_url)
1867             if id_m:
1868                 break
1869         else:
1870             raise ExtractorError('Cannot identify player %r' % player_url)
1871         return id_m.group('id')
1872
1873     def _load_player(self, video_id, player_url, fatal=True) -> bool:
1874         player_id = self._extract_player_info(player_url)
1875         if player_id not in self._code_cache:
1876             self._code_cache[player_id] = self._download_webpage(
1877                 player_url, video_id, fatal=fatal,
1878                 note='Downloading player ' + player_id,
1879                 errnote='Download of %s failed' % player_url)
1880         return player_id in self._code_cache
1881
1882     def _extract_signature_function(self, video_id, player_url, example_sig):
1883         player_id = self._extract_player_info(player_url)
1884
1885         # Read from filesystem cache
1886         func_id = 'js_%s_%s' % (
1887             player_id, self._signature_cache_id(example_sig))
1888         assert os.path.basename(func_id) == func_id
1889
1890         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
1891         if cache_spec is not None:
1892             return lambda s: ''.join(s[i] for i in cache_spec)
1893
1894         if self._load_player(video_id, player_url):
1895             code = self._code_cache[player_id]
1896             res = self._parse_sig_js(code)
1897
1898             test_string = ''.join(map(compat_chr, range(len(example_sig))))
1899             cache_res = res(test_string)
1900             cache_spec = [ord(c) for c in cache_res]
1901
1902             self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
1903             return res
1904
1905     def _print_sig_code(self, func, example_sig):
1906         def gen_sig_code(idxs):
1907             def _genslice(start, end, step):
1908                 starts = '' if start == 0 else str(start)
1909                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
1910                 steps = '' if step == 1 else (':%d' % step)
1911                 return 's[%s%s%s]' % (starts, ends, steps)
1912
1913             step = None
1914             # Quelch pyflakes warnings - start will be set when step is set
1915             start = '(Never used)'
1916             for i, prev in zip(idxs[1:], idxs[:-1]):
1917                 if step is not None:
1918                     if i - prev == step:
1919                         continue
1920                     yield _genslice(start, prev, step)
1921                     step = None
1922                     continue
1923                 if i - prev in [-1, 1]:
1924                     step = i - prev
1925                     start = prev
1926                     continue
1927                 else:
1928                     yield 's[%d]' % prev
1929             if step is None:
1930                 yield 's[%d]' % i
1931             else:
1932                 yield _genslice(start, i, step)
1933
1934         test_string = ''.join(map(compat_chr, range(len(example_sig))))
1935         cache_res = func(test_string)
1936         cache_spec = [ord(c) for c in cache_res]
1937         expr_code = ' + '.join(gen_sig_code(cache_spec))
1938         signature_id_tuple = '(%s)' % (
1939             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1940         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1941                 '    return %s\n') % (signature_id_tuple, expr_code)
1942         self.to_screen('Extracted signature function:\n' + code)
1943
1944     def _parse_sig_js(self, jscode):
1945         funcname = self._search_regex(
1946             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1947              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1948              r'\bm=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(h\.s\)\)',
1949              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(c\)\)',
1950              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
1951              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1952              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1953              # Obsolete patterns
1954              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1955              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
1956              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1957              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1958              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1959              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1960              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1961              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
1962             jscode, 'Initial JS player signature function name', group='sig')
1963
1964         jsi = JSInterpreter(jscode)
1965         initial_function = jsi.extract_function(funcname)
1966         return lambda s: initial_function([s])
1967
1968     def _decrypt_signature(self, s, video_id, player_url):
1969         """Turn the encrypted s field into a working signature"""
1970
1971         if player_url is None:
1972             raise ExtractorError('Cannot decrypt signature without player_url')
1973
1974         try:
1975             player_id = (player_url, self._signature_cache_id(s))
1976             if player_id not in self._player_cache:
1977                 func = self._extract_signature_function(
1978                     video_id, player_url, s
1979                 )
1980                 self._player_cache[player_id] = func
1981             func = self._player_cache[player_id]
1982             if self.get_param('youtube_print_sig_code'):
1983                 self._print_sig_code(func, s)
1984             return func(s)
1985         except Exception as e:
1986             tb = traceback.format_exc()
1987             raise ExtractorError(
1988                 'Signature extraction failed: ' + tb, cause=e)
1989
1990     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
1991         """
1992         Extract signatureTimestamp (sts)
1993         Required to tell API what sig/player version is in use.
1994         """
1995         sts = None
1996         if isinstance(ytcfg, dict):
1997             sts = int_or_none(ytcfg.get('STS'))
1998
1999         if not sts:
2000             # Attempt to extract from player
2001             if player_url is None:
2002                 error_msg = 'Cannot extract signature timestamp without player_url.'
2003                 if fatal:
2004                     raise ExtractorError(error_msg)
2005                 self.report_warning(error_msg)
2006                 return
2007             if self._load_player(video_id, player_url, fatal=fatal):
2008                 player_id = self._extract_player_info(player_url)
2009                 code = self._code_cache[player_id]
2010                 sts = int_or_none(self._search_regex(
2011                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
2012                     'JS player signature timestamp', group='sts', fatal=fatal))
2013         return sts
2014
2015     def _mark_watched(self, video_id, player_responses):
2016         playback_url = traverse_obj(
2017             player_responses, (..., 'playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
2018             expected_type=url_or_none, get_all=False)
2019         if not playback_url:
2020             self.report_warning('Unable to mark watched')
2021             return
2022         parsed_playback_url = compat_urlparse.urlparse(playback_url)
2023         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
2024
2025         # cpn generation algorithm is reverse engineered from base.js.
2026         # In fact it works even with dummy cpn.
2027         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
2028         cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
2029
2030         qs.update({
2031             'ver': ['2'],
2032             'cpn': [cpn],
2033         })
2034         playback_url = compat_urlparse.urlunparse(
2035             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
2036
2037         self._download_webpage(
2038             playback_url, video_id, 'Marking watched',
2039             'Unable to mark watched', fatal=False)
2040
2041     @staticmethod
2042     def _extract_urls(webpage):
2043         # Embedded YouTube player
2044         entries = [
2045             unescapeHTML(mobj.group('url'))
2046             for mobj in re.finditer(r'''(?x)
2047             (?:
2048                 <iframe[^>]+?src=|
2049                 data-video-url=|
2050                 <embed[^>]+?src=|
2051                 embedSWF\(?:\s*|
2052                 <object[^>]+data=|
2053                 new\s+SWFObject\(
2054             )
2055             (["\'])
2056                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
2057                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
2058             \1''', webpage)]
2059
2060         # lazyYT YouTube embed
2061         entries.extend(list(map(
2062             unescapeHTML,
2063             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
2064
2065         # Wordpress "YouTube Video Importer" plugin
2066         matches = re.findall(r'''(?x)<div[^>]+
2067             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
2068             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
2069         entries.extend(m[-1] for m in matches)
2070
2071         return entries
2072
2073     @staticmethod
2074     def _extract_url(webpage):
2075         urls = YoutubeIE._extract_urls(webpage)
2076         return urls[0] if urls else None
2077
2078     @classmethod
2079     def extract_id(cls, url):
2080         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
2081         if mobj is None:
2082             raise ExtractorError('Invalid URL: %s' % url)
2083         video_id = mobj.group(2)
2084         return video_id
2085
2086     def _extract_chapters_from_json(self, data, duration):
2087         chapter_list = traverse_obj(
2088             data, (
2089                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
2090                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
2091             ), expected_type=list)
2092
2093         return self._extract_chapters(
2094             chapter_list,
2095             chapter_time=lambda chapter: float_or_none(
2096                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
2097             chapter_title=lambda chapter: traverse_obj(
2098                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
2099             duration=duration)
2100
2101     def _extract_chapters_from_engagement_panel(self, data, duration):
2102         content_list = traverse_obj(
2103             data,
2104             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
2105             expected_type=list, default=[])
2106         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
2107         chapter_title = lambda chapter: self._get_text(chapter, 'title')
2108
2109         return next((
2110             filter(None, (
2111                 self._extract_chapters(
2112                     traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
2113                     chapter_time, chapter_title, duration)
2114                 for contents in content_list
2115             ))), [])
2116
2117     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration):
2118         chapters = []
2119         last_chapter = {'start_time': 0}
2120         for idx, chapter in enumerate(chapter_list or []):
2121             title = chapter_title(chapter)
2122             start_time = chapter_time(chapter)
2123             if start_time is None:
2124                 continue
2125             last_chapter['end_time'] = start_time
2126             if start_time < last_chapter['start_time']:
2127                 if idx == 1:
2128                     chapters.pop()
2129                     self.report_warning('Invalid start time for chapter "%s"' % last_chapter['title'])
2130                 else:
2131                     self.report_warning(f'Invalid start time for chapter "{title}"')
2132                     continue
2133             last_chapter = {'start_time': start_time, 'title': title}
2134             chapters.append(last_chapter)
2135         last_chapter['end_time'] = duration
2136         return chapters
2137
2138     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
2139         return self._parse_json(self._search_regex(
2140             (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
2141              regex), webpage, name, default='{}'), video_id, fatal=False)
2142
2143     @staticmethod
2144     def parse_time_text(time_text):
2145         """
2146         Parse the comment time text
2147         time_text is in the format 'X units ago (edited)'
2148         """
2149         time_text_split = time_text.split(' ')
2150         if len(time_text_split) >= 3:
2151             try:
2152                 return datetime_from_str('now-%s%s' % (time_text_split[0], time_text_split[1]), precision='auto')
2153             except ValueError:
2154                 return None
2155
2156     def _extract_comment(self, comment_renderer, parent=None):
2157         comment_id = comment_renderer.get('commentId')
2158         if not comment_id:
2159             return
2160
2161         text = self._get_text(comment_renderer, 'contentText')
2162
2163         # note: timestamp is an estimate calculated from the current time and time_text
2164         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
2165         time_text_dt = self.parse_time_text(time_text)
2166         if isinstance(time_text_dt, datetime.datetime):
2167             timestamp = calendar.timegm(time_text_dt.timetuple())
2168         author = self._get_text(comment_renderer, 'authorText')
2169         author_id = try_get(comment_renderer,
2170                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
2171
2172         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
2173                                                        lambda x: x['likeCount']), compat_str)) or 0
2174         author_thumbnail = try_get(comment_renderer,
2175                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], compat_str)
2176
2177         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
2178         is_favorited = 'creatorHeart' in (try_get(
2179             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
2180         return {
2181             'id': comment_id,
2182             'text': text,
2183             'timestamp': timestamp,
2184             'time_text': time_text,
2185             'like_count': votes,
2186             'is_favorited': is_favorited,
2187             'author': author,
2188             'author_id': author_id,
2189             'author_thumbnail': author_thumbnail,
2190             'author_is_uploader': author_is_uploader,
2191             'parent': parent or 'root'
2192         }
2193
2194     def _comment_entries(self, root_continuation_data, identity_token, account_syncid,
2195                          ytcfg, video_id, parent=None, comment_counts=None):
2196
2197         def extract_header(contents):
2198             _total_comments = 0
2199             _continuation = None
2200             for content in contents:
2201                 comments_header_renderer = try_get(content, lambda x: x['commentsHeaderRenderer'])
2202                 expected_comment_count = parse_count(self._get_text(
2203                     comments_header_renderer, 'countText', 'commentsCount', max_runs=1))
2204
2205                 if expected_comment_count:
2206                     comment_counts[1] = expected_comment_count
2207                     self.to_screen('Downloading ~%d comments' % expected_comment_count)
2208                     _total_comments = comment_counts[1]
2209                 sort_mode_str = self._configuration_arg('comment_sort', [''])[0]
2210                 comment_sort_index = int(sort_mode_str != 'top')  # 1 = new, 0 = top
2211
2212                 sort_menu_item = try_get(
2213                     comments_header_renderer,
2214                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
2215                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
2216
2217                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
2218                 if not _continuation:
2219                     continue
2220
2221                 sort_text = sort_menu_item.get('title')
2222                 if isinstance(sort_text, compat_str):
2223                     sort_text = sort_text.lower()
2224                 else:
2225                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
2226                 self.to_screen('Sorting comments by %s' % sort_text)
2227                 break
2228             return _total_comments, _continuation
2229
2230         def extract_thread(contents):
2231             if not parent:
2232                 comment_counts[2] = 0
2233             for content in contents:
2234                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
2235                 comment_renderer = try_get(
2236                     comment_thread_renderer, (lambda x: x['comment']['commentRenderer'], dict)) or try_get(
2237                     content, (lambda x: x['commentRenderer'], dict))
2238
2239                 if not comment_renderer:
2240                     continue
2241                 comment = self._extract_comment(comment_renderer, parent)
2242                 if not comment:
2243                     continue
2244                 comment_counts[0] += 1
2245                 yield comment
2246                 # Attempt to get the replies
2247                 comment_replies_renderer = try_get(
2248                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
2249
2250                 if comment_replies_renderer:
2251                     comment_counts[2] += 1
2252                     comment_entries_iter = self._comment_entries(
2253                         comment_replies_renderer, identity_token, account_syncid, ytcfg,
2254                         video_id, parent=comment.get('id'), comment_counts=comment_counts)
2255
2256                     for reply_comment in comment_entries_iter:
2257                         yield reply_comment
2258
2259         # YouTube comments have a max depth of 2
2260         max_depth = int_or_none(self._configuration_arg('max_comment_depth', [''])[0]) or float('inf')
2261         if max_depth == 1 and parent:
2262             return
2263         if not comment_counts:
2264             # comment so far, est. total comments, current comment thread #
2265             comment_counts = [0, 0, 0]
2266
2267         continuation = self._extract_continuation(root_continuation_data)
2268         if continuation and len(continuation['continuation']) < 27:
2269             self.write_debug('Detected old API continuation token. Generating new API compatible token.')
2270             continuation_token = self._generate_comment_continuation(video_id)
2271             continuation = self._build_api_continuation_query(continuation_token, None)
2272
2273         visitor_data = None
2274         is_first_continuation = parent is None
2275
2276         for page_num in itertools.count(0):
2277             if not continuation:
2278                 break
2279             headers = self.generate_api_headers(ytcfg, identity_token, account_syncid, visitor_data)
2280             comment_prog_str = '(%d/%d)' % (comment_counts[0], comment_counts[1])
2281             if page_num == 0:
2282                 if is_first_continuation:
2283                     note_prefix = 'Downloading comment section API JSON'
2284                 else:
2285                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
2286                         comment_counts[2], comment_prog_str)
2287             else:
2288                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
2289                     '       ' if parent else '', ' replies' if parent else '',
2290                     page_num, comment_prog_str)
2291
2292             response = self._extract_response(
2293                 item_id=None, query=continuation,
2294                 ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
2295                 check_get_keys=('onResponseReceivedEndpoints', 'continuationContents'))
2296             if not response:
2297                 break
2298             visitor_data = try_get(
2299                 response,
2300                 lambda x: x['responseContext']['webResponseContextExtensionData']['ytConfigData']['visitorData'],
2301                 compat_str) or visitor_data
2302
2303             continuation_contents = dict_get(response, ('onResponseReceivedEndpoints', 'continuationContents'))
2304
2305             continuation = None
2306             if isinstance(continuation_contents, list):
2307                 for continuation_section in continuation_contents:
2308                     if not isinstance(continuation_section, dict):
2309                         continue
2310                     continuation_items = try_get(
2311                         continuation_section,
2312                         (lambda x: x['reloadContinuationItemsCommand']['continuationItems'],
2313                          lambda x: x['appendContinuationItemsAction']['continuationItems']),
2314                         list) or []
2315                     if is_first_continuation:
2316                         total_comments, continuation = extract_header(continuation_items)
2317                         if total_comments:
2318                             yield total_comments
2319                         is_first_continuation = False
2320                         if continuation:
2321                             break
2322                         continue
2323                     count = 0
2324                     for count, entry in enumerate(extract_thread(continuation_items)):
2325                         yield entry
2326                     continuation = self._extract_continuation({'contents': continuation_items})
2327                     if continuation:
2328                         # Sometimes YouTube provides a continuation without any comments
2329                         # In most cases we end up just downloading these with very little comments to come.
2330                         if count == 0:
2331                             if not parent:
2332                                 self.report_warning('No comments received - assuming end of comments')
2333                             continuation = None
2334                         break
2335
2336             # Deprecated response structure
2337             elif isinstance(continuation_contents, dict):
2338                 known_continuation_renderers = ('itemSectionContinuation', 'commentRepliesContinuation')
2339                 for key, continuation_renderer in continuation_contents.items():
2340                     if key not in known_continuation_renderers:
2341                         continue
2342                     if not isinstance(continuation_renderer, dict):
2343                         continue
2344                     if is_first_continuation:
2345                         header_continuation_items = [continuation_renderer.get('header') or {}]
2346                         total_comments, continuation = extract_header(header_continuation_items)
2347                         if total_comments:
2348                             yield total_comments
2349                         is_first_continuation = False
2350                         if continuation:
2351                             break
2352
2353                     # Sometimes YouTube provides a continuation without any comments
2354                     # In most cases we end up just downloading these with very little comments to come.
2355                     count = 0
2356                     for count, entry in enumerate(extract_thread(continuation_renderer.get('contents') or {})):
2357                         yield entry
2358                     continuation = self._extract_continuation(continuation_renderer)
2359                     if count == 0:
2360                         if not parent:
2361                             self.report_warning('No comments received - assuming end of comments')
2362                         continuation = None
2363                     break
2364
2365     @staticmethod
2366     def _generate_comment_continuation(video_id):
2367         """
2368         Generates initial comment section continuation token from given video id
2369         """
2370         b64_vid_id = base64.b64encode(bytes(video_id.encode('utf-8')))
2371         parts = ('Eg0SCw==', b64_vid_id, 'GAYyJyIRIgs=', b64_vid_id, 'MAB4AjAAQhBjb21tZW50cy1zZWN0aW9u')
2372         new_continuation_intlist = list(itertools.chain.from_iterable(
2373             [bytes_to_intlist(base64.b64decode(part)) for part in parts]))
2374         return base64.b64encode(intlist_to_bytes(new_continuation_intlist)).decode('utf-8')
2375
2376     def _extract_comments(self, ytcfg, video_id, contents, webpage):
2377         """Entry for comment extraction"""
2378         def _real_comment_extract(contents):
2379             if isinstance(contents, list):
2380                 for entry in contents:
2381                     for key, renderer in entry.items():
2382                         if key not in known_entry_comment_renderers:
2383                             continue
2384                         yield from self._comment_entries(
2385                             renderer, video_id=video_id, ytcfg=ytcfg,
2386                             identity_token=self._extract_identity_token(webpage, item_id=video_id),
2387                             account_syncid=self._extract_account_syncid(ytcfg))
2388                         break
2389         comments = []
2390         known_entry_comment_renderers = ('itemSectionRenderer',)
2391         estimated_total = 0
2392         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0]) or float('inf')
2393         # Force English regardless of account setting to prevent parsing issues
2394         # See: https://github.com/yt-dlp/yt-dlp/issues/532
2395         ytcfg = copy.deepcopy(ytcfg)
2396         traverse_obj(
2397             ytcfg, ('INNERTUBE_CONTEXT', 'client'), expected_type=dict, default={})['hl'] = 'en'
2398         try:
2399             for comment in _real_comment_extract(contents):
2400                 if len(comments) >= max_comments:
2401                     break
2402                 if isinstance(comment, int):
2403                     estimated_total = comment
2404                     continue
2405                 comments.append(comment)
2406         except KeyboardInterrupt:
2407             self.to_screen('Interrupted by user')
2408         self.to_screen('Downloaded %d/%d comments' % (len(comments), estimated_total))
2409         return {
2410             'comments': comments,
2411             'comment_count': len(comments),
2412         }
2413
2414     @staticmethod
2415     def _generate_player_context(sts=None):
2416         context = {
2417             'html5Preference': 'HTML5_PREF_WANTS',
2418         }
2419         if sts is not None:
2420             context['signatureTimestamp'] = sts
2421         return {
2422             'playbackContext': {
2423                 'contentPlaybackContext': context
2424             },
2425             'contentCheckOk': True,
2426             'racyCheckOk': True
2427         }
2428
2429     @staticmethod
2430     def _is_agegated(player_response):
2431         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
2432             return True
2433
2434         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')), default=[])
2435         AGE_GATE_REASONS = (
2436             'confirm your age', 'age-restricted', 'inappropriate',  # reason
2437             'age_verification_required', 'age_check_required',  # status
2438         )
2439         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
2440
2441     @staticmethod
2442     def _is_unplayable(player_response):
2443         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
2444
2445     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, identity_token, player_url, initial_pr):
2446
2447         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
2448         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
2449         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False)
2450         headers = self.generate_api_headers(
2451             player_ytcfg, identity_token, syncid,
2452             default_client=client, session_index=session_index)
2453
2454         yt_query = {'videoId': video_id}
2455         yt_query.update(self._generate_player_context(sts))
2456         return self._extract_response(
2457             item_id=video_id, ep='player', query=yt_query,
2458             ytcfg=player_ytcfg, headers=headers, fatal=True,
2459             default_client=client,
2460             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
2461         ) or None
2462
2463     def _get_requested_clients(self, url, smuggled_data):
2464         requested_clients = []
2465         allowed_clients = sorted(
2466             [client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'],
2467             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
2468         for client in self._configuration_arg('player_client'):
2469             if client in allowed_clients:
2470                 requested_clients.append(client)
2471             elif client == 'all':
2472                 requested_clients.extend(allowed_clients)
2473             else:
2474                 self.report_warning(f'Skipping unsupported client {client}')
2475         if not requested_clients:
2476             requested_clients = ['android', 'web']
2477
2478         if smuggled_data.get('is_music_url') or self.is_music_url(url):
2479             requested_clients.extend(
2480                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
2481
2482         return orderedSet(requested_clients)
2483
2484     def _extract_player_ytcfg(self, client, video_id):
2485         url = {
2486             'web_music': 'https://music.youtube.com',
2487             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
2488         }.get(client)
2489         if not url:
2490             return {}
2491         webpage = self._download_webpage(url, video_id, fatal=False, note=f'Downloading {client} config')
2492         return self.extract_ytcfg(video_id, webpage) or {}
2493
2494     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, player_url, identity_token):
2495         initial_pr = None
2496         if webpage:
2497             initial_pr = self._extract_yt_initial_variable(
2498                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
2499                 video_id, 'initial player response')
2500
2501         original_clients = clients
2502         clients = clients[::-1]
2503
2504         def append_client(client_name):
2505             if client_name in INNERTUBE_CLIENTS and client_name not in original_clients:
2506                 clients.append(client_name)
2507
2508         # Android player_response does not have microFormats which are needed for
2509         # extraction of some data. So we return the initial_pr with formats
2510         # stripped out even if not requested by the user
2511         # See: https://github.com/yt-dlp/yt-dlp/issues/501
2512         yielded_pr = False
2513         if initial_pr:
2514             pr = dict(initial_pr)
2515             pr['streamingData'] = None
2516             yielded_pr = True
2517             yield pr
2518
2519         last_error = None
2520         while clients:
2521             client = clients.pop()
2522             player_ytcfg = master_ytcfg if client == 'web' else {}
2523             if 'configs' not in self._configuration_arg('player_skip'):
2524                 player_ytcfg = self._extract_player_ytcfg(client, video_id) or player_ytcfg
2525
2526             try:
2527                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
2528                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, identity_token, player_url, initial_pr)
2529             except ExtractorError as e:
2530                 if last_error:
2531                     self.report_warning(last_error)
2532                 last_error = e
2533                 continue
2534
2535             if pr:
2536                 yielded_pr = True
2537                 yield pr
2538
2539             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
2540             if client.endswith('_agegate') and self._is_unplayable(pr) and self._generate_sapisidhash_header():
2541                 append_client(client.replace('_agegate', '_creator'))
2542             elif self._is_agegated(pr):
2543                 append_client(f'{client}_agegate')
2544
2545         if last_error:
2546             if not yielded_pr:
2547                 raise last_error
2548             self.report_warning(last_error)
2549
2550     def _extract_formats(self, streaming_data, video_id, player_url, is_live):
2551         itags, stream_ids = [], []
2552         itag_qualities, res_qualities = {}, {}
2553         q = qualities([
2554             # Normally tiny is the smallest video-only formats. But
2555             # audio-only formats with unknown quality may get tagged as tiny
2556             'tiny',
2557             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
2558             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
2559         ])
2560         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...), default=[])
2561
2562         for fmt in streaming_formats:
2563             if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
2564                 continue
2565
2566             itag = str_or_none(fmt.get('itag'))
2567             audio_track = fmt.get('audioTrack') or {}
2568             stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
2569             if stream_id in stream_ids:
2570                 continue
2571
2572             quality = fmt.get('quality')
2573             height = int_or_none(fmt.get('height'))
2574             if quality == 'tiny' or not quality:
2575                 quality = fmt.get('audioQuality', '').lower() or quality
2576             # The 3gp format (17) in android client has a quality of "small",
2577             # but is actually worse than other formats
2578             if itag == '17':
2579                 quality = 'tiny'
2580             if quality:
2581                 if itag:
2582                     itag_qualities[itag] = quality
2583                 if height:
2584                     res_qualities[height] = quality
2585             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
2586             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
2587             # number of fragment that would subsequently requested with (`&sq=N`)
2588             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
2589                 continue
2590
2591             fmt_url = fmt.get('url')
2592             if not fmt_url:
2593                 sc = compat_parse_qs(fmt.get('signatureCipher'))
2594                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
2595                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
2596                 if not (sc and fmt_url and encrypted_sig):
2597                     continue
2598                 if not player_url:
2599                     continue
2600                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
2601                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
2602                 fmt_url += '&' + sp + '=' + signature
2603
2604             if itag:
2605                 itags.append(itag)
2606                 stream_ids.append(stream_id)
2607
2608             tbr = float_or_none(
2609                 fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
2610             dct = {
2611                 'asr': int_or_none(fmt.get('audioSampleRate')),
2612                 'filesize': int_or_none(fmt.get('contentLength')),
2613                 'format_id': itag,
2614                 'format_note': ', '.join(filter(None, (
2615                     audio_track.get('displayName'),
2616                     fmt.get('qualityLabel') or quality.replace('audio_quality_', '')))),
2617                 'fps': int_or_none(fmt.get('fps')),
2618                 'height': height,
2619                 'quality': q(quality),
2620                 'tbr': tbr,
2621                 'url': fmt_url,
2622                 'width': int_or_none(fmt.get('width')),
2623                 'language': audio_track.get('id', '').split('.')[0],
2624             }
2625             mime_mobj = re.match(
2626                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
2627             if mime_mobj:
2628                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
2629                 dct.update(parse_codecs(mime_mobj.group(2)))
2630             no_audio = dct.get('acodec') == 'none'
2631             no_video = dct.get('vcodec') == 'none'
2632             if no_audio:
2633                 dct['vbr'] = tbr
2634             if no_video:
2635                 dct['abr'] = tbr
2636             if no_audio or no_video:
2637                 dct['downloader_options'] = {
2638                     # Youtube throttles chunks >~10M
2639                     'http_chunk_size': 10485760,
2640                 }
2641                 if dct.get('ext'):
2642                     dct['container'] = dct['ext'] + '_dash'
2643             yield dct
2644
2645         skip_manifests = self._configuration_arg('skip')
2646         get_dash = (
2647             (not is_live or self._configuration_arg('include_live_dash'))
2648             and 'dash' not in skip_manifests and self.get_param('youtube_include_dash_manifest', True))
2649         get_hls = 'hls' not in skip_manifests and self.get_param('youtube_include_hls_manifest', True)
2650
2651         def guess_quality(f):
2652             for val, qdict in ((f.get('format_id'), itag_qualities), (f.get('height'), res_qualities)):
2653                 if val in qdict:
2654                     return q(qdict[val])
2655             return -1
2656
2657         for sd in streaming_data:
2658             hls_manifest_url = get_hls and sd.get('hlsManifestUrl')
2659             if hls_manifest_url:
2660                 for f in self._extract_m3u8_formats(hls_manifest_url, video_id, 'mp4', fatal=False):
2661                     itag = self._search_regex(
2662                         r'/itag/(\d+)', f['url'], 'itag', default=None)
2663                     if itag in itags:
2664                         continue
2665                     if itag:
2666                         f['format_id'] = itag
2667                         itags.append(itag)
2668                     f['quality'] = guess_quality(f)
2669                     yield f
2670
2671             dash_manifest_url = get_dash and sd.get('dashManifestUrl')
2672             if dash_manifest_url:
2673                 for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
2674                     itag = f['format_id']
2675                     if itag in itags:
2676                         continue
2677                     if itag:
2678                         itags.append(itag)
2679                     f['quality'] = guess_quality(f)
2680                     filesize = int_or_none(self._search_regex(
2681                         r'/clen/(\d+)', f.get('fragment_base_url')
2682                         or f['url'], 'file size', default=None))
2683                     if filesize:
2684                         f['filesize'] = filesize
2685                     yield f
2686
2687     def _real_extract(self, url):
2688         url, smuggled_data = unsmuggle_url(url, {})
2689         video_id = self._match_id(url)
2690
2691         base_url = self.http_scheme() + '//www.youtube.com/'
2692         webpage_url = base_url + 'watch?v=' + video_id
2693         webpage = self._download_webpage(
2694             webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
2695
2696         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
2697         player_url = self._extract_player_url(master_ytcfg, webpage)
2698         identity_token = self._extract_identity_token(webpage, video_id)
2699
2700         player_responses = list(self._extract_player_responses(
2701             self._get_requested_clients(url, smuggled_data),
2702             video_id, webpage, master_ytcfg, player_url, identity_token))
2703
2704         get_first = lambda obj, keys, **kwargs: traverse_obj(obj, (..., *variadic(keys)), **kwargs, get_all=False)
2705
2706         playability_statuses = traverse_obj(
2707             player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
2708
2709         trailer_video_id = get_first(
2710             playability_statuses,
2711             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
2712             expected_type=str)
2713         if trailer_video_id:
2714             return self.url_result(
2715                 trailer_video_id, self.ie_key(), trailer_video_id)
2716
2717         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
2718                        if webpage else (lambda x: None))
2719
2720         video_details = traverse_obj(
2721             player_responses, (..., 'videoDetails'), expected_type=dict, default=[])
2722         microformats = traverse_obj(
2723             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
2724             expected_type=dict, default=[])
2725         video_title = (
2726             get_first(video_details, 'title')
2727             or self._get_text(microformats, (..., 'title'))
2728             or search_meta(['og:title', 'twitter:title', 'title']))
2729         video_description = get_first(video_details, 'shortDescription')
2730
2731         if not smuggled_data.get('force_singlefeed', False):
2732             if not self.get_param('noplaylist'):
2733                 multifeed_metadata_list = get_first(
2734                     player_responses,
2735                     ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
2736                     expected_type=str)
2737                 if multifeed_metadata_list:
2738                     entries = []
2739                     feed_ids = []
2740                     for feed in multifeed_metadata_list.split(','):
2741                         # Unquote should take place before split on comma (,) since textual
2742                         # fields may contain comma as well (see
2743                         # https://github.com/ytdl-org/youtube-dl/issues/8536)
2744                         feed_data = compat_parse_qs(
2745                             compat_urllib_parse_unquote_plus(feed))
2746
2747                         def feed_entry(name):
2748                             return try_get(
2749                                 feed_data, lambda x: x[name][0], compat_str)
2750
2751                         feed_id = feed_entry('id')
2752                         if not feed_id:
2753                             continue
2754                         feed_title = feed_entry('title')
2755                         title = video_title
2756                         if feed_title:
2757                             title += ' (%s)' % feed_title
2758                         entries.append({
2759                             '_type': 'url_transparent',
2760                             'ie_key': 'Youtube',
2761                             'url': smuggle_url(
2762                                 '%swatch?v=%s' % (base_url, feed_data['id'][0]),
2763                                 {'force_singlefeed': True}),
2764                             'title': title,
2765                         })
2766                         feed_ids.append(feed_id)
2767                     self.to_screen(
2768                         'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
2769                         % (', '.join(feed_ids), video_id))
2770                     return self.playlist_result(
2771                         entries, video_id, video_title, video_description)
2772             else:
2773                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2774
2775         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
2776         is_live = get_first(video_details, 'isLive')
2777         if is_live is None:
2778             is_live = get_first(live_broadcast_details, 'isLiveNow')
2779
2780         streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
2781         formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live))
2782
2783         if not formats:
2784             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
2785                 self.raise_no_formats(
2786                     'This video is DRM protected.', expected=True)
2787             pemr = get_first(
2788                 playability_statuses,
2789                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
2790             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
2791             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
2792             if subreason:
2793                 if subreason == 'The uploader has not made this video available in your country.':
2794                     countries = get_first(microformats, 'availableCountries')
2795                     if not countries:
2796                         regions_allowed = search_meta('regionsAllowed')
2797                         countries = regions_allowed.split(',') if regions_allowed else None
2798                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
2799                 reason += f'. {subreason}'
2800             if reason:
2801                 self.raise_no_formats(reason, expected=True)
2802
2803         for f in formats:
2804             if '&c=WEB&' in f['url'] and '&ratebypass=yes&' not in f['url']:  # throttled
2805                 f['source_preference'] = -10
2806                 # TODO: this method is not reliable
2807                 f['format_note'] = format_field(f, 'format_note', '%s ') + '(maybe throttled)'
2808
2809         # Source is given priority since formats that throttle are given lower source_preference
2810         # When throttling issue is fully fixed, remove this
2811         self._sort_formats(formats, ('quality', 'height', 'fps', 'source'))
2812
2813         keywords = get_first(video_details, 'keywords', expected_type=list) or []
2814         if not keywords and webpage:
2815             keywords = [
2816                 unescapeHTML(m.group('content'))
2817                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
2818         for keyword in keywords:
2819             if keyword.startswith('yt:stretch='):
2820                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
2821                 if mobj:
2822                     # NB: float is intentional for forcing float division
2823                     w, h = (float(v) for v in mobj.groups())
2824                     if w > 0 and h > 0:
2825                         ratio = w / h
2826                         for f in formats:
2827                             if f.get('vcodec') != 'none':
2828                                 f['stretched_ratio'] = ratio
2829                         break
2830
2831         thumbnails = []
2832         thumbnail_dicts = traverse_obj(
2833             (video_details, microformats), (..., ..., 'thumbnail', 'thumbnails', ...),
2834             expected_type=dict, default=[])
2835         for thumbnail in thumbnail_dicts:
2836             thumbnail_url = thumbnail.get('url')
2837             if not thumbnail_url:
2838                 continue
2839             # Sometimes youtube gives a wrong thumbnail URL. See:
2840             # https://github.com/yt-dlp/yt-dlp/issues/233
2841             # https://github.com/ytdl-org/youtube-dl/issues/28023
2842             if 'maxresdefault' in thumbnail_url:
2843                 thumbnail_url = thumbnail_url.split('?')[0]
2844             thumbnails.append({
2845                 'url': thumbnail_url,
2846                 'height': int_or_none(thumbnail.get('height')),
2847                 'width': int_or_none(thumbnail.get('width')),
2848             })
2849         thumbnail_url = search_meta(['og:image', 'twitter:image'])
2850         if thumbnail_url:
2851             thumbnails.append({
2852                 'url': thumbnail_url,
2853             })
2854         # The best resolution thumbnails sometimes does not appear in the webpage
2855         # See: https://github.com/ytdl-org/youtube-dl/issues/29049, https://github.com/yt-dlp/yt-dlp/issues/340
2856         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
2857         hq_thumbnail_names = ['maxresdefault', 'hq720', 'sddefault', 'sd1', 'sd2', 'sd3']
2858         # TODO: Test them also? - For some videos, even these don't exist
2859         guaranteed_thumbnail_names = [
2860             'hqdefault', 'hq1', 'hq2', 'hq3', '0',
2861             'mqdefault', 'mq1', 'mq2', 'mq3',
2862             'default', '1', '2', '3'
2863         ]
2864         thumbnail_names = hq_thumbnail_names + guaranteed_thumbnail_names
2865         n_thumbnail_names = len(thumbnail_names)
2866
2867         thumbnails.extend({
2868             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
2869                 video_id=video_id, name=name, ext=ext,
2870                 webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''),
2871             '_test_url': name in hq_thumbnail_names,
2872         } for name in thumbnail_names for ext in ('webp', 'jpg'))
2873         for thumb in thumbnails:
2874             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
2875             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
2876         self._remove_duplicate_formats(thumbnails)
2877
2878         category = get_first(microformats, 'category') or search_meta('genre')
2879         channel_id = str_or_none(
2880             get_first(video_details, 'channelId')
2881             or get_first(microformats, 'externalChannelId')
2882             or search_meta('channelId'))
2883         duration = int_or_none(
2884             get_first(video_details, 'lengthSeconds')
2885             or get_first(microformats, 'lengthSeconds')
2886             or parse_duration(search_meta('duration'))) or None
2887         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
2888
2889         live_content = get_first(video_details, 'isLiveContent')
2890         is_upcoming = get_first(video_details, 'isUpcoming')
2891         if is_live is None:
2892             if is_upcoming or live_content is False:
2893                 is_live = False
2894         if is_upcoming is None and (live_content or is_live):
2895             is_upcoming = False
2896         live_starttime = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
2897         live_endtime = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
2898         if not duration and live_endtime and live_starttime:
2899             duration = live_endtime - live_starttime
2900
2901         info = {
2902             'id': video_id,
2903             'title': self._live_title(video_title) if is_live else video_title,
2904             'formats': formats,
2905             'thumbnails': thumbnails,
2906             'description': video_description,
2907             'upload_date': unified_strdate(
2908                 get_first(microformats, 'uploadDate')
2909                 or search_meta('uploadDate')),
2910             'uploader': get_first(video_details, 'author'),
2911             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
2912             'uploader_url': owner_profile_url,
2913             'channel_id': channel_id,
2914             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
2915             'duration': duration,
2916             'view_count': int_or_none(
2917                 get_first((video_details, microformats), (..., 'viewCount'))
2918                 or search_meta('interactionCount')),
2919             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
2920             'age_limit': 18 if (
2921                 get_first(microformats, 'isFamilySafe') is False
2922                 or search_meta('isFamilyFriendly') == 'false'
2923                 or search_meta('og:restrictions:age') == '18+') else 0,
2924             'webpage_url': webpage_url,
2925             'categories': [category] if category else None,
2926             'tags': keywords,
2927             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
2928             'is_live': is_live,
2929             'was_live': (False if is_live or is_upcoming or live_content is False
2930                          else None if is_live is None or is_upcoming is None
2931                          else live_content),
2932             'live_status': 'is_upcoming' if is_upcoming else None,  # rest will be set by YoutubeDL
2933             'release_timestamp': live_starttime,
2934         }
2935
2936         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
2937         # Converted into dicts to remove duplicates
2938         captions = {
2939             sub.get('baseUrl'): sub
2940             for sub in traverse_obj(pctr, (..., 'captionTracks', ...), default=[])}
2941         translation_languages = {
2942             lang.get('languageCode'): lang.get('languageName')
2943             for lang in traverse_obj(pctr, (..., 'translationLanguages', ...), default=[])}
2944         subtitles = {}
2945         if pctr:
2946             def process_language(container, base_url, lang_code, sub_name, query):
2947                 lang_subs = container.setdefault(lang_code, [])
2948                 for fmt in self._SUBTITLE_FORMATS:
2949                     query.update({
2950                         'fmt': fmt,
2951                     })
2952                     lang_subs.append({
2953                         'ext': fmt,
2954                         'url': update_url_query(base_url, query),
2955                         'name': sub_name,
2956                     })
2957
2958             for base_url, caption_track in captions.items():
2959                 if not base_url:
2960                     continue
2961                 if caption_track.get('kind') != 'asr':
2962                     lang_code = (
2963                         remove_start(caption_track.get('vssId') or '', '.').replace('.', '-')
2964                         or caption_track.get('languageCode'))
2965                     if not lang_code:
2966                         continue
2967                     process_language(
2968                         subtitles, base_url, lang_code,
2969                         traverse_obj(caption_track, ('name', 'simpleText')),
2970                         {})
2971                     continue
2972                 automatic_captions = {}
2973                 for trans_code, trans_name in translation_languages.items():
2974                     if not trans_code:
2975                         continue
2976                     process_language(
2977                         automatic_captions, base_url, trans_code,
2978                         self._get_text(trans_name, max_runs=1),
2979                         {'tlang': trans_code})
2980                 info['automatic_captions'] = automatic_captions
2981         info['subtitles'] = subtitles
2982
2983         parsed_url = compat_urllib_parse_urlparse(url)
2984         for component in [parsed_url.fragment, parsed_url.query]:
2985             query = compat_parse_qs(component)
2986             for k, v in query.items():
2987                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
2988                     d_k += '_time'
2989                     if d_k not in info and k in s_ks:
2990                         info[d_k] = parse_duration(query[k][0])
2991
2992         # Youtube Music Auto-generated description
2993         if video_description:
2994             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
2995             if mobj:
2996                 release_year = mobj.group('release_year')
2997                 release_date = mobj.group('release_date')
2998                 if release_date:
2999                     release_date = release_date.replace('-', '')
3000                     if not release_year:
3001                         release_year = release_date[:4]
3002                 info.update({
3003                     'album': mobj.group('album'.strip()),
3004                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
3005                     'track': mobj.group('track').strip(),
3006                     'release_date': release_date,
3007                     'release_year': int_or_none(release_year),
3008                 })
3009
3010         initial_data = None
3011         if webpage:
3012             initial_data = self._extract_yt_initial_variable(
3013                 webpage, self._YT_INITIAL_DATA_RE, video_id,
3014                 'yt initial data')
3015         if not initial_data:
3016             headers = self.generate_api_headers(
3017                 master_ytcfg, identity_token, self._extract_account_syncid(master_ytcfg),
3018                 session_index=self._extract_session_index(master_ytcfg))
3019
3020             initial_data = self._extract_response(
3021                 item_id=video_id, ep='next', fatal=False,
3022                 ytcfg=master_ytcfg, headers=headers, query={'videoId': video_id},
3023                 note='Downloading initial data API JSON')
3024
3025         try:
3026             # This will error if there is no livechat
3027             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
3028             info['subtitles']['live_chat'] = [{
3029                 'url': 'https://www.youtube.com/watch?v=%s' % video_id,  # url is needed to set cookies
3030                 'video_id': video_id,
3031                 'ext': 'json',
3032                 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
3033             }]
3034         except (KeyError, IndexError, TypeError):
3035             pass
3036
3037         if initial_data:
3038             info['chapters'] = (
3039                 self._extract_chapters_from_json(initial_data, duration)
3040                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
3041                 or None)
3042
3043             contents = try_get(
3044                 initial_data,
3045                 lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'],
3046                 list) or []
3047             for content in contents:
3048                 vpir = content.get('videoPrimaryInfoRenderer')
3049                 if vpir:
3050                     stl = vpir.get('superTitleLink')
3051                     if stl:
3052                         stl = self._get_text(stl)
3053                         if try_get(
3054                                 vpir,
3055                                 lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
3056                             info['location'] = stl
3057                         else:
3058                             mobj = re.search(r'(.+?)\s*S(\d+)\s*•\s*E(\d+)', stl)
3059                             if mobj:
3060                                 info.update({
3061                                     'series': mobj.group(1),
3062                                     'season_number': int(mobj.group(2)),
3063                                     'episode_number': int(mobj.group(3)),
3064                                 })
3065                     for tlb in (try_get(
3066                             vpir,
3067                             lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
3068                             list) or []):
3069                         tbr = tlb.get('toggleButtonRenderer') or {}
3070                         for getter, regex in [(
3071                                 lambda x: x['defaultText']['accessibility']['accessibilityData'],
3072                                 r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
3073                                     lambda x: x['accessibility'],
3074                                     lambda x: x['accessibilityData']['accessibilityData'],
3075                                 ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
3076                             label = (try_get(tbr, getter, dict) or {}).get('label')
3077                             if label:
3078                                 mobj = re.match(regex, label)
3079                                 if mobj:
3080                                     info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
3081                                     break
3082                     sbr_tooltip = try_get(
3083                         vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
3084                     if sbr_tooltip:
3085                         like_count, dislike_count = sbr_tooltip.split(' / ')
3086                         info.update({
3087                             'like_count': str_to_int(like_count),
3088                             'dislike_count': str_to_int(dislike_count),
3089                         })
3090                 vsir = content.get('videoSecondaryInfoRenderer')
3091                 if vsir:
3092                     info['channel'] = self._get_text(vsir, ('owner', 'videoOwnerRenderer', 'title'))
3093                     rows = try_get(
3094                         vsir,
3095                         lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
3096                         list) or []
3097                     multiple_songs = False
3098                     for row in rows:
3099                         if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
3100                             multiple_songs = True
3101                             break
3102                     for row in rows:
3103                         mrr = row.get('metadataRowRenderer') or {}
3104                         mrr_title = mrr.get('title')
3105                         if not mrr_title:
3106                             continue
3107                         mrr_title = self._get_text(mrr, 'title')
3108                         mrr_contents_text = self._get_text(mrr, ('contents', 0))
3109                         if mrr_title == 'License':
3110                             info['license'] = mrr_contents_text
3111                         elif not multiple_songs:
3112                             if mrr_title == 'Album':
3113                                 info['album'] = mrr_contents_text
3114                             elif mrr_title == 'Artist':
3115                                 info['artist'] = mrr_contents_text
3116                             elif mrr_title == 'Song':
3117                                 info['track'] = mrr_contents_text
3118
3119         fallbacks = {
3120             'channel': 'uploader',
3121             'channel_id': 'uploader_id',
3122             'channel_url': 'uploader_url',
3123         }
3124         for to, frm in fallbacks.items():
3125             if not info.get(to):
3126                 info[to] = info.get(frm)
3127
3128         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
3129             v = info.get(s_k)
3130             if v:
3131                 info[d_k] = v
3132
3133         is_private = get_first(video_details, 'isPrivate', expected_type=bool)
3134         is_unlisted = get_first(microformats, 'isUnlisted', expected_type=bool)
3135         is_membersonly = None
3136         is_premium = None
3137         if initial_data and is_private is not None:
3138             is_membersonly = False
3139             is_premium = False
3140             contents = try_get(initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
3141             badge_labels = set()
3142             for content in contents:
3143                 if not isinstance(content, dict):
3144                     continue
3145                 badge_labels.update(self._extract_badges(content.get('videoPrimaryInfoRenderer')))
3146             for badge_label in badge_labels:
3147                 if badge_label.lower() == 'members only':
3148                     is_membersonly = True
3149                 elif badge_label.lower() == 'premium':
3150                     is_premium = True
3151                 elif badge_label.lower() == 'unlisted':
3152                     is_unlisted = True
3153
3154         info['availability'] = self._availability(
3155             is_private=is_private,
3156             needs_premium=is_premium,
3157             needs_subscription=is_membersonly,
3158             needs_auth=info['age_limit'] >= 18,
3159             is_unlisted=None if is_private is None else is_unlisted)
3160
3161         # get xsrf for annotations or comments
3162         get_annotations = self.get_param('writeannotations', False)
3163         get_comments = self.get_param('getcomments', False)
3164         if get_annotations or get_comments:
3165             xsrf_token = None
3166             if master_ytcfg:
3167                 xsrf_token = try_get(master_ytcfg, lambda x: x['XSRF_TOKEN'], compat_str)
3168             if not xsrf_token:
3169                 xsrf_token = self._search_regex(
3170                     r'([\'"])XSRF_TOKEN\1\s*:\s*([\'"])(?P<xsrf_token>(?:(?!\2).)+)\2',
3171                     webpage, 'xsrf token', group='xsrf_token', fatal=False)
3172
3173         # annotations
3174         if get_annotations:
3175             invideo_url = get_first(
3176                 player_responses,
3177                 ('annotations', 0, 'playerAnnotationsUrlsRenderer', 'invideoUrl'),
3178                 expected_type=str)
3179             if xsrf_token and invideo_url:
3180                 xsrf_field_name = None
3181                 if master_ytcfg:
3182                     xsrf_field_name = try_get(master_ytcfg, lambda x: x['XSRF_FIELD_NAME'], compat_str)
3183                 if not xsrf_field_name:
3184                     xsrf_field_name = self._search_regex(
3185                         r'([\'"])XSRF_FIELD_NAME\1\s*:\s*([\'"])(?P<xsrf_field_name>\w+)\2',
3186                         webpage, 'xsrf field name',
3187                         group='xsrf_field_name', default='session_token')
3188                 info['annotations'] = self._download_webpage(
3189                     self._proto_relative_url(invideo_url),
3190                     video_id, note='Downloading annotations',
3191                     errnote='Unable to download video annotations', fatal=False,
3192                     data=urlencode_postdata({xsrf_field_name: xsrf_token}))
3193
3194         if get_comments:
3195             info['__post_extractor'] = lambda: self._extract_comments(master_ytcfg, video_id, contents, webpage)
3196
3197         self.mark_watched(video_id, player_responses)
3198
3199         return info
3200
3201
3202 class YoutubeTabIE(YoutubeBaseInfoExtractor):
3203     IE_DESC = 'YouTube.com tab'
3204     _VALID_URL = r'''(?x)
3205                     https?://
3206                         (?:\w+\.)?
3207                         (?:
3208                             youtube(?:kids)?\.com|
3209                             invidio\.us
3210                         )/
3211                         (?:
3212                             (?P<channel_type>channel|c|user|browse)/|
3213                             (?P<not_channel>
3214                                 feed/|hashtag/|
3215                                 (?:playlist|watch)\?.*?\blist=
3216                             )|
3217                             (?!(?:%s)\b)  # Direct URLs
3218                         )
3219                         (?P<id>[^/?\#&]+)
3220                     ''' % YoutubeBaseInfoExtractor._RESERVED_NAMES
3221     IE_NAME = 'youtube:tab'
3222
3223     _TESTS = [{
3224         'note': 'playlists, multipage',
3225         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
3226         'playlist_mincount': 94,
3227         'info_dict': {
3228             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
3229             'title': 'Игорь Клейнер - Playlists',
3230             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
3231             'uploader': 'Игорь Клейнер',
3232             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
3233         },
3234     }, {
3235         'note': 'playlists, multipage, different order',
3236         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
3237         'playlist_mincount': 94,
3238         'info_dict': {
3239             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
3240             'title': 'Игорь Клейнер - Playlists',
3241             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
3242             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
3243             'uploader': 'Игорь Клейнер',
3244         },
3245     }, {
3246         'note': 'playlists, series',
3247         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
3248         'playlist_mincount': 5,
3249         'info_dict': {
3250             'id': 'UCYO_jab_esuFRV4b17AJtAw',
3251             'title': '3Blue1Brown - Playlists',
3252             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
3253             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
3254             'uploader': '3Blue1Brown',
3255         },
3256     }, {
3257         'note': 'playlists, singlepage',
3258         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
3259         'playlist_mincount': 4,
3260         'info_dict': {
3261             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
3262             'title': 'ThirstForScience - Playlists',
3263             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
3264             'uploader': 'ThirstForScience',
3265             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
3266         }
3267     }, {
3268         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
3269         'only_matching': True,
3270     }, {
3271         'note': 'basic, single video playlist',
3272         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
3273         'info_dict': {
3274             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
3275             'uploader': 'Sergey M.',
3276             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
3277             'title': 'youtube-dl public playlist',
3278         },
3279         'playlist_count': 1,
3280     }, {
3281         'note': 'empty playlist',
3282         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
3283         'info_dict': {
3284             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
3285             'uploader': 'Sergey M.',
3286             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
3287             'title': 'youtube-dl empty playlist',
3288         },
3289         'playlist_count': 0,
3290     }, {
3291         'note': 'Home tab',
3292         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
3293         'info_dict': {
3294             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3295             'title': 'lex will - Home',
3296             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
3297             'uploader': 'lex will',
3298             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3299         },
3300         'playlist_mincount': 2,
3301     }, {
3302         'note': 'Videos tab',
3303         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
3304         'info_dict': {
3305             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3306             'title': 'lex will - Videos',
3307             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
3308             'uploader': 'lex will',
3309             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3310         },
3311         'playlist_mincount': 975,
3312     }, {
3313         'note': 'Videos tab, sorted by popular',
3314         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
3315         'info_dict': {
3316             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3317             'title': 'lex will - Videos',
3318             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
3319             'uploader': 'lex will',
3320             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3321         },
3322         'playlist_mincount': 199,
3323     }, {
3324         'note': 'Playlists tab',
3325         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
3326         'info_dict': {
3327             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3328             'title': 'lex will - Playlists',
3329             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
3330             'uploader': 'lex will',
3331             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3332         },
3333         'playlist_mincount': 17,
3334     }, {
3335         'note': 'Community tab',
3336         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
3337         'info_dict': {
3338             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3339             'title': 'lex will - Community',
3340             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
3341             'uploader': 'lex will',
3342             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3343         },
3344         'playlist_mincount': 18,
3345     }, {
3346         'note': 'Channels tab',
3347         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
3348         'info_dict': {
3349             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3350             'title': 'lex will - Channels',
3351             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
3352             'uploader': 'lex will',
3353             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
3354         },
3355         'playlist_mincount': 12,
3356     }, {
3357         'note': 'Search tab',
3358         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
3359         'playlist_mincount': 40,
3360         'info_dict': {
3361             'id': 'UCYO_jab_esuFRV4b17AJtAw',
3362             'title': '3Blue1Brown - Search - linear algebra',
3363             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
3364             'uploader': '3Blue1Brown',
3365             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
3366         },
3367     }, {
3368         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
3369         'only_matching': True,
3370     }, {
3371         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
3372         'only_matching': True,
3373     }, {
3374         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
3375         'only_matching': True,
3376     }, {
3377         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
3378         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
3379         'info_dict': {
3380             'title': '29C3: Not my department',
3381             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
3382             'uploader': 'Christiaan008',
3383             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
3384             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
3385         },
3386         'playlist_count': 96,
3387     }, {
3388         'note': 'Large playlist',
3389         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
3390         'info_dict': {
3391             'title': 'Uploads from Cauchemar',
3392             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
3393             'uploader': 'Cauchemar',
3394             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
3395         },
3396         'playlist_mincount': 1123,
3397     }, {
3398         'note': 'even larger playlist, 8832 videos',
3399         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
3400         'only_matching': True,
3401     }, {
3402         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
3403         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
3404         'info_dict': {
3405             'title': 'Uploads from Interstellar Movie',
3406             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
3407             'uploader': 'Interstellar Movie',
3408             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
3409         },
3410         'playlist_mincount': 21,
3411     }, {
3412         'note': 'Playlist with "show unavailable videos" button',
3413         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
3414         'info_dict': {
3415             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
3416             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
3417             'uploader': 'Phim Siêu Nhân Nhật Bản',
3418             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
3419         },
3420         'playlist_mincount': 200,
3421     }, {
3422         'note': 'Playlist with unavailable videos in page 7',
3423         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
3424         'info_dict': {
3425             'title': 'Uploads from BlankTV',
3426             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
3427             'uploader': 'BlankTV',
3428             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
3429         },
3430         'playlist_mincount': 1000,
3431     }, {
3432         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
3433         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
3434         'info_dict': {
3435             'title': 'Data Analysis with Dr Mike Pound',
3436             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
3437             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
3438             'uploader': 'Computerphile',
3439             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
3440         },
3441         'playlist_mincount': 11,
3442     }, {
3443         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
3444         'only_matching': True,
3445     }, {
3446         'note': 'Playlist URL that does not actually serve a playlist',
3447         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
3448         'info_dict': {
3449             'id': 'FqZTN594JQw',
3450             'ext': 'webm',
3451             'title': "Smiley's People 01 detective, Adventure Series, Action",
3452             'uploader': 'STREEM',
3453             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
3454             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
3455             'upload_date': '20150526',
3456             'license': 'Standard YouTube License',
3457             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
3458             'categories': ['People & Blogs'],
3459             'tags': list,
3460             'view_count': int,
3461             'like_count': int,
3462             'dislike_count': int,
3463         },
3464         'params': {
3465             'skip_download': True,
3466         },
3467         'skip': 'This video is not available.',
3468         'add_ie': [YoutubeIE.ie_key()],
3469     }, {
3470         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
3471         'only_matching': True,
3472     }, {
3473         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
3474         'only_matching': True,
3475     }, {
3476         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
3477         'info_dict': {
3478             'id': '3yImotZU3tw',  # This will keep changing
3479             'ext': 'mp4',
3480             'title': compat_str,
3481             'uploader': 'Sky News',
3482             'uploader_id': 'skynews',
3483             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
3484             'upload_date': r're:\d{8}',
3485             'description': compat_str,
3486             'categories': ['News & Politics'],
3487             'tags': list,
3488             'like_count': int,
3489             'dislike_count': int,
3490         },
3491         'params': {
3492             'skip_download': True,
3493         },
3494         'expected_warnings': ['Downloading just video ', 'Ignoring subtitle tracks found in '],
3495     }, {
3496         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
3497         'info_dict': {
3498             'id': 'a48o2S1cPoo',
3499             'ext': 'mp4',
3500             'title': 'The Young Turks - Live Main Show',
3501             'uploader': 'The Young Turks',
3502             'uploader_id': 'TheYoungTurks',
3503             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
3504             'upload_date': '20150715',
3505             'license': 'Standard YouTube License',
3506             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
3507             'categories': ['News & Politics'],
3508             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
3509             'like_count': int,
3510             'dislike_count': int,
3511         },
3512         'params': {
3513             'skip_download': True,
3514         },
3515         'only_matching': True,
3516     }, {
3517         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
3518         'only_matching': True,
3519     }, {
3520         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
3521         'only_matching': True,
3522     }, {
3523         'note': 'A channel that is not live. Should raise error',
3524         'url': 'https://www.youtube.com/user/numberphile/live',
3525         'only_matching': True,
3526     }, {
3527         'url': 'https://www.youtube.com/feed/trending',
3528         'only_matching': True,
3529     }, {
3530         'url': 'https://www.youtube.com/feed/library',
3531         'only_matching': True,
3532     }, {
3533         'url': 'https://www.youtube.com/feed/history',
3534         'only_matching': True,
3535     }, {
3536         'url': 'https://www.youtube.com/feed/subscriptions',
3537         'only_matching': True,
3538     }, {
3539         'url': 'https://www.youtube.com/feed/watch_later',
3540         'only_matching': True,
3541     }, {
3542         'note': 'Recommended - redirects to home page',
3543         'url': 'https://www.youtube.com/feed/recommended',
3544         'only_matching': True,
3545     }, {
3546         'note': 'inline playlist with not always working continuations',
3547         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
3548         'only_matching': True,
3549     }, {
3550         'url': 'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8',
3551         'only_matching': True,
3552     }, {
3553         'url': 'https://www.youtube.com/course',
3554         'only_matching': True,
3555     }, {
3556         'url': 'https://www.youtube.com/zsecurity',
3557         'only_matching': True,
3558     }, {
3559         'url': 'http://www.youtube.com/NASAgovVideo/videos',
3560         'only_matching': True,
3561     }, {
3562         'url': 'https://www.youtube.com/TheYoungTurks/live',
3563         'only_matching': True,
3564     }, {
3565         'url': 'https://www.youtube.com/hashtag/cctv9',
3566         'info_dict': {
3567             'id': 'cctv9',
3568             'title': '#cctv9',
3569         },
3570         'playlist_mincount': 350,
3571     }, {
3572         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
3573         'only_matching': True,
3574     }, {
3575         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
3576         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
3577         'only_matching': True
3578     }, {
3579         'note': '/browse/ should redirect to /channel/',
3580         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
3581         'only_matching': True
3582     }, {
3583         'note': 'VLPL, should redirect to playlist?list=PL...',
3584         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
3585         'info_dict': {
3586             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
3587             'uploader': 'NoCopyrightSounds',
3588             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
3589             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
3590             'title': 'NCS Releases',
3591         },
3592         'playlist_mincount': 166,
3593     }, {
3594         'note': 'Topic, should redirect to playlist?list=UU...',
3595         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
3596         'info_dict': {
3597             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
3598             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
3599             'title': 'Uploads from Royalty Free Music - Topic',
3600             'uploader': 'Royalty Free Music - Topic',
3601         },
3602         'expected_warnings': [
3603             'A channel/user page was given',
3604             'The URL does not have a videos tab',
3605         ],
3606         'playlist_mincount': 101,
3607     }, {
3608         'note': 'Topic without a UU playlist',
3609         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
3610         'info_dict': {
3611             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
3612             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
3613         },
3614         'expected_warnings': [
3615             'A channel/user page was given',
3616             'The URL does not have a videos tab',
3617             'Falling back to channel URL',
3618         ],
3619         'playlist_mincount': 9,
3620     }, {
3621         'note': 'Youtube music Album',
3622         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
3623         'info_dict': {
3624             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
3625             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
3626         },
3627         'playlist_count': 50,
3628     }, {
3629         'note': 'unlisted single video playlist',
3630         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
3631         'info_dict': {
3632             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
3633             'uploader': 'colethedj',
3634             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
3635             'title': 'yt-dlp unlisted playlist test',
3636             'availability': 'unlisted'
3637         },
3638         'playlist_count': 1,
3639     }]
3640
3641     @classmethod
3642     def suitable(cls, url):
3643         return False if YoutubeIE.suitable(url) else super(
3644             YoutubeTabIE, cls).suitable(url)
3645
3646     def _extract_channel_id(self, webpage):
3647         channel_id = self._html_search_meta(
3648             'channelId', webpage, 'channel id', default=None)
3649         if channel_id:
3650             return channel_id
3651         channel_url = self._html_search_meta(
3652             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
3653              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
3654              'twitter:app:url:googleplay'), webpage, 'channel url')
3655         return self._search_regex(
3656             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
3657             channel_url, 'channel id')
3658
3659     @staticmethod
3660     def _extract_basic_item_renderer(item):
3661         # Modified from _extract_grid_item_renderer
3662         known_basic_renderers = (
3663             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer'
3664         )
3665         for key, renderer in item.items():
3666             if not isinstance(renderer, dict):
3667                 continue
3668             elif key in known_basic_renderers:
3669                 return renderer
3670             elif key.startswith('grid') and key.endswith('Renderer'):
3671                 return renderer
3672
3673     def _grid_entries(self, grid_renderer):
3674         for item in grid_renderer['items']:
3675             if not isinstance(item, dict):
3676                 continue
3677             renderer = self._extract_basic_item_renderer(item)
3678             if not isinstance(renderer, dict):
3679                 continue
3680             title = self._get_text(renderer, 'title')
3681
3682             # playlist
3683             playlist_id = renderer.get('playlistId')
3684             if playlist_id:
3685                 yield self.url_result(
3686                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
3687                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
3688                     video_title=title)
3689                 continue
3690             # video
3691             video_id = renderer.get('videoId')
3692             if video_id:
3693                 yield self._extract_video(renderer)
3694                 continue
3695             # channel
3696             channel_id = renderer.get('channelId')
3697             if channel_id:
3698                 yield self.url_result(
3699                     'https://www.youtube.com/channel/%s' % channel_id,
3700                     ie=YoutubeTabIE.ie_key(), video_title=title)
3701                 continue
3702             # generic endpoint URL support
3703             ep_url = urljoin('https://www.youtube.com/', try_get(
3704                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
3705                 compat_str))
3706             if ep_url:
3707                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
3708                     if ie.suitable(ep_url):
3709                         yield self.url_result(
3710                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
3711                         break
3712
3713     def _shelf_entries_from_content(self, shelf_renderer):
3714         content = shelf_renderer.get('content')
3715         if not isinstance(content, dict):
3716             return
3717         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
3718         if renderer:
3719             # TODO: add support for nested playlists so each shelf is processed
3720             # as separate playlist
3721             # TODO: this includes only first N items
3722             for entry in self._grid_entries(renderer):
3723                 yield entry
3724         renderer = content.get('horizontalListRenderer')
3725         if renderer:
3726             # TODO
3727             pass
3728
3729     def _shelf_entries(self, shelf_renderer, skip_channels=False):
3730         ep = try_get(
3731             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
3732             compat_str)
3733         shelf_url = urljoin('https://www.youtube.com', ep)
3734         if shelf_url:
3735             # Skipping links to another channels, note that checking for
3736             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
3737             # will not work
3738             if skip_channels and '/channels?' in shelf_url:
3739                 return
3740             title = self._get_text(shelf_renderer, 'title')
3741             yield self.url_result(shelf_url, video_title=title)
3742         # Shelf may not contain shelf URL, fallback to extraction from content
3743         for entry in self._shelf_entries_from_content(shelf_renderer):
3744             yield entry
3745
3746     def _playlist_entries(self, video_list_renderer):
3747         for content in video_list_renderer['contents']:
3748             if not isinstance(content, dict):
3749                 continue
3750             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
3751             if not isinstance(renderer, dict):
3752                 continue
3753             video_id = renderer.get('videoId')
3754             if not video_id:
3755                 continue
3756             yield self._extract_video(renderer)
3757
3758     def _rich_entries(self, rich_grid_renderer):
3759         renderer = try_get(
3760             rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
3761         video_id = renderer.get('videoId')
3762         if not video_id:
3763             return
3764         yield self._extract_video(renderer)
3765
3766     def _video_entry(self, video_renderer):
3767         video_id = video_renderer.get('videoId')
3768         if video_id:
3769             return self._extract_video(video_renderer)
3770
3771     def _post_thread_entries(self, post_thread_renderer):
3772         post_renderer = try_get(
3773             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
3774         if not post_renderer:
3775             return
3776         # video attachment
3777         video_renderer = try_get(
3778             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
3779         video_id = video_renderer.get('videoId')
3780         if video_id:
3781             entry = self._extract_video(video_renderer)
3782             if entry:
3783                 yield entry
3784         # playlist attachment
3785         playlist_id = try_get(
3786             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], compat_str)
3787         if playlist_id:
3788             yield self.url_result(
3789                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
3790                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
3791         # inline video links
3792         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
3793         for run in runs:
3794             if not isinstance(run, dict):
3795                 continue
3796             ep_url = try_get(
3797                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
3798             if not ep_url:
3799                 continue
3800             if not YoutubeIE.suitable(ep_url):
3801                 continue
3802             ep_video_id = YoutubeIE._match_id(ep_url)
3803             if video_id == ep_video_id:
3804                 continue
3805             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
3806
3807     def _post_thread_continuation_entries(self, post_thread_continuation):
3808         contents = post_thread_continuation.get('contents')
3809         if not isinstance(contents, list):
3810             return
3811         for content in contents:
3812             renderer = content.get('backstagePostThreadRenderer')
3813             if not isinstance(renderer, dict):
3814                 continue
3815             for entry in self._post_thread_entries(renderer):
3816                 yield entry
3817
3818     r''' # unused
3819     def _rich_grid_entries(self, contents):
3820         for content in contents:
3821             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
3822             if video_renderer:
3823                 entry = self._video_entry(video_renderer)
3824                 if entry:
3825                     yield entry
3826     '''
3827     def _entries(self, tab, item_id, identity_token, account_syncid, ytcfg):
3828
3829         def extract_entries(parent_renderer):  # this needs to called again for continuation to work with feeds
3830             contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
3831             for content in contents:
3832                 if not isinstance(content, dict):
3833                     continue
3834                 is_renderer = try_get(content, lambda x: x['itemSectionRenderer'], dict)
3835                 if not is_renderer:
3836                     renderer = content.get('richItemRenderer')
3837                     if renderer:
3838                         for entry in self._rich_entries(renderer):
3839                             yield entry
3840                         continuation_list[0] = self._extract_continuation(parent_renderer)
3841                     continue
3842                 isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
3843                 for isr_content in isr_contents:
3844                     if not isinstance(isr_content, dict):
3845                         continue
3846
3847                     known_renderers = {
3848                         'playlistVideoListRenderer': self._playlist_entries,
3849                         'gridRenderer': self._grid_entries,
3850                         'shelfRenderer': lambda x: self._shelf_entries(x, tab.get('title') != 'Channels'),
3851                         'backstagePostThreadRenderer': self._post_thread_entries,
3852                         'videoRenderer': lambda x: [self._video_entry(x)],
3853                     }
3854                     for key, renderer in isr_content.items():
3855                         if key not in known_renderers:
3856                             continue
3857                         for entry in known_renderers[key](renderer):
3858                             if entry:
3859                                 yield entry
3860                         continuation_list[0] = self._extract_continuation(renderer)
3861                         break
3862
3863                 if not continuation_list[0]:
3864                     continuation_list[0] = self._extract_continuation(is_renderer)
3865
3866             if not continuation_list[0]:
3867                 continuation_list[0] = self._extract_continuation(parent_renderer)
3868
3869         continuation_list = [None]  # Python 2 doesnot support nonlocal
3870         tab_content = try_get(tab, lambda x: x['content'], dict)
3871         if not tab_content:
3872             return
3873         parent_renderer = (
3874             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
3875             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
3876         for entry in extract_entries(parent_renderer):
3877             yield entry
3878         continuation = continuation_list[0]
3879         visitor_data = None
3880
3881         for page_num in itertools.count(1):
3882             if not continuation:
3883                 break
3884             headers = self.generate_api_headers(ytcfg, identity_token, account_syncid, visitor_data)
3885             response = self._extract_response(
3886                 item_id='%s page %s' % (item_id, page_num),
3887                 query=continuation, headers=headers, ytcfg=ytcfg,
3888                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
3889
3890             if not response:
3891                 break
3892             visitor_data = try_get(
3893                 response, lambda x: x['responseContext']['visitorData'], compat_str) or visitor_data
3894
3895             known_continuation_renderers = {
3896                 'playlistVideoListContinuation': self._playlist_entries,
3897                 'gridContinuation': self._grid_entries,
3898                 'itemSectionContinuation': self._post_thread_continuation_entries,
3899                 'sectionListContinuation': extract_entries,  # for feeds
3900             }
3901             continuation_contents = try_get(
3902                 response, lambda x: x['continuationContents'], dict) or {}
3903             continuation_renderer = None
3904             for key, value in continuation_contents.items():
3905                 if key not in known_continuation_renderers:
3906                     continue
3907                 continuation_renderer = value
3908                 continuation_list = [None]
3909                 for entry in known_continuation_renderers[key](continuation_renderer):
3910                     yield entry
3911                 continuation = continuation_list[0] or self._extract_continuation(continuation_renderer)
3912                 break
3913             if continuation_renderer:
3914                 continue
3915
3916             known_renderers = {
3917                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
3918                 'gridVideoRenderer': (self._grid_entries, 'items'),
3919                 'gridChannelRenderer': (self._grid_entries, 'items'),
3920                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
3921                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
3922                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
3923                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents')
3924             }
3925             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
3926             continuation_items = try_get(
3927                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
3928             continuation_item = try_get(continuation_items, lambda x: x[0], dict) or {}
3929             video_items_renderer = None
3930             for key, value in continuation_item.items():
3931                 if key not in known_renderers:
3932                     continue
3933                 video_items_renderer = {known_renderers[key][1]: continuation_items}
3934                 continuation_list = [None]
3935                 for entry in known_renderers[key][0](video_items_renderer):
3936                     yield entry
3937                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
3938                 break
3939             if video_items_renderer:
3940                 continue
3941             break
3942
3943     @staticmethod
3944     def _extract_selected_tab(tabs):
3945         for tab in tabs:
3946             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
3947             if renderer.get('selected') is True:
3948                 return renderer
3949         else:
3950             raise ExtractorError('Unable to find selected tab')
3951
3952     @classmethod
3953     def _extract_uploader(cls, data):
3954         uploader = {}
3955         renderer = cls._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer') or {}
3956         owner = try_get(
3957             renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
3958         if owner:
3959             uploader['uploader'] = owner.get('text')
3960             uploader['uploader_id'] = try_get(
3961                 owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
3962             uploader['uploader_url'] = urljoin(
3963                 'https://www.youtube.com/',
3964                 try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
3965         return {k: v for k, v in uploader.items() if v is not None}
3966
3967     def _extract_from_tabs(self, item_id, webpage, data, tabs):
3968         playlist_id = title = description = channel_url = channel_name = channel_id = None
3969         thumbnails_list = tags = []
3970
3971         selected_tab = self._extract_selected_tab(tabs)
3972         renderer = try_get(
3973             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
3974         if renderer:
3975             channel_name = renderer.get('title')
3976             channel_url = renderer.get('channelUrl')
3977             channel_id = renderer.get('externalId')
3978         else:
3979             renderer = try_get(
3980                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
3981
3982         if renderer:
3983             title = renderer.get('title')
3984             description = renderer.get('description', '')
3985             playlist_id = channel_id
3986             tags = renderer.get('keywords', '').split()
3987             thumbnails_list = (
3988                 try_get(renderer, lambda x: x['avatar']['thumbnails'], list)
3989                 or try_get(
3990                     self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer'),
3991                     lambda x: x['thumbnailRenderer']['playlistVideoThumbnailRenderer']['thumbnail']['thumbnails'],
3992                     list)
3993                 or [])
3994
3995         thumbnails = []
3996         for t in thumbnails_list:
3997             if not isinstance(t, dict):
3998                 continue
3999             thumbnail_url = url_or_none(t.get('url'))
4000             if not thumbnail_url:
4001                 continue
4002             thumbnails.append({
4003                 'url': thumbnail_url,
4004                 'width': int_or_none(t.get('width')),
4005                 'height': int_or_none(t.get('height')),
4006             })
4007         if playlist_id is None:
4008             playlist_id = item_id
4009         if title is None:
4010             title = (
4011                 try_get(data, lambda x: x['header']['hashtagHeaderRenderer']['hashtag']['simpleText'])
4012                 or playlist_id)
4013         title += format_field(selected_tab, 'title', ' - %s')
4014         title += format_field(selected_tab, 'expandedText', ' - %s')
4015         metadata = {
4016             'playlist_id': playlist_id,
4017             'playlist_title': title,
4018             'playlist_description': description,
4019             'uploader': channel_name,
4020             'uploader_id': channel_id,
4021             'uploader_url': channel_url,
4022             'thumbnails': thumbnails,
4023             'tags': tags,
4024         }
4025         availability = self._extract_availability(data)
4026         if availability:
4027             metadata['availability'] = availability
4028         if not channel_id:
4029             metadata.update(self._extract_uploader(data))
4030         metadata.update({
4031             'channel': metadata['uploader'],
4032             'channel_id': metadata['uploader_id'],
4033             'channel_url': metadata['uploader_url']})
4034         ytcfg = self.extract_ytcfg(item_id, webpage)
4035         return self.playlist_result(
4036             self._entries(
4037                 selected_tab, playlist_id,
4038                 self._extract_identity_token(webpage, item_id),
4039                 self._extract_account_syncid(ytcfg, data), ytcfg),
4040             **metadata)
4041
4042     def _extract_mix_playlist(self, playlist, playlist_id, data, webpage):
4043         first_id = last_id = None
4044         ytcfg = self.extract_ytcfg(playlist_id, webpage)
4045         headers = self.generate_api_headers(
4046             ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4047             identity_token=self._extract_identity_token(webpage, item_id=playlist_id))
4048         for page_num in itertools.count(1):
4049             videos = list(self._playlist_entries(playlist))
4050             if not videos:
4051                 return
4052             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4053             if start >= len(videos):
4054                 return
4055             for video in videos[start:]:
4056                 if video['id'] == first_id:
4057                     self.to_screen('First video %s found again; Assuming end of Mix' % first_id)
4058                     return
4059                 yield video
4060             first_id = first_id or videos[0]['id']
4061             last_id = videos[-1]['id']
4062             watch_endpoint = try_get(
4063                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4064             query = {
4065                 'playlistId': playlist_id,
4066                 'videoId': watch_endpoint.get('videoId') or last_id,
4067                 'index': watch_endpoint.get('index') or len(videos),
4068                 'params': watch_endpoint.get('params') or 'OAE%3D'
4069             }
4070             response = self._extract_response(
4071                 item_id='%s page %d' % (playlist_id, page_num),
4072                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4073                 check_get_keys='contents'
4074             )
4075             playlist = try_get(
4076                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4077
4078     def _extract_from_playlist(self, item_id, url, data, playlist, webpage):
4079         title = playlist.get('title') or try_get(
4080             data, lambda x: x['titleText']['simpleText'], compat_str)
4081         playlist_id = playlist.get('playlistId') or item_id
4082
4083         # Delegating everything except mix playlists to regular tab-based playlist URL
4084         playlist_url = urljoin(url, try_get(
4085             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4086             compat_str))
4087         if playlist_url and playlist_url != url:
4088             return self.url_result(
4089                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4090                 video_title=title)
4091
4092         return self.playlist_result(
4093             self._extract_mix_playlist(playlist, playlist_id, data, webpage),
4094             playlist_id=playlist_id, playlist_title=title)
4095
4096     def _extract_availability(self, data):
4097         """
4098         Gets the availability of a given playlist/tab.
4099         Note: Unless YouTube tells us explicitly, we do not assume it is public
4100         @param data: response
4101         """
4102         is_private = is_unlisted = None
4103         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4104         badge_labels = self._extract_badges(renderer)
4105
4106         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4107         privacy_dropdown_entries = try_get(
4108             renderer, lambda x: x['privacyForm']['dropdownFormFieldRenderer']['dropdown']['dropdownRenderer']['entries'], list) or []
4109         for renderer_dict in privacy_dropdown_entries:
4110             is_selected = try_get(
4111                 renderer_dict, lambda x: x['privacyDropdownItemRenderer']['isSelected'], bool) or False
4112             if not is_selected:
4113                 continue
4114             label = self._get_text(renderer_dict, ('privacyDropdownItemRenderer', 'label'))
4115             if label:
4116                 badge_labels.add(label.lower())
4117                 break
4118
4119         for badge_label in badge_labels:
4120             if badge_label == 'unlisted':
4121                 is_unlisted = True
4122             elif badge_label == 'private':
4123                 is_private = True
4124             elif badge_label == 'public':
4125                 is_unlisted = is_private = False
4126         return self._availability(is_private, False, False, False, is_unlisted)
4127
4128     @staticmethod
4129     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
4130         sidebar_renderer = try_get(
4131             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
4132         for item in sidebar_renderer:
4133             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
4134             if renderer:
4135                 return renderer
4136
4137     def _reload_with_unavailable_videos(self, item_id, data, webpage):
4138         """
4139         Get playlist with unavailable videos if the 'show unavailable videos' button exists.
4140         """
4141         browse_id = params = None
4142         renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4143         if not renderer:
4144             return
4145         menu_renderer = try_get(
4146             renderer, lambda x: x['menu']['menuRenderer']['items'], list) or []
4147         for menu_item in menu_renderer:
4148             if not isinstance(menu_item, dict):
4149                 continue
4150             nav_item_renderer = menu_item.get('menuNavigationItemRenderer')
4151             text = try_get(
4152                 nav_item_renderer, lambda x: x['text']['simpleText'], compat_str)
4153             if not text or text.lower() != 'show unavailable videos':
4154                 continue
4155             browse_endpoint = try_get(
4156                 nav_item_renderer, lambda x: x['navigationEndpoint']['browseEndpoint'], dict) or {}
4157             browse_id = browse_endpoint.get('browseId')
4158             params = browse_endpoint.get('params')
4159             break
4160
4161         ytcfg = self.extract_ytcfg(item_id, webpage)
4162         headers = self.generate_api_headers(
4163             ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4164             identity_token=self._extract_identity_token(webpage, item_id=item_id),
4165             visitor_data=try_get(
4166                 self._extract_context(ytcfg), lambda x: x['client']['visitorData'], compat_str))
4167         query = {
4168             'params': params or 'wgYCCAA=',
4169             'browseId': browse_id or 'VL%s' % item_id
4170         }
4171         return self._extract_response(
4172             item_id=item_id, headers=headers, query=query,
4173             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
4174             note='Downloading API JSON with unavailable videos')
4175
4176     def _extract_webpage(self, url, item_id):
4177         retries = self.get_param('extractor_retries', 3)
4178         count = -1
4179         last_error = 'Incomplete yt initial data recieved'
4180         while count < retries:
4181             count += 1
4182             # Sometimes youtube returns a webpage with incomplete ytInitialData
4183             # See: https://github.com/yt-dlp/yt-dlp/issues/116
4184             if count:
4185                 self.report_warning('%s. Retrying ...' % last_error)
4186             webpage = self._download_webpage(
4187                 url, item_id,
4188                 'Downloading webpage%s' % (' (retry #%d)' % count if count else ''))
4189             data = self.extract_yt_initial_data(item_id, webpage)
4190             if data.get('contents') or data.get('currentVideoEndpoint'):
4191                 break
4192             # Extract alerts here only when there is error
4193             self._extract_and_report_alerts(data)
4194             if count >= retries:
4195                 raise ExtractorError(last_error)
4196         return webpage, data
4197
4198     @staticmethod
4199     def _smuggle_data(entries, data):
4200         for entry in entries:
4201             if data:
4202                 entry['url'] = smuggle_url(entry['url'], data)
4203             yield entry
4204
4205     def _real_extract(self, url):
4206         url, smuggled_data = unsmuggle_url(url, {})
4207         if self.is_music_url(url):
4208             smuggled_data['is_music_url'] = True
4209         info_dict = self.__real_extract(url, smuggled_data)
4210         if info_dict.get('entries'):
4211             info_dict['entries'] = self._smuggle_data(info_dict['entries'], smuggled_data)
4212         return info_dict
4213
4214     _url_re = re.compile(r'(?P<pre>%s)(?(channel_type)(?P<tab>/\w+))?(?P<post>.*)$' % _VALID_URL)
4215
4216     def __real_extract(self, url, smuggled_data):
4217         item_id = self._match_id(url)
4218         url = compat_urlparse.urlunparse(
4219             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
4220         compat_opts = self.get_param('compat_opts', [])
4221
4222         def get_mobj(url):
4223             mobj = self._url_re.match(url).groupdict()
4224             mobj.update((k, '') for k, v in mobj.items() if v is None)
4225             return mobj
4226
4227         mobj = get_mobj(url)
4228         # Youtube returns incomplete data if tabname is not lower case
4229         pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
4230
4231         if is_channel:
4232             if smuggled_data.get('is_music_url'):
4233                 if item_id[:2] == 'VL':
4234                     # Youtube music VL channels have an equivalent playlist
4235                     item_id = item_id[2:]
4236                     pre, tab, post, is_channel = 'https://www.youtube.com/playlist?list=%s' % item_id, '', '', False
4237                 elif item_id[:2] == 'MP':
4238                     # Youtube music albums (/channel/MP...) have a OLAK playlist that can be extracted from the webpage
4239                     item_id = self._search_regex(
4240                         r'\\x22audioPlaylistId\\x22:\\x22([0-9A-Za-z_-]+)\\x22',
4241                         self._download_webpage('https://music.youtube.com/channel/%s' % item_id, item_id),
4242                         'playlist id')
4243                     pre, tab, post, is_channel = 'https://www.youtube.com/playlist?list=%s' % item_id, '', '', False
4244                 elif mobj['channel_type'] == 'browse':
4245                     # Youtube music /browse/ should be changed to /channel/
4246                     pre = 'https://www.youtube.com/channel/%s' % item_id
4247         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
4248             # Home URLs should redirect to /videos/
4249             self.report_warning(
4250                 'A channel/user page was given. All the channel\'s videos will be downloaded. '
4251                 'To download only the videos in the home page, add a "/featured" to the URL')
4252             tab = '/videos'
4253
4254         url = ''.join((pre, tab, post))
4255         mobj = get_mobj(url)
4256
4257         # Handle both video/playlist URLs
4258         qs = parse_qs(url)
4259         video_id = qs.get('v', [None])[0]
4260         playlist_id = qs.get('list', [None])[0]
4261
4262         if not video_id and mobj['not_channel'].startswith('watch'):
4263             if not playlist_id:
4264                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
4265                 raise ExtractorError('Unable to recognize tab page')
4266             # Common mistake: https://www.youtube.com/watch?list=playlist_id
4267             self.report_warning('A video URL was given without video ID. Trying to download playlist %s' % playlist_id)
4268             url = 'https://www.youtube.com/playlist?list=%s' % playlist_id
4269             mobj = get_mobj(url)
4270
4271         if video_id and playlist_id:
4272             if self.get_param('noplaylist'):
4273                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
4274                 return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
4275             self.to_screen('Downloading playlist %s; add --no-playlist to just download video %s' % (playlist_id, video_id))
4276
4277         webpage, data = self._extract_webpage(url, item_id)
4278
4279         tabs = try_get(
4280             data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
4281         if tabs:
4282             selected_tab = self._extract_selected_tab(tabs)
4283             tab_name = selected_tab.get('title', '')
4284             if 'no-youtube-channel-redirect' not in compat_opts:
4285                 if mobj['tab'] == '/live':
4286                     # Live tab should have redirected to the video
4287                     raise ExtractorError('The channel is not currently live', expected=True)
4288                 if mobj['tab'] == '/videos' and tab_name.lower() != mobj['tab'][1:]:
4289                     if not mobj['not_channel'] and item_id[:2] == 'UC':
4290                         # Topic channels don't have /videos. Use the equivalent playlist instead
4291                         self.report_warning('The URL does not have a %s tab. Trying to redirect to playlist UU%s instead' % (mobj['tab'][1:], item_id[2:]))
4292                         pl_id = 'UU%s' % item_id[2:]
4293                         pl_url = 'https://www.youtube.com/playlist?list=%s%s' % (pl_id, mobj['post'])
4294                         try:
4295                             pl_webpage, pl_data = self._extract_webpage(pl_url, pl_id)
4296                             for alert_type, alert_message in self._extract_alerts(pl_data):
4297                                 if alert_type == 'error':
4298                                     raise ExtractorError('Youtube said: %s' % alert_message)
4299                             item_id, url, webpage, data = pl_id, pl_url, pl_webpage, pl_data
4300                         except ExtractorError:
4301                             self.report_warning('The playlist gave error. Falling back to channel URL')
4302                     else:
4303                         self.report_warning('The URL does not have a %s tab. %s is being downloaded instead' % (mobj['tab'][1:], tab_name))
4304
4305         self.write_debug('Final URL: %s' % url)
4306
4307         # YouTube sometimes provides a button to reload playlist with unavailable videos.
4308         if 'no-youtube-unavailable-videos' not in compat_opts:
4309             data = self._reload_with_unavailable_videos(item_id, data, webpage) or data
4310         self._extract_and_report_alerts(data)
4311         tabs = try_get(
4312             data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
4313         if tabs:
4314             return self._extract_from_tabs(item_id, webpage, data, tabs)
4315
4316         playlist = try_get(
4317             data, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4318         if playlist:
4319             return self._extract_from_playlist(item_id, url, data, playlist, webpage)
4320
4321         video_id = try_get(
4322             data, lambda x: x['currentVideoEndpoint']['watchEndpoint']['videoId'],
4323             compat_str) or video_id
4324         if video_id:
4325             if mobj['tab'] != '/live':  # live tab is expected to redirect to video
4326                 self.report_warning('Unable to recognize playlist. Downloading just video %s' % video_id)
4327             return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
4328
4329         raise ExtractorError('Unable to recognize tab page')
4330
4331
4332 class YoutubePlaylistIE(InfoExtractor):
4333     IE_DESC = 'YouTube.com playlists'
4334     _VALID_URL = r'''(?x)(?:
4335                         (?:https?://)?
4336                         (?:\w+\.)?
4337                         (?:
4338                             (?:
4339                                 youtube(?:kids)?\.com|
4340                                 invidio\.us
4341                             )
4342                             /.*?\?.*?\blist=
4343                         )?
4344                         (?P<id>%(playlist_id)s)
4345                      )''' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
4346     IE_NAME = 'youtube:playlist'
4347     _TESTS = [{
4348         'note': 'issue #673',
4349         'url': 'PLBB231211A4F62143',
4350         'info_dict': {
4351             'title': '[OLD]Team Fortress 2 (Class-based LP)',
4352             'id': 'PLBB231211A4F62143',
4353             'uploader': 'Wickydoo',
4354             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
4355             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
4356         },
4357         'playlist_mincount': 29,
4358     }, {
4359         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
4360         'info_dict': {
4361             'title': 'YDL_safe_search',
4362             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
4363         },
4364         'playlist_count': 2,
4365         'skip': 'This playlist is private',
4366     }, {
4367         'note': 'embedded',
4368         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
4369         'playlist_count': 4,
4370         'info_dict': {
4371             'title': 'JODA15',
4372             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
4373             'uploader': 'milan',
4374             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
4375         }
4376     }, {
4377         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
4378         'playlist_mincount': 654,
4379         'info_dict': {
4380             'title': '2018 Chinese New Singles (11/6 updated)',
4381             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
4382             'uploader': 'LBK',
4383             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
4384             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
4385         }
4386     }, {
4387         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
4388         'only_matching': True,
4389     }, {
4390         # music album playlist
4391         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
4392         'only_matching': True,
4393     }]
4394
4395     @classmethod
4396     def suitable(cls, url):
4397         if YoutubeTabIE.suitable(url):
4398             return False
4399         # Hack for lazy extractors until more generic solution is implemented
4400         # (see #28780)
4401         from .youtube import parse_qs
4402         qs = parse_qs(url)
4403         if qs.get('v', [None])[0]:
4404             return False
4405         return super(YoutubePlaylistIE, cls).suitable(url)
4406
4407     def _real_extract(self, url):
4408         playlist_id = self._match_id(url)
4409         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
4410         url = update_url_query(
4411             'https://www.youtube.com/playlist',
4412             parse_qs(url) or {'list': playlist_id})
4413         if is_music_url:
4414             url = smuggle_url(url, {'is_music_url': True})
4415         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4416
4417
4418 class YoutubeYtBeIE(InfoExtractor):
4419     IE_DESC = 'youtu.be'
4420     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
4421     _TESTS = [{
4422         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
4423         'info_dict': {
4424             'id': 'yeWKywCrFtk',
4425             'ext': 'mp4',
4426             'title': 'Small Scale Baler and Braiding Rugs',
4427             'uploader': 'Backus-Page House Museum',
4428             'uploader_id': 'backuspagemuseum',
4429             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
4430             'upload_date': '20161008',
4431             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
4432             'categories': ['Nonprofits & Activism'],
4433             'tags': list,
4434             'like_count': int,
4435             'dislike_count': int,
4436         },
4437         'params': {
4438             'noplaylist': True,
4439             'skip_download': True,
4440         },
4441     }, {
4442         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
4443         'only_matching': True,
4444     }]
4445
4446     def _real_extract(self, url):
4447         mobj = re.match(self._VALID_URL, url)
4448         video_id = mobj.group('id')
4449         playlist_id = mobj.group('playlist_id')
4450         return self.url_result(
4451             update_url_query('https://www.youtube.com/watch', {
4452                 'v': video_id,
4453                 'list': playlist_id,
4454                 'feature': 'youtu.be',
4455             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4456
4457
4458 class YoutubeYtUserIE(InfoExtractor):
4459     IE_DESC = 'YouTube.com user videos, URL or "ytuser" keyword'
4460     _VALID_URL = r'ytuser:(?P<id>.+)'
4461     _TESTS = [{
4462         'url': 'ytuser:phihag',
4463         'only_matching': True,
4464     }]
4465
4466     def _real_extract(self, url):
4467         user_id = self._match_id(url)
4468         return self.url_result(
4469             'https://www.youtube.com/user/%s' % user_id,
4470             ie=YoutubeTabIE.ie_key(), video_id=user_id)
4471
4472
4473 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
4474     IE_NAME = 'youtube:favorites'
4475     IE_DESC = 'YouTube.com liked videos, ":ytfav" for short (requires authentication)'
4476     _VALID_URL = r':ytfav(?:ou?rite)?s?'
4477     _LOGIN_REQUIRED = True
4478     _TESTS = [{
4479         'url': ':ytfav',
4480         'only_matching': True,
4481     }, {
4482         'url': ':ytfavorites',
4483         'only_matching': True,
4484     }]
4485
4486     def _real_extract(self, url):
4487         return self.url_result(
4488             'https://www.youtube.com/playlist?list=LL',
4489             ie=YoutubeTabIE.ie_key())
4490
4491
4492 class YoutubeSearchIE(SearchInfoExtractor, YoutubeTabIE):
4493     IE_DESC = 'YouTube.com searches, "ytsearch" keyword'
4494     # there doesn't appear to be a real limit, for example if you search for
4495     # 'python' you get more than 8.000.000 results
4496     _MAX_RESULTS = float('inf')
4497     IE_NAME = 'youtube:search'
4498     _SEARCH_KEY = 'ytsearch'
4499     _SEARCH_PARAMS = None
4500     _TESTS = []
4501
4502     def _entries(self, query, n):
4503         data = {'query': query}
4504         if self._SEARCH_PARAMS:
4505             data['params'] = self._SEARCH_PARAMS
4506         total = 0
4507         continuation = {}
4508         for page_num in itertools.count(1):
4509             data.update(continuation)
4510             search = self._extract_response(
4511                 item_id='query "%s" page %s' % (query, page_num), ep='search', query=data,
4512                 check_get_keys=('contents', 'onResponseReceivedCommands')
4513             )
4514             if not search:
4515                 break
4516             slr_contents = try_get(
4517                 search,
4518                 (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
4519                  lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
4520                 list)
4521             if not slr_contents:
4522                 break
4523
4524             # Youtube sometimes adds promoted content to searches,
4525             # changing the index location of videos and token.
4526             # So we search through all entries till we find them.
4527             continuation = None
4528             for slr_content in slr_contents:
4529                 if not continuation:
4530                     continuation = self._extract_continuation({'contents': [slr_content]})
4531
4532                 isr_contents = try_get(
4533                     slr_content,
4534                     lambda x: x['itemSectionRenderer']['contents'],
4535                     list)
4536                 if not isr_contents:
4537                     continue
4538                 for content in isr_contents:
4539                     if not isinstance(content, dict):
4540                         continue
4541                     video = content.get('videoRenderer')
4542                     if not isinstance(video, dict):
4543                         continue
4544                     video_id = video.get('videoId')
4545                     if not video_id:
4546                         continue
4547
4548                     yield self._extract_video(video)
4549                     total += 1
4550                     if total == n:
4551                         return
4552
4553             if not continuation:
4554                 break
4555
4556     def _get_n_results(self, query, n):
4557         """Get a specified number of results for a query"""
4558         return self.playlist_result(self._entries(query, n), query, query)
4559
4560
4561 class YoutubeSearchDateIE(YoutubeSearchIE):
4562     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
4563     _SEARCH_KEY = 'ytsearchdate'
4564     IE_DESC = 'YouTube.com searches, newest videos first, "ytsearchdate" keyword'
4565     _SEARCH_PARAMS = 'CAI%3D'
4566
4567
4568 class YoutubeSearchURLIE(YoutubeSearchIE):
4569     IE_DESC = 'YouTube.com search URLs'
4570     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
4571     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?:[^&]+)(?:[&]|$)'
4572     # _MAX_RESULTS = 100
4573     _TESTS = [{
4574         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
4575         'playlist_mincount': 5,
4576         'info_dict': {
4577             'id': 'youtube-dl test video',
4578             'title': 'youtube-dl test video',
4579         }
4580     }, {
4581         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
4582         'only_matching': True,
4583     }]
4584
4585     @classmethod
4586     def _make_valid_url(cls):
4587         return cls._VALID_URL
4588
4589     def _real_extract(self, url):
4590         qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
4591         query = (qs.get('search_query') or qs.get('q'))[0]
4592         self._SEARCH_PARAMS = qs.get('sp', ('',))[0]
4593         return self._get_n_results(query, self._MAX_RESULTS)
4594
4595
4596 class YoutubeFeedsInfoExtractor(YoutubeTabIE):
4597     """
4598     Base class for feed extractors
4599     Subclasses must define the _FEED_NAME property.
4600     """
4601     _LOGIN_REQUIRED = True
4602     _TESTS = []
4603
4604     @property
4605     def IE_NAME(self):
4606         return 'youtube:%s' % self._FEED_NAME
4607
4608     def _real_extract(self, url):
4609         return self.url_result(
4610             'https://www.youtube.com/feed/%s' % self._FEED_NAME,
4611             ie=YoutubeTabIE.ie_key())
4612
4613
4614 class YoutubeWatchLaterIE(InfoExtractor):
4615     IE_NAME = 'youtube:watchlater'
4616     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
4617     _VALID_URL = r':ytwatchlater'
4618     _TESTS = [{
4619         'url': ':ytwatchlater',
4620         'only_matching': True,
4621     }]
4622
4623     def _real_extract(self, url):
4624         return self.url_result(
4625             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
4626
4627
4628 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
4629     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
4630     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
4631     _FEED_NAME = 'recommended'
4632     _LOGIN_REQUIRED = False
4633     _TESTS = [{
4634         'url': ':ytrec',
4635         'only_matching': True,
4636     }, {
4637         'url': ':ytrecommended',
4638         'only_matching': True,
4639     }, {
4640         'url': 'https://youtube.com',
4641         'only_matching': True,
4642     }]
4643
4644
4645 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
4646     IE_DESC = 'YouTube.com subscriptions feed, ":ytsubs" for short (requires authentication)'
4647     _VALID_URL = r':ytsub(?:scription)?s?'
4648     _FEED_NAME = 'subscriptions'
4649     _TESTS = [{
4650         'url': ':ytsubs',
4651         'only_matching': True,
4652     }, {
4653         'url': ':ytsubscriptions',
4654         'only_matching': True,
4655     }]
4656
4657
4658 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
4659     IE_DESC = 'Youtube watch history, ":ythis" for short (requires authentication)'
4660     _VALID_URL = r':ythis(?:tory)?'
4661     _FEED_NAME = 'history'
4662     _TESTS = [{
4663         'url': ':ythistory',
4664         'only_matching': True,
4665     }]
4666
4667
4668 class YoutubeTruncatedURLIE(InfoExtractor):
4669     IE_NAME = 'youtube:truncated_url'
4670     IE_DESC = False  # Do not list
4671     _VALID_URL = r'''(?x)
4672         (?:https?://)?
4673         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
4674         (?:watch\?(?:
4675             feature=[a-z_]+|
4676             annotation_id=annotation_[^&]+|
4677             x-yt-cl=[0-9]+|
4678             hl=[^&]*|
4679             t=[0-9]+
4680         )?
4681         |
4682             attribution_link\?a=[^&]+
4683         )
4684         $
4685     '''
4686
4687     _TESTS = [{
4688         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
4689         'only_matching': True,
4690     }, {
4691         'url': 'https://www.youtube.com/watch?',
4692         'only_matching': True,
4693     }, {
4694         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
4695         'only_matching': True,
4696     }, {
4697         'url': 'https://www.youtube.com/watch?feature=foo',
4698         'only_matching': True,
4699     }, {
4700         'url': 'https://www.youtube.com/watch?hl=en-GB',
4701         'only_matching': True,
4702     }, {
4703         'url': 'https://www.youtube.com/watch?t=2372',
4704         'only_matching': True,
4705     }]
4706
4707     def _real_extract(self, url):
4708         raise ExtractorError(
4709             'Did you forget to quote the URL? Remember that & is a meta '
4710             'character in most shells, so you want to put the URL in quotes, '
4711             'like  youtube-dl '
4712             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
4713             ' or simply  youtube-dl BaW_jenozKc  .',
4714             expected=True)
4715
4716
4717 class YoutubeTruncatedIDIE(InfoExtractor):
4718     IE_NAME = 'youtube:truncated_id'
4719     IE_DESC = False  # Do not list
4720     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
4721
4722     _TESTS = [{
4723         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
4724         'only_matching': True,
4725     }]
4726
4727     def _real_extract(self, url):
4728         video_id = self._match_id(url)
4729         raise ExtractorError(
4730             'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
4731             expected=True)