yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import collections
   4 import copy
   5 import datetime
   6 import enum
   7 import hashlib
   8 import itertools
   9 import json
  10 import math
  11 import os.path
  12 import random
  13 import re
  14 import sys
  15 import threading
  16 import time
  17 import traceback
  18 import urllib.error
  19 import urllib.parse
  20
  21 from .common import InfoExtractor, SearchInfoExtractor
  22 from .openload import PhantomJSwrapper
  23 from ..compat import functools
  24 from ..jsinterp import JSInterpreter
  25 from ..utils import (
  26     NO_DEFAULT,
  27     ExtractorError,
  28     LazyList,
  29     UserNotLive,
  30     bug_reports_message,
  31     classproperty,
  32     clean_html,
  33     datetime_from_str,
  34     dict_get,
  35     filter_dict,
  36     float_or_none,
  37     format_field,
  38     get_first,
  39     int_or_none,
  40     is_html,
  41     join_nonempty,
  42     js_to_json,
  43     mimetype2ext,
  44     network_exceptions,
  45     orderedSet,
  46     parse_codecs,
  47     parse_count,
  48     parse_duration,
  49     parse_iso8601,
  50     parse_qs,
  51     qualities,
  52     remove_start,
  53     smuggle_url,
  54     str_or_none,
  55     str_to_int,
  56     strftime_or_none,
  57     traverse_obj,
  58     try_get,
  59     unescapeHTML,
  60     unified_strdate,
  61     unified_timestamp,
  62     unsmuggle_url,
  63     update_url_query,
  64     url_or_none,
  65     urljoin,
  66     variadic,
  67 )
  68
  69 # any clients starting with _ cannot be explicitly requested by the user
  70 INNERTUBE_CLIENTS = {
  71     'web': {
  72         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  73         'INNERTUBE_CONTEXT': {
  74             'client': {
  75                 'clientName': 'WEB',
  76                 'clientVersion': '2.20220801.00.00',
  77             }
  78         },
  79         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  80     },
  81     'web_embedded': {
  82         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  83         'INNERTUBE_CONTEXT': {
  84             'client': {
  85                 'clientName': 'WEB_EMBEDDED_PLAYER',
  86                 'clientVersion': '1.20220731.00.00',
  87             },
  88         },
  89         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  90     },
  91     'web_music': {
  92         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  93         'INNERTUBE_HOST': 'music.youtube.com',
  94         'INNERTUBE_CONTEXT': {
  95             'client': {
  96                 'clientName': 'WEB_REMIX',
  97                 'clientVersion': '1.20220727.01.00',
  98             }
  99         },
 100         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 101     },
 102     'web_creator': {
 103         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 104         'INNERTUBE_CONTEXT': {
 105             'client': {
 106                 'clientName': 'WEB_CREATOR',
 107                 'clientVersion': '1.20220726.00.00',
 108             }
 109         },
 110         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 111     },
 112     'android': {
 113         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 114         'INNERTUBE_CONTEXT': {
 115             'client': {
 116                 'clientName': 'ANDROID',
 117                 'clientVersion': '17.31.35',
 118                 'androidSdkVersion': 30,
 119                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 120             }
 121         },
 122         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 123         'REQUIRE_JS_PLAYER': False
 124     },
 125     'android_embedded': {
 126         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 127         'INNERTUBE_CONTEXT': {
 128             'client': {
 129                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 130                 'clientVersion': '17.31.35',
 131                 'androidSdkVersion': 30,
 132                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 133             },
 134         },
 135         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 136         'REQUIRE_JS_PLAYER': False
 137     },
 138     'android_music': {
 139         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 140         'INNERTUBE_CONTEXT': {
 141             'client': {
 142                 'clientName': 'ANDROID_MUSIC',
 143                 'clientVersion': '5.16.51',
 144                 'androidSdkVersion': 30,
 145                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 146             }
 147         },
 148         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 149         'REQUIRE_JS_PLAYER': False
 150     },
 151     'android_creator': {
 152         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 153         'INNERTUBE_CONTEXT': {
 154             'client': {
 155                 'clientName': 'ANDROID_CREATOR',
 156                 'clientVersion': '22.30.100',
 157                 'androidSdkVersion': 30,
 158                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 159             },
 160         },
 161         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 162         'REQUIRE_JS_PLAYER': False
 163     },
 164     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 165     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 166     'ios': {
 167         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 168         'INNERTUBE_CONTEXT': {
 169             'client': {
 170                 'clientName': 'IOS',
 171                 'clientVersion': '17.33.2',
 172                 'deviceModel': 'iPhone14,3',
 173                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 174             }
 175         },
 176         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 177         'REQUIRE_JS_PLAYER': False
 178     },
 179     'ios_embedded': {
 180         'INNERTUBE_CONTEXT': {
 181             'client': {
 182                 'clientName': 'IOS_MESSAGES_EXTENSION',
 183                 'clientVersion': '17.33.2',
 184                 'deviceModel': 'iPhone14,3',
 185                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 186             },
 187         },
 188         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 189         'REQUIRE_JS_PLAYER': False
 190     },
 191     'ios_music': {
 192         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 193         'INNERTUBE_CONTEXT': {
 194             'client': {
 195                 'clientName': 'IOS_MUSIC',
 196                 'clientVersion': '5.21',
 197                 'deviceModel': 'iPhone14,3',
 198                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 199             },
 200         },
 201         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 202         'REQUIRE_JS_PLAYER': False
 203     },
 204     'ios_creator': {
 205         'INNERTUBE_CONTEXT': {
 206             'client': {
 207                 'clientName': 'IOS_CREATOR',
 208                 'clientVersion': '22.33.101',
 209                 'deviceModel': 'iPhone14,3',
 210                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 211             },
 212         },
 213         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 214         'REQUIRE_JS_PLAYER': False
 215     },
 216     # mweb has 'ultralow' formats
 217     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 218     'mweb': {
 219         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 220         'INNERTUBE_CONTEXT': {
 221             'client': {
 222                 'clientName': 'MWEB',
 223                 'clientVersion': '2.20220801.00.00',
 224             }
 225         },
 226         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 227     },
 228     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 229     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 230     'tv_embedded': {
 231         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 232         'INNERTUBE_CONTEXT': {
 233             'client': {
 234                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 235                 'clientVersion': '2.0',
 236             },
 237         },
 238         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 239     },
 240 }
 241
 242
 243 def _split_innertube_client(client_name):
 244     variant, *base = client_name.rsplit('.', 1)
 245     if base:
 246         return variant, base[0], variant
 247     base, *variant = client_name.split('_', 1)
 248     return client_name, base, variant[0] if variant else None
 249
 250
 251 def build_innertube_clients():
 252     THIRD_PARTY = {
 253         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 254     }
 255     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 256     priority = qualities(BASE_CLIENTS[::-1])
 257
 258     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 259         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 260         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 261         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 262         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 263
 264         _, base_client, variant = _split_innertube_client(client)
 265         ytcfg['priority'] = 10 * priority(base_client)
 266
 267         if not variant:
 268             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 269             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 270             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 271             embedscreen['priority'] -= 3
 272         elif variant == 'embedded':
 273             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 274             ytcfg['priority'] -= 2
 275         else:
 276             ytcfg['priority'] -= 3
 277
 278
 279 build_innertube_clients()
 280
 281
 282 class BadgeType(enum.Enum):
 283     AVAILABILITY_UNLISTED = enum.auto()
 284     AVAILABILITY_PRIVATE = enum.auto()
 285     AVAILABILITY_PUBLIC = enum.auto()
 286     AVAILABILITY_PREMIUM = enum.auto()
 287     AVAILABILITY_SUBSCRIPTION = enum.auto()
 288     LIVE_NOW = enum.auto()
 289
 290
 291 class YoutubeBaseInfoExtractor(InfoExtractor):
 292     """Provide base functions for Youtube extractors"""
 293
 294     _RESERVED_NAMES = (
 295         r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
 296         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 297         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 298         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 299
 300     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 301
 302     # _NETRC_MACHINE = 'youtube'
 303
 304     # If True it will raise an error if no login info is provided
 305     _LOGIN_REQUIRED = False
 306
 307     _INVIDIOUS_SITES = (
 308         # invidious-redirect websites
 309         r'(?:www\.)?redirect\.invidious\.io',
 310         r'(?:(?:www|dev)\.)?invidio\.us',
 311         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 312         r'(?:www\.)?invidious\.pussthecat\.org',
 313         r'(?:www\.)?invidious\.zee\.li',
 314         r'(?:www\.)?invidious\.ethibox\.fr',
 315         r'(?:www\.)?iv\.ggtyler\.dev',
 316         r'(?:www\.)?inv\.vern\.i2p',
 317         r'(?:www\.)?am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd\.onion',
 318         r'(?:www\.)?inv\.riverside\.rocks',
 319         r'(?:www\.)?invidious\.silur\.me',
 320         r'(?:www\.)?inv\.bp\.projectsegfau\.lt',
 321         r'(?:www\.)?invidious\.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid\.onion',
 322         r'(?:www\.)?invidious\.slipfox\.xyz',
 323         r'(?:www\.)?invidious\.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd\.onion',
 324         r'(?:www\.)?inv\.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad\.onion',
 325         r'(?:www\.)?invidious\.tiekoetter\.com',
 326         r'(?:www\.)?iv\.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd\.onion',
 327         r'(?:www\.)?invidious\.nerdvpn\.de',
 328         r'(?:www\.)?invidious\.weblibre\.org',
 329         r'(?:www\.)?inv\.odyssey346\.dev',
 330         r'(?:www\.)?invidious\.dhusch\.de',
 331         r'(?:www\.)?iv\.melmac\.space',
 332         r'(?:www\.)?watch\.thekitty\.zone',
 333         r'(?:www\.)?invidious\.privacydev\.net',
 334         r'(?:www\.)?ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid\.onion',
 335         r'(?:www\.)?invidious\.drivet\.xyz',
 336         r'(?:www\.)?vid\.priv\.au',
 337         r'(?:www\.)?euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd\.onion',
 338         r'(?:www\.)?inv\.vern\.cc',
 339         r'(?:www\.)?invidious\.esmailelbob\.xyz',
 340         r'(?:www\.)?invidious\.sethforprivacy\.com',
 341         r'(?:www\.)?yt\.oelrichsgarcia\.de',
 342         r'(?:www\.)?yt\.artemislena\.eu',
 343         r'(?:www\.)?invidious\.flokinet\.to',
 344         r'(?:www\.)?invidious\.baczek\.me',
 345         r'(?:www\.)?y\.com\.sb',
 346         r'(?:www\.)?invidious\.epicsite\.xyz',
 347         r'(?:www\.)?invidious\.lidarshield\.cloud',
 348         r'(?:www\.)?yt\.funami\.tech',
 349         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 350         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 351         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 352         # youtube-dl invidious instances list
 353         r'(?:(?:www|no)\.)?invidiou\.sh',
 354         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 355         r'(?:www\.)?invidious\.kabi\.tk',
 356         r'(?:www\.)?invidious\.mastodon\.host',
 357         r'(?:www\.)?invidious\.zapashcanon\.fr',
 358         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 359         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 360         r'(?:www\.)?invidious\.himiko\.cloud',
 361         r'(?:www\.)?invidious\.reallyancient\.tech',
 362         r'(?:www\.)?invidious\.tube',
 363         r'(?:www\.)?invidiou\.site',
 364         r'(?:www\.)?invidious\.site',
 365         r'(?:www\.)?invidious\.xyz',
 366         r'(?:www\.)?invidious\.nixnet\.xyz',
 367         r'(?:www\.)?invidious\.048596\.xyz',
 368         r'(?:www\.)?invidious\.drycat\.fr',
 369         r'(?:www\.)?inv\.skyn3t\.in',
 370         r'(?:www\.)?tube\.poal\.co',
 371         r'(?:www\.)?tube\.connect\.cafe',
 372         r'(?:www\.)?vid\.wxzm\.sx',
 373         r'(?:www\.)?vid\.mint\.lgbt',
 374         r'(?:www\.)?vid\.puffyan\.us',
 375         r'(?:www\.)?yewtu\.be',
 376         r'(?:www\.)?yt\.elukerio\.org',
 377         r'(?:www\.)?yt\.lelux\.fi',
 378         r'(?:www\.)?invidious\.ggc-project\.de',
 379         r'(?:www\.)?yt\.maisputain\.ovh',
 380         r'(?:www\.)?ytprivate\.com',
 381         r'(?:www\.)?invidious\.13ad\.de',
 382         r'(?:www\.)?invidious\.toot\.koeln',
 383         r'(?:www\.)?invidious\.fdn\.fr',
 384         r'(?:www\.)?watch\.nettohikari\.com',
 385         r'(?:www\.)?invidious\.namazso\.eu',
 386         r'(?:www\.)?invidious\.silkky\.cloud',
 387         r'(?:www\.)?invidious\.exonip\.de',
 388         r'(?:www\.)?invidious\.riverside\.rocks',
 389         r'(?:www\.)?invidious\.blamefran\.net',
 390         r'(?:www\.)?invidious\.moomoo\.de',
 391         r'(?:www\.)?ytb\.trom\.tf',
 392         r'(?:www\.)?yt\.cyberhost\.uk',
 393         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 394         r'(?:www\.)?qklhadlycap4cnod\.onion',
 395         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 396         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 397         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 398         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 399         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 400         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 401         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 402         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 403         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 404         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 405         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 406         r'(?:www\.)?piped\.kavin\.rocks',
 407         r'(?:www\.)?piped\.tokhmi\.xyz',
 408         r'(?:www\.)?piped\.syncpundit\.io',
 409         r'(?:www\.)?piped\.mha\.fi',
 410         r'(?:www\.)?watch\.whatever\.social',
 411         r'(?:www\.)?piped\.garudalinux\.org',
 412         r'(?:www\.)?piped\.rivo\.lol',
 413         r'(?:www\.)?piped-libre\.kavin\.rocks',
 414         r'(?:www\.)?yt\.jae\.fi',
 415         r'(?:www\.)?piped\.mint\.lgbt',
 416         r'(?:www\.)?il\.ax',
 417         r'(?:www\.)?piped\.esmailelbob\.xyz',
 418         r'(?:www\.)?piped\.projectsegfau\.lt',
 419         r'(?:www\.)?piped\.privacydev\.net',
 420         r'(?:www\.)?piped\.palveluntarjoaja\.eu',
 421         r'(?:www\.)?piped\.smnz\.de',
 422         r'(?:www\.)?piped\.adminforge\.de',
 423         r'(?:www\.)?watch\.whatevertinfoil\.de',
 424         r'(?:www\.)?piped\.qdi\.fi',
 425         r'(?:www\.)?piped\.video',
 426         r'(?:www\.)?piped\.aeong\.one',
 427         r'(?:www\.)?piped\.moomoo\.me',
 428         r'(?:www\.)?piped\.chauvet\.pro',
 429         r'(?:www\.)?watch\.leptons\.xyz',
 430         r'(?:www\.)?pd\.vern\.cc',
 431         r'(?:www\.)?piped\.hostux\.net',
 432         r'(?:www\.)?piped\.lunar\.icu',
 433         # Hyperpipe instances from https://hyperpipe.codeberg.page/
 434         r'(?:www\.)?hyperpipe\.surge\.sh',
 435         r'(?:www\.)?hyperpipe\.esmailelbob\.xyz',
 436         r'(?:www\.)?listen\.whatever\.social',
 437         r'(?:www\.)?music\.adminforge\.de',
 438     )
 439
 440     # extracted from account/account_menu ep
 441     # XXX: These are the supported YouTube UI and API languages,
 442     # which is slightly different from languages supported for translation in YouTube studio
 443     _SUPPORTED_LANG_CODES = [
 444         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 445         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 446         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 447         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 448         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 449         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 450     ]
 451
 452     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 453
 454     @functools.cached_property
 455     def _preferred_lang(self):
 456         """
 457         Returns a language code supported by YouTube for the user preferred language.
 458         Returns None if no preferred language set.
 459         """
 460         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 461         if not preferred_lang:
 462             return
 463         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 464             raise ExtractorError(
 465                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 466                 expected=True)
 467         elif preferred_lang != 'en':
 468             self.report_warning(
 469                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 470         return preferred_lang
 471
 472     def _initialize_consent(self):
 473         cookies = self._get_cookies('https://www.youtube.com/')
 474         if cookies.get('__Secure-3PSID'):
 475             return
 476         consent_id = None
 477         consent = cookies.get('CONSENT')
 478         if consent:
 479             if 'YES' in consent.value:
 480                 return
 481             consent_id = self._search_regex(
 482                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 483         if not consent_id:
 484             consent_id = random.randint(100, 999)
 485         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 486
 487     def _initialize_pref(self):
 488         cookies = self._get_cookies('https://www.youtube.com/')
 489         pref_cookie = cookies.get('PREF')
 490         pref = {}
 491         if pref_cookie:
 492             try:
 493                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 494             except ValueError:
 495                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 496         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 497         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 498
 499     def _real_initialize(self):
 500         self._initialize_pref()
 501         self._initialize_consent()
 502         self._check_login_required()
 503
 504     def _check_login_required(self):
 505         if self._LOGIN_REQUIRED and not self._cookies_passed:
 506             self.raise_login_required('Login details are needed to download this content', method='cookies')
 507
 508     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 509     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 510
 511     def _get_default_ytcfg(self, client='web'):
 512         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 513
 514     def _get_innertube_host(self, client='web'):
 515         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 516
 517     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 518         # try_get but with fallback to default ytcfg client values when present
 519         _func = lambda y: try_get(y, getter, expected_type)
 520         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 521
 522     def _extract_client_name(self, ytcfg, default_client='web'):
 523         return self._ytcfg_get_safe(
 524             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 525                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 526
 527     def _extract_client_version(self, ytcfg, default_client='web'):
 528         return self._ytcfg_get_safe(
 529             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 530                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 531
 532     def _select_api_hostname(self, req_api_hostname, default_client=None):
 533         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 534                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 535
 536     def _extract_api_key(self, ytcfg=None, default_client='web'):
 537         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 538
 539     def _extract_context(self, ytcfg=None, default_client='web'):
 540         context = get_first(
 541             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 542         # Enforce language and tz for extraction
 543         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 544         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 545         return context
 546
 547     _SAPISID = None
 548
 549     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 550         time_now = round(time.time())
 551         if self._SAPISID is None:
 552             yt_cookies = self._get_cookies('https://www.youtube.com')
 553             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 554             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 555             sapisid_cookie = dict_get(
 556                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 557             if sapisid_cookie and sapisid_cookie.value:
 558                 self._SAPISID = sapisid_cookie.value
 559                 self.write_debug('Extracted SAPISID cookie')
 560                 # SAPISID cookie is required if not already present
 561                 if not yt_cookies.get('SAPISID'):
 562                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 563                     self._set_cookie(
 564                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 565             else:
 566                 self._SAPISID = False
 567         if not self._SAPISID:
 568             return None
 569         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 570         sapisidhash = hashlib.sha1(
 571             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 572         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 573
 574     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 575                   note='Downloading API JSON', errnote='Unable to download API page',
 576                   context=None, api_key=None, api_hostname=None, default_client='web'):
 577
 578         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 579         data.update(query)
 580         real_headers = self.generate_api_headers(default_client=default_client)
 581         real_headers.update({'content-type': 'application/json'})
 582         if headers:
 583             real_headers.update(headers)
 584         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 585                    or api_key or self._extract_api_key(default_client=default_client))
 586         return self._download_json(
 587             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 588             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 589             data=json.dumps(data).encode('utf8'), headers=real_headers,
 590             query={'key': api_key, 'prettyPrint': 'false'})
 591
 592     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 593         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 594
 595     @staticmethod
 596     def _extract_session_index(*data):
 597         """
 598         Index of current account in account list.
 599         See: https://github.com/yt-dlp/yt-dlp/pull/519
 600         """
 601         for ytcfg in data:
 602             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 603             if session_index is not None:
 604                 return session_index
 605
 606     # Deprecated?
 607     def _extract_identity_token(self, ytcfg=None, webpage=None):
 608         if ytcfg:
 609             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 610             if token:
 611                 return token
 612         if webpage:
 613             return self._search_regex(
 614                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 615                 'identity token', default=None, fatal=False)
 616
 617     @staticmethod
 618     def _extract_account_syncid(*args):
 619         """
 620         Extract syncId required to download private playlists of secondary channels
 621         @params response and/or ytcfg
 622         """
 623         for data in args:
 624             # ytcfg includes channel_syncid if on secondary channel
 625             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 626             if delegated_sid:
 627                 return delegated_sid
 628             sync_ids = (try_get(
 629                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 630                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 631             if len(sync_ids) >= 2 and sync_ids[1]:
 632                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 633                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 634                 return sync_ids[0]
 635
 636     @staticmethod
 637     def _extract_visitor_data(*args):
 638         """
 639         Extracts visitorData from an API response or ytcfg
 640         Appears to be used to track session state
 641         """
 642         return get_first(
 643             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 644             expected_type=str)
 645
 646     @functools.cached_property
 647     def is_authenticated(self):
 648         return bool(self._generate_sapisidhash_header())
 649
 650     def extract_ytcfg(self, video_id, webpage):
 651         if not webpage:
 652             return {}
 653         return self._parse_json(
 654             self._search_regex(
 655                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 656                 default='{}'), video_id, fatal=False) or {}
 657
 658     def generate_api_headers(
 659             self, *, ytcfg=None, account_syncid=None, session_index=None,
 660             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 661
 662         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 663         headers = {
 664             'X-YouTube-Client-Name': str(
 665                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 666             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 667             'Origin': origin,
 668             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 669             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 670             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 671             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 672         }
 673         if session_index is None:
 674             session_index = self._extract_session_index(ytcfg)
 675         if account_syncid or session_index is not None:
 676             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 677
 678         auth = self._generate_sapisidhash_header(origin)
 679         if auth is not None:
 680             headers['Authorization'] = auth
 681             headers['X-Origin'] = origin
 682         return filter_dict(headers)
 683
 684     def _download_ytcfg(self, client, video_id):
 685         url = {
 686             'web': 'https://www.youtube.com',
 687             'web_music': 'https://music.youtube.com',
 688             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 689         }.get(client)
 690         if not url:
 691             return {}
 692         webpage = self._download_webpage(
 693             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 694         return self.extract_ytcfg(video_id, webpage) or {}
 695
 696     @staticmethod
 697     def _build_api_continuation_query(continuation, ctp=None):
 698         query = {
 699             'continuation': continuation
 700         }
 701         # TODO: Inconsistency with clickTrackingParams.
 702         # Currently we have a fixed ctp contained within context (from ytcfg)
 703         # and a ctp in root query for continuation.
 704         if ctp:
 705             query['clickTracking'] = {'clickTrackingParams': ctp}
 706         return query
 707
 708     @classmethod
 709     def _extract_next_continuation_data(cls, renderer):
 710         next_continuation = try_get(
 711             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 712                        lambda x: x['continuation']['reloadContinuationData']), dict)
 713         if not next_continuation:
 714             return
 715         continuation = next_continuation.get('continuation')
 716         if not continuation:
 717             return
 718         ctp = next_continuation.get('clickTrackingParams')
 719         return cls._build_api_continuation_query(continuation, ctp)
 720
 721     @classmethod
 722     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 723         if isinstance(continuation_ep, dict):
 724             continuation = try_get(
 725                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 726             if not continuation:
 727                 return
 728             ctp = continuation_ep.get('clickTrackingParams')
 729             return cls._build_api_continuation_query(continuation, ctp)
 730
 731     @classmethod
 732     def _extract_continuation(cls, renderer):
 733         next_continuation = cls._extract_next_continuation_data(renderer)
 734         if next_continuation:
 735             return next_continuation
 736
 737         return traverse_obj(renderer, (
 738             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 739             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 740         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 741
 742     @classmethod
 743     def _extract_alerts(cls, data):
 744         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 745             if not isinstance(alert_dict, dict):
 746                 continue
 747             for alert in alert_dict.values():
 748                 alert_type = alert.get('type')
 749                 if not alert_type:
 750                     continue
 751                 message = cls._get_text(alert, 'text')
 752                 if message:
 753                     yield alert_type, message
 754
 755     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 756         errors, warnings = [], []
 757         for alert_type, alert_message in alerts:
 758             if alert_type.lower() == 'error' and fatal:
 759                 errors.append([alert_type, alert_message])
 760             elif alert_message not in self._IGNORED_WARNINGS:
 761                 warnings.append([alert_type, alert_message])
 762
 763         for alert_type, alert_message in (warnings + errors[:-1]):
 764             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 765         if errors:
 766             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 767
 768     def _extract_and_report_alerts(self, data, *args, **kwargs):
 769         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 770
 771     def _extract_badges(self, renderer: dict):
 772         privacy_icon_map = {
 773             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 774             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 775             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 776         }
 777
 778         badge_style_map = {
 779             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 780             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 781             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 782         }
 783
 784         label_map = {
 785             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 786             'private': BadgeType.AVAILABILITY_PRIVATE,
 787             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 788             'live': BadgeType.LIVE_NOW,
 789             'premium': BadgeType.AVAILABILITY_PREMIUM
 790         }
 791
 792         badges = []
 793         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer')):
 794             badge_type = (
 795                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 796                 or badge_style_map.get(traverse_obj(badge, 'style'))
 797             )
 798             if badge_type:
 799                 badges.append({'type': badge_type})
 800                 continue
 801
 802             # fallback, won't work in some languages
 803             label = traverse_obj(badge, 'label', expected_type=str, default='')
 804             for match, label_badge_type in label_map.items():
 805                 if match in label.lower():
 806                     badges.append({'type': badge_type})
 807                     continue
 808
 809         return badges
 810
 811     @staticmethod
 812     def _has_badge(badges, badge_type):
 813         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 814
 815     @staticmethod
 816     def _get_text(data, *path_list, max_runs=None):
 817         for path in path_list or [None]:
 818             if path is None:
 819                 obj = [data]
 820             else:
 821                 obj = traverse_obj(data, path, default=[])
 822                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 823                     obj = [obj]
 824             for item in obj:
 825                 text = try_get(item, lambda x: x['simpleText'], str)
 826                 if text:
 827                     return text
 828                 runs = try_get(item, lambda x: x['runs'], list) or []
 829                 if not runs and isinstance(item, list):
 830                     runs = item
 831
 832                 runs = runs[:min(len(runs), max_runs or len(runs))]
 833                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str))
 834                 if text:
 835                     return text
 836
 837     def _get_count(self, data, *path_list):
 838         count_text = self._get_text(data, *path_list) or ''
 839         count = parse_count(count_text)
 840         if count is None:
 841             count = str_to_int(
 842                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 843         return count
 844
 845     @staticmethod
 846     def _extract_thumbnails(data, *path_list):
 847         """
 848         Extract thumbnails from thumbnails dict
 849         @param path_list: path list to level that contains 'thumbnails' key
 850         """
 851         thumbnails = []
 852         for path in path_list or [()]:
 853             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...)):
 854                 thumbnail_url = url_or_none(thumbnail.get('url'))
 855                 if not thumbnail_url:
 856                     continue
 857                 # Sometimes youtube gives a wrong thumbnail URL. See:
 858                 # https://github.com/yt-dlp/yt-dlp/issues/233
 859                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 860                 if 'maxresdefault' in thumbnail_url:
 861                     thumbnail_url = thumbnail_url.split('?')[0]
 862                 thumbnails.append({
 863                     'url': thumbnail_url,
 864                     'height': int_or_none(thumbnail.get('height')),
 865                     'width': int_or_none(thumbnail.get('width')),
 866                 })
 867         return thumbnails
 868
 869     @staticmethod
 870     def extract_relative_time(relative_time_text):
 871         """
 872         Extracts a relative time from string and converts to dt object
 873         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 874         """
 875         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 876         if mobj:
 877             start = mobj.group('start')
 878             if start:
 879                 return datetime_from_str(start)
 880             try:
 881                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 882             except ValueError:
 883                 return None
 884
 885     def _parse_time_text(self, text):
 886         if not text:
 887             return
 888         dt = self.extract_relative_time(text)
 889         timestamp = None
 890         if isinstance(dt, datetime.datetime):
 891             timestamp = calendar.timegm(dt.timetuple())
 892
 893         if timestamp is None:
 894             timestamp = (
 895                 unified_timestamp(text) or unified_timestamp(
 896                     self._search_regex(
 897                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 898                         text.lower(), 'time text', default=None)))
 899
 900         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 901             self.report_warning(
 902                 f'Cannot parse localized time text "{text}"', only_once=True)
 903         return timestamp
 904
 905     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 906                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 907                           default_client='web'):
 908         for retry in self.RetryManager():
 909             try:
 910                 response = self._call_api(
 911                     ep=ep, fatal=True, headers=headers,
 912                     video_id=item_id, query=query, note=note,
 913                     context=self._extract_context(ytcfg, default_client),
 914                     api_key=self._extract_api_key(ytcfg, default_client),
 915                     api_hostname=api_hostname, default_client=default_client)
 916             except ExtractorError as e:
 917                 if not isinstance(e.cause, network_exceptions):
 918                     return self._error_or_warning(e, fatal=fatal)
 919                 elif not isinstance(e.cause, urllib.error.HTTPError):
 920                     retry.error = e
 921                     continue
 922
 923                 first_bytes = e.cause.read(512)
 924                 if not is_html(first_bytes):
 925                     yt_error = try_get(
 926                         self._parse_json(
 927                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 928                         lambda x: x['error']['message'], str)
 929                     if yt_error:
 930                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 931                 # Downloading page may result in intermittent 5xx HTTP error
 932                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 933                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 934                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 935                 if e.cause.code not in (403, 429):
 936                     retry.error = e
 937                     continue
 938                 return self._error_or_warning(e, fatal=fatal)
 939
 940             try:
 941                 self._extract_and_report_alerts(response, only_once=True)
 942             except ExtractorError as e:
 943                 # YouTube servers may return errors we want to retry on in a 200 OK response
 944                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 945                 if 'unknown error' in e.msg.lower():
 946                     retry.error = e
 947                     continue
 948                 return self._error_or_warning(e, fatal=fatal)
 949             # Youtube sometimes sends incomplete data
 950             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 951             if not traverse_obj(response, *variadic(check_get_keys)):
 952                 retry.error = ExtractorError('Incomplete data received', expected=True)
 953                 continue
 954
 955             return response
 956
 957     @staticmethod
 958     def is_music_url(url):
 959         return re.match(r'(https?://)?music\.youtube\.com/', url) is not None
 960
 961     def _extract_video(self, renderer):
 962         video_id = renderer.get('videoId')
 963
 964         reel_header_renderer = traverse_obj(renderer, (
 965             'navigationEndpoint', 'reelWatchEndpoint', 'overlay', 'reelPlayerOverlayRenderer',
 966             'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
 967
 968         title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
 969         description = self._get_text(renderer, 'descriptionSnippet')
 970
 971         duration = int_or_none(renderer.get('lengthSeconds'))
 972         if duration is None:
 973             duration = parse_duration(self._get_text(
 974                 renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 975         if duration is None:
 976             # XXX: should write a parser to be more general to support more cases (e.g. shorts in shorts tab)
 977             duration = parse_duration(self._search_regex(
 978                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 979                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 980                 video_id, default=None, group='duration'))
 981
 982         channel_id = traverse_obj(
 983             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 984             expected_type=str, get_all=False)
 985         if not channel_id:
 986             channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
 987
 988         overlay_style = traverse_obj(
 989             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 990             get_all=False, expected_type=str)
 991         badges = self._extract_badges(renderer)
 992
 993         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
 994             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
 995             expected_type=str)) or ''
 996         url = f'https://www.youtube.com/watch?v={video_id}'
 997         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
 998             url = f'https://www.youtube.com/shorts/{video_id}'
 999
1000         time_text = (self._get_text(renderer, 'publishedTimeText', 'videoInfo')
1001                      or self._get_text(reel_header_renderer, 'timestampText') or '')
1002         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
1003
1004         live_status = (
1005             'is_upcoming' if scheduled_timestamp is not None
1006             else 'was_live' if 'streamed' in time_text.lower()
1007             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
1008             else None)
1009
1010         # videoInfo is a string like '50K views • 10 years ago'.
1011         view_count_text = self._get_text(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo') or ''
1012         view_count = (0 if 'no views' in view_count_text.lower()
1013                       else self._get_count({'simpleText': view_count_text}))
1014         view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
1015
1016         return {
1017             '_type': 'url',
1018             'ie_key': YoutubeIE.ie_key(),
1019             'id': video_id,
1020             'url': url,
1021             'title': title,
1022             'description': description,
1023             'duration': duration,
1024             'channel_id': channel_id,
1025             'channel': (self._get_text(renderer, 'ownerText', 'shortBylineText')
1026                         or self._get_text(reel_header_renderer, 'channelTitleText')),
1027             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
1028             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
1029             'timestamp': (self._parse_time_text(time_text)
1030                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
1031                           else None),
1032             'release_timestamp': scheduled_timestamp,
1033             'availability':
1034                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
1035                 else self._availability(
1036                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
1037                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
1038                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
1039                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
1040             view_count_field: view_count,
1041             'live_status': live_status
1042         }
1043
1044
1045 class YoutubeIE(YoutubeBaseInfoExtractor):
1046     IE_DESC = 'YouTube'
1047     _VALID_URL = r"""(?x)^
1048                      (
1049                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
1050                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
1051                             (?:www\.)?deturl\.com/www\.youtube\.com|
1052                             (?:www\.)?pwnyoutube\.com|
1053                             (?:www\.)?hooktube\.com|
1054                             (?:www\.)?yourepeat\.com|
1055                             tube\.majestyc\.net|
1056                             %(invidious)s|
1057                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
1058                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
1059                          (?:                                                  # the various things that can precede the ID:
1060                              (?:(?:v|embed|e|shorts|live)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
1061                              |(?:                                             # or the v= param in all its forms
1062                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
1063                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
1064                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
1065                                  v=
1066                              )
1067                          ))
1068                          |(?:
1069                             youtu\.be|                                        # just youtu.be/xxxx
1070                             vid\.plus|                                        # or vid.plus/xxxx
1071                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
1072                             %(invidious)s
1073                          )/
1074                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
1075                          )
1076                      )?                                                       # all until now is optional -> you can pass the naked ID
1077                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
1078                      (?(1).+)?                                                # if we found the ID, everything can follow
1079                      (?:\#|$)""" % {
1080         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1081     }
1082     _EMBED_REGEX = [
1083         r'''(?x)
1084             (?:
1085                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1086                 data-video-url=|
1087                 <embed[^>]+?src=|
1088                 embedSWF\(?:\s*|
1089                 <object[^>]+data=|
1090                 new\s+SWFObject\(
1091             )
1092             (["\'])
1093                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1094                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1095             \1''',
1096         # https://wordpress.org/plugins/lazy-load-for-videos/
1097         r'''(?xs)
1098             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1099             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1100     ]
1101     _RETURN_TYPE = 'video'  # XXX: How to handle multifeed?
1102
1103     _PLAYER_INFO_RE = (
1104         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1105         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1106         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1107     )
1108     _formats = {
1109         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1110         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1111         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1112         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1113         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1114         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1115         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1116         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1117         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1118         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1119         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1120         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1121         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1122         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1123         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1124         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1125         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1126         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1127
1128
1129         # 3D videos
1130         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1131         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1132         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1133         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1134         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1135         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1136         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1137
1138         # Apple HTTP Live Streaming
1139         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1140         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1141         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1142         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1143         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1144         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1145         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1146         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1147
1148         # DASH mp4 video
1149         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1150         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1151         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1152         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1153         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1154         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1155         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1156         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1157         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1158         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1159         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1160         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1161
1162         # Dash mp4 audio
1163         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1164         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1165         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1166         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1167         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1168         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1169         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1170
1171         # Dash webm
1172         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1173         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1174         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1175         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1176         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1177         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1178         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1179         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1180         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1181         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1182         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1183         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1184         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1185         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1186         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1187         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1188         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1189         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1190         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1191         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1192         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1193         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1194
1195         # Dash webm audio
1196         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1197         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1198
1199         # Dash webm audio with opus inside
1200         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1201         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1202         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1203
1204         # RTMP (unnamed)
1205         '_rtmp': {'protocol': 'rtmp'},
1206
1207         # av01 video only formats sometimes served with "unknown" codecs
1208         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1209         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1210         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1211         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1212         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1213         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1214         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1215         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1216     }
1217     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1218
1219     _GEO_BYPASS = False
1220
1221     IE_NAME = 'youtube'
1222     _TESTS = [
1223         {
1224             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1225             'info_dict': {
1226                 'id': 'BaW_jenozKc',
1227                 'ext': 'mp4',
1228                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1229                 'uploader': 'Philipp Hagemeister',
1230                 'uploader_id': 'phihag',
1231                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1232                 'channel': 'Philipp Hagemeister',
1233                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1234                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1235                 'upload_date': '20121002',
1236                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1237                 'categories': ['Science & Technology'],
1238                 'tags': ['youtube-dl'],
1239                 'duration': 10,
1240                 'view_count': int,
1241                 'like_count': int,
1242                 'availability': 'public',
1243                 'playable_in_embed': True,
1244                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1245                 'live_status': 'not_live',
1246                 'age_limit': 0,
1247                 'start_time': 1,
1248                 'end_time': 9,
1249                 'comment_count': int,
1250                 'channel_follower_count': int
1251             }
1252         },
1253         {
1254             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1255             'note': 'Embed-only video (#1746)',
1256             'info_dict': {
1257                 'id': 'yZIXLfi8CZQ',
1258                 'ext': 'mp4',
1259                 'upload_date': '20120608',
1260                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1261                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1262                 'uploader': 'SET India',
1263                 'uploader_id': 'setindia',
1264                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1265                 'age_limit': 18,
1266             },
1267             'skip': 'Private video',
1268         },
1269         {
1270             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1271             'note': 'Use the first video ID in the URL',
1272             'info_dict': {
1273                 'id': 'BaW_jenozKc',
1274                 'ext': 'mp4',
1275                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1276                 'uploader': 'Philipp Hagemeister',
1277                 'uploader_id': 'phihag',
1278                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1279                 'channel': 'Philipp Hagemeister',
1280                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1281                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1282                 'upload_date': '20121002',
1283                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1284                 'categories': ['Science & Technology'],
1285                 'tags': ['youtube-dl'],
1286                 'duration': 10,
1287                 'view_count': int,
1288                 'like_count': int,
1289                 'availability': 'public',
1290                 'playable_in_embed': True,
1291                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1292                 'live_status': 'not_live',
1293                 'age_limit': 0,
1294                 'comment_count': int,
1295                 'channel_follower_count': int
1296             },
1297             'params': {
1298                 'skip_download': True,
1299             },
1300         },
1301         {
1302             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1303             'note': '256k DASH audio (format 141) via DASH manifest',
1304             'info_dict': {
1305                 'id': 'a9LDPn-MO4I',
1306                 'ext': 'm4a',
1307                 'upload_date': '20121002',
1308                 'uploader_id': '8KVIDEO',
1309                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1310                 'description': '',
1311                 'uploader': '8KVIDEO',
1312                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1313             },
1314             'params': {
1315                 'youtube_include_dash_manifest': True,
1316                 'format': '141',
1317             },
1318             'skip': 'format 141 not served anymore',
1319         },
1320         # DASH manifest with encrypted signature
1321         {
1322             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1323             'info_dict': {
1324                 'id': 'IB3lcPjvWLA',
1325                 'ext': 'm4a',
1326                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1327                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1328                 'duration': 244,
1329                 'uploader': 'AfrojackVEVO',
1330                 'uploader_id': 'AfrojackVEVO',
1331                 'upload_date': '20131011',
1332                 'abr': 129.495,
1333                 'like_count': int,
1334                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1335                 'playable_in_embed': True,
1336                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1337                 'view_count': int,
1338                 'track': 'The Spark',
1339                 'live_status': 'not_live',
1340                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1341                 'channel': 'Afrojack',
1342                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1343                 'tags': 'count:19',
1344                 'availability': 'public',
1345                 'categories': ['Music'],
1346                 'age_limit': 0,
1347                 'alt_title': 'The Spark',
1348                 'channel_follower_count': int
1349             },
1350             'params': {
1351                 'youtube_include_dash_manifest': True,
1352                 'format': '141/bestaudio[ext=m4a]',
1353             },
1354         },
1355         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1356         {
1357             'note': 'Embed allowed age-gate video',
1358             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1359             'info_dict': {
1360                 'id': 'HtVdAasjOgU',
1361                 'ext': 'mp4',
1362                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1363                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1364                 'duration': 142,
1365                 'uploader': 'The Witcher',
1366                 'uploader_id': 'WitcherGame',
1367                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1368                 'upload_date': '20140605',
1369                 'age_limit': 18,
1370                 'categories': ['Gaming'],
1371                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1372                 'availability': 'needs_auth',
1373                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1374                 'like_count': int,
1375                 'channel': 'The Witcher',
1376                 'live_status': 'not_live',
1377                 'tags': 'count:17',
1378                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1379                 'playable_in_embed': True,
1380                 'view_count': int,
1381                 'channel_follower_count': int
1382             },
1383         },
1384         {
1385             'note': 'Age-gate video with embed allowed in public site',
1386             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1387             'info_dict': {
1388                 'id': 'HsUATh_Nc2U',
1389                 'ext': 'mp4',
1390                 'title': 'Godzilla 2 (Official Video)',
1391                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1392                 'upload_date': '20200408',
1393                 'uploader_id': 'FlyingKitty900',
1394                 'uploader': 'FlyingKitty',
1395                 'age_limit': 18,
1396                 'availability': 'needs_auth',
1397                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1398                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1399                 'channel': 'FlyingKitty',
1400                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1401                 'view_count': int,
1402                 'categories': ['Entertainment'],
1403                 'live_status': 'not_live',
1404                 'tags': ['Flyingkitty', 'godzilla 2'],
1405                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1406                 'like_count': int,
1407                 'duration': 177,
1408                 'playable_in_embed': True,
1409                 'channel_follower_count': int
1410             },
1411         },
1412         {
1413             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1414             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1415             'info_dict': {
1416                 'id': 'Tq92D6wQ1mg',
1417                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1418                 'ext': 'mp4',
1419                 'upload_date': '20191228',
1420                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1421                 'uploader': 'Projekt Melody',
1422                 'description': 'md5:17eccca93a786d51bc67646756894066',
1423                 'age_limit': 18,
1424                 'like_count': int,
1425                 'availability': 'needs_auth',
1426                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1427                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1428                 'view_count': int,
1429                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1430                 'channel': 'Projekt Melody',
1431                 'live_status': 'not_live',
1432                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1433                 'playable_in_embed': True,
1434                 'categories': ['Entertainment'],
1435                 'duration': 106,
1436                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1437                 'comment_count': int,
1438                 'channel_follower_count': int
1439             },
1440         },
1441         {
1442             'note': 'Non-Agegated non-embeddable video',
1443             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1444             'info_dict': {
1445                 'id': 'MeJVWBSsPAY',
1446                 'ext': 'mp4',
1447                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1448                 'uploader': 'Herr Lurik',
1449                 'uploader_id': 'st3in234',
1450                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1451                 'upload_date': '20130730',
1452                 'track': 'Such mich find mich',
1453                 'age_limit': 0,
1454                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1455                 'like_count': int,
1456                 'playable_in_embed': False,
1457                 'creator': 'OOMPH!',
1458                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1459                 'view_count': int,
1460                 'alt_title': 'Such mich find mich',
1461                 'duration': 210,
1462                 'channel': 'Herr Lurik',
1463                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1464                 'categories': ['Music'],
1465                 'availability': 'public',
1466                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1467                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1468                 'live_status': 'not_live',
1469                 'artist': 'OOMPH!',
1470                 'channel_follower_count': int
1471             },
1472         },
1473         {
1474             'note': 'Non-bypassable age-gated video',
1475             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1476             'only_matching': True,
1477         },
1478         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1479         # YouTube Red ad is not captured for creator
1480         {
1481             'url': '__2ABJjxzNo',
1482             'info_dict': {
1483                 'id': '__2ABJjxzNo',
1484                 'ext': 'mp4',
1485                 'duration': 266,
1486                 'upload_date': '20100430',
1487                 'uploader_id': 'deadmau5',
1488                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1489                 'creator': 'deadmau5',
1490                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1491                 'uploader': 'deadmau5',
1492                 'title': 'Deadmau5 - Some Chords (HD)',
1493                 'alt_title': 'Some Chords',
1494                 'availability': 'public',
1495                 'tags': 'count:14',
1496                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1497                 'view_count': int,
1498                 'live_status': 'not_live',
1499                 'channel': 'deadmau5',
1500                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1501                 'like_count': int,
1502                 'track': 'Some Chords',
1503                 'artist': 'deadmau5',
1504                 'playable_in_embed': True,
1505                 'age_limit': 0,
1506                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1507                 'categories': ['Music'],
1508                 'album': 'Some Chords',
1509                 'channel_follower_count': int
1510             },
1511             'expected_warnings': [
1512                 'DASH manifest missing',
1513             ]
1514         },
1515         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1516         {
1517             'url': 'lqQg6PlCWgI',
1518             'info_dict': {
1519                 'id': 'lqQg6PlCWgI',
1520                 'ext': 'mp4',
1521                 'duration': 6085,
1522                 'upload_date': '20150827',
1523                 'uploader_id': 'olympic',
1524                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1525                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1526                 'uploader': 'Olympics',
1527                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1528                 'like_count': int,
1529                 'release_timestamp': 1343767800,
1530                 'playable_in_embed': True,
1531                 'categories': ['Sports'],
1532                 'release_date': '20120731',
1533                 'channel': 'Olympics',
1534                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1535                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1536                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1537                 'age_limit': 0,
1538                 'availability': 'public',
1539                 'live_status': 'was_live',
1540                 'view_count': int,
1541                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1542                 'channel_follower_count': int
1543             },
1544             'params': {
1545                 'skip_download': 'requires avconv',
1546             }
1547         },
1548         # Non-square pixels
1549         {
1550             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1551             'info_dict': {
1552                 'id': '_b-2C3KPAM0',
1553                 'ext': 'mp4',
1554                 'stretched_ratio': 16 / 9.,
1555                 'duration': 85,
1556                 'upload_date': '20110310',
1557                 'uploader_id': 'AllenMeow',
1558                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1559                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1560                 'uploader': '孫ᄋᄅ',
1561                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1562                 'playable_in_embed': True,
1563                 'channel': '孫ᄋᄅ',
1564                 'age_limit': 0,
1565                 'tags': 'count:11',
1566                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1567                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1568                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1569                 'view_count': int,
1570                 'categories': ['People & Blogs'],
1571                 'like_count': int,
1572                 'live_status': 'not_live',
1573                 'availability': 'unlisted',
1574                 'comment_count': int,
1575                 'channel_follower_count': int
1576             },
1577         },
1578         # url_encoded_fmt_stream_map is empty string
1579         {
1580             'url': 'qEJwOuvDf7I',
1581             'info_dict': {
1582                 'id': 'qEJwOuvDf7I',
1583                 'ext': 'webm',
1584                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1585                 'description': '',
1586                 'upload_date': '20150404',
1587                 'uploader_id': 'spbelect',
1588                 'uploader': 'Наблюдатели Петербурга',
1589             },
1590             'params': {
1591                 'skip_download': 'requires avconv',
1592             },
1593             'skip': 'This live event has ended.',
1594         },
1595         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1596         {
1597             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1598             'info_dict': {
1599                 'id': 'FIl7x6_3R5Y',
1600                 'ext': 'webm',
1601                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1602                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1603                 'duration': 220,
1604                 'upload_date': '20150625',
1605                 'uploader_id': 'dorappi2000',
1606                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1607                 'uploader': 'dorappi2000',
1608                 'formats': 'mincount:31',
1609             },
1610             'skip': 'not actual anymore',
1611         },
1612         # DASH manifest with segment_list
1613         {
1614             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1615             'md5': '8ce563a1d667b599d21064e982ab9e31',
1616             'info_dict': {
1617                 'id': 'CsmdDsKjzN8',
1618                 'ext': 'mp4',
1619                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1620                 'uploader': 'Airtek',
1621                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1622                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1623                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1624             },
1625             'params': {
1626                 'youtube_include_dash_manifest': True,
1627                 'format': '135',  # bestvideo
1628             },
1629             'skip': 'This live event has ended.',
1630         },
1631         {
1632             # Multifeed videos (multiple cameras), URL can be of any Camera
1633             'url': 'https://www.youtube.com/watch?v=zaPI8MvL8pg',
1634             'info_dict': {
1635                 'id': 'zaPI8MvL8pg',
1636                 'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04',
1637                 'description': 'md5:563ccbc698b39298481ca3c571169519',
1638             },
1639             'playlist': [{
1640                 'info_dict': {
1641                     'id': 'j5yGuxZ8lLU',
1642                     'ext': 'mp4',
1643                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Chris)',
1644                     'uploader': 'WiiLikeToPlay',
1645                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1646                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1647                     'duration': 10120,
1648                     'channel_follower_count': int,
1649                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1650                     'availability': 'public',
1651                     'playable_in_embed': True,
1652                     'upload_date': '20131105',
1653                     'uploader_id': 'WiiRikeToPray',
1654                     'categories': ['Gaming'],
1655                     'live_status': 'was_live',
1656                     'tags': 'count:24',
1657                     'release_timestamp': 1383701910,
1658                     'thumbnail': 'https://i.ytimg.com/vi/j5yGuxZ8lLU/maxresdefault.jpg',
1659                     'comment_count': int,
1660                     'age_limit': 0,
1661                     'like_count': int,
1662                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1663                     'channel': 'WiiLikeToPlay',
1664                     'view_count': int,
1665                     'release_date': '20131106',
1666                 },
1667             }, {
1668                 'info_dict': {
1669                     'id': 'zaPI8MvL8pg',
1670                     'ext': 'mp4',
1671                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Tyson)',
1672                     'uploader_id': 'WiiRikeToPray',
1673                     'availability': 'public',
1674                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1675                     'channel': 'WiiLikeToPlay',
1676                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1677                     'channel_follower_count': int,
1678                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1679                     'duration': 10108,
1680                     'age_limit': 0,
1681                     'like_count': int,
1682                     'tags': 'count:24',
1683                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1684                     'uploader': 'WiiLikeToPlay',
1685                     'release_timestamp': 1383701915,
1686                     'comment_count': int,
1687                     'upload_date': '20131105',
1688                     'thumbnail': 'https://i.ytimg.com/vi/zaPI8MvL8pg/maxresdefault.jpg',
1689                     'release_date': '20131106',
1690                     'playable_in_embed': True,
1691                     'live_status': 'was_live',
1692                     'categories': ['Gaming'],
1693                     'view_count': int,
1694                 },
1695             }, {
1696                 'info_dict': {
1697                     'id': 'R7r3vfO7Hao',
1698                     'ext': 'mp4',
1699                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Spencer)',
1700                     'thumbnail': 'https://i.ytimg.com/vi/R7r3vfO7Hao/maxresdefault.jpg',
1701                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1702                     'like_count': int,
1703                     'availability': 'public',
1704                     'playable_in_embed': True,
1705                     'upload_date': '20131105',
1706                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1707                     'uploader_id': 'WiiRikeToPray',
1708                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1709                     'channel_follower_count': int,
1710                     'tags': 'count:24',
1711                     'release_date': '20131106',
1712                     'uploader': 'WiiLikeToPlay',
1713                     'comment_count': int,
1714                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1715                     'channel': 'WiiLikeToPlay',
1716                     'categories': ['Gaming'],
1717                     'release_timestamp': 1383701914,
1718                     'live_status': 'was_live',
1719                     'age_limit': 0,
1720                     'duration': 10128,
1721                     'view_count': int,
1722                 },
1723             }],
1724             'params': {'skip_download': True},
1725         },
1726         {
1727             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1728             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1729             'info_dict': {
1730                 'id': 'gVfLd0zydlo',
1731                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1732             },
1733             'playlist_count': 2,
1734             'skip': 'Not multifeed anymore',
1735         },
1736         {
1737             'url': 'https://vid.plus/FlRa-iH7PGw',
1738             'only_matching': True,
1739         },
1740         {
1741             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1742             'only_matching': True,
1743         },
1744         {
1745             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1746             # Also tests cut-off URL expansion in video description (see
1747             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1748             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1749             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1750             'info_dict': {
1751                 'id': 'lsguqyKfVQg',
1752                 'ext': 'mp4',
1753                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1754                 'alt_title': 'Dark Walk',
1755                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1756                 'duration': 133,
1757                 'upload_date': '20151119',
1758                 'uploader_id': 'IronSoulElf',
1759                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1760                 'uploader': 'IronSoulElf',
1761                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1762                 'track': 'Dark Walk',
1763                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1764                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1765                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1766                 'categories': ['Film & Animation'],
1767                 'view_count': int,
1768                 'live_status': 'not_live',
1769                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1770                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1771                 'tags': 'count:13',
1772                 'availability': 'public',
1773                 'channel': 'IronSoulElf',
1774                 'playable_in_embed': True,
1775                 'like_count': int,
1776                 'age_limit': 0,
1777                 'channel_follower_count': int
1778             },
1779             'params': {
1780                 'skip_download': True,
1781             },
1782         },
1783         {
1784             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1785             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1786             'only_matching': True,
1787         },
1788         {
1789             # Video with yt:stretch=17:0
1790             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1791             'info_dict': {
1792                 'id': 'Q39EVAstoRM',
1793                 'ext': 'mp4',
1794                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1795                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1796                 'upload_date': '20151107',
1797                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1798                 'uploader': 'CH GAMER DROID',
1799             },
1800             'params': {
1801                 'skip_download': True,
1802             },
1803             'skip': 'This video does not exist.',
1804         },
1805         {
1806             # Video with incomplete 'yt:stretch=16:'
1807             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1808             'only_matching': True,
1809         },
1810         {
1811             # Video licensed under Creative Commons
1812             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1813             'info_dict': {
1814                 'id': 'M4gD1WSo5mA',
1815                 'ext': 'mp4',
1816                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1817                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1818                 'duration': 721,
1819                 'upload_date': '20150128',
1820                 'uploader_id': 'BerkmanCenter',
1821                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1822                 'uploader': 'The Berkman Klein Center for Internet & Society',
1823                 'license': 'Creative Commons Attribution license (reuse allowed)',
1824                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1825                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1826                 'like_count': int,
1827                 'age_limit': 0,
1828                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1829                 'channel': 'The Berkman Klein Center for Internet & Society',
1830                 'availability': 'public',
1831                 'view_count': int,
1832                 'categories': ['Education'],
1833                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1834                 'live_status': 'not_live',
1835                 'playable_in_embed': True,
1836                 'comment_count': int,
1837                 'channel_follower_count': int,
1838                 'chapters': list,
1839             },
1840             'params': {
1841                 'skip_download': True,
1842             },
1843         },
1844         {
1845             # Channel-like uploader_url
1846             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1847             'info_dict': {
1848                 'id': 'eQcmzGIKrzg',
1849                 'ext': 'mp4',
1850                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1851                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1852                 'duration': 4060,
1853                 'upload_date': '20151120',
1854                 'uploader': 'Bernie Sanders',
1855                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1856                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1857                 'license': 'Creative Commons Attribution license (reuse allowed)',
1858                 'playable_in_embed': True,
1859                 'tags': 'count:12',
1860                 'like_count': int,
1861                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1862                 'age_limit': 0,
1863                 'availability': 'public',
1864                 'categories': ['News & Politics'],
1865                 'channel': 'Bernie Sanders',
1866                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1867                 'view_count': int,
1868                 'live_status': 'not_live',
1869                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1870                 'comment_count': int,
1871                 'channel_follower_count': int,
1872                 'chapters': list,
1873             },
1874             'params': {
1875                 'skip_download': True,
1876             },
1877         },
1878         {
1879             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1880             'only_matching': True,
1881         },
1882         {
1883             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1884             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1885             'only_matching': True,
1886         },
1887         {
1888             # Rental video preview
1889             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1890             'info_dict': {
1891                 'id': 'uGpuVWrhIzE',
1892                 'ext': 'mp4',
1893                 'title': 'Piku - Trailer',
1894                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1895                 'upload_date': '20150811',
1896                 'uploader': 'FlixMatrix',
1897                 'uploader_id': 'FlixMatrixKaravan',
1898                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1899                 'license': 'Standard YouTube License',
1900             },
1901             'params': {
1902                 'skip_download': True,
1903             },
1904             'skip': 'This video is not available.',
1905         },
1906         {
1907             # YouTube Red video with episode data
1908             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1909             'info_dict': {
1910                 'id': 'iqKdEhx-dD4',
1911                 'ext': 'mp4',
1912                 'title': 'Isolation - Mind Field (Ep 1)',
1913                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1914                 'duration': 2085,
1915                 'upload_date': '20170118',
1916                 'uploader': 'Vsauce',
1917                 'uploader_id': 'Vsauce',
1918                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1919                 'series': 'Mind Field',
1920                 'season_number': 1,
1921                 'episode_number': 1,
1922                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1923                 'tags': 'count:12',
1924                 'view_count': int,
1925                 'availability': 'public',
1926                 'age_limit': 0,
1927                 'channel': 'Vsauce',
1928                 'episode': 'Episode 1',
1929                 'categories': ['Entertainment'],
1930                 'season': 'Season 1',
1931                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1932                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1933                 'like_count': int,
1934                 'playable_in_embed': True,
1935                 'live_status': 'not_live',
1936                 'channel_follower_count': int
1937             },
1938             'params': {
1939                 'skip_download': True,
1940             },
1941             'expected_warnings': [
1942                 'Skipping DASH manifest',
1943             ],
1944         },
1945         {
1946             # The following content has been identified by the YouTube community
1947             # as inappropriate or offensive to some audiences.
1948             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1949             'info_dict': {
1950                 'id': '6SJNVb0GnPI',
1951                 'ext': 'mp4',
1952                 'title': 'Race Differences in Intelligence',
1953                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1954                 'duration': 965,
1955                 'upload_date': '20140124',
1956                 'uploader': 'New Century Foundation',
1957                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1958                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1959             },
1960             'params': {
1961                 'skip_download': True,
1962             },
1963             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1964         },
1965         {
1966             # itag 212
1967             'url': '1t24XAntNCY',
1968             'only_matching': True,
1969         },
1970         {
1971             # geo restricted to JP
1972             'url': 'sJL6WA-aGkQ',
1973             'only_matching': True,
1974         },
1975         {
1976             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1977             'only_matching': True,
1978         },
1979         {
1980             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1981             'only_matching': True,
1982         },
1983         {
1984             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1985             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1986             'only_matching': True,
1987         },
1988         {
1989             # DRM protected
1990             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1991             'only_matching': True,
1992         },
1993         {
1994             # Video with unsupported adaptive stream type formats
1995             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1996             'info_dict': {
1997                 'id': 'Z4Vy8R84T1U',
1998                 'ext': 'mp4',
1999                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
2000                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
2001                 'duration': 433,
2002                 'upload_date': '20130923',
2003                 'uploader': 'Amelia Putri Harwita',
2004                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
2005                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
2006                 'formats': 'maxcount:10',
2007             },
2008             'params': {
2009                 'skip_download': True,
2010                 'youtube_include_dash_manifest': False,
2011             },
2012             'skip': 'not actual anymore',
2013         },
2014         {
2015             # Youtube Music Auto-generated description
2016             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2017             'info_dict': {
2018                 'id': 'MgNrAu2pzNs',
2019                 'ext': 'mp4',
2020                 'title': 'Voyeur Girl',
2021                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
2022                 'upload_date': '20190312',
2023                 'uploader': 'Stephen - Topic',
2024                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
2025                 'artist': 'Stephen',
2026                 'track': 'Voyeur Girl',
2027                 'album': 'it\'s too much love to know my dear',
2028                 'release_date': '20190313',
2029                 'release_year': 2019,
2030                 'alt_title': 'Voyeur Girl',
2031                 'view_count': int,
2032                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
2033                 'playable_in_embed': True,
2034                 'like_count': int,
2035                 'categories': ['Music'],
2036                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
2037                 'channel': 'Stephen',
2038                 'availability': 'public',
2039                 'creator': 'Stephen',
2040                 'duration': 169,
2041                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
2042                 'age_limit': 0,
2043                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
2044                 'tags': 'count:11',
2045                 'live_status': 'not_live',
2046                 'channel_follower_count': int
2047             },
2048             'params': {
2049                 'skip_download': True,
2050             },
2051         },
2052         {
2053             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
2054             'only_matching': True,
2055         },
2056         {
2057             # invalid -> valid video id redirection
2058             'url': 'DJztXj2GPfl',
2059             'info_dict': {
2060                 'id': 'DJztXj2GPfk',
2061                 'ext': 'mp4',
2062                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
2063                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
2064                 'upload_date': '20090125',
2065                 'uploader': 'Prochorowka',
2066                 'uploader_id': 'Prochorowka',
2067                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
2068                 'artist': 'Panjabi MC',
2069                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
2070                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
2071             },
2072             'params': {
2073                 'skip_download': True,
2074             },
2075             'skip': 'Video unavailable',
2076         },
2077         {
2078             # empty description results in an empty string
2079             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
2080             'info_dict': {
2081                 'id': 'x41yOUIvK2k',
2082                 'ext': 'mp4',
2083                 'title': 'IMG 3456',
2084                 'description': '',
2085                 'upload_date': '20170613',
2086                 'uploader_id': 'ElevageOrVert',
2087                 'uploader': 'ElevageOrVert',
2088                 'view_count': int,
2089                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
2090                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
2091                 'like_count': int,
2092                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
2093                 'tags': [],
2094                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
2095                 'availability': 'public',
2096                 'age_limit': 0,
2097                 'categories': ['Pets & Animals'],
2098                 'duration': 7,
2099                 'playable_in_embed': True,
2100                 'live_status': 'not_live',
2101                 'channel': 'ElevageOrVert',
2102                 'channel_follower_count': int
2103             },
2104             'params': {
2105                 'skip_download': True,
2106             },
2107         },
2108         {
2109             # with '};' inside yt initial data (see [1])
2110             # see [2] for an example with '};' inside ytInitialPlayerResponse
2111             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
2112             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2113             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
2114             'info_dict': {
2115                 'id': 'CHqg6qOn4no',
2116                 'ext': 'mp4',
2117                 'title': 'Part 77   Sort a list of simple types in c#',
2118                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2119                 'upload_date': '20130831',
2120                 'uploader_id': 'kudvenkat',
2121                 'uploader': 'kudvenkat',
2122                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2123                 'like_count': int,
2124                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
2125                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2126                 'live_status': 'not_live',
2127                 'categories': ['Education'],
2128                 'availability': 'public',
2129                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2130                 'tags': 'count:12',
2131                 'playable_in_embed': True,
2132                 'age_limit': 0,
2133                 'view_count': int,
2134                 'duration': 522,
2135                 'channel': 'kudvenkat',
2136                 'comment_count': int,
2137                 'channel_follower_count': int,
2138                 'chapters': list,
2139             },
2140             'params': {
2141                 'skip_download': True,
2142             },
2143         },
2144         {
2145             # another example of '};' in ytInitialData
2146             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2147             'only_matching': True,
2148         },
2149         {
2150             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2151             'only_matching': True,
2152         },
2153         {
2154             # https://github.com/ytdl-org/youtube-dl/pull/28094
2155             'url': 'OtqTfy26tG0',
2156             'info_dict': {
2157                 'id': 'OtqTfy26tG0',
2158                 'ext': 'mp4',
2159                 'title': 'Burn Out',
2160                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2161                 'upload_date': '20141120',
2162                 'uploader': 'The Cinematic Orchestra - Topic',
2163                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2164                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2165                 'artist': 'The Cinematic Orchestra',
2166                 'track': 'Burn Out',
2167                 'album': 'Every Day',
2168                 'like_count': int,
2169                 'live_status': 'not_live',
2170                 'alt_title': 'Burn Out',
2171                 'duration': 614,
2172                 'age_limit': 0,
2173                 'view_count': int,
2174                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2175                 'creator': 'The Cinematic Orchestra',
2176                 'channel': 'The Cinematic Orchestra',
2177                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2178                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2179                 'availability': 'public',
2180                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2181                 'categories': ['Music'],
2182                 'playable_in_embed': True,
2183                 'channel_follower_count': int
2184             },
2185             'params': {
2186                 'skip_download': True,
2187             },
2188         },
2189         {
2190             # controversial video, only works with bpctr when authenticated with cookies
2191             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2192             'only_matching': True,
2193         },
2194         {
2195             # controversial video, requires bpctr/contentCheckOk
2196             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2197             'info_dict': {
2198                 'id': 'SZJvDhaSDnc',
2199                 'ext': 'mp4',
2200                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2201                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2202                 'uploader': 'CBS Mornings',
2203                 'uploader_id': 'CBSThisMorning',
2204                 'upload_date': '20140716',
2205                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2206                 'duration': 170,
2207                 'categories': ['News & Politics'],
2208                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2209                 'view_count': int,
2210                 'channel': 'CBS Mornings',
2211                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2212                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2213                 'age_limit': 18,
2214                 'availability': 'needs_auth',
2215                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2216                 'like_count': int,
2217                 'live_status': 'not_live',
2218                 'playable_in_embed': True,
2219                 'channel_follower_count': int
2220             }
2221         },
2222         {
2223             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2224             'url': 'cBvYw8_A0vQ',
2225             'info_dict': {
2226                 'id': 'cBvYw8_A0vQ',
2227                 'ext': 'mp4',
2228                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2229                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2230                 'upload_date': '20201120',
2231                 'uploader': 'Walk around Japan',
2232                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2233                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2234                 'duration': 1456,
2235                 'categories': ['Travel & Events'],
2236                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2237                 'view_count': int,
2238                 'channel': 'Walk around Japan',
2239                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2240                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2241                 'age_limit': 0,
2242                 'availability': 'public',
2243                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2244                 'live_status': 'not_live',
2245                 'playable_in_embed': True,
2246                 'channel_follower_count': int
2247             },
2248             'params': {
2249                 'skip_download': True,
2250             },
2251         }, {
2252             # Has multiple audio streams
2253             'url': 'WaOKSUlf4TM',
2254             'only_matching': True
2255         }, {
2256             # Requires Premium: has format 141 when requested using YTM url
2257             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2258             'only_matching': True
2259         }, {
2260             # multiple subtitles with same lang_code
2261             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2262             'only_matching': True,
2263         }, {
2264             # Force use android client fallback
2265             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2266             'info_dict': {
2267                 'id': 'YOelRv7fMxY',
2268                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2269                 'ext': '3gp',
2270                 'upload_date': '20210624',
2271                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2272                 'uploader': 'colinfurze',
2273                 'uploader_id': 'colinfurze',
2274                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2275                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2276                 'duration': 596,
2277                 'categories': ['Entertainment'],
2278                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2279                 'view_count': int,
2280                 'channel': 'colinfurze',
2281                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2282                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2283                 'age_limit': 0,
2284                 'availability': 'public',
2285                 'like_count': int,
2286                 'live_status': 'not_live',
2287                 'playable_in_embed': True,
2288                 'channel_follower_count': int,
2289                 'chapters': list,
2290             },
2291             'params': {
2292                 'format': '17',  # 3gp format available on android
2293                 'extractor_args': {'youtube': {'player_client': ['android']}},
2294             },
2295         },
2296         {
2297             # Skip download of additional client configs (remix client config in this case)
2298             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2299             'only_matching': True,
2300             'params': {
2301                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2302             },
2303         }, {
2304             # shorts
2305             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2306             'only_matching': True,
2307         }, {
2308             'note': 'Storyboards',
2309             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2310             'info_dict': {
2311                 'id': '5KLPxDtMqe8',
2312                 'ext': 'mhtml',
2313                 'format_id': 'sb0',
2314                 'title': 'Your Brain is Plastic',
2315                 'uploader_id': 'scishow',
2316                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2317                 'upload_date': '20140324',
2318                 'uploader': 'SciShow',
2319                 'like_count': int,
2320                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2321                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2322                 'view_count': int,
2323                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2324                 'playable_in_embed': True,
2325                 'tags': 'count:12',
2326                 'uploader_url': 'http://www.youtube.com/user/scishow',
2327                 'availability': 'public',
2328                 'channel': 'SciShow',
2329                 'live_status': 'not_live',
2330                 'duration': 248,
2331                 'categories': ['Education'],
2332                 'age_limit': 0,
2333                 'channel_follower_count': int,
2334                 'chapters': list,
2335             }, 'params': {'format': 'mhtml', 'skip_download': True}
2336         }, {
2337             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2338             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2339             'info_dict': {
2340                 'id': '2NUZ8W2llS4',
2341                 'ext': 'mp4',
2342                 'title': 'The NP that test your phone performance 🙂',
2343                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2344                 'uploader': 'Leon Nguyen',
2345                 'uploader_id': 'VNSXIII',
2346                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2347                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2348                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2349                 'duration': 21,
2350                 'view_count': int,
2351                 'age_limit': 0,
2352                 'categories': ['Gaming'],
2353                 'tags': 'count:23',
2354                 'playable_in_embed': True,
2355                 'live_status': 'not_live',
2356                 'upload_date': '20220103',
2357                 'like_count': int,
2358                 'availability': 'public',
2359                 'channel': 'Leon Nguyen',
2360                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2361                 'comment_count': int,
2362                 'channel_follower_count': int
2363             }
2364         }, {
2365             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2366             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2367             'info_dict': {
2368                 'id': '2NUZ8W2llS4',
2369                 'ext': 'mp4',
2370                 'title': 'The NP that test your phone performance 🙂',
2371                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2372                 'uploader': 'Leon Nguyen',
2373                 'uploader_id': 'VNSXIII',
2374                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2375                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2376                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2377                 'duration': 21,
2378                 'view_count': int,
2379                 'age_limit': 0,
2380                 'categories': ['Gaming'],
2381                 'tags': 'count:23',
2382                 'playable_in_embed': True,
2383                 'live_status': 'not_live',
2384                 'upload_date': '20220102',
2385                 'like_count': int,
2386                 'availability': 'public',
2387                 'channel': 'Leon Nguyen',
2388                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2389                 'comment_count': int,
2390                 'channel_follower_count': int
2391             },
2392             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2393         }, {
2394             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2395             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2396             'info_dict': {
2397                 'id': 'mzZzzBU6lrM',
2398                 'ext': 'mp4',
2399                 'title': 'I Met GeorgeNotFound In Real Life...',
2400                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2401                 'uploader': 'Quackity',
2402                 'uploader_id': 'QuackityHQ',
2403                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2404                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2405                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2406                 'duration': 955,
2407                 'view_count': int,
2408                 'age_limit': 0,
2409                 'categories': ['Entertainment'],
2410                 'tags': 'count:26',
2411                 'playable_in_embed': True,
2412                 'live_status': 'not_live',
2413                 'release_timestamp': 1641172509,
2414                 'release_date': '20220103',
2415                 'upload_date': '20220103',
2416                 'like_count': int,
2417                 'availability': 'public',
2418                 'channel': 'Quackity',
2419                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2420                 'channel_follower_count': int
2421             }
2422         },
2423         {   # continuous livestream. Microformat upload date should be preferred.
2424             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2425             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2426             'info_dict': {
2427                 'id': 'kgx4WGK0oNU',
2428                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2429                 'ext': 'mp4',
2430                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2431                 'availability': 'public',
2432                 'age_limit': 0,
2433                 'release_timestamp': 1637975704,
2434                 'upload_date': '20210619',
2435                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2436                 'live_status': 'is_live',
2437                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2438                 'uploader': '阿鲍Abao',
2439                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2440                 'channel': 'Abao in Tokyo',
2441                 'channel_follower_count': int,
2442                 'release_date': '20211127',
2443                 'tags': 'count:39',
2444                 'categories': ['People & Blogs'],
2445                 'like_count': int,
2446                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2447                 'view_count': int,
2448                 'playable_in_embed': True,
2449                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2450                 'concurrent_view_count': int,
2451             },
2452             'params': {'skip_download': True}
2453         }, {
2454             # Story. Requires specific player params to work.
2455             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2456             'info_dict': {
2457                 'id': 'vv8qTUWmulI',
2458                 'ext': 'mp4',
2459                 'availability': 'unlisted',
2460                 'view_count': int,
2461                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2462                 'upload_date': '20220526',
2463                 'categories': ['Education'],
2464                 'title': 'Story',
2465                 'channel': 'IT\'S HISTORY',
2466                 'description': '',
2467                 'uploader_id': 'BlastfromthePast',
2468                 'duration': 12,
2469                 'uploader': 'IT\'S HISTORY',
2470                 'playable_in_embed': True,
2471                 'age_limit': 0,
2472                 'live_status': 'not_live',
2473                 'tags': [],
2474                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2475                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2476                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2477             },
2478             'skip': 'stories get removed after some period of time',
2479         }, {
2480             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2481             'info_dict': {
2482                 'id': 'tjjjtzRLHvA',
2483                 'ext': 'mp4',
2484                 'title': 'ハッシュタグ無し };if window.ytcsi',
2485                 'upload_date': '20220323',
2486                 'like_count': int,
2487                 'availability': 'unlisted',
2488                 'channel': 'nao20010128nao',
2489                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2490                 'age_limit': 0,
2491                 'uploader': 'nao20010128nao',
2492                 'uploader_id': 'nao20010128nao',
2493                 'categories': ['Music'],
2494                 'view_count': int,
2495                 'description': '',
2496                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2497                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2498                 'live_status': 'not_live',
2499                 'playable_in_embed': True,
2500                 'channel_follower_count': int,
2501                 'duration': 6,
2502                 'tags': [],
2503                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2504             }
2505         }, {
2506             # Prefer primary title+description language metadata by default
2507             # Do not prefer translated description if primary is empty
2508             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2509             'info_dict': {
2510                 'id': 'el3E4MbxRqQ',
2511                 'ext': 'mp4',
2512                 'title': 'dlp test video 2 - primary sv no desc',
2513                 'description': '',
2514                 'channel': 'cole-dlp-test-acc',
2515                 'tags': [],
2516                 'view_count': int,
2517                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2518                 'like_count': int,
2519                 'playable_in_embed': True,
2520                 'availability': 'unlisted',
2521                 'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
2522                 'age_limit': 0,
2523                 'duration': 5,
2524                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2525                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2526                 'live_status': 'not_live',
2527                 'upload_date': '20220908',
2528                 'categories': ['People & Blogs'],
2529                 'uploader': 'cole-dlp-test-acc',
2530                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2531             },
2532             'params': {'skip_download': True}
2533         }, {
2534             # Extractor argument: prefer translated title+description
2535             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2536             'info_dict': {
2537                 'id': 'gHKT4uU8Zng',
2538                 'ext': 'mp4',
2539                 'channel': 'cole-dlp-test-acc',
2540                 'tags': [],
2541                 'duration': 5,
2542                 'live_status': 'not_live',
2543                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2544                 'upload_date': '20220728',
2545                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2546                 'view_count': int,
2547                 'categories': ['People & Blogs'],
2548                 'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
2549                 'title': 'dlp test video title translated (fr)',
2550                 'availability': 'public',
2551                 'uploader': 'cole-dlp-test-acc',
2552                 'age_limit': 0,
2553                 'description': 'dlp test video description translated (fr)',
2554                 'playable_in_embed': True,
2555                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2556                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2557             },
2558             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2559             'expected_warnings': [r'Preferring "fr" translated fields'],
2560         }, {
2561             'note': '6 channel audio',
2562             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2563             'only_matching': True,
2564         }, {
2565             'note': 'Multiple HLS formats with same itag',
2566             'url': 'https://www.youtube.com/watch?v=kX3nB4PpJko',
2567             'info_dict': {
2568                 'id': 'kX3nB4PpJko',
2569                 'ext': 'mp4',
2570                 'categories': ['Entertainment'],
2571                 'description': 'md5:e8031ff6e426cdb6a77670c9b81f6fa6',
2572                 'uploader_url': 'http://www.youtube.com/user/MrBeast6000',
2573                 'live_status': 'not_live',
2574                 'duration': 937,
2575                 'channel_follower_count': int,
2576                 'thumbnail': 'https://i.ytimg.com/vi_webp/kX3nB4PpJko/maxresdefault.webp',
2577                 'title': 'Last To Take Hand Off Jet, Keeps It!',
2578                 'channel': 'MrBeast',
2579                 'playable_in_embed': True,
2580                 'view_count': int,
2581                 'upload_date': '20221112',
2582                 'uploader': 'MrBeast',
2583                 'uploader_id': 'MrBeast6000',
2584                 'channel_url': 'https://www.youtube.com/channel/UCX6OQ3DkcsbYNE6H8uQQuVA',
2585                 'age_limit': 0,
2586                 'availability': 'public',
2587                 'channel_id': 'UCX6OQ3DkcsbYNE6H8uQQuVA',
2588                 'like_count': int,
2589                 'tags': [],
2590             },
2591             'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
2592         }, {
2593             'note': 'Audio formats with Dynamic Range Compression',
2594             'url': 'https://www.youtube.com/watch?v=Tq92D6wQ1mg',
2595             'info_dict': {
2596                 'id': 'Tq92D6wQ1mg',
2597                 'ext': 'weba',
2598                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
2599                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2600                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2601                 'channel_follower_count': int,
2602                 'description': 'md5:17eccca93a786d51bc67646756894066',
2603                 'upload_date': '20191228',
2604                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2605                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
2606                 'playable_in_embed': True,
2607                 'like_count': int,
2608                 'categories': ['Entertainment'],
2609                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
2610                 'age_limit': 18,
2611                 'channel': 'Projekt Melody',
2612                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2613                 'view_count': int,
2614                 'availability': 'needs_auth',
2615                 'comment_count': int,
2616                 'live_status': 'not_live',
2617                 'uploader': 'Projekt Melody',
2618                 'duration': 106,
2619             },
2620             'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
2621         },
2622         {
2623             'url': 'https://www.youtube.com/live/qVv6vCqciTM',
2624             'info_dict': {
2625                 'id': 'qVv6vCqciTM',
2626                 'ext': 'mp4',
2627                 'age_limit': 0,
2628                 'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
2629                 'comment_count': int,
2630                 'chapters': 'count:13',
2631                 'upload_date': '20221223',
2632                 'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
2633                 'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
2634                 'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
2635                 'like_count': int,
2636                 'release_date': '20221223',
2637                 'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
2638                 'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
2639                 'view_count': int,
2640                 'playable_in_embed': True,
2641                 'duration': 4438,
2642                 'availability': 'public',
2643                 'channel_follower_count': int,
2644                 'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
2645                 'categories': ['Entertainment'],
2646                 'live_status': 'was_live',
2647                 'release_timestamp': 1671793345,
2648                 'channel': 'さなちゃんねる',
2649                 'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
2650                 'uploader': 'さなちゃんねる',
2651             },
2652         },
2653     ]
2654
2655     _WEBPAGE_TESTS = [
2656         # YouTube <object> embed
2657         {
2658             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2659             'md5': '873c81d308b979f0e23ee7e620b312a3',
2660             'info_dict': {
2661                 'id': 'msN87y-iEx0',
2662                 'ext': 'mp4',
2663                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2664                 'upload_date': '20080526',
2665                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2666                 'uploader': 'Christopher Sykes',
2667                 'uploader_id': 'ChristopherJSykes',
2668                 'age_limit': 0,
2669                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2670                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2671                 'playable_in_embed': True,
2672                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2673                 'like_count': int,
2674                 'comment_count': int,
2675                 'channel': 'Christopher Sykes',
2676                 'live_status': 'not_live',
2677                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2678                 'availability': 'public',
2679                 'duration': 195,
2680                 'view_count': int,
2681                 'categories': ['Science & Technology'],
2682                 'channel_follower_count': int,
2683                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2684             },
2685             'params': {
2686                 'skip_download': True,
2687             }
2688         },
2689     ]
2690
2691     @classmethod
2692     def suitable(cls, url):
2693         from ..utils import parse_qs
2694
2695         qs = parse_qs(url)
2696         if qs.get('list', [None])[0]:
2697             return False
2698         return super().suitable(url)
2699
2700     def __init__(self, *args, **kwargs):
2701         super().__init__(*args, **kwargs)
2702         self._code_cache = {}
2703         self._player_cache = {}
2704
2705     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2706         lock = threading.Lock()
2707         start_time = time.time()
2708         formats = [f for f in formats if f.get('is_from_start')]
2709
2710         def refetch_manifest(format_id, delay):
2711             nonlocal formats, start_time, is_live
2712             if time.time() <= start_time + delay:
2713                 return
2714
2715             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2716             video_details = traverse_obj(prs, (..., 'videoDetails'), expected_type=dict)
2717             microformats = traverse_obj(
2718                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2719                 expected_type=dict)
2720             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2721             is_live = live_status == 'is_live'
2722             start_time = time.time()
2723
2724         def mpd_feed(format_id, delay):
2725             """
2726             @returns (manifest_url, manifest_stream_number, is_live) or None
2727             """
2728             for retry in self.RetryManager(fatal=False):
2729                 with lock:
2730                     refetch_manifest(format_id, delay)
2731
2732                 f = next((f for f in formats if f['format_id'] == format_id), None)
2733                 if not f:
2734                     if not is_live:
2735                         retry.error = f'{video_id}: Video is no longer live'
2736                     else:
2737                         retry.error = f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}'
2738                     continue
2739                 return f['manifest_url'], f['manifest_stream_number'], is_live
2740             return None
2741
2742         for f in formats:
2743             f['is_live'] = is_live
2744             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2745                                     live_start_time, mpd_feed, not is_live and f.copy())
2746             if is_live:
2747                 f['fragments'] = gen
2748                 f['protocol'] = 'http_dash_segments_generator'
2749             else:
2750                 f['fragments'] = LazyList(gen({}))
2751                 del f['is_from_start']
2752
2753     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2754         FETCH_SPAN, MAX_DURATION = 5, 432000
2755
2756         mpd_url, stream_number, is_live = None, None, True
2757
2758         begin_index = 0
2759         download_start_time = ctx.get('start') or time.time()
2760
2761         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2762         if lack_early_segments:
2763             self.report_warning(bug_reports_message(
2764                 'Starting download from the last 120 hours of the live stream since '
2765                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2766             lack_early_segments = True
2767
2768         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2769         fragments, fragment_base_url = None, None
2770
2771         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2772             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2773             # Obtain from MPD's maximum seq value
2774             old_mpd_url = mpd_url
2775             last_error = ctx.pop('last_error', None)
2776             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2777             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2778                                                or (mpd_url, stream_number, False))
2779             if not refresh_sequence:
2780                 if expire_fast and not is_live:
2781                     return False, last_seq
2782                 elif old_mpd_url == mpd_url:
2783                     return True, last_seq
2784             if manifestless_orig_fmt:
2785                 fmt_info = manifestless_orig_fmt
2786             else:
2787                 try:
2788                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2789                         mpd_url, None, note=False, errnote=False, fatal=False)
2790                 except ExtractorError:
2791                     fmts = None
2792                 if not fmts:
2793                     no_fragment_score += 2
2794                     return False, last_seq
2795                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2796             fragments = fmt_info['fragments']
2797             fragment_base_url = fmt_info['fragment_base_url']
2798             assert fragment_base_url
2799
2800             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2801             return True, _last_seq
2802
2803         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2804         while is_live:
2805             fetch_time = time.time()
2806             if no_fragment_score > 30:
2807                 return
2808             if last_segment_url:
2809                 # Obtain from "X-Head-Seqnum" header value from each segment
2810                 try:
2811                     urlh = self._request_webpage(
2812                         last_segment_url, None, note=False, errnote=False, fatal=False)
2813                 except ExtractorError:
2814                     urlh = None
2815                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2816                 if last_seq is None:
2817                     no_fragment_score += 2
2818                     last_segment_url = None
2819                     continue
2820             else:
2821                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2822                 no_fragment_score += 2
2823                 if not should_continue:
2824                     continue
2825
2826             if known_idx > last_seq:
2827                 last_segment_url = None
2828                 continue
2829
2830             last_seq += 1
2831
2832             if begin_index < 0 and known_idx < 0:
2833                 # skip from the start when it's negative value
2834                 known_idx = last_seq + begin_index
2835             if lack_early_segments:
2836                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2837             try:
2838                 for idx in range(known_idx, last_seq):
2839                     # do not update sequence here or you'll get skipped some part of it
2840                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2841                     if not should_continue:
2842                         known_idx = idx - 1
2843                         raise ExtractorError('breaking out of outer loop')
2844                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2845                     yield {
2846                         'url': last_segment_url,
2847                         'fragment_count': last_seq,
2848                     }
2849                 if known_idx == last_seq:
2850                     no_fragment_score += 5
2851                 else:
2852                     no_fragment_score = 0
2853                 known_idx = last_seq
2854             except ExtractorError:
2855                 continue
2856
2857             if manifestless_orig_fmt:
2858                 # Stop at the first iteration if running for post-live manifestless;
2859                 # fragment count no longer increase since it starts
2860                 break
2861
2862             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2863
2864     def _extract_player_url(self, *ytcfgs, webpage=None):
2865         player_url = traverse_obj(
2866             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2867             get_all=False, expected_type=str)
2868         if not player_url:
2869             return
2870         return urljoin('https://www.youtube.com', player_url)
2871
2872     def _download_player_url(self, video_id, fatal=False):
2873         res = self._download_webpage(
2874             'https://www.youtube.com/iframe_api',
2875             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2876         if res:
2877             player_version = self._search_regex(
2878                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2879             if player_version:
2880                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2881
2882     def _signature_cache_id(self, example_sig):
2883         """ Return a string representation of a signature """
2884         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2885
2886     @classmethod
2887     def _extract_player_info(cls, player_url):
2888         for player_re in cls._PLAYER_INFO_RE:
2889             id_m = re.search(player_re, player_url)
2890             if id_m:
2891                 break
2892         else:
2893             raise ExtractorError('Cannot identify player %r' % player_url)
2894         return id_m.group('id')
2895
2896     def _load_player(self, video_id, player_url, fatal=True):
2897         player_id = self._extract_player_info(player_url)
2898         if player_id not in self._code_cache:
2899             code = self._download_webpage(
2900                 player_url, video_id, fatal=fatal,
2901                 note='Downloading player ' + player_id,
2902                 errnote='Download of %s failed' % player_url)
2903             if code:
2904                 self._code_cache[player_id] = code
2905         return self._code_cache.get(player_id)
2906
2907     def _extract_signature_function(self, video_id, player_url, example_sig):
2908         player_id = self._extract_player_info(player_url)
2909
2910         # Read from filesystem cache
2911         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2912         assert os.path.basename(func_id) == func_id
2913
2914         self.write_debug(f'Extracting signature function {func_id}')
2915         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2916
2917         if not cache_spec:
2918             code = self._load_player(video_id, player_url)
2919         if code:
2920             res = self._parse_sig_js(code)
2921             test_string = ''.join(map(chr, range(len(example_sig))))
2922             cache_spec = [ord(c) for c in res(test_string)]
2923             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2924
2925         return lambda s: ''.join(s[i] for i in cache_spec)
2926
2927     def _print_sig_code(self, func, example_sig):
2928         if not self.get_param('youtube_print_sig_code'):
2929             return
2930
2931         def gen_sig_code(idxs):
2932             def _genslice(start, end, step):
2933                 starts = '' if start == 0 else str(start)
2934                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2935                 steps = '' if step == 1 else (':%d' % step)
2936                 return f's[{starts}{ends}{steps}]'
2937
2938             step = None
2939             # Quelch pyflakes warnings - start will be set when step is set
2940             start = '(Never used)'
2941             for i, prev in zip(idxs[1:], idxs[:-1]):
2942                 if step is not None:
2943                     if i - prev == step:
2944                         continue
2945                     yield _genslice(start, prev, step)
2946                     step = None
2947                     continue
2948                 if i - prev in [-1, 1]:
2949                     step = i - prev
2950                     start = prev
2951                     continue
2952                 else:
2953                     yield 's[%d]' % prev
2954             if step is None:
2955                 yield 's[%d]' % i
2956             else:
2957                 yield _genslice(start, i, step)
2958
2959         test_string = ''.join(map(chr, range(len(example_sig))))
2960         cache_res = func(test_string)
2961         cache_spec = [ord(c) for c in cache_res]
2962         expr_code = ' + '.join(gen_sig_code(cache_spec))
2963         signature_id_tuple = '(%s)' % (
2964             ', '.join(str(len(p)) for p in example_sig.split('.')))
2965         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2966                 '    return %s\n') % (signature_id_tuple, expr_code)
2967         self.to_screen('Extracted signature function:\n' + code)
2968
2969     def _parse_sig_js(self, jscode):
2970         funcname = self._search_regex(
2971             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2972              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2973              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2974              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2975              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2976              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2977              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2978              # Obsolete patterns
2979              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2980              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2981              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2982              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2983              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2984              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2985              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2986              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2987             jscode, 'Initial JS player signature function name', group='sig')
2988
2989         jsi = JSInterpreter(jscode)
2990         initial_function = jsi.extract_function(funcname)
2991         return lambda s: initial_function([s])
2992
2993     def _cached(self, func, *cache_id):
2994         def inner(*args, **kwargs):
2995             if cache_id not in self._player_cache:
2996                 try:
2997                     self._player_cache[cache_id] = func(*args, **kwargs)
2998                 except ExtractorError as e:
2999                     self._player_cache[cache_id] = e
3000                 except Exception as e:
3001                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
3002
3003             ret = self._player_cache[cache_id]
3004             if isinstance(ret, Exception):
3005                 raise ret
3006             return ret
3007         return inner
3008
3009     def _decrypt_signature(self, s, video_id, player_url):
3010         """Turn the encrypted s field into a working signature"""
3011         extract_sig = self._cached(
3012             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
3013         func = extract_sig(video_id, player_url, s)
3014         self._print_sig_code(func, s)
3015         return func(s)
3016
3017     def _decrypt_nsig(self, s, video_id, player_url):
3018         """Turn the encrypted n field into a working signature"""
3019         if player_url is None:
3020             raise ExtractorError('Cannot decrypt nsig without player_url')
3021         player_url = urljoin('https://www.youtube.com', player_url)
3022
3023         try:
3024             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
3025         except ExtractorError as e:
3026             raise ExtractorError('Unable to extract nsig function code', cause=e)
3027         if self.get_param('youtube_print_sig_code'):
3028             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
3029
3030         try:
3031             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
3032             ret = extract_nsig(jsi, func_code)(s)
3033         except JSInterpreter.Exception as e:
3034             try:
3035                 jsi = PhantomJSwrapper(self, timeout=5000)
3036             except ExtractorError:
3037                 raise e
3038             self.report_warning(
3039                 f'Native nsig extraction failed: Trying with PhantomJS\n'
3040                 f'         n = {s} ; player = {player_url}', video_id)
3041             self.write_debug(e, only_once=True)
3042
3043             args, func_body = func_code
3044             ret = jsi.execute(
3045                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
3046                 video_id=video_id, note='Executing signature code').strip()
3047
3048         self.write_debug(f'Decrypted nsig {s} => {ret}')
3049         return ret
3050
3051     def _extract_n_function_name(self, jscode):
3052         funcname, idx = self._search_regex(
3053             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
3054             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
3055         if not idx:
3056             return funcname
3057
3058         return json.loads(js_to_json(self._search_regex(
3059             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
3060             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
3061
3062     def _extract_n_function_code(self, video_id, player_url):
3063         player_id = self._extract_player_info(player_url)
3064         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
3065         jscode = func_code or self._load_player(video_id, player_url)
3066         jsi = JSInterpreter(jscode)
3067
3068         if func_code:
3069             return jsi, player_id, func_code
3070
3071         func_name = self._extract_n_function_name(jscode)
3072
3073         # For redundancy
3074         func_code = self._search_regex(
3075             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
3076                      # NB: The end of the regex is intentionally kept strict
3077                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
3078             jscode, 'nsig function', group=('var', 'code'), default=None)
3079         if func_code:
3080             func_code = ([func_code[0]], func_code[1])
3081         else:
3082             self.write_debug('Extracting nsig function with jsinterp')
3083             func_code = jsi.extract_function_code(func_name)
3084
3085         self.cache.store('youtube-nsig', player_id, func_code)
3086         return jsi, player_id, func_code
3087
3088     def _extract_n_function_from_code(self, jsi, func_code):
3089         func = jsi.extract_function_from_code(*func_code)
3090
3091         def extract_nsig(s):
3092             try:
3093                 ret = func([s])
3094             except JSInterpreter.Exception:
3095                 raise
3096             except Exception as e:
3097                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
3098
3099             if ret.startswith('enhanced_except_'):
3100                 raise JSInterpreter.Exception('Signature function returned an exception')
3101             return ret
3102
3103         return extract_nsig
3104
3105     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
3106         """
3107         Extract signatureTimestamp (sts)
3108         Required to tell API what sig/player version is in use.
3109         """
3110         sts = None
3111         if isinstance(ytcfg, dict):
3112             sts = int_or_none(ytcfg.get('STS'))
3113
3114         if not sts:
3115             # Attempt to extract from player
3116             if player_url is None:
3117                 error_msg = 'Cannot extract signature timestamp without player_url.'
3118                 if fatal:
3119                     raise ExtractorError(error_msg)
3120                 self.report_warning(error_msg)
3121                 return
3122             code = self._load_player(video_id, player_url, fatal=fatal)
3123             if code:
3124                 sts = int_or_none(self._search_regex(
3125                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
3126                     'JS player signature timestamp', group='sts', fatal=fatal))
3127         return sts
3128
3129     def _mark_watched(self, video_id, player_responses):
3130         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
3131             label = 'fully ' if is_full else ''
3132             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
3133                             expected_type=url_or_none)
3134             if not url:
3135                 self.report_warning(f'Unable to mark {label}watched')
3136                 return
3137             parsed_url = urllib.parse.urlparse(url)
3138             qs = urllib.parse.parse_qs(parsed_url.query)
3139
3140             # cpn generation algorithm is reverse engineered from base.js.
3141             # In fact it works even with dummy cpn.
3142             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
3143             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
3144
3145             # # more consistent results setting it to right before the end
3146             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
3147
3148             qs.update({
3149                 'ver': ['2'],
3150                 'cpn': [cpn],
3151                 'cmt': video_length,
3152                 'el': 'detailpage',  # otherwise defaults to "shorts"
3153             })
3154
3155             if is_full:
3156                 # these seem to mark watchtime "history" in the real world
3157                 # they're required, so send in a single value
3158                 qs.update({
3159                     'st': 0,
3160                     'et': video_length,
3161                 })
3162
3163             url = urllib.parse.urlunparse(
3164                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
3165
3166             self._download_webpage(
3167                 url, video_id, f'Marking {label}watched',
3168                 'Unable to mark watched', fatal=False)
3169
3170     @classmethod
3171     def _extract_from_webpage(cls, url, webpage):
3172         # Invidious Instances
3173         # https://github.com/yt-dlp/yt-dlp/issues/195
3174         # https://github.com/iv-org/invidious/pull/1730
3175         mobj = re.search(
3176             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
3177             webpage)
3178         if mobj:
3179             yield cls.url_result(mobj.group('url'), cls)
3180             raise cls.StopExtraction()
3181
3182         yield from super()._extract_from_webpage(url, webpage)
3183
3184         # lazyYT YouTube embed
3185         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
3186             yield cls.url_result(unescapeHTML(id_), cls, id_)
3187
3188         # Wordpress "YouTube Video Importer" plugin
3189         for m in re.findall(r'''(?x)<div[^>]+
3190                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
3191                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
3192             yield cls.url_result(m[-1], cls, m[-1])
3193
3194     @classmethod
3195     def extract_id(cls, url):
3196         video_id = cls.get_temp_id(url)
3197         if not video_id:
3198             raise ExtractorError(f'Invalid URL: {url}')
3199         return video_id
3200
3201     def _extract_chapters_from_json(self, data, duration):
3202         chapter_list = traverse_obj(
3203             data, (
3204                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
3205                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3206             ), expected_type=list)
3207
3208         return self._extract_chapters(
3209             chapter_list,
3210             chapter_time=lambda chapter: float_or_none(
3211                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3212             chapter_title=lambda chapter: traverse_obj(
3213                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3214             duration=duration)
3215
3216     def _extract_chapters_from_engagement_panel(self, data, duration):
3217         content_list = traverse_obj(
3218             data,
3219             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3220             expected_type=list)
3221         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3222         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3223
3224         return next(filter(None, (
3225             self._extract_chapters(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3226                                    chapter_time, chapter_title, duration)
3227             for contents in content_list)), [])
3228
3229     def _extract_chapters_from_description(self, description, duration):
3230         duration_re = r'(?:\d+:)?\d{1,2}:\d{2}'
3231         sep_re = r'(?m)^\s*(%s)\b\W*\s(%s)\s*$'
3232         return self._extract_chapters(
3233             re.findall(sep_re % (duration_re, r'.+?'), description or ''),
3234             chapter_time=lambda x: parse_duration(x[0]), chapter_title=lambda x: x[1],
3235             duration=duration, strict=False) or self._extract_chapters(
3236             re.findall(sep_re % (r'.+?', duration_re), description or ''),
3237             chapter_time=lambda x: parse_duration(x[1]), chapter_title=lambda x: x[0],
3238             duration=duration, strict=False)
3239
3240     def _extract_chapters(self, chapter_list, chapter_time, chapter_title, duration, strict=True):
3241         if not duration:
3242             return
3243         chapter_list = [{
3244             'start_time': chapter_time(chapter),
3245             'title': chapter_title(chapter),
3246         } for chapter in chapter_list or []]
3247         if not strict:
3248             chapter_list.sort(key=lambda c: c['start_time'] or 0)
3249
3250         chapters = [{'start_time': 0}]
3251         for idx, chapter in enumerate(chapter_list):
3252             if chapter['start_time'] is None:
3253                 self.report_warning(f'Incomplete chapter {idx}')
3254             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
3255                 chapters.append(chapter)
3256             elif chapter not in chapters:
3257                 self.report_warning(
3258                     f'Invalid start time ({chapter["start_time"]} < {chapters[-1]["start_time"]}) for chapter "{chapter["title"]}"')
3259         return chapters[1:]
3260
3261     def _extract_comment(self, comment_renderer, parent=None):
3262         comment_id = comment_renderer.get('commentId')
3263         if not comment_id:
3264             return
3265
3266         text = self._get_text(comment_renderer, 'contentText')
3267
3268         # Timestamp is an estimate calculated from the current time and time_text
3269         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3270         timestamp = self._parse_time_text(time_text)
3271
3272         author = self._get_text(comment_renderer, 'authorText')
3273         author_id = try_get(comment_renderer,
3274                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3275
3276         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3277                                                        lambda x: x['likeCount']), str)) or 0
3278         author_thumbnail = try_get(comment_renderer,
3279                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3280
3281         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3282         is_favorited = 'creatorHeart' in (try_get(
3283             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3284         return {
3285             'id': comment_id,
3286             'text': text,
3287             'timestamp': timestamp,
3288             'time_text': time_text,
3289             'like_count': votes,
3290             'is_favorited': is_favorited,
3291             'author': author,
3292             'author_id': author_id,
3293             'author_thumbnail': author_thumbnail,
3294             'author_is_uploader': author_is_uploader,
3295             'parent': parent or 'root'
3296         }
3297
3298     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3299
3300         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3301
3302         def extract_header(contents):
3303             _continuation = None
3304             for content in contents:
3305                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3306                 expected_comment_count = self._get_count(
3307                     comments_header_renderer, 'countText', 'commentsCount')
3308
3309                 if expected_comment_count:
3310                     tracker['est_total'] = expected_comment_count
3311                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3312                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3313
3314                 sort_menu_item = try_get(
3315                     comments_header_renderer,
3316                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3317                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3318
3319                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3320                 if not _continuation:
3321                     continue
3322
3323                 sort_text = str_or_none(sort_menu_item.get('title'))
3324                 if not sort_text:
3325                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3326                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3327                 break
3328             return _continuation
3329
3330         def extract_thread(contents):
3331             if not parent:
3332                 tracker['current_page_thread'] = 0
3333             for content in contents:
3334                 if not parent and tracker['total_parent_comments'] >= max_parents:
3335                     yield
3336                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3337                 comment_renderer = get_first(
3338                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3339                     expected_type=dict, default={})
3340
3341                 comment = self._extract_comment(comment_renderer, parent)
3342                 if not comment:
3343                     continue
3344                 # Sometimes YouTube may break and give us infinite looping comments.
3345                 # See: https://github.com/yt-dlp/yt-dlp/issues/6290
3346                 if comment['id'] in tracker['seen_comment_ids']:
3347                     self.report_warning('Detected YouTube comments looping. Stopping comment extraction as we probably cannot get any more.')
3348                     yield
3349                 else:
3350                     tracker['seen_comment_ids'].add(comment['id'])
3351
3352                 tracker['running_total'] += 1
3353                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3354                 yield comment
3355
3356                 # Attempt to get the replies
3357                 comment_replies_renderer = try_get(
3358                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3359
3360                 if comment_replies_renderer:
3361                     tracker['current_page_thread'] += 1
3362                     comment_entries_iter = self._comment_entries(
3363                         comment_replies_renderer, ytcfg, video_id,
3364                         parent=comment.get('id'), tracker=tracker)
3365                     yield from itertools.islice(comment_entries_iter, min(
3366                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3367
3368         # Keeps track of counts across recursive calls
3369         if not tracker:
3370             tracker = dict(
3371                 running_total=0,
3372                 est_total=0,
3373                 current_page_thread=0,
3374                 total_parent_comments=0,
3375                 total_reply_comments=0,
3376                 seen_comment_ids=set())
3377
3378         # TODO: Deprecated
3379         # YouTube comments have a max depth of 2
3380         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3381         if max_depth:
3382             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3383                                                 'Set max replies in the max-comments extractor argument instead')
3384         if max_depth == 1 and parent:
3385             return
3386
3387         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3388             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3389
3390         continuation = self._extract_continuation(root_continuation_data)
3391
3392         response = None
3393         is_forced_continuation = False
3394         is_first_continuation = parent is None
3395         if is_first_continuation and not continuation:
3396             # Sometimes you can get comments by generating the continuation yourself,
3397             # even if YouTube initially reports them being disabled - e.g. stories comments.
3398             # Note: if the comment section is actually disabled, YouTube may return a response with
3399             # required check_get_keys missing. So we will disable that check initially in this case.
3400             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3401             is_forced_continuation = True
3402
3403         for page_num in itertools.count(0):
3404             if not continuation:
3405                 break
3406             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3407             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3408             if page_num == 0:
3409                 if is_first_continuation:
3410                     note_prefix = 'Downloading comment section API JSON'
3411                 else:
3412                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3413                         tracker['current_page_thread'], comment_prog_str)
3414             else:
3415                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3416                     '       ' if parent else '', ' replies' if parent else '',
3417                     page_num, comment_prog_str)
3418             try:
3419                 response = self._extract_response(
3420                     item_id=None, query=continuation,
3421                     ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3422                     check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3423             except ExtractorError as e:
3424                 # Ignore incomplete data error for replies if retries didn't work.
3425                 # This is to allow any other parent comments and comment threads to be downloaded.
3426                 # See: https://github.com/yt-dlp/yt-dlp/issues/4669
3427                 if 'incomplete data' in str(e).lower() and parent and self.get_param('ignoreerrors') is True:
3428                     self.report_warning(
3429                         'Received incomplete data for a comment reply thread and retrying did not help. '
3430                         'Ignoring to let other comments be downloaded.')
3431                 else:
3432                     raise
3433             is_forced_continuation = False
3434             continuation_contents = traverse_obj(
3435                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3436
3437             continuation = None
3438             for continuation_section in continuation_contents:
3439                 continuation_items = traverse_obj(
3440                     continuation_section,
3441                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3442                     get_all=False, expected_type=list) or []
3443                 if is_first_continuation:
3444                     continuation = extract_header(continuation_items)
3445                     is_first_continuation = False
3446                     if continuation:
3447                         break
3448                     continue
3449
3450                 for entry in extract_thread(continuation_items):
3451                     if not entry:
3452                         return
3453                     yield entry
3454                 continuation = self._extract_continuation({'contents': continuation_items})
3455                 if continuation:
3456                     break
3457
3458         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3459         if message and not parent and tracker['running_total'] == 0:
3460             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3461             raise self.CommentsDisabled
3462
3463     @staticmethod
3464     def _generate_comment_continuation(video_id):
3465         """
3466         Generates initial comment section continuation token from given video id
3467         """
3468         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3469         return base64.b64encode(token.encode()).decode()
3470
3471     def _get_comments(self, ytcfg, video_id, contents, webpage):
3472         """Entry for comment extraction"""
3473         def _real_comment_extract(contents):
3474             renderer = next((
3475                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3476                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3477             yield from self._comment_entries(renderer, ytcfg, video_id)
3478
3479         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3480         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3481
3482     @staticmethod
3483     def _get_checkok_params():
3484         return {'contentCheckOk': True, 'racyCheckOk': True}
3485
3486     @classmethod
3487     def _generate_player_context(cls, sts=None):
3488         context = {
3489             'html5Preference': 'HTML5_PREF_WANTS',
3490         }
3491         if sts is not None:
3492             context['signatureTimestamp'] = sts
3493         return {
3494             'playbackContext': {
3495                 'contentPlaybackContext': context
3496             },
3497             **cls._get_checkok_params()
3498         }
3499
3500     @staticmethod
3501     def _is_agegated(player_response):
3502         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3503             return True
3504
3505         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')))
3506         AGE_GATE_REASONS = (
3507             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3508             'age_verification_required', 'age_check_required',  # status
3509         )
3510         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3511
3512     @staticmethod
3513     def _is_unplayable(player_response):
3514         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3515
3516     _STORY_PLAYER_PARAMS = '8AEB'
3517
3518     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3519
3520         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3521         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3522         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3523         headers = self.generate_api_headers(
3524             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3525
3526         yt_query = {
3527             'videoId': video_id,
3528         }
3529         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3530             yt_query['params'] = self._STORY_PLAYER_PARAMS
3531
3532         yt_query.update(self._generate_player_context(sts))
3533         return self._extract_response(
3534             item_id=video_id, ep='player', query=yt_query,
3535             ytcfg=player_ytcfg, headers=headers, fatal=True,
3536             default_client=client,
3537             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3538         ) or None
3539
3540     def _get_requested_clients(self, url, smuggled_data):
3541         requested_clients = []
3542         default = ['android', 'web']
3543         allowed_clients = sorted(
3544             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3545             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3546         for client in self._configuration_arg('player_client'):
3547             if client in allowed_clients:
3548                 requested_clients.append(client)
3549             elif client == 'default':
3550                 requested_clients.extend(default)
3551             elif client == 'all':
3552                 requested_clients.extend(allowed_clients)
3553             else:
3554                 self.report_warning(f'Skipping unsupported client {client}')
3555         if not requested_clients:
3556             requested_clients = default
3557
3558         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3559             requested_clients.extend(
3560                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3561
3562         return orderedSet(requested_clients)
3563
3564     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3565         initial_pr = None
3566         if webpage:
3567             initial_pr = self._search_json(
3568                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3569
3570         all_clients = set(clients)
3571         clients = clients[::-1]
3572         prs = []
3573
3574         def append_client(*client_names):
3575             """ Append the first client name that exists but not already used """
3576             for client_name in client_names:
3577                 actual_client = _split_innertube_client(client_name)[0]
3578                 if actual_client in INNERTUBE_CLIENTS:
3579                     if actual_client not in all_clients:
3580                         clients.append(client_name)
3581                         all_clients.add(actual_client)
3582                         return
3583
3584         # Android player_response does not have microFormats which are needed for
3585         # extraction of some data. So we return the initial_pr with formats
3586         # stripped out even if not requested by the user
3587         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3588         if initial_pr:
3589             pr = dict(initial_pr)
3590             pr['streamingData'] = None
3591             prs.append(pr)
3592
3593         last_error = None
3594         tried_iframe_fallback = False
3595         player_url = None
3596         while clients:
3597             client, base_client, variant = _split_innertube_client(clients.pop())
3598             player_ytcfg = master_ytcfg if client == 'web' else {}
3599             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3600                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3601
3602             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3603             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3604             if 'js' in self._configuration_arg('player_skip'):
3605                 require_js_player = False
3606                 player_url = None
3607
3608             if not player_url and not tried_iframe_fallback and require_js_player:
3609                 player_url = self._download_player_url(video_id)
3610                 tried_iframe_fallback = True
3611
3612             try:
3613                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3614                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3615             except ExtractorError as e:
3616                 if last_error:
3617                     self.report_warning(last_error)
3618                 last_error = e
3619                 continue
3620
3621             if pr:
3622                 # YouTube may return a different video player response than expected.
3623                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3624                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3625                 if pr_video_id and pr_video_id != video_id:
3626                     self.report_warning(
3627                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3628                 else:
3629                     prs.append(pr)
3630
3631             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3632             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3633                 append_client(f'{base_client}_creator')
3634             elif self._is_agegated(pr):
3635                 if variant == 'tv_embedded':
3636                     append_client(f'{base_client}_embedded')
3637                 elif not variant:
3638                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3639
3640         if last_error:
3641             if not len(prs):
3642                 raise last_error
3643             self.report_warning(last_error)
3644         return prs, player_url
3645
3646     def _needs_live_processing(self, live_status, duration):
3647         if (live_status == 'is_live' and self.get_param('live_from_start')
3648                 or live_status == 'post_live' and (duration or 0) > 4 * 3600):
3649             return live_status
3650
3651     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3652         itags, stream_ids = collections.defaultdict(set), []
3653         itag_qualities, res_qualities = {}, {0: None}
3654         q = qualities([
3655             # Normally tiny is the smallest video-only formats. But
3656             # audio-only formats with unknown quality may get tagged as tiny
3657             'tiny',
3658             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3659             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3660         ])
3661         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
3662
3663         for fmt in streaming_formats:
3664             if fmt.get('targetDurationSec'):
3665                 continue
3666
3667             itag = str_or_none(fmt.get('itag'))
3668             audio_track = fmt.get('audioTrack') or {}
3669             stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
3670             if stream_id in stream_ids:
3671                 continue
3672
3673             quality = fmt.get('quality')
3674             height = int_or_none(fmt.get('height'))
3675             if quality == 'tiny' or not quality:
3676                 quality = fmt.get('audioQuality', '').lower() or quality
3677             # The 3gp format (17) in android client has a quality of "small",
3678             # but is actually worse than other formats
3679             if itag == '17':
3680                 quality = 'tiny'
3681             if quality:
3682                 if itag:
3683                     itag_qualities[itag] = quality
3684                 if height:
3685                     res_qualities[height] = quality
3686             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3687             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3688             # number of fragment that would subsequently requested with (`&sq=N`)
3689             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3690                 continue
3691
3692             fmt_url = fmt.get('url')
3693             if not fmt_url:
3694                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3695                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3696                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3697                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3698                     continue
3699                 try:
3700                     fmt_url += '&%s=%s' % (
3701                         traverse_obj(sc, ('sp', -1)) or 'signature',
3702                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3703                     )
3704                 except ExtractorError as e:
3705                     self.report_warning('Signature extraction failed: Some formats may be missing',
3706                                         video_id=video_id, only_once=True)
3707                     self.write_debug(e, only_once=True)
3708                     continue
3709
3710             query = parse_qs(fmt_url)
3711             throttled = False
3712             if query.get('n'):
3713                 try:
3714                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3715                     fmt_url = update_url_query(fmt_url, {
3716                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3717                     })
3718                 except ExtractorError as e:
3719                     phantomjs_hint = ''
3720                     if isinstance(e, JSInterpreter.Exception):
3721                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3722                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3723                     if player_url:
3724                         self.report_warning(
3725                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3726                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3727                         self.write_debug(e, only_once=True)
3728                     else:
3729                         self.report_warning(
3730                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3731                             video_id=video_id, only_once=True)
3732                     throttled = True
3733
3734             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3735             language_preference = (
3736                 10 if audio_track.get('audioIsDefault') and 10
3737                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3738                 else -1)
3739             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3740             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3741             # Make sure to avoid false positives with small duration differences.
3742             # E.g. __2ABJjxzNo, ySuUZEjARPY
3743             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3744             if is_damaged:
3745                 self.report_warning(
3746                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3747             dct = {
3748                 'asr': int_or_none(fmt.get('audioSampleRate')),
3749                 'filesize': int_or_none(fmt.get('contentLength')),
3750                 'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
3751                 'format_note': join_nonempty(
3752                     '%s%s' % (audio_track.get('displayName') or '',
3753                               ' (default)' if language_preference > 0 else ''),
3754                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3755                     'DRC' if fmt.get('isDrc') else None,
3756                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3757                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3758                     throttled and 'THROTTLED', is_damaged and 'DAMAGED', delim=', '),
3759                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3760                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3761                 'fps': int_or_none(fmt.get('fps')) or None,
3762                 'audio_channels': fmt.get('audioChannels'),
3763                 'height': height,
3764                 'quality': q(quality) - bool(fmt.get('isDrc')) / 2,
3765                 'has_drm': bool(fmt.get('drmFamilies')),
3766                 'tbr': tbr,
3767                 'url': fmt_url,
3768                 'width': int_or_none(fmt.get('width')),
3769                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3770                                           'desc' if language_preference < -1 else '') or None,
3771                 'language_preference': language_preference,
3772                 # Strictly de-prioritize damaged and 3gp formats
3773                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3774             }
3775             mime_mobj = re.match(
3776                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3777             if mime_mobj:
3778                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3779                 dct.update(parse_codecs(mime_mobj.group(2)))
3780             no_audio = dct.get('acodec') == 'none'
3781             no_video = dct.get('vcodec') == 'none'
3782             if no_audio:
3783                 dct['vbr'] = tbr
3784             if no_video:
3785                 dct['abr'] = tbr
3786             if no_audio or no_video:
3787                 CHUNK_SIZE = 10 << 20
3788                 dct.update({
3789                     'protocol': 'http_dash_segments',
3790                     'fragments': [{
3791                         'url': update_url_query(dct['url'], {
3792                             'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
3793                         })
3794                     } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
3795                 } if dct['filesize'] else {
3796                     'downloader_options': {'http_chunk_size': CHUNK_SIZE}  # No longer useful?
3797                 })
3798
3799                 if dct.get('ext'):
3800                     dct['container'] = dct['ext'] + '_dash'
3801
3802             if itag:
3803                 itags[itag].add(('https', dct.get('language')))
3804                 stream_ids.append(stream_id)
3805             yield dct
3806
3807         needs_live_processing = self._needs_live_processing(live_status, duration)
3808         skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
3809
3810         skip_manifests = set(self._configuration_arg('skip'))
3811         if (not self.get_param('youtube_include_hls_manifest', True)
3812                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3813                 or needs_live_processing and skip_bad_formats):
3814             skip_manifests.add('hls')
3815
3816         if not self.get_param('youtube_include_dash_manifest', True):
3817             skip_manifests.add('dash')
3818         if self._configuration_arg('include_live_dash'):
3819             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3820                                                 'Use include_incomplete_formats extractor argument instead')
3821         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3822             skip_manifests.add('dash')
3823
3824         def process_manifest_format(f, proto, itag):
3825             key = (proto, f.get('language'))
3826             if key in itags[itag]:
3827                 return False
3828             itags[itag].add(key)
3829
3830             if any(p != proto for p, _ in itags[itag]):
3831                 f['format_id'] = f'{itag}-{proto}'
3832             elif itag:
3833                 f['format_id'] = itag
3834
3835             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3836             if f['quality'] == -1 and f.get('height'):
3837                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3838             return True
3839
3840         subtitles = {}
3841         for sd in streaming_data:
3842             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3843             if hls_manifest_url:
3844                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3845                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3846                 subtitles = self._merge_subtitles(subs, subtitles)
3847                 for f in fmts:
3848                     if process_manifest_format(f, 'hls', self._search_regex(
3849                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3850                         yield f
3851
3852             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3853             if dash_manifest_url:
3854                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3855                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3856                 for f in formats:
3857                     if process_manifest_format(f, 'dash', f['format_id']):
3858                         f['filesize'] = int_or_none(self._search_regex(
3859                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3860                         if needs_live_processing:
3861                             f['is_from_start'] = True
3862
3863                         yield f
3864         yield subtitles
3865
3866     def _extract_storyboard(self, player_responses, duration):
3867         spec = get_first(
3868             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3869         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3870         if not base_url:
3871             return
3872         L = len(spec) - 1
3873         for i, args in enumerate(spec):
3874             args = args.split('#')
3875             counts = list(map(int_or_none, args[:5]))
3876             if len(args) != 8 or not all(counts):
3877                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3878                 continue
3879             width, height, frame_count, cols, rows = counts
3880             N, sigh = args[6:]
3881
3882             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3883             fragment_count = frame_count / (cols * rows)
3884             fragment_duration = duration / fragment_count
3885             yield {
3886                 'format_id': f'sb{i}',
3887                 'format_note': 'storyboard',
3888                 'ext': 'mhtml',
3889                 'protocol': 'mhtml',
3890                 'acodec': 'none',
3891                 'vcodec': 'none',
3892                 'url': url,
3893                 'width': width,
3894                 'height': height,
3895                 'fps': frame_count / duration,
3896                 'rows': rows,
3897                 'columns': cols,
3898                 'fragments': [{
3899                     'url': url.replace('$M', str(j)),
3900                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3901                 } for j in range(math.ceil(fragment_count))],
3902             }
3903
3904     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3905         webpage = None
3906         if 'webpage' not in self._configuration_arg('player_skip'):
3907             query = {'bpctr': '9999999999', 'has_verified': '1'}
3908             if smuggled_data.get('is_story'):
3909                 query['pp'] = self._STORY_PLAYER_PARAMS
3910             webpage = self._download_webpage(
3911                 webpage_url, video_id, fatal=False, query=query)
3912
3913         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3914
3915         player_responses, player_url = self._extract_player_responses(
3916             self._get_requested_clients(url, smuggled_data),
3917             video_id, webpage, master_ytcfg, smuggled_data)
3918
3919         return webpage, master_ytcfg, player_responses, player_url
3920
3921     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3922         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3923         is_live = get_first(video_details, 'isLive')
3924         if is_live is None:
3925             is_live = get_first(live_broadcast_details, 'isLiveNow')
3926         live_content = get_first(video_details, 'isLiveContent')
3927         is_upcoming = get_first(video_details, 'isUpcoming')
3928         post_live = get_first(video_details, 'isPostLiveDvr')
3929         live_status = ('post_live' if post_live
3930                        else 'is_live' if is_live
3931                        else 'is_upcoming' if is_upcoming
3932                        else 'was_live' if live_content
3933                        else 'not_live' if False in (is_live, live_content)
3934                        else None)
3935         streaming_data = traverse_obj(player_responses, (..., 'streamingData'))
3936         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
3937
3938         return live_broadcast_details, live_status, streaming_data, formats, subtitles
3939
3940     def _real_extract(self, url):
3941         url, smuggled_data = unsmuggle_url(url, {})
3942         video_id = self._match_id(url)
3943
3944         base_url = self.http_scheme() + '//www.youtube.com/'
3945         webpage_url = base_url + 'watch?v=' + video_id
3946
3947         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3948
3949         playability_statuses = traverse_obj(
3950             player_responses, (..., 'playabilityStatus'), expected_type=dict)
3951
3952         trailer_video_id = get_first(
3953             playability_statuses,
3954             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3955             expected_type=str)
3956         if trailer_video_id:
3957             return self.url_result(
3958                 trailer_video_id, self.ie_key(), trailer_video_id)
3959
3960         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3961                        if webpage else (lambda x: None))
3962
3963         video_details = traverse_obj(player_responses, (..., 'videoDetails'), expected_type=dict)
3964         microformats = traverse_obj(
3965             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3966             expected_type=dict)
3967
3968         translated_title = self._get_text(microformats, (..., 'title'))
3969         video_title = (self._preferred_lang and translated_title
3970                        or get_first(video_details, 'title')  # primary
3971                        or translated_title
3972                        or search_meta(['og:title', 'twitter:title', 'title']))
3973         translated_description = self._get_text(microformats, (..., 'description'))
3974         original_description = get_first(video_details, 'shortDescription')
3975         video_description = (
3976             self._preferred_lang and translated_description
3977             # If original description is blank, it will be an empty string.
3978             # Do not prefer translated description in this case.
3979             or original_description if original_description is not None else translated_description)
3980
3981         multifeed_metadata_list = get_first(
3982             player_responses,
3983             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3984             expected_type=str)
3985         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3986             if self.get_param('noplaylist'):
3987                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3988             else:
3989                 entries = []
3990                 feed_ids = []
3991                 for feed in multifeed_metadata_list.split(','):
3992                     # Unquote should take place before split on comma (,) since textual
3993                     # fields may contain comma as well (see
3994                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3995                     feed_data = urllib.parse.parse_qs(
3996                         urllib.parse.unquote_plus(feed))
3997
3998                     def feed_entry(name):
3999                         return try_get(
4000                             feed_data, lambda x: x[name][0], str)
4001
4002                     feed_id = feed_entry('id')
4003                     if not feed_id:
4004                         continue
4005                     feed_title = feed_entry('title')
4006                     title = video_title
4007                     if feed_title:
4008                         title += ' (%s)' % feed_title
4009                     entries.append({
4010                         '_type': 'url_transparent',
4011                         'ie_key': 'Youtube',
4012                         'url': smuggle_url(
4013                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
4014                             {'force_singlefeed': True}),
4015                         'title': title,
4016                     })
4017                     feed_ids.append(feed_id)
4018                 self.to_screen(
4019                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
4020                     % (', '.join(feed_ids), video_id))
4021                 return self.playlist_result(
4022                     entries, video_id, video_title, video_description)
4023
4024         duration = (int_or_none(get_first(video_details, 'lengthSeconds'))
4025                     or int_or_none(get_first(microformats, 'lengthSeconds'))
4026                     or parse_duration(search_meta('duration')) or None)
4027
4028         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
4029             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
4030         if live_status == 'post_live':
4031             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
4032
4033         if not formats:
4034             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
4035                 self.report_drm(video_id)
4036             pemr = get_first(
4037                 playability_statuses,
4038                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
4039             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
4040             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
4041             if subreason:
4042                 if subreason == 'The uploader has not made this video available in your country.':
4043                     countries = get_first(microformats, 'availableCountries')
4044                     if not countries:
4045                         regions_allowed = search_meta('regionsAllowed')
4046                         countries = regions_allowed.split(',') if regions_allowed else None
4047                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
4048                 reason += f'. {subreason}'
4049             if reason:
4050                 self.raise_no_formats(reason, expected=True)
4051
4052         keywords = get_first(video_details, 'keywords', expected_type=list) or []
4053         if not keywords and webpage:
4054             keywords = [
4055                 unescapeHTML(m.group('content'))
4056                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
4057         for keyword in keywords:
4058             if keyword.startswith('yt:stretch='):
4059                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
4060                 if mobj:
4061                     # NB: float is intentional for forcing float division
4062                     w, h = (float(v) for v in mobj.groups())
4063                     if w > 0 and h > 0:
4064                         ratio = w / h
4065                         for f in formats:
4066                             if f.get('vcodec') != 'none':
4067                                 f['stretched_ratio'] = ratio
4068                         break
4069         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
4070         thumbnail_url = search_meta(['og:image', 'twitter:image'])
4071         if thumbnail_url:
4072             thumbnails.append({
4073                 'url': thumbnail_url,
4074             })
4075         original_thumbnails = thumbnails.copy()
4076
4077         # The best resolution thumbnails sometimes does not appear in the webpage
4078         # See: https://github.com/yt-dlp/yt-dlp/issues/340
4079         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
4080         thumbnail_names = [
4081             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
4082             # in resolution, these are not the custom thumbnail. So de-prioritize them
4083             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
4084             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
4085         ]
4086         n_thumbnail_names = len(thumbnail_names)
4087         thumbnails.extend({
4088             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
4089                 video_id=video_id, name=name, ext=ext,
4090                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
4091         } for name in thumbnail_names for ext in ('webp', 'jpg'))
4092         for thumb in thumbnails:
4093             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
4094             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
4095         self._remove_duplicate_formats(thumbnails)
4096         self._downloader._sort_thumbnails(original_thumbnails)
4097
4098         category = get_first(microformats, 'category') or search_meta('genre')
4099         channel_id = str_or_none(
4100             get_first(video_details, 'channelId')
4101             or get_first(microformats, 'externalChannelId')
4102             or search_meta('channelId'))
4103         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
4104
4105         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
4106         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
4107         if not duration and live_end_time and live_start_time:
4108             duration = live_end_time - live_start_time
4109
4110         needs_live_processing = self._needs_live_processing(live_status, duration)
4111
4112         def is_bad_format(fmt):
4113             if needs_live_processing and not fmt.get('is_from_start'):
4114                 return True
4115             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
4116                     and fmt.get('protocol') == 'http_dash_segments'):
4117                 return True
4118
4119         for fmt in filter(is_bad_format, formats):
4120             fmt['preference'] = (fmt.get('preference') or -1) - 10
4121             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
4122
4123         if needs_live_processing:
4124             self._prepare_live_from_start_formats(
4125                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
4126
4127         formats.extend(self._extract_storyboard(player_responses, duration))
4128
4129         info = {
4130             'id': video_id,
4131             'title': video_title,
4132             'formats': formats,
4133             'thumbnails': thumbnails,
4134             # The best thumbnail that we are sure exists. Prevents unnecessary
4135             # URL checking if user don't care about getting the best possible thumbnail
4136             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
4137             'description': video_description,
4138             'uploader': get_first(video_details, 'author'),
4139             'uploader_id': self._search_regex(r'/(?:channel/|user/|(?=@))([^/?&#]+)', owner_profile_url, 'uploader id', default=None),
4140             'uploader_url': owner_profile_url,
4141             'channel_id': channel_id,
4142             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
4143             'duration': duration,
4144             'view_count': int_or_none(
4145                 get_first((video_details, microformats), (..., 'viewCount'))
4146                 or search_meta('interactionCount')),
4147             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
4148             'age_limit': 18 if (
4149                 get_first(microformats, 'isFamilySafe') is False
4150                 or search_meta('isFamilyFriendly') == 'false'
4151                 or search_meta('og:restrictions:age') == '18+') else 0,
4152             'webpage_url': webpage_url,
4153             'categories': [category] if category else None,
4154             'tags': keywords,
4155             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
4156             'live_status': live_status,
4157             'release_timestamp': live_start_time,
4158             '_format_sort_fields': (  # source_preference is lower for throttled/potentially damaged formats
4159                 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto')
4160         }
4161
4162         subtitles = {}
4163         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
4164         if pctr:
4165             def get_lang_code(track):
4166                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
4167                         or track.get('languageCode'))
4168
4169             # Converted into dicts to remove duplicates
4170             captions = {
4171                 get_lang_code(sub): sub
4172                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...))}
4173             translation_languages = {
4174                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
4175                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...))}
4176
4177             def process_language(container, base_url, lang_code, sub_name, query):
4178                 lang_subs = container.setdefault(lang_code, [])
4179                 for fmt in self._SUBTITLE_FORMATS:
4180                     query.update({
4181                         'fmt': fmt,
4182                     })
4183                     lang_subs.append({
4184                         'ext': fmt,
4185                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
4186                         'name': sub_name,
4187                     })
4188
4189             # NB: Constructing the full subtitle dictionary is slow
4190             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
4191                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
4192             for lang_code, caption_track in captions.items():
4193                 base_url = caption_track.get('baseUrl')
4194                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
4195                 if not base_url:
4196                     continue
4197                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
4198                 if caption_track.get('kind') != 'asr':
4199                     if not lang_code:
4200                         continue
4201                     process_language(
4202                         subtitles, base_url, lang_code, lang_name, {})
4203                     if not caption_track.get('isTranslatable'):
4204                         continue
4205                 for trans_code, trans_name in translation_languages.items():
4206                     if not trans_code:
4207                         continue
4208                     orig_trans_code = trans_code
4209                     if caption_track.get('kind') != 'asr' and trans_code != 'und':
4210                         if not get_translated_subs:
4211                             continue
4212                         trans_code += f'-{lang_code}'
4213                         trans_name += format_field(lang_name, None, ' from %s')
4214                     # Add an "-orig" label to the original language so that it can be distinguished.
4215                     # The subs are returned without "-orig" as well for compatibility
4216                     if lang_code == f'a-{orig_trans_code}':
4217                         process_language(
4218                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
4219                     # Setting tlang=lang returns damaged subtitles.
4220                     process_language(automatic_captions, base_url, trans_code, trans_name,
4221                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
4222
4223         info['automatic_captions'] = automatic_captions
4224         info['subtitles'] = subtitles
4225
4226         parsed_url = urllib.parse.urlparse(url)
4227         for component in [parsed_url.fragment, parsed_url.query]:
4228             query = urllib.parse.parse_qs(component)
4229             for k, v in query.items():
4230                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
4231                     d_k += '_time'
4232                     if d_k not in info and k in s_ks:
4233                         info[d_k] = parse_duration(query[k][0])
4234
4235         # Youtube Music Auto-generated description
4236         if video_description:
4237             mobj = re.search(
4238                 r'''(?xs)
4239                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
4240                     (?P<album>[^\n]+)
4241                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4242                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4243                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
4244                     .+\nAuto-generated\ by\ YouTube\.\s*$
4245                 ''', video_description)
4246             if mobj:
4247                 release_year = mobj.group('release_year')
4248                 release_date = mobj.group('release_date')
4249                 if release_date:
4250                     release_date = release_date.replace('-', '')
4251                     if not release_year:
4252                         release_year = release_date[:4]
4253                 info.update({
4254                     'album': mobj.group('album'.strip()),
4255                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
4256                     'track': mobj.group('track').strip(),
4257                     'release_date': release_date,
4258                     'release_year': int_or_none(release_year),
4259                 })
4260
4261         initial_data = None
4262         if webpage:
4263             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4264         if not initial_data:
4265             query = {'videoId': video_id}
4266             query.update(self._get_checkok_params())
4267             initial_data = self._extract_response(
4268                 item_id=video_id, ep='next', fatal=False,
4269                 ytcfg=master_ytcfg, query=query,
4270                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4271                 note='Downloading initial data API JSON')
4272
4273         info['comment_count'] = traverse_obj(initial_data, (
4274             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4275             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
4276         ), (
4277             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4278             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
4279         ), expected_type=int_or_none, get_all=False)
4280
4281         try:  # This will error if there is no livechat
4282             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4283         except (KeyError, IndexError, TypeError):
4284             pass
4285         else:
4286             info.setdefault('subtitles', {})['live_chat'] = [{
4287                 # url is needed to set cookies
4288                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4289                 'video_id': video_id,
4290                 'ext': 'json',
4291                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4292                              else 'youtube_live_chat_replay'),
4293             }]
4294
4295         if initial_data:
4296             info['chapters'] = (
4297                 self._extract_chapters_from_json(initial_data, duration)
4298                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4299                 or self._extract_chapters_from_description(video_description, duration)
4300                 or None)
4301
4302         contents = traverse_obj(
4303             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4304             expected_type=list, default=[])
4305
4306         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4307         if vpir:
4308             stl = vpir.get('superTitleLink')
4309             if stl:
4310                 stl = self._get_text(stl)
4311                 if try_get(
4312                         vpir,
4313                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4314                     info['location'] = stl
4315                 else:
4316                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4317                     if mobj:
4318                         info.update({
4319                             'series': mobj.group(1),
4320                             'season_number': int(mobj.group(2)),
4321                             'episode_number': int(mobj.group(3)),
4322                         })
4323             for tlb in (try_get(
4324                     vpir,
4325                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4326                     list) or []):
4327                 tbrs = variadic(
4328                     traverse_obj(
4329                         tlb, ('toggleButtonRenderer', ...),
4330                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer')))
4331                 for tbr in tbrs:
4332                     for getter, regex in [(
4333                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4334                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4335                                 lambda x: x['accessibility'],
4336                                 lambda x: x['accessibilityData']['accessibilityData'],
4337                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4338                         label = (try_get(tbr, getter, dict) or {}).get('label')
4339                         if label:
4340                             mobj = re.match(regex, label)
4341                             if mobj:
4342                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4343                                 break
4344             sbr_tooltip = try_get(
4345                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4346             if sbr_tooltip:
4347                 like_count, dislike_count = sbr_tooltip.split(' / ')
4348                 info.update({
4349                     'like_count': str_to_int(like_count),
4350                     'dislike_count': str_to_int(dislike_count),
4351                 })
4352             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4353             if vcr:
4354                 vc = self._get_count(vcr, 'viewCount')
4355                 # Upcoming premieres with waiting count are treated as live here
4356                 if vcr.get('isLive'):
4357                     info['concurrent_view_count'] = vc
4358                 elif info.get('view_count') is None:
4359                     info['view_count'] = vc
4360
4361         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4362         if vsir:
4363             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4364             info.update({
4365                 'channel': self._get_text(vor, 'title'),
4366                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4367
4368             rows = try_get(
4369                 vsir,
4370                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4371                 list) or []
4372             multiple_songs = False
4373             for row in rows:
4374                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4375                     multiple_songs = True
4376                     break
4377             for row in rows:
4378                 mrr = row.get('metadataRowRenderer') or {}
4379                 mrr_title = mrr.get('title')
4380                 if not mrr_title:
4381                     continue
4382                 mrr_title = self._get_text(mrr, 'title')
4383                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4384                 if mrr_title == 'License':
4385                     info['license'] = mrr_contents_text
4386                 elif not multiple_songs:
4387                     if mrr_title == 'Album':
4388                         info['album'] = mrr_contents_text
4389                     elif mrr_title == 'Artist':
4390                         info['artist'] = mrr_contents_text
4391                     elif mrr_title == 'Song':
4392                         info['track'] = mrr_contents_text
4393
4394         fallbacks = {
4395             'channel': 'uploader',
4396             'channel_id': 'uploader_id',
4397             'channel_url': 'uploader_url',
4398         }
4399
4400         # The upload date for scheduled, live and past live streams / premieres in microformats
4401         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4402         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4403         upload_date = (
4404             unified_strdate(get_first(microformats, 'uploadDate'))
4405             or unified_strdate(search_meta('uploadDate')))
4406         if not upload_date or (
4407             live_status in ('not_live', None)
4408             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4409         ):
4410             upload_date = strftime_or_none(
4411                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4412         info['upload_date'] = upload_date
4413
4414         for to, frm in fallbacks.items():
4415             if not info.get(to):
4416                 info[to] = info.get(frm)
4417
4418         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4419             v = info.get(s_k)
4420             if v:
4421                 info[d_k] = v
4422
4423         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4424
4425         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4426                       or get_first(video_details, 'isPrivate', expected_type=bool))
4427
4428         info['availability'] = (
4429             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4430             else self._availability(
4431                 is_private=is_private,
4432                 needs_premium=(
4433                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4434                     or False if initial_data and is_private is not None else None),
4435                 needs_subscription=(
4436                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4437                     or False if initial_data and is_private is not None else None),
4438                 needs_auth=info['age_limit'] >= 18,
4439                 is_unlisted=None if is_private is None else (
4440                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4441                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4442
4443         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4444
4445         self.mark_watched(video_id, player_responses)
4446
4447         return info
4448
4449
4450 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4451     @staticmethod
4452     def passthrough_smuggled_data(func):
4453         def _smuggle(info, smuggled_data):
4454             if info.get('_type') not in ('url', 'url_transparent'):
4455                 return info
4456             if smuggled_data.get('is_music_url'):
4457                 parsed_url = urllib.parse.urlparse(info['url'])
4458                 if parsed_url.netloc in ('www.youtube.com', 'music.youtube.com'):
4459                     smuggled_data.pop('is_music_url')
4460                     info['url'] = urllib.parse.urlunparse(parsed_url._replace(netloc='music.youtube.com'))
4461             if smuggled_data:
4462                 info['url'] = smuggle_url(info['url'], smuggled_data)
4463             return info
4464
4465         @functools.wraps(func)
4466         def wrapper(self, url):
4467             url, smuggled_data = unsmuggle_url(url, {})
4468             if self.is_music_url(url):
4469                 smuggled_data['is_music_url'] = True
4470             info_dict = func(self, url, smuggled_data)
4471             if smuggled_data:
4472                 _smuggle(info_dict, smuggled_data)
4473                 if info_dict.get('entries'):
4474                     info_dict['entries'] = (_smuggle(i, smuggled_data.copy()) for i in info_dict['entries'])
4475             return info_dict
4476         return wrapper
4477
4478     @staticmethod
4479     def _extract_basic_item_renderer(item):
4480         # Modified from _extract_grid_item_renderer
4481         known_basic_renderers = (
4482             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4483         )
4484         for key, renderer in item.items():
4485             if not isinstance(renderer, dict):
4486                 continue
4487             elif key in known_basic_renderers:
4488                 return renderer
4489             elif key.startswith('grid') and key.endswith('Renderer'):
4490                 return renderer
4491
4492     def _extract_channel_renderer(self, renderer):
4493         channel_id = renderer['channelId']
4494         title = self._get_text(renderer, 'title')
4495         channel_url = f'https://www.youtube.com/channel/{channel_id}'
4496         return {
4497             '_type': 'url',
4498             'url': channel_url,
4499             'id': channel_id,
4500             'ie_key': YoutubeTabIE.ie_key(),
4501             'channel': title,
4502             'channel_id': channel_id,
4503             'channel_url': channel_url,
4504             'title': title,
4505             'channel_follower_count': self._get_count(renderer, 'subscriberCountText'),
4506             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
4507             'playlist_count': self._get_count(renderer, 'videoCountText'),
4508             'description': self._get_text(renderer, 'descriptionSnippet'),
4509         }
4510
4511     def _grid_entries(self, grid_renderer):
4512         for item in grid_renderer['items']:
4513             if not isinstance(item, dict):
4514                 continue
4515             renderer = self._extract_basic_item_renderer(item)
4516             if not isinstance(renderer, dict):
4517                 continue
4518             title = self._get_text(renderer, 'title')
4519
4520             # playlist
4521             playlist_id = renderer.get('playlistId')
4522             if playlist_id:
4523                 yield self.url_result(
4524                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4525                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4526                     video_title=title)
4527                 continue
4528             # video
4529             video_id = renderer.get('videoId')
4530             if video_id:
4531                 yield self._extract_video(renderer)
4532                 continue
4533             # channel
4534             channel_id = renderer.get('channelId')
4535             if channel_id:
4536                 yield self._extract_channel_renderer(renderer)
4537                 continue
4538             # generic endpoint URL support
4539             ep_url = urljoin('https://www.youtube.com/', try_get(
4540                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4541                 str))
4542             if ep_url:
4543                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4544                     if ie.suitable(ep_url):
4545                         yield self.url_result(
4546                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4547                         break
4548
4549     def _music_reponsive_list_entry(self, renderer):
4550         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4551         if video_id:
4552             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4553                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4554         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4555         if playlist_id:
4556             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4557             if video_id:
4558                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4559                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4560             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4561                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4562         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4563         if browse_id:
4564             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4565                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4566
4567     def _shelf_entries_from_content(self, shelf_renderer):
4568         content = shelf_renderer.get('content')
4569         if not isinstance(content, dict):
4570             return
4571         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4572         if renderer:
4573             # TODO: add support for nested playlists so each shelf is processed
4574             # as separate playlist
4575             # TODO: this includes only first N items
4576             yield from self._grid_entries(renderer)
4577         renderer = content.get('horizontalListRenderer')
4578         if renderer:
4579             # TODO
4580             pass
4581
4582     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4583         ep = try_get(
4584             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4585             str)
4586         shelf_url = urljoin('https://www.youtube.com', ep)
4587         if shelf_url:
4588             # Skipping links to another channels, note that checking for
4589             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4590             # will not work
4591             if skip_channels and '/channels?' in shelf_url:
4592                 return
4593             title = self._get_text(shelf_renderer, 'title')
4594             yield self.url_result(shelf_url, video_title=title)
4595         # Shelf may not contain shelf URL, fallback to extraction from content
4596         yield from self._shelf_entries_from_content(shelf_renderer)
4597
4598     def _playlist_entries(self, video_list_renderer):
4599         for content in video_list_renderer['contents']:
4600             if not isinstance(content, dict):
4601                 continue
4602             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4603             if not isinstance(renderer, dict):
4604                 continue
4605             video_id = renderer.get('videoId')
4606             if not video_id:
4607                 continue
4608             yield self._extract_video(renderer)
4609
4610     def _rich_entries(self, rich_grid_renderer):
4611         renderer = traverse_obj(
4612             rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
4613         video_id = renderer.get('videoId')
4614         if not video_id:
4615             return
4616         yield self._extract_video(renderer)
4617
4618     def _video_entry(self, video_renderer):
4619         video_id = video_renderer.get('videoId')
4620         if video_id:
4621             return self._extract_video(video_renderer)
4622
4623     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4624         url = urljoin('https://youtube.com', traverse_obj(
4625             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4626         if url:
4627             return self.url_result(
4628                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4629
4630     def _post_thread_entries(self, post_thread_renderer):
4631         post_renderer = try_get(
4632             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4633         if not post_renderer:
4634             return
4635         # video attachment
4636         video_renderer = try_get(
4637             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4638         video_id = video_renderer.get('videoId')
4639         if video_id:
4640             entry = self._extract_video(video_renderer)
4641             if entry:
4642                 yield entry
4643         # playlist attachment
4644         playlist_id = try_get(
4645             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4646         if playlist_id:
4647             yield self.url_result(
4648                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4649                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4650         # inline video links
4651         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4652         for run in runs:
4653             if not isinstance(run, dict):
4654                 continue
4655             ep_url = try_get(
4656                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4657             if not ep_url:
4658                 continue
4659             if not YoutubeIE.suitable(ep_url):
4660                 continue
4661             ep_video_id = YoutubeIE._match_id(ep_url)
4662             if video_id == ep_video_id:
4663                 continue
4664             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4665
4666     def _post_thread_continuation_entries(self, post_thread_continuation):
4667         contents = post_thread_continuation.get('contents')
4668         if not isinstance(contents, list):
4669             return
4670         for content in contents:
4671             renderer = content.get('backstagePostThreadRenderer')
4672             if isinstance(renderer, dict):
4673                 yield from self._post_thread_entries(renderer)
4674                 continue
4675             renderer = content.get('videoRenderer')
4676             if isinstance(renderer, dict):
4677                 yield self._video_entry(renderer)
4678
4679     r''' # unused
4680     def _rich_grid_entries(self, contents):
4681         for content in contents:
4682             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4683             if video_renderer:
4684                 entry = self._video_entry(video_renderer)
4685                 if entry:
4686                     yield entry
4687     '''
4688
4689     def _report_history_entries(self, renderer):
4690         for url in traverse_obj(renderer, (
4691                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4692                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4693                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4694             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4695
4696     def _extract_entries(self, parent_renderer, continuation_list):
4697         # continuation_list is modified in-place with continuation_list = [continuation_token]
4698         continuation_list[:] = [None]
4699         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4700         for content in contents:
4701             if not isinstance(content, dict):
4702                 continue
4703             is_renderer = traverse_obj(
4704                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4705                 expected_type=dict)
4706             if not is_renderer:
4707                 if content.get('richItemRenderer'):
4708                     for entry in self._rich_entries(content['richItemRenderer']):
4709                         yield entry
4710                     continuation_list[0] = self._extract_continuation(parent_renderer)
4711                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4712                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4713                     yield from self._report_history_entries(table)
4714                     continuation_list[0] = self._extract_continuation(table)
4715                 continue
4716
4717             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4718             for isr_content in isr_contents:
4719                 if not isinstance(isr_content, dict):
4720                     continue
4721
4722                 known_renderers = {
4723                     'playlistVideoListRenderer': self._playlist_entries,
4724                     'gridRenderer': self._grid_entries,
4725                     'reelShelfRenderer': self._grid_entries,
4726                     'shelfRenderer': self._shelf_entries,
4727                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4728                     'backstagePostThreadRenderer': self._post_thread_entries,
4729                     'videoRenderer': lambda x: [self._video_entry(x)],
4730                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4731                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4732                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4733                 }
4734                 for key, renderer in isr_content.items():
4735                     if key not in known_renderers:
4736                         continue
4737                     for entry in known_renderers[key](renderer):
4738                         if entry:
4739                             yield entry
4740                     continuation_list[0] = self._extract_continuation(renderer)
4741                     break
4742
4743             if not continuation_list[0]:
4744                 continuation_list[0] = self._extract_continuation(is_renderer)
4745
4746         if not continuation_list[0]:
4747             continuation_list[0] = self._extract_continuation(parent_renderer)
4748
4749     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4750         continuation_list = [None]
4751         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4752         tab_content = try_get(tab, lambda x: x['content'], dict)
4753         if not tab_content:
4754             return
4755         parent_renderer = (
4756             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4757             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4758         yield from extract_entries(parent_renderer)
4759         continuation = continuation_list[0]
4760
4761         for page_num in itertools.count(1):
4762             if not continuation:
4763                 break
4764             headers = self.generate_api_headers(
4765                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4766             response = self._extract_response(
4767                 item_id=f'{item_id} page {page_num}',
4768                 query=continuation, headers=headers, ytcfg=ytcfg,
4769                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4770
4771             if not response:
4772                 break
4773             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4774             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4775             visitor_data = self._extract_visitor_data(response) or visitor_data
4776
4777             known_renderers = {
4778                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4779                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4780                 'gridVideoRenderer': (self._grid_entries, 'items'),
4781                 'gridChannelRenderer': (self._grid_entries, 'items'),
4782                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4783                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4784                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4785                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4786                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4787                 'playlistVideoListContinuation': (self._playlist_entries, None),
4788                 'gridContinuation': (self._grid_entries, None),
4789                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4790                 'sectionListContinuation': (extract_entries, None),  # for feeds
4791             }
4792
4793             continuation_items = traverse_obj(response, (
4794                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4795                 'appendContinuationItemsAction', 'continuationItems'
4796             ), 'continuationContents', get_all=False)
4797             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4798
4799             video_items_renderer = None
4800             for key in continuation_item.keys():
4801                 if key not in known_renderers:
4802                     continue
4803                 func, parent_key = known_renderers[key]
4804                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4805                 continuation_list = [None]
4806                 yield from func(video_items_renderer)
4807                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4808
4809             if not video_items_renderer:
4810                 break
4811
4812     @staticmethod
4813     def _extract_selected_tab(tabs, fatal=True):
4814         for tab_renderer in tabs:
4815             if tab_renderer.get('selected'):
4816                 return tab_renderer
4817         if fatal:
4818             raise ExtractorError('Unable to find selected tab')
4819
4820     @staticmethod
4821     def _extract_tab_renderers(response):
4822         return traverse_obj(
4823             response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
4824
4825     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4826         metadata = self._extract_metadata_from_tabs(item_id, data)
4827
4828         selected_tab = self._extract_selected_tab(tabs)
4829         metadata['title'] += format_field(selected_tab, 'title', ' - %s')
4830         metadata['title'] += format_field(selected_tab, 'expandedText', ' - %s')
4831
4832         return self.playlist_result(
4833             self._entries(
4834                 selected_tab, metadata['id'], ytcfg,
4835                 self._extract_account_syncid(ytcfg, data),
4836                 self._extract_visitor_data(data, ytcfg)),
4837             **metadata)
4838
4839     def _extract_metadata_from_tabs(self, item_id, data):
4840         info = {'id': item_id}
4841
4842         metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
4843         if metadata_renderer:
4844             info.update({
4845                 'uploader': metadata_renderer.get('title'),
4846                 'uploader_id': metadata_renderer.get('externalId'),
4847                 'uploader_url': metadata_renderer.get('channelUrl'),
4848             })
4849             if info['uploader_id']:
4850                 info['id'] = info['uploader_id']
4851         else:
4852             metadata_renderer = traverse_obj(data, ('metadata', 'playlistMetadataRenderer'), expected_type=dict)
4853
4854         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4855         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4856         def _get_uncropped(url):
4857             return url_or_none((url or '').split('=')[0] + '=s0')
4858
4859         avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
4860         if avatar_thumbnails:
4861             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4862             if uncropped_avatar:
4863                 avatar_thumbnails.append({
4864                     'url': uncropped_avatar,
4865                     'id': 'avatar_uncropped',
4866                     'preference': 1
4867                 })
4868
4869         channel_banners = self._extract_thumbnails(
4870             data, ('header', ..., ('banner', 'mobileBanner', 'tvBanner')))
4871         for banner in channel_banners:
4872             banner['preference'] = -10
4873
4874         if channel_banners:
4875             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4876             if uncropped_banner:
4877                 channel_banners.append({
4878                     'url': uncropped_banner,
4879                     'id': 'banner_uncropped',
4880                     'preference': -5
4881                 })
4882
4883         # Deprecated - remove primary_sidebar_renderer when layout discontinued
4884         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4885         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
4886
4887         primary_thumbnails = self._extract_thumbnails(
4888             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4889         playlist_thumbnails = self._extract_thumbnails(
4890             playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
4891
4892         info.update({
4893             'title': (traverse_obj(metadata_renderer, 'title')
4894                       or self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag'))
4895                       or info['id']),
4896             'availability': self._extract_availability(data),
4897             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4898             'description': try_get(metadata_renderer, lambda x: x.get('description', '')),
4899             'tags': try_get(metadata_renderer or {}, lambda x: x.get('keywords', '').split()),
4900             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
4901         })
4902
4903         # Playlist stats is a text runs array containing [video count, view count, last updated].
4904         # last updated or (view count and last updated) may be missing.
4905         playlist_stats = get_first(
4906             (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'), ))
4907
4908         last_updated_unix = self._parse_time_text(
4909             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
4910             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
4911         info['modified_date'] = strftime_or_none(last_updated_unix, '%Y%m%d')
4912
4913         info['view_count'] = self._get_count(playlist_stats, 1)
4914         if info['view_count'] is None:  # 0 is allowed
4915             info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText')
4916         if info['view_count'] is None:
4917             info['view_count'] = self._get_count(data, (
4918                 'contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., 'tabRenderer', 'content', 'sectionListRenderer',
4919                 'contents', ..., 'itemSectionRenderer', 'contents', ..., 'channelAboutFullMetadataRenderer', 'viewCountText'))
4920
4921         info['playlist_count'] = self._get_count(playlist_stats, 0)
4922         if info['playlist_count'] is None:  # 0 is allowed
4923             info['playlist_count'] = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
4924
4925         if not info.get('uploader_id'):
4926             owner = traverse_obj(playlist_header_renderer, 'ownerText')
4927             if not owner:  # Deprecated
4928                 owner = traverse_obj(
4929                     self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
4930                     ('videoOwner', 'videoOwnerRenderer', 'title'))
4931             owner_text = self._get_text(owner)
4932             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
4933             info.update({
4934                 'uploader': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
4935                 'uploader_id': browse_ep.get('browseId'),
4936                 'uploader_url': urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))
4937             })
4938
4939         info.update({
4940             'channel': info['uploader'],
4941             'channel_id': info['uploader_id'],
4942             'channel_url': info['uploader_url']
4943         })
4944         return info
4945
4946     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4947         first_id = last_id = response = None
4948         for page_num in itertools.count(1):
4949             videos = list(self._playlist_entries(playlist))
4950             if not videos:
4951                 return
4952             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4953             if start >= len(videos):
4954                 return
4955             yield from videos[start:]
4956             first_id = first_id or videos[0]['id']
4957             last_id = videos[-1]['id']
4958             watch_endpoint = try_get(
4959                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4960             headers = self.generate_api_headers(
4961                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4962                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4963             query = {
4964                 'playlistId': playlist_id,
4965                 'videoId': watch_endpoint.get('videoId') or last_id,
4966                 'index': watch_endpoint.get('index') or len(videos),
4967                 'params': watch_endpoint.get('params') or 'OAE%3D'
4968             }
4969             response = self._extract_response(
4970                 item_id='%s page %d' % (playlist_id, page_num),
4971                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4972                 check_get_keys='contents'
4973             )
4974             playlist = try_get(
4975                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4976
4977     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4978         title = playlist.get('title') or try_get(
4979             data, lambda x: x['titleText']['simpleText'], str)
4980         playlist_id = playlist.get('playlistId') or item_id
4981
4982         # Delegating everything except mix playlists to regular tab-based playlist URL
4983         playlist_url = urljoin(url, try_get(
4984             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4985             str))
4986
4987         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4988         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4989         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4990
4991         if playlist_url and playlist_url != url and not is_known_unviewable:
4992             return self.url_result(
4993                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4994                 video_title=title)
4995
4996         return self.playlist_result(
4997             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4998             playlist_id=playlist_id, playlist_title=title)
4999
5000     def _extract_availability(self, data):
5001         """
5002         Gets the availability of a given playlist/tab.
5003         Note: Unless YouTube tells us explicitly, we do not assume it is public
5004         @param data: response
5005         """
5006         sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
5007         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
5008         player_header_privacy = playlist_header_renderer.get('privacy')
5009
5010         badges = self._extract_badges(sidebar_renderer)
5011
5012         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
5013         privacy_setting_icon = get_first(
5014             (playlist_header_renderer, sidebar_renderer),
5015             ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
5016              lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
5017             expected_type=str)
5018
5019         microformats_is_unlisted = traverse_obj(
5020             data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
5021
5022         return (
5023             'public' if (
5024                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
5025                 or player_header_privacy == 'PUBLIC'
5026                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
5027             else self._availability(
5028                 is_private=(
5029                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
5030                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
5031                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
5032                 is_unlisted=(
5033                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
5034                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
5035                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
5036                     else microformats_is_unlisted if microformats_is_unlisted is not None else None),
5037                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
5038                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
5039                 needs_auth=False))
5040
5041     @staticmethod
5042     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
5043         sidebar_renderer = try_get(
5044             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
5045         for item in sidebar_renderer:
5046             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
5047             if renderer:
5048                 return renderer
5049
5050     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
5051         """
5052         Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
5053         """
5054         is_playlist = bool(traverse_obj(
5055             data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
5056         if not is_playlist:
5057             return
5058         headers = self.generate_api_headers(
5059             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
5060             visitor_data=self._extract_visitor_data(data, ytcfg))
5061         query = {
5062             'params': 'wgYCCAA=',
5063             'browseId': f'VL{item_id}'
5064         }
5065         return self._extract_response(
5066             item_id=item_id, headers=headers, query=query,
5067             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
5068             note='Redownloading playlist API JSON with unavailable videos')
5069
5070     @functools.cached_property
5071     def skip_webpage(self):
5072         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
5073
5074     def _extract_webpage(self, url, item_id, fatal=True):
5075         webpage, data = None, None
5076         for retry in self.RetryManager(fatal=fatal):
5077             try:
5078                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
5079                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
5080             except ExtractorError as e:
5081                 if isinstance(e.cause, network_exceptions):
5082                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
5083                         retry.error = e
5084                         continue
5085                 self._error_or_warning(e, fatal=fatal)
5086                 break
5087
5088             try:
5089                 self._extract_and_report_alerts(data)
5090             except ExtractorError as e:
5091                 self._error_or_warning(e, fatal=fatal)
5092                 break
5093
5094             # Sometimes youtube returns a webpage with incomplete ytInitialData
5095             # See: https://github.com/yt-dlp/yt-dlp/issues/116
5096             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
5097                 retry.error = ExtractorError('Incomplete yt initial data received')
5098                 continue
5099
5100         return webpage, data
5101
5102     def _report_playlist_authcheck(self, ytcfg, fatal=True):
5103         """Use if failed to extract ytcfg (and data) from initial webpage"""
5104         if not ytcfg and self.is_authenticated:
5105             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
5106             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
5107                 raise ExtractorError(
5108                     f'{msg}. If you are not downloading private content, or '
5109                     'your cookies are only for the first account and channel,'
5110                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
5111                     expected=True)
5112             self.report_warning(msg, only_once=True)
5113
5114     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
5115         data = None
5116         if not self.skip_webpage:
5117             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
5118             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
5119             # Reject webpage data if redirected to home page without explicitly requesting
5120             selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
5121             if (url != 'https://www.youtube.com/feed/recommended'
5122                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
5123                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
5124                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
5125                 if fatal:
5126                     raise ExtractorError(msg, expected=True)
5127                 self.report_warning(msg, only_once=True)
5128         if not data:
5129             self._report_playlist_authcheck(ytcfg, fatal=fatal)
5130             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
5131         return data, ytcfg
5132
5133     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
5134         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
5135         resolve_response = self._extract_response(
5136             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
5137             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
5138         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
5139         for ep_key, ep in endpoints.items():
5140             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
5141             if params:
5142                 return self._extract_response(
5143                     item_id=item_id, query=params, ep=ep, headers=headers,
5144                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
5145                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
5146         err_note = 'Failed to resolve url (does the playlist exist?)'
5147         if fatal:
5148             raise ExtractorError(err_note, expected=True)
5149         self.report_warning(err_note, item_id)
5150
5151     _SEARCH_PARAMS = None
5152
5153     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
5154         data = {'query': query}
5155         if params is NO_DEFAULT:
5156             params = self._SEARCH_PARAMS
5157         if params:
5158             data['params'] = params
5159
5160         content_keys = (
5161             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
5162             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
5163             # ytmusic search
5164             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
5165             ('continuationContents', ),
5166         )
5167         display_id = f'query "{query}"'
5168         check_get_keys = tuple({keys[0] for keys in content_keys})
5169         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
5170         self._report_playlist_authcheck(ytcfg, fatal=False)
5171
5172         continuation_list = [None]
5173         search = None
5174         for page_num in itertools.count(1):
5175             data.update(continuation_list[0] or {})
5176             headers = self.generate_api_headers(
5177                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
5178             search = self._extract_response(
5179                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
5180                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
5181             slr_contents = traverse_obj(search, *content_keys)
5182             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
5183             if not continuation_list[0]:
5184                 break
5185
5186
5187 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
5188     IE_DESC = 'YouTube Tabs'
5189     _VALID_URL = r'''(?x:
5190         https?://
5191             (?!consent\.)(?:\w+\.)?
5192             (?:
5193                 youtube(?:kids)?\.com|
5194                 %(invidious)s
5195             )/
5196             (?:
5197                 (?P<channel_type>channel|c|user|browse)/|
5198                 (?P<not_channel>
5199                     feed/|hashtag/|
5200                     (?:playlist|watch)\?.*?\blist=
5201                 )|
5202                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
5203             )
5204             (?P<id>[^/?\#&]+)
5205     )''' % {
5206         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
5207         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5208     }
5209     IE_NAME = 'youtube:tab'
5210
5211     _TESTS = [{
5212         'note': 'playlists, multipage',
5213         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
5214         'playlist_mincount': 94,
5215         'info_dict': {
5216             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5217             'title': 'Igor Kleiner - Playlists',
5218             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5219             'uploader': 'Igor Kleiner',
5220             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5221             'channel': 'Igor Kleiner',
5222             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5223             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5224             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5225             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5226             'channel_follower_count': int
5227         },
5228     }, {
5229         'note': 'playlists, multipage, different order',
5230         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
5231         'playlist_mincount': 94,
5232         'info_dict': {
5233             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5234             'title': 'Igor Kleiner - Playlists',
5235             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5236             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5237             'uploader': 'Igor Kleiner',
5238             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5239             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5240             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5241             'channel': 'Igor Kleiner',
5242             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5243             'channel_follower_count': int
5244         },
5245     }, {
5246         'note': 'playlists, series',
5247         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5248         'playlist_mincount': 5,
5249         'info_dict': {
5250             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5251             'title': '3Blue1Brown - Playlists',
5252             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5253             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5254             'uploader': '3Blue1Brown',
5255             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5256             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5257             'channel': '3Blue1Brown',
5258             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5259             'tags': ['Mathematics'],
5260             'channel_follower_count': int
5261         },
5262     }, {
5263         'note': 'playlists, singlepage',
5264         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5265         'playlist_mincount': 4,
5266         'info_dict': {
5267             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5268             'title': 'ThirstForScience - Playlists',
5269             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5270             'uploader': 'ThirstForScience',
5271             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5272             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5273             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5274             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5275             'tags': 'count:13',
5276             'channel': 'ThirstForScience',
5277             'channel_follower_count': int
5278         }
5279     }, {
5280         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5281         'only_matching': True,
5282     }, {
5283         'note': 'basic, single video playlist',
5284         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5285         'info_dict': {
5286             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5287             'uploader': 'Sergey M.',
5288             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5289             'title': 'youtube-dl public playlist',
5290             'description': '',
5291             'tags': [],
5292             'view_count': int,
5293             'modified_date': '20201130',
5294             'channel': 'Sergey M.',
5295             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5296             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5297             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5298             'availability': 'public',
5299         },
5300         'playlist_count': 1,
5301     }, {
5302         'note': 'empty playlist',
5303         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5304         'info_dict': {
5305             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5306             'uploader': 'Sergey M.',
5307             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5308             'title': 'youtube-dl empty playlist',
5309             'tags': [],
5310             'channel': 'Sergey M.',
5311             'description': '',
5312             'modified_date': '20160902',
5313             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5314             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5315             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5316             'availability': 'public',
5317         },
5318         'playlist_count': 0,
5319     }, {
5320         'note': 'Home tab',
5321         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5322         'info_dict': {
5323             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5324             'title': 'lex will - Home',
5325             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5326             'uploader': 'lex will',
5327             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5328             'channel': 'lex will',
5329             'tags': ['bible', 'history', 'prophesy'],
5330             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5331             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5332             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5333             'channel_follower_count': int
5334         },
5335         'playlist_mincount': 2,
5336     }, {
5337         'note': 'Videos tab',
5338         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5339         'info_dict': {
5340             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5341             'title': 'lex will - Videos',
5342             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5343             'uploader': 'lex will',
5344             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5345             'tags': ['bible', 'history', 'prophesy'],
5346             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5347             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5348             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5349             'channel': 'lex will',
5350             'channel_follower_count': int
5351         },
5352         'playlist_mincount': 975,
5353     }, {
5354         'note': 'Videos tab, sorted by popular',
5355         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5356         'info_dict': {
5357             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5358             'title': 'lex will - Videos',
5359             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5360             'uploader': 'lex will',
5361             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5362             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5363             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5364             'channel': 'lex will',
5365             'tags': ['bible', 'history', 'prophesy'],
5366             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5367             'channel_follower_count': int
5368         },
5369         'playlist_mincount': 199,
5370     }, {
5371         'note': 'Playlists tab',
5372         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5373         'info_dict': {
5374             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5375             'title': 'lex will - Playlists',
5376             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5377             'uploader': 'lex will',
5378             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5379             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5380             'channel': 'lex will',
5381             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5382             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5383             'tags': ['bible', 'history', 'prophesy'],
5384             'channel_follower_count': int
5385         },
5386         'playlist_mincount': 17,
5387     }, {
5388         'note': 'Community tab',
5389         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5390         'info_dict': {
5391             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5392             'title': 'lex will - Community',
5393             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5394             'uploader': 'lex will',
5395             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5396             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5397             'channel': 'lex will',
5398             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5399             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5400             'tags': ['bible', 'history', 'prophesy'],
5401             'channel_follower_count': int
5402         },
5403         'playlist_mincount': 18,
5404     }, {
5405         'note': 'Channels tab',
5406         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5407         'info_dict': {
5408             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5409             'title': 'lex will - Channels',
5410             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5411             'uploader': 'lex will',
5412             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5413             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5414             'channel': 'lex will',
5415             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5416             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5417             'tags': ['bible', 'history', 'prophesy'],
5418             'channel_follower_count': int
5419         },
5420         'playlist_mincount': 12,
5421     }, {
5422         'note': 'Search tab',
5423         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5424         'playlist_mincount': 40,
5425         'info_dict': {
5426             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5427             'title': '3Blue1Brown - Search - linear algebra',
5428             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5429             'uploader': '3Blue1Brown',
5430             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5431             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5432             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5433             'tags': ['Mathematics'],
5434             'channel': '3Blue1Brown',
5435             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5436             'channel_follower_count': int
5437         },
5438     }, {
5439         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5440         'only_matching': True,
5441     }, {
5442         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5443         'only_matching': True,
5444     }, {
5445         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5446         'only_matching': True,
5447     }, {
5448         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5449         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5450         'info_dict': {
5451             'title': '29C3: Not my department',
5452             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5453             'uploader': 'Christiaan008',
5454             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5455             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5456             'tags': [],
5457             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5458             'view_count': int,
5459             'modified_date': '20150605',
5460             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5461             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5462             'channel': 'Christiaan008',
5463             'availability': 'public',
5464         },
5465         'playlist_count': 96,
5466     }, {
5467         'note': 'Large playlist',
5468         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5469         'info_dict': {
5470             'title': 'Uploads from Cauchemar',
5471             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5472             'uploader': 'Cauchemar',
5473             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5474             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5475             'tags': [],
5476             'modified_date': r're:\d{8}',
5477             'channel': 'Cauchemar',
5478             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5479             'view_count': int,
5480             'description': '',
5481             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5482             'availability': 'public',
5483         },
5484         'playlist_mincount': 1123,
5485         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5486     }, {
5487         'note': 'even larger playlist, 8832 videos',
5488         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5489         'only_matching': True,
5490     }, {
5491         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5492         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5493         'info_dict': {
5494             'title': 'Uploads from Interstellar Movie',
5495             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5496             'uploader': 'Interstellar Movie',
5497             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5498             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5499             'tags': [],
5500             'view_count': int,
5501             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5502             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5503             'channel': 'Interstellar Movie',
5504             'description': '',
5505             'modified_date': r're:\d{8}',
5506             'availability': 'public',
5507         },
5508         'playlist_mincount': 21,
5509     }, {
5510         'note': 'Playlist with "show unavailable videos" button',
5511         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5512         'info_dict': {
5513             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5514             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5515             'uploader': 'Phim Siêu Nhân Nhật Bản',
5516             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5517             'view_count': int,
5518             'channel': 'Phim Siêu Nhân Nhật Bản',
5519             'tags': [],
5520             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5521             'description': '',
5522             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5523             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5524             'modified_date': r're:\d{8}',
5525             'availability': 'public',
5526         },
5527         'playlist_mincount': 200,
5528         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5529     }, {
5530         'note': 'Playlist with unavailable videos in page 7',
5531         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5532         'info_dict': {
5533             'title': 'Uploads from BlankTV',
5534             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5535             'uploader': 'BlankTV',
5536             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5537             'channel': 'BlankTV',
5538             'channel_url': 'https://www.youtube.com/c/blanktv',
5539             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5540             'view_count': int,
5541             'tags': [],
5542             'uploader_url': 'https://www.youtube.com/c/blanktv',
5543             'modified_date': r're:\d{8}',
5544             'description': '',
5545             'availability': 'public',
5546         },
5547         'playlist_mincount': 1000,
5548         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5549     }, {
5550         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5551         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5552         'info_dict': {
5553             'title': 'Data Analysis with Dr Mike Pound',
5554             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5555             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5556             'uploader': 'Computerphile',
5557             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5558             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5559             'tags': [],
5560             'view_count': int,
5561             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5562             'channel_url': 'https://www.youtube.com/user/Computerphile',
5563             'channel': 'Computerphile',
5564             'availability': 'public',
5565             'modified_date': '20190712',
5566         },
5567         'playlist_mincount': 11,
5568     }, {
5569         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5570         'only_matching': True,
5571     }, {
5572         'note': 'Playlist URL that does not actually serve a playlist',
5573         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5574         'info_dict': {
5575             'id': 'FqZTN594JQw',
5576             'ext': 'webm',
5577             'title': "Smiley's People 01 detective, Adventure Series, Action",
5578             'uploader': 'STREEM',
5579             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5580             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5581             'upload_date': '20150526',
5582             'license': 'Standard YouTube License',
5583             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5584             'categories': ['People & Blogs'],
5585             'tags': list,
5586             'view_count': int,
5587             'like_count': int,
5588         },
5589         'params': {
5590             'skip_download': True,
5591         },
5592         'skip': 'This video is not available.',
5593         'add_ie': [YoutubeIE.ie_key()],
5594     }, {
5595         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5596         'only_matching': True,
5597     }, {
5598         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5599         'only_matching': True,
5600     }, {
5601         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5602         'info_dict': {
5603             'id': 'Wq15eF5vCbI',  # This will keep changing
5604             'ext': 'mp4',
5605             'title': str,
5606             'uploader': 'Sky News',
5607             'uploader_id': 'skynews',
5608             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5609             'upload_date': r're:\d{8}',
5610             'description': str,
5611             'categories': ['News & Politics'],
5612             'tags': list,
5613             'like_count': int,
5614             'release_timestamp': int,
5615             'channel': 'Sky News',
5616             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5617             'age_limit': 0,
5618             'view_count': int,
5619             'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
5620             'playable_in_embed': True,
5621             'release_date': r're:\d+',
5622             'availability': 'public',
5623             'live_status': 'is_live',
5624             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5625             'channel_follower_count': int,
5626             'concurrent_view_count': int,
5627         },
5628         'params': {
5629             'skip_download': True,
5630         },
5631         'expected_warnings': ['Ignoring subtitle tracks found in '],
5632     }, {
5633         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5634         'info_dict': {
5635             'id': 'a48o2S1cPoo',
5636             'ext': 'mp4',
5637             'title': 'The Young Turks - Live Main Show',
5638             'uploader': 'The Young Turks',
5639             'uploader_id': 'TheYoungTurks',
5640             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5641             'upload_date': '20150715',
5642             'license': 'Standard YouTube License',
5643             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5644             'categories': ['News & Politics'],
5645             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5646             'like_count': int,
5647         },
5648         'params': {
5649             'skip_download': True,
5650         },
5651         'only_matching': True,
5652     }, {
5653         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5654         'only_matching': True,
5655     }, {
5656         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5657         'only_matching': True,
5658     }, {
5659         'note': 'A channel that is not live. Should raise error',
5660         'url': 'https://www.youtube.com/user/numberphile/live',
5661         'only_matching': True,
5662     }, {
5663         'url': 'https://www.youtube.com/feed/trending',
5664         'only_matching': True,
5665     }, {
5666         'url': 'https://www.youtube.com/feed/library',
5667         'only_matching': True,
5668     }, {
5669         'url': 'https://www.youtube.com/feed/history',
5670         'only_matching': True,
5671     }, {
5672         'url': 'https://www.youtube.com/feed/subscriptions',
5673         'only_matching': True,
5674     }, {
5675         'url': 'https://www.youtube.com/feed/watch_later',
5676         'only_matching': True,
5677     }, {
5678         'note': 'Recommended - redirects to home page.',
5679         'url': 'https://www.youtube.com/feed/recommended',
5680         'only_matching': True,
5681     }, {
5682         'note': 'inline playlist with not always working continuations',
5683         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5684         'only_matching': True,
5685     }, {
5686         'url': 'https://www.youtube.com/course',
5687         'only_matching': True,
5688     }, {
5689         'url': 'https://www.youtube.com/zsecurity',
5690         'only_matching': True,
5691     }, {
5692         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5693         'only_matching': True,
5694     }, {
5695         'url': 'https://www.youtube.com/TheYoungTurks/live',
5696         'only_matching': True,
5697     }, {
5698         'url': 'https://www.youtube.com/hashtag/cctv9',
5699         'info_dict': {
5700             'id': 'cctv9',
5701             'title': '#cctv9',
5702             'tags': [],
5703         },
5704         'playlist_mincount': 300,  # not consistent but should be over 300
5705     }, {
5706         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5707         'only_matching': True,
5708     }, {
5709         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5710         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5711         'only_matching': True
5712     }, {
5713         'note': '/browse/ should redirect to /channel/',
5714         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5715         'only_matching': True
5716     }, {
5717         'note': 'VLPL, should redirect to playlist?list=PL...',
5718         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5719         'info_dict': {
5720             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5721             'uploader': 'NoCopyrightSounds',
5722             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5723             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5724             'title': 'NCS : All Releases 💿',
5725             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5726             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5727             'modified_date': r're:\d{8}',
5728             'view_count': int,
5729             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5730             'tags': [],
5731             'channel': 'NoCopyrightSounds',
5732             'availability': 'public',
5733         },
5734         'playlist_mincount': 166,
5735         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5736     }, {
5737         'note': 'Topic, should redirect to playlist?list=UU...',
5738         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5739         'info_dict': {
5740             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5741             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5742             'title': 'Uploads from Royalty Free Music - Topic',
5743             'uploader': 'Royalty Free Music - Topic',
5744             'tags': [],
5745             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5746             'channel': 'Royalty Free Music - Topic',
5747             'view_count': int,
5748             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5749             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5750             'modified_date': r're:\d{8}',
5751             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5752             'description': '',
5753             'availability': 'public',
5754         },
5755         'playlist_mincount': 101,
5756     }, {
5757         # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
5758         # Treat as a general feed
5759         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5760         'info_dict': {
5761             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5762             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5763             'tags': [],
5764         },
5765         'playlist_mincount': 9,
5766     }, {
5767         'note': 'Youtube music Album',
5768         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5769         'info_dict': {
5770             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5771             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5772             'tags': [],
5773             'view_count': int,
5774             'description': '',
5775             'availability': 'unlisted',
5776             'modified_date': r're:\d{8}',
5777         },
5778         'playlist_count': 50,
5779     }, {
5780         'note': 'unlisted single video playlist',
5781         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5782         'info_dict': {
5783             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5784             'uploader': 'colethedj',
5785             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5786             'title': 'yt-dlp unlisted playlist test',
5787             'availability': 'unlisted',
5788             'tags': [],
5789             'modified_date': '20220418',
5790             'channel': 'colethedj',
5791             'view_count': int,
5792             'description': '',
5793             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5794             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5795             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5796         },
5797         'playlist_count': 1,
5798     }, {
5799         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5800         'url': 'https://www.youtube.com/feed/recommended',
5801         'info_dict': {
5802             'id': 'recommended',
5803             'title': 'recommended',
5804             'tags': [],
5805         },
5806         'playlist_mincount': 50,
5807         'params': {
5808             'skip_download': True,
5809             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5810         },
5811     }, {
5812         'note': 'API Fallback: /videos tab, sorted by oldest first',
5813         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5814         'info_dict': {
5815             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5816             'title': 'Cody\'sLab - Videos',
5817             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5818             'uploader': 'Cody\'sLab',
5819             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5820             'channel': 'Cody\'sLab',
5821             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5822             'tags': [],
5823             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5824             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5825             'channel_follower_count': int
5826         },
5827         'playlist_mincount': 650,
5828         'params': {
5829             'skip_download': True,
5830             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5831         },
5832         'skip': 'Query for sorting no longer works',
5833     }, {
5834         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5835         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5836         'info_dict': {
5837             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5838             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5839             'title': 'Uploads from Royalty Free Music - Topic',
5840             'uploader': 'Royalty Free Music - Topic',
5841             'modified_date': r're:\d{8}',
5842             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5843             'description': '',
5844             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5845             'tags': [],
5846             'channel': 'Royalty Free Music - Topic',
5847             'view_count': int,
5848             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5849             'availability': 'public',
5850         },
5851         'playlist_mincount': 101,
5852         'params': {
5853             'skip_download': True,
5854             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5855         },
5856     }, {
5857         'note': 'non-standard redirect to regional channel',
5858         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5859         'only_matching': True
5860     }, {
5861         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5862         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5863         'info_dict': {
5864             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5865             'modified_date': '20220407',
5866             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5867             'tags': [],
5868             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5869             'uploader': 'pukkandan',
5870             'availability': 'unlisted',
5871             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5872             'channel': 'pukkandan',
5873             'description': 'Test for collaborative playlist',
5874             'title': 'yt-dlp test - collaborative playlist',
5875             'view_count': int,
5876             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5877         },
5878         'playlist_mincount': 2
5879     }, {
5880         'note': 'translated tab name',
5881         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5882         'info_dict': {
5883             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5884             'tags': [],
5885             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5886             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5887             'description': 'test description',
5888             'title': 'cole-dlp-test-acc - 再生リスト',
5889             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5890             'uploader': 'cole-dlp-test-acc',
5891             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5892             'channel': 'cole-dlp-test-acc',
5893         },
5894         'playlist_mincount': 1,
5895         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5896         'expected_warnings': ['Preferring "ja"'],
5897     }, {
5898         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5899         'note': 'preferred lang set with playlist with translated video titles',
5900         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5901         'info_dict': {
5902             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5903             'tags': [],
5904             'view_count': int,
5905             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5906             'uploader': 'cole-dlp-test-acc',
5907             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5908             'channel': 'cole-dlp-test-acc',
5909             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5910             'description': 'test',
5911             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5912             'title': 'dlp test playlist',
5913             'availability': 'public',
5914         },
5915         'playlist_mincount': 1,
5916         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5917         'expected_warnings': ['Preferring "ja"'],
5918     }, {
5919         # shorts audio pivot for 2GtVksBMYFM.
5920         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
5921         'info_dict': {
5922             'id': 'sfv_audio_pivot',
5923             'title': 'sfv_audio_pivot',
5924             'tags': [],
5925         },
5926         'playlist_mincount': 50,
5927
5928     }, {
5929         # Channel with a real live tab (not to be mistaken with streams tab)
5930         # Do not treat like it should redirect to live stream
5931         'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
5932         'info_dict': {
5933             'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
5934             'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
5935             'tags': [],
5936         },
5937         'playlist_mincount': 20,
5938     }, {
5939         # Tab name is not the same as tab id
5940         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
5941         'info_dict': {
5942             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5943             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
5944             'tags': [],
5945         },
5946         'playlist_mincount': 8,
5947     }, {
5948         # Home tab id is literally home. Not to get mistaken with featured
5949         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
5950         'info_dict': {
5951             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5952             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
5953             'tags': [],
5954         },
5955         'playlist_mincount': 8,
5956     }, {
5957         # Should get three playlists for videos, shorts and streams tabs
5958         'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5959         'info_dict': {
5960             'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5961             'title': 'Polka Ch. 尾丸ポルカ',
5962             'channel_follower_count': int,
5963             'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5964             'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5965             'uploader': 'Polka Ch. 尾丸ポルカ',
5966             'description': 'md5:3b8df1ac5af337aa206e37ee3d181ec9',
5967             'channel': 'Polka Ch. 尾丸ポルカ',
5968             'tags': 'count:35',
5969             'uploader_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5970             'uploader_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5971         },
5972         'playlist_count': 3,
5973     }, {
5974         # Shorts tab with channel with handle
5975         'url': 'https://www.youtube.com/@NotJustBikes/shorts',
5976         'info_dict': {
5977             'id': 'UC0intLFzLaudFG-xAvUEO-A',
5978             'title': 'Not Just Bikes - Shorts',
5979             'tags': 'count:12',
5980             'uploader': 'Not Just Bikes',
5981             'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5982             'description': 'md5:7513148b1f02b924783157d84c4ea555',
5983             'channel_follower_count': int,
5984             'uploader_id': 'UC0intLFzLaudFG-xAvUEO-A',
5985             'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
5986             'uploader_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5987             'channel': 'Not Just Bikes',
5988         },
5989         'playlist_mincount': 10,
5990     }, {
5991         # Streams tab
5992         'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
5993         'info_dict': {
5994             'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5995             'title': '中村悠一 - Live',
5996             'tags': 'count:7',
5997             'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5998             'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
5999             'uploader_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
6000             'channel': '中村悠一',
6001             'uploader_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
6002             'channel_follower_count': int,
6003             'uploader': '中村悠一',
6004             'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
6005         },
6006         'playlist_mincount': 60,
6007     }, {
6008         # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
6009         # See test_youtube_lists
6010         'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
6011         'only_matching': True,
6012     }, {
6013         # No uploads and no UCID given. Should fail with no uploads error
6014         # See test_youtube_lists
6015         'url': 'https://www.youtube.com/news',
6016         'only_matching': True
6017     }, {
6018         # No videos tab but has a shorts tab
6019         'url': 'https://www.youtube.com/c/TKFShorts',
6020         'info_dict': {
6021             'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6022             'title': 'Shorts Break - Shorts',
6023             'tags': 'count:32',
6024             'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6025             'channel': 'Shorts Break',
6026             'description': 'md5:a6c234cf3d50d878ef8721e34457cd11',
6027             'uploader': 'Shorts Break',
6028             'channel_follower_count': int,
6029             'uploader_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6030             'uploader_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
6031             'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
6032         },
6033         'playlist_mincount': 30,
6034     }, {
6035         # Trending Now Tab. tab id is empty
6036         'url': 'https://www.youtube.com/feed/trending',
6037         'info_dict': {
6038             'id': 'trending',
6039             'title': 'trending - Now',
6040             'tags': [],
6041         },
6042         'playlist_mincount': 30,
6043     }, {
6044         # Trending Gaming Tab. tab id is empty
6045         'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
6046         'info_dict': {
6047             'id': 'trending',
6048             'title': 'trending - Gaming',
6049             'tags': [],
6050         },
6051         'playlist_mincount': 30,
6052     }, {
6053         # Shorts url result in shorts tab
6054         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
6055         'info_dict': {
6056             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6057             'title': 'cole-dlp-test-acc - Shorts',
6058             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
6059             'channel': 'cole-dlp-test-acc',
6060             'description': 'test description',
6061             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6062             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6063             'tags': [],
6064             'uploader': 'cole-dlp-test-acc',
6065             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6066
6067         },
6068         'playlist': [{
6069             'info_dict': {
6070                 '_type': 'url',
6071                 'ie_key': 'Youtube',
6072                 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
6073                 'id': 'sSM9J5YH_60',
6074                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6075                 'title': 'SHORT short',
6076                 'channel': 'cole-dlp-test-acc',
6077                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6078                 'view_count': int,
6079                 'thumbnails': list,
6080             }
6081         }],
6082         'params': {'extract_flat': True},
6083     }, {
6084         # Live video status should be extracted
6085         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
6086         'info_dict': {
6087             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6088             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live',  # TODO, should be Minecraft - Live or Minecraft - Topic - Live
6089             'tags': []
6090         },
6091         'playlist': [{
6092             'info_dict': {
6093                 '_type': 'url',
6094                 'ie_key': 'Youtube',
6095                 'url': 'startswith:https://www.youtube.com/watch?v=',
6096                 'id': str,
6097                 'title': str,
6098                 'live_status': 'is_live',
6099                 'channel_id': str,
6100                 'channel_url': str,
6101                 'concurrent_view_count': int,
6102                 'channel': str,
6103             }
6104         }],
6105         'params': {'extract_flat': True, 'playlist_items': '1'},
6106         'playlist_mincount': 1
6107     }, {
6108         # Channel renderer metadata. Contains number of videos on the channel
6109         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
6110         'info_dict': {
6111             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6112             'title': 'cole-dlp-test-acc - Channels',
6113             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
6114             'channel': 'cole-dlp-test-acc',
6115             'description': 'test description',
6116             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6117             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6118             'tags': [],
6119             'uploader': 'cole-dlp-test-acc',
6120             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6121
6122         },
6123         'playlist': [{
6124             'info_dict': {
6125                 '_type': 'url',
6126                 'ie_key': 'YoutubeTab',
6127                 'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6128                 'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6129                 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6130                 'title': 'PewDiePie',
6131                 'channel': 'PewDiePie',
6132                 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6133                 'thumbnails': list,
6134                 'channel_follower_count': int,
6135                 'playlist_count': int
6136             }
6137         }],
6138         'params': {'extract_flat': True},
6139     }, {
6140         'url': 'https://www.youtube.com/@3blue1brown/about',
6141         'info_dict': {
6142             'id': 'UCYO_jab_esuFRV4b17AJtAw',
6143             'tags': ['Mathematics'],
6144             'title': '3Blue1Brown - About',
6145             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
6146             'channel_follower_count': int,
6147             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
6148             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
6149             'channel': '3Blue1Brown',
6150             'uploader': '3Blue1Brown',
6151             'view_count': int,
6152             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
6153             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
6154         },
6155         'playlist_count': 0,
6156     }]
6157
6158     @classmethod
6159     def suitable(cls, url):
6160         return False if YoutubeIE.suitable(url) else super().suitable(url)
6161
6162     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
6163
6164     def _get_url_mobj(self, url):
6165         mobj = self._URL_RE.match(url).groupdict()
6166         mobj.update((k, '') for k, v in mobj.items() if v is None)
6167         return mobj
6168
6169     def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
6170         tab_name = (tab.get('title') or '').lower()
6171         tab_url = urljoin(base_url, traverse_obj(
6172             tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
6173
6174         tab_id = (tab_url and self._get_url_mobj(tab_url)['tab'][1:]
6175                   or traverse_obj(tab, 'tabIdentifier', expected_type=str))
6176         if tab_id:
6177             return {
6178                 'TAB_ID_SPONSORSHIPS': 'membership',
6179             }.get(tab_id, tab_id), tab_name
6180
6181         # Fallback to tab name if we cannot get the tab id.
6182         # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
6183         # Note that in the case of translated tab name this may result in an empty string, which we don't want.
6184         if tab_name:
6185             self.write_debug(f'Falling back to selected tab name: {tab_name}')
6186         return {
6187             'home': 'featured',
6188             'live': 'streams',
6189         }.get(tab_name, tab_name), tab_name
6190
6191     def _has_tab(self, tabs, tab_id):
6192         return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
6193
6194     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
6195     def _real_extract(self, url, smuggled_data):
6196         item_id = self._match_id(url)
6197         url = urllib.parse.urlunparse(
6198             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
6199         compat_opts = self.get_param('compat_opts', [])
6200
6201         mobj = self._get_url_mobj(url)
6202         pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
6203         if is_channel and smuggled_data.get('is_music_url'):
6204             if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
6205                 return self.url_result(
6206                     f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
6207             elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
6208                 mdata = self._extract_tab_endpoint(
6209                     f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
6210                 murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
6211                                     get_all=False, expected_type=str)
6212                 if not murl:
6213                     raise ExtractorError('Failed to resolve album to playlist')
6214                 return self.url_result(murl, YoutubeTabIE)
6215             elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
6216                 return self.url_result(
6217                     f'https://music.youtube.com/channel/{item_id}{tab}{post}', YoutubeTabIE, item_id)
6218
6219         original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
6220         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
6221             url = f'{pre}/videos{post}'
6222         if smuggled_data.get('is_music_url'):
6223             self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}')
6224
6225         # Handle both video/playlist URLs
6226         qs = parse_qs(url)
6227         video_id, playlist_id = [traverse_obj(qs, (key, 0)) for key in ('v', 'list')]
6228         if not video_id and mobj['not_channel'].startswith('watch'):
6229             if not playlist_id:
6230                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
6231                 raise ExtractorError('A video URL was given without video ID', expected=True)
6232             # Common mistake: https://www.youtube.com/watch?list=playlist_id
6233             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
6234             return self.url_result(
6235                 f'https://www.youtube.com/playlist?list={playlist_id}', YoutubeTabIE, playlist_id)
6236
6237         if not self._yes_playlist(playlist_id, video_id):
6238             return self.url_result(
6239                 f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6240
6241         data, ytcfg = self._extract_data(url, display_id)
6242
6243         # YouTube may provide a non-standard redirect to the regional channel
6244         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
6245         # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
6246         redirect_url = traverse_obj(
6247             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
6248         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
6249             redirect_url = ''.join((urljoin('https://www.youtube.com', redirect_url), tab, post))
6250             self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
6251             return self.url_result(redirect_url, YoutubeTabIE)
6252
6253         tabs, extra_tabs = self._extract_tab_renderers(data), []
6254         if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
6255             selected_tab = self._extract_selected_tab(tabs)
6256             selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url)  # NB: Name may be translated
6257             self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
6258
6259             if not original_tab_id and selected_tab_name:
6260                 self.to_screen('Downloading all uploads of the channel. '
6261                                'To download only the videos in a specific tab, pass the tab\'s URL')
6262                 if self._has_tab(tabs, 'streams'):
6263                     extra_tabs.append(''.join((pre, '/streams', post)))
6264                 if self._has_tab(tabs, 'shorts'):
6265                     extra_tabs.append(''.join((pre, '/shorts', post)))
6266                 # XXX: Members-only tab should also be extracted
6267
6268                 if not extra_tabs and selected_tab_id != 'videos':
6269                     # Channel does not have streams, shorts or videos tabs
6270                     if item_id[:2] != 'UC':
6271                         raise ExtractorError('This channel has no uploads', expected=True)
6272
6273                     # Topic channels don't have /videos. Use the equivalent playlist instead
6274                     pl_id = f'UU{item_id[2:]}'
6275                     pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
6276                     try:
6277                         data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
6278                     except ExtractorError:
6279                         raise ExtractorError('This channel has no uploads', expected=True)
6280                     else:
6281                         item_id, url = pl_id, pl_url
6282                         self.to_screen(
6283                             f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
6284
6285                 elif extra_tabs and selected_tab_id != 'videos':
6286                     # When there are shorts/live tabs but not videos tab
6287                     url, data = f'{pre}{post}', None
6288
6289             elif (original_tab_id or 'videos') != selected_tab_id:
6290                 if original_tab_id == 'live':
6291                     # Live tab should have redirected to the video
6292                     # Except in the case the channel has an actual live tab
6293                     # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
6294                     raise UserNotLive(video_id=item_id)
6295                 elif selected_tab_name:
6296                     raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
6297
6298                 # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
6299                 url = f'{pre}{post}'
6300
6301         # YouTube sometimes provides a button to reload playlist with unavailable videos.
6302         if 'no-youtube-unavailable-videos' not in compat_opts:
6303             data = self._reload_with_unavailable_videos(display_id, data, ytcfg) or data
6304         self._extract_and_report_alerts(data, only_once=True)
6305
6306         tabs, entries = self._extract_tab_renderers(data), []
6307         if tabs:
6308             entries = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
6309             entries[0].update({
6310                 'extractor_key': YoutubeTabIE.ie_key(),
6311                 'extractor': YoutubeTabIE.IE_NAME,
6312                 'webpage_url': url,
6313             })
6314         if self.get_param('playlist_items') == '0':
6315             entries.extend(self.url_result(u, YoutubeTabIE) for u in extra_tabs)
6316         else:  # Users expect to get all `video_id`s even with `--flat-playlist`. So don't return `url_result`
6317             entries.extend(map(self._real_extract, extra_tabs))
6318
6319         if len(entries) == 1:
6320             return entries[0]
6321         elif entries:
6322             metadata = self._extract_metadata_from_tabs(item_id, data)
6323             uploads_url = 'the Uploads (UU) playlist URL'
6324             if try_get(metadata, lambda x: x['channel_id'].startswith('UC')):
6325                 uploads_url = f'https://www.youtube.com/playlist?list=UU{metadata["channel_id"][2:]}'
6326             self.to_screen(
6327                 'Downloading as multiple playlists, separated by tabs. '
6328                 f'To download as a single playlist instead, pass {uploads_url}')
6329             return self.playlist_result(entries, item_id, **metadata)
6330
6331         # Inline playlist
6332         playlist = traverse_obj(
6333             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
6334         if playlist:
6335             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
6336
6337         video_id = traverse_obj(
6338             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
6339         if video_id:
6340             if tab != '/live':  # live tab is expected to redirect to video
6341                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
6342             return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6343
6344         raise ExtractorError('Unable to recognize tab page')
6345
6346
6347 class YoutubePlaylistIE(InfoExtractor):
6348     IE_DESC = 'YouTube playlists'
6349     _VALID_URL = r'''(?x)(?:
6350                         (?:https?://)?
6351                         (?:\w+\.)?
6352                         (?:
6353                             (?:
6354                                 youtube(?:kids)?\.com|
6355                                 %(invidious)s
6356                             )
6357                             /.*?\?.*?\blist=
6358                         )?
6359                         (?P<id>%(playlist_id)s)
6360                      )''' % {
6361         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
6362         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
6363     }
6364     IE_NAME = 'youtube:playlist'
6365     _TESTS = [{
6366         'note': 'issue #673',
6367         'url': 'PLBB231211A4F62143',
6368         'info_dict': {
6369             'title': '[OLD]Team Fortress 2 (Class-based LP)',
6370             'id': 'PLBB231211A4F62143',
6371             'uploader': 'Wickman',
6372             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6373             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
6374             'view_count': int,
6375             'uploader_url': 'https://www.youtube.com/c/WickmanVT',
6376             'modified_date': r're:\d{8}',
6377             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6378             'channel': 'Wickman',
6379             'tags': [],
6380             'channel_url': 'https://www.youtube.com/c/WickmanVT',
6381             'availability': 'public',
6382         },
6383         'playlist_mincount': 29,
6384     }, {
6385         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6386         'info_dict': {
6387             'title': 'YDL_safe_search',
6388             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6389         },
6390         'playlist_count': 2,
6391         'skip': 'This playlist is private',
6392     }, {
6393         'note': 'embedded',
6394         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6395         'playlist_count': 4,
6396         'info_dict': {
6397             'title': 'JODA15',
6398             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6399             'uploader': 'milan',
6400             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6401             'description': '',
6402             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6403             'tags': [],
6404             'modified_date': '20140919',
6405             'view_count': int,
6406             'channel': 'milan',
6407             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6408             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6409             'availability': 'public',
6410         },
6411         'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'],
6412     }, {
6413         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6414         'playlist_mincount': 455,
6415         'info_dict': {
6416             'title': '2018 Chinese New Singles (11/6 updated)',
6417             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6418             'uploader': 'LBK',
6419             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6420             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
6421             'channel': 'LBK',
6422             'view_count': int,
6423             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
6424             'tags': [],
6425             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
6426             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6427             'modified_date': r're:\d{8}',
6428             'availability': 'public',
6429         },
6430         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
6431     }, {
6432         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
6433         'only_matching': True,
6434     }, {
6435         # music album playlist
6436         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
6437         'only_matching': True,
6438     }]
6439
6440     @classmethod
6441     def suitable(cls, url):
6442         if YoutubeTabIE.suitable(url):
6443             return False
6444         from ..utils import parse_qs
6445         qs = parse_qs(url)
6446         if qs.get('v', [None])[0]:
6447             return False
6448         return super().suitable(url)
6449
6450     def _real_extract(self, url):
6451         playlist_id = self._match_id(url)
6452         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
6453         url = update_url_query(
6454             'https://www.youtube.com/playlist',
6455             parse_qs(url) or {'list': playlist_id})
6456         if is_music_url:
6457             url = smuggle_url(url, {'is_music_url': True})
6458         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6459
6460
6461 class YoutubeYtBeIE(InfoExtractor):
6462     IE_DESC = 'youtu.be'
6463     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
6464     _TESTS = [{
6465         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
6466         'info_dict': {
6467             'id': 'yeWKywCrFtk',
6468             'ext': 'mp4',
6469             'title': 'Small Scale Baler and Braiding Rugs',
6470             'uploader': 'Backus-Page House Museum',
6471             'uploader_id': 'backuspagemuseum',
6472             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
6473             'upload_date': '20161008',
6474             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
6475             'categories': ['Nonprofits & Activism'],
6476             'tags': list,
6477             'like_count': int,
6478             'age_limit': 0,
6479             'playable_in_embed': True,
6480             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
6481             'channel': 'Backus-Page House Museum',
6482             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
6483             'live_status': 'not_live',
6484             'view_count': int,
6485             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
6486             'availability': 'public',
6487             'duration': 59,
6488             'comment_count': int,
6489             'channel_follower_count': int
6490         },
6491         'params': {
6492             'noplaylist': True,
6493             'skip_download': True,
6494         },
6495     }, {
6496         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
6497         'only_matching': True,
6498     }]
6499
6500     def _real_extract(self, url):
6501         mobj = self._match_valid_url(url)
6502         video_id = mobj.group('id')
6503         playlist_id = mobj.group('playlist_id')
6504         return self.url_result(
6505             update_url_query('https://www.youtube.com/watch', {
6506                 'v': video_id,
6507                 'list': playlist_id,
6508                 'feature': 'youtu.be',
6509             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6510
6511
6512 class YoutubeLivestreamEmbedIE(InfoExtractor):
6513     IE_DESC = 'YouTube livestream embeds'
6514     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6515     _TESTS = [{
6516         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6517         'only_matching': True,
6518     }]
6519
6520     def _real_extract(self, url):
6521         channel_id = self._match_id(url)
6522         return self.url_result(
6523             f'https://www.youtube.com/channel/{channel_id}/live',
6524             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6525
6526
6527 class YoutubeYtUserIE(InfoExtractor):
6528     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6529     IE_NAME = 'youtube:user'
6530     _VALID_URL = r'ytuser:(?P<id>.+)'
6531     _TESTS = [{
6532         'url': 'ytuser:phihag',
6533         'only_matching': True,
6534     }]
6535
6536     def _real_extract(self, url):
6537         user_id = self._match_id(url)
6538         return self.url_result(f'https://www.youtube.com/user/{user_id}', YoutubeTabIE, user_id)
6539
6540
6541 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6542     IE_NAME = 'youtube:favorites'
6543     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6544     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6545     _LOGIN_REQUIRED = True
6546     _TESTS = [{
6547         'url': ':ytfav',
6548         'only_matching': True,
6549     }, {
6550         'url': ':ytfavorites',
6551         'only_matching': True,
6552     }]
6553
6554     def _real_extract(self, url):
6555         return self.url_result(
6556             'https://www.youtube.com/playlist?list=LL',
6557             ie=YoutubeTabIE.ie_key())
6558
6559
6560 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6561     IE_NAME = 'youtube:notif'
6562     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6563     _VALID_URL = r':ytnotif(?:ication)?s?'
6564     _LOGIN_REQUIRED = True
6565     _TESTS = [{
6566         'url': ':ytnotif',
6567         'only_matching': True,
6568     }, {
6569         'url': ':ytnotifications',
6570         'only_matching': True,
6571     }]
6572
6573     def _extract_notification_menu(self, response, continuation_list):
6574         notification_list = traverse_obj(
6575             response,
6576             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6577             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6578             expected_type=list) or []
6579         continuation_list[0] = None
6580         for item in notification_list:
6581             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6582             if entry:
6583                 yield entry
6584             continuation = item.get('continuationItemRenderer')
6585             if continuation:
6586                 continuation_list[0] = continuation
6587
6588     def _extract_notification_renderer(self, notification):
6589         video_id = traverse_obj(
6590             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6591         url = f'https://www.youtube.com/watch?v={video_id}'
6592         channel_id = None
6593         if not video_id:
6594             browse_ep = traverse_obj(
6595                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6596             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
6597             post_id = self._search_regex(
6598                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6599                 'post id', default=None)
6600             if not channel_id or not post_id:
6601                 return
6602             # The direct /post url redirects to this in the browser
6603             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6604
6605         channel = traverse_obj(
6606             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6607             expected_type=str)
6608         notification_title = self._get_text(notification, 'shortMessage')
6609         if notification_title:
6610             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6611         # TODO: handle recommended videos
6612         title = self._search_regex(
6613             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6614             'video title', default=None)
6615         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6616                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6617                      else None)
6618         return {
6619             '_type': 'url',
6620             'url': url,
6621             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6622             'video_id': video_id,
6623             'title': title,
6624             'channel_id': channel_id,
6625             'channel': channel,
6626             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6627             'timestamp': timestamp,
6628         }
6629
6630     def _notification_menu_entries(self, ytcfg):
6631         continuation_list = [None]
6632         response = None
6633         for page in itertools.count(1):
6634             ctoken = traverse_obj(
6635                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6636             response = self._extract_response(
6637                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6638                 ep='notification/get_notification_menu', check_get_keys='actions',
6639                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6640             yield from self._extract_notification_menu(response, continuation_list)
6641             if not continuation_list[0]:
6642                 break
6643
6644     def _real_extract(self, url):
6645         display_id = 'notifications'
6646         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6647         self._report_playlist_authcheck(ytcfg)
6648         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6649
6650
6651 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6652     IE_DESC = 'YouTube search'
6653     IE_NAME = 'youtube:search'
6654     _SEARCH_KEY = 'ytsearch'
6655     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6656     _TESTS = [{
6657         'url': 'ytsearch5:youtube-dl test video',
6658         'playlist_count': 5,
6659         'info_dict': {
6660             'id': 'youtube-dl test video',
6661             'title': 'youtube-dl test video',
6662         }
6663     }]
6664
6665
6666 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6667     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6668     _SEARCH_KEY = 'ytsearchdate'
6669     IE_DESC = 'YouTube search, newest videos first'
6670     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6671     _TESTS = [{
6672         'url': 'ytsearchdate5:youtube-dl test video',
6673         'playlist_count': 5,
6674         'info_dict': {
6675             'id': 'youtube-dl test video',
6676             'title': 'youtube-dl test video',
6677         }
6678     }]
6679
6680
6681 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6682     IE_DESC = 'YouTube search URLs with sorting and filter support'
6683     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6684     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6685     _TESTS = [{
6686         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6687         'playlist_mincount': 5,
6688         'info_dict': {
6689             'id': 'youtube-dl test video',
6690             'title': 'youtube-dl test video',
6691         }
6692     }, {
6693         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6694         'playlist_mincount': 5,
6695         'info_dict': {
6696             'id': 'python',
6697             'title': 'python',
6698         }
6699     }, {
6700         'url': 'https://www.youtube.com/results?search_query=%23cats',
6701         'playlist_mincount': 1,
6702         'info_dict': {
6703             'id': '#cats',
6704             'title': '#cats',
6705             # The test suite does not have support for nested playlists
6706             # 'entries': [{
6707             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6708             #     'title': '#cats',
6709             # }],
6710         },
6711     }, {
6712         # Channel results
6713         'url': 'https://www.youtube.com/results?search_query=kurzgesagt&sp=EgIQAg%253D%253D',
6714         'info_dict': {
6715             'id': 'kurzgesagt',
6716             'title': 'kurzgesagt',
6717         },
6718         'playlist': [{
6719             'info_dict': {
6720                 '_type': 'url',
6721                 'id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6722                 'url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6723                 'ie_key': 'YoutubeTab',
6724                 'channel': 'Kurzgesagt – In a Nutshell',
6725                 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
6726                 'title': 'Kurzgesagt – In a Nutshell',
6727                 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6728                 'playlist_count': int,  # XXX: should have a way of saying > 1
6729                 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6730                 'thumbnails': list
6731             }
6732         }],
6733         'params': {'extract_flat': True, 'playlist_items': '1'},
6734         'playlist_mincount': 1,
6735     }, {
6736         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6737         'only_matching': True,
6738     }]
6739
6740     def _real_extract(self, url):
6741         qs = parse_qs(url)
6742         query = (qs.get('search_query') or qs.get('q'))[0]
6743         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6744
6745
6746 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6747     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6748     IE_NAME = 'youtube:music:search_url'
6749     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6750     _TESTS = [{
6751         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6752         'playlist_count': 16,
6753         'info_dict': {
6754             'id': 'royalty free music',
6755             'title': 'royalty free music',
6756         }
6757     }, {
6758         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6759         'playlist_mincount': 30,
6760         'info_dict': {
6761             'id': 'royalty free music - songs',
6762             'title': 'royalty free music - songs',
6763         },
6764         'params': {'extract_flat': 'in_playlist'}
6765     }, {
6766         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6767         'playlist_mincount': 30,
6768         'info_dict': {
6769             'id': 'royalty free music - community playlists',
6770             'title': 'royalty free music - community playlists',
6771         },
6772         'params': {'extract_flat': 'in_playlist'}
6773     }]
6774
6775     _SECTIONS = {
6776         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6777         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6778         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6779         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6780         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6781         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6782     }
6783
6784     def _real_extract(self, url):
6785         qs = parse_qs(url)
6786         query = (qs.get('search_query') or qs.get('q'))[0]
6787         params = qs.get('sp', (None,))[0]
6788         if params:
6789             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6790         else:
6791             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6792             params = self._SECTIONS.get(section)
6793             if not params:
6794                 section = None
6795         title = join_nonempty(query, section, delim=' - ')
6796         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6797
6798
6799 class YoutubeFeedsInfoExtractor(InfoExtractor):
6800     """
6801     Base class for feed extractors
6802     Subclasses must re-define the _FEED_NAME property.
6803     """
6804     _LOGIN_REQUIRED = True
6805     _FEED_NAME = 'feeds'
6806
6807     def _real_initialize(self):
6808         YoutubeBaseInfoExtractor._check_login_required(self)
6809
6810     @classproperty
6811     def IE_NAME(self):
6812         return f'youtube:{self._FEED_NAME}'
6813
6814     def _real_extract(self, url):
6815         return self.url_result(
6816             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6817
6818
6819 class YoutubeWatchLaterIE(InfoExtractor):
6820     IE_NAME = 'youtube:watchlater'
6821     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6822     _VALID_URL = r':ytwatchlater'
6823     _TESTS = [{
6824         'url': ':ytwatchlater',
6825         'only_matching': True,
6826     }]
6827
6828     def _real_extract(self, url):
6829         return self.url_result(
6830             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6831
6832
6833 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6834     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6835     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6836     _FEED_NAME = 'recommended'
6837     _LOGIN_REQUIRED = False
6838     _TESTS = [{
6839         'url': ':ytrec',
6840         'only_matching': True,
6841     }, {
6842         'url': ':ytrecommended',
6843         'only_matching': True,
6844     }, {
6845         'url': 'https://youtube.com',
6846         'only_matching': True,
6847     }]
6848
6849
6850 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6851     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6852     _VALID_URL = r':ytsub(?:scription)?s?'
6853     _FEED_NAME = 'subscriptions'
6854     _TESTS = [{
6855         'url': ':ytsubs',
6856         'only_matching': True,
6857     }, {
6858         'url': ':ytsubscriptions',
6859         'only_matching': True,
6860     }]
6861
6862
6863 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6864     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6865     _VALID_URL = r':ythis(?:tory)?'
6866     _FEED_NAME = 'history'
6867     _TESTS = [{
6868         'url': ':ythistory',
6869         'only_matching': True,
6870     }]
6871
6872
6873 class YoutubeStoriesIE(InfoExtractor):
6874     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6875     IE_NAME = 'youtube:stories'
6876     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6877     _TESTS = [{
6878         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6879         'only_matching': True,
6880     }]
6881
6882     def _real_extract(self, url):
6883         playlist_id = f'RLTD{self._match_id(url)}'
6884         return self.url_result(
6885             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6886             ie=YoutubeTabIE, video_id=playlist_id)
6887
6888
6889 class YoutubeShortsAudioPivotIE(InfoExtractor):
6890     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
6891     IE_NAME = 'youtube:shorts:pivot:audio'
6892     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
6893     _TESTS = [{
6894         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
6895         'only_matching': True,
6896     }]
6897
6898     @staticmethod
6899     def _generate_audio_pivot_params(video_id):
6900         """
6901         Generates sfv_audio_pivot browse params for this video id
6902         """
6903         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
6904         return urllib.parse.quote(base64.b64encode(pb_params).decode())
6905
6906     def _real_extract(self, url):
6907         video_id = self._match_id(url)
6908         return self.url_result(
6909             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
6910             ie=YoutubeTabIE)
6911
6912
6913 class YoutubeTruncatedURLIE(InfoExtractor):
6914     IE_NAME = 'youtube:truncated_url'
6915     IE_DESC = False  # Do not list
6916     _VALID_URL = r'''(?x)
6917         (?:https?://)?
6918         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6919         (?:watch\?(?:
6920             feature=[a-z_]+|
6921             annotation_id=annotation_[^&]+|
6922             x-yt-cl=[0-9]+|
6923             hl=[^&]*|
6924             t=[0-9]+
6925         )?
6926         |
6927             attribution_link\?a=[^&]+
6928         )
6929         $
6930     '''
6931
6932     _TESTS = [{
6933         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6934         'only_matching': True,
6935     }, {
6936         'url': 'https://www.youtube.com/watch?',
6937         'only_matching': True,
6938     }, {
6939         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6940         'only_matching': True,
6941     }, {
6942         'url': 'https://www.youtube.com/watch?feature=foo',
6943         'only_matching': True,
6944     }, {
6945         'url': 'https://www.youtube.com/watch?hl=en-GB',
6946         'only_matching': True,
6947     }, {
6948         'url': 'https://www.youtube.com/watch?t=2372',
6949         'only_matching': True,
6950     }]
6951
6952     def _real_extract(self, url):
6953         raise ExtractorError(
6954             'Did you forget to quote the URL? Remember that & is a meta '
6955             'character in most shells, so you want to put the URL in quotes, '
6956             'like  youtube-dl '
6957             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6958             ' or simply  youtube-dl BaW_jenozKc  .',
6959             expected=True)
6960
6961
6962 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6963     IE_NAME = 'youtube:clip'
6964     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6965     _TESTS = [{
6966         # FIXME: Other metadata should be extracted from the clip, not from the base video
6967         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6968         'info_dict': {
6969             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6970             'ext': 'mp4',
6971             'section_start': 29.0,
6972             'section_end': 39.7,
6973             'duration': 10.7,
6974             'age_limit': 0,
6975             'availability': 'public',
6976             'categories': ['Gaming'],
6977             'channel': 'Scott The Woz',
6978             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6979             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6980             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6981             'like_count': int,
6982             'playable_in_embed': True,
6983             'tags': 'count:17',
6984             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6985             'title': 'Mobile Games on Console - Scott The Woz',
6986             'upload_date': '20210920',
6987             'uploader': 'Scott The Woz',
6988             'uploader_id': 'scottthewoz',
6989             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6990             'view_count': int,
6991             'live_status': 'not_live',
6992             'channel_follower_count': int
6993         }
6994     }]
6995
6996     def _real_extract(self, url):
6997         clip_id = self._match_id(url)
6998         _, data = self._extract_webpage(url, clip_id)
6999
7000         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
7001         if not video_id:
7002             raise ExtractorError('Unable to find video ID')
7003
7004         clip_data = traverse_obj(data, (
7005             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
7006             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
7007             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
7008             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
7009
7010         return {
7011             '_type': 'url_transparent',
7012             'url': f'https://www.youtube.com/watch?v={video_id}',
7013             'ie_key': YoutubeIE.ie_key(),
7014             'id': clip_id,
7015             'section_start': int(clip_data['startTimeMs']) / 1000,
7016             'section_end': int(clip_data['endTimeMs']) / 1000,
7017         }
7018
7019
7020 class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
7021     IE_NAME = 'youtube:consent'
7022     IE_DESC = False  # Do not list
7023     _VALID_URL = r'https?://consent\.youtube\.com/m\?'
7024     _TESTS = [{
7025         'url': 'https://consent.youtube.com/m?continue=https%3A%2F%2Fwww.youtube.com%2Flive%2FqVv6vCqciTM%3Fcbrd%3D1&gl=NL&m=0&pc=yt&hl=en&src=1',
7026         'info_dict': {
7027             'id': 'qVv6vCqciTM',
7028             'ext': 'mp4',
7029             'age_limit': 0,
7030             'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
7031             'comment_count': int,
7032             'chapters': 'count:13',
7033             'upload_date': '20221223',
7034             'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
7035             'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
7036             'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
7037             'like_count': int,
7038             'release_date': '20221223',
7039             'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
7040             'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
7041             'view_count': int,
7042             'playable_in_embed': True,
7043             'duration': 4438,
7044             'availability': 'public',
7045             'channel_follower_count': int,
7046             'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
7047             'categories': ['Entertainment'],
7048             'live_status': 'was_live',
7049             'release_timestamp': 1671793345,
7050             'channel': 'さなちゃんねる',
7051             'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
7052             'uploader': 'さなちゃんねる',
7053         },
7054         'add_ie': ['Youtube'],
7055         'params': {'skip_download': 'Youtube'},
7056     }]
7057
7058     def _real_extract(self, url):
7059         redirect_url = url_or_none(parse_qs(url).get('continue', [None])[-1])
7060         if not redirect_url:
7061             raise ExtractorError('Invalid cookie consent redirect URL', expected=True)
7062         return self.url_result(redirect_url)
7063
7064
7065 class YoutubeTruncatedIDIE(InfoExtractor):
7066     IE_NAME = 'youtube:truncated_id'
7067     IE_DESC = False  # Do not list
7068     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
7069
7070     _TESTS = [{
7071         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
7072         'only_matching': True,
7073     }]
7074
7075     def _real_extract(self, url):
7076         video_id = self._match_id(url)
7077         raise ExtractorError(
7078             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
7079             expected=True)