yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import collections
   4 import copy
   5 import datetime
   6 import enum
   7 import hashlib
   8 import itertools
   9 import json
  10 import math
  11 import os.path
  12 import random
  13 import re
  14 import sys
  15 import threading
  16 import time
  17 import traceback
  18 import urllib.error
  19 import urllib.parse
  20
  21 from .common import InfoExtractor, SearchInfoExtractor
  22 from .openload import PhantomJSwrapper
  23 from ..compat import functools
  24 from ..jsinterp import JSInterpreter
  25 from ..utils import (
  26     NO_DEFAULT,
  27     ExtractorError,
  28     LazyList,
  29     UserNotLive,
  30     bug_reports_message,
  31     classproperty,
  32     clean_html,
  33     datetime_from_str,
  34     dict_get,
  35     filter_dict,
  36     float_or_none,
  37     format_field,
  38     get_first,
  39     int_or_none,
  40     is_html,
  41     join_nonempty,
  42     js_to_json,
  43     mimetype2ext,
  44     network_exceptions,
  45     orderedSet,
  46     parse_codecs,
  47     parse_count,
  48     parse_duration,
  49     parse_iso8601,
  50     parse_qs,
  51     qualities,
  52     remove_start,
  53     smuggle_url,
  54     str_or_none,
  55     str_to_int,
  56     strftime_or_none,
  57     traverse_obj,
  58     try_get,
  59     unescapeHTML,
  60     unified_strdate,
  61     unified_timestamp,
  62     unsmuggle_url,
  63     update_url_query,
  64     url_or_none,
  65     urljoin,
  66     variadic,
  67 )
  68
  69 STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client'
  70 # any clients starting with _ cannot be explicitly requested by the user
  71 INNERTUBE_CLIENTS = {
  72     'web': {
  73         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  74         'INNERTUBE_CONTEXT': {
  75             'client': {
  76                 'clientName': 'WEB',
  77                 'clientVersion': '2.20220801.00.00',
  78             }
  79         },
  80         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  81     },
  82     'web_embedded': {
  83         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  84         'INNERTUBE_CONTEXT': {
  85             'client': {
  86                 'clientName': 'WEB_EMBEDDED_PLAYER',
  87                 'clientVersion': '1.20220731.00.00',
  88             },
  89         },
  90         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  91     },
  92     'web_music': {
  93         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  94         'INNERTUBE_HOST': 'music.youtube.com',
  95         'INNERTUBE_CONTEXT': {
  96             'client': {
  97                 'clientName': 'WEB_REMIX',
  98                 'clientVersion': '1.20220727.01.00',
  99             }
 100         },
 101         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 102     },
 103     'web_creator': {
 104         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 105         'INNERTUBE_CONTEXT': {
 106             'client': {
 107                 'clientName': 'WEB_CREATOR',
 108                 'clientVersion': '1.20220726.00.00',
 109             }
 110         },
 111         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 112     },
 113     'android': {
 114         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 115         'INNERTUBE_CONTEXT': {
 116             'client': {
 117                 'clientName': 'ANDROID',
 118                 'clientVersion': '17.31.35',
 119                 'androidSdkVersion': 30,
 120                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 121             }
 122         },
 123         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 124         'REQUIRE_JS_PLAYER': False
 125     },
 126     'android_embedded': {
 127         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 128         'INNERTUBE_CONTEXT': {
 129             'client': {
 130                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 131                 'clientVersion': '17.31.35',
 132                 'androidSdkVersion': 30,
 133                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 134             },
 135         },
 136         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 137         'REQUIRE_JS_PLAYER': False
 138     },
 139     'android_music': {
 140         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 141         'INNERTUBE_CONTEXT': {
 142             'client': {
 143                 'clientName': 'ANDROID_MUSIC',
 144                 'clientVersion': '5.16.51',
 145                 'androidSdkVersion': 30,
 146                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 147             }
 148         },
 149         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 150         'REQUIRE_JS_PLAYER': False
 151     },
 152     'android_creator': {
 153         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 154         'INNERTUBE_CONTEXT': {
 155             'client': {
 156                 'clientName': 'ANDROID_CREATOR',
 157                 'clientVersion': '22.30.100',
 158                 'androidSdkVersion': 30,
 159                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 160             },
 161         },
 162         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 163         'REQUIRE_JS_PLAYER': False
 164     },
 165     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 166     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 167     'ios': {
 168         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 169         'INNERTUBE_CONTEXT': {
 170             'client': {
 171                 'clientName': 'IOS',
 172                 'clientVersion': '17.33.2',
 173                 'deviceModel': 'iPhone14,3',
 174                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 175             }
 176         },
 177         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 178         'REQUIRE_JS_PLAYER': False
 179     },
 180     'ios_embedded': {
 181         'INNERTUBE_CONTEXT': {
 182             'client': {
 183                 'clientName': 'IOS_MESSAGES_EXTENSION',
 184                 'clientVersion': '17.33.2',
 185                 'deviceModel': 'iPhone14,3',
 186                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 187             },
 188         },
 189         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 190         'REQUIRE_JS_PLAYER': False
 191     },
 192     'ios_music': {
 193         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 194         'INNERTUBE_CONTEXT': {
 195             'client': {
 196                 'clientName': 'IOS_MUSIC',
 197                 'clientVersion': '5.21',
 198                 'deviceModel': 'iPhone14,3',
 199                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 200             },
 201         },
 202         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 203         'REQUIRE_JS_PLAYER': False
 204     },
 205     'ios_creator': {
 206         'INNERTUBE_CONTEXT': {
 207             'client': {
 208                 'clientName': 'IOS_CREATOR',
 209                 'clientVersion': '22.33.101',
 210                 'deviceModel': 'iPhone14,3',
 211                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 212             },
 213         },
 214         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 215         'REQUIRE_JS_PLAYER': False
 216     },
 217     # mweb has 'ultralow' formats
 218     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 219     'mweb': {
 220         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 221         'INNERTUBE_CONTEXT': {
 222             'client': {
 223                 'clientName': 'MWEB',
 224                 'clientVersion': '2.20220801.00.00',
 225             }
 226         },
 227         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 228     },
 229     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 230     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 231     'tv_embedded': {
 232         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 233         'INNERTUBE_CONTEXT': {
 234             'client': {
 235                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 236                 'clientVersion': '2.0',
 237             },
 238         },
 239         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 240     },
 241 }
 242
 243
 244 def _split_innertube_client(client_name):
 245     variant, *base = client_name.rsplit('.', 1)
 246     if base:
 247         return variant, base[0], variant
 248     base, *variant = client_name.split('_', 1)
 249     return client_name, base, variant[0] if variant else None
 250
 251
 252 def short_client_name(client_name):
 253     main, *parts = _split_innertube_client(client_name)[0].replace('embedscreen', 'e_s').split('_')
 254     return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
 255
 256
 257 def build_innertube_clients():
 258     THIRD_PARTY = {
 259         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 260     }
 261     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 262     priority = qualities(BASE_CLIENTS[::-1])
 263
 264     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 265         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 266         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 267         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 268         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 269
 270         _, base_client, variant = _split_innertube_client(client)
 271         ytcfg['priority'] = 10 * priority(base_client)
 272
 273         if not variant:
 274             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 275             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 276             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 277             embedscreen['priority'] -= 3
 278         elif variant == 'embedded':
 279             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 280             ytcfg['priority'] -= 2
 281         else:
 282             ytcfg['priority'] -= 3
 283
 284
 285 build_innertube_clients()
 286
 287
 288 class BadgeType(enum.Enum):
 289     AVAILABILITY_UNLISTED = enum.auto()
 290     AVAILABILITY_PRIVATE = enum.auto()
 291     AVAILABILITY_PUBLIC = enum.auto()
 292     AVAILABILITY_PREMIUM = enum.auto()
 293     AVAILABILITY_SUBSCRIPTION = enum.auto()
 294     LIVE_NOW = enum.auto()
 295
 296
 297 class YoutubeBaseInfoExtractor(InfoExtractor):
 298     """Provide base functions for Youtube extractors"""
 299
 300     _RESERVED_NAMES = (
 301         r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
 302         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 303         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 304         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 305
 306     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 307
 308     # _NETRC_MACHINE = 'youtube'
 309
 310     # If True it will raise an error if no login info is provided
 311     _LOGIN_REQUIRED = False
 312
 313     _INVIDIOUS_SITES = (
 314         # invidious-redirect websites
 315         r'(?:www\.)?redirect\.invidious\.io',
 316         r'(?:(?:www|dev)\.)?invidio\.us',
 317         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 318         r'(?:www\.)?invidious\.pussthecat\.org',
 319         r'(?:www\.)?invidious\.zee\.li',
 320         r'(?:www\.)?invidious\.ethibox\.fr',
 321         r'(?:www\.)?iv\.ggtyler\.dev',
 322         r'(?:www\.)?inv\.vern\.i2p',
 323         r'(?:www\.)?am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd\.onion',
 324         r'(?:www\.)?inv\.riverside\.rocks',
 325         r'(?:www\.)?invidious\.silur\.me',
 326         r'(?:www\.)?inv\.bp\.projectsegfau\.lt',
 327         r'(?:www\.)?invidious\.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid\.onion',
 328         r'(?:www\.)?invidious\.slipfox\.xyz',
 329         r'(?:www\.)?invidious\.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd\.onion',
 330         r'(?:www\.)?inv\.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad\.onion',
 331         r'(?:www\.)?invidious\.tiekoetter\.com',
 332         r'(?:www\.)?iv\.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd\.onion',
 333         r'(?:www\.)?invidious\.nerdvpn\.de',
 334         r'(?:www\.)?invidious\.weblibre\.org',
 335         r'(?:www\.)?inv\.odyssey346\.dev',
 336         r'(?:www\.)?invidious\.dhusch\.de',
 337         r'(?:www\.)?iv\.melmac\.space',
 338         r'(?:www\.)?watch\.thekitty\.zone',
 339         r'(?:www\.)?invidious\.privacydev\.net',
 340         r'(?:www\.)?ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid\.onion',
 341         r'(?:www\.)?invidious\.drivet\.xyz',
 342         r'(?:www\.)?vid\.priv\.au',
 343         r'(?:www\.)?euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd\.onion',
 344         r'(?:www\.)?inv\.vern\.cc',
 345         r'(?:www\.)?invidious\.esmailelbob\.xyz',
 346         r'(?:www\.)?invidious\.sethforprivacy\.com',
 347         r'(?:www\.)?yt\.oelrichsgarcia\.de',
 348         r'(?:www\.)?yt\.artemislena\.eu',
 349         r'(?:www\.)?invidious\.flokinet\.to',
 350         r'(?:www\.)?invidious\.baczek\.me',
 351         r'(?:www\.)?y\.com\.sb',
 352         r'(?:www\.)?invidious\.epicsite\.xyz',
 353         r'(?:www\.)?invidious\.lidarshield\.cloud',
 354         r'(?:www\.)?yt\.funami\.tech',
 355         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 356         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 357         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 358         # youtube-dl invidious instances list
 359         r'(?:(?:www|no)\.)?invidiou\.sh',
 360         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 361         r'(?:www\.)?invidious\.kabi\.tk',
 362         r'(?:www\.)?invidious\.mastodon\.host',
 363         r'(?:www\.)?invidious\.zapashcanon\.fr',
 364         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 365         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 366         r'(?:www\.)?invidious\.himiko\.cloud',
 367         r'(?:www\.)?invidious\.reallyancient\.tech',
 368         r'(?:www\.)?invidious\.tube',
 369         r'(?:www\.)?invidiou\.site',
 370         r'(?:www\.)?invidious\.site',
 371         r'(?:www\.)?invidious\.xyz',
 372         r'(?:www\.)?invidious\.nixnet\.xyz',
 373         r'(?:www\.)?invidious\.048596\.xyz',
 374         r'(?:www\.)?invidious\.drycat\.fr',
 375         r'(?:www\.)?inv\.skyn3t\.in',
 376         r'(?:www\.)?tube\.poal\.co',
 377         r'(?:www\.)?tube\.connect\.cafe',
 378         r'(?:www\.)?vid\.wxzm\.sx',
 379         r'(?:www\.)?vid\.mint\.lgbt',
 380         r'(?:www\.)?vid\.puffyan\.us',
 381         r'(?:www\.)?yewtu\.be',
 382         r'(?:www\.)?yt\.elukerio\.org',
 383         r'(?:www\.)?yt\.lelux\.fi',
 384         r'(?:www\.)?invidious\.ggc-project\.de',
 385         r'(?:www\.)?yt\.maisputain\.ovh',
 386         r'(?:www\.)?ytprivate\.com',
 387         r'(?:www\.)?invidious\.13ad\.de',
 388         r'(?:www\.)?invidious\.toot\.koeln',
 389         r'(?:www\.)?invidious\.fdn\.fr',
 390         r'(?:www\.)?watch\.nettohikari\.com',
 391         r'(?:www\.)?invidious\.namazso\.eu',
 392         r'(?:www\.)?invidious\.silkky\.cloud',
 393         r'(?:www\.)?invidious\.exonip\.de',
 394         r'(?:www\.)?invidious\.riverside\.rocks',
 395         r'(?:www\.)?invidious\.blamefran\.net',
 396         r'(?:www\.)?invidious\.moomoo\.de',
 397         r'(?:www\.)?ytb\.trom\.tf',
 398         r'(?:www\.)?yt\.cyberhost\.uk',
 399         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 400         r'(?:www\.)?qklhadlycap4cnod\.onion',
 401         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 402         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 403         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 404         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 405         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 406         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 407         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 408         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 409         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 410         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 411         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 412         r'(?:www\.)?piped\.kavin\.rocks',
 413         r'(?:www\.)?piped\.tokhmi\.xyz',
 414         r'(?:www\.)?piped\.syncpundit\.io',
 415         r'(?:www\.)?piped\.mha\.fi',
 416         r'(?:www\.)?watch\.whatever\.social',
 417         r'(?:www\.)?piped\.garudalinux\.org',
 418         r'(?:www\.)?piped\.rivo\.lol',
 419         r'(?:www\.)?piped-libre\.kavin\.rocks',
 420         r'(?:www\.)?yt\.jae\.fi',
 421         r'(?:www\.)?piped\.mint\.lgbt',
 422         r'(?:www\.)?il\.ax',
 423         r'(?:www\.)?piped\.esmailelbob\.xyz',
 424         r'(?:www\.)?piped\.projectsegfau\.lt',
 425         r'(?:www\.)?piped\.privacydev\.net',
 426         r'(?:www\.)?piped\.palveluntarjoaja\.eu',
 427         r'(?:www\.)?piped\.smnz\.de',
 428         r'(?:www\.)?piped\.adminforge\.de',
 429         r'(?:www\.)?watch\.whatevertinfoil\.de',
 430         r'(?:www\.)?piped\.qdi\.fi',
 431         r'(?:www\.)?piped\.video',
 432         r'(?:www\.)?piped\.aeong\.one',
 433         r'(?:www\.)?piped\.moomoo\.me',
 434         r'(?:www\.)?piped\.chauvet\.pro',
 435         r'(?:www\.)?watch\.leptons\.xyz',
 436         r'(?:www\.)?pd\.vern\.cc',
 437         r'(?:www\.)?piped\.hostux\.net',
 438         r'(?:www\.)?piped\.lunar\.icu',
 439         # Hyperpipe instances from https://hyperpipe.codeberg.page/
 440         r'(?:www\.)?hyperpipe\.surge\.sh',
 441         r'(?:www\.)?hyperpipe\.esmailelbob\.xyz',
 442         r'(?:www\.)?listen\.whatever\.social',
 443         r'(?:www\.)?music\.adminforge\.de',
 444     )
 445
 446     # extracted from account/account_menu ep
 447     # XXX: These are the supported YouTube UI and API languages,
 448     # which is slightly different from languages supported for translation in YouTube studio
 449     _SUPPORTED_LANG_CODES = [
 450         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 451         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 452         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 453         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 454         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 455         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 456     ]
 457
 458     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 459
 460     _YT_HANDLE_RE = r'@[\w.-]{3,30}'  # https://support.google.com/youtube/answer/11585688?hl=en
 461     _YT_CHANNEL_UCID_RE = r'UC[\w-]{22}'
 462
 463     def ucid_or_none(self, ucid):
 464         return self._search_regex(rf'^({self._YT_CHANNEL_UCID_RE})$', ucid, 'UC-id', default=None)
 465
 466     def handle_or_none(self, handle):
 467         return self._search_regex(rf'^({self._YT_HANDLE_RE})$', handle, '@-handle', default=None)
 468
 469     def handle_from_url(self, url):
 470         return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_HANDLE_RE})',
 471                                   url, 'channel handle', default=None)
 472
 473     def ucid_from_url(self, url):
 474         return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_CHANNEL_UCID_RE})',
 475                                   url, 'channel id', default=None)
 476
 477     @functools.cached_property
 478     def _preferred_lang(self):
 479         """
 480         Returns a language code supported by YouTube for the user preferred language.
 481         Returns None if no preferred language set.
 482         """
 483         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 484         if not preferred_lang:
 485             return
 486         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 487             raise ExtractorError(
 488                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 489                 expected=True)
 490         elif preferred_lang != 'en':
 491             self.report_warning(
 492                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 493         return preferred_lang
 494
 495     def _initialize_consent(self):
 496         cookies = self._get_cookies('https://www.youtube.com/')
 497         if cookies.get('__Secure-3PSID'):
 498             return
 499         consent_id = None
 500         consent = cookies.get('CONSENT')
 501         if consent:
 502             if 'YES' in consent.value:
 503                 return
 504             consent_id = self._search_regex(
 505                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 506         if not consent_id:
 507             consent_id = random.randint(100, 999)
 508         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 509
 510     def _initialize_pref(self):
 511         cookies = self._get_cookies('https://www.youtube.com/')
 512         pref_cookie = cookies.get('PREF')
 513         pref = {}
 514         if pref_cookie:
 515             try:
 516                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 517             except ValueError:
 518                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 519         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 520         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 521
 522     def _real_initialize(self):
 523         self._initialize_pref()
 524         self._initialize_consent()
 525         self._check_login_required()
 526
 527     def _check_login_required(self):
 528         if self._LOGIN_REQUIRED and not self._cookies_passed:
 529             self.raise_login_required('Login details are needed to download this content', method='cookies')
 530
 531     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 532     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 533
 534     def _get_default_ytcfg(self, client='web'):
 535         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 536
 537     def _get_innertube_host(self, client='web'):
 538         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 539
 540     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 541         # try_get but with fallback to default ytcfg client values when present
 542         _func = lambda y: try_get(y, getter, expected_type)
 543         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 544
 545     def _extract_client_name(self, ytcfg, default_client='web'):
 546         return self._ytcfg_get_safe(
 547             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 548                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 549
 550     def _extract_client_version(self, ytcfg, default_client='web'):
 551         return self._ytcfg_get_safe(
 552             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 553                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 554
 555     def _select_api_hostname(self, req_api_hostname, default_client=None):
 556         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 557                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 558
 559     def _extract_api_key(self, ytcfg=None, default_client='web'):
 560         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 561
 562     def _extract_context(self, ytcfg=None, default_client='web'):
 563         context = get_first(
 564             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 565         # Enforce language and tz for extraction
 566         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 567         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 568         return context
 569
 570     _SAPISID = None
 571
 572     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 573         time_now = round(time.time())
 574         if self._SAPISID is None:
 575             yt_cookies = self._get_cookies('https://www.youtube.com')
 576             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 577             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 578             sapisid_cookie = dict_get(
 579                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 580             if sapisid_cookie and sapisid_cookie.value:
 581                 self._SAPISID = sapisid_cookie.value
 582                 self.write_debug('Extracted SAPISID cookie')
 583                 # SAPISID cookie is required if not already present
 584                 if not yt_cookies.get('SAPISID'):
 585                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 586                     self._set_cookie(
 587                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 588             else:
 589                 self._SAPISID = False
 590         if not self._SAPISID:
 591             return None
 592         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 593         sapisidhash = hashlib.sha1(
 594             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 595         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 596
 597     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 598                   note='Downloading API JSON', errnote='Unable to download API page',
 599                   context=None, api_key=None, api_hostname=None, default_client='web'):
 600
 601         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 602         data.update(query)
 603         real_headers = self.generate_api_headers(default_client=default_client)
 604         real_headers.update({'content-type': 'application/json'})
 605         if headers:
 606             real_headers.update(headers)
 607         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 608                    or api_key or self._extract_api_key(default_client=default_client))
 609         return self._download_json(
 610             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 611             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 612             data=json.dumps(data).encode('utf8'), headers=real_headers,
 613             query={'key': api_key, 'prettyPrint': 'false'})
 614
 615     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 616         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 617
 618     @staticmethod
 619     def _extract_session_index(*data):
 620         """
 621         Index of current account in account list.
 622         See: https://github.com/yt-dlp/yt-dlp/pull/519
 623         """
 624         for ytcfg in data:
 625             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 626             if session_index is not None:
 627                 return session_index
 628
 629     # Deprecated?
 630     def _extract_identity_token(self, ytcfg=None, webpage=None):
 631         if ytcfg:
 632             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 633             if token:
 634                 return token
 635         if webpage:
 636             return self._search_regex(
 637                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 638                 'identity token', default=None, fatal=False)
 639
 640     @staticmethod
 641     def _extract_account_syncid(*args):
 642         """
 643         Extract syncId required to download private playlists of secondary channels
 644         @params response and/or ytcfg
 645         """
 646         for data in args:
 647             # ytcfg includes channel_syncid if on secondary channel
 648             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 649             if delegated_sid:
 650                 return delegated_sid
 651             sync_ids = (try_get(
 652                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 653                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 654             if len(sync_ids) >= 2 and sync_ids[1]:
 655                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 656                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 657                 return sync_ids[0]
 658
 659     @staticmethod
 660     def _extract_visitor_data(*args):
 661         """
 662         Extracts visitorData from an API response or ytcfg
 663         Appears to be used to track session state
 664         """
 665         return get_first(
 666             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 667             expected_type=str)
 668
 669     @functools.cached_property
 670     def is_authenticated(self):
 671         return bool(self._generate_sapisidhash_header())
 672
 673     def extract_ytcfg(self, video_id, webpage):
 674         if not webpage:
 675             return {}
 676         return self._parse_json(
 677             self._search_regex(
 678                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 679                 default='{}'), video_id, fatal=False) or {}
 680
 681     def generate_api_headers(
 682             self, *, ytcfg=None, account_syncid=None, session_index=None,
 683             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 684
 685         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 686         headers = {
 687             'X-YouTube-Client-Name': str(
 688                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 689             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 690             'Origin': origin,
 691             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 692             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 693             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 694             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 695         }
 696         if session_index is None:
 697             session_index = self._extract_session_index(ytcfg)
 698         if account_syncid or session_index is not None:
 699             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 700
 701         auth = self._generate_sapisidhash_header(origin)
 702         if auth is not None:
 703             headers['Authorization'] = auth
 704             headers['X-Origin'] = origin
 705         return filter_dict(headers)
 706
 707     def _download_ytcfg(self, client, video_id):
 708         url = {
 709             'web': 'https://www.youtube.com',
 710             'web_music': 'https://music.youtube.com',
 711             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 712         }.get(client)
 713         if not url:
 714             return {}
 715         webpage = self._download_webpage(
 716             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 717         return self.extract_ytcfg(video_id, webpage) or {}
 718
 719     @staticmethod
 720     def _build_api_continuation_query(continuation, ctp=None):
 721         query = {
 722             'continuation': continuation
 723         }
 724         # TODO: Inconsistency with clickTrackingParams.
 725         # Currently we have a fixed ctp contained within context (from ytcfg)
 726         # and a ctp in root query for continuation.
 727         if ctp:
 728             query['clickTracking'] = {'clickTrackingParams': ctp}
 729         return query
 730
 731     @classmethod
 732     def _extract_next_continuation_data(cls, renderer):
 733         next_continuation = try_get(
 734             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 735                        lambda x: x['continuation']['reloadContinuationData']), dict)
 736         if not next_continuation:
 737             return
 738         continuation = next_continuation.get('continuation')
 739         if not continuation:
 740             return
 741         ctp = next_continuation.get('clickTrackingParams')
 742         return cls._build_api_continuation_query(continuation, ctp)
 743
 744     @classmethod
 745     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 746         if isinstance(continuation_ep, dict):
 747             continuation = try_get(
 748                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 749             if not continuation:
 750                 return
 751             ctp = continuation_ep.get('clickTrackingParams')
 752             return cls._build_api_continuation_query(continuation, ctp)
 753
 754     @classmethod
 755     def _extract_continuation(cls, renderer):
 756         next_continuation = cls._extract_next_continuation_data(renderer)
 757         if next_continuation:
 758             return next_continuation
 759
 760         return traverse_obj(renderer, (
 761             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 762             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 763         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 764
 765     @classmethod
 766     def _extract_alerts(cls, data):
 767         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 768             if not isinstance(alert_dict, dict):
 769                 continue
 770             for alert in alert_dict.values():
 771                 alert_type = alert.get('type')
 772                 if not alert_type:
 773                     continue
 774                 message = cls._get_text(alert, 'text')
 775                 if message:
 776                     yield alert_type, message
 777
 778     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 779         errors, warnings = [], []
 780         for alert_type, alert_message in alerts:
 781             if alert_type.lower() == 'error' and fatal:
 782                 errors.append([alert_type, alert_message])
 783             elif alert_message not in self._IGNORED_WARNINGS:
 784                 warnings.append([alert_type, alert_message])
 785
 786         for alert_type, alert_message in (warnings + errors[:-1]):
 787             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 788         if errors:
 789             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 790
 791     def _extract_and_report_alerts(self, data, *args, **kwargs):
 792         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 793
 794     def _extract_badges(self, renderer: dict):
 795         privacy_icon_map = {
 796             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 797             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 798             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 799         }
 800
 801         badge_style_map = {
 802             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 803             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 804             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 805         }
 806
 807         label_map = {
 808             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 809             'private': BadgeType.AVAILABILITY_PRIVATE,
 810             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 811             'live': BadgeType.LIVE_NOW,
 812             'premium': BadgeType.AVAILABILITY_PREMIUM
 813         }
 814
 815         badges = []
 816         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer')):
 817             badge_type = (
 818                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 819                 or badge_style_map.get(traverse_obj(badge, 'style'))
 820             )
 821             if badge_type:
 822                 badges.append({'type': badge_type})
 823                 continue
 824
 825             # fallback, won't work in some languages
 826             label = traverse_obj(badge, 'label', expected_type=str, default='')
 827             for match, label_badge_type in label_map.items():
 828                 if match in label.lower():
 829                     badges.append({'type': badge_type})
 830                     continue
 831
 832         return badges
 833
 834     @staticmethod
 835     def _has_badge(badges, badge_type):
 836         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 837
 838     @staticmethod
 839     def _get_text(data, *path_list, max_runs=None):
 840         for path in path_list or [None]:
 841             if path is None:
 842                 obj = [data]
 843             else:
 844                 obj = traverse_obj(data, path, default=[])
 845                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 846                     obj = [obj]
 847             for item in obj:
 848                 text = try_get(item, lambda x: x['simpleText'], str)
 849                 if text:
 850                     return text
 851                 runs = try_get(item, lambda x: x['runs'], list) or []
 852                 if not runs and isinstance(item, list):
 853                     runs = item
 854
 855                 runs = runs[:min(len(runs), max_runs or len(runs))]
 856                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str))
 857                 if text:
 858                     return text
 859
 860     def _get_count(self, data, *path_list):
 861         count_text = self._get_text(data, *path_list) or ''
 862         count = parse_count(count_text)
 863         if count is None:
 864             count = str_to_int(
 865                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 866         return count
 867
 868     @staticmethod
 869     def _extract_thumbnails(data, *path_list):
 870         """
 871         Extract thumbnails from thumbnails dict
 872         @param path_list: path list to level that contains 'thumbnails' key
 873         """
 874         thumbnails = []
 875         for path in path_list or [()]:
 876             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...)):
 877                 thumbnail_url = url_or_none(thumbnail.get('url'))
 878                 if not thumbnail_url:
 879                     continue
 880                 # Sometimes youtube gives a wrong thumbnail URL. See:
 881                 # https://github.com/yt-dlp/yt-dlp/issues/233
 882                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 883                 if 'maxresdefault' in thumbnail_url:
 884                     thumbnail_url = thumbnail_url.split('?')[0]
 885                 thumbnails.append({
 886                     'url': thumbnail_url,
 887                     'height': int_or_none(thumbnail.get('height')),
 888                     'width': int_or_none(thumbnail.get('width')),
 889                 })
 890         return thumbnails
 891
 892     @staticmethod
 893     def extract_relative_time(relative_time_text):
 894         """
 895         Extracts a relative time from string and converts to dt object
 896         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 897         """
 898         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 899         if mobj:
 900             start = mobj.group('start')
 901             if start:
 902                 return datetime_from_str(start)
 903             try:
 904                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 905             except ValueError:
 906                 return None
 907
 908     def _parse_time_text(self, text):
 909         if not text:
 910             return
 911         dt = self.extract_relative_time(text)
 912         timestamp = None
 913         if isinstance(dt, datetime.datetime):
 914             timestamp = calendar.timegm(dt.timetuple())
 915
 916         if timestamp is None:
 917             timestamp = (
 918                 unified_timestamp(text) or unified_timestamp(
 919                     self._search_regex(
 920                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 921                         text.lower(), 'time text', default=None)))
 922
 923         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 924             self.report_warning(
 925                 f'Cannot parse localized time text "{text}"', only_once=True)
 926         return timestamp
 927
 928     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 929                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 930                           default_client='web'):
 931         for retry in self.RetryManager():
 932             try:
 933                 response = self._call_api(
 934                     ep=ep, fatal=True, headers=headers,
 935                     video_id=item_id, query=query, note=note,
 936                     context=self._extract_context(ytcfg, default_client),
 937                     api_key=self._extract_api_key(ytcfg, default_client),
 938                     api_hostname=api_hostname, default_client=default_client)
 939             except ExtractorError as e:
 940                 if not isinstance(e.cause, network_exceptions):
 941                     return self._error_or_warning(e, fatal=fatal)
 942                 elif not isinstance(e.cause, urllib.error.HTTPError):
 943                     retry.error = e
 944                     continue
 945
 946                 first_bytes = e.cause.read(512)
 947                 if not is_html(first_bytes):
 948                     yt_error = try_get(
 949                         self._parse_json(
 950                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 951                         lambda x: x['error']['message'], str)
 952                     if yt_error:
 953                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 954                 # Downloading page may result in intermittent 5xx HTTP error
 955                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 956                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 957                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 958                 if e.cause.code not in (403, 429):
 959                     retry.error = e
 960                     continue
 961                 return self._error_or_warning(e, fatal=fatal)
 962
 963             try:
 964                 self._extract_and_report_alerts(response, only_once=True)
 965             except ExtractorError as e:
 966                 # YouTube servers may return errors we want to retry on in a 200 OK response
 967                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 968                 if 'unknown error' in e.msg.lower():
 969                     retry.error = e
 970                     continue
 971                 return self._error_or_warning(e, fatal=fatal)
 972             # Youtube sometimes sends incomplete data
 973             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 974             if not traverse_obj(response, *variadic(check_get_keys)):
 975                 retry.error = ExtractorError('Incomplete data received', expected=True)
 976                 continue
 977
 978             return response
 979
 980     @staticmethod
 981     def is_music_url(url):
 982         return re.match(r'(https?://)?music\.youtube\.com/', url) is not None
 983
 984     def _extract_video(self, renderer):
 985         video_id = renderer.get('videoId')
 986
 987         reel_header_renderer = traverse_obj(renderer, (
 988             'navigationEndpoint', 'reelWatchEndpoint', 'overlay', 'reelPlayerOverlayRenderer',
 989             'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
 990
 991         title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
 992         description = self._get_text(renderer, 'descriptionSnippet')
 993
 994         duration = int_or_none(renderer.get('lengthSeconds'))
 995         if duration is None:
 996             duration = parse_duration(self._get_text(
 997                 renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 998         if duration is None:
 999             # XXX: should write a parser to be more general to support more cases (e.g. shorts in shorts tab)
1000             duration = parse_duration(self._search_regex(
1001                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
1002                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
1003                 video_id, default=None, group='duration'))
1004
1005         channel_id = traverse_obj(
1006             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
1007             expected_type=str, get_all=False)
1008         if not channel_id:
1009             channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
1010
1011         channel_id = self.ucid_or_none(channel_id)
1012
1013         overlay_style = traverse_obj(
1014             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
1015             get_all=False, expected_type=str)
1016         badges = self._extract_badges(renderer)
1017
1018         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
1019             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
1020             expected_type=str)) or ''
1021         url = f'https://www.youtube.com/watch?v={video_id}'
1022         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
1023             url = f'https://www.youtube.com/shorts/{video_id}'
1024
1025         time_text = (self._get_text(renderer, 'publishedTimeText', 'videoInfo')
1026                      or self._get_text(reel_header_renderer, 'timestampText') or '')
1027         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
1028
1029         live_status = (
1030             'is_upcoming' if scheduled_timestamp is not None
1031             else 'was_live' if 'streamed' in time_text.lower()
1032             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
1033             else None)
1034
1035         # videoInfo is a string like '50K views • 10 years ago'.
1036         view_count_text = self._get_text(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo') or ''
1037         view_count = (0 if 'no views' in view_count_text.lower()
1038                       else self._get_count({'simpleText': view_count_text}))
1039         view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
1040
1041         channel = (self._get_text(renderer, 'ownerText', 'shortBylineText')
1042                    or self._get_text(reel_header_renderer, 'channelTitleText'))
1043
1044         channel_handle = traverse_obj(renderer, (
1045             'shortBylineText', 'runs', ..., 'navigationEndpoint',
1046             (('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl'))),
1047             expected_type=self.handle_from_url, get_all=False)
1048         return {
1049             '_type': 'url',
1050             'ie_key': YoutubeIE.ie_key(),
1051             'id': video_id,
1052             'url': url,
1053             'title': title,
1054             'description': description,
1055             'duration': duration,
1056             'channel_id': channel_id,
1057             'channel': channel,
1058             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
1059             'uploader': channel,
1060             'uploader_id': channel_handle,
1061             'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
1062             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
1063             'timestamp': (self._parse_time_text(time_text)
1064                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
1065                           else None),
1066             'release_timestamp': scheduled_timestamp,
1067             'availability':
1068                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
1069                 else self._availability(
1070                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
1071                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
1072                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
1073                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
1074             view_count_field: view_count,
1075             'live_status': live_status
1076         }
1077
1078
1079 class YoutubeIE(YoutubeBaseInfoExtractor):
1080     IE_DESC = 'YouTube'
1081     _VALID_URL = r"""(?x)^
1082                      (
1083                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
1084                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
1085                             (?:www\.)?deturl\.com/www\.youtube\.com|
1086                             (?:www\.)?pwnyoutube\.com|
1087                             (?:www\.)?hooktube\.com|
1088                             (?:www\.)?yourepeat\.com|
1089                             tube\.majestyc\.net|
1090                             %(invidious)s|
1091                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
1092                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
1093                          (?:                                                  # the various things that can precede the ID:
1094                              (?:(?:v|embed|e|shorts|live)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
1095                              |(?:                                             # or the v= param in all its forms
1096                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
1097                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
1098                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
1099                                  v=
1100                              )
1101                          ))
1102                          |(?:
1103                             youtu\.be|                                        # just youtu.be/xxxx
1104                             vid\.plus|                                        # or vid.plus/xxxx
1105                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
1106                             %(invidious)s
1107                          )/
1108                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
1109                          )
1110                      )?                                                       # all until now is optional -> you can pass the naked ID
1111                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
1112                      (?(1).+)?                                                # if we found the ID, everything can follow
1113                      (?:\#|$)""" % {
1114         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1115     }
1116     _EMBED_REGEX = [
1117         r'''(?x)
1118             (?:
1119                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1120                 data-video-url=|
1121                 <embed[^>]+?src=|
1122                 embedSWF\(?:\s*|
1123                 <object[^>]+data=|
1124                 new\s+SWFObject\(
1125             )
1126             (["\'])
1127                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1128                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1129             \1''',
1130         # https://wordpress.org/plugins/lazy-load-for-videos/
1131         r'''(?xs)
1132             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1133             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1134     ]
1135     _RETURN_TYPE = 'video'  # XXX: How to handle multifeed?
1136
1137     _PLAYER_INFO_RE = (
1138         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1139         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1140         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1141     )
1142     _formats = {
1143         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1144         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1145         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1146         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1147         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1148         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1149         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1150         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1151         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1152         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1153         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1154         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1155         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1156         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1157         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1158         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1159         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1160         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1161
1162
1163         # 3D videos
1164         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1165         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1166         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1167         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1168         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1169         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1170         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1171
1172         # Apple HTTP Live Streaming
1173         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1174         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1175         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1176         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1177         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1178         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1179         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1180         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1181
1182         # DASH mp4 video
1183         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1184         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1185         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1186         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1187         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1188         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1189         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1190         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1191         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1192         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1193         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1194         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1195
1196         # Dash mp4 audio
1197         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1198         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1199         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1200         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1201         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1202         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1203         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1204
1205         # Dash webm
1206         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1207         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1208         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1209         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1210         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1211         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1212         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1213         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1214         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1215         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1216         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1217         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1218         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1219         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1220         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1221         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1222         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1223         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1224         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1225         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1226         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1227         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1228
1229         # Dash webm audio
1230         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1231         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1232
1233         # Dash webm audio with opus inside
1234         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1235         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1236         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1237
1238         # RTMP (unnamed)
1239         '_rtmp': {'protocol': 'rtmp'},
1240
1241         # av01 video only formats sometimes served with "unknown" codecs
1242         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1243         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1244         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1245         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1246         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1247         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1248         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1249         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1250     }
1251     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1252
1253     _GEO_BYPASS = False
1254
1255     IE_NAME = 'youtube'
1256     _TESTS = [
1257         {
1258             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1259             'info_dict': {
1260                 'id': 'BaW_jenozKc',
1261                 'ext': 'mp4',
1262                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1263                 'channel': 'Philipp Hagemeister',
1264                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1265                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1266                 'upload_date': '20121002',
1267                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1268                 'categories': ['Science & Technology'],
1269                 'tags': ['youtube-dl'],
1270                 'duration': 10,
1271                 'view_count': int,
1272                 'like_count': int,
1273                 'availability': 'public',
1274                 'playable_in_embed': True,
1275                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1276                 'live_status': 'not_live',
1277                 'age_limit': 0,
1278                 'start_time': 1,
1279                 'end_time': 9,
1280                 'comment_count': int,
1281                 'channel_follower_count': int,
1282                 'uploader': 'Philipp Hagemeister',
1283                 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
1284                 'uploader_id': '@PhilippHagemeister',
1285                 'heatmap': 'count:100',
1286             }
1287         },
1288         {
1289             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1290             'note': 'Embed-only video (#1746)',
1291             'info_dict': {
1292                 'id': 'yZIXLfi8CZQ',
1293                 'ext': 'mp4',
1294                 'upload_date': '20120608',
1295                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1296                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1297                 'age_limit': 18,
1298             },
1299             'skip': 'Private video',
1300         },
1301         {
1302             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1303             'note': 'Use the first video ID in the URL',
1304             'info_dict': {
1305                 'id': 'BaW_jenozKc',
1306                 'ext': 'mp4',
1307                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1308                 'channel': 'Philipp Hagemeister',
1309                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1310                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1311                 'upload_date': '20121002',
1312                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1313                 'categories': ['Science & Technology'],
1314                 'tags': ['youtube-dl'],
1315                 'duration': 10,
1316                 'view_count': int,
1317                 'like_count': int,
1318                 'availability': 'public',
1319                 'playable_in_embed': True,
1320                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1321                 'live_status': 'not_live',
1322                 'age_limit': 0,
1323                 'comment_count': int,
1324                 'channel_follower_count': int,
1325                 'uploader': 'Philipp Hagemeister',
1326                 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
1327                 'uploader_id': '@PhilippHagemeister',
1328             },
1329             'params': {
1330                 'skip_download': True,
1331             },
1332         },
1333         {
1334             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1335             'note': '256k DASH audio (format 141) via DASH manifest',
1336             'info_dict': {
1337                 'id': 'a9LDPn-MO4I',
1338                 'ext': 'm4a',
1339                 'upload_date': '20121002',
1340                 'description': '',
1341                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1342             },
1343             'params': {
1344                 'youtube_include_dash_manifest': True,
1345                 'format': '141',
1346             },
1347             'skip': 'format 141 not served anymore',
1348         },
1349         # DASH manifest with encrypted signature
1350         {
1351             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1352             'info_dict': {
1353                 'id': 'IB3lcPjvWLA',
1354                 'ext': 'm4a',
1355                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1356                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1357                 'duration': 244,
1358                 'upload_date': '20131011',
1359                 'abr': 129.495,
1360                 'like_count': int,
1361                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1362                 'playable_in_embed': True,
1363                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1364                 'view_count': int,
1365                 'track': 'The Spark',
1366                 'live_status': 'not_live',
1367                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1368                 'channel': 'Afrojack',
1369                 'tags': 'count:19',
1370                 'availability': 'public',
1371                 'categories': ['Music'],
1372                 'age_limit': 0,
1373                 'alt_title': 'The Spark',
1374                 'channel_follower_count': int,
1375                 'uploader': 'Afrojack',
1376                 'uploader_url': 'https://www.youtube.com/@Afrojack',
1377                 'uploader_id': '@Afrojack',
1378             },
1379             'params': {
1380                 'youtube_include_dash_manifest': True,
1381                 'format': '141/bestaudio[ext=m4a]',
1382             },
1383         },
1384         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1385         {
1386             'note': 'Embed allowed age-gate video',
1387             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1388             'info_dict': {
1389                 'id': 'HtVdAasjOgU',
1390                 'ext': 'mp4',
1391                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1392                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1393                 'duration': 142,
1394                 'upload_date': '20140605',
1395                 'age_limit': 18,
1396                 'categories': ['Gaming'],
1397                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1398                 'availability': 'needs_auth',
1399                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1400                 'like_count': int,
1401                 'channel': 'The Witcher',
1402                 'live_status': 'not_live',
1403                 'tags': 'count:17',
1404                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1405                 'playable_in_embed': True,
1406                 'view_count': int,
1407                 'channel_follower_count': int,
1408                 'uploader': 'The Witcher',
1409                 'uploader_url': 'https://www.youtube.com/@thewitcher',
1410                 'uploader_id': '@thewitcher',
1411             },
1412         },
1413         {
1414             'note': 'Age-gate video with embed allowed in public site',
1415             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1416             'info_dict': {
1417                 'id': 'HsUATh_Nc2U',
1418                 'ext': 'mp4',
1419                 'title': 'Godzilla 2 (Official Video)',
1420                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1421                 'upload_date': '20200408',
1422                 'age_limit': 18,
1423                 'availability': 'needs_auth',
1424                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1425                 'channel': 'FlyingKitty',
1426                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1427                 'view_count': int,
1428                 'categories': ['Entertainment'],
1429                 'live_status': 'not_live',
1430                 'tags': ['Flyingkitty', 'godzilla 2'],
1431                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1432                 'like_count': int,
1433                 'duration': 177,
1434                 'playable_in_embed': True,
1435                 'channel_follower_count': int,
1436                 'uploader': 'FlyingKitty',
1437                 'uploader_url': 'https://www.youtube.com/@FlyingKitty900',
1438                 'uploader_id': '@FlyingKitty900',
1439                 'comment_count': int,
1440             },
1441         },
1442         {
1443             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1444             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1445             'info_dict': {
1446                 'id': 'Tq92D6wQ1mg',
1447                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1448                 'ext': 'mp4',
1449                 'upload_date': '20191228',
1450                 'description': 'md5:17eccca93a786d51bc67646756894066',
1451                 'age_limit': 18,
1452                 'like_count': int,
1453                 'availability': 'needs_auth',
1454                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1455                 'view_count': int,
1456                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1457                 'channel': 'Projekt Melody',
1458                 'live_status': 'not_live',
1459                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1460                 'playable_in_embed': True,
1461                 'categories': ['Entertainment'],
1462                 'duration': 106,
1463                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1464                 'comment_count': int,
1465                 'channel_follower_count': int,
1466                 'uploader': 'Projekt Melody',
1467                 'uploader_url': 'https://www.youtube.com/@ProjektMelody',
1468                 'uploader_id': '@ProjektMelody',
1469             },
1470         },
1471         {
1472             'note': 'Non-Agegated non-embeddable video',
1473             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1474             'info_dict': {
1475                 'id': 'MeJVWBSsPAY',
1476                 'ext': 'mp4',
1477                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1478                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1479                 'upload_date': '20130730',
1480                 'track': 'Such mich find mich',
1481                 'age_limit': 0,
1482                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1483                 'like_count': int,
1484                 'playable_in_embed': False,
1485                 'creator': 'OOMPH!',
1486                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1487                 'view_count': int,
1488                 'alt_title': 'Such mich find mich',
1489                 'duration': 210,
1490                 'channel': 'Herr Lurik',
1491                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1492                 'categories': ['Music'],
1493                 'availability': 'public',
1494                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1495                 'live_status': 'not_live',
1496                 'artist': 'OOMPH!',
1497                 'channel_follower_count': int,
1498                 'uploader': 'Herr Lurik',
1499                 'uploader_url': 'https://www.youtube.com/@HerrLurik',
1500                 'uploader_id': '@HerrLurik',
1501             },
1502         },
1503         {
1504             'note': 'Non-bypassable age-gated video',
1505             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1506             'only_matching': True,
1507         },
1508         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1509         # YouTube Red ad is not captured for creator
1510         {
1511             'url': '__2ABJjxzNo',
1512             'info_dict': {
1513                 'id': '__2ABJjxzNo',
1514                 'ext': 'mp4',
1515                 'duration': 266,
1516                 'upload_date': '20100430',
1517                 'creator': 'deadmau5',
1518                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1519                 'title': 'Deadmau5 - Some Chords (HD)',
1520                 'alt_title': 'Some Chords',
1521                 'availability': 'public',
1522                 'tags': 'count:14',
1523                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1524                 'view_count': int,
1525                 'live_status': 'not_live',
1526                 'channel': 'deadmau5',
1527                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1528                 'like_count': int,
1529                 'track': 'Some Chords',
1530                 'artist': 'deadmau5',
1531                 'playable_in_embed': True,
1532                 'age_limit': 0,
1533                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1534                 'categories': ['Music'],
1535                 'album': 'Some Chords',
1536                 'channel_follower_count': int,
1537                 'uploader': 'deadmau5',
1538                 'uploader_url': 'https://www.youtube.com/@deadmau5',
1539                 'uploader_id': '@deadmau5',
1540             },
1541             'expected_warnings': [
1542                 'DASH manifest missing',
1543             ]
1544         },
1545         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1546         {
1547             'url': 'lqQg6PlCWgI',
1548             'info_dict': {
1549                 'id': 'lqQg6PlCWgI',
1550                 'ext': 'mp4',
1551                 'duration': 6085,
1552                 'upload_date': '20150827',
1553                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1554                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1555                 'like_count': int,
1556                 'release_timestamp': 1343767800,
1557                 'playable_in_embed': True,
1558                 'categories': ['Sports'],
1559                 'release_date': '20120731',
1560                 'channel': 'Olympics',
1561                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1562                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1563                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1564                 'age_limit': 0,
1565                 'availability': 'public',
1566                 'live_status': 'was_live',
1567                 'view_count': int,
1568                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1569                 'channel_follower_count': int,
1570                 'uploader': 'Olympics',
1571                 'uploader_url': 'https://www.youtube.com/@Olympics',
1572                 'uploader_id': '@Olympics',
1573             },
1574             'params': {
1575                 'skip_download': 'requires avconv',
1576             }
1577         },
1578         # Non-square pixels
1579         {
1580             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1581             'info_dict': {
1582                 'id': '_b-2C3KPAM0',
1583                 'ext': 'mp4',
1584                 'stretched_ratio': 16 / 9.,
1585                 'duration': 85,
1586                 'upload_date': '20110310',
1587                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1588                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1589                 'playable_in_embed': True,
1590                 'channel': '孫ᄋᄅ',
1591                 'age_limit': 0,
1592                 'tags': 'count:11',
1593                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1594                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1595                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1596                 'view_count': int,
1597                 'categories': ['People & Blogs'],
1598                 'like_count': int,
1599                 'live_status': 'not_live',
1600                 'availability': 'unlisted',
1601                 'comment_count': int,
1602                 'channel_follower_count': int,
1603                 'uploader': '孫ᄋᄅ',
1604                 'uploader_url': 'https://www.youtube.com/@AllenMeow',
1605                 'uploader_id': '@AllenMeow',
1606             },
1607         },
1608         # url_encoded_fmt_stream_map is empty string
1609         {
1610             'url': 'qEJwOuvDf7I',
1611             'info_dict': {
1612                 'id': 'qEJwOuvDf7I',
1613                 'ext': 'webm',
1614                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1615                 'description': '',
1616                 'upload_date': '20150404',
1617             },
1618             'params': {
1619                 'skip_download': 'requires avconv',
1620             },
1621             'skip': 'This live event has ended.',
1622         },
1623         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1624         {
1625             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1626             'info_dict': {
1627                 'id': 'FIl7x6_3R5Y',
1628                 'ext': 'webm',
1629                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1630                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1631                 'duration': 220,
1632                 'upload_date': '20150625',
1633                 'formats': 'mincount:31',
1634             },
1635             'skip': 'not actual anymore',
1636         },
1637         # DASH manifest with segment_list
1638         {
1639             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1640             'md5': '8ce563a1d667b599d21064e982ab9e31',
1641             'info_dict': {
1642                 'id': 'CsmdDsKjzN8',
1643                 'ext': 'mp4',
1644                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1645                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1646                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1647             },
1648             'params': {
1649                 'youtube_include_dash_manifest': True,
1650                 'format': '135',  # bestvideo
1651             },
1652             'skip': 'This live event has ended.',
1653         },
1654         {
1655             # Multifeed videos (multiple cameras), URL can be of any Camera
1656             # TODO: fix multifeed titles
1657             'url': 'https://www.youtube.com/watch?v=zaPI8MvL8pg',
1658             'info_dict': {
1659                 'id': 'zaPI8MvL8pg',
1660                 'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04',
1661                 'description': 'md5:563ccbc698b39298481ca3c571169519',
1662             },
1663             'playlist': [{
1664                 'info_dict': {
1665                     'id': 'j5yGuxZ8lLU',
1666                     'ext': 'mp4',
1667                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Chris)',
1668                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1669                     'duration': 10120,
1670                     'channel_follower_count': int,
1671                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1672                     'availability': 'public',
1673                     'playable_in_embed': True,
1674                     'upload_date': '20131105',
1675                     'categories': ['Gaming'],
1676                     'live_status': 'was_live',
1677                     'tags': 'count:24',
1678                     'release_timestamp': 1383701910,
1679                     'thumbnail': 'https://i.ytimg.com/vi/j5yGuxZ8lLU/maxresdefault.jpg',
1680                     'comment_count': int,
1681                     'age_limit': 0,
1682                     'like_count': int,
1683                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1684                     'channel': 'WiiLikeToPlay',
1685                     'view_count': int,
1686                     'release_date': '20131106',
1687                     'uploader': 'WiiLikeToPlay',
1688                     'uploader_id': '@WLTP',
1689                     'uploader_url': 'https://www.youtube.com/@WLTP',
1690                 },
1691             }, {
1692                 'info_dict': {
1693                     'id': 'zaPI8MvL8pg',
1694                     'ext': 'mp4',
1695                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Tyson)',
1696                     'availability': 'public',
1697                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1698                     'channel': 'WiiLikeToPlay',
1699                     'channel_follower_count': int,
1700                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1701                     'duration': 10108,
1702                     'age_limit': 0,
1703                     'like_count': int,
1704                     'tags': 'count:24',
1705                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1706                     'release_timestamp': 1383701915,
1707                     'comment_count': int,
1708                     'upload_date': '20131105',
1709                     'thumbnail': 'https://i.ytimg.com/vi/zaPI8MvL8pg/maxresdefault.jpg',
1710                     'release_date': '20131106',
1711                     'playable_in_embed': True,
1712                     'live_status': 'was_live',
1713                     'categories': ['Gaming'],
1714                     'view_count': int,
1715                     'uploader': 'WiiLikeToPlay',
1716                     'uploader_id': '@WLTP',
1717                     'uploader_url': 'https://www.youtube.com/@WLTP',
1718                 },
1719             }, {
1720                 'info_dict': {
1721                     'id': 'R7r3vfO7Hao',
1722                     'ext': 'mp4',
1723                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Spencer)',
1724                     'thumbnail': 'https://i.ytimg.com/vi/R7r3vfO7Hao/maxresdefault.jpg',
1725                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1726                     'like_count': int,
1727                     'availability': 'public',
1728                     'playable_in_embed': True,
1729                     'upload_date': '20131105',
1730                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1731                     'channel_follower_count': int,
1732                     'tags': 'count:24',
1733                     'release_date': '20131106',
1734                     'comment_count': int,
1735                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1736                     'channel': 'WiiLikeToPlay',
1737                     'categories': ['Gaming'],
1738                     'release_timestamp': 1383701914,
1739                     'live_status': 'was_live',
1740                     'age_limit': 0,
1741                     'duration': 10128,
1742                     'view_count': int,
1743                     'uploader': 'WiiLikeToPlay',
1744                     'uploader_id': '@WLTP',
1745                     'uploader_url': 'https://www.youtube.com/@WLTP',
1746                 },
1747             }],
1748             'params': {'skip_download': True},
1749         },
1750         {
1751             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1752             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1753             'info_dict': {
1754                 'id': 'gVfLd0zydlo',
1755                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1756             },
1757             'playlist_count': 2,
1758             'skip': 'Not multifeed anymore',
1759         },
1760         {
1761             'url': 'https://vid.plus/FlRa-iH7PGw',
1762             'only_matching': True,
1763         },
1764         {
1765             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1766             'only_matching': True,
1767         },
1768         {
1769             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1770             # Also tests cut-off URL expansion in video description (see
1771             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1772             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1773             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1774             'info_dict': {
1775                 'id': 'lsguqyKfVQg',
1776                 'ext': 'mp4',
1777                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1778                 'alt_title': 'Dark Walk',
1779                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1780                 'duration': 133,
1781                 'upload_date': '20151119',
1782                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1783                 'track': 'Dark Walk',
1784                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1785                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1786                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1787                 'categories': ['Film & Animation'],
1788                 'view_count': int,
1789                 'live_status': 'not_live',
1790                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1791                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1792                 'tags': 'count:13',
1793                 'availability': 'public',
1794                 'channel': 'IronSoulElf',
1795                 'playable_in_embed': True,
1796                 'like_count': int,
1797                 'age_limit': 0,
1798                 'channel_follower_count': int
1799             },
1800             'params': {
1801                 'skip_download': True,
1802             },
1803         },
1804         {
1805             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1806             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1807             'only_matching': True,
1808         },
1809         {
1810             # Video with yt:stretch=17:0
1811             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1812             'info_dict': {
1813                 'id': 'Q39EVAstoRM',
1814                 'ext': 'mp4',
1815                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1816                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1817                 'upload_date': '20151107',
1818             },
1819             'params': {
1820                 'skip_download': True,
1821             },
1822             'skip': 'This video does not exist.',
1823         },
1824         {
1825             # Video with incomplete 'yt:stretch=16:'
1826             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1827             'only_matching': True,
1828         },
1829         {
1830             # Video licensed under Creative Commons
1831             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1832             'info_dict': {
1833                 'id': 'M4gD1WSo5mA',
1834                 'ext': 'mp4',
1835                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1836                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1837                 'duration': 721,
1838                 'upload_date': '20150128',
1839                 'license': 'Creative Commons Attribution license (reuse allowed)',
1840                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1841                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1842                 'like_count': int,
1843                 'age_limit': 0,
1844                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1845                 'channel': 'The Berkman Klein Center for Internet & Society',
1846                 'availability': 'public',
1847                 'view_count': int,
1848                 'categories': ['Education'],
1849                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1850                 'live_status': 'not_live',
1851                 'playable_in_embed': True,
1852                 'channel_follower_count': int,
1853                 'chapters': list,
1854                 'uploader': 'The Berkman Klein Center for Internet & Society',
1855                 'uploader_id': '@BKCHarvard',
1856                 'uploader_url': 'https://www.youtube.com/@BKCHarvard',
1857             },
1858             'params': {
1859                 'skip_download': True,
1860             },
1861         },
1862         {
1863             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1864             'info_dict': {
1865                 'id': 'eQcmzGIKrzg',
1866                 'ext': 'mp4',
1867                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1868                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1869                 'duration': 4060,
1870                 'upload_date': '20151120',
1871                 'license': 'Creative Commons Attribution license (reuse allowed)',
1872                 'playable_in_embed': True,
1873                 'tags': 'count:12',
1874                 'like_count': int,
1875                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1876                 'age_limit': 0,
1877                 'availability': 'public',
1878                 'categories': ['News & Politics'],
1879                 'channel': 'Bernie Sanders',
1880                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1881                 'view_count': int,
1882                 'live_status': 'not_live',
1883                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1884                 'comment_count': int,
1885                 'channel_follower_count': int,
1886                 'chapters': list,
1887                 'uploader': 'Bernie Sanders',
1888                 'uploader_url': 'https://www.youtube.com/@BernieSanders',
1889                 'uploader_id': '@BernieSanders',
1890             },
1891             'params': {
1892                 'skip_download': True,
1893             },
1894         },
1895         {
1896             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1897             'only_matching': True,
1898         },
1899         {
1900             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1901             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1902             'only_matching': True,
1903         },
1904         {
1905             # Rental video preview
1906             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1907             'info_dict': {
1908                 'id': 'uGpuVWrhIzE',
1909                 'ext': 'mp4',
1910                 'title': 'Piku - Trailer',
1911                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1912                 'upload_date': '20150811',
1913                 'license': 'Standard YouTube License',
1914             },
1915             'params': {
1916                 'skip_download': True,
1917             },
1918             'skip': 'This video is not available.',
1919         },
1920         {
1921             # YouTube Red video with episode data
1922             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1923             'info_dict': {
1924                 'id': 'iqKdEhx-dD4',
1925                 'ext': 'mp4',
1926                 'title': 'Isolation - Mind Field (Ep 1)',
1927                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1928                 'duration': 2085,
1929                 'upload_date': '20170118',
1930                 'series': 'Mind Field',
1931                 'season_number': 1,
1932                 'episode_number': 1,
1933                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1934                 'tags': 'count:12',
1935                 'view_count': int,
1936                 'availability': 'public',
1937                 'age_limit': 0,
1938                 'channel': 'Vsauce',
1939                 'episode': 'Episode 1',
1940                 'categories': ['Entertainment'],
1941                 'season': 'Season 1',
1942                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1943                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1944                 'like_count': int,
1945                 'playable_in_embed': True,
1946                 'live_status': 'not_live',
1947                 'channel_follower_count': int,
1948                 'uploader': 'Vsauce',
1949                 'uploader_url': 'https://www.youtube.com/@Vsauce',
1950                 'uploader_id': '@Vsauce',
1951             },
1952             'params': {
1953                 'skip_download': True,
1954             },
1955             'expected_warnings': [
1956                 'Skipping DASH manifest',
1957             ],
1958         },
1959         {
1960             # The following content has been identified by the YouTube community
1961             # as inappropriate or offensive to some audiences.
1962             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1963             'info_dict': {
1964                 'id': '6SJNVb0GnPI',
1965                 'ext': 'mp4',
1966                 'title': 'Race Differences in Intelligence',
1967                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1968                 'duration': 965,
1969                 'upload_date': '20140124',
1970             },
1971             'params': {
1972                 'skip_download': True,
1973             },
1974             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1975         },
1976         {
1977             # itag 212
1978             'url': '1t24XAntNCY',
1979             'only_matching': True,
1980         },
1981         {
1982             # geo restricted to JP
1983             'url': 'sJL6WA-aGkQ',
1984             'only_matching': True,
1985         },
1986         {
1987             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1988             'only_matching': True,
1989         },
1990         {
1991             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1992             'only_matching': True,
1993         },
1994         {
1995             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1996             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1997             'only_matching': True,
1998         },
1999         {
2000             # DRM protected
2001             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
2002             'only_matching': True,
2003         },
2004         {
2005             # Video with unsupported adaptive stream type formats
2006             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
2007             'info_dict': {
2008                 'id': 'Z4Vy8R84T1U',
2009                 'ext': 'mp4',
2010                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
2011                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
2012                 'duration': 433,
2013                 'upload_date': '20130923',
2014                 'formats': 'maxcount:10',
2015             },
2016             'params': {
2017                 'skip_download': True,
2018                 'youtube_include_dash_manifest': False,
2019             },
2020             'skip': 'not actual anymore',
2021         },
2022         {
2023             # Youtube Music Auto-generated description
2024             # TODO: fix metadata extraction
2025             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2026             'info_dict': {
2027                 'id': 'MgNrAu2pzNs',
2028                 'ext': 'mp4',
2029                 'title': 'Voyeur Girl',
2030                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
2031                 'upload_date': '20190312',
2032                 'artist': 'Stephen',
2033                 'track': 'Voyeur Girl',
2034                 'album': 'it\'s too much love to know my dear',
2035                 'release_date': '20190313',
2036                 'release_year': 2019,
2037                 'alt_title': 'Voyeur Girl',
2038                 'view_count': int,
2039                 'playable_in_embed': True,
2040                 'like_count': int,
2041                 'categories': ['Music'],
2042                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
2043                 'channel': 'Stephen',  # TODO: should be "Stephen - Topic"
2044                 'uploader': 'Stephen',
2045                 'availability': 'public',
2046                 'creator': 'Stephen',
2047                 'duration': 169,
2048                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
2049                 'age_limit': 0,
2050                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
2051                 'tags': 'count:11',
2052                 'live_status': 'not_live',
2053                 'channel_follower_count': int
2054             },
2055             'params': {
2056                 'skip_download': True,
2057             },
2058         },
2059         {
2060             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
2061             'only_matching': True,
2062         },
2063         {
2064             # invalid -> valid video id redirection
2065             'url': 'DJztXj2GPfl',
2066             'info_dict': {
2067                 'id': 'DJztXj2GPfk',
2068                 'ext': 'mp4',
2069                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
2070                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
2071                 'upload_date': '20090125',
2072                 'artist': 'Panjabi MC',
2073                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
2074                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
2075             },
2076             'params': {
2077                 'skip_download': True,
2078             },
2079             'skip': 'Video unavailable',
2080         },
2081         {
2082             # empty description results in an empty string
2083             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
2084             'info_dict': {
2085                 'id': 'x41yOUIvK2k',
2086                 'ext': 'mp4',
2087                 'title': 'IMG 3456',
2088                 'description': '',
2089                 'upload_date': '20170613',
2090                 'view_count': int,
2091                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
2092                 'like_count': int,
2093                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
2094                 'tags': [],
2095                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
2096                 'availability': 'public',
2097                 'age_limit': 0,
2098                 'categories': ['Pets & Animals'],
2099                 'duration': 7,
2100                 'playable_in_embed': True,
2101                 'live_status': 'not_live',
2102                 'channel': 'l\'Or Vert asbl',
2103                 'channel_follower_count': int,
2104                 'uploader': 'l\'Or Vert asbl',
2105                 'uploader_url': 'https://www.youtube.com/@ElevageOrVert',
2106                 'uploader_id': '@ElevageOrVert',
2107             },
2108             'params': {
2109                 'skip_download': True,
2110             },
2111         },
2112         {
2113             # with '};' inside yt initial data (see [1])
2114             # see [2] for an example with '};' inside ytInitialPlayerResponse
2115             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
2116             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2117             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
2118             'info_dict': {
2119                 'id': 'CHqg6qOn4no',
2120                 'ext': 'mp4',
2121                 'title': 'Part 77   Sort a list of simple types in c#',
2122                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2123                 'upload_date': '20130831',
2124                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2125                 'like_count': int,
2126                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2127                 'live_status': 'not_live',
2128                 'categories': ['Education'],
2129                 'availability': 'public',
2130                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2131                 'tags': 'count:12',
2132                 'playable_in_embed': True,
2133                 'age_limit': 0,
2134                 'view_count': int,
2135                 'duration': 522,
2136                 'channel': 'kudvenkat',
2137                 'comment_count': int,
2138                 'channel_follower_count': int,
2139                 'chapters': list,
2140                 'uploader': 'kudvenkat',
2141                 'uploader_url': 'https://www.youtube.com/@Csharp-video-tutorialsBlogspot',
2142                 'uploader_id': '@Csharp-video-tutorialsBlogspot',
2143             },
2144             'params': {
2145                 'skip_download': True,
2146             },
2147         },
2148         {
2149             # another example of '};' in ytInitialData
2150             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2151             'only_matching': True,
2152         },
2153         {
2154             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2155             'only_matching': True,
2156         },
2157         {
2158             # https://github.com/ytdl-org/youtube-dl/pull/28094
2159             'url': 'OtqTfy26tG0',
2160             'info_dict': {
2161                 'id': 'OtqTfy26tG0',
2162                 'ext': 'mp4',
2163                 'title': 'Burn Out',
2164                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2165                 'upload_date': '20141120',
2166                 'artist': 'The Cinematic Orchestra',
2167                 'track': 'Burn Out',
2168                 'album': 'Every Day',
2169                 'like_count': int,
2170                 'live_status': 'not_live',
2171                 'alt_title': 'Burn Out',
2172                 'duration': 614,
2173                 'age_limit': 0,
2174                 'view_count': int,
2175                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2176                 'creator': 'The Cinematic Orchestra',
2177                 'channel': 'The Cinematic Orchestra',
2178                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2179                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2180                 'availability': 'public',
2181                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2182                 'categories': ['Music'],
2183                 'playable_in_embed': True,
2184                 'channel_follower_count': int,
2185                 'uploader': 'The Cinematic Orchestra',
2186                 'comment_count': int,
2187             },
2188             'params': {
2189                 'skip_download': True,
2190             },
2191         },
2192         {
2193             # controversial video, only works with bpctr when authenticated with cookies
2194             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2195             'only_matching': True,
2196         },
2197         {
2198             # controversial video, requires bpctr/contentCheckOk
2199             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2200             'info_dict': {
2201                 'id': 'SZJvDhaSDnc',
2202                 'ext': 'mp4',
2203                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2204                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2205                 'upload_date': '20140716',
2206                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2207                 'duration': 170,
2208                 'categories': ['News & Politics'],
2209                 'view_count': int,
2210                 'channel': 'CBS Mornings',
2211                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2212                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2213                 'age_limit': 18,
2214                 'availability': 'needs_auth',
2215                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2216                 'like_count': int,
2217                 'live_status': 'not_live',
2218                 'playable_in_embed': True,
2219                 'channel_follower_count': int,
2220                 'uploader': 'CBS Mornings',
2221                 'uploader_url': 'https://www.youtube.com/@CBSMornings',
2222                 'uploader_id': '@CBSMornings',
2223             }
2224         },
2225         {
2226             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2227             'url': 'cBvYw8_A0vQ',
2228             'info_dict': {
2229                 'id': 'cBvYw8_A0vQ',
2230                 'ext': 'mp4',
2231                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2232                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2233                 'upload_date': '20201120',
2234                 'duration': 1456,
2235                 'categories': ['Travel & Events'],
2236                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2237                 'view_count': int,
2238                 'channel': 'Walk around Japan',
2239                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2240                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2241                 'age_limit': 0,
2242                 'availability': 'public',
2243                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2244                 'live_status': 'not_live',
2245                 'playable_in_embed': True,
2246                 'channel_follower_count': int,
2247                 'uploader': 'Walk around Japan',
2248                 'uploader_url': 'https://www.youtube.com/@walkaroundjapan7124',
2249                 'uploader_id': '@walkaroundjapan7124',
2250             },
2251             'params': {
2252                 'skip_download': True,
2253             },
2254         }, {
2255             # Has multiple audio streams
2256             'url': 'WaOKSUlf4TM',
2257             'only_matching': True
2258         }, {
2259             # Requires Premium: has format 141 when requested using YTM url
2260             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2261             'only_matching': True
2262         }, {
2263             # multiple subtitles with same lang_code
2264             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2265             'only_matching': True,
2266         }, {
2267             # Force use android client fallback
2268             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2269             'info_dict': {
2270                 'id': 'YOelRv7fMxY',
2271                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2272                 'ext': '3gp',
2273                 'upload_date': '20210624',
2274                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2275                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2276                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2277                 'duration': 596,
2278                 'categories': ['Entertainment'],
2279                 'view_count': int,
2280                 'channel': 'colinfurze',
2281                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2282                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2283                 'age_limit': 0,
2284                 'availability': 'public',
2285                 'like_count': int,
2286                 'live_status': 'not_live',
2287                 'playable_in_embed': True,
2288                 'channel_follower_count': int,
2289                 'chapters': list,
2290                 'uploader': 'colinfurze',
2291                 'uploader_url': 'https://www.youtube.com/@colinfurze',
2292                 'uploader_id': '@colinfurze',
2293             },
2294             'params': {
2295                 'format': '17',  # 3gp format available on android
2296                 'extractor_args': {'youtube': {'player_client': ['android']}},
2297             },
2298         },
2299         {
2300             # Skip download of additional client configs (remix client config in this case)
2301             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2302             'only_matching': True,
2303             'params': {
2304                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2305             },
2306         }, {
2307             # shorts
2308             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2309             'only_matching': True,
2310         }, {
2311             'note': 'Storyboards',
2312             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2313             'info_dict': {
2314                 'id': '5KLPxDtMqe8',
2315                 'ext': 'mhtml',
2316                 'format_id': 'sb0',
2317                 'title': 'Your Brain is Plastic',
2318                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2319                 'upload_date': '20140324',
2320                 'like_count': int,
2321                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2322                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2323                 'view_count': int,
2324                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2325                 'playable_in_embed': True,
2326                 'tags': 'count:12',
2327                 'availability': 'public',
2328                 'channel': 'SciShow',
2329                 'live_status': 'not_live',
2330                 'duration': 248,
2331                 'categories': ['Education'],
2332                 'age_limit': 0,
2333                 'channel_follower_count': int,
2334                 'chapters': list,
2335                 'uploader': 'SciShow',
2336                 'uploader_url': 'https://www.youtube.com/@SciShow',
2337                 'uploader_id': '@SciShow',
2338             }, 'params': {'format': 'mhtml', 'skip_download': True}
2339         }, {
2340             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2341             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2342             'info_dict': {
2343                 'id': '2NUZ8W2llS4',
2344                 'ext': 'mp4',
2345                 'title': 'The NP that test your phone performance 🙂',
2346                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2347                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2348                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2349                 'duration': 21,
2350                 'view_count': int,
2351                 'age_limit': 0,
2352                 'categories': ['Gaming'],
2353                 'tags': 'count:23',
2354                 'playable_in_embed': True,
2355                 'live_status': 'not_live',
2356                 'upload_date': '20220103',
2357                 'like_count': int,
2358                 'availability': 'public',
2359                 'channel': 'Leon Nguyen',
2360                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2361                 'comment_count': int,
2362                 'channel_follower_count': int,
2363                 'uploader': 'Leon Nguyen',
2364                 'uploader_url': 'https://www.youtube.com/@LeonNguyen',
2365                 'uploader_id': '@LeonNguyen',
2366             }
2367         }, {
2368             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2369             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2370             'info_dict': {
2371                 'id': '2NUZ8W2llS4',
2372                 'ext': 'mp4',
2373                 'title': 'The NP that test your phone performance 🙂',
2374                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2375                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2376                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2377                 'duration': 21,
2378                 'view_count': int,
2379                 'age_limit': 0,
2380                 'categories': ['Gaming'],
2381                 'tags': 'count:23',
2382                 'playable_in_embed': True,
2383                 'live_status': 'not_live',
2384                 'upload_date': '20220102',
2385                 'like_count': int,
2386                 'availability': 'public',
2387                 'channel': 'Leon Nguyen',
2388                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2389                 'comment_count': int,
2390                 'channel_follower_count': int,
2391                 'uploader': 'Leon Nguyen',
2392                 'uploader_url': 'https://www.youtube.com/@LeonNguyen',
2393                 'uploader_id': '@LeonNguyen',
2394             },
2395             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2396         }, {
2397             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2398             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2399             'info_dict': {
2400                 'id': 'mzZzzBU6lrM',
2401                 'ext': 'mp4',
2402                 'title': 'I Met GeorgeNotFound In Real Life...',
2403                 'description': 'md5:978296ec9783a031738b684d4ebf302d',
2404                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2405                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2406                 'duration': 955,
2407                 'view_count': int,
2408                 'age_limit': 0,
2409                 'categories': ['Entertainment'],
2410                 'tags': 'count:26',
2411                 'playable_in_embed': True,
2412                 'live_status': 'not_live',
2413                 'release_timestamp': 1641172509,
2414                 'release_date': '20220103',
2415                 'upload_date': '20220103',
2416                 'like_count': int,
2417                 'availability': 'public',
2418                 'channel': 'Quackity',
2419                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2420                 'channel_follower_count': int,
2421                 'uploader': 'Quackity',
2422                 'uploader_id': '@Quackity',
2423                 'uploader_url': 'https://www.youtube.com/@Quackity',
2424             }
2425         },
2426         {   # continuous livestream. Microformat upload date should be preferred.
2427             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2428             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2429             'info_dict': {
2430                 'id': 'kgx4WGK0oNU',
2431                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2432                 'ext': 'mp4',
2433                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2434                 'availability': 'public',
2435                 'age_limit': 0,
2436                 'release_timestamp': 1637975704,
2437                 'upload_date': '20210619',
2438                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2439                 'live_status': 'is_live',
2440                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2441                 'channel': 'Abao in Tokyo',
2442                 'channel_follower_count': int,
2443                 'release_date': '20211127',
2444                 'tags': 'count:39',
2445                 'categories': ['People & Blogs'],
2446                 'like_count': int,
2447                 'view_count': int,
2448                 'playable_in_embed': True,
2449                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2450                 'concurrent_view_count': int,
2451                 'uploader': 'Abao in Tokyo',
2452                 'uploader_url': 'https://www.youtube.com/@abaointokyo',
2453                 'uploader_id': '@abaointokyo',
2454             },
2455             'params': {'skip_download': True}
2456         }, {
2457             # Story. Requires specific player params to work.
2458             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2459             'info_dict': {
2460                 'id': 'vv8qTUWmulI',
2461                 'ext': 'mp4',
2462                 'availability': 'unlisted',
2463                 'view_count': int,
2464                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2465                 'upload_date': '20220526',
2466                 'categories': ['Education'],
2467                 'title': 'Story',
2468                 'channel': 'IT\'S HISTORY',
2469                 'description': '',
2470                 'duration': 12,
2471                 'playable_in_embed': True,
2472                 'age_limit': 0,
2473                 'live_status': 'not_live',
2474                 'tags': [],
2475                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2476                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2477             },
2478             'skip': 'stories get removed after some period of time',
2479         }, {
2480             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2481             'info_dict': {
2482                 'id': 'tjjjtzRLHvA',
2483                 'ext': 'mp4',
2484                 'title': 'ハッシュタグ無し };if window.ytcsi',
2485                 'upload_date': '20220323',
2486                 'like_count': int,
2487                 'availability': 'unlisted',
2488                 'channel': 'Lesmiscore',
2489                 'thumbnail': r're:^https?://.*\.jpg',
2490                 'age_limit': 0,
2491                 'categories': ['Music'],
2492                 'view_count': int,
2493                 'description': '',
2494                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2495                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2496                 'live_status': 'not_live',
2497                 'playable_in_embed': True,
2498                 'channel_follower_count': int,
2499                 'duration': 6,
2500                 'tags': [],
2501                 'uploader_id': '@lesmiscore',
2502                 'uploader': 'Lesmiscore',
2503                 'uploader_url': 'https://www.youtube.com/@lesmiscore',
2504             }
2505         }, {
2506             # Prefer primary title+description language metadata by default
2507             # Do not prefer translated description if primary is empty
2508             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2509             'info_dict': {
2510                 'id': 'el3E4MbxRqQ',
2511                 'ext': 'mp4',
2512                 'title': 'dlp test video 2 - primary sv no desc',
2513                 'description': '',
2514                 'channel': 'cole-dlp-test-acc',
2515                 'tags': [],
2516                 'view_count': int,
2517                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2518                 'like_count': int,
2519                 'playable_in_embed': True,
2520                 'availability': 'unlisted',
2521                 'thumbnail': r're:^https?://.*\.jpg',
2522                 'age_limit': 0,
2523                 'duration': 5,
2524                 'live_status': 'not_live',
2525                 'upload_date': '20220908',
2526                 'categories': ['People & Blogs'],
2527                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2528                 'uploader_url': 'https://www.youtube.com/@coletdjnz',
2529                 'uploader_id': '@coletdjnz',
2530                 'uploader': 'cole-dlp-test-acc',
2531             },
2532             'params': {'skip_download': True}
2533         }, {
2534             # Extractor argument: prefer translated title+description
2535             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2536             'info_dict': {
2537                 'id': 'gHKT4uU8Zng',
2538                 'ext': 'mp4',
2539                 'channel': 'cole-dlp-test-acc',
2540                 'tags': [],
2541                 'duration': 5,
2542                 'live_status': 'not_live',
2543                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2544                 'upload_date': '20220728',
2545                 'view_count': int,
2546                 'categories': ['People & Blogs'],
2547                 'thumbnail': r're:^https?://.*\.jpg',
2548                 'title': 'dlp test video title translated (fr)',
2549                 'availability': 'public',
2550                 'age_limit': 0,
2551                 'description': 'dlp test video description translated (fr)',
2552                 'playable_in_embed': True,
2553                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2554                 'uploader_url': 'https://www.youtube.com/@coletdjnz',
2555                 'uploader_id': '@coletdjnz',
2556                 'uploader': 'cole-dlp-test-acc',
2557             },
2558             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2559             'expected_warnings': [r'Preferring "fr" translated fields'],
2560         }, {
2561             'note': '6 channel audio',
2562             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2563             'only_matching': True,
2564         }, {
2565             'note': 'Multiple HLS formats with same itag',
2566             'url': 'https://www.youtube.com/watch?v=kX3nB4PpJko',
2567             'info_dict': {
2568                 'id': 'kX3nB4PpJko',
2569                 'ext': 'mp4',
2570                 'categories': ['Entertainment'],
2571                 'description': 'md5:e8031ff6e426cdb6a77670c9b81f6fa6',
2572                 'live_status': 'not_live',
2573                 'duration': 937,
2574                 'channel_follower_count': int,
2575                 'thumbnail': 'https://i.ytimg.com/vi_webp/kX3nB4PpJko/maxresdefault.webp',
2576                 'title': 'Last To Take Hand Off Jet, Keeps It!',
2577                 'channel': 'MrBeast',
2578                 'playable_in_embed': True,
2579                 'view_count': int,
2580                 'upload_date': '20221112',
2581                 'channel_url': 'https://www.youtube.com/channel/UCX6OQ3DkcsbYNE6H8uQQuVA',
2582                 'age_limit': 0,
2583                 'availability': 'public',
2584                 'channel_id': 'UCX6OQ3DkcsbYNE6H8uQQuVA',
2585                 'like_count': int,
2586                 'tags': [],
2587                 'uploader': 'MrBeast',
2588                 'uploader_url': 'https://www.youtube.com/@MrBeast',
2589                 'uploader_id': '@MrBeast',
2590             },
2591             'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
2592         }, {
2593             'note': 'Audio formats with Dynamic Range Compression',
2594             'url': 'https://www.youtube.com/watch?v=Tq92D6wQ1mg',
2595             'info_dict': {
2596                 'id': 'Tq92D6wQ1mg',
2597                 'ext': 'webm',
2598                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
2599                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2600                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2601                 'channel_follower_count': int,
2602                 'description': 'md5:17eccca93a786d51bc67646756894066',
2603                 'upload_date': '20191228',
2604                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
2605                 'playable_in_embed': True,
2606                 'like_count': int,
2607                 'categories': ['Entertainment'],
2608                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
2609                 'age_limit': 18,
2610                 'channel': 'Projekt Melody',
2611                 'view_count': int,
2612                 'availability': 'needs_auth',
2613                 'comment_count': int,
2614                 'live_status': 'not_live',
2615                 'duration': 106,
2616                 'uploader': 'Projekt Melody',
2617                 'uploader_id': '@ProjektMelody',
2618                 'uploader_url': 'https://www.youtube.com/@ProjektMelody',
2619             },
2620             'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
2621         },
2622         {
2623             'url': 'https://www.youtube.com/live/qVv6vCqciTM',
2624             'info_dict': {
2625                 'id': 'qVv6vCqciTM',
2626                 'ext': 'mp4',
2627                 'age_limit': 0,
2628                 'comment_count': int,
2629                 'chapters': 'count:13',
2630                 'upload_date': '20221223',
2631                 'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
2632                 'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
2633                 'like_count': int,
2634                 'release_date': '20221223',
2635                 'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
2636                 'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
2637                 'view_count': int,
2638                 'playable_in_embed': True,
2639                 'duration': 4438,
2640                 'availability': 'public',
2641                 'channel_follower_count': int,
2642                 'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
2643                 'categories': ['Entertainment'],
2644                 'live_status': 'was_live',
2645                 'release_timestamp': 1671793345,
2646                 'channel': 'さなちゃんねる',
2647                 'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
2648                 'uploader': 'さなちゃんねる',
2649                 'uploader_url': 'https://www.youtube.com/@sana_natori',
2650                 'uploader_id': '@sana_natori',
2651             },
2652         },
2653         {
2654             # Fallbacks when webpage and web client is unavailable
2655             'url': 'https://www.youtube.com/watch?v=wSSmNUl9Snw',
2656             'info_dict': {
2657                 'id': 'wSSmNUl9Snw',
2658                 'ext': 'mp4',
2659                 # 'categories': ['Science & Technology'],
2660                 'view_count': int,
2661                 'chapters': 'count:2',
2662                 'channel': 'Scott Manley',
2663                 'like_count': int,
2664                 'age_limit': 0,
2665                 # 'availability': 'public',
2666                 'channel_follower_count': int,
2667                 'live_status': 'not_live',
2668                 'upload_date': '20170831',
2669                 'duration': 682,
2670                 'tags': 'count:8',
2671                 'uploader_url': 'https://www.youtube.com/@scottmanley',
2672                 'description': 'md5:f4bed7b200404b72a394c2f97b782c02',
2673                 'uploader': 'Scott Manley',
2674                 'uploader_id': '@scottmanley',
2675                 'title': 'The Computer Hack That Saved Apollo 14',
2676                 'channel_id': 'UCxzC4EngIsMrPmbm6Nxvb-A',
2677                 'thumbnail': r're:^https?://.*\.webp',
2678                 'channel_url': 'https://www.youtube.com/channel/UCxzC4EngIsMrPmbm6Nxvb-A',
2679                 'playable_in_embed': True,
2680             },
2681             'params': {
2682                 'extractor_args': {'youtube': {'player_client': ['android'], 'player_skip': ['webpage']}},
2683             },
2684         },
2685     ]
2686
2687     _WEBPAGE_TESTS = [
2688         # YouTube <object> embed
2689         {
2690             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2691             'md5': '873c81d308b979f0e23ee7e620b312a3',
2692             'info_dict': {
2693                 'id': 'msN87y-iEx0',
2694                 'ext': 'mp4',
2695                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2696                 'upload_date': '20080526',
2697                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2698                 'age_limit': 0,
2699                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2700                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2701                 'playable_in_embed': True,
2702                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2703                 'like_count': int,
2704                 'comment_count': int,
2705                 'channel': 'Christopher Sykes',
2706                 'live_status': 'not_live',
2707                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2708                 'availability': 'public',
2709                 'duration': 195,
2710                 'view_count': int,
2711                 'categories': ['Science & Technology'],
2712                 'channel_follower_count': int,
2713                 'uploader': 'Christopher Sykes',
2714                 'uploader_url': 'https://www.youtube.com/@ChristopherSykesDocumentaries',
2715                 'uploader_id': '@ChristopherSykesDocumentaries',
2716             },
2717             'params': {
2718                 'skip_download': True,
2719             }
2720         },
2721     ]
2722
2723     @classmethod
2724     def suitable(cls, url):
2725         from ..utils import parse_qs
2726
2727         qs = parse_qs(url)
2728         if qs.get('list', [None])[0]:
2729             return False
2730         return super().suitable(url)
2731
2732     def __init__(self, *args, **kwargs):
2733         super().__init__(*args, **kwargs)
2734         self._code_cache = {}
2735         self._player_cache = {}
2736
2737     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2738         lock = threading.Lock()
2739         start_time = time.time()
2740         formats = [f for f in formats if f.get('is_from_start')]
2741
2742         def refetch_manifest(format_id, delay):
2743             nonlocal formats, start_time, is_live
2744             if time.time() <= start_time + delay:
2745                 return
2746
2747             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2748             video_details = traverse_obj(prs, (..., 'videoDetails'), expected_type=dict)
2749             microformats = traverse_obj(
2750                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2751                 expected_type=dict)
2752             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2753             is_live = live_status == 'is_live'
2754             start_time = time.time()
2755
2756         def mpd_feed(format_id, delay):
2757             """
2758             @returns (manifest_url, manifest_stream_number, is_live) or None
2759             """
2760             for retry in self.RetryManager(fatal=False):
2761                 with lock:
2762                     refetch_manifest(format_id, delay)
2763
2764                 f = next((f for f in formats if f['format_id'] == format_id), None)
2765                 if not f:
2766                     if not is_live:
2767                         retry.error = f'{video_id}: Video is no longer live'
2768                     else:
2769                         retry.error = f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}'
2770                     continue
2771                 return f['manifest_url'], f['manifest_stream_number'], is_live
2772             return None
2773
2774         for f in formats:
2775             f['is_live'] = is_live
2776             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2777                                     live_start_time, mpd_feed, not is_live and f.copy())
2778             if is_live:
2779                 f['fragments'] = gen
2780                 f['protocol'] = 'http_dash_segments_generator'
2781             else:
2782                 f['fragments'] = LazyList(gen({}))
2783                 del f['is_from_start']
2784
2785     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2786         FETCH_SPAN, MAX_DURATION = 5, 432000
2787
2788         mpd_url, stream_number, is_live = None, None, True
2789
2790         begin_index = 0
2791         download_start_time = ctx.get('start') or time.time()
2792
2793         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2794         if lack_early_segments:
2795             self.report_warning(bug_reports_message(
2796                 'Starting download from the last 120 hours of the live stream since '
2797                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2798             lack_early_segments = True
2799
2800         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2801         fragments, fragment_base_url = None, None
2802
2803         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2804             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2805             # Obtain from MPD's maximum seq value
2806             old_mpd_url = mpd_url
2807             last_error = ctx.pop('last_error', None)
2808             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2809             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2810                                                or (mpd_url, stream_number, False))
2811             if not refresh_sequence:
2812                 if expire_fast and not is_live:
2813                     return False, last_seq
2814                 elif old_mpd_url == mpd_url:
2815                     return True, last_seq
2816             if manifestless_orig_fmt:
2817                 fmt_info = manifestless_orig_fmt
2818             else:
2819                 try:
2820                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2821                         mpd_url, None, note=False, errnote=False, fatal=False)
2822                 except ExtractorError:
2823                     fmts = None
2824                 if not fmts:
2825                     no_fragment_score += 2
2826                     return False, last_seq
2827                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2828             fragments = fmt_info['fragments']
2829             fragment_base_url = fmt_info['fragment_base_url']
2830             assert fragment_base_url
2831
2832             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2833             return True, _last_seq
2834
2835         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2836         while is_live:
2837             fetch_time = time.time()
2838             if no_fragment_score > 30:
2839                 return
2840             if last_segment_url:
2841                 # Obtain from "X-Head-Seqnum" header value from each segment
2842                 try:
2843                     urlh = self._request_webpage(
2844                         last_segment_url, None, note=False, errnote=False, fatal=False)
2845                 except ExtractorError:
2846                     urlh = None
2847                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2848                 if last_seq is None:
2849                     no_fragment_score += 2
2850                     last_segment_url = None
2851                     continue
2852             else:
2853                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2854                 no_fragment_score += 2
2855                 if not should_continue:
2856                     continue
2857
2858             if known_idx > last_seq:
2859                 last_segment_url = None
2860                 continue
2861
2862             last_seq += 1
2863
2864             if begin_index < 0 and known_idx < 0:
2865                 # skip from the start when it's negative value
2866                 known_idx = last_seq + begin_index
2867             if lack_early_segments:
2868                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2869             try:
2870                 for idx in range(known_idx, last_seq):
2871                     # do not update sequence here or you'll get skipped some part of it
2872                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2873                     if not should_continue:
2874                         known_idx = idx - 1
2875                         raise ExtractorError('breaking out of outer loop')
2876                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2877                     yield {
2878                         'url': last_segment_url,
2879                         'fragment_count': last_seq,
2880                     }
2881                 if known_idx == last_seq:
2882                     no_fragment_score += 5
2883                 else:
2884                     no_fragment_score = 0
2885                 known_idx = last_seq
2886             except ExtractorError:
2887                 continue
2888
2889             if manifestless_orig_fmt:
2890                 # Stop at the first iteration if running for post-live manifestless;
2891                 # fragment count no longer increase since it starts
2892                 break
2893
2894             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2895
2896     def _extract_player_url(self, *ytcfgs, webpage=None):
2897         player_url = traverse_obj(
2898             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2899             get_all=False, expected_type=str)
2900         if not player_url:
2901             return
2902         return urljoin('https://www.youtube.com', player_url)
2903
2904     def _download_player_url(self, video_id, fatal=False):
2905         res = self._download_webpage(
2906             'https://www.youtube.com/iframe_api',
2907             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2908         if res:
2909             player_version = self._search_regex(
2910                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2911             if player_version:
2912                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2913
2914     def _signature_cache_id(self, example_sig):
2915         """ Return a string representation of a signature """
2916         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2917
2918     @classmethod
2919     def _extract_player_info(cls, player_url):
2920         for player_re in cls._PLAYER_INFO_RE:
2921             id_m = re.search(player_re, player_url)
2922             if id_m:
2923                 break
2924         else:
2925             raise ExtractorError('Cannot identify player %r' % player_url)
2926         return id_m.group('id')
2927
2928     def _load_player(self, video_id, player_url, fatal=True):
2929         player_id = self._extract_player_info(player_url)
2930         if player_id not in self._code_cache:
2931             code = self._download_webpage(
2932                 player_url, video_id, fatal=fatal,
2933                 note='Downloading player ' + player_id,
2934                 errnote='Download of %s failed' % player_url)
2935             if code:
2936                 self._code_cache[player_id] = code
2937         return self._code_cache.get(player_id)
2938
2939     def _extract_signature_function(self, video_id, player_url, example_sig):
2940         player_id = self._extract_player_info(player_url)
2941
2942         # Read from filesystem cache
2943         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2944         assert os.path.basename(func_id) == func_id
2945
2946         self.write_debug(f'Extracting signature function {func_id}')
2947         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2948
2949         if not cache_spec:
2950             code = self._load_player(video_id, player_url)
2951         if code:
2952             res = self._parse_sig_js(code)
2953             test_string = ''.join(map(chr, range(len(example_sig))))
2954             cache_spec = [ord(c) for c in res(test_string)]
2955             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2956
2957         return lambda s: ''.join(s[i] for i in cache_spec)
2958
2959     def _print_sig_code(self, func, example_sig):
2960         if not self.get_param('youtube_print_sig_code'):
2961             return
2962
2963         def gen_sig_code(idxs):
2964             def _genslice(start, end, step):
2965                 starts = '' if start == 0 else str(start)
2966                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2967                 steps = '' if step == 1 else (':%d' % step)
2968                 return f's[{starts}{ends}{steps}]'
2969
2970             step = None
2971             # Quelch pyflakes warnings - start will be set when step is set
2972             start = '(Never used)'
2973             for i, prev in zip(idxs[1:], idxs[:-1]):
2974                 if step is not None:
2975                     if i - prev == step:
2976                         continue
2977                     yield _genslice(start, prev, step)
2978                     step = None
2979                     continue
2980                 if i - prev in [-1, 1]:
2981                     step = i - prev
2982                     start = prev
2983                     continue
2984                 else:
2985                     yield 's[%d]' % prev
2986             if step is None:
2987                 yield 's[%d]' % i
2988             else:
2989                 yield _genslice(start, i, step)
2990
2991         test_string = ''.join(map(chr, range(len(example_sig))))
2992         cache_res = func(test_string)
2993         cache_spec = [ord(c) for c in cache_res]
2994         expr_code = ' + '.join(gen_sig_code(cache_spec))
2995         signature_id_tuple = '(%s)' % (
2996             ', '.join(str(len(p)) for p in example_sig.split('.')))
2997         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2998                 '    return %s\n') % (signature_id_tuple, expr_code)
2999         self.to_screen('Extracted signature function:\n' + code)
3000
3001     def _parse_sig_js(self, jscode):
3002         funcname = self._search_regex(
3003             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3004              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3005              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
3006              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
3007              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
3008              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
3009              # Obsolete patterns
3010              r'("|\')signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3011              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
3012              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3013              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3014              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
3015              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
3016             jscode, 'Initial JS player signature function name', group='sig')
3017
3018         jsi = JSInterpreter(jscode)
3019         initial_function = jsi.extract_function(funcname)
3020         return lambda s: initial_function([s])
3021
3022     def _cached(self, func, *cache_id):
3023         def inner(*args, **kwargs):
3024             if cache_id not in self._player_cache:
3025                 try:
3026                     self._player_cache[cache_id] = func(*args, **kwargs)
3027                 except ExtractorError as e:
3028                     self._player_cache[cache_id] = e
3029                 except Exception as e:
3030                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
3031
3032             ret = self._player_cache[cache_id]
3033             if isinstance(ret, Exception):
3034                 raise ret
3035             return ret
3036         return inner
3037
3038     def _decrypt_signature(self, s, video_id, player_url):
3039         """Turn the encrypted s field into a working signature"""
3040         extract_sig = self._cached(
3041             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
3042         func = extract_sig(video_id, player_url, s)
3043         self._print_sig_code(func, s)
3044         return func(s)
3045
3046     def _decrypt_nsig(self, s, video_id, player_url):
3047         """Turn the encrypted n field into a working signature"""
3048         if player_url is None:
3049             raise ExtractorError('Cannot decrypt nsig without player_url')
3050         player_url = urljoin('https://www.youtube.com', player_url)
3051
3052         try:
3053             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
3054         except ExtractorError as e:
3055             raise ExtractorError('Unable to extract nsig function code', cause=e)
3056         if self.get_param('youtube_print_sig_code'):
3057             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
3058
3059         try:
3060             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
3061             ret = extract_nsig(jsi, func_code)(s)
3062         except JSInterpreter.Exception as e:
3063             try:
3064                 jsi = PhantomJSwrapper(self, timeout=5000)
3065             except ExtractorError:
3066                 raise e
3067             self.report_warning(
3068                 f'Native nsig extraction failed: Trying with PhantomJS\n'
3069                 f'         n = {s} ; player = {player_url}', video_id)
3070             self.write_debug(e, only_once=True)
3071
3072             args, func_body = func_code
3073             ret = jsi.execute(
3074                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
3075                 video_id=video_id, note='Executing signature code').strip()
3076
3077         self.write_debug(f'Decrypted nsig {s} => {ret}')
3078         return ret
3079
3080     def _extract_n_function_name(self, jscode):
3081         funcname, idx = self._search_regex(
3082             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
3083             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
3084         if not idx:
3085             return funcname
3086
3087         return json.loads(js_to_json(self._search_regex(
3088             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
3089             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
3090
3091     def _extract_n_function_code(self, video_id, player_url):
3092         player_id = self._extract_player_info(player_url)
3093         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
3094         jscode = func_code or self._load_player(video_id, player_url)
3095         jsi = JSInterpreter(jscode)
3096
3097         if func_code:
3098             return jsi, player_id, func_code
3099
3100         func_name = self._extract_n_function_name(jscode)
3101
3102         # For redundancy
3103         func_code = self._search_regex(
3104             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
3105                      # NB: The end of the regex is intentionally kept strict
3106                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
3107             jscode, 'nsig function', group=('var', 'code'), default=None)
3108         if func_code:
3109             func_code = ([func_code[0]], func_code[1])
3110         else:
3111             self.write_debug('Extracting nsig function with jsinterp')
3112             func_code = jsi.extract_function_code(func_name)
3113
3114         self.cache.store('youtube-nsig', player_id, func_code)
3115         return jsi, player_id, func_code
3116
3117     def _extract_n_function_from_code(self, jsi, func_code):
3118         func = jsi.extract_function_from_code(*func_code)
3119
3120         def extract_nsig(s):
3121             try:
3122                 ret = func([s])
3123             except JSInterpreter.Exception:
3124                 raise
3125             except Exception as e:
3126                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
3127
3128             if ret.startswith('enhanced_except_'):
3129                 raise JSInterpreter.Exception('Signature function returned an exception')
3130             return ret
3131
3132         return extract_nsig
3133
3134     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
3135         """
3136         Extract signatureTimestamp (sts)
3137         Required to tell API what sig/player version is in use.
3138         """
3139         sts = None
3140         if isinstance(ytcfg, dict):
3141             sts = int_or_none(ytcfg.get('STS'))
3142
3143         if not sts:
3144             # Attempt to extract from player
3145             if player_url is None:
3146                 error_msg = 'Cannot extract signature timestamp without player_url.'
3147                 if fatal:
3148                     raise ExtractorError(error_msg)
3149                 self.report_warning(error_msg)
3150                 return
3151             code = self._load_player(video_id, player_url, fatal=fatal)
3152             if code:
3153                 sts = int_or_none(self._search_regex(
3154                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
3155                     'JS player signature timestamp', group='sts', fatal=fatal))
3156         return sts
3157
3158     def _mark_watched(self, video_id, player_responses):
3159         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
3160             label = 'fully ' if is_full else ''
3161             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
3162                             expected_type=url_or_none)
3163             if not url:
3164                 self.report_warning(f'Unable to mark {label}watched')
3165                 return
3166             parsed_url = urllib.parse.urlparse(url)
3167             qs = urllib.parse.parse_qs(parsed_url.query)
3168
3169             # cpn generation algorithm is reverse engineered from base.js.
3170             # In fact it works even with dummy cpn.
3171             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
3172             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
3173
3174             # # more consistent results setting it to right before the end
3175             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
3176
3177             qs.update({
3178                 'ver': ['2'],
3179                 'cpn': [cpn],
3180                 'cmt': video_length,
3181                 'el': 'detailpage',  # otherwise defaults to "shorts"
3182             })
3183
3184             if is_full:
3185                 # these seem to mark watchtime "history" in the real world
3186                 # they're required, so send in a single value
3187                 qs.update({
3188                     'st': 0,
3189                     'et': video_length,
3190                 })
3191
3192             url = urllib.parse.urlunparse(
3193                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
3194
3195             self._download_webpage(
3196                 url, video_id, f'Marking {label}watched',
3197                 'Unable to mark watched', fatal=False)
3198
3199     @classmethod
3200     def _extract_from_webpage(cls, url, webpage):
3201         # Invidious Instances
3202         # https://github.com/yt-dlp/yt-dlp/issues/195
3203         # https://github.com/iv-org/invidious/pull/1730
3204         mobj = re.search(
3205             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
3206             webpage)
3207         if mobj:
3208             yield cls.url_result(mobj.group('url'), cls)
3209             raise cls.StopExtraction()
3210
3211         yield from super()._extract_from_webpage(url, webpage)
3212
3213         # lazyYT YouTube embed
3214         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
3215             yield cls.url_result(unescapeHTML(id_), cls, id_)
3216
3217         # Wordpress "YouTube Video Importer" plugin
3218         for m in re.findall(r'''(?x)<div[^>]+
3219                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
3220                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
3221             yield cls.url_result(m[-1], cls, m[-1])
3222
3223     @classmethod
3224     def extract_id(cls, url):
3225         video_id = cls.get_temp_id(url)
3226         if not video_id:
3227             raise ExtractorError(f'Invalid URL: {url}')
3228         return video_id
3229
3230     def _extract_chapters_from_json(self, data, duration):
3231         chapter_list = traverse_obj(
3232             data, (
3233                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
3234                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3235             ), expected_type=list)
3236
3237         return self._extract_chapters_helper(
3238             chapter_list,
3239             start_function=lambda chapter: float_or_none(
3240                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3241             title_function=lambda chapter: traverse_obj(
3242                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3243             duration=duration)
3244
3245     def _extract_chapters_from_engagement_panel(self, data, duration):
3246         content_list = traverse_obj(
3247             data,
3248             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3249             expected_type=list)
3250         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3251         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3252
3253         return next(filter(None, (
3254             self._extract_chapters_helper(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3255                                           chapter_time, chapter_title, duration)
3256             for contents in content_list)), [])
3257
3258     def _extract_heatmap_from_player_overlay(self, data):
3259         content_list = traverse_obj(data, (
3260             'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer', 'decoratedPlayerBarRenderer', 'playerBar',
3261             'multiMarkersPlayerBarRenderer', 'markersMap', ..., 'value', 'heatmap', 'heatmapRenderer', 'heatMarkers', {list}))
3262         return next(filter(None, (
3263             traverse_obj(contents, (..., 'heatMarkerRenderer', {
3264                 'start_time': ('timeRangeStartMillis', {functools.partial(float_or_none, scale=1000)}),
3265                 'end_time': {lambda x: (x['timeRangeStartMillis'] + x['markerDurationMillis']) / 1000},
3266                 'value': ('heatMarkerIntensityScoreNormalized', {float_or_none}),
3267             })) for contents in content_list)), None)
3268
3269     def _extract_comment(self, comment_renderer, parent=None):
3270         comment_id = comment_renderer.get('commentId')
3271         if not comment_id:
3272             return
3273
3274         text = self._get_text(comment_renderer, 'contentText')
3275
3276         # Timestamp is an estimate calculated from the current time and time_text
3277         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3278         timestamp = self._parse_time_text(time_text)
3279
3280         author = self._get_text(comment_renderer, 'authorText')
3281         author_id = try_get(comment_renderer,
3282                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3283
3284         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3285                                                        lambda x: x['likeCount']), str)) or 0
3286         author_thumbnail = try_get(comment_renderer,
3287                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3288
3289         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3290         is_favorited = 'creatorHeart' in (try_get(
3291             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3292         return {
3293             'id': comment_id,
3294             'text': text,
3295             'timestamp': timestamp,
3296             'time_text': time_text,
3297             'like_count': votes,
3298             'is_favorited': is_favorited,
3299             'author': author,
3300             'author_id': author_id,
3301             'author_thumbnail': author_thumbnail,
3302             'author_is_uploader': author_is_uploader,
3303             'parent': parent or 'root'
3304         }
3305
3306     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3307
3308         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3309
3310         def extract_header(contents):
3311             _continuation = None
3312             for content in contents:
3313                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3314                 expected_comment_count = self._get_count(
3315                     comments_header_renderer, 'countText', 'commentsCount')
3316
3317                 if expected_comment_count:
3318                     tracker['est_total'] = expected_comment_count
3319                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3320                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3321
3322                 sort_menu_item = try_get(
3323                     comments_header_renderer,
3324                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3325                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3326
3327                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3328                 if not _continuation:
3329                     continue
3330
3331                 sort_text = str_or_none(sort_menu_item.get('title'))
3332                 if not sort_text:
3333                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3334                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3335                 break
3336             return _continuation
3337
3338         def extract_thread(contents):
3339             if not parent:
3340                 tracker['current_page_thread'] = 0
3341             for content in contents:
3342                 if not parent and tracker['total_parent_comments'] >= max_parents:
3343                     yield
3344                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3345                 comment_renderer = get_first(
3346                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3347                     expected_type=dict, default={})
3348
3349                 comment = self._extract_comment(comment_renderer, parent)
3350                 if not comment:
3351                     continue
3352                 is_pinned = bool(traverse_obj(comment_renderer, 'pinnedCommentBadge'))
3353                 comment_id = comment['id']
3354                 if is_pinned:
3355                     tracker['pinned_comment_ids'].add(comment_id)
3356                 # Sometimes YouTube may break and give us infinite looping comments.
3357                 # See: https://github.com/yt-dlp/yt-dlp/issues/6290
3358                 if comment_id in tracker['seen_comment_ids']:
3359                     if comment_id in tracker['pinned_comment_ids'] and not is_pinned:
3360                         # Pinned comments may appear a second time in newest first sort
3361                         # See: https://github.com/yt-dlp/yt-dlp/issues/6712
3362                         continue
3363                     self.report_warning('Detected YouTube comments looping. Stopping comment extraction as we probably cannot get any more.')
3364                     yield
3365                 else:
3366                     tracker['seen_comment_ids'].add(comment['id'])
3367
3368                 tracker['running_total'] += 1
3369                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3370                 yield comment
3371
3372                 # Attempt to get the replies
3373                 comment_replies_renderer = try_get(
3374                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3375
3376                 if comment_replies_renderer:
3377                     tracker['current_page_thread'] += 1
3378                     comment_entries_iter = self._comment_entries(
3379                         comment_replies_renderer, ytcfg, video_id,
3380                         parent=comment.get('id'), tracker=tracker)
3381                     yield from itertools.islice(comment_entries_iter, min(
3382                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3383
3384         # Keeps track of counts across recursive calls
3385         if not tracker:
3386             tracker = dict(
3387                 running_total=0,
3388                 est_total=0,
3389                 current_page_thread=0,
3390                 total_parent_comments=0,
3391                 total_reply_comments=0,
3392                 seen_comment_ids=set(),
3393                 pinned_comment_ids=set()
3394             )
3395
3396         # TODO: Deprecated
3397         # YouTube comments have a max depth of 2
3398         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3399         if max_depth:
3400             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3401                                                 'Set max replies in the max-comments extractor argument instead')
3402         if max_depth == 1 and parent:
3403             return
3404
3405         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3406             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3407
3408         continuation = self._extract_continuation(root_continuation_data)
3409
3410         response = None
3411         is_forced_continuation = False
3412         is_first_continuation = parent is None
3413         if is_first_continuation and not continuation:
3414             # Sometimes you can get comments by generating the continuation yourself,
3415             # even if YouTube initially reports them being disabled - e.g. stories comments.
3416             # Note: if the comment section is actually disabled, YouTube may return a response with
3417             # required check_get_keys missing. So we will disable that check initially in this case.
3418             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3419             is_forced_continuation = True
3420
3421         for page_num in itertools.count(0):
3422             if not continuation:
3423                 break
3424             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3425             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3426             if page_num == 0:
3427                 if is_first_continuation:
3428                     note_prefix = 'Downloading comment section API JSON'
3429                 else:
3430                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3431                         tracker['current_page_thread'], comment_prog_str)
3432             else:
3433                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3434                     '       ' if parent else '', ' replies' if parent else '',
3435                     page_num, comment_prog_str)
3436             try:
3437                 response = self._extract_response(
3438                     item_id=None, query=continuation,
3439                     ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3440                     check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3441             except ExtractorError as e:
3442                 # Ignore incomplete data error for replies if retries didn't work.
3443                 # This is to allow any other parent comments and comment threads to be downloaded.
3444                 # See: https://github.com/yt-dlp/yt-dlp/issues/4669
3445                 if 'incomplete data' in str(e).lower() and parent and self.get_param('ignoreerrors') is True:
3446                     self.report_warning(
3447                         'Received incomplete data for a comment reply thread and retrying did not help. '
3448                         'Ignoring to let other comments be downloaded.')
3449                 else:
3450                     raise
3451             is_forced_continuation = False
3452             continuation_contents = traverse_obj(
3453                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3454
3455             continuation = None
3456             for continuation_section in continuation_contents:
3457                 continuation_items = traverse_obj(
3458                     continuation_section,
3459                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3460                     get_all=False, expected_type=list) or []
3461                 if is_first_continuation:
3462                     continuation = extract_header(continuation_items)
3463                     is_first_continuation = False
3464                     if continuation:
3465                         break
3466                     continue
3467
3468                 for entry in extract_thread(continuation_items):
3469                     if not entry:
3470                         return
3471                     yield entry
3472                 continuation = self._extract_continuation({'contents': continuation_items})
3473                 if continuation:
3474                     break
3475
3476         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3477         if message and not parent and tracker['running_total'] == 0:
3478             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3479             raise self.CommentsDisabled
3480
3481     @staticmethod
3482     def _generate_comment_continuation(video_id):
3483         """
3484         Generates initial comment section continuation token from given video id
3485         """
3486         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3487         return base64.b64encode(token.encode()).decode()
3488
3489     def _get_comments(self, ytcfg, video_id, contents, webpage):
3490         """Entry for comment extraction"""
3491         def _real_comment_extract(contents):
3492             renderer = next((
3493                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3494                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3495             yield from self._comment_entries(renderer, ytcfg, video_id)
3496
3497         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3498         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3499
3500     @staticmethod
3501     def _get_checkok_params():
3502         return {'contentCheckOk': True, 'racyCheckOk': True}
3503
3504     @classmethod
3505     def _generate_player_context(cls, sts=None):
3506         context = {
3507             'html5Preference': 'HTML5_PREF_WANTS',
3508         }
3509         if sts is not None:
3510             context['signatureTimestamp'] = sts
3511         return {
3512             'playbackContext': {
3513                 'contentPlaybackContext': context
3514             },
3515             **cls._get_checkok_params()
3516         }
3517
3518     @staticmethod
3519     def _is_agegated(player_response):
3520         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3521             return True
3522
3523         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')))
3524         AGE_GATE_REASONS = (
3525             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3526             'age_verification_required', 'age_check_required',  # status
3527         )
3528         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3529
3530     @staticmethod
3531     def _is_unplayable(player_response):
3532         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3533
3534     _STORY_PLAYER_PARAMS = '8AEB'
3535
3536     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3537
3538         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3539         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3540         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3541         headers = self.generate_api_headers(
3542             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3543
3544         yt_query = {
3545             'videoId': video_id,
3546         }
3547         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3548             yt_query['params'] = self._STORY_PLAYER_PARAMS
3549
3550         yt_query.update(self._generate_player_context(sts))
3551         return self._extract_response(
3552             item_id=video_id, ep='player', query=yt_query,
3553             ytcfg=player_ytcfg, headers=headers, fatal=True,
3554             default_client=client,
3555             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3556         ) or None
3557
3558     def _get_requested_clients(self, url, smuggled_data):
3559         requested_clients = []
3560         default = ['android', 'web']
3561         allowed_clients = sorted(
3562             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3563             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3564         for client in self._configuration_arg('player_client'):
3565             if client in allowed_clients:
3566                 requested_clients.append(client)
3567             elif client == 'default':
3568                 requested_clients.extend(default)
3569             elif client == 'all':
3570                 requested_clients.extend(allowed_clients)
3571             else:
3572                 self.report_warning(f'Skipping unsupported client {client}')
3573         if not requested_clients:
3574             requested_clients = default
3575
3576         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3577             requested_clients.extend(
3578                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3579
3580         return orderedSet(requested_clients)
3581
3582     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3583         initial_pr = None
3584         if webpage:
3585             initial_pr = self._search_json(
3586                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3587
3588         all_clients = set(clients)
3589         clients = clients[::-1]
3590         prs = []
3591
3592         def append_client(*client_names):
3593             """ Append the first client name that exists but not already used """
3594             for client_name in client_names:
3595                 actual_client = _split_innertube_client(client_name)[0]
3596                 if actual_client in INNERTUBE_CLIENTS:
3597                     if actual_client not in all_clients:
3598                         clients.append(client_name)
3599                         all_clients.add(actual_client)
3600                         return
3601
3602         # Android player_response does not have microFormats which are needed for
3603         # extraction of some data. So we return the initial_pr with formats
3604         # stripped out even if not requested by the user
3605         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3606         if initial_pr:
3607             pr = dict(initial_pr)
3608             pr['streamingData'] = None
3609             prs.append(pr)
3610
3611         last_error = None
3612         tried_iframe_fallback = False
3613         player_url = None
3614         while clients:
3615             client, base_client, variant = _split_innertube_client(clients.pop())
3616             player_ytcfg = master_ytcfg if client == 'web' else {}
3617             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3618                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3619
3620             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3621             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3622             if 'js' in self._configuration_arg('player_skip'):
3623                 require_js_player = False
3624                 player_url = None
3625
3626             if not player_url and not tried_iframe_fallback and require_js_player:
3627                 player_url = self._download_player_url(video_id)
3628                 tried_iframe_fallback = True
3629
3630             try:
3631                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3632                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3633             except ExtractorError as e:
3634                 if last_error:
3635                     self.report_warning(last_error)
3636                 last_error = e
3637                 continue
3638
3639             if pr:
3640                 # YouTube may return a different video player response than expected.
3641                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3642                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3643                 if pr_video_id and pr_video_id != video_id:
3644                     self.report_warning(
3645                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3646                 else:
3647                     # Save client name for introspection later
3648                     name = short_client_name(client)
3649                     sd = traverse_obj(pr, ('streamingData', {dict})) or {}
3650                     sd[STREAMING_DATA_CLIENT_NAME] = name
3651                     for f in traverse_obj(sd, (('formats', 'adaptiveFormats'), ..., {dict})):
3652                         f[STREAMING_DATA_CLIENT_NAME] = name
3653                     prs.append(pr)
3654
3655             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3656             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3657                 append_client(f'{base_client}_creator')
3658             elif self._is_agegated(pr):
3659                 if variant == 'tv_embedded':
3660                     append_client(f'{base_client}_embedded')
3661                 elif not variant:
3662                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3663
3664         if last_error:
3665             if not len(prs):
3666                 raise last_error
3667             self.report_warning(last_error)
3668         return prs, player_url
3669
3670     def _needs_live_processing(self, live_status, duration):
3671         if (live_status == 'is_live' and self.get_param('live_from_start')
3672                 or live_status == 'post_live' and (duration or 0) > 4 * 3600):
3673             return live_status
3674
3675     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3676         CHUNK_SIZE = 10 << 20
3677         itags, stream_ids = collections.defaultdict(set), []
3678         itag_qualities, res_qualities = {}, {0: None}
3679         q = qualities([
3680             # Normally tiny is the smallest video-only formats. But
3681             # audio-only formats with unknown quality may get tagged as tiny
3682             'tiny',
3683             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3684             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3685         ])
3686         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
3687         all_formats = self._configuration_arg('include_duplicate_formats')
3688
3689         def build_fragments(f):
3690             return LazyList({
3691                 'url': update_url_query(f['url'], {
3692                     'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, f["filesize"])}'
3693                 })
3694             } for range_start in range(0, f['filesize'], CHUNK_SIZE))
3695
3696         for fmt in streaming_formats:
3697             if fmt.get('targetDurationSec'):
3698                 continue
3699
3700             itag = str_or_none(fmt.get('itag'))
3701             audio_track = fmt.get('audioTrack') or {}
3702             stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
3703             if not all_formats:
3704                 if stream_id in stream_ids:
3705                     continue
3706
3707             quality = fmt.get('quality')
3708             height = int_or_none(fmt.get('height'))
3709             if quality == 'tiny' or not quality:
3710                 quality = fmt.get('audioQuality', '').lower() or quality
3711             # The 3gp format (17) in android client has a quality of "small",
3712             # but is actually worse than other formats
3713             if itag == '17':
3714                 quality = 'tiny'
3715             if quality:
3716                 if itag:
3717                     itag_qualities[itag] = quality
3718                 if height:
3719                     res_qualities[height] = quality
3720             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3721             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3722             # number of fragment that would subsequently requested with (`&sq=N`)
3723             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3724                 continue
3725
3726             fmt_url = fmt.get('url')
3727             if not fmt_url:
3728                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3729                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3730                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3731                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3732                     continue
3733                 try:
3734                     fmt_url += '&%s=%s' % (
3735                         traverse_obj(sc, ('sp', -1)) or 'signature',
3736                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3737                     )
3738                 except ExtractorError as e:
3739                     self.report_warning('Signature extraction failed: Some formats may be missing',
3740                                         video_id=video_id, only_once=True)
3741                     self.write_debug(e, only_once=True)
3742                     continue
3743
3744             query = parse_qs(fmt_url)
3745             throttled = False
3746             if query.get('n'):
3747                 try:
3748                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3749                     fmt_url = update_url_query(fmt_url, {
3750                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3751                     })
3752                 except ExtractorError as e:
3753                     phantomjs_hint = ''
3754                     if isinstance(e, JSInterpreter.Exception):
3755                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3756                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3757                     if player_url:
3758                         self.report_warning(
3759                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3760                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3761                         self.write_debug(e, only_once=True)
3762                     else:
3763                         self.report_warning(
3764                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3765                             video_id=video_id, only_once=True)
3766                     throttled = True
3767
3768             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3769             language_preference = (
3770                 10 if audio_track.get('audioIsDefault') and 10
3771                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3772                 else -1)
3773             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3774             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3775             # Make sure to avoid false positives with small duration differences.
3776             # E.g. __2ABJjxzNo, ySuUZEjARPY
3777             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3778             if is_damaged:
3779                 self.report_warning(
3780                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3781
3782             client_name = fmt.get(STREAMING_DATA_CLIENT_NAME)
3783             dct = {
3784                 'asr': int_or_none(fmt.get('audioSampleRate')),
3785                 'filesize': int_or_none(fmt.get('contentLength')),
3786                 'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
3787                 'format_note': join_nonempty(
3788                     join_nonempty(audio_track.get('displayName'),
3789                                   language_preference > 0 and ' (default)', delim=''),
3790                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3791                     fmt.get('isDrc') and 'DRC',
3792                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3793                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3794                     throttled and 'THROTTLED', is_damaged and 'DAMAGED',
3795                     (self.get_param('verbose') or all_formats) and client_name,
3796                     delim=', '),
3797                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3798                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3799                 'fps': int_or_none(fmt.get('fps')) or None,
3800                 'audio_channels': fmt.get('audioChannels'),
3801                 'height': height,
3802                 'quality': q(quality) - bool(fmt.get('isDrc')) / 2,
3803                 'has_drm': bool(fmt.get('drmFamilies')),
3804                 'tbr': tbr,
3805                 'url': fmt_url,
3806                 'width': int_or_none(fmt.get('width')),
3807                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3808                                           'desc' if language_preference < -1 else '') or None,
3809                 'language_preference': language_preference,
3810                 # Strictly de-prioritize damaged and 3gp formats
3811                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3812             }
3813             mime_mobj = re.match(
3814                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3815             if mime_mobj:
3816                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3817                 dct.update(parse_codecs(mime_mobj.group(2)))
3818             if itag:
3819                 itags[itag].add(('https', dct.get('language')))
3820                 stream_ids.append(stream_id)
3821             single_stream = 'none' in (dct.get('acodec'), dct.get('vcodec'))
3822             if single_stream and dct.get('ext'):
3823                 dct['container'] = dct['ext'] + '_dash'
3824
3825             if all_formats and dct['filesize']:
3826                 yield {
3827                     **dct,
3828                     'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
3829                     'protocol': 'http_dash_segments',
3830                     'fragments': build_fragments(dct),
3831                 }
3832             dct['downloader_options'] = {'http_chunk_size': CHUNK_SIZE}
3833             yield dct
3834
3835         needs_live_processing = self._needs_live_processing(live_status, duration)
3836         skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
3837
3838         skip_manifests = set(self._configuration_arg('skip'))
3839         if (not self.get_param('youtube_include_hls_manifest', True)
3840                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3841                 or needs_live_processing and skip_bad_formats):
3842             skip_manifests.add('hls')
3843
3844         if not self.get_param('youtube_include_dash_manifest', True):
3845             skip_manifests.add('dash')
3846         if self._configuration_arg('include_live_dash'):
3847             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3848                                                 'Use include_incomplete_formats extractor argument instead')
3849         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3850             skip_manifests.add('dash')
3851
3852         def process_manifest_format(f, proto, client_name, itag):
3853             key = (proto, f.get('language'))
3854             if not all_formats and key in itags[itag]:
3855                 return False
3856             itags[itag].add(key)
3857
3858             if itag and all_formats:
3859                 f['format_id'] = f'{itag}-{proto}'
3860             elif any(p != proto for p, _ in itags[itag]):
3861                 f['format_id'] = f'{itag}-{proto}'
3862             elif itag:
3863                 f['format_id'] = itag
3864
3865             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3866             if f['quality'] == -1 and f.get('height'):
3867                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3868             if self.get_param('verbose'):
3869                 f['format_note'] = join_nonempty(f.get('format_note'), client_name, delim=', ')
3870             return True
3871
3872         subtitles = {}
3873         for sd in streaming_data:
3874             client_name = sd.get(STREAMING_DATA_CLIENT_NAME)
3875
3876             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3877             if hls_manifest_url:
3878                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3879                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3880                 subtitles = self._merge_subtitles(subs, subtitles)
3881                 for f in fmts:
3882                     if process_manifest_format(f, 'hls', client_name, self._search_regex(
3883                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3884                         yield f
3885
3886             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3887             if dash_manifest_url:
3888                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3889                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3890                 for f in formats:
3891                     if process_manifest_format(f, 'dash', client_name, f['format_id']):
3892                         f['filesize'] = int_or_none(self._search_regex(
3893                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3894                         if needs_live_processing:
3895                             f['is_from_start'] = True
3896
3897                         yield f
3898         yield subtitles
3899
3900     def _extract_storyboard(self, player_responses, duration):
3901         spec = get_first(
3902             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3903         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3904         if not base_url:
3905             return
3906         L = len(spec) - 1
3907         for i, args in enumerate(spec):
3908             args = args.split('#')
3909             counts = list(map(int_or_none, args[:5]))
3910             if len(args) != 8 or not all(counts):
3911                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3912                 continue
3913             width, height, frame_count, cols, rows = counts
3914             N, sigh = args[6:]
3915
3916             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3917             fragment_count = frame_count / (cols * rows)
3918             fragment_duration = duration / fragment_count
3919             yield {
3920                 'format_id': f'sb{i}',
3921                 'format_note': 'storyboard',
3922                 'ext': 'mhtml',
3923                 'protocol': 'mhtml',
3924                 'acodec': 'none',
3925                 'vcodec': 'none',
3926                 'url': url,
3927                 'width': width,
3928                 'height': height,
3929                 'fps': frame_count / duration,
3930                 'rows': rows,
3931                 'columns': cols,
3932                 'fragments': [{
3933                     'url': url.replace('$M', str(j)),
3934                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3935                 } for j in range(math.ceil(fragment_count))],
3936             }
3937
3938     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3939         webpage = None
3940         if 'webpage' not in self._configuration_arg('player_skip'):
3941             query = {'bpctr': '9999999999', 'has_verified': '1'}
3942             if smuggled_data.get('is_story'):
3943                 query['pp'] = self._STORY_PLAYER_PARAMS
3944             webpage = self._download_webpage(
3945                 webpage_url, video_id, fatal=False, query=query)
3946
3947         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3948
3949         player_responses, player_url = self._extract_player_responses(
3950             self._get_requested_clients(url, smuggled_data),
3951             video_id, webpage, master_ytcfg, smuggled_data)
3952
3953         return webpage, master_ytcfg, player_responses, player_url
3954
3955     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3956         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3957         is_live = get_first(video_details, 'isLive')
3958         if is_live is None:
3959             is_live = get_first(live_broadcast_details, 'isLiveNow')
3960         live_content = get_first(video_details, 'isLiveContent')
3961         is_upcoming = get_first(video_details, 'isUpcoming')
3962         post_live = get_first(video_details, 'isPostLiveDvr')
3963         live_status = ('post_live' if post_live
3964                        else 'is_live' if is_live
3965                        else 'is_upcoming' if is_upcoming
3966                        else 'was_live' if live_content
3967                        else 'not_live' if False in (is_live, live_content)
3968                        else None)
3969         streaming_data = traverse_obj(player_responses, (..., 'streamingData'))
3970         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
3971
3972         return live_broadcast_details, live_status, streaming_data, formats, subtitles
3973
3974     def _real_extract(self, url):
3975         url, smuggled_data = unsmuggle_url(url, {})
3976         video_id = self._match_id(url)
3977
3978         base_url = self.http_scheme() + '//www.youtube.com/'
3979         webpage_url = base_url + 'watch?v=' + video_id
3980
3981         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3982
3983         playability_statuses = traverse_obj(
3984             player_responses, (..., 'playabilityStatus'), expected_type=dict)
3985
3986         trailer_video_id = get_first(
3987             playability_statuses,
3988             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3989             expected_type=str)
3990         if trailer_video_id:
3991             return self.url_result(
3992                 trailer_video_id, self.ie_key(), trailer_video_id)
3993
3994         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3995                        if webpage else (lambda x: None))
3996
3997         video_details = traverse_obj(player_responses, (..., 'videoDetails'), expected_type=dict)
3998         microformats = traverse_obj(
3999             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
4000             expected_type=dict)
4001
4002         translated_title = self._get_text(microformats, (..., 'title'))
4003         video_title = (self._preferred_lang and translated_title
4004                        or get_first(video_details, 'title')  # primary
4005                        or translated_title
4006                        or search_meta(['og:title', 'twitter:title', 'title']))
4007         translated_description = self._get_text(microformats, (..., 'description'))
4008         original_description = get_first(video_details, 'shortDescription')
4009         video_description = (
4010             self._preferred_lang and translated_description
4011             # If original description is blank, it will be an empty string.
4012             # Do not prefer translated description in this case.
4013             or original_description if original_description is not None else translated_description)
4014
4015         multifeed_metadata_list = get_first(
4016             player_responses,
4017             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
4018             expected_type=str)
4019         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
4020             if self.get_param('noplaylist'):
4021                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
4022             else:
4023                 entries = []
4024                 feed_ids = []
4025                 for feed in multifeed_metadata_list.split(','):
4026                     # Unquote should take place before split on comma (,) since textual
4027                     # fields may contain comma as well (see
4028                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
4029                     feed_data = urllib.parse.parse_qs(
4030                         urllib.parse.unquote_plus(feed))
4031
4032                     def feed_entry(name):
4033                         return try_get(
4034                             feed_data, lambda x: x[name][0], str)
4035
4036                     feed_id = feed_entry('id')
4037                     if not feed_id:
4038                         continue
4039                     feed_title = feed_entry('title')
4040                     title = video_title
4041                     if feed_title:
4042                         title += ' (%s)' % feed_title
4043                     entries.append({
4044                         '_type': 'url_transparent',
4045                         'ie_key': 'Youtube',
4046                         'url': smuggle_url(
4047                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
4048                             {'force_singlefeed': True}),
4049                         'title': title,
4050                     })
4051                     feed_ids.append(feed_id)
4052                 self.to_screen(
4053                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
4054                     % (', '.join(feed_ids), video_id))
4055                 return self.playlist_result(
4056                     entries, video_id, video_title, video_description)
4057
4058         duration = (int_or_none(get_first(video_details, 'lengthSeconds'))
4059                     or int_or_none(get_first(microformats, 'lengthSeconds'))
4060                     or parse_duration(search_meta('duration')) or None)
4061
4062         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
4063             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
4064         if live_status == 'post_live':
4065             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
4066
4067         if not formats:
4068             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
4069                 self.report_drm(video_id)
4070             pemr = get_first(
4071                 playability_statuses,
4072                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
4073             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
4074             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
4075             if subreason:
4076                 if subreason == 'The uploader has not made this video available in your country.':
4077                     countries = get_first(microformats, 'availableCountries')
4078                     if not countries:
4079                         regions_allowed = search_meta('regionsAllowed')
4080                         countries = regions_allowed.split(',') if regions_allowed else None
4081                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
4082                 reason += f'. {subreason}'
4083             if reason:
4084                 self.raise_no_formats(reason, expected=True)
4085
4086         keywords = get_first(video_details, 'keywords', expected_type=list) or []
4087         if not keywords and webpage:
4088             keywords = [
4089                 unescapeHTML(m.group('content'))
4090                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
4091         for keyword in keywords:
4092             if keyword.startswith('yt:stretch='):
4093                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
4094                 if mobj:
4095                     # NB: float is intentional for forcing float division
4096                     w, h = (float(v) for v in mobj.groups())
4097                     if w > 0 and h > 0:
4098                         ratio = w / h
4099                         for f in formats:
4100                             if f.get('vcodec') != 'none':
4101                                 f['stretched_ratio'] = ratio
4102                         break
4103         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
4104         thumbnail_url = search_meta(['og:image', 'twitter:image'])
4105         if thumbnail_url:
4106             thumbnails.append({
4107                 'url': thumbnail_url,
4108             })
4109         original_thumbnails = thumbnails.copy()
4110
4111         # The best resolution thumbnails sometimes does not appear in the webpage
4112         # See: https://github.com/yt-dlp/yt-dlp/issues/340
4113         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
4114         thumbnail_names = [
4115             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
4116             # in resolution, these are not the custom thumbnail. So de-prioritize them
4117             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
4118             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
4119         ]
4120         n_thumbnail_names = len(thumbnail_names)
4121         thumbnails.extend({
4122             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
4123                 video_id=video_id, name=name, ext=ext,
4124                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
4125         } for name in thumbnail_names for ext in ('webp', 'jpg'))
4126         for thumb in thumbnails:
4127             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
4128             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
4129         self._remove_duplicate_formats(thumbnails)
4130         self._downloader._sort_thumbnails(original_thumbnails)
4131
4132         category = get_first(microformats, 'category') or search_meta('genre')
4133         channel_id = self.ucid_or_none(str_or_none(
4134             get_first(video_details, 'channelId')
4135             or get_first(microformats, 'externalChannelId')
4136             or search_meta('channelId')))
4137         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
4138
4139         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
4140         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
4141         if not duration and live_end_time and live_start_time:
4142             duration = live_end_time - live_start_time
4143
4144         needs_live_processing = self._needs_live_processing(live_status, duration)
4145
4146         def is_bad_format(fmt):
4147             if needs_live_processing and not fmt.get('is_from_start'):
4148                 return True
4149             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
4150                     and fmt.get('protocol') == 'http_dash_segments'):
4151                 return True
4152
4153         for fmt in filter(is_bad_format, formats):
4154             fmt['preference'] = (fmt.get('preference') or -1) - 10
4155             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
4156
4157         if needs_live_processing:
4158             self._prepare_live_from_start_formats(
4159                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
4160
4161         formats.extend(self._extract_storyboard(player_responses, duration))
4162
4163         channel_handle = self.handle_from_url(owner_profile_url)
4164
4165         info = {
4166             'id': video_id,
4167             'title': video_title,
4168             'formats': formats,
4169             'thumbnails': thumbnails,
4170             # The best thumbnail that we are sure exists. Prevents unnecessary
4171             # URL checking if user don't care about getting the best possible thumbnail
4172             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
4173             'description': video_description,
4174             'channel_id': channel_id,
4175             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None),
4176             'duration': duration,
4177             'view_count': int_or_none(
4178                 get_first((video_details, microformats), (..., 'viewCount'))
4179                 or search_meta('interactionCount')),
4180             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
4181             'age_limit': 18 if (
4182                 get_first(microformats, 'isFamilySafe') is False
4183                 or search_meta('isFamilyFriendly') == 'false'
4184                 or search_meta('og:restrictions:age') == '18+') else 0,
4185             'webpage_url': webpage_url,
4186             'categories': [category] if category else None,
4187             'tags': keywords,
4188             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
4189             'live_status': live_status,
4190             'release_timestamp': live_start_time,
4191             '_format_sort_fields': (  # source_preference is lower for throttled/potentially damaged formats
4192                 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto')
4193         }
4194
4195         subtitles = {}
4196         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
4197         if pctr:
4198             def get_lang_code(track):
4199                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
4200                         or track.get('languageCode'))
4201
4202             # Converted into dicts to remove duplicates
4203             captions = {
4204                 get_lang_code(sub): sub
4205                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...))}
4206             translation_languages = {
4207                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
4208                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...))}
4209
4210             def process_language(container, base_url, lang_code, sub_name, query):
4211                 lang_subs = container.setdefault(lang_code, [])
4212                 for fmt in self._SUBTITLE_FORMATS:
4213                     query.update({
4214                         'fmt': fmt,
4215                     })
4216                     lang_subs.append({
4217                         'ext': fmt,
4218                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
4219                         'name': sub_name,
4220                     })
4221
4222             # NB: Constructing the full subtitle dictionary is slow
4223             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
4224                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
4225             for lang_code, caption_track in captions.items():
4226                 base_url = caption_track.get('baseUrl')
4227                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
4228                 if not base_url:
4229                     continue
4230                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
4231                 if caption_track.get('kind') != 'asr':
4232                     if not lang_code:
4233                         continue
4234                     process_language(
4235                         subtitles, base_url, lang_code, lang_name, {})
4236                     if not caption_track.get('isTranslatable'):
4237                         continue
4238                 for trans_code, trans_name in translation_languages.items():
4239                     if not trans_code:
4240                         continue
4241                     orig_trans_code = trans_code
4242                     if caption_track.get('kind') != 'asr' and trans_code != 'und':
4243                         if not get_translated_subs:
4244                             continue
4245                         trans_code += f'-{lang_code}'
4246                         trans_name += format_field(lang_name, None, ' from %s')
4247                     # Add an "-orig" label to the original language so that it can be distinguished.
4248                     # The subs are returned without "-orig" as well for compatibility
4249                     if lang_code == f'a-{orig_trans_code}':
4250                         process_language(
4251                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
4252                     # Setting tlang=lang returns damaged subtitles.
4253                     process_language(automatic_captions, base_url, trans_code, trans_name,
4254                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
4255
4256         info['automatic_captions'] = automatic_captions
4257         info['subtitles'] = subtitles
4258
4259         parsed_url = urllib.parse.urlparse(url)
4260         for component in [parsed_url.fragment, parsed_url.query]:
4261             query = urllib.parse.parse_qs(component)
4262             for k, v in query.items():
4263                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
4264                     d_k += '_time'
4265                     if d_k not in info and k in s_ks:
4266                         info[d_k] = parse_duration(query[k][0])
4267
4268         # Youtube Music Auto-generated description
4269         if video_description:
4270             mobj = re.search(
4271                 r'''(?xs)
4272                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
4273                     (?P<album>[^\n]+)
4274                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4275                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4276                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
4277                     .+\nAuto-generated\ by\ YouTube\.\s*$
4278                 ''', video_description)
4279             if mobj:
4280                 release_year = mobj.group('release_year')
4281                 release_date = mobj.group('release_date')
4282                 if release_date:
4283                     release_date = release_date.replace('-', '')
4284                     if not release_year:
4285                         release_year = release_date[:4]
4286                 info.update({
4287                     'album': mobj.group('album'.strip()),
4288                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
4289                     'track': mobj.group('track').strip(),
4290                     'release_date': release_date,
4291                     'release_year': int_or_none(release_year),
4292                 })
4293
4294         initial_data = None
4295         if webpage:
4296             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4297             if not traverse_obj(initial_data, 'contents'):
4298                 self.report_warning('Incomplete data received in embedded initial data; re-fetching using API.')
4299                 initial_data = None
4300         if not initial_data:
4301             query = {'videoId': video_id}
4302             query.update(self._get_checkok_params())
4303             initial_data = self._extract_response(
4304                 item_id=video_id, ep='next', fatal=False,
4305                 ytcfg=master_ytcfg, query=query, check_get_keys='contents',
4306                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4307                 note='Downloading initial data API JSON')
4308
4309         info['comment_count'] = traverse_obj(initial_data, (
4310             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4311             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount'
4312         ), (
4313             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4314             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo'
4315         ), expected_type=self._get_count, get_all=False)
4316
4317         try:  # This will error if there is no livechat
4318             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4319         except (KeyError, IndexError, TypeError):
4320             pass
4321         else:
4322             info.setdefault('subtitles', {})['live_chat'] = [{
4323                 # url is needed to set cookies
4324                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4325                 'video_id': video_id,
4326                 'ext': 'json',
4327                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4328                              else 'youtube_live_chat_replay'),
4329             }]
4330
4331         if initial_data:
4332             info['chapters'] = (
4333                 self._extract_chapters_from_json(initial_data, duration)
4334                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4335                 or self._extract_chapters_from_description(video_description, duration)
4336                 or None)
4337
4338             info['heatmap'] = self._extract_heatmap_from_player_overlay(initial_data)
4339
4340         contents = traverse_obj(
4341             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4342             expected_type=list, default=[])
4343
4344         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4345         if vpir:
4346             stl = vpir.get('superTitleLink')
4347             if stl:
4348                 stl = self._get_text(stl)
4349                 if try_get(
4350                         vpir,
4351                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4352                     info['location'] = stl
4353                 else:
4354                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4355                     if mobj:
4356                         info.update({
4357                             'series': mobj.group(1),
4358                             'season_number': int(mobj.group(2)),
4359                             'episode_number': int(mobj.group(3)),
4360                         })
4361             for tlb in (try_get(
4362                     vpir,
4363                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4364                     list) or []):
4365                 tbrs = variadic(
4366                     traverse_obj(
4367                         tlb, ('toggleButtonRenderer', ...),
4368                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer')))
4369                 for tbr in tbrs:
4370                     for getter, regex in [(
4371                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4372                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4373                                 lambda x: x['accessibility'],
4374                                 lambda x: x['accessibilityData']['accessibilityData'],
4375                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4376                         label = (try_get(tbr, getter, dict) or {}).get('label')
4377                         if label:
4378                             mobj = re.match(regex, label)
4379                             if mobj:
4380                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4381                                 break
4382             sbr_tooltip = try_get(
4383                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4384             if sbr_tooltip:
4385                 like_count, dislike_count = sbr_tooltip.split(' / ')
4386                 info.update({
4387                     'like_count': str_to_int(like_count),
4388                     'dislike_count': str_to_int(dislike_count),
4389                 })
4390             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4391             if vcr:
4392                 vc = self._get_count(vcr, 'viewCount')
4393                 # Upcoming premieres with waiting count are treated as live here
4394                 if vcr.get('isLive'):
4395                     info['concurrent_view_count'] = vc
4396                 elif info.get('view_count') is None:
4397                     info['view_count'] = vc
4398
4399         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4400         if vsir:
4401             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4402             info.update({
4403                 'channel': self._get_text(vor, 'title'),
4404                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4405
4406             if not channel_handle:
4407                 channel_handle = self.handle_from_url(
4408                     traverse_obj(vor, (
4409                         ('navigationEndpoint', ('title', 'runs', ..., 'navigationEndpoint')),
4410                         (('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl')),
4411                         {str}), get_all=False))
4412
4413             rows = try_get(
4414                 vsir,
4415                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4416                 list) or []
4417             multiple_songs = False
4418             for row in rows:
4419                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4420                     multiple_songs = True
4421                     break
4422             for row in rows:
4423                 mrr = row.get('metadataRowRenderer') or {}
4424                 mrr_title = mrr.get('title')
4425                 if not mrr_title:
4426                     continue
4427                 mrr_title = self._get_text(mrr, 'title')
4428                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4429                 if mrr_title == 'License':
4430                     info['license'] = mrr_contents_text
4431                 elif not multiple_songs:
4432                     if mrr_title == 'Album':
4433                         info['album'] = mrr_contents_text
4434                     elif mrr_title == 'Artist':
4435                         info['artist'] = mrr_contents_text
4436                     elif mrr_title == 'Song':
4437                         info['track'] = mrr_contents_text
4438
4439         info.update({
4440             'uploader': info.get('channel'),
4441             'uploader_id': channel_handle,
4442             'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
4443         })
4444         # The upload date for scheduled, live and past live streams / premieres in microformats
4445         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4446         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4447         upload_date = (
4448             unified_strdate(get_first(microformats, 'uploadDate'))
4449             or unified_strdate(search_meta('uploadDate')))
4450         if not upload_date or (
4451             live_status in ('not_live', None)
4452             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4453         ):
4454             upload_date = strftime_or_none(
4455                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4456         info['upload_date'] = upload_date
4457
4458         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4459             v = info.get(s_k)
4460             if v:
4461                 info[d_k] = v
4462
4463         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4464
4465         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4466                       or get_first(video_details, 'isPrivate', expected_type=bool))
4467
4468         info['availability'] = (
4469             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4470             else self._availability(
4471                 is_private=is_private,
4472                 needs_premium=(
4473                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4474                     or False if initial_data and is_private is not None else None),
4475                 needs_subscription=(
4476                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4477                     or False if initial_data and is_private is not None else None),
4478                 needs_auth=info['age_limit'] >= 18,
4479                 is_unlisted=None if is_private is None else (
4480                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4481                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4482
4483         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4484
4485         self.mark_watched(video_id, player_responses)
4486
4487         return info
4488
4489
4490 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4491     @staticmethod
4492     def passthrough_smuggled_data(func):
4493         def _smuggle(info, smuggled_data):
4494             if info.get('_type') not in ('url', 'url_transparent'):
4495                 return info
4496             if smuggled_data.get('is_music_url'):
4497                 parsed_url = urllib.parse.urlparse(info['url'])
4498                 if parsed_url.netloc in ('www.youtube.com', 'music.youtube.com'):
4499                     smuggled_data.pop('is_music_url')
4500                     info['url'] = urllib.parse.urlunparse(parsed_url._replace(netloc='music.youtube.com'))
4501             if smuggled_data:
4502                 info['url'] = smuggle_url(info['url'], smuggled_data)
4503             return info
4504
4505         @functools.wraps(func)
4506         def wrapper(self, url):
4507             url, smuggled_data = unsmuggle_url(url, {})
4508             if self.is_music_url(url):
4509                 smuggled_data['is_music_url'] = True
4510             info_dict = func(self, url, smuggled_data)
4511             if smuggled_data:
4512                 _smuggle(info_dict, smuggled_data)
4513                 if info_dict.get('entries'):
4514                     info_dict['entries'] = (_smuggle(i, smuggled_data.copy()) for i in info_dict['entries'])
4515             return info_dict
4516         return wrapper
4517
4518     @staticmethod
4519     def _extract_basic_item_renderer(item):
4520         # Modified from _extract_grid_item_renderer
4521         known_basic_renderers = (
4522             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4523         )
4524         for key, renderer in item.items():
4525             if not isinstance(renderer, dict):
4526                 continue
4527             elif key in known_basic_renderers:
4528                 return renderer
4529             elif key.startswith('grid') and key.endswith('Renderer'):
4530                 return renderer
4531
4532     def _extract_channel_renderer(self, renderer):
4533         channel_id = self.ucid_or_none(renderer['channelId'])
4534         title = self._get_text(renderer, 'title')
4535         channel_url = format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None)
4536         # As of 2023-03-01 YouTube doesn't use the channel handles on these renderers yet.
4537         # However we can expect them to change that in the future.
4538         channel_handle = self.handle_from_url(
4539             traverse_obj(renderer, (
4540                 'navigationEndpoint', (('commandMetadata', 'webCommandMetadata', 'url'),
4541                                        ('browseEndpoint', 'canonicalBaseUrl')),
4542                 {str}), get_all=False))
4543         return {
4544             '_type': 'url',
4545             'url': channel_url,
4546             'id': channel_id,
4547             'ie_key': YoutubeTabIE.ie_key(),
4548             'channel': title,
4549             'uploader': title,
4550             'channel_id': channel_id,
4551             'channel_url': channel_url,
4552             'title': title,
4553             'uploader_id': channel_handle,
4554             'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
4555             'channel_follower_count': self._get_count(renderer, 'subscriberCountText'),
4556             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
4557             'playlist_count': self._get_count(renderer, 'videoCountText'),
4558             'description': self._get_text(renderer, 'descriptionSnippet'),
4559         }
4560
4561     def _grid_entries(self, grid_renderer):
4562         for item in grid_renderer['items']:
4563             if not isinstance(item, dict):
4564                 continue
4565             renderer = self._extract_basic_item_renderer(item)
4566             if not isinstance(renderer, dict):
4567                 continue
4568             title = self._get_text(renderer, 'title')
4569
4570             # playlist
4571             playlist_id = renderer.get('playlistId')
4572             if playlist_id:
4573                 yield self.url_result(
4574                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4575                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4576                     video_title=title)
4577                 continue
4578             # video
4579             video_id = renderer.get('videoId')
4580             if video_id:
4581                 yield self._extract_video(renderer)
4582                 continue
4583             # channel
4584             channel_id = renderer.get('channelId')
4585             if channel_id:
4586                 yield self._extract_channel_renderer(renderer)
4587                 continue
4588             # generic endpoint URL support
4589             ep_url = urljoin('https://www.youtube.com/', try_get(
4590                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4591                 str))
4592             if ep_url:
4593                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4594                     if ie.suitable(ep_url):
4595                         yield self.url_result(
4596                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4597                         break
4598
4599     def _music_reponsive_list_entry(self, renderer):
4600         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4601         if video_id:
4602             title = traverse_obj(renderer, (
4603                 'flexColumns', 0, 'musicResponsiveListItemFlexColumnRenderer',
4604                 'text', 'runs', 0, 'text'))
4605             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4606                                    ie=YoutubeIE.ie_key(), video_id=video_id, title=title)
4607         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4608         if playlist_id:
4609             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4610             if video_id:
4611                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4612                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4613             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4614                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4615         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4616         if browse_id:
4617             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4618                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4619
4620     def _shelf_entries_from_content(self, shelf_renderer):
4621         content = shelf_renderer.get('content')
4622         if not isinstance(content, dict):
4623             return
4624         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4625         if renderer:
4626             # TODO: add support for nested playlists so each shelf is processed
4627             # as separate playlist
4628             # TODO: this includes only first N items
4629             yield from self._grid_entries(renderer)
4630         renderer = content.get('horizontalListRenderer')
4631         if renderer:
4632             # TODO
4633             pass
4634
4635     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4636         ep = try_get(
4637             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4638             str)
4639         shelf_url = urljoin('https://www.youtube.com', ep)
4640         if shelf_url:
4641             # Skipping links to another channels, note that checking for
4642             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4643             # will not work
4644             if skip_channels and '/channels?' in shelf_url:
4645                 return
4646             title = self._get_text(shelf_renderer, 'title')
4647             yield self.url_result(shelf_url, video_title=title)
4648         # Shelf may not contain shelf URL, fallback to extraction from content
4649         yield from self._shelf_entries_from_content(shelf_renderer)
4650
4651     def _playlist_entries(self, video_list_renderer):
4652         for content in video_list_renderer['contents']:
4653             if not isinstance(content, dict):
4654                 continue
4655             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4656             if not isinstance(renderer, dict):
4657                 continue
4658             video_id = renderer.get('videoId')
4659             if not video_id:
4660                 continue
4661             yield self._extract_video(renderer)
4662
4663     def _rich_entries(self, rich_grid_renderer):
4664         renderer = traverse_obj(
4665             rich_grid_renderer,
4666             ('content', ('videoRenderer', 'reelItemRenderer', 'playlistRenderer')), get_all=False) or {}
4667         video_id = renderer.get('videoId')
4668         if video_id:
4669             yield self._extract_video(renderer)
4670             return
4671         playlist_id = renderer.get('playlistId')
4672         if playlist_id:
4673             yield self.url_result(
4674                 f'https://www.youtube.com/playlist?list={playlist_id}',
4675                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4676                 video_title=self._get_text(renderer, 'title'))
4677             return
4678
4679     def _video_entry(self, video_renderer):
4680         video_id = video_renderer.get('videoId')
4681         if video_id:
4682             return self._extract_video(video_renderer)
4683
4684     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4685         url = urljoin('https://youtube.com', traverse_obj(
4686             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4687         if url:
4688             return self.url_result(
4689                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4690
4691     def _post_thread_entries(self, post_thread_renderer):
4692         post_renderer = try_get(
4693             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4694         if not post_renderer:
4695             return
4696         # video attachment
4697         video_renderer = try_get(
4698             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4699         video_id = video_renderer.get('videoId')
4700         if video_id:
4701             entry = self._extract_video(video_renderer)
4702             if entry:
4703                 yield entry
4704         # playlist attachment
4705         playlist_id = try_get(
4706             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4707         if playlist_id:
4708             yield self.url_result(
4709                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4710                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4711         # inline video links
4712         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4713         for run in runs:
4714             if not isinstance(run, dict):
4715                 continue
4716             ep_url = try_get(
4717                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4718             if not ep_url:
4719                 continue
4720             if not YoutubeIE.suitable(ep_url):
4721                 continue
4722             ep_video_id = YoutubeIE._match_id(ep_url)
4723             if video_id == ep_video_id:
4724                 continue
4725             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4726
4727     def _post_thread_continuation_entries(self, post_thread_continuation):
4728         contents = post_thread_continuation.get('contents')
4729         if not isinstance(contents, list):
4730             return
4731         for content in contents:
4732             renderer = content.get('backstagePostThreadRenderer')
4733             if isinstance(renderer, dict):
4734                 yield from self._post_thread_entries(renderer)
4735                 continue
4736             renderer = content.get('videoRenderer')
4737             if isinstance(renderer, dict):
4738                 yield self._video_entry(renderer)
4739
4740     r''' # unused
4741     def _rich_grid_entries(self, contents):
4742         for content in contents:
4743             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4744             if video_renderer:
4745                 entry = self._video_entry(video_renderer)
4746                 if entry:
4747                     yield entry
4748     '''
4749
4750     def _report_history_entries(self, renderer):
4751         for url in traverse_obj(renderer, (
4752                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4753                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4754                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4755             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4756
4757     def _extract_entries(self, parent_renderer, continuation_list):
4758         # continuation_list is modified in-place with continuation_list = [continuation_token]
4759         continuation_list[:] = [None]
4760         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4761         for content in contents:
4762             if not isinstance(content, dict):
4763                 continue
4764             is_renderer = traverse_obj(
4765                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4766                 expected_type=dict)
4767             if not is_renderer:
4768                 if content.get('richItemRenderer'):
4769                     for entry in self._rich_entries(content['richItemRenderer']):
4770                         yield entry
4771                     continuation_list[0] = self._extract_continuation(parent_renderer)
4772                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4773                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4774                     yield from self._report_history_entries(table)
4775                     continuation_list[0] = self._extract_continuation(table)
4776                 continue
4777
4778             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4779             for isr_content in isr_contents:
4780                 if not isinstance(isr_content, dict):
4781                     continue
4782
4783                 known_renderers = {
4784                     'playlistVideoListRenderer': self._playlist_entries,
4785                     'gridRenderer': self._grid_entries,
4786                     'reelShelfRenderer': self._grid_entries,
4787                     'shelfRenderer': self._shelf_entries,
4788                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4789                     'backstagePostThreadRenderer': self._post_thread_entries,
4790                     'videoRenderer': lambda x: [self._video_entry(x)],
4791                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4792                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4793                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4794                 }
4795                 for key, renderer in isr_content.items():
4796                     if key not in known_renderers:
4797                         continue
4798                     for entry in known_renderers[key](renderer):
4799                         if entry:
4800                             yield entry
4801                     continuation_list[0] = self._extract_continuation(renderer)
4802                     break
4803
4804             if not continuation_list[0]:
4805                 continuation_list[0] = self._extract_continuation(is_renderer)
4806
4807         if not continuation_list[0]:
4808             continuation_list[0] = self._extract_continuation(parent_renderer)
4809
4810     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4811         continuation_list = [None]
4812         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4813         tab_content = try_get(tab, lambda x: x['content'], dict)
4814         if not tab_content:
4815             return
4816         parent_renderer = (
4817             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4818             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4819         yield from extract_entries(parent_renderer)
4820         continuation = continuation_list[0]
4821
4822         for page_num in itertools.count(1):
4823             if not continuation:
4824                 break
4825             headers = self.generate_api_headers(
4826                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4827             response = self._extract_response(
4828                 item_id=f'{item_id} page {page_num}',
4829                 query=continuation, headers=headers, ytcfg=ytcfg,
4830                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4831
4832             if not response:
4833                 break
4834             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4835             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4836             visitor_data = self._extract_visitor_data(response) or visitor_data
4837
4838             known_renderers = {
4839                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4840                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4841                 'gridVideoRenderer': (self._grid_entries, 'items'),
4842                 'gridChannelRenderer': (self._grid_entries, 'items'),
4843                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4844                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4845                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4846                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4847                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4848                 'playlistVideoListContinuation': (self._playlist_entries, None),
4849                 'gridContinuation': (self._grid_entries, None),
4850                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4851                 'sectionListContinuation': (extract_entries, None),  # for feeds
4852             }
4853
4854             continuation_items = traverse_obj(response, (
4855                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4856                 'appendContinuationItemsAction', 'continuationItems'
4857             ), 'continuationContents', get_all=False)
4858             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4859
4860             video_items_renderer = None
4861             for key in continuation_item.keys():
4862                 if key not in known_renderers:
4863                     continue
4864                 func, parent_key = known_renderers[key]
4865                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4866                 continuation_list = [None]
4867                 yield from func(video_items_renderer)
4868                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4869
4870             if not video_items_renderer:
4871                 break
4872
4873     @staticmethod
4874     def _extract_selected_tab(tabs, fatal=True):
4875         for tab_renderer in tabs:
4876             if tab_renderer.get('selected'):
4877                 return tab_renderer
4878         if fatal:
4879             raise ExtractorError('Unable to find selected tab')
4880
4881     @staticmethod
4882     def _extract_tab_renderers(response):
4883         return traverse_obj(
4884             response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
4885
4886     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4887         metadata = self._extract_metadata_from_tabs(item_id, data)
4888
4889         selected_tab = self._extract_selected_tab(tabs)
4890         metadata['title'] += format_field(selected_tab, 'title', ' - %s')
4891         metadata['title'] += format_field(selected_tab, 'expandedText', ' - %s')
4892
4893         return self.playlist_result(
4894             self._entries(
4895                 selected_tab, metadata['id'], ytcfg,
4896                 self._extract_account_syncid(ytcfg, data),
4897                 self._extract_visitor_data(data, ytcfg)),
4898             **metadata)
4899
4900     def _extract_metadata_from_tabs(self, item_id, data):
4901         info = {'id': item_id}
4902
4903         metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
4904         if metadata_renderer:
4905             channel_id = traverse_obj(metadata_renderer, ('externalId', {self.ucid_or_none}),
4906                                       ('channelUrl', {self.ucid_from_url}))
4907             info.update({
4908                 'channel': metadata_renderer.get('title'),
4909                 'channel_id': channel_id,
4910             })
4911             if info['channel_id']:
4912                 info['id'] = info['channel_id']
4913         else:
4914             metadata_renderer = traverse_obj(data, ('metadata', 'playlistMetadataRenderer'), expected_type=dict)
4915
4916         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4917         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4918         def _get_uncropped(url):
4919             return url_or_none((url or '').split('=')[0] + '=s0')
4920
4921         avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
4922         if avatar_thumbnails:
4923             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4924             if uncropped_avatar:
4925                 avatar_thumbnails.append({
4926                     'url': uncropped_avatar,
4927                     'id': 'avatar_uncropped',
4928                     'preference': 1
4929                 })
4930
4931         channel_banners = self._extract_thumbnails(
4932             data, ('header', ..., ('banner', 'mobileBanner', 'tvBanner')))
4933         for banner in channel_banners:
4934             banner['preference'] = -10
4935
4936         if channel_banners:
4937             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4938             if uncropped_banner:
4939                 channel_banners.append({
4940                     'url': uncropped_banner,
4941                     'id': 'banner_uncropped',
4942                     'preference': -5
4943                 })
4944
4945         # Deprecated - remove primary_sidebar_renderer when layout discontinued
4946         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4947         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
4948
4949         primary_thumbnails = self._extract_thumbnails(
4950             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4951         playlist_thumbnails = self._extract_thumbnails(
4952             playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
4953
4954         info.update({
4955             'title': (traverse_obj(metadata_renderer, 'title')
4956                       or self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag'))
4957                       or info['id']),
4958             'availability': self._extract_availability(data),
4959             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4960             'description': try_get(metadata_renderer, lambda x: x.get('description', '')),
4961             'tags': try_get(metadata_renderer or {}, lambda x: x.get('keywords', '').split()),
4962             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
4963         })
4964
4965         channel_handle = (
4966             traverse_obj(metadata_renderer, (('vanityChannelUrl', ('ownerUrls', ...)), {self.handle_from_url}), get_all=False)
4967             or traverse_obj(data, ('header', ..., 'channelHandleText', {self.handle_or_none}), get_all=False))
4968
4969         if channel_handle:
4970             info.update({
4971                 'uploader_id': channel_handle,
4972                 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
4973             })
4974         # Playlist stats is a text runs array containing [video count, view count, last updated].
4975         # last updated or (view count and last updated) may be missing.
4976         playlist_stats = get_first(
4977             (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'), ))
4978
4979         last_updated_unix = self._parse_time_text(
4980             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
4981             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
4982         info['modified_date'] = strftime_or_none(last_updated_unix, '%Y%m%d')
4983
4984         info['view_count'] = self._get_count(playlist_stats, 1)
4985         if info['view_count'] is None:  # 0 is allowed
4986             info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText')
4987         if info['view_count'] is None:
4988             info['view_count'] = self._get_count(data, (
4989                 'contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., 'tabRenderer', 'content', 'sectionListRenderer',
4990                 'contents', ..., 'itemSectionRenderer', 'contents', ..., 'channelAboutFullMetadataRenderer', 'viewCountText'))
4991
4992         info['playlist_count'] = self._get_count(playlist_stats, 0)
4993         if info['playlist_count'] is None:  # 0 is allowed
4994             info['playlist_count'] = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
4995
4996         if not info.get('channel_id'):
4997             owner = traverse_obj(playlist_header_renderer, 'ownerText')
4998             if not owner:  # Deprecated
4999                 owner = traverse_obj(
5000                     self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
5001                     ('videoOwner', 'videoOwnerRenderer', 'title'))
5002             owner_text = self._get_text(owner)
5003             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
5004             info.update({
5005                 'channel': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
5006                 'channel_id': self.ucid_or_none(browse_ep.get('browseId')),
5007                 'uploader_id': self.handle_from_url(urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl')))
5008             })
5009
5010         info.update({
5011             'uploader': info['channel'],
5012             'channel_url': format_field(info.get('channel_id'), None, 'https://www.youtube.com/channel/%s', default=None),
5013             'uploader_url': format_field(info.get('uploader_id'), None, 'https://www.youtube.com/%s', default=None),
5014         })
5015
5016         return info
5017
5018     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
5019         first_id = last_id = response = None
5020         for page_num in itertools.count(1):
5021             videos = list(self._playlist_entries(playlist))
5022             if not videos:
5023                 return
5024             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
5025             if start >= len(videos):
5026                 return
5027             yield from videos[start:]
5028             first_id = first_id or videos[0]['id']
5029             last_id = videos[-1]['id']
5030             watch_endpoint = try_get(
5031                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
5032             headers = self.generate_api_headers(
5033                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
5034                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
5035             query = {
5036                 'playlistId': playlist_id,
5037                 'videoId': watch_endpoint.get('videoId') or last_id,
5038                 'index': watch_endpoint.get('index') or len(videos),
5039                 'params': watch_endpoint.get('params') or 'OAE%3D'
5040             }
5041             response = self._extract_response(
5042                 item_id='%s page %d' % (playlist_id, page_num),
5043                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
5044                 check_get_keys='contents'
5045             )
5046             playlist = try_get(
5047                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
5048
5049     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
5050         title = playlist.get('title') or try_get(
5051             data, lambda x: x['titleText']['simpleText'], str)
5052         playlist_id = playlist.get('playlistId') or item_id
5053
5054         # Delegating everything except mix playlists to regular tab-based playlist URL
5055         playlist_url = urljoin(url, try_get(
5056             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
5057             str))
5058
5059         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
5060         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
5061         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
5062
5063         if playlist_url and playlist_url != url and not is_known_unviewable:
5064             return self.url_result(
5065                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
5066                 video_title=title)
5067
5068         return self.playlist_result(
5069             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
5070             playlist_id=playlist_id, playlist_title=title)
5071
5072     def _extract_availability(self, data):
5073         """
5074         Gets the availability of a given playlist/tab.
5075         Note: Unless YouTube tells us explicitly, we do not assume it is public
5076         @param data: response
5077         """
5078         sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
5079         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
5080         player_header_privacy = playlist_header_renderer.get('privacy')
5081
5082         badges = self._extract_badges(sidebar_renderer)
5083
5084         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
5085         privacy_setting_icon = get_first(
5086             (playlist_header_renderer, sidebar_renderer),
5087             ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
5088              lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
5089             expected_type=str)
5090
5091         microformats_is_unlisted = traverse_obj(
5092             data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
5093
5094         return (
5095             'public' if (
5096                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
5097                 or player_header_privacy == 'PUBLIC'
5098                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
5099             else self._availability(
5100                 is_private=(
5101                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
5102                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
5103                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
5104                 is_unlisted=(
5105                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
5106                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
5107                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
5108                     else microformats_is_unlisted if microformats_is_unlisted is not None else None),
5109                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
5110                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
5111                 needs_auth=False))
5112
5113     @staticmethod
5114     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
5115         sidebar_renderer = try_get(
5116             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
5117         for item in sidebar_renderer:
5118             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
5119             if renderer:
5120                 return renderer
5121
5122     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
5123         """
5124         Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
5125         """
5126         is_playlist = bool(traverse_obj(
5127             data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
5128         if not is_playlist:
5129             return
5130         headers = self.generate_api_headers(
5131             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
5132             visitor_data=self._extract_visitor_data(data, ytcfg))
5133         query = {
5134             'params': 'wgYCCAA=',
5135             'browseId': f'VL{item_id}'
5136         }
5137         return self._extract_response(
5138             item_id=item_id, headers=headers, query=query,
5139             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
5140             note='Redownloading playlist API JSON with unavailable videos')
5141
5142     @functools.cached_property
5143     def skip_webpage(self):
5144         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
5145
5146     def _extract_webpage(self, url, item_id, fatal=True):
5147         webpage, data = None, None
5148         for retry in self.RetryManager(fatal=fatal):
5149             try:
5150                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
5151                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
5152             except ExtractorError as e:
5153                 if isinstance(e.cause, network_exceptions):
5154                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
5155                         retry.error = e
5156                         continue
5157                 self._error_or_warning(e, fatal=fatal)
5158                 break
5159
5160             try:
5161                 self._extract_and_report_alerts(data)
5162             except ExtractorError as e:
5163                 self._error_or_warning(e, fatal=fatal)
5164                 break
5165
5166             # Sometimes youtube returns a webpage with incomplete ytInitialData
5167             # See: https://github.com/yt-dlp/yt-dlp/issues/116
5168             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
5169                 retry.error = ExtractorError('Incomplete yt initial data received')
5170                 continue
5171
5172         return webpage, data
5173
5174     def _report_playlist_authcheck(self, ytcfg, fatal=True):
5175         """Use if failed to extract ytcfg (and data) from initial webpage"""
5176         if not ytcfg and self.is_authenticated:
5177             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
5178             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
5179                 raise ExtractorError(
5180                     f'{msg}. If you are not downloading private content, or '
5181                     'your cookies are only for the first account and channel,'
5182                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
5183                     expected=True)
5184             self.report_warning(msg, only_once=True)
5185
5186     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
5187         data = None
5188         if not self.skip_webpage:
5189             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
5190             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
5191             # Reject webpage data if redirected to home page without explicitly requesting
5192             selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
5193             if (url != 'https://www.youtube.com/feed/recommended'
5194                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
5195                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
5196                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
5197                 if fatal:
5198                     raise ExtractorError(msg, expected=True)
5199                 self.report_warning(msg, only_once=True)
5200         if not data:
5201             self._report_playlist_authcheck(ytcfg, fatal=fatal)
5202             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
5203         return data, ytcfg
5204
5205     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
5206         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
5207         resolve_response = self._extract_response(
5208             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
5209             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
5210         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
5211         for ep_key, ep in endpoints.items():
5212             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
5213             if params:
5214                 return self._extract_response(
5215                     item_id=item_id, query=params, ep=ep, headers=headers,
5216                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
5217                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
5218         err_note = 'Failed to resolve url (does the playlist exist?)'
5219         if fatal:
5220             raise ExtractorError(err_note, expected=True)
5221         self.report_warning(err_note, item_id)
5222
5223     _SEARCH_PARAMS = None
5224
5225     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
5226         data = {'query': query}
5227         if params is NO_DEFAULT:
5228             params = self._SEARCH_PARAMS
5229         if params:
5230             data['params'] = params
5231
5232         content_keys = (
5233             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
5234             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
5235             # ytmusic search
5236             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
5237             ('continuationContents', ),
5238         )
5239         display_id = f'query "{query}"'
5240         check_get_keys = tuple({keys[0] for keys in content_keys})
5241         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
5242         self._report_playlist_authcheck(ytcfg, fatal=False)
5243
5244         continuation_list = [None]
5245         search = None
5246         for page_num in itertools.count(1):
5247             data.update(continuation_list[0] or {})
5248             headers = self.generate_api_headers(
5249                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
5250             search = self._extract_response(
5251                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
5252                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
5253             slr_contents = traverse_obj(search, *content_keys)
5254             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
5255             if not continuation_list[0]:
5256                 break
5257
5258
5259 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
5260     IE_DESC = 'YouTube Tabs'
5261     _VALID_URL = r'''(?x:
5262         https?://
5263             (?!consent\.)(?:\w+\.)?
5264             (?:
5265                 youtube(?:kids)?\.com|
5266                 %(invidious)s
5267             )/
5268             (?:
5269                 (?P<channel_type>channel|c|user|browse)/|
5270                 (?P<not_channel>
5271                     feed/|hashtag/|
5272                     (?:playlist|watch)\?.*?\blist=
5273                 )|
5274                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
5275             )
5276             (?P<id>[^/?\#&]+)
5277     )''' % {
5278         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
5279         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5280     }
5281     IE_NAME = 'youtube:tab'
5282
5283     _TESTS = [{
5284         'note': 'playlists, multipage',
5285         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
5286         'playlist_mincount': 94,
5287         'info_dict': {
5288             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5289             'title': 'Igor Kleiner - Playlists',
5290             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5291             'uploader': 'Igor Kleiner',
5292             'uploader_id': '@IgorDataScience',
5293             'uploader_url': 'https://www.youtube.com/@IgorDataScience',
5294             'channel': 'Igor Kleiner',
5295             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5296             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5297             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5298             'channel_follower_count': int
5299         },
5300     }, {
5301         'note': 'playlists, multipage, different order',
5302         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
5303         'playlist_mincount': 94,
5304         'info_dict': {
5305             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5306             'title': 'Igor Kleiner - Playlists',
5307             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5308             'uploader': 'Igor Kleiner',
5309             'uploader_id': '@IgorDataScience',
5310             'uploader_url': 'https://www.youtube.com/@IgorDataScience',
5311             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5312             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5313             'channel': 'Igor Kleiner',
5314             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5315             'channel_follower_count': int
5316         },
5317     }, {
5318         'note': 'playlists, series',
5319         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5320         'playlist_mincount': 5,
5321         'info_dict': {
5322             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5323             'title': '3Blue1Brown - Playlists',
5324             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5325             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5326             'channel': '3Blue1Brown',
5327             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5328             'uploader_id': '@3blue1brown',
5329             'uploader_url': 'https://www.youtube.com/@3blue1brown',
5330             'uploader': '3Blue1Brown',
5331             'tags': ['Mathematics'],
5332             'channel_follower_count': int
5333         },
5334     }, {
5335         'note': 'playlists, singlepage',
5336         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5337         'playlist_mincount': 4,
5338         'info_dict': {
5339             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5340             'title': 'ThirstForScience - Playlists',
5341             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5342             'uploader': 'ThirstForScience',
5343             'uploader_url': 'https://www.youtube.com/@ThirstForScience',
5344             'uploader_id': '@ThirstForScience',
5345             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5346             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5347             'tags': 'count:13',
5348             'channel': 'ThirstForScience',
5349             'channel_follower_count': int
5350         }
5351     }, {
5352         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5353         'only_matching': True,
5354     }, {
5355         'note': 'basic, single video playlist',
5356         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5357         'info_dict': {
5358             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5359             'title': 'youtube-dl public playlist',
5360             'description': '',
5361             'tags': [],
5362             'view_count': int,
5363             'modified_date': '20201130',
5364             'channel': 'Sergey M.',
5365             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5366             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5367             'availability': 'public',
5368             'uploader': 'Sergey M.',
5369             'uploader_url': 'https://www.youtube.com/@sergeym.6173',
5370             'uploader_id': '@sergeym.6173',
5371         },
5372         'playlist_count': 1,
5373     }, {
5374         'note': 'empty playlist',
5375         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5376         'info_dict': {
5377             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5378             'title': 'youtube-dl empty playlist',
5379             'tags': [],
5380             'channel': 'Sergey M.',
5381             'description': '',
5382             'modified_date': '20160902',
5383             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5384             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5385             'availability': 'public',
5386             'uploader_url': 'https://www.youtube.com/@sergeym.6173',
5387             'uploader_id': '@sergeym.6173',
5388             'uploader': 'Sergey M.',
5389         },
5390         'playlist_count': 0,
5391     }, {
5392         'note': 'Home tab',
5393         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5394         'info_dict': {
5395             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5396             'title': 'lex will - Home',
5397             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5398             'uploader': 'lex will',
5399             'uploader_id': '@lexwill718',
5400             'channel': 'lex will',
5401             'tags': ['bible', 'history', 'prophesy'],
5402             'uploader_url': 'https://www.youtube.com/@lexwill718',
5403             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5404             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5405             'channel_follower_count': int
5406         },
5407         'playlist_mincount': 2,
5408     }, {
5409         'note': 'Videos tab',
5410         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5411         'info_dict': {
5412             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5413             'title': 'lex will - Videos',
5414             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5415             'uploader': 'lex will',
5416             'uploader_id': '@lexwill718',
5417             'tags': ['bible', 'history', 'prophesy'],
5418             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5419             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5420             'uploader_url': 'https://www.youtube.com/@lexwill718',
5421             'channel': 'lex will',
5422             'channel_follower_count': int
5423         },
5424         'playlist_mincount': 975,
5425     }, {
5426         'note': 'Videos tab, sorted by popular',
5427         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5428         'info_dict': {
5429             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5430             'title': 'lex will - Videos',
5431             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5432             'uploader': 'lex will',
5433             'uploader_id': '@lexwill718',
5434             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5435             'uploader_url': 'https://www.youtube.com/@lexwill718',
5436             'channel': 'lex will',
5437             'tags': ['bible', 'history', 'prophesy'],
5438             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5439             'channel_follower_count': int
5440         },
5441         'playlist_mincount': 199,
5442     }, {
5443         'note': 'Playlists tab',
5444         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5445         'info_dict': {
5446             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5447             'title': 'lex will - Playlists',
5448             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5449             'uploader': 'lex will',
5450             'uploader_id': '@lexwill718',
5451             'uploader_url': 'https://www.youtube.com/@lexwill718',
5452             'channel': 'lex will',
5453             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5454             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5455             'tags': ['bible', 'history', 'prophesy'],
5456             'channel_follower_count': int
5457         },
5458         'playlist_mincount': 17,
5459     }, {
5460         'note': 'Community tab',
5461         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5462         'info_dict': {
5463             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5464             'title': 'lex will - Community',
5465             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5466             'channel': 'lex will',
5467             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5468             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5469             'tags': ['bible', 'history', 'prophesy'],
5470             'channel_follower_count': int,
5471             'uploader_url': 'https://www.youtube.com/@lexwill718',
5472             'uploader_id': '@lexwill718',
5473             'uploader': 'lex will',
5474         },
5475         'playlist_mincount': 18,
5476     }, {
5477         'note': 'Channels tab',
5478         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5479         'info_dict': {
5480             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5481             'title': 'lex will - Channels',
5482             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5483             'channel': 'lex will',
5484             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5485             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5486             'tags': ['bible', 'history', 'prophesy'],
5487             'channel_follower_count': int,
5488             'uploader_url': 'https://www.youtube.com/@lexwill718',
5489             'uploader_id': '@lexwill718',
5490             'uploader': 'lex will',
5491         },
5492         'playlist_mincount': 12,
5493     }, {
5494         'note': 'Search tab',
5495         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5496         'playlist_mincount': 40,
5497         'info_dict': {
5498             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5499             'title': '3Blue1Brown - Search - linear algebra',
5500             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5501             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5502             'tags': ['Mathematics'],
5503             'channel': '3Blue1Brown',
5504             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5505             'channel_follower_count': int,
5506             'uploader_url': 'https://www.youtube.com/@3blue1brown',
5507             'uploader_id': '@3blue1brown',
5508             'uploader': '3Blue1Brown',
5509         },
5510     }, {
5511         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5512         'only_matching': True,
5513     }, {
5514         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5515         'only_matching': True,
5516     }, {
5517         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5518         'only_matching': True,
5519     }, {
5520         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5521         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5522         'info_dict': {
5523             'title': '29C3: Not my department',
5524             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5525             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5526             'tags': [],
5527             'view_count': int,
5528             'modified_date': '20150605',
5529             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5530             'channel_url': 'https://www.youtube.com/channel/UCEPzS1rYsrkqzSLNp76nrcg',
5531             'channel': 'Christiaan008',
5532             'availability': 'public',
5533             'uploader_id': '@ChRiStIaAn008',
5534             'uploader': 'Christiaan008',
5535             'uploader_url': 'https://www.youtube.com/@ChRiStIaAn008',
5536         },
5537         'playlist_count': 96,
5538     }, {
5539         'note': 'Large playlist',
5540         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5541         'info_dict': {
5542             'title': 'Uploads from Cauchemar',
5543             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5544             'channel_url': 'https://www.youtube.com/channel/UCBABnxM4Ar9ten8Mdjj1j0Q',
5545             'tags': [],
5546             'modified_date': r're:\d{8}',
5547             'channel': 'Cauchemar',
5548             'view_count': int,
5549             'description': '',
5550             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5551             'availability': 'public',
5552             'uploader_id': '@Cauchemar89',
5553             'uploader': 'Cauchemar',
5554             'uploader_url': 'https://www.youtube.com/@Cauchemar89',
5555         },
5556         'playlist_mincount': 1123,
5557         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5558     }, {
5559         'note': 'even larger playlist, 8832 videos',
5560         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5561         'only_matching': True,
5562     }, {
5563         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5564         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5565         'info_dict': {
5566             'title': 'Uploads from Interstellar Movie',
5567             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5568             'tags': [],
5569             'view_count': int,
5570             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5571             'channel_url': 'https://www.youtube.com/channel/UCXw-G3eDE9trcvY2sBMM_aA',
5572             'channel': 'Interstellar Movie',
5573             'description': '',
5574             'modified_date': r're:\d{8}',
5575             'availability': 'public',
5576             'uploader_id': '@InterstellarMovie',
5577             'uploader': 'Interstellar Movie',
5578             'uploader_url': 'https://www.youtube.com/@InterstellarMovie',
5579         },
5580         'playlist_mincount': 21,
5581     }, {
5582         'note': 'Playlist with "show unavailable videos" button',
5583         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5584         'info_dict': {
5585             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5586             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5587             'view_count': int,
5588             'channel': 'Phim Siêu Nhân Nhật Bản',
5589             'tags': [],
5590             'description': '',
5591             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5592             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5593             'modified_date': r're:\d{8}',
5594             'availability': 'public',
5595             'uploader_url': 'https://www.youtube.com/@phimsieunhannhatban',
5596             'uploader_id': '@phimsieunhannhatban',
5597             'uploader': 'Phim Siêu Nhân Nhật Bản',
5598         },
5599         'playlist_mincount': 200,
5600         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5601     }, {
5602         'note': 'Playlist with unavailable videos in page 7',
5603         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5604         'info_dict': {
5605             'title': 'Uploads from BlankTV',
5606             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5607             'channel': 'BlankTV',
5608             'channel_url': 'https://www.youtube.com/channel/UC8l9frL61Yl5KFOl87nIm2w',
5609             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5610             'view_count': int,
5611             'tags': [],
5612             'modified_date': r're:\d{8}',
5613             'description': '',
5614             'availability': 'public',
5615             'uploader_id': '@blanktv',
5616             'uploader': 'BlankTV',
5617             'uploader_url': 'https://www.youtube.com/@blanktv',
5618         },
5619         'playlist_mincount': 1000,
5620         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5621     }, {
5622         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5623         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5624         'info_dict': {
5625             'title': 'Data Analysis with Dr Mike Pound',
5626             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5627             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5628             'tags': [],
5629             'view_count': int,
5630             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5631             'channel_url': 'https://www.youtube.com/channel/UC9-y-6csu5WGm29I7JiwpnA',
5632             'channel': 'Computerphile',
5633             'availability': 'public',
5634             'modified_date': '20190712',
5635             'uploader_id': '@Computerphile',
5636             'uploader': 'Computerphile',
5637             'uploader_url': 'https://www.youtube.com/@Computerphile',
5638         },
5639         'playlist_mincount': 11,
5640     }, {
5641         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5642         'only_matching': True,
5643     }, {
5644         'note': 'Playlist URL that does not actually serve a playlist',
5645         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5646         'info_dict': {
5647             'id': 'FqZTN594JQw',
5648             'ext': 'webm',
5649             'title': "Smiley's People 01 detective, Adventure Series, Action",
5650             'upload_date': '20150526',
5651             'license': 'Standard YouTube License',
5652             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5653             'categories': ['People & Blogs'],
5654             'tags': list,
5655             'view_count': int,
5656             'like_count': int,
5657         },
5658         'params': {
5659             'skip_download': True,
5660         },
5661         'skip': 'This video is not available.',
5662         'add_ie': [YoutubeIE.ie_key()],
5663     }, {
5664         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5665         'only_matching': True,
5666     }, {
5667         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5668         'only_matching': True,
5669     }, {
5670         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5671         'info_dict': {
5672             'id': 'AlTsmyW4auo',  # This will keep changing
5673             'ext': 'mp4',
5674             'title': str,
5675             'upload_date': r're:\d{8}',
5676             'description': str,
5677             'categories': ['News & Politics'],
5678             'tags': list,
5679             'like_count': int,
5680             'release_timestamp': int,
5681             'channel': 'Sky News',
5682             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5683             'age_limit': 0,
5684             'view_count': int,
5685             'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
5686             'playable_in_embed': True,
5687             'release_date': r're:\d+',
5688             'availability': 'public',
5689             'live_status': 'is_live',
5690             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5691             'channel_follower_count': int,
5692             'concurrent_view_count': int,
5693             'uploader_url': 'https://www.youtube.com/@SkyNews',
5694             'uploader_id': '@SkyNews',
5695             'uploader': 'Sky News',
5696         },
5697         'params': {
5698             'skip_download': True,
5699         },
5700         'expected_warnings': ['Ignoring subtitle tracks found in '],
5701     }, {
5702         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5703         'info_dict': {
5704             'id': 'a48o2S1cPoo',
5705             'ext': 'mp4',
5706             'title': 'The Young Turks - Live Main Show',
5707             'upload_date': '20150715',
5708             'license': 'Standard YouTube License',
5709             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5710             'categories': ['News & Politics'],
5711             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5712             'like_count': int,
5713         },
5714         'params': {
5715             'skip_download': True,
5716         },
5717         'only_matching': True,
5718     }, {
5719         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5720         'only_matching': True,
5721     }, {
5722         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5723         'only_matching': True,
5724     }, {
5725         'note': 'A channel that is not live. Should raise error',
5726         'url': 'https://www.youtube.com/user/numberphile/live',
5727         'only_matching': True,
5728     }, {
5729         'url': 'https://www.youtube.com/feed/trending',
5730         'only_matching': True,
5731     }, {
5732         'url': 'https://www.youtube.com/feed/library',
5733         'only_matching': True,
5734     }, {
5735         'url': 'https://www.youtube.com/feed/history',
5736         'only_matching': True,
5737     }, {
5738         'url': 'https://www.youtube.com/feed/subscriptions',
5739         'only_matching': True,
5740     }, {
5741         'url': 'https://www.youtube.com/feed/watch_later',
5742         'only_matching': True,
5743     }, {
5744         'note': 'Recommended - redirects to home page.',
5745         'url': 'https://www.youtube.com/feed/recommended',
5746         'only_matching': True,
5747     }, {
5748         'note': 'inline playlist with not always working continuations',
5749         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5750         'only_matching': True,
5751     }, {
5752         'url': 'https://www.youtube.com/course',
5753         'only_matching': True,
5754     }, {
5755         'url': 'https://www.youtube.com/zsecurity',
5756         'only_matching': True,
5757     }, {
5758         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5759         'only_matching': True,
5760     }, {
5761         'url': 'https://www.youtube.com/TheYoungTurks/live',
5762         'only_matching': True,
5763     }, {
5764         'url': 'https://www.youtube.com/hashtag/cctv9',
5765         'info_dict': {
5766             'id': 'cctv9',
5767             'title': '#cctv9',
5768             'tags': [],
5769         },
5770         'playlist_mincount': 300,  # not consistent but should be over 300
5771     }, {
5772         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5773         'only_matching': True,
5774     }, {
5775         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5776         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5777         'only_matching': True
5778     }, {
5779         'note': '/browse/ should redirect to /channel/',
5780         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5781         'only_matching': True
5782     }, {
5783         'note': 'VLPL, should redirect to playlist?list=PL...',
5784         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5785         'info_dict': {
5786             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5787             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5788             'title': 'NCS : All Releases 💿',
5789             'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
5790             'modified_date': r're:\d{8}',
5791             'view_count': int,
5792             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5793             'tags': [],
5794             'channel': 'NoCopyrightSounds',
5795             'availability': 'public',
5796             'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
5797             'uploader': 'NoCopyrightSounds',
5798             'uploader_id': '@NoCopyrightSounds',
5799         },
5800         'playlist_mincount': 166,
5801         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden', 'YouTube Music is not directly supported'],
5802     }, {
5803         # TODO: fix 'unviewable' issue with this playlist when reloading with unavailable videos
5804         'note': 'Topic, should redirect to playlist?list=UU...',
5805         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5806         'info_dict': {
5807             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5808             'title': 'Uploads from Royalty Free Music - Topic',
5809             'tags': [],
5810             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5811             'channel': 'Royalty Free Music - Topic',
5812             'view_count': int,
5813             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5814             'modified_date': r're:\d{8}',
5815             'description': '',
5816             'availability': 'public',
5817             'uploader': 'Royalty Free Music - Topic',
5818         },
5819         'playlist_mincount': 101,
5820         'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
5821     }, {
5822         # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
5823         # Treat as a general feed
5824         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5825         'info_dict': {
5826             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5827             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5828             'tags': [],
5829         },
5830         'playlist_mincount': 9,
5831     }, {
5832         'note': 'Youtube music Album',
5833         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5834         'info_dict': {
5835             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5836             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5837             'tags': [],
5838             'view_count': int,
5839             'description': '',
5840             'availability': 'unlisted',
5841             'modified_date': r're:\d{8}',
5842         },
5843         'playlist_count': 50,
5844         'expected_warnings': ['YouTube Music is not directly supported'],
5845     }, {
5846         'note': 'unlisted single video playlist',
5847         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5848         'info_dict': {
5849             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5850             'title': 'yt-dlp unlisted playlist test',
5851             'availability': 'unlisted',
5852             'tags': [],
5853             'modified_date': '20220418',
5854             'channel': 'colethedj',
5855             'view_count': int,
5856             'description': '',
5857             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5858             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5859             'uploader_url': 'https://www.youtube.com/@colethedj1894',
5860             'uploader_id': '@colethedj1894',
5861             'uploader': 'colethedj',
5862         },
5863         'playlist': [{
5864             'info_dict': {
5865                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
5866                 'id': 'BaW_jenozKc',
5867                 '_type': 'url',
5868                 'ie_key': 'Youtube',
5869                 'duration': 10,
5870                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
5871                 'channel_url': 'https://www.youtube.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
5872                 'view_count': int,
5873                 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc',
5874                 'channel': 'Philipp Hagemeister',
5875                 'uploader_id': '@PhilippHagemeister',
5876                 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
5877                 'uploader': 'Philipp Hagemeister',
5878             }
5879         }],
5880         'playlist_count': 1,
5881         'params': {'extract_flat': True},
5882     }, {
5883         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5884         'url': 'https://www.youtube.com/feed/recommended',
5885         'info_dict': {
5886             'id': 'recommended',
5887             'title': 'recommended',
5888             'tags': [],
5889         },
5890         'playlist_mincount': 50,
5891         'params': {
5892             'skip_download': True,
5893             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5894         },
5895     }, {
5896         'note': 'API Fallback: /videos tab, sorted by oldest first',
5897         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5898         'info_dict': {
5899             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5900             'title': 'Cody\'sLab - Videos',
5901             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5902             'channel': 'Cody\'sLab',
5903             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5904             'tags': [],
5905             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5906             'channel_follower_count': int
5907         },
5908         'playlist_mincount': 650,
5909         'params': {
5910             'skip_download': True,
5911             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5912         },
5913         'skip': 'Query for sorting no longer works',
5914     }, {
5915         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5916         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5917         'info_dict': {
5918             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5919             'title': 'Uploads from Royalty Free Music - Topic',
5920             'modified_date': r're:\d{8}',
5921             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5922             'description': '',
5923             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5924             'tags': [],
5925             'channel': 'Royalty Free Music - Topic',
5926             'view_count': int,
5927             'availability': 'public',
5928             'uploader': 'Royalty Free Music - Topic',
5929         },
5930         'playlist_mincount': 101,
5931         'params': {
5932             'skip_download': True,
5933             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5934         },
5935         'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
5936     }, {
5937         'note': 'non-standard redirect to regional channel',
5938         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5939         'only_matching': True
5940     }, {
5941         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5942         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5943         'info_dict': {
5944             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5945             'modified_date': '20220407',
5946             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5947             'tags': [],
5948             'availability': 'unlisted',
5949             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5950             'channel': 'pukkandan',
5951             'description': 'Test for collaborative playlist',
5952             'title': 'yt-dlp test - collaborative playlist',
5953             'view_count': int,
5954             'uploader_url': 'https://www.youtube.com/@pukkandan',
5955             'uploader_id': '@pukkandan',
5956             'uploader': 'pukkandan',
5957         },
5958         'playlist_mincount': 2
5959     }, {
5960         'note': 'translated tab name',
5961         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5962         'info_dict': {
5963             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5964             'tags': [],
5965             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5966             'description': 'test description',
5967             'title': 'cole-dlp-test-acc - 再生リスト',
5968             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5969             'channel': 'cole-dlp-test-acc',
5970             'uploader_url': 'https://www.youtube.com/@coletdjnz',
5971             'uploader_id': '@coletdjnz',
5972             'uploader': 'cole-dlp-test-acc',
5973         },
5974         'playlist_mincount': 1,
5975         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5976         'expected_warnings': ['Preferring "ja"'],
5977     }, {
5978         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5979         'note': 'preferred lang set with playlist with translated video titles',
5980         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5981         'info_dict': {
5982             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5983             'tags': [],
5984             'view_count': int,
5985             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5986             'channel': 'cole-dlp-test-acc',
5987             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5988             'description': 'test',
5989             'title': 'dlp test playlist',
5990             'availability': 'public',
5991             'uploader_url': 'https://www.youtube.com/@coletdjnz',
5992             'uploader_id': '@coletdjnz',
5993             'uploader': 'cole-dlp-test-acc',
5994         },
5995         'playlist_mincount': 1,
5996         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5997         'expected_warnings': ['Preferring "ja"'],
5998     }, {
5999         # shorts audio pivot for 2GtVksBMYFM.
6000         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
6001         'info_dict': {
6002             'id': 'sfv_audio_pivot',
6003             'title': 'sfv_audio_pivot',
6004             'tags': [],
6005         },
6006         'playlist_mincount': 50,
6007
6008     }, {
6009         # Channel with a real live tab (not to be mistaken with streams tab)
6010         # Do not treat like it should redirect to live stream
6011         'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
6012         'info_dict': {
6013             'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
6014             'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
6015             'tags': [],
6016         },
6017         'playlist_mincount': 20,
6018     }, {
6019         # Tab name is not the same as tab id
6020         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
6021         'info_dict': {
6022             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6023             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
6024             'tags': [],
6025         },
6026         'playlist_mincount': 8,
6027     }, {
6028         # Home tab id is literally home. Not to get mistaken with featured
6029         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
6030         'info_dict': {
6031             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6032             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
6033             'tags': [],
6034         },
6035         'playlist_mincount': 8,
6036     }, {
6037         # Should get three playlists for videos, shorts and streams tabs
6038         'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
6039         'info_dict': {
6040             'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
6041             'title': 'Polka Ch. 尾丸ポルカ',
6042             'channel_follower_count': int,
6043             'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
6044             'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
6045             'description': 'md5:e56b74b5bb7e9c701522162e9abfb822',
6046             'channel': 'Polka Ch. 尾丸ポルカ',
6047             'tags': 'count:35',
6048             'uploader_url': 'https://www.youtube.com/@OmaruPolka',
6049             'uploader': 'Polka Ch. 尾丸ポルカ',
6050             'uploader_id': '@OmaruPolka',
6051         },
6052         'playlist_count': 3,
6053     }, {
6054         # Shorts tab with channel with handle
6055         # TODO: fix channel description
6056         'url': 'https://www.youtube.com/@NotJustBikes/shorts',
6057         'info_dict': {
6058             'id': 'UC0intLFzLaudFG-xAvUEO-A',
6059             'title': 'Not Just Bikes - Shorts',
6060             'tags': 'count:12',
6061             'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
6062             'description': 'md5:26bc55af26855a608a5cf89dfa595c8d',
6063             'channel_follower_count': int,
6064             'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
6065             'channel': 'Not Just Bikes',
6066             'uploader_url': 'https://www.youtube.com/@NotJustBikes',
6067             'uploader': 'Not Just Bikes',
6068             'uploader_id': '@NotJustBikes',
6069         },
6070         'playlist_mincount': 10,
6071     }, {
6072         # Streams tab
6073         'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
6074         'info_dict': {
6075             'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
6076             'title': '中村悠一 - Live',
6077             'tags': 'count:7',
6078             'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
6079             'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
6080             'channel': '中村悠一',
6081             'channel_follower_count': int,
6082             'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
6083             'uploader_url': 'https://www.youtube.com/@Yuichi-Nakamura',
6084             'uploader_id': '@Yuichi-Nakamura',
6085             'uploader': '中村悠一',
6086         },
6087         'playlist_mincount': 60,
6088     }, {
6089         # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
6090         # See test_youtube_lists
6091         'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
6092         'only_matching': True,
6093     }, {
6094         # No uploads and no UCID given. Should fail with no uploads error
6095         # See test_youtube_lists
6096         'url': 'https://www.youtube.com/news',
6097         'only_matching': True
6098     }, {
6099         # No videos tab but has a shorts tab
6100         'url': 'https://www.youtube.com/c/TKFShorts',
6101         'info_dict': {
6102             'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6103             'title': 'Shorts Break - Shorts',
6104             'tags': 'count:48',
6105             'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6106             'channel': 'Shorts Break',
6107             'description': 'md5:6de33c5e7ba686e5f3efd4e19c7ef499',
6108             'channel_follower_count': int,
6109             'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
6110             'uploader_url': 'https://www.youtube.com/@ShortsBreak_Official',
6111             'uploader': 'Shorts Break',
6112             'uploader_id': '@ShortsBreak_Official',
6113         },
6114         'playlist_mincount': 30,
6115     }, {
6116         # Trending Now Tab. tab id is empty
6117         'url': 'https://www.youtube.com/feed/trending',
6118         'info_dict': {
6119             'id': 'trending',
6120             'title': 'trending - Now',
6121             'tags': [],
6122         },
6123         'playlist_mincount': 30,
6124     }, {
6125         # Trending Gaming Tab. tab id is empty
6126         'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
6127         'info_dict': {
6128             'id': 'trending',
6129             'title': 'trending - Gaming',
6130             'tags': [],
6131         },
6132         'playlist_mincount': 30,
6133     }, {
6134         # Shorts url result in shorts tab
6135         # TODO: Fix channel id extraction
6136         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
6137         'info_dict': {
6138             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6139             'title': 'cole-dlp-test-acc - Shorts',
6140             'channel': 'cole-dlp-test-acc',
6141             'description': 'test description',
6142             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6143             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6144             'tags': [],
6145             'uploader_url': 'https://www.youtube.com/@coletdjnz',
6146             'uploader_id': '@coletdjnz',
6147             'uploader': 'cole-dlp-test-acc',
6148         },
6149         'playlist': [{
6150             'info_dict': {
6151                 # Channel data is not currently available for short renderers (as of 2023-03-01)
6152                 '_type': 'url',
6153                 'ie_key': 'Youtube',
6154                 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
6155                 'id': 'sSM9J5YH_60',
6156                 'title': 'SHORT short',
6157                 'view_count': int,
6158                 'thumbnails': list,
6159             }
6160         }],
6161         'params': {'extract_flat': True},
6162     }, {
6163         # Live video status should be extracted
6164         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
6165         'info_dict': {
6166             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6167             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live',  # TODO, should be Minecraft - Live or Minecraft - Topic - Live
6168             'tags': []
6169         },
6170         'playlist': [{
6171             'info_dict': {
6172                 '_type': 'url',
6173                 'ie_key': 'Youtube',
6174                 'url': 'startswith:https://www.youtube.com/watch?v=',
6175                 'id': str,
6176                 'title': str,
6177                 'live_status': 'is_live',
6178                 'channel_id': str,
6179                 'channel_url': str,
6180                 'concurrent_view_count': int,
6181                 'channel': str,
6182                 'uploader': str,
6183                 'uploader_url': str,
6184                 'uploader_id': str
6185             }
6186         }],
6187         'params': {'extract_flat': True, 'playlist_items': '1'},
6188         'playlist_mincount': 1
6189     }, {
6190         # Channel renderer metadata. Contains number of videos on the channel
6191         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
6192         'info_dict': {
6193             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6194             'title': 'cole-dlp-test-acc - Channels',
6195             'channel': 'cole-dlp-test-acc',
6196             'description': 'test description',
6197             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6198             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6199             'tags': [],
6200             'uploader_url': 'https://www.youtube.com/@coletdjnz',
6201             'uploader_id': '@coletdjnz',
6202             'uploader': 'cole-dlp-test-acc',
6203         },
6204         'playlist': [{
6205             'info_dict': {
6206                 '_type': 'url',
6207                 'ie_key': 'YoutubeTab',
6208                 'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6209                 'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6210                 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6211                 'title': 'PewDiePie',
6212                 'channel': 'PewDiePie',
6213                 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6214                 'thumbnails': list,
6215                 'channel_follower_count': int,
6216                 'playlist_count': int,
6217                 'uploader': 'PewDiePie',
6218                 'uploader_url': 'https://www.youtube.com/@PewDiePie',
6219                 'uploader_id': '@PewDiePie',
6220             }
6221         }],
6222         'params': {'extract_flat': True},
6223     }, {
6224         'url': 'https://www.youtube.com/@3blue1brown/about',
6225         'info_dict': {
6226             'id': 'UCYO_jab_esuFRV4b17AJtAw',
6227             'tags': ['Mathematics'],
6228             'title': '3Blue1Brown - About',
6229             'channel_follower_count': int,
6230             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
6231             'channel': '3Blue1Brown',
6232             'view_count': int,
6233             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
6234             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
6235             'uploader_url': 'https://www.youtube.com/@3blue1brown',
6236             'uploader_id': '@3blue1brown',
6237             'uploader': '3Blue1Brown',
6238         },
6239         'playlist_count': 0,
6240     }, {
6241         # Podcasts tab, with rich entry playlistRenderers
6242         'url': 'https://www.youtube.com/@99percentinvisiblepodcast/podcasts',
6243         'info_dict': {
6244             'id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
6245             'channel_id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
6246             'uploader_url': 'https://www.youtube.com/@99percentinvisiblepodcast',
6247             'description': 'md5:3a0ed38f1ad42a68ef0428c04a15695c',
6248             'title': '99 Percent Invisible - Podcasts',
6249             'uploader': '99 Percent Invisible',
6250             'channel_follower_count': int,
6251             'channel_url': 'https://www.youtube.com/channel/UCVMF2HD4ZgC0QHpU9Yq5Xrw',
6252             'tags': [],
6253             'channel': '99 Percent Invisible',
6254             'uploader_id': '@99percentinvisiblepodcast',
6255         },
6256         'playlist_count': 1,
6257     }, {
6258         # Releases tab, with rich entry playlistRenderers (same as Podcasts tab)
6259         'url': 'https://www.youtube.com/@AHimitsu/releases',
6260         'info_dict': {
6261             'id': 'UCgFwu-j5-xNJml2FtTrrB3A',
6262             'channel': 'A Himitsu',
6263             'uploader_url': 'https://www.youtube.com/@AHimitsu',
6264             'title': 'A Himitsu - Releases',
6265             'uploader_id': '@AHimitsu',
6266             'uploader': 'A Himitsu',
6267             'channel_id': 'UCgFwu-j5-xNJml2FtTrrB3A',
6268             'tags': 'count:16',
6269             'description': 'I make music',
6270             'channel_url': 'https://www.youtube.com/channel/UCgFwu-j5-xNJml2FtTrrB3A',
6271             'channel_follower_count': int,
6272         },
6273         'playlist_mincount': 10,
6274     }]
6275
6276     @classmethod
6277     def suitable(cls, url):
6278         return False if YoutubeIE.suitable(url) else super().suitable(url)
6279
6280     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
6281
6282     def _get_url_mobj(self, url):
6283         mobj = self._URL_RE.match(url).groupdict()
6284         mobj.update((k, '') for k, v in mobj.items() if v is None)
6285         return mobj
6286
6287     def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
6288         tab_name = (tab.get('title') or '').lower()
6289         tab_url = urljoin(base_url, traverse_obj(
6290             tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
6291
6292         tab_id = (tab_url and self._get_url_mobj(tab_url)['tab'][1:]
6293                   or traverse_obj(tab, 'tabIdentifier', expected_type=str))
6294         if tab_id:
6295             return {
6296                 'TAB_ID_SPONSORSHIPS': 'membership',
6297             }.get(tab_id, tab_id), tab_name
6298
6299         # Fallback to tab name if we cannot get the tab id.
6300         # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
6301         # Note that in the case of translated tab name this may result in an empty string, which we don't want.
6302         if tab_name:
6303             self.write_debug(f'Falling back to selected tab name: {tab_name}')
6304         return {
6305             'home': 'featured',
6306             'live': 'streams',
6307         }.get(tab_name, tab_name), tab_name
6308
6309     def _has_tab(self, tabs, tab_id):
6310         return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
6311
6312     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
6313     def _real_extract(self, url, smuggled_data):
6314         item_id = self._match_id(url)
6315         url = urllib.parse.urlunparse(
6316             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
6317         compat_opts = self.get_param('compat_opts', [])
6318
6319         mobj = self._get_url_mobj(url)
6320         pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
6321         if is_channel and smuggled_data.get('is_music_url'):
6322             if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
6323                 return self.url_result(
6324                     f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
6325             elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
6326                 mdata = self._extract_tab_endpoint(
6327                     f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
6328                 murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
6329                                     get_all=False, expected_type=str)
6330                 if not murl:
6331                     raise ExtractorError('Failed to resolve album to playlist')
6332                 return self.url_result(murl, YoutubeTabIE)
6333             elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
6334                 return self.url_result(
6335                     f'https://music.youtube.com/channel/{item_id}{tab}{post}', YoutubeTabIE, item_id)
6336
6337         original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
6338         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
6339             url = f'{pre}/videos{post}'
6340         if smuggled_data.get('is_music_url'):
6341             self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}')
6342
6343         # Handle both video/playlist URLs
6344         qs = parse_qs(url)
6345         video_id, playlist_id = [traverse_obj(qs, (key, 0)) for key in ('v', 'list')]
6346         if not video_id and mobj['not_channel'].startswith('watch'):
6347             if not playlist_id:
6348                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
6349                 raise ExtractorError('A video URL was given without video ID', expected=True)
6350             # Common mistake: https://www.youtube.com/watch?list=playlist_id
6351             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
6352             return self.url_result(
6353                 f'https://www.youtube.com/playlist?list={playlist_id}', YoutubeTabIE, playlist_id)
6354
6355         if not self._yes_playlist(playlist_id, video_id):
6356             return self.url_result(
6357                 f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6358
6359         data, ytcfg = self._extract_data(url, display_id)
6360
6361         # YouTube may provide a non-standard redirect to the regional channel
6362         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
6363         # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
6364         redirect_url = traverse_obj(
6365             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
6366         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
6367             redirect_url = ''.join((urljoin('https://www.youtube.com', redirect_url), tab, post))
6368             self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
6369             return self.url_result(redirect_url, YoutubeTabIE)
6370
6371         tabs, extra_tabs = self._extract_tab_renderers(data), []
6372         if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
6373             selected_tab = self._extract_selected_tab(tabs)
6374             selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url)  # NB: Name may be translated
6375             self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
6376
6377             if not original_tab_id and selected_tab_name:
6378                 self.to_screen('Downloading all uploads of the channel. '
6379                                'To download only the videos in a specific tab, pass the tab\'s URL')
6380                 if self._has_tab(tabs, 'streams'):
6381                     extra_tabs.append(''.join((pre, '/streams', post)))
6382                 if self._has_tab(tabs, 'shorts'):
6383                     extra_tabs.append(''.join((pre, '/shorts', post)))
6384                 # XXX: Members-only tab should also be extracted
6385
6386                 if not extra_tabs and selected_tab_id != 'videos':
6387                     # Channel does not have streams, shorts or videos tabs
6388                     if item_id[:2] != 'UC':
6389                         raise ExtractorError('This channel has no uploads', expected=True)
6390
6391                     # Topic channels don't have /videos. Use the equivalent playlist instead
6392                     pl_id = f'UU{item_id[2:]}'
6393                     pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
6394                     try:
6395                         data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
6396                     except ExtractorError:
6397                         raise ExtractorError('This channel has no uploads', expected=True)
6398                     else:
6399                         item_id, url = pl_id, pl_url
6400                         self.to_screen(
6401                             f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
6402
6403                 elif extra_tabs and selected_tab_id != 'videos':
6404                     # When there are shorts/live tabs but not videos tab
6405                     url, data = f'{pre}{post}', None
6406
6407             elif (original_tab_id or 'videos') != selected_tab_id:
6408                 if original_tab_id == 'live':
6409                     # Live tab should have redirected to the video
6410                     # Except in the case the channel has an actual live tab
6411                     # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
6412                     raise UserNotLive(video_id=item_id)
6413                 elif selected_tab_name:
6414                     raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
6415
6416                 # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
6417                 url = f'{pre}{post}'
6418
6419         # YouTube sometimes provides a button to reload playlist with unavailable videos.
6420         if 'no-youtube-unavailable-videos' not in compat_opts:
6421             data = self._reload_with_unavailable_videos(display_id, data, ytcfg) or data
6422         self._extract_and_report_alerts(data, only_once=True)
6423
6424         tabs, entries = self._extract_tab_renderers(data), []
6425         if tabs:
6426             entries = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
6427             entries[0].update({
6428                 'extractor_key': YoutubeTabIE.ie_key(),
6429                 'extractor': YoutubeTabIE.IE_NAME,
6430                 'webpage_url': url,
6431             })
6432         if self.get_param('playlist_items') == '0':
6433             entries.extend(self.url_result(u, YoutubeTabIE) for u in extra_tabs)
6434         else:  # Users expect to get all `video_id`s even with `--flat-playlist`. So don't return `url_result`
6435             entries.extend(map(self._real_extract, extra_tabs))
6436
6437         if len(entries) == 1:
6438             return entries[0]
6439         elif entries:
6440             metadata = self._extract_metadata_from_tabs(item_id, data)
6441             uploads_url = 'the Uploads (UU) playlist URL'
6442             if try_get(metadata, lambda x: x['channel_id'].startswith('UC')):
6443                 uploads_url = f'https://www.youtube.com/playlist?list=UU{metadata["channel_id"][2:]}'
6444             self.to_screen(
6445                 'Downloading as multiple playlists, separated by tabs. '
6446                 f'To download as a single playlist instead, pass {uploads_url}')
6447             return self.playlist_result(entries, item_id, **metadata)
6448
6449         # Inline playlist
6450         playlist = traverse_obj(
6451             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
6452         if playlist:
6453             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
6454
6455         video_id = traverse_obj(
6456             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
6457         if video_id:
6458             if tab != '/live':  # live tab is expected to redirect to video
6459                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
6460             return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6461
6462         raise ExtractorError('Unable to recognize tab page')
6463
6464
6465 class YoutubePlaylistIE(InfoExtractor):
6466     IE_DESC = 'YouTube playlists'
6467     _VALID_URL = r'''(?x)(?:
6468                         (?:https?://)?
6469                         (?:\w+\.)?
6470                         (?:
6471                             (?:
6472                                 youtube(?:kids)?\.com|
6473                                 %(invidious)s
6474                             )
6475                             /.*?\?.*?\blist=
6476                         )?
6477                         (?P<id>%(playlist_id)s)
6478                      )''' % {
6479         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
6480         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
6481     }
6482     IE_NAME = 'youtube:playlist'
6483     _TESTS = [{
6484         'note': 'issue #673',
6485         'url': 'PLBB231211A4F62143',
6486         'info_dict': {
6487             'title': '[OLD]Team Fortress 2 (Class-based LP)',
6488             'id': 'PLBB231211A4F62143',
6489             'uploader': 'Wickman',
6490             'uploader_id': '@WickmanVT',
6491             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
6492             'view_count': int,
6493             'uploader_url': 'https://www.youtube.com/@WickmanVT',
6494             'modified_date': r're:\d{8}',
6495             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6496             'channel': 'Wickman',
6497             'tags': [],
6498             'channel_url': 'https://www.youtube.com/channel/UCKSpbfbl5kRQpTdL7kMc-1Q',
6499             'availability': 'public',
6500         },
6501         'playlist_mincount': 29,
6502     }, {
6503         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6504         'info_dict': {
6505             'title': 'YDL_safe_search',
6506             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6507         },
6508         'playlist_count': 2,
6509         'skip': 'This playlist is private',
6510     }, {
6511         'note': 'embedded',
6512         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6513         'playlist_count': 4,
6514         'info_dict': {
6515             'title': 'JODA15',
6516             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6517             'uploader': 'milan',
6518             'uploader_id': '@milan5503',
6519             'description': '',
6520             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6521             'tags': [],
6522             'modified_date': '20140919',
6523             'view_count': int,
6524             'channel': 'milan',
6525             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6526             'uploader_url': 'https://www.youtube.com/@milan5503',
6527             'availability': 'public',
6528         },
6529         'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'],
6530     }, {
6531         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6532         'playlist_mincount': 455,
6533         'info_dict': {
6534             'title': '2018 Chinese New Singles (11/6 updated)',
6535             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6536             'uploader': 'LBK',
6537             'uploader_id': '@music_king',
6538             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
6539             'channel': 'LBK',
6540             'view_count': int,
6541             'channel_url': 'https://www.youtube.com/channel/UC21nz3_MesPLqtDqwdvnoxA',
6542             'tags': [],
6543             'uploader_url': 'https://www.youtube.com/@music_king',
6544             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6545             'modified_date': r're:\d{8}',
6546             'availability': 'public',
6547         },
6548         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
6549     }, {
6550         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
6551         'only_matching': True,
6552     }, {
6553         # music album playlist
6554         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
6555         'only_matching': True,
6556     }]
6557
6558     @classmethod
6559     def suitable(cls, url):
6560         if YoutubeTabIE.suitable(url):
6561             return False
6562         from ..utils import parse_qs
6563         qs = parse_qs(url)
6564         if qs.get('v', [None])[0]:
6565             return False
6566         return super().suitable(url)
6567
6568     def _real_extract(self, url):
6569         playlist_id = self._match_id(url)
6570         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
6571         url = update_url_query(
6572             'https://www.youtube.com/playlist',
6573             parse_qs(url) or {'list': playlist_id})
6574         if is_music_url:
6575             url = smuggle_url(url, {'is_music_url': True})
6576         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6577
6578
6579 class YoutubeYtBeIE(InfoExtractor):
6580     IE_DESC = 'youtu.be'
6581     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
6582     _TESTS = [{
6583         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
6584         'info_dict': {
6585             'id': 'yeWKywCrFtk',
6586             'ext': 'mp4',
6587             'title': 'Small Scale Baler and Braiding Rugs',
6588             'uploader': 'Backus-Page House Museum',
6589             'uploader_id': '@backuspagemuseum',
6590             'uploader_url': r're:https?://(?:www\.)?youtube\.com/@backuspagemuseum',
6591             'upload_date': '20161008',
6592             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
6593             'categories': ['Nonprofits & Activism'],
6594             'tags': list,
6595             'like_count': int,
6596             'age_limit': 0,
6597             'playable_in_embed': True,
6598             'thumbnail': r're:^https?://.*\.webp',
6599             'channel': 'Backus-Page House Museum',
6600             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
6601             'live_status': 'not_live',
6602             'view_count': int,
6603             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
6604             'availability': 'public',
6605             'duration': 59,
6606             'comment_count': int,
6607             'channel_follower_count': int
6608         },
6609         'params': {
6610             'noplaylist': True,
6611             'skip_download': True,
6612         },
6613     }, {
6614         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
6615         'only_matching': True,
6616     }]
6617
6618     def _real_extract(self, url):
6619         mobj = self._match_valid_url(url)
6620         video_id = mobj.group('id')
6621         playlist_id = mobj.group('playlist_id')
6622         return self.url_result(
6623             update_url_query('https://www.youtube.com/watch', {
6624                 'v': video_id,
6625                 'list': playlist_id,
6626                 'feature': 'youtu.be',
6627             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6628
6629
6630 class YoutubeLivestreamEmbedIE(InfoExtractor):
6631     IE_DESC = 'YouTube livestream embeds'
6632     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6633     _TESTS = [{
6634         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6635         'only_matching': True,
6636     }]
6637
6638     def _real_extract(self, url):
6639         channel_id = self._match_id(url)
6640         return self.url_result(
6641             f'https://www.youtube.com/channel/{channel_id}/live',
6642             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6643
6644
6645 class YoutubeYtUserIE(InfoExtractor):
6646     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6647     IE_NAME = 'youtube:user'
6648     _VALID_URL = r'ytuser:(?P<id>.+)'
6649     _TESTS = [{
6650         'url': 'ytuser:phihag',
6651         'only_matching': True,
6652     }]
6653
6654     def _real_extract(self, url):
6655         user_id = self._match_id(url)
6656         return self.url_result(f'https://www.youtube.com/user/{user_id}', YoutubeTabIE, user_id)
6657
6658
6659 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6660     IE_NAME = 'youtube:favorites'
6661     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6662     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6663     _LOGIN_REQUIRED = True
6664     _TESTS = [{
6665         'url': ':ytfav',
6666         'only_matching': True,
6667     }, {
6668         'url': ':ytfavorites',
6669         'only_matching': True,
6670     }]
6671
6672     def _real_extract(self, url):
6673         return self.url_result(
6674             'https://www.youtube.com/playlist?list=LL',
6675             ie=YoutubeTabIE.ie_key())
6676
6677
6678 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6679     IE_NAME = 'youtube:notif'
6680     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6681     _VALID_URL = r':ytnotif(?:ication)?s?'
6682     _LOGIN_REQUIRED = True
6683     _TESTS = [{
6684         'url': ':ytnotif',
6685         'only_matching': True,
6686     }, {
6687         'url': ':ytnotifications',
6688         'only_matching': True,
6689     }]
6690
6691     def _extract_notification_menu(self, response, continuation_list):
6692         notification_list = traverse_obj(
6693             response,
6694             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6695             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6696             expected_type=list) or []
6697         continuation_list[0] = None
6698         for item in notification_list:
6699             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6700             if entry:
6701                 yield entry
6702             continuation = item.get('continuationItemRenderer')
6703             if continuation:
6704                 continuation_list[0] = continuation
6705
6706     def _extract_notification_renderer(self, notification):
6707         video_id = traverse_obj(
6708             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6709         url = f'https://www.youtube.com/watch?v={video_id}'
6710         channel_id = None
6711         if not video_id:
6712             browse_ep = traverse_obj(
6713                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6714             channel_id = self.ucid_or_none(traverse_obj(browse_ep, 'browseId', expected_type=str))
6715             post_id = self._search_regex(
6716                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6717                 'post id', default=None)
6718             if not channel_id or not post_id:
6719                 return
6720             # The direct /post url redirects to this in the browser
6721             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6722
6723         channel = traverse_obj(
6724             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6725             expected_type=str)
6726         notification_title = self._get_text(notification, 'shortMessage')
6727         if notification_title:
6728             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6729         # TODO: handle recommended videos
6730         title = self._search_regex(
6731             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6732             'video title', default=None)
6733         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6734                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6735                      else None)
6736         return {
6737             '_type': 'url',
6738             'url': url,
6739             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6740             'video_id': video_id,
6741             'title': title,
6742             'channel_id': channel_id,
6743             'channel': channel,
6744             'uploader': channel,
6745             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6746             'timestamp': timestamp,
6747         }
6748
6749     def _notification_menu_entries(self, ytcfg):
6750         continuation_list = [None]
6751         response = None
6752         for page in itertools.count(1):
6753             ctoken = traverse_obj(
6754                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6755             response = self._extract_response(
6756                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6757                 ep='notification/get_notification_menu', check_get_keys='actions',
6758                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6759             yield from self._extract_notification_menu(response, continuation_list)
6760             if not continuation_list[0]:
6761                 break
6762
6763     def _real_extract(self, url):
6764         display_id = 'notifications'
6765         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6766         self._report_playlist_authcheck(ytcfg)
6767         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6768
6769
6770 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6771     IE_DESC = 'YouTube search'
6772     IE_NAME = 'youtube:search'
6773     _SEARCH_KEY = 'ytsearch'
6774     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6775     _TESTS = [{
6776         'url': 'ytsearch5:youtube-dl test video',
6777         'playlist_count': 5,
6778         'info_dict': {
6779             'id': 'youtube-dl test video',
6780             'title': 'youtube-dl test video',
6781         }
6782     }]
6783
6784
6785 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6786     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6787     _SEARCH_KEY = 'ytsearchdate'
6788     IE_DESC = 'YouTube search, newest videos first'
6789     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6790     _TESTS = [{
6791         'url': 'ytsearchdate5:youtube-dl test video',
6792         'playlist_count': 5,
6793         'info_dict': {
6794             'id': 'youtube-dl test video',
6795             'title': 'youtube-dl test video',
6796         }
6797     }]
6798
6799
6800 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6801     IE_DESC = 'YouTube search URLs with sorting and filter support'
6802     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6803     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6804     _TESTS = [{
6805         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6806         'playlist_mincount': 5,
6807         'info_dict': {
6808             'id': 'youtube-dl test video',
6809             'title': 'youtube-dl test video',
6810         }
6811     }, {
6812         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6813         'playlist_mincount': 5,
6814         'info_dict': {
6815             'id': 'python',
6816             'title': 'python',
6817         }
6818     }, {
6819         'url': 'https://www.youtube.com/results?search_query=%23cats',
6820         'playlist_mincount': 1,
6821         'info_dict': {
6822             'id': '#cats',
6823             'title': '#cats',
6824             # The test suite does not have support for nested playlists
6825             # 'entries': [{
6826             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6827             #     'title': '#cats',
6828             # }],
6829         },
6830     }, {
6831         # Channel results
6832         'url': 'https://www.youtube.com/results?search_query=kurzgesagt&sp=EgIQAg%253D%253D',
6833         'info_dict': {
6834             'id': 'kurzgesagt',
6835             'title': 'kurzgesagt',
6836         },
6837         'playlist': [{
6838             'info_dict': {
6839                 '_type': 'url',
6840                 'id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6841                 'url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6842                 'ie_key': 'YoutubeTab',
6843                 'channel': 'Kurzgesagt – In a Nutshell',
6844                 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
6845                 'title': 'Kurzgesagt – In a Nutshell',
6846                 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6847                 'playlist_count': int,  # XXX: should have a way of saying > 1
6848                 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6849                 'thumbnails': list,
6850                 'uploader_id': '@kurzgesagt',
6851                 'uploader_url': 'https://www.youtube.com/@kurzgesagt',
6852                 'uploader': 'Kurzgesagt – In a Nutshell',
6853             }
6854         }],
6855         'params': {'extract_flat': True, 'playlist_items': '1'},
6856         'playlist_mincount': 1,
6857     }, {
6858         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6859         'only_matching': True,
6860     }]
6861
6862     def _real_extract(self, url):
6863         qs = parse_qs(url)
6864         query = (qs.get('search_query') or qs.get('q'))[0]
6865         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6866
6867
6868 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6869     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6870     IE_NAME = 'youtube:music:search_url'
6871     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6872     _TESTS = [{
6873         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6874         'playlist_count': 16,
6875         'info_dict': {
6876             'id': 'royalty free music',
6877             'title': 'royalty free music',
6878         }
6879     }, {
6880         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6881         'playlist_mincount': 30,
6882         'info_dict': {
6883             'id': 'royalty free music - songs',
6884             'title': 'royalty free music - songs',
6885         },
6886         'params': {'extract_flat': 'in_playlist'}
6887     }, {
6888         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6889         'playlist_mincount': 30,
6890         'info_dict': {
6891             'id': 'royalty free music - community playlists',
6892             'title': 'royalty free music - community playlists',
6893         },
6894         'params': {'extract_flat': 'in_playlist'}
6895     }]
6896
6897     _SECTIONS = {
6898         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6899         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6900         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6901         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6902         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6903         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6904     }
6905
6906     def _real_extract(self, url):
6907         qs = parse_qs(url)
6908         query = (qs.get('search_query') or qs.get('q'))[0]
6909         params = qs.get('sp', (None,))[0]
6910         if params:
6911             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6912         else:
6913             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6914             params = self._SECTIONS.get(section)
6915             if not params:
6916                 section = None
6917         title = join_nonempty(query, section, delim=' - ')
6918         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6919
6920
6921 class YoutubeFeedsInfoExtractor(InfoExtractor):
6922     """
6923     Base class for feed extractors
6924     Subclasses must re-define the _FEED_NAME property.
6925     """
6926     _LOGIN_REQUIRED = True
6927     _FEED_NAME = 'feeds'
6928
6929     def _real_initialize(self):
6930         YoutubeBaseInfoExtractor._check_login_required(self)
6931
6932     @classproperty
6933     def IE_NAME(self):
6934         return f'youtube:{self._FEED_NAME}'
6935
6936     def _real_extract(self, url):
6937         return self.url_result(
6938             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6939
6940
6941 class YoutubeWatchLaterIE(InfoExtractor):
6942     IE_NAME = 'youtube:watchlater'
6943     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6944     _VALID_URL = r':ytwatchlater'
6945     _TESTS = [{
6946         'url': ':ytwatchlater',
6947         'only_matching': True,
6948     }]
6949
6950     def _real_extract(self, url):
6951         return self.url_result(
6952             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6953
6954
6955 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6956     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6957     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6958     _FEED_NAME = 'recommended'
6959     _LOGIN_REQUIRED = False
6960     _TESTS = [{
6961         'url': ':ytrec',
6962         'only_matching': True,
6963     }, {
6964         'url': ':ytrecommended',
6965         'only_matching': True,
6966     }, {
6967         'url': 'https://youtube.com',
6968         'only_matching': True,
6969     }]
6970
6971
6972 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6973     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6974     _VALID_URL = r':ytsub(?:scription)?s?'
6975     _FEED_NAME = 'subscriptions'
6976     _TESTS = [{
6977         'url': ':ytsubs',
6978         'only_matching': True,
6979     }, {
6980         'url': ':ytsubscriptions',
6981         'only_matching': True,
6982     }]
6983
6984
6985 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6986     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6987     _VALID_URL = r':ythis(?:tory)?'
6988     _FEED_NAME = 'history'
6989     _TESTS = [{
6990         'url': ':ythistory',
6991         'only_matching': True,
6992     }]
6993
6994
6995 class YoutubeStoriesIE(InfoExtractor):
6996     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6997     IE_NAME = 'youtube:stories'
6998     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6999     _TESTS = [{
7000         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
7001         'only_matching': True,
7002     }]
7003
7004     def _real_extract(self, url):
7005         playlist_id = f'RLTD{self._match_id(url)}'
7006         return self.url_result(
7007             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
7008             ie=YoutubeTabIE, video_id=playlist_id)
7009
7010
7011 class YoutubeShortsAudioPivotIE(InfoExtractor):
7012     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
7013     IE_NAME = 'youtube:shorts:pivot:audio'
7014     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
7015     _TESTS = [{
7016         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
7017         'only_matching': True,
7018     }]
7019
7020     @staticmethod
7021     def _generate_audio_pivot_params(video_id):
7022         """
7023         Generates sfv_audio_pivot browse params for this video id
7024         """
7025         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
7026         return urllib.parse.quote(base64.b64encode(pb_params).decode())
7027
7028     def _real_extract(self, url):
7029         video_id = self._match_id(url)
7030         return self.url_result(
7031             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
7032             ie=YoutubeTabIE)
7033
7034
7035 class YoutubeTruncatedURLIE(InfoExtractor):
7036     IE_NAME = 'youtube:truncated_url'
7037     IE_DESC = False  # Do not list
7038     _VALID_URL = r'''(?x)
7039         (?:https?://)?
7040         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
7041         (?:watch\?(?:
7042             feature=[a-z_]+|
7043             annotation_id=annotation_[^&]+|
7044             x-yt-cl=[0-9]+|
7045             hl=[^&]*|
7046             t=[0-9]+
7047         )?
7048         |
7049             attribution_link\?a=[^&]+
7050         )
7051         $
7052     '''
7053
7054     _TESTS = [{
7055         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
7056         'only_matching': True,
7057     }, {
7058         'url': 'https://www.youtube.com/watch?',
7059         'only_matching': True,
7060     }, {
7061         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
7062         'only_matching': True,
7063     }, {
7064         'url': 'https://www.youtube.com/watch?feature=foo',
7065         'only_matching': True,
7066     }, {
7067         'url': 'https://www.youtube.com/watch?hl=en-GB',
7068         'only_matching': True,
7069     }, {
7070         'url': 'https://www.youtube.com/watch?t=2372',
7071         'only_matching': True,
7072     }]
7073
7074     def _real_extract(self, url):
7075         raise ExtractorError(
7076             'Did you forget to quote the URL? Remember that & is a meta '
7077             'character in most shells, so you want to put the URL in quotes, '
7078             'like  youtube-dl '
7079             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
7080             ' or simply  youtube-dl BaW_jenozKc  .',
7081             expected=True)
7082
7083
7084 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
7085     IE_NAME = 'youtube:clip'
7086     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
7087     _TESTS = [{
7088         # FIXME: Other metadata should be extracted from the clip, not from the base video
7089         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
7090         'info_dict': {
7091             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
7092             'ext': 'mp4',
7093             'section_start': 29.0,
7094             'section_end': 39.7,
7095             'duration': 10.7,
7096             'age_limit': 0,
7097             'availability': 'public',
7098             'categories': ['Gaming'],
7099             'channel': 'Scott The Woz',
7100             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
7101             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
7102             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
7103             'like_count': int,
7104             'playable_in_embed': True,
7105             'tags': 'count:17',
7106             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
7107             'title': 'Mobile Games on Console - Scott The Woz',
7108             'upload_date': '20210920',
7109             'uploader': 'Scott The Woz',
7110             'uploader_id': '@ScottTheWoz',
7111             'uploader_url': 'https://www.youtube.com/@ScottTheWoz',
7112             'view_count': int,
7113             'live_status': 'not_live',
7114             'channel_follower_count': int,
7115             'chapters': 'count:20',
7116         }
7117     }]
7118
7119     def _real_extract(self, url):
7120         clip_id = self._match_id(url)
7121         _, data = self._extract_webpage(url, clip_id)
7122
7123         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
7124         if not video_id:
7125             raise ExtractorError('Unable to find video ID')
7126
7127         clip_data = traverse_obj(data, (
7128             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
7129             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
7130             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
7131             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
7132
7133         return {
7134             '_type': 'url_transparent',
7135             'url': f'https://www.youtube.com/watch?v={video_id}',
7136             'ie_key': YoutubeIE.ie_key(),
7137             'id': clip_id,
7138             'section_start': int(clip_data['startTimeMs']) / 1000,
7139             'section_end': int(clip_data['endTimeMs']) / 1000,
7140         }
7141
7142
7143 class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
7144     IE_NAME = 'youtube:consent'
7145     IE_DESC = False  # Do not list
7146     _VALID_URL = r'https?://consent\.youtube\.com/m\?'
7147     _TESTS = [{
7148         'url': 'https://consent.youtube.com/m?continue=https%3A%2F%2Fwww.youtube.com%2Flive%2FqVv6vCqciTM%3Fcbrd%3D1&gl=NL&m=0&pc=yt&hl=en&src=1',
7149         'info_dict': {
7150             'id': 'qVv6vCqciTM',
7151             'ext': 'mp4',
7152             'age_limit': 0,
7153             'uploader_id': '@sana_natori',
7154             'comment_count': int,
7155             'chapters': 'count:13',
7156             'upload_date': '20221223',
7157             'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
7158             'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
7159             'uploader_url': 'https://www.youtube.com/@sana_natori',
7160             'like_count': int,
7161             'release_date': '20221223',
7162             'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
7163             'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
7164             'view_count': int,
7165             'playable_in_embed': True,
7166             'duration': 4438,
7167             'availability': 'public',
7168             'channel_follower_count': int,
7169             'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
7170             'categories': ['Entertainment'],
7171             'live_status': 'was_live',
7172             'release_timestamp': 1671793345,
7173             'channel': 'さなちゃんねる',
7174             'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
7175             'uploader': 'さなちゃんねる',
7176         },
7177         'add_ie': ['Youtube'],
7178         'params': {'skip_download': 'Youtube'},
7179     }]
7180
7181     def _real_extract(self, url):
7182         redirect_url = url_or_none(parse_qs(url).get('continue', [None])[-1])
7183         if not redirect_url:
7184             raise ExtractorError('Invalid cookie consent redirect URL', expected=True)
7185         return self.url_result(redirect_url)
7186
7187
7188 class YoutubeTruncatedIDIE(InfoExtractor):
7189     IE_NAME = 'youtube:truncated_id'
7190     IE_DESC = False  # Do not list
7191     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
7192
7193     _TESTS = [{
7194         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
7195         'only_matching': True,
7196     }]
7197
7198     def _real_extract(self, url):
7199         video_id = self._match_id(url)
7200         raise ExtractorError(
7201             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
7202             expected=True)