yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import collections
   4 import copy
   5 import datetime
   6 import enum
   7 import hashlib
   8 import itertools
   9 import json
  10 import math
  11 import os.path
  12 import random
  13 import re
  14 import sys
  15 import threading
  16 import time
  17 import traceback
  18 import urllib.error
  19 import urllib.parse
  20
  21 from .common import InfoExtractor, SearchInfoExtractor
  22 from .openload import PhantomJSwrapper
  23 from ..compat import functools
  24 from ..jsinterp import JSInterpreter
  25 from ..utils import (
  26     NO_DEFAULT,
  27     ExtractorError,
  28     LazyList,
  29     UserNotLive,
  30     bug_reports_message,
  31     classproperty,
  32     clean_html,
  33     datetime_from_str,
  34     dict_get,
  35     filter_dict,
  36     float_or_none,
  37     format_field,
  38     get_first,
  39     int_or_none,
  40     is_html,
  41     join_nonempty,
  42     js_to_json,
  43     mimetype2ext,
  44     network_exceptions,
  45     orderedSet,
  46     parse_codecs,
  47     parse_count,
  48     parse_duration,
  49     parse_iso8601,
  50     parse_qs,
  51     qualities,
  52     remove_start,
  53     smuggle_url,
  54     str_or_none,
  55     str_to_int,
  56     strftime_or_none,
  57     traverse_obj,
  58     try_get,
  59     unescapeHTML,
  60     unified_strdate,
  61     unified_timestamp,
  62     unsmuggle_url,
  63     update_url_query,
  64     url_or_none,
  65     urljoin,
  66     variadic,
  67 )
  68
  69
  70 STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client'
  71 # any clients starting with _ cannot be explicitly requested by the user
  72 INNERTUBE_CLIENTS = {
  73     'web': {
  74         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  75         'INNERTUBE_CONTEXT': {
  76             'client': {
  77                 'clientName': 'WEB',
  78                 'clientVersion': '2.20220801.00.00',
  79             }
  80         },
  81         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  82     },
  83     'web_embedded': {
  84         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  85         'INNERTUBE_CONTEXT': {
  86             'client': {
  87                 'clientName': 'WEB_EMBEDDED_PLAYER',
  88                 'clientVersion': '1.20220731.00.00',
  89             },
  90         },
  91         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  92     },
  93     'web_music': {
  94         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  95         'INNERTUBE_HOST': 'music.youtube.com',
  96         'INNERTUBE_CONTEXT': {
  97             'client': {
  98                 'clientName': 'WEB_REMIX',
  99                 'clientVersion': '1.20220727.01.00',
 100             }
 101         },
 102         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 103     },
 104     'web_creator': {
 105         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 106         'INNERTUBE_CONTEXT': {
 107             'client': {
 108                 'clientName': 'WEB_CREATOR',
 109                 'clientVersion': '1.20220726.00.00',
 110             }
 111         },
 112         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 113     },
 114     'android': {
 115         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 116         'INNERTUBE_CONTEXT': {
 117             'client': {
 118                 'clientName': 'ANDROID',
 119                 'clientVersion': '17.31.35',
 120                 'androidSdkVersion': 30,
 121                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 122             }
 123         },
 124         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 125         'REQUIRE_JS_PLAYER': False
 126     },
 127     'android_embedded': {
 128         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 129         'INNERTUBE_CONTEXT': {
 130             'client': {
 131                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 132                 'clientVersion': '17.31.35',
 133                 'androidSdkVersion': 30,
 134                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 135             },
 136         },
 137         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 138         'REQUIRE_JS_PLAYER': False
 139     },
 140     'android_music': {
 141         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 142         'INNERTUBE_CONTEXT': {
 143             'client': {
 144                 'clientName': 'ANDROID_MUSIC',
 145                 'clientVersion': '5.16.51',
 146                 'androidSdkVersion': 30,
 147                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 148             }
 149         },
 150         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 151         'REQUIRE_JS_PLAYER': False
 152     },
 153     'android_creator': {
 154         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 155         'INNERTUBE_CONTEXT': {
 156             'client': {
 157                 'clientName': 'ANDROID_CREATOR',
 158                 'clientVersion': '22.30.100',
 159                 'androidSdkVersion': 30,
 160                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 161             },
 162         },
 163         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 164         'REQUIRE_JS_PLAYER': False
 165     },
 166     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 167     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 168     'ios': {
 169         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 170         'INNERTUBE_CONTEXT': {
 171             'client': {
 172                 'clientName': 'IOS',
 173                 'clientVersion': '17.33.2',
 174                 'deviceModel': 'iPhone14,3',
 175                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 176             }
 177         },
 178         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 179         'REQUIRE_JS_PLAYER': False
 180     },
 181     'ios_embedded': {
 182         'INNERTUBE_CONTEXT': {
 183             'client': {
 184                 'clientName': 'IOS_MESSAGES_EXTENSION',
 185                 'clientVersion': '17.33.2',
 186                 'deviceModel': 'iPhone14,3',
 187                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 188             },
 189         },
 190         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 191         'REQUIRE_JS_PLAYER': False
 192     },
 193     'ios_music': {
 194         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 195         'INNERTUBE_CONTEXT': {
 196             'client': {
 197                 'clientName': 'IOS_MUSIC',
 198                 'clientVersion': '5.21',
 199                 'deviceModel': 'iPhone14,3',
 200                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 201             },
 202         },
 203         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 204         'REQUIRE_JS_PLAYER': False
 205     },
 206     'ios_creator': {
 207         'INNERTUBE_CONTEXT': {
 208             'client': {
 209                 'clientName': 'IOS_CREATOR',
 210                 'clientVersion': '22.33.101',
 211                 'deviceModel': 'iPhone14,3',
 212                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 213             },
 214         },
 215         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 216         'REQUIRE_JS_PLAYER': False
 217     },
 218     # mweb has 'ultralow' formats
 219     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 220     'mweb': {
 221         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 222         'INNERTUBE_CONTEXT': {
 223             'client': {
 224                 'clientName': 'MWEB',
 225                 'clientVersion': '2.20220801.00.00',
 226             }
 227         },
 228         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 229     },
 230     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 231     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 232     'tv_embedded': {
 233         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 234         'INNERTUBE_CONTEXT': {
 235             'client': {
 236                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 237                 'clientVersion': '2.0',
 238             },
 239         },
 240         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 241     },
 242 }
 243
 244
 245 def _split_innertube_client(client_name):
 246     variant, *base = client_name.rsplit('.', 1)
 247     if base:
 248         return variant, base[0], variant
 249     base, *variant = client_name.split('_', 1)
 250     return client_name, base, variant[0] if variant else None
 251
 252
 253 def short_client_name(client_name):
 254     main, *parts = _split_innertube_client(client_name)[0].replace('embedscreen', 'e_s').split('_')
 255     return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
 256
 257
 258 def build_innertube_clients():
 259     THIRD_PARTY = {
 260         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 261     }
 262     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 263     priority = qualities(BASE_CLIENTS[::-1])
 264
 265     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 266         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 267         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 268         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 269         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 270
 271         _, base_client, variant = _split_innertube_client(client)
 272         ytcfg['priority'] = 10 * priority(base_client)
 273
 274         if not variant:
 275             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 276             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 277             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 278             embedscreen['priority'] -= 3
 279         elif variant == 'embedded':
 280             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 281             ytcfg['priority'] -= 2
 282         else:
 283             ytcfg['priority'] -= 3
 284
 285
 286 build_innertube_clients()
 287
 288
 289 class BadgeType(enum.Enum):
 290     AVAILABILITY_UNLISTED = enum.auto()
 291     AVAILABILITY_PRIVATE = enum.auto()
 292     AVAILABILITY_PUBLIC = enum.auto()
 293     AVAILABILITY_PREMIUM = enum.auto()
 294     AVAILABILITY_SUBSCRIPTION = enum.auto()
 295     LIVE_NOW = enum.auto()
 296
 297
 298 class YoutubeBaseInfoExtractor(InfoExtractor):
 299     """Provide base functions for Youtube extractors"""
 300
 301     _RESERVED_NAMES = (
 302         r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
 303         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 304         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 305         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 306
 307     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 308
 309     # _NETRC_MACHINE = 'youtube'
 310
 311     # If True it will raise an error if no login info is provided
 312     _LOGIN_REQUIRED = False
 313
 314     _INVIDIOUS_SITES = (
 315         # invidious-redirect websites
 316         r'(?:www\.)?redirect\.invidious\.io',
 317         r'(?:(?:www|dev)\.)?invidio\.us',
 318         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 319         r'(?:www\.)?invidious\.pussthecat\.org',
 320         r'(?:www\.)?invidious\.zee\.li',
 321         r'(?:www\.)?invidious\.ethibox\.fr',
 322         r'(?:www\.)?iv\.ggtyler\.dev',
 323         r'(?:www\.)?inv\.vern\.i2p',
 324         r'(?:www\.)?am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd\.onion',
 325         r'(?:www\.)?inv\.riverside\.rocks',
 326         r'(?:www\.)?invidious\.silur\.me',
 327         r'(?:www\.)?inv\.bp\.projectsegfau\.lt',
 328         r'(?:www\.)?invidious\.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid\.onion',
 329         r'(?:www\.)?invidious\.slipfox\.xyz',
 330         r'(?:www\.)?invidious\.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd\.onion',
 331         r'(?:www\.)?inv\.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad\.onion',
 332         r'(?:www\.)?invidious\.tiekoetter\.com',
 333         r'(?:www\.)?iv\.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd\.onion',
 334         r'(?:www\.)?invidious\.nerdvpn\.de',
 335         r'(?:www\.)?invidious\.weblibre\.org',
 336         r'(?:www\.)?inv\.odyssey346\.dev',
 337         r'(?:www\.)?invidious\.dhusch\.de',
 338         r'(?:www\.)?iv\.melmac\.space',
 339         r'(?:www\.)?watch\.thekitty\.zone',
 340         r'(?:www\.)?invidious\.privacydev\.net',
 341         r'(?:www\.)?ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid\.onion',
 342         r'(?:www\.)?invidious\.drivet\.xyz',
 343         r'(?:www\.)?vid\.priv\.au',
 344         r'(?:www\.)?euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd\.onion',
 345         r'(?:www\.)?inv\.vern\.cc',
 346         r'(?:www\.)?invidious\.esmailelbob\.xyz',
 347         r'(?:www\.)?invidious\.sethforprivacy\.com',
 348         r'(?:www\.)?yt\.oelrichsgarcia\.de',
 349         r'(?:www\.)?yt\.artemislena\.eu',
 350         r'(?:www\.)?invidious\.flokinet\.to',
 351         r'(?:www\.)?invidious\.baczek\.me',
 352         r'(?:www\.)?y\.com\.sb',
 353         r'(?:www\.)?invidious\.epicsite\.xyz',
 354         r'(?:www\.)?invidious\.lidarshield\.cloud',
 355         r'(?:www\.)?yt\.funami\.tech',
 356         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 357         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 358         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 359         # youtube-dl invidious instances list
 360         r'(?:(?:www|no)\.)?invidiou\.sh',
 361         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 362         r'(?:www\.)?invidious\.kabi\.tk',
 363         r'(?:www\.)?invidious\.mastodon\.host',
 364         r'(?:www\.)?invidious\.zapashcanon\.fr',
 365         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 366         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 367         r'(?:www\.)?invidious\.himiko\.cloud',
 368         r'(?:www\.)?invidious\.reallyancient\.tech',
 369         r'(?:www\.)?invidious\.tube',
 370         r'(?:www\.)?invidiou\.site',
 371         r'(?:www\.)?invidious\.site',
 372         r'(?:www\.)?invidious\.xyz',
 373         r'(?:www\.)?invidious\.nixnet\.xyz',
 374         r'(?:www\.)?invidious\.048596\.xyz',
 375         r'(?:www\.)?invidious\.drycat\.fr',
 376         r'(?:www\.)?inv\.skyn3t\.in',
 377         r'(?:www\.)?tube\.poal\.co',
 378         r'(?:www\.)?tube\.connect\.cafe',
 379         r'(?:www\.)?vid\.wxzm\.sx',
 380         r'(?:www\.)?vid\.mint\.lgbt',
 381         r'(?:www\.)?vid\.puffyan\.us',
 382         r'(?:www\.)?yewtu\.be',
 383         r'(?:www\.)?yt\.elukerio\.org',
 384         r'(?:www\.)?yt\.lelux\.fi',
 385         r'(?:www\.)?invidious\.ggc-project\.de',
 386         r'(?:www\.)?yt\.maisputain\.ovh',
 387         r'(?:www\.)?ytprivate\.com',
 388         r'(?:www\.)?invidious\.13ad\.de',
 389         r'(?:www\.)?invidious\.toot\.koeln',
 390         r'(?:www\.)?invidious\.fdn\.fr',
 391         r'(?:www\.)?watch\.nettohikari\.com',
 392         r'(?:www\.)?invidious\.namazso\.eu',
 393         r'(?:www\.)?invidious\.silkky\.cloud',
 394         r'(?:www\.)?invidious\.exonip\.de',
 395         r'(?:www\.)?invidious\.riverside\.rocks',
 396         r'(?:www\.)?invidious\.blamefran\.net',
 397         r'(?:www\.)?invidious\.moomoo\.de',
 398         r'(?:www\.)?ytb\.trom\.tf',
 399         r'(?:www\.)?yt\.cyberhost\.uk',
 400         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 401         r'(?:www\.)?qklhadlycap4cnod\.onion',
 402         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 403         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 404         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 405         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 406         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 407         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 408         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 409         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 410         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 411         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 412         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 413         r'(?:www\.)?piped\.kavin\.rocks',
 414         r'(?:www\.)?piped\.tokhmi\.xyz',
 415         r'(?:www\.)?piped\.syncpundit\.io',
 416         r'(?:www\.)?piped\.mha\.fi',
 417         r'(?:www\.)?watch\.whatever\.social',
 418         r'(?:www\.)?piped\.garudalinux\.org',
 419         r'(?:www\.)?piped\.rivo\.lol',
 420         r'(?:www\.)?piped-libre\.kavin\.rocks',
 421         r'(?:www\.)?yt\.jae\.fi',
 422         r'(?:www\.)?piped\.mint\.lgbt',
 423         r'(?:www\.)?il\.ax',
 424         r'(?:www\.)?piped\.esmailelbob\.xyz',
 425         r'(?:www\.)?piped\.projectsegfau\.lt',
 426         r'(?:www\.)?piped\.privacydev\.net',
 427         r'(?:www\.)?piped\.palveluntarjoaja\.eu',
 428         r'(?:www\.)?piped\.smnz\.de',
 429         r'(?:www\.)?piped\.adminforge\.de',
 430         r'(?:www\.)?watch\.whatevertinfoil\.de',
 431         r'(?:www\.)?piped\.qdi\.fi',
 432         r'(?:www\.)?piped\.video',
 433         r'(?:www\.)?piped\.aeong\.one',
 434         r'(?:www\.)?piped\.moomoo\.me',
 435         r'(?:www\.)?piped\.chauvet\.pro',
 436         r'(?:www\.)?watch\.leptons\.xyz',
 437         r'(?:www\.)?pd\.vern\.cc',
 438         r'(?:www\.)?piped\.hostux\.net',
 439         r'(?:www\.)?piped\.lunar\.icu',
 440         # Hyperpipe instances from https://hyperpipe.codeberg.page/
 441         r'(?:www\.)?hyperpipe\.surge\.sh',
 442         r'(?:www\.)?hyperpipe\.esmailelbob\.xyz',
 443         r'(?:www\.)?listen\.whatever\.social',
 444         r'(?:www\.)?music\.adminforge\.de',
 445     )
 446
 447     # extracted from account/account_menu ep
 448     # XXX: These are the supported YouTube UI and API languages,
 449     # which is slightly different from languages supported for translation in YouTube studio
 450     _SUPPORTED_LANG_CODES = [
 451         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 452         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 453         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 454         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 455         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 456         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 457     ]
 458
 459     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 460
 461     @functools.cached_property
 462     def _preferred_lang(self):
 463         """
 464         Returns a language code supported by YouTube for the user preferred language.
 465         Returns None if no preferred language set.
 466         """
 467         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 468         if not preferred_lang:
 469             return
 470         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 471             raise ExtractorError(
 472                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 473                 expected=True)
 474         elif preferred_lang != 'en':
 475             self.report_warning(
 476                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 477         return preferred_lang
 478
 479     def _initialize_consent(self):
 480         cookies = self._get_cookies('https://www.youtube.com/')
 481         if cookies.get('__Secure-3PSID'):
 482             return
 483         consent_id = None
 484         consent = cookies.get('CONSENT')
 485         if consent:
 486             if 'YES' in consent.value:
 487                 return
 488             consent_id = self._search_regex(
 489                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 490         if not consent_id:
 491             consent_id = random.randint(100, 999)
 492         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 493
 494     def _initialize_pref(self):
 495         cookies = self._get_cookies('https://www.youtube.com/')
 496         pref_cookie = cookies.get('PREF')
 497         pref = {}
 498         if pref_cookie:
 499             try:
 500                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 501             except ValueError:
 502                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 503         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 504         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 505
 506     def _real_initialize(self):
 507         self._initialize_pref()
 508         self._initialize_consent()
 509         self._check_login_required()
 510
 511     def _check_login_required(self):
 512         if self._LOGIN_REQUIRED and not self._cookies_passed:
 513             self.raise_login_required('Login details are needed to download this content', method='cookies')
 514
 515     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 516     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 517
 518     def _get_default_ytcfg(self, client='web'):
 519         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 520
 521     def _get_innertube_host(self, client='web'):
 522         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 523
 524     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 525         # try_get but with fallback to default ytcfg client values when present
 526         _func = lambda y: try_get(y, getter, expected_type)
 527         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 528
 529     def _extract_client_name(self, ytcfg, default_client='web'):
 530         return self._ytcfg_get_safe(
 531             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 532                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 533
 534     def _extract_client_version(self, ytcfg, default_client='web'):
 535         return self._ytcfg_get_safe(
 536             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 537                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 538
 539     def _select_api_hostname(self, req_api_hostname, default_client=None):
 540         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 541                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 542
 543     def _extract_api_key(self, ytcfg=None, default_client='web'):
 544         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 545
 546     def _extract_context(self, ytcfg=None, default_client='web'):
 547         context = get_first(
 548             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 549         # Enforce language and tz for extraction
 550         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 551         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 552         return context
 553
 554     _SAPISID = None
 555
 556     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 557         time_now = round(time.time())
 558         if self._SAPISID is None:
 559             yt_cookies = self._get_cookies('https://www.youtube.com')
 560             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 561             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 562             sapisid_cookie = dict_get(
 563                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 564             if sapisid_cookie and sapisid_cookie.value:
 565                 self._SAPISID = sapisid_cookie.value
 566                 self.write_debug('Extracted SAPISID cookie')
 567                 # SAPISID cookie is required if not already present
 568                 if not yt_cookies.get('SAPISID'):
 569                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 570                     self._set_cookie(
 571                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 572             else:
 573                 self._SAPISID = False
 574         if not self._SAPISID:
 575             return None
 576         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 577         sapisidhash = hashlib.sha1(
 578             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 579         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 580
 581     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 582                   note='Downloading API JSON', errnote='Unable to download API page',
 583                   context=None, api_key=None, api_hostname=None, default_client='web'):
 584
 585         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 586         data.update(query)
 587         real_headers = self.generate_api_headers(default_client=default_client)
 588         real_headers.update({'content-type': 'application/json'})
 589         if headers:
 590             real_headers.update(headers)
 591         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 592                    or api_key or self._extract_api_key(default_client=default_client))
 593         return self._download_json(
 594             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 595             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 596             data=json.dumps(data).encode('utf8'), headers=real_headers,
 597             query={'key': api_key, 'prettyPrint': 'false'})
 598
 599     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 600         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 601
 602     @staticmethod
 603     def _extract_session_index(*data):
 604         """
 605         Index of current account in account list.
 606         See: https://github.com/yt-dlp/yt-dlp/pull/519
 607         """
 608         for ytcfg in data:
 609             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 610             if session_index is not None:
 611                 return session_index
 612
 613     # Deprecated?
 614     def _extract_identity_token(self, ytcfg=None, webpage=None):
 615         if ytcfg:
 616             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 617             if token:
 618                 return token
 619         if webpage:
 620             return self._search_regex(
 621                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 622                 'identity token', default=None, fatal=False)
 623
 624     @staticmethod
 625     def _extract_account_syncid(*args):
 626         """
 627         Extract syncId required to download private playlists of secondary channels
 628         @params response and/or ytcfg
 629         """
 630         for data in args:
 631             # ytcfg includes channel_syncid if on secondary channel
 632             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 633             if delegated_sid:
 634                 return delegated_sid
 635             sync_ids = (try_get(
 636                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 637                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 638             if len(sync_ids) >= 2 and sync_ids[1]:
 639                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 640                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 641                 return sync_ids[0]
 642
 643     @staticmethod
 644     def _extract_visitor_data(*args):
 645         """
 646         Extracts visitorData from an API response or ytcfg
 647         Appears to be used to track session state
 648         """
 649         return get_first(
 650             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 651             expected_type=str)
 652
 653     @functools.cached_property
 654     def is_authenticated(self):
 655         return bool(self._generate_sapisidhash_header())
 656
 657     def extract_ytcfg(self, video_id, webpage):
 658         if not webpage:
 659             return {}
 660         return self._parse_json(
 661             self._search_regex(
 662                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 663                 default='{}'), video_id, fatal=False) or {}
 664
 665     def generate_api_headers(
 666             self, *, ytcfg=None, account_syncid=None, session_index=None,
 667             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 668
 669         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 670         headers = {
 671             'X-YouTube-Client-Name': str(
 672                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 673             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 674             'Origin': origin,
 675             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 676             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 677             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 678             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 679         }
 680         if session_index is None:
 681             session_index = self._extract_session_index(ytcfg)
 682         if account_syncid or session_index is not None:
 683             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 684
 685         auth = self._generate_sapisidhash_header(origin)
 686         if auth is not None:
 687             headers['Authorization'] = auth
 688             headers['X-Origin'] = origin
 689         return filter_dict(headers)
 690
 691     def _download_ytcfg(self, client, video_id):
 692         url = {
 693             'web': 'https://www.youtube.com',
 694             'web_music': 'https://music.youtube.com',
 695             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 696         }.get(client)
 697         if not url:
 698             return {}
 699         webpage = self._download_webpage(
 700             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 701         return self.extract_ytcfg(video_id, webpage) or {}
 702
 703     @staticmethod
 704     def _build_api_continuation_query(continuation, ctp=None):
 705         query = {
 706             'continuation': continuation
 707         }
 708         # TODO: Inconsistency with clickTrackingParams.
 709         # Currently we have a fixed ctp contained within context (from ytcfg)
 710         # and a ctp in root query for continuation.
 711         if ctp:
 712             query['clickTracking'] = {'clickTrackingParams': ctp}
 713         return query
 714
 715     @classmethod
 716     def _extract_next_continuation_data(cls, renderer):
 717         next_continuation = try_get(
 718             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 719                        lambda x: x['continuation']['reloadContinuationData']), dict)
 720         if not next_continuation:
 721             return
 722         continuation = next_continuation.get('continuation')
 723         if not continuation:
 724             return
 725         ctp = next_continuation.get('clickTrackingParams')
 726         return cls._build_api_continuation_query(continuation, ctp)
 727
 728     @classmethod
 729     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 730         if isinstance(continuation_ep, dict):
 731             continuation = try_get(
 732                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 733             if not continuation:
 734                 return
 735             ctp = continuation_ep.get('clickTrackingParams')
 736             return cls._build_api_continuation_query(continuation, ctp)
 737
 738     @classmethod
 739     def _extract_continuation(cls, renderer):
 740         next_continuation = cls._extract_next_continuation_data(renderer)
 741         if next_continuation:
 742             return next_continuation
 743
 744         return traverse_obj(renderer, (
 745             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 746             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 747         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 748
 749     @classmethod
 750     def _extract_alerts(cls, data):
 751         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 752             if not isinstance(alert_dict, dict):
 753                 continue
 754             for alert in alert_dict.values():
 755                 alert_type = alert.get('type')
 756                 if not alert_type:
 757                     continue
 758                 message = cls._get_text(alert, 'text')
 759                 if message:
 760                     yield alert_type, message
 761
 762     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 763         errors, warnings = [], []
 764         for alert_type, alert_message in alerts:
 765             if alert_type.lower() == 'error' and fatal:
 766                 errors.append([alert_type, alert_message])
 767             elif alert_message not in self._IGNORED_WARNINGS:
 768                 warnings.append([alert_type, alert_message])
 769
 770         for alert_type, alert_message in (warnings + errors[:-1]):
 771             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 772         if errors:
 773             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 774
 775     def _extract_and_report_alerts(self, data, *args, **kwargs):
 776         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 777
 778     def _extract_badges(self, renderer: dict):
 779         privacy_icon_map = {
 780             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 781             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 782             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 783         }
 784
 785         badge_style_map = {
 786             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 787             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 788             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 789         }
 790
 791         label_map = {
 792             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 793             'private': BadgeType.AVAILABILITY_PRIVATE,
 794             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 795             'live': BadgeType.LIVE_NOW,
 796             'premium': BadgeType.AVAILABILITY_PREMIUM
 797         }
 798
 799         badges = []
 800         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer')):
 801             badge_type = (
 802                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 803                 or badge_style_map.get(traverse_obj(badge, 'style'))
 804             )
 805             if badge_type:
 806                 badges.append({'type': badge_type})
 807                 continue
 808
 809             # fallback, won't work in some languages
 810             label = traverse_obj(badge, 'label', expected_type=str, default='')
 811             for match, label_badge_type in label_map.items():
 812                 if match in label.lower():
 813                     badges.append({'type': badge_type})
 814                     continue
 815
 816         return badges
 817
 818     @staticmethod
 819     def _has_badge(badges, badge_type):
 820         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 821
 822     @staticmethod
 823     def _get_text(data, *path_list, max_runs=None):
 824         for path in path_list or [None]:
 825             if path is None:
 826                 obj = [data]
 827             else:
 828                 obj = traverse_obj(data, path, default=[])
 829                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 830                     obj = [obj]
 831             for item in obj:
 832                 text = try_get(item, lambda x: x['simpleText'], str)
 833                 if text:
 834                     return text
 835                 runs = try_get(item, lambda x: x['runs'], list) or []
 836                 if not runs and isinstance(item, list):
 837                     runs = item
 838
 839                 runs = runs[:min(len(runs), max_runs or len(runs))]
 840                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str))
 841                 if text:
 842                     return text
 843
 844     def _get_count(self, data, *path_list):
 845         count_text = self._get_text(data, *path_list) or ''
 846         count = parse_count(count_text)
 847         if count is None:
 848             count = str_to_int(
 849                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 850         return count
 851
 852     @staticmethod
 853     def _extract_thumbnails(data, *path_list):
 854         """
 855         Extract thumbnails from thumbnails dict
 856         @param path_list: path list to level that contains 'thumbnails' key
 857         """
 858         thumbnails = []
 859         for path in path_list or [()]:
 860             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...)):
 861                 thumbnail_url = url_or_none(thumbnail.get('url'))
 862                 if not thumbnail_url:
 863                     continue
 864                 # Sometimes youtube gives a wrong thumbnail URL. See:
 865                 # https://github.com/yt-dlp/yt-dlp/issues/233
 866                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 867                 if 'maxresdefault' in thumbnail_url:
 868                     thumbnail_url = thumbnail_url.split('?')[0]
 869                 thumbnails.append({
 870                     'url': thumbnail_url,
 871                     'height': int_or_none(thumbnail.get('height')),
 872                     'width': int_or_none(thumbnail.get('width')),
 873                 })
 874         return thumbnails
 875
 876     @staticmethod
 877     def extract_relative_time(relative_time_text):
 878         """
 879         Extracts a relative time from string and converts to dt object
 880         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 881         """
 882         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 883         if mobj:
 884             start = mobj.group('start')
 885             if start:
 886                 return datetime_from_str(start)
 887             try:
 888                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 889             except ValueError:
 890                 return None
 891
 892     def _parse_time_text(self, text):
 893         if not text:
 894             return
 895         dt = self.extract_relative_time(text)
 896         timestamp = None
 897         if isinstance(dt, datetime.datetime):
 898             timestamp = calendar.timegm(dt.timetuple())
 899
 900         if timestamp is None:
 901             timestamp = (
 902                 unified_timestamp(text) or unified_timestamp(
 903                     self._search_regex(
 904                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 905                         text.lower(), 'time text', default=None)))
 906
 907         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 908             self.report_warning(
 909                 f'Cannot parse localized time text "{text}"', only_once=True)
 910         return timestamp
 911
 912     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 913                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 914                           default_client='web'):
 915         for retry in self.RetryManager():
 916             try:
 917                 response = self._call_api(
 918                     ep=ep, fatal=True, headers=headers,
 919                     video_id=item_id, query=query, note=note,
 920                     context=self._extract_context(ytcfg, default_client),
 921                     api_key=self._extract_api_key(ytcfg, default_client),
 922                     api_hostname=api_hostname, default_client=default_client)
 923             except ExtractorError as e:
 924                 if not isinstance(e.cause, network_exceptions):
 925                     return self._error_or_warning(e, fatal=fatal)
 926                 elif not isinstance(e.cause, urllib.error.HTTPError):
 927                     retry.error = e
 928                     continue
 929
 930                 first_bytes = e.cause.read(512)
 931                 if not is_html(first_bytes):
 932                     yt_error = try_get(
 933                         self._parse_json(
 934                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 935                         lambda x: x['error']['message'], str)
 936                     if yt_error:
 937                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 938                 # Downloading page may result in intermittent 5xx HTTP error
 939                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 940                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 941                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 942                 if e.cause.code not in (403, 429):
 943                     retry.error = e
 944                     continue
 945                 return self._error_or_warning(e, fatal=fatal)
 946
 947             try:
 948                 self._extract_and_report_alerts(response, only_once=True)
 949             except ExtractorError as e:
 950                 # YouTube servers may return errors we want to retry on in a 200 OK response
 951                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 952                 if 'unknown error' in e.msg.lower():
 953                     retry.error = e
 954                     continue
 955                 return self._error_or_warning(e, fatal=fatal)
 956             # Youtube sometimes sends incomplete data
 957             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 958             if not traverse_obj(response, *variadic(check_get_keys)):
 959                 retry.error = ExtractorError('Incomplete data received', expected=True)
 960                 continue
 961
 962             return response
 963
 964     @staticmethod
 965     def is_music_url(url):
 966         return re.match(r'(https?://)?music\.youtube\.com/', url) is not None
 967
 968     def _extract_video(self, renderer):
 969         video_id = renderer.get('videoId')
 970
 971         reel_header_renderer = traverse_obj(renderer, (
 972             'navigationEndpoint', 'reelWatchEndpoint', 'overlay', 'reelPlayerOverlayRenderer',
 973             'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
 974
 975         title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
 976         description = self._get_text(renderer, 'descriptionSnippet')
 977
 978         duration = int_or_none(renderer.get('lengthSeconds'))
 979         if duration is None:
 980             duration = parse_duration(self._get_text(
 981                 renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 982         if duration is None:
 983             # XXX: should write a parser to be more general to support more cases (e.g. shorts in shorts tab)
 984             duration = parse_duration(self._search_regex(
 985                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 986                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 987                 video_id, default=None, group='duration'))
 988
 989         channel_id = traverse_obj(
 990             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 991             expected_type=str, get_all=False)
 992         if not channel_id:
 993             channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
 994
 995         overlay_style = traverse_obj(
 996             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 997             get_all=False, expected_type=str)
 998         badges = self._extract_badges(renderer)
 999
1000         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
1001             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
1002             expected_type=str)) or ''
1003         url = f'https://www.youtube.com/watch?v={video_id}'
1004         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
1005             url = f'https://www.youtube.com/shorts/{video_id}'
1006
1007         time_text = (self._get_text(renderer, 'publishedTimeText', 'videoInfo')
1008                      or self._get_text(reel_header_renderer, 'timestampText') or '')
1009         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
1010
1011         live_status = (
1012             'is_upcoming' if scheduled_timestamp is not None
1013             else 'was_live' if 'streamed' in time_text.lower()
1014             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
1015             else None)
1016
1017         # videoInfo is a string like '50K views • 10 years ago'.
1018         view_count_text = self._get_text(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo') or ''
1019         view_count = (0 if 'no views' in view_count_text.lower()
1020                       else self._get_count({'simpleText': view_count_text}))
1021         view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
1022
1023         return {
1024             '_type': 'url',
1025             'ie_key': YoutubeIE.ie_key(),
1026             'id': video_id,
1027             'url': url,
1028             'title': title,
1029             'description': description,
1030             'duration': duration,
1031             'channel_id': channel_id,
1032             'channel': (self._get_text(renderer, 'ownerText', 'shortBylineText')
1033                         or self._get_text(reel_header_renderer, 'channelTitleText')),
1034             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
1035             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
1036             'timestamp': (self._parse_time_text(time_text)
1037                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
1038                           else None),
1039             'release_timestamp': scheduled_timestamp,
1040             'availability':
1041                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
1042                 else self._availability(
1043                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
1044                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
1045                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
1046                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
1047             view_count_field: view_count,
1048             'live_status': live_status
1049         }
1050
1051
1052 class YoutubeIE(YoutubeBaseInfoExtractor):
1053     IE_DESC = 'YouTube'
1054     _VALID_URL = r"""(?x)^
1055                      (
1056                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
1057                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
1058                             (?:www\.)?deturl\.com/www\.youtube\.com|
1059                             (?:www\.)?pwnyoutube\.com|
1060                             (?:www\.)?hooktube\.com|
1061                             (?:www\.)?yourepeat\.com|
1062                             tube\.majestyc\.net|
1063                             %(invidious)s|
1064                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
1065                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
1066                          (?:                                                  # the various things that can precede the ID:
1067                              (?:(?:v|embed|e|shorts|live)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
1068                              |(?:                                             # or the v= param in all its forms
1069                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
1070                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
1071                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
1072                                  v=
1073                              )
1074                          ))
1075                          |(?:
1076                             youtu\.be|                                        # just youtu.be/xxxx
1077                             vid\.plus|                                        # or vid.plus/xxxx
1078                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
1079                             %(invidious)s
1080                          )/
1081                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
1082                          )
1083                      )?                                                       # all until now is optional -> you can pass the naked ID
1084                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
1085                      (?(1).+)?                                                # if we found the ID, everything can follow
1086                      (?:\#|$)""" % {
1087         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1088     }
1089     _EMBED_REGEX = [
1090         r'''(?x)
1091             (?:
1092                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1093                 data-video-url=|
1094                 <embed[^>]+?src=|
1095                 embedSWF\(?:\s*|
1096                 <object[^>]+data=|
1097                 new\s+SWFObject\(
1098             )
1099             (["\'])
1100                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1101                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1102             \1''',
1103         # https://wordpress.org/plugins/lazy-load-for-videos/
1104         r'''(?xs)
1105             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1106             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1107     ]
1108     _RETURN_TYPE = 'video'  # XXX: How to handle multifeed?
1109
1110     _PLAYER_INFO_RE = (
1111         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1112         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1113         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1114     )
1115     _formats = {
1116         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1117         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1118         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1119         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1120         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1121         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1122         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1123         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1124         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1125         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1126         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1127         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1128         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1129         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1130         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1131         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1132         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1133         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1134
1135
1136         # 3D videos
1137         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1138         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1139         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1140         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1141         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1142         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1143         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1144
1145         # Apple HTTP Live Streaming
1146         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1147         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1148         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1149         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1150         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1151         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1152         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1153         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1154
1155         # DASH mp4 video
1156         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1157         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1158         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1159         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1160         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1161         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1162         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1163         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1164         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1165         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1166         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1167         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1168
1169         # Dash mp4 audio
1170         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1171         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1172         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1173         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1174         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1175         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1176         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1177
1178         # Dash webm
1179         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1180         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1181         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1182         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1183         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1184         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1185         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1186         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1187         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1188         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1189         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1190         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1191         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1192         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1193         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1194         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1195         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1196         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1197         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1198         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1199         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1200         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1201
1202         # Dash webm audio
1203         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1204         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1205
1206         # Dash webm audio with opus inside
1207         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1208         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1209         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1210
1211         # RTMP (unnamed)
1212         '_rtmp': {'protocol': 'rtmp'},
1213
1214         # av01 video only formats sometimes served with "unknown" codecs
1215         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1216         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1217         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1218         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1219         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1220         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1221         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1222         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1223     }
1224     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1225
1226     _GEO_BYPASS = False
1227
1228     IE_NAME = 'youtube'
1229     _TESTS = [
1230         {
1231             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1232             'info_dict': {
1233                 'id': 'BaW_jenozKc',
1234                 'ext': 'mp4',
1235                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1236                 'uploader': 'Philipp Hagemeister',
1237                 'uploader_id': 'phihag',
1238                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1239                 'channel': 'Philipp Hagemeister',
1240                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1241                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1242                 'upload_date': '20121002',
1243                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1244                 'categories': ['Science & Technology'],
1245                 'tags': ['youtube-dl'],
1246                 'duration': 10,
1247                 'view_count': int,
1248                 'like_count': int,
1249                 'availability': 'public',
1250                 'playable_in_embed': True,
1251                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1252                 'live_status': 'not_live',
1253                 'age_limit': 0,
1254                 'start_time': 1,
1255                 'end_time': 9,
1256                 'comment_count': int,
1257                 'channel_follower_count': int
1258             }
1259         },
1260         {
1261             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1262             'note': 'Embed-only video (#1746)',
1263             'info_dict': {
1264                 'id': 'yZIXLfi8CZQ',
1265                 'ext': 'mp4',
1266                 'upload_date': '20120608',
1267                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1268                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1269                 'uploader': 'SET India',
1270                 'uploader_id': 'setindia',
1271                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1272                 'age_limit': 18,
1273             },
1274             'skip': 'Private video',
1275         },
1276         {
1277             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1278             'note': 'Use the first video ID in the URL',
1279             'info_dict': {
1280                 'id': 'BaW_jenozKc',
1281                 'ext': 'mp4',
1282                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1283                 'uploader': 'Philipp Hagemeister',
1284                 'uploader_id': 'phihag',
1285                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1286                 'channel': 'Philipp Hagemeister',
1287                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1288                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1289                 'upload_date': '20121002',
1290                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1291                 'categories': ['Science & Technology'],
1292                 'tags': ['youtube-dl'],
1293                 'duration': 10,
1294                 'view_count': int,
1295                 'like_count': int,
1296                 'availability': 'public',
1297                 'playable_in_embed': True,
1298                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1299                 'live_status': 'not_live',
1300                 'age_limit': 0,
1301                 'comment_count': int,
1302                 'channel_follower_count': int
1303             },
1304             'params': {
1305                 'skip_download': True,
1306             },
1307         },
1308         {
1309             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1310             'note': '256k DASH audio (format 141) via DASH manifest',
1311             'info_dict': {
1312                 'id': 'a9LDPn-MO4I',
1313                 'ext': 'm4a',
1314                 'upload_date': '20121002',
1315                 'uploader_id': '8KVIDEO',
1316                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1317                 'description': '',
1318                 'uploader': '8KVIDEO',
1319                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1320             },
1321             'params': {
1322                 'youtube_include_dash_manifest': True,
1323                 'format': '141',
1324             },
1325             'skip': 'format 141 not served anymore',
1326         },
1327         # DASH manifest with encrypted signature
1328         {
1329             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1330             'info_dict': {
1331                 'id': 'IB3lcPjvWLA',
1332                 'ext': 'm4a',
1333                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1334                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1335                 'duration': 244,
1336                 'uploader': 'AfrojackVEVO',
1337                 'uploader_id': 'AfrojackVEVO',
1338                 'upload_date': '20131011',
1339                 'abr': 129.495,
1340                 'like_count': int,
1341                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1342                 'playable_in_embed': True,
1343                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1344                 'view_count': int,
1345                 'track': 'The Spark',
1346                 'live_status': 'not_live',
1347                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1348                 'channel': 'Afrojack',
1349                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1350                 'tags': 'count:19',
1351                 'availability': 'public',
1352                 'categories': ['Music'],
1353                 'age_limit': 0,
1354                 'alt_title': 'The Spark',
1355                 'channel_follower_count': int
1356             },
1357             'params': {
1358                 'youtube_include_dash_manifest': True,
1359                 'format': '141/bestaudio[ext=m4a]',
1360             },
1361         },
1362         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1363         {
1364             'note': 'Embed allowed age-gate video',
1365             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1366             'info_dict': {
1367                 'id': 'HtVdAasjOgU',
1368                 'ext': 'mp4',
1369                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1370                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1371                 'duration': 142,
1372                 'uploader': 'The Witcher',
1373                 'uploader_id': 'WitcherGame',
1374                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1375                 'upload_date': '20140605',
1376                 'age_limit': 18,
1377                 'categories': ['Gaming'],
1378                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1379                 'availability': 'needs_auth',
1380                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1381                 'like_count': int,
1382                 'channel': 'The Witcher',
1383                 'live_status': 'not_live',
1384                 'tags': 'count:17',
1385                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1386                 'playable_in_embed': True,
1387                 'view_count': int,
1388                 'channel_follower_count': int
1389             },
1390         },
1391         {
1392             'note': 'Age-gate video with embed allowed in public site',
1393             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1394             'info_dict': {
1395                 'id': 'HsUATh_Nc2U',
1396                 'ext': 'mp4',
1397                 'title': 'Godzilla 2 (Official Video)',
1398                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1399                 'upload_date': '20200408',
1400                 'uploader_id': 'FlyingKitty900',
1401                 'uploader': 'FlyingKitty',
1402                 'age_limit': 18,
1403                 'availability': 'needs_auth',
1404                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1405                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1406                 'channel': 'FlyingKitty',
1407                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1408                 'view_count': int,
1409                 'categories': ['Entertainment'],
1410                 'live_status': 'not_live',
1411                 'tags': ['Flyingkitty', 'godzilla 2'],
1412                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1413                 'like_count': int,
1414                 'duration': 177,
1415                 'playable_in_embed': True,
1416                 'channel_follower_count': int
1417             },
1418         },
1419         {
1420             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1421             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1422             'info_dict': {
1423                 'id': 'Tq92D6wQ1mg',
1424                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1425                 'ext': 'mp4',
1426                 'upload_date': '20191228',
1427                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1428                 'uploader': 'Projekt Melody',
1429                 'description': 'md5:17eccca93a786d51bc67646756894066',
1430                 'age_limit': 18,
1431                 'like_count': int,
1432                 'availability': 'needs_auth',
1433                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1434                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1435                 'view_count': int,
1436                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1437                 'channel': 'Projekt Melody',
1438                 'live_status': 'not_live',
1439                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1440                 'playable_in_embed': True,
1441                 'categories': ['Entertainment'],
1442                 'duration': 106,
1443                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1444                 'comment_count': int,
1445                 'channel_follower_count': int
1446             },
1447         },
1448         {
1449             'note': 'Non-Agegated non-embeddable video',
1450             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1451             'info_dict': {
1452                 'id': 'MeJVWBSsPAY',
1453                 'ext': 'mp4',
1454                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1455                 'uploader': 'Herr Lurik',
1456                 'uploader_id': 'st3in234',
1457                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1458                 'upload_date': '20130730',
1459                 'track': 'Such mich find mich',
1460                 'age_limit': 0,
1461                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1462                 'like_count': int,
1463                 'playable_in_embed': False,
1464                 'creator': 'OOMPH!',
1465                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1466                 'view_count': int,
1467                 'alt_title': 'Such mich find mich',
1468                 'duration': 210,
1469                 'channel': 'Herr Lurik',
1470                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1471                 'categories': ['Music'],
1472                 'availability': 'public',
1473                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1474                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1475                 'live_status': 'not_live',
1476                 'artist': 'OOMPH!',
1477                 'channel_follower_count': int
1478             },
1479         },
1480         {
1481             'note': 'Non-bypassable age-gated video',
1482             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1483             'only_matching': True,
1484         },
1485         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1486         # YouTube Red ad is not captured for creator
1487         {
1488             'url': '__2ABJjxzNo',
1489             'info_dict': {
1490                 'id': '__2ABJjxzNo',
1491                 'ext': 'mp4',
1492                 'duration': 266,
1493                 'upload_date': '20100430',
1494                 'uploader_id': 'deadmau5',
1495                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1496                 'creator': 'deadmau5',
1497                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1498                 'uploader': 'deadmau5',
1499                 'title': 'Deadmau5 - Some Chords (HD)',
1500                 'alt_title': 'Some Chords',
1501                 'availability': 'public',
1502                 'tags': 'count:14',
1503                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1504                 'view_count': int,
1505                 'live_status': 'not_live',
1506                 'channel': 'deadmau5',
1507                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1508                 'like_count': int,
1509                 'track': 'Some Chords',
1510                 'artist': 'deadmau5',
1511                 'playable_in_embed': True,
1512                 'age_limit': 0,
1513                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1514                 'categories': ['Music'],
1515                 'album': 'Some Chords',
1516                 'channel_follower_count': int
1517             },
1518             'expected_warnings': [
1519                 'DASH manifest missing',
1520             ]
1521         },
1522         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1523         {
1524             'url': 'lqQg6PlCWgI',
1525             'info_dict': {
1526                 'id': 'lqQg6PlCWgI',
1527                 'ext': 'mp4',
1528                 'duration': 6085,
1529                 'upload_date': '20150827',
1530                 'uploader_id': 'olympic',
1531                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1532                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1533                 'uploader': 'Olympics',
1534                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1535                 'like_count': int,
1536                 'release_timestamp': 1343767800,
1537                 'playable_in_embed': True,
1538                 'categories': ['Sports'],
1539                 'release_date': '20120731',
1540                 'channel': 'Olympics',
1541                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1542                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1543                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1544                 'age_limit': 0,
1545                 'availability': 'public',
1546                 'live_status': 'was_live',
1547                 'view_count': int,
1548                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1549                 'channel_follower_count': int
1550             },
1551             'params': {
1552                 'skip_download': 'requires avconv',
1553             }
1554         },
1555         # Non-square pixels
1556         {
1557             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1558             'info_dict': {
1559                 'id': '_b-2C3KPAM0',
1560                 'ext': 'mp4',
1561                 'stretched_ratio': 16 / 9.,
1562                 'duration': 85,
1563                 'upload_date': '20110310',
1564                 'uploader_id': 'AllenMeow',
1565                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1566                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1567                 'uploader': '孫ᄋᄅ',
1568                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1569                 'playable_in_embed': True,
1570                 'channel': '孫ᄋᄅ',
1571                 'age_limit': 0,
1572                 'tags': 'count:11',
1573                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1574                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1575                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1576                 'view_count': int,
1577                 'categories': ['People & Blogs'],
1578                 'like_count': int,
1579                 'live_status': 'not_live',
1580                 'availability': 'unlisted',
1581                 'comment_count': int,
1582                 'channel_follower_count': int
1583             },
1584         },
1585         # url_encoded_fmt_stream_map is empty string
1586         {
1587             'url': 'qEJwOuvDf7I',
1588             'info_dict': {
1589                 'id': 'qEJwOuvDf7I',
1590                 'ext': 'webm',
1591                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1592                 'description': '',
1593                 'upload_date': '20150404',
1594                 'uploader_id': 'spbelect',
1595                 'uploader': 'Наблюдатели Петербурга',
1596             },
1597             'params': {
1598                 'skip_download': 'requires avconv',
1599             },
1600             'skip': 'This live event has ended.',
1601         },
1602         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1603         {
1604             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1605             'info_dict': {
1606                 'id': 'FIl7x6_3R5Y',
1607                 'ext': 'webm',
1608                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1609                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1610                 'duration': 220,
1611                 'upload_date': '20150625',
1612                 'uploader_id': 'dorappi2000',
1613                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1614                 'uploader': 'dorappi2000',
1615                 'formats': 'mincount:31',
1616             },
1617             'skip': 'not actual anymore',
1618         },
1619         # DASH manifest with segment_list
1620         {
1621             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1622             'md5': '8ce563a1d667b599d21064e982ab9e31',
1623             'info_dict': {
1624                 'id': 'CsmdDsKjzN8',
1625                 'ext': 'mp4',
1626                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1627                 'uploader': 'Airtek',
1628                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1629                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1630                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1631             },
1632             'params': {
1633                 'youtube_include_dash_manifest': True,
1634                 'format': '135',  # bestvideo
1635             },
1636             'skip': 'This live event has ended.',
1637         },
1638         {
1639             # Multifeed videos (multiple cameras), URL can be of any Camera
1640             'url': 'https://www.youtube.com/watch?v=zaPI8MvL8pg',
1641             'info_dict': {
1642                 'id': 'zaPI8MvL8pg',
1643                 'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04',
1644                 'description': 'md5:563ccbc698b39298481ca3c571169519',
1645             },
1646             'playlist': [{
1647                 'info_dict': {
1648                     'id': 'j5yGuxZ8lLU',
1649                     'ext': 'mp4',
1650                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Chris)',
1651                     'uploader': 'WiiLikeToPlay',
1652                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1653                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1654                     'duration': 10120,
1655                     'channel_follower_count': int,
1656                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1657                     'availability': 'public',
1658                     'playable_in_embed': True,
1659                     'upload_date': '20131105',
1660                     'uploader_id': 'WiiRikeToPray',
1661                     'categories': ['Gaming'],
1662                     'live_status': 'was_live',
1663                     'tags': 'count:24',
1664                     'release_timestamp': 1383701910,
1665                     'thumbnail': 'https://i.ytimg.com/vi/j5yGuxZ8lLU/maxresdefault.jpg',
1666                     'comment_count': int,
1667                     'age_limit': 0,
1668                     'like_count': int,
1669                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1670                     'channel': 'WiiLikeToPlay',
1671                     'view_count': int,
1672                     'release_date': '20131106',
1673                 },
1674             }, {
1675                 'info_dict': {
1676                     'id': 'zaPI8MvL8pg',
1677                     'ext': 'mp4',
1678                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Tyson)',
1679                     'uploader_id': 'WiiRikeToPray',
1680                     'availability': 'public',
1681                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1682                     'channel': 'WiiLikeToPlay',
1683                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1684                     'channel_follower_count': int,
1685                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1686                     'duration': 10108,
1687                     'age_limit': 0,
1688                     'like_count': int,
1689                     'tags': 'count:24',
1690                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1691                     'uploader': 'WiiLikeToPlay',
1692                     'release_timestamp': 1383701915,
1693                     'comment_count': int,
1694                     'upload_date': '20131105',
1695                     'thumbnail': 'https://i.ytimg.com/vi/zaPI8MvL8pg/maxresdefault.jpg',
1696                     'release_date': '20131106',
1697                     'playable_in_embed': True,
1698                     'live_status': 'was_live',
1699                     'categories': ['Gaming'],
1700                     'view_count': int,
1701                 },
1702             }, {
1703                 'info_dict': {
1704                     'id': 'R7r3vfO7Hao',
1705                     'ext': 'mp4',
1706                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Spencer)',
1707                     'thumbnail': 'https://i.ytimg.com/vi/R7r3vfO7Hao/maxresdefault.jpg',
1708                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1709                     'like_count': int,
1710                     'availability': 'public',
1711                     'playable_in_embed': True,
1712                     'upload_date': '20131105',
1713                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1714                     'uploader_id': 'WiiRikeToPray',
1715                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1716                     'channel_follower_count': int,
1717                     'tags': 'count:24',
1718                     'release_date': '20131106',
1719                     'uploader': 'WiiLikeToPlay',
1720                     'comment_count': int,
1721                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1722                     'channel': 'WiiLikeToPlay',
1723                     'categories': ['Gaming'],
1724                     'release_timestamp': 1383701914,
1725                     'live_status': 'was_live',
1726                     'age_limit': 0,
1727                     'duration': 10128,
1728                     'view_count': int,
1729                 },
1730             }],
1731             'params': {'skip_download': True},
1732         },
1733         {
1734             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1735             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1736             'info_dict': {
1737                 'id': 'gVfLd0zydlo',
1738                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1739             },
1740             'playlist_count': 2,
1741             'skip': 'Not multifeed anymore',
1742         },
1743         {
1744             'url': 'https://vid.plus/FlRa-iH7PGw',
1745             'only_matching': True,
1746         },
1747         {
1748             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1749             'only_matching': True,
1750         },
1751         {
1752             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1753             # Also tests cut-off URL expansion in video description (see
1754             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1755             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1756             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1757             'info_dict': {
1758                 'id': 'lsguqyKfVQg',
1759                 'ext': 'mp4',
1760                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1761                 'alt_title': 'Dark Walk',
1762                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1763                 'duration': 133,
1764                 'upload_date': '20151119',
1765                 'uploader_id': 'IronSoulElf',
1766                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1767                 'uploader': 'IronSoulElf',
1768                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1769                 'track': 'Dark Walk',
1770                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1771                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1772                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1773                 'categories': ['Film & Animation'],
1774                 'view_count': int,
1775                 'live_status': 'not_live',
1776                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1777                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1778                 'tags': 'count:13',
1779                 'availability': 'public',
1780                 'channel': 'IronSoulElf',
1781                 'playable_in_embed': True,
1782                 'like_count': int,
1783                 'age_limit': 0,
1784                 'channel_follower_count': int
1785             },
1786             'params': {
1787                 'skip_download': True,
1788             },
1789         },
1790         {
1791             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1792             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1793             'only_matching': True,
1794         },
1795         {
1796             # Video with yt:stretch=17:0
1797             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1798             'info_dict': {
1799                 'id': 'Q39EVAstoRM',
1800                 'ext': 'mp4',
1801                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1802                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1803                 'upload_date': '20151107',
1804                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1805                 'uploader': 'CH GAMER DROID',
1806             },
1807             'params': {
1808                 'skip_download': True,
1809             },
1810             'skip': 'This video does not exist.',
1811         },
1812         {
1813             # Video with incomplete 'yt:stretch=16:'
1814             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1815             'only_matching': True,
1816         },
1817         {
1818             # Video licensed under Creative Commons
1819             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1820             'info_dict': {
1821                 'id': 'M4gD1WSo5mA',
1822                 'ext': 'mp4',
1823                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1824                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1825                 'duration': 721,
1826                 'upload_date': '20150128',
1827                 'uploader_id': 'BerkmanCenter',
1828                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1829                 'uploader': 'The Berkman Klein Center for Internet & Society',
1830                 'license': 'Creative Commons Attribution license (reuse allowed)',
1831                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1832                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1833                 'like_count': int,
1834                 'age_limit': 0,
1835                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1836                 'channel': 'The Berkman Klein Center for Internet & Society',
1837                 'availability': 'public',
1838                 'view_count': int,
1839                 'categories': ['Education'],
1840                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1841                 'live_status': 'not_live',
1842                 'playable_in_embed': True,
1843                 'comment_count': int,
1844                 'channel_follower_count': int,
1845                 'chapters': list,
1846             },
1847             'params': {
1848                 'skip_download': True,
1849             },
1850         },
1851         {
1852             # Channel-like uploader_url
1853             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1854             'info_dict': {
1855                 'id': 'eQcmzGIKrzg',
1856                 'ext': 'mp4',
1857                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1858                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1859                 'duration': 4060,
1860                 'upload_date': '20151120',
1861                 'uploader': 'Bernie Sanders',
1862                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1863                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1864                 'license': 'Creative Commons Attribution license (reuse allowed)',
1865                 'playable_in_embed': True,
1866                 'tags': 'count:12',
1867                 'like_count': int,
1868                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1869                 'age_limit': 0,
1870                 'availability': 'public',
1871                 'categories': ['News & Politics'],
1872                 'channel': 'Bernie Sanders',
1873                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1874                 'view_count': int,
1875                 'live_status': 'not_live',
1876                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1877                 'comment_count': int,
1878                 'channel_follower_count': int,
1879                 'chapters': list,
1880             },
1881             'params': {
1882                 'skip_download': True,
1883             },
1884         },
1885         {
1886             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1887             'only_matching': True,
1888         },
1889         {
1890             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1891             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1892             'only_matching': True,
1893         },
1894         {
1895             # Rental video preview
1896             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1897             'info_dict': {
1898                 'id': 'uGpuVWrhIzE',
1899                 'ext': 'mp4',
1900                 'title': 'Piku - Trailer',
1901                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1902                 'upload_date': '20150811',
1903                 'uploader': 'FlixMatrix',
1904                 'uploader_id': 'FlixMatrixKaravan',
1905                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1906                 'license': 'Standard YouTube License',
1907             },
1908             'params': {
1909                 'skip_download': True,
1910             },
1911             'skip': 'This video is not available.',
1912         },
1913         {
1914             # YouTube Red video with episode data
1915             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1916             'info_dict': {
1917                 'id': 'iqKdEhx-dD4',
1918                 'ext': 'mp4',
1919                 'title': 'Isolation - Mind Field (Ep 1)',
1920                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1921                 'duration': 2085,
1922                 'upload_date': '20170118',
1923                 'uploader': 'Vsauce',
1924                 'uploader_id': 'Vsauce',
1925                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1926                 'series': 'Mind Field',
1927                 'season_number': 1,
1928                 'episode_number': 1,
1929                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1930                 'tags': 'count:12',
1931                 'view_count': int,
1932                 'availability': 'public',
1933                 'age_limit': 0,
1934                 'channel': 'Vsauce',
1935                 'episode': 'Episode 1',
1936                 'categories': ['Entertainment'],
1937                 'season': 'Season 1',
1938                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1939                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1940                 'like_count': int,
1941                 'playable_in_embed': True,
1942                 'live_status': 'not_live',
1943                 'channel_follower_count': int
1944             },
1945             'params': {
1946                 'skip_download': True,
1947             },
1948             'expected_warnings': [
1949                 'Skipping DASH manifest',
1950             ],
1951         },
1952         {
1953             # The following content has been identified by the YouTube community
1954             # as inappropriate or offensive to some audiences.
1955             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1956             'info_dict': {
1957                 'id': '6SJNVb0GnPI',
1958                 'ext': 'mp4',
1959                 'title': 'Race Differences in Intelligence',
1960                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1961                 'duration': 965,
1962                 'upload_date': '20140124',
1963                 'uploader': 'New Century Foundation',
1964                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1965                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1966             },
1967             'params': {
1968                 'skip_download': True,
1969             },
1970             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1971         },
1972         {
1973             # itag 212
1974             'url': '1t24XAntNCY',
1975             'only_matching': True,
1976         },
1977         {
1978             # geo restricted to JP
1979             'url': 'sJL6WA-aGkQ',
1980             'only_matching': True,
1981         },
1982         {
1983             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1984             'only_matching': True,
1985         },
1986         {
1987             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1988             'only_matching': True,
1989         },
1990         {
1991             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1992             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1993             'only_matching': True,
1994         },
1995         {
1996             # DRM protected
1997             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1998             'only_matching': True,
1999         },
2000         {
2001             # Video with unsupported adaptive stream type formats
2002             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
2003             'info_dict': {
2004                 'id': 'Z4Vy8R84T1U',
2005                 'ext': 'mp4',
2006                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
2007                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
2008                 'duration': 433,
2009                 'upload_date': '20130923',
2010                 'uploader': 'Amelia Putri Harwita',
2011                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
2012                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
2013                 'formats': 'maxcount:10',
2014             },
2015             'params': {
2016                 'skip_download': True,
2017                 'youtube_include_dash_manifest': False,
2018             },
2019             'skip': 'not actual anymore',
2020         },
2021         {
2022             # Youtube Music Auto-generated description
2023             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2024             'info_dict': {
2025                 'id': 'MgNrAu2pzNs',
2026                 'ext': 'mp4',
2027                 'title': 'Voyeur Girl',
2028                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
2029                 'upload_date': '20190312',
2030                 'uploader': 'Stephen - Topic',
2031                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
2032                 'artist': 'Stephen',
2033                 'track': 'Voyeur Girl',
2034                 'album': 'it\'s too much love to know my dear',
2035                 'release_date': '20190313',
2036                 'release_year': 2019,
2037                 'alt_title': 'Voyeur Girl',
2038                 'view_count': int,
2039                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
2040                 'playable_in_embed': True,
2041                 'like_count': int,
2042                 'categories': ['Music'],
2043                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
2044                 'channel': 'Stephen',
2045                 'availability': 'public',
2046                 'creator': 'Stephen',
2047                 'duration': 169,
2048                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
2049                 'age_limit': 0,
2050                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
2051                 'tags': 'count:11',
2052                 'live_status': 'not_live',
2053                 'channel_follower_count': int
2054             },
2055             'params': {
2056                 'skip_download': True,
2057             },
2058         },
2059         {
2060             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
2061             'only_matching': True,
2062         },
2063         {
2064             # invalid -> valid video id redirection
2065             'url': 'DJztXj2GPfl',
2066             'info_dict': {
2067                 'id': 'DJztXj2GPfk',
2068                 'ext': 'mp4',
2069                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
2070                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
2071                 'upload_date': '20090125',
2072                 'uploader': 'Prochorowka',
2073                 'uploader_id': 'Prochorowka',
2074                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
2075                 'artist': 'Panjabi MC',
2076                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
2077                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
2078             },
2079             'params': {
2080                 'skip_download': True,
2081             },
2082             'skip': 'Video unavailable',
2083         },
2084         {
2085             # empty description results in an empty string
2086             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
2087             'info_dict': {
2088                 'id': 'x41yOUIvK2k',
2089                 'ext': 'mp4',
2090                 'title': 'IMG 3456',
2091                 'description': '',
2092                 'upload_date': '20170613',
2093                 'uploader_id': 'ElevageOrVert',
2094                 'uploader': 'ElevageOrVert',
2095                 'view_count': int,
2096                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
2097                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
2098                 'like_count': int,
2099                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
2100                 'tags': [],
2101                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
2102                 'availability': 'public',
2103                 'age_limit': 0,
2104                 'categories': ['Pets & Animals'],
2105                 'duration': 7,
2106                 'playable_in_embed': True,
2107                 'live_status': 'not_live',
2108                 'channel': 'ElevageOrVert',
2109                 'channel_follower_count': int
2110             },
2111             'params': {
2112                 'skip_download': True,
2113             },
2114         },
2115         {
2116             # with '};' inside yt initial data (see [1])
2117             # see [2] for an example with '};' inside ytInitialPlayerResponse
2118             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
2119             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2120             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
2121             'info_dict': {
2122                 'id': 'CHqg6qOn4no',
2123                 'ext': 'mp4',
2124                 'title': 'Part 77   Sort a list of simple types in c#',
2125                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2126                 'upload_date': '20130831',
2127                 'uploader_id': 'kudvenkat',
2128                 'uploader': 'kudvenkat',
2129                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2130                 'like_count': int,
2131                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
2132                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2133                 'live_status': 'not_live',
2134                 'categories': ['Education'],
2135                 'availability': 'public',
2136                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2137                 'tags': 'count:12',
2138                 'playable_in_embed': True,
2139                 'age_limit': 0,
2140                 'view_count': int,
2141                 'duration': 522,
2142                 'channel': 'kudvenkat',
2143                 'comment_count': int,
2144                 'channel_follower_count': int,
2145                 'chapters': list,
2146             },
2147             'params': {
2148                 'skip_download': True,
2149             },
2150         },
2151         {
2152             # another example of '};' in ytInitialData
2153             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2154             'only_matching': True,
2155         },
2156         {
2157             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2158             'only_matching': True,
2159         },
2160         {
2161             # https://github.com/ytdl-org/youtube-dl/pull/28094
2162             'url': 'OtqTfy26tG0',
2163             'info_dict': {
2164                 'id': 'OtqTfy26tG0',
2165                 'ext': 'mp4',
2166                 'title': 'Burn Out',
2167                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2168                 'upload_date': '20141120',
2169                 'uploader': 'The Cinematic Orchestra - Topic',
2170                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2171                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2172                 'artist': 'The Cinematic Orchestra',
2173                 'track': 'Burn Out',
2174                 'album': 'Every Day',
2175                 'like_count': int,
2176                 'live_status': 'not_live',
2177                 'alt_title': 'Burn Out',
2178                 'duration': 614,
2179                 'age_limit': 0,
2180                 'view_count': int,
2181                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2182                 'creator': 'The Cinematic Orchestra',
2183                 'channel': 'The Cinematic Orchestra',
2184                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2185                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2186                 'availability': 'public',
2187                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2188                 'categories': ['Music'],
2189                 'playable_in_embed': True,
2190                 'channel_follower_count': int
2191             },
2192             'params': {
2193                 'skip_download': True,
2194             },
2195         },
2196         {
2197             # controversial video, only works with bpctr when authenticated with cookies
2198             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2199             'only_matching': True,
2200         },
2201         {
2202             # controversial video, requires bpctr/contentCheckOk
2203             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2204             'info_dict': {
2205                 'id': 'SZJvDhaSDnc',
2206                 'ext': 'mp4',
2207                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2208                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2209                 'uploader': 'CBS Mornings',
2210                 'uploader_id': 'CBSThisMorning',
2211                 'upload_date': '20140716',
2212                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2213                 'duration': 170,
2214                 'categories': ['News & Politics'],
2215                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2216                 'view_count': int,
2217                 'channel': 'CBS Mornings',
2218                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2219                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2220                 'age_limit': 18,
2221                 'availability': 'needs_auth',
2222                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2223                 'like_count': int,
2224                 'live_status': 'not_live',
2225                 'playable_in_embed': True,
2226                 'channel_follower_count': int
2227             }
2228         },
2229         {
2230             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2231             'url': 'cBvYw8_A0vQ',
2232             'info_dict': {
2233                 'id': 'cBvYw8_A0vQ',
2234                 'ext': 'mp4',
2235                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2236                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2237                 'upload_date': '20201120',
2238                 'uploader': 'Walk around Japan',
2239                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2240                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2241                 'duration': 1456,
2242                 'categories': ['Travel & Events'],
2243                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2244                 'view_count': int,
2245                 'channel': 'Walk around Japan',
2246                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2247                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2248                 'age_limit': 0,
2249                 'availability': 'public',
2250                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2251                 'live_status': 'not_live',
2252                 'playable_in_embed': True,
2253                 'channel_follower_count': int
2254             },
2255             'params': {
2256                 'skip_download': True,
2257             },
2258         }, {
2259             # Has multiple audio streams
2260             'url': 'WaOKSUlf4TM',
2261             'only_matching': True
2262         }, {
2263             # Requires Premium: has format 141 when requested using YTM url
2264             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2265             'only_matching': True
2266         }, {
2267             # multiple subtitles with same lang_code
2268             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2269             'only_matching': True,
2270         }, {
2271             # Force use android client fallback
2272             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2273             'info_dict': {
2274                 'id': 'YOelRv7fMxY',
2275                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2276                 'ext': '3gp',
2277                 'upload_date': '20210624',
2278                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2279                 'uploader': 'colinfurze',
2280                 'uploader_id': 'colinfurze',
2281                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2282                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2283                 'duration': 596,
2284                 'categories': ['Entertainment'],
2285                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2286                 'view_count': int,
2287                 'channel': 'colinfurze',
2288                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2289                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2290                 'age_limit': 0,
2291                 'availability': 'public',
2292                 'like_count': int,
2293                 'live_status': 'not_live',
2294                 'playable_in_embed': True,
2295                 'channel_follower_count': int,
2296                 'chapters': list,
2297             },
2298             'params': {
2299                 'format': '17',  # 3gp format available on android
2300                 'extractor_args': {'youtube': {'player_client': ['android']}},
2301             },
2302         },
2303         {
2304             # Skip download of additional client configs (remix client config in this case)
2305             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2306             'only_matching': True,
2307             'params': {
2308                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2309             },
2310         }, {
2311             # shorts
2312             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2313             'only_matching': True,
2314         }, {
2315             'note': 'Storyboards',
2316             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2317             'info_dict': {
2318                 'id': '5KLPxDtMqe8',
2319                 'ext': 'mhtml',
2320                 'format_id': 'sb0',
2321                 'title': 'Your Brain is Plastic',
2322                 'uploader_id': 'scishow',
2323                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2324                 'upload_date': '20140324',
2325                 'uploader': 'SciShow',
2326                 'like_count': int,
2327                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2328                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2329                 'view_count': int,
2330                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2331                 'playable_in_embed': True,
2332                 'tags': 'count:12',
2333                 'uploader_url': 'http://www.youtube.com/user/scishow',
2334                 'availability': 'public',
2335                 'channel': 'SciShow',
2336                 'live_status': 'not_live',
2337                 'duration': 248,
2338                 'categories': ['Education'],
2339                 'age_limit': 0,
2340                 'channel_follower_count': int,
2341                 'chapters': list,
2342             }, 'params': {'format': 'mhtml', 'skip_download': True}
2343         }, {
2344             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2345             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2346             'info_dict': {
2347                 'id': '2NUZ8W2llS4',
2348                 'ext': 'mp4',
2349                 'title': 'The NP that test your phone performance 🙂',
2350                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2351                 'uploader': 'Leon Nguyen',
2352                 'uploader_id': 'VNSXIII',
2353                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2354                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2355                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2356                 'duration': 21,
2357                 'view_count': int,
2358                 'age_limit': 0,
2359                 'categories': ['Gaming'],
2360                 'tags': 'count:23',
2361                 'playable_in_embed': True,
2362                 'live_status': 'not_live',
2363                 'upload_date': '20220103',
2364                 'like_count': int,
2365                 'availability': 'public',
2366                 'channel': 'Leon Nguyen',
2367                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2368                 'comment_count': int,
2369                 'channel_follower_count': int
2370             }
2371         }, {
2372             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2373             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2374             'info_dict': {
2375                 'id': '2NUZ8W2llS4',
2376                 'ext': 'mp4',
2377                 'title': 'The NP that test your phone performance 🙂',
2378                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2379                 'uploader': 'Leon Nguyen',
2380                 'uploader_id': 'VNSXIII',
2381                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2382                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2383                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2384                 'duration': 21,
2385                 'view_count': int,
2386                 'age_limit': 0,
2387                 'categories': ['Gaming'],
2388                 'tags': 'count:23',
2389                 'playable_in_embed': True,
2390                 'live_status': 'not_live',
2391                 'upload_date': '20220102',
2392                 'like_count': int,
2393                 'availability': 'public',
2394                 'channel': 'Leon Nguyen',
2395                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2396                 'comment_count': int,
2397                 'channel_follower_count': int
2398             },
2399             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2400         }, {
2401             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2402             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2403             'info_dict': {
2404                 'id': 'mzZzzBU6lrM',
2405                 'ext': 'mp4',
2406                 'title': 'I Met GeorgeNotFound In Real Life...',
2407                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2408                 'uploader': 'Quackity',
2409                 'uploader_id': 'QuackityHQ',
2410                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2411                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2412                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2413                 'duration': 955,
2414                 'view_count': int,
2415                 'age_limit': 0,
2416                 'categories': ['Entertainment'],
2417                 'tags': 'count:26',
2418                 'playable_in_embed': True,
2419                 'live_status': 'not_live',
2420                 'release_timestamp': 1641172509,
2421                 'release_date': '20220103',
2422                 'upload_date': '20220103',
2423                 'like_count': int,
2424                 'availability': 'public',
2425                 'channel': 'Quackity',
2426                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2427                 'channel_follower_count': int
2428             }
2429         },
2430         {   # continuous livestream. Microformat upload date should be preferred.
2431             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2432             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2433             'info_dict': {
2434                 'id': 'kgx4WGK0oNU',
2435                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2436                 'ext': 'mp4',
2437                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2438                 'availability': 'public',
2439                 'age_limit': 0,
2440                 'release_timestamp': 1637975704,
2441                 'upload_date': '20210619',
2442                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2443                 'live_status': 'is_live',
2444                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2445                 'uploader': '阿鲍Abao',
2446                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2447                 'channel': 'Abao in Tokyo',
2448                 'channel_follower_count': int,
2449                 'release_date': '20211127',
2450                 'tags': 'count:39',
2451                 'categories': ['People & Blogs'],
2452                 'like_count': int,
2453                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2454                 'view_count': int,
2455                 'playable_in_embed': True,
2456                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2457                 'concurrent_view_count': int,
2458             },
2459             'params': {'skip_download': True}
2460         }, {
2461             # Story. Requires specific player params to work.
2462             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2463             'info_dict': {
2464                 'id': 'vv8qTUWmulI',
2465                 'ext': 'mp4',
2466                 'availability': 'unlisted',
2467                 'view_count': int,
2468                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2469                 'upload_date': '20220526',
2470                 'categories': ['Education'],
2471                 'title': 'Story',
2472                 'channel': 'IT\'S HISTORY',
2473                 'description': '',
2474                 'uploader_id': 'BlastfromthePast',
2475                 'duration': 12,
2476                 'uploader': 'IT\'S HISTORY',
2477                 'playable_in_embed': True,
2478                 'age_limit': 0,
2479                 'live_status': 'not_live',
2480                 'tags': [],
2481                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2482                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2483                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2484             },
2485             'skip': 'stories get removed after some period of time',
2486         }, {
2487             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2488             'info_dict': {
2489                 'id': 'tjjjtzRLHvA',
2490                 'ext': 'mp4',
2491                 'title': 'ハッシュタグ無し };if window.ytcsi',
2492                 'upload_date': '20220323',
2493                 'like_count': int,
2494                 'availability': 'unlisted',
2495                 'channel': 'nao20010128nao',
2496                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2497                 'age_limit': 0,
2498                 'uploader': 'nao20010128nao',
2499                 'uploader_id': 'nao20010128nao',
2500                 'categories': ['Music'],
2501                 'view_count': int,
2502                 'description': '',
2503                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2504                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2505                 'live_status': 'not_live',
2506                 'playable_in_embed': True,
2507                 'channel_follower_count': int,
2508                 'duration': 6,
2509                 'tags': [],
2510                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2511             }
2512         }, {
2513             # Prefer primary title+description language metadata by default
2514             # Do not prefer translated description if primary is empty
2515             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2516             'info_dict': {
2517                 'id': 'el3E4MbxRqQ',
2518                 'ext': 'mp4',
2519                 'title': 'dlp test video 2 - primary sv no desc',
2520                 'description': '',
2521                 'channel': 'cole-dlp-test-acc',
2522                 'tags': [],
2523                 'view_count': int,
2524                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2525                 'like_count': int,
2526                 'playable_in_embed': True,
2527                 'availability': 'unlisted',
2528                 'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
2529                 'age_limit': 0,
2530                 'duration': 5,
2531                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2532                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2533                 'live_status': 'not_live',
2534                 'upload_date': '20220908',
2535                 'categories': ['People & Blogs'],
2536                 'uploader': 'cole-dlp-test-acc',
2537                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2538             },
2539             'params': {'skip_download': True}
2540         }, {
2541             # Extractor argument: prefer translated title+description
2542             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2543             'info_dict': {
2544                 'id': 'gHKT4uU8Zng',
2545                 'ext': 'mp4',
2546                 'channel': 'cole-dlp-test-acc',
2547                 'tags': [],
2548                 'duration': 5,
2549                 'live_status': 'not_live',
2550                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2551                 'upload_date': '20220728',
2552                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2553                 'view_count': int,
2554                 'categories': ['People & Blogs'],
2555                 'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
2556                 'title': 'dlp test video title translated (fr)',
2557                 'availability': 'public',
2558                 'uploader': 'cole-dlp-test-acc',
2559                 'age_limit': 0,
2560                 'description': 'dlp test video description translated (fr)',
2561                 'playable_in_embed': True,
2562                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2563                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2564             },
2565             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2566             'expected_warnings': [r'Preferring "fr" translated fields'],
2567         }, {
2568             'note': '6 channel audio',
2569             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2570             'only_matching': True,
2571         }, {
2572             'note': 'Multiple HLS formats with same itag',
2573             'url': 'https://www.youtube.com/watch?v=kX3nB4PpJko',
2574             'info_dict': {
2575                 'id': 'kX3nB4PpJko',
2576                 'ext': 'mp4',
2577                 'categories': ['Entertainment'],
2578                 'description': 'md5:e8031ff6e426cdb6a77670c9b81f6fa6',
2579                 'uploader_url': 'http://www.youtube.com/user/MrBeast6000',
2580                 'live_status': 'not_live',
2581                 'duration': 937,
2582                 'channel_follower_count': int,
2583                 'thumbnail': 'https://i.ytimg.com/vi_webp/kX3nB4PpJko/maxresdefault.webp',
2584                 'title': 'Last To Take Hand Off Jet, Keeps It!',
2585                 'channel': 'MrBeast',
2586                 'playable_in_embed': True,
2587                 'view_count': int,
2588                 'upload_date': '20221112',
2589                 'uploader': 'MrBeast',
2590                 'uploader_id': 'MrBeast6000',
2591                 'channel_url': 'https://www.youtube.com/channel/UCX6OQ3DkcsbYNE6H8uQQuVA',
2592                 'age_limit': 0,
2593                 'availability': 'public',
2594                 'channel_id': 'UCX6OQ3DkcsbYNE6H8uQQuVA',
2595                 'like_count': int,
2596                 'tags': [],
2597             },
2598             'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
2599         }, {
2600             'note': 'Audio formats with Dynamic Range Compression',
2601             'url': 'https://www.youtube.com/watch?v=Tq92D6wQ1mg',
2602             'info_dict': {
2603                 'id': 'Tq92D6wQ1mg',
2604                 'ext': 'weba',
2605                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
2606                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2607                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2608                 'channel_follower_count': int,
2609                 'description': 'md5:17eccca93a786d51bc67646756894066',
2610                 'upload_date': '20191228',
2611                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2612                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
2613                 'playable_in_embed': True,
2614                 'like_count': int,
2615                 'categories': ['Entertainment'],
2616                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
2617                 'age_limit': 18,
2618                 'channel': 'Projekt Melody',
2619                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2620                 'view_count': int,
2621                 'availability': 'needs_auth',
2622                 'comment_count': int,
2623                 'live_status': 'not_live',
2624                 'uploader': 'Projekt Melody',
2625                 'duration': 106,
2626             },
2627             'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
2628         },
2629         {
2630             'url': 'https://www.youtube.com/live/qVv6vCqciTM',
2631             'info_dict': {
2632                 'id': 'qVv6vCqciTM',
2633                 'ext': 'mp4',
2634                 'age_limit': 0,
2635                 'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
2636                 'comment_count': int,
2637                 'chapters': 'count:13',
2638                 'upload_date': '20221223',
2639                 'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
2640                 'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
2641                 'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
2642                 'like_count': int,
2643                 'release_date': '20221223',
2644                 'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
2645                 'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
2646                 'view_count': int,
2647                 'playable_in_embed': True,
2648                 'duration': 4438,
2649                 'availability': 'public',
2650                 'channel_follower_count': int,
2651                 'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
2652                 'categories': ['Entertainment'],
2653                 'live_status': 'was_live',
2654                 'release_timestamp': 1671793345,
2655                 'channel': 'さなちゃんねる',
2656                 'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
2657                 'uploader': 'さなちゃんねる',
2658             },
2659         },
2660     ]
2661
2662     _WEBPAGE_TESTS = [
2663         # YouTube <object> embed
2664         {
2665             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2666             'md5': '873c81d308b979f0e23ee7e620b312a3',
2667             'info_dict': {
2668                 'id': 'msN87y-iEx0',
2669                 'ext': 'mp4',
2670                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2671                 'upload_date': '20080526',
2672                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2673                 'uploader': 'Christopher Sykes',
2674                 'uploader_id': 'ChristopherJSykes',
2675                 'age_limit': 0,
2676                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2677                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2678                 'playable_in_embed': True,
2679                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2680                 'like_count': int,
2681                 'comment_count': int,
2682                 'channel': 'Christopher Sykes',
2683                 'live_status': 'not_live',
2684                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2685                 'availability': 'public',
2686                 'duration': 195,
2687                 'view_count': int,
2688                 'categories': ['Science & Technology'],
2689                 'channel_follower_count': int,
2690                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2691             },
2692             'params': {
2693                 'skip_download': True,
2694             }
2695         },
2696     ]
2697
2698     @classmethod
2699     def suitable(cls, url):
2700         from ..utils import parse_qs
2701
2702         qs = parse_qs(url)
2703         if qs.get('list', [None])[0]:
2704             return False
2705         return super().suitable(url)
2706
2707     def __init__(self, *args, **kwargs):
2708         super().__init__(*args, **kwargs)
2709         self._code_cache = {}
2710         self._player_cache = {}
2711
2712     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2713         lock = threading.Lock()
2714         start_time = time.time()
2715         formats = [f for f in formats if f.get('is_from_start')]
2716
2717         def refetch_manifest(format_id, delay):
2718             nonlocal formats, start_time, is_live
2719             if time.time() <= start_time + delay:
2720                 return
2721
2722             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2723             video_details = traverse_obj(prs, (..., 'videoDetails'), expected_type=dict)
2724             microformats = traverse_obj(
2725                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2726                 expected_type=dict)
2727             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2728             is_live = live_status == 'is_live'
2729             start_time = time.time()
2730
2731         def mpd_feed(format_id, delay):
2732             """
2733             @returns (manifest_url, manifest_stream_number, is_live) or None
2734             """
2735             for retry in self.RetryManager(fatal=False):
2736                 with lock:
2737                     refetch_manifest(format_id, delay)
2738
2739                 f = next((f for f in formats if f['format_id'] == format_id), None)
2740                 if not f:
2741                     if not is_live:
2742                         retry.error = f'{video_id}: Video is no longer live'
2743                     else:
2744                         retry.error = f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}'
2745                     continue
2746                 return f['manifest_url'], f['manifest_stream_number'], is_live
2747             return None
2748
2749         for f in formats:
2750             f['is_live'] = is_live
2751             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2752                                     live_start_time, mpd_feed, not is_live and f.copy())
2753             if is_live:
2754                 f['fragments'] = gen
2755                 f['protocol'] = 'http_dash_segments_generator'
2756             else:
2757                 f['fragments'] = LazyList(gen({}))
2758                 del f['is_from_start']
2759
2760     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2761         FETCH_SPAN, MAX_DURATION = 5, 432000
2762
2763         mpd_url, stream_number, is_live = None, None, True
2764
2765         begin_index = 0
2766         download_start_time = ctx.get('start') or time.time()
2767
2768         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2769         if lack_early_segments:
2770             self.report_warning(bug_reports_message(
2771                 'Starting download from the last 120 hours of the live stream since '
2772                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2773             lack_early_segments = True
2774
2775         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2776         fragments, fragment_base_url = None, None
2777
2778         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2779             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2780             # Obtain from MPD's maximum seq value
2781             old_mpd_url = mpd_url
2782             last_error = ctx.pop('last_error', None)
2783             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2784             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2785                                                or (mpd_url, stream_number, False))
2786             if not refresh_sequence:
2787                 if expire_fast and not is_live:
2788                     return False, last_seq
2789                 elif old_mpd_url == mpd_url:
2790                     return True, last_seq
2791             if manifestless_orig_fmt:
2792                 fmt_info = manifestless_orig_fmt
2793             else:
2794                 try:
2795                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2796                         mpd_url, None, note=False, errnote=False, fatal=False)
2797                 except ExtractorError:
2798                     fmts = None
2799                 if not fmts:
2800                     no_fragment_score += 2
2801                     return False, last_seq
2802                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2803             fragments = fmt_info['fragments']
2804             fragment_base_url = fmt_info['fragment_base_url']
2805             assert fragment_base_url
2806
2807             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2808             return True, _last_seq
2809
2810         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2811         while is_live:
2812             fetch_time = time.time()
2813             if no_fragment_score > 30:
2814                 return
2815             if last_segment_url:
2816                 # Obtain from "X-Head-Seqnum" header value from each segment
2817                 try:
2818                     urlh = self._request_webpage(
2819                         last_segment_url, None, note=False, errnote=False, fatal=False)
2820                 except ExtractorError:
2821                     urlh = None
2822                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2823                 if last_seq is None:
2824                     no_fragment_score += 2
2825                     last_segment_url = None
2826                     continue
2827             else:
2828                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2829                 no_fragment_score += 2
2830                 if not should_continue:
2831                     continue
2832
2833             if known_idx > last_seq:
2834                 last_segment_url = None
2835                 continue
2836
2837             last_seq += 1
2838
2839             if begin_index < 0 and known_idx < 0:
2840                 # skip from the start when it's negative value
2841                 known_idx = last_seq + begin_index
2842             if lack_early_segments:
2843                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2844             try:
2845                 for idx in range(known_idx, last_seq):
2846                     # do not update sequence here or you'll get skipped some part of it
2847                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2848                     if not should_continue:
2849                         known_idx = idx - 1
2850                         raise ExtractorError('breaking out of outer loop')
2851                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2852                     yield {
2853                         'url': last_segment_url,
2854                         'fragment_count': last_seq,
2855                     }
2856                 if known_idx == last_seq:
2857                     no_fragment_score += 5
2858                 else:
2859                     no_fragment_score = 0
2860                 known_idx = last_seq
2861             except ExtractorError:
2862                 continue
2863
2864             if manifestless_orig_fmt:
2865                 # Stop at the first iteration if running for post-live manifestless;
2866                 # fragment count no longer increase since it starts
2867                 break
2868
2869             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2870
2871     def _extract_player_url(self, *ytcfgs, webpage=None):
2872         player_url = traverse_obj(
2873             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2874             get_all=False, expected_type=str)
2875         if not player_url:
2876             return
2877         return urljoin('https://www.youtube.com', player_url)
2878
2879     def _download_player_url(self, video_id, fatal=False):
2880         res = self._download_webpage(
2881             'https://www.youtube.com/iframe_api',
2882             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2883         if res:
2884             player_version = self._search_regex(
2885                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2886             if player_version:
2887                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2888
2889     def _signature_cache_id(self, example_sig):
2890         """ Return a string representation of a signature """
2891         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2892
2893     @classmethod
2894     def _extract_player_info(cls, player_url):
2895         for player_re in cls._PLAYER_INFO_RE:
2896             id_m = re.search(player_re, player_url)
2897             if id_m:
2898                 break
2899         else:
2900             raise ExtractorError('Cannot identify player %r' % player_url)
2901         return id_m.group('id')
2902
2903     def _load_player(self, video_id, player_url, fatal=True):
2904         player_id = self._extract_player_info(player_url)
2905         if player_id not in self._code_cache:
2906             code = self._download_webpage(
2907                 player_url, video_id, fatal=fatal,
2908                 note='Downloading player ' + player_id,
2909                 errnote='Download of %s failed' % player_url)
2910             if code:
2911                 self._code_cache[player_id] = code
2912         return self._code_cache.get(player_id)
2913
2914     def _extract_signature_function(self, video_id, player_url, example_sig):
2915         player_id = self._extract_player_info(player_url)
2916
2917         # Read from filesystem cache
2918         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2919         assert os.path.basename(func_id) == func_id
2920
2921         self.write_debug(f'Extracting signature function {func_id}')
2922         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2923
2924         if not cache_spec:
2925             code = self._load_player(video_id, player_url)
2926         if code:
2927             res = self._parse_sig_js(code)
2928             test_string = ''.join(map(chr, range(len(example_sig))))
2929             cache_spec = [ord(c) for c in res(test_string)]
2930             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2931
2932         return lambda s: ''.join(s[i] for i in cache_spec)
2933
2934     def _print_sig_code(self, func, example_sig):
2935         if not self.get_param('youtube_print_sig_code'):
2936             return
2937
2938         def gen_sig_code(idxs):
2939             def _genslice(start, end, step):
2940                 starts = '' if start == 0 else str(start)
2941                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2942                 steps = '' if step == 1 else (':%d' % step)
2943                 return f's[{starts}{ends}{steps}]'
2944
2945             step = None
2946             # Quelch pyflakes warnings - start will be set when step is set
2947             start = '(Never used)'
2948             for i, prev in zip(idxs[1:], idxs[:-1]):
2949                 if step is not None:
2950                     if i - prev == step:
2951                         continue
2952                     yield _genslice(start, prev, step)
2953                     step = None
2954                     continue
2955                 if i - prev in [-1, 1]:
2956                     step = i - prev
2957                     start = prev
2958                     continue
2959                 else:
2960                     yield 's[%d]' % prev
2961             if step is None:
2962                 yield 's[%d]' % i
2963             else:
2964                 yield _genslice(start, i, step)
2965
2966         test_string = ''.join(map(chr, range(len(example_sig))))
2967         cache_res = func(test_string)
2968         cache_spec = [ord(c) for c in cache_res]
2969         expr_code = ' + '.join(gen_sig_code(cache_spec))
2970         signature_id_tuple = '(%s)' % (
2971             ', '.join(str(len(p)) for p in example_sig.split('.')))
2972         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2973                 '    return %s\n') % (signature_id_tuple, expr_code)
2974         self.to_screen('Extracted signature function:\n' + code)
2975
2976     def _parse_sig_js(self, jscode):
2977         funcname = self._search_regex(
2978             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2979              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2980              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2981              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2982              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2983              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2984              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2985              # Obsolete patterns
2986              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2987              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2988              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2989              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2990              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2991              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2992              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2993              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2994             jscode, 'Initial JS player signature function name', group='sig')
2995
2996         jsi = JSInterpreter(jscode)
2997         initial_function = jsi.extract_function(funcname)
2998         return lambda s: initial_function([s])
2999
3000     def _cached(self, func, *cache_id):
3001         def inner(*args, **kwargs):
3002             if cache_id not in self._player_cache:
3003                 try:
3004                     self._player_cache[cache_id] = func(*args, **kwargs)
3005                 except ExtractorError as e:
3006                     self._player_cache[cache_id] = e
3007                 except Exception as e:
3008                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
3009
3010             ret = self._player_cache[cache_id]
3011             if isinstance(ret, Exception):
3012                 raise ret
3013             return ret
3014         return inner
3015
3016     def _decrypt_signature(self, s, video_id, player_url):
3017         """Turn the encrypted s field into a working signature"""
3018         extract_sig = self._cached(
3019             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
3020         func = extract_sig(video_id, player_url, s)
3021         self._print_sig_code(func, s)
3022         return func(s)
3023
3024     def _decrypt_nsig(self, s, video_id, player_url):
3025         """Turn the encrypted n field into a working signature"""
3026         if player_url is None:
3027             raise ExtractorError('Cannot decrypt nsig without player_url')
3028         player_url = urljoin('https://www.youtube.com', player_url)
3029
3030         try:
3031             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
3032         except ExtractorError as e:
3033             raise ExtractorError('Unable to extract nsig function code', cause=e)
3034         if self.get_param('youtube_print_sig_code'):
3035             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
3036
3037         try:
3038             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
3039             ret = extract_nsig(jsi, func_code)(s)
3040         except JSInterpreter.Exception as e:
3041             try:
3042                 jsi = PhantomJSwrapper(self, timeout=5000)
3043             except ExtractorError:
3044                 raise e
3045             self.report_warning(
3046                 f'Native nsig extraction failed: Trying with PhantomJS\n'
3047                 f'         n = {s} ; player = {player_url}', video_id)
3048             self.write_debug(e, only_once=True)
3049
3050             args, func_body = func_code
3051             ret = jsi.execute(
3052                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
3053                 video_id=video_id, note='Executing signature code').strip()
3054
3055         self.write_debug(f'Decrypted nsig {s} => {ret}')
3056         return ret
3057
3058     def _extract_n_function_name(self, jscode):
3059         funcname, idx = self._search_regex(
3060             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
3061             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
3062         if not idx:
3063             return funcname
3064
3065         return json.loads(js_to_json(self._search_regex(
3066             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
3067             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
3068
3069     def _extract_n_function_code(self, video_id, player_url):
3070         player_id = self._extract_player_info(player_url)
3071         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
3072         jscode = func_code or self._load_player(video_id, player_url)
3073         jsi = JSInterpreter(jscode)
3074
3075         if func_code:
3076             return jsi, player_id, func_code
3077
3078         func_name = self._extract_n_function_name(jscode)
3079
3080         # For redundancy
3081         func_code = self._search_regex(
3082             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
3083                      # NB: The end of the regex is intentionally kept strict
3084                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
3085             jscode, 'nsig function', group=('var', 'code'), default=None)
3086         if func_code:
3087             func_code = ([func_code[0]], func_code[1])
3088         else:
3089             self.write_debug('Extracting nsig function with jsinterp')
3090             func_code = jsi.extract_function_code(func_name)
3091
3092         self.cache.store('youtube-nsig', player_id, func_code)
3093         return jsi, player_id, func_code
3094
3095     def _extract_n_function_from_code(self, jsi, func_code):
3096         func = jsi.extract_function_from_code(*func_code)
3097
3098         def extract_nsig(s):
3099             try:
3100                 ret = func([s])
3101             except JSInterpreter.Exception:
3102                 raise
3103             except Exception as e:
3104                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
3105
3106             if ret.startswith('enhanced_except_'):
3107                 raise JSInterpreter.Exception('Signature function returned an exception')
3108             return ret
3109
3110         return extract_nsig
3111
3112     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
3113         """
3114         Extract signatureTimestamp (sts)
3115         Required to tell API what sig/player version is in use.
3116         """
3117         sts = None
3118         if isinstance(ytcfg, dict):
3119             sts = int_or_none(ytcfg.get('STS'))
3120
3121         if not sts:
3122             # Attempt to extract from player
3123             if player_url is None:
3124                 error_msg = 'Cannot extract signature timestamp without player_url.'
3125                 if fatal:
3126                     raise ExtractorError(error_msg)
3127                 self.report_warning(error_msg)
3128                 return
3129             code = self._load_player(video_id, player_url, fatal=fatal)
3130             if code:
3131                 sts = int_or_none(self._search_regex(
3132                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
3133                     'JS player signature timestamp', group='sts', fatal=fatal))
3134         return sts
3135
3136     def _mark_watched(self, video_id, player_responses):
3137         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
3138             label = 'fully ' if is_full else ''
3139             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
3140                             expected_type=url_or_none)
3141             if not url:
3142                 self.report_warning(f'Unable to mark {label}watched')
3143                 return
3144             parsed_url = urllib.parse.urlparse(url)
3145             qs = urllib.parse.parse_qs(parsed_url.query)
3146
3147             # cpn generation algorithm is reverse engineered from base.js.
3148             # In fact it works even with dummy cpn.
3149             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
3150             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
3151
3152             # # more consistent results setting it to right before the end
3153             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
3154
3155             qs.update({
3156                 'ver': ['2'],
3157                 'cpn': [cpn],
3158                 'cmt': video_length,
3159                 'el': 'detailpage',  # otherwise defaults to "shorts"
3160             })
3161
3162             if is_full:
3163                 # these seem to mark watchtime "history" in the real world
3164                 # they're required, so send in a single value
3165                 qs.update({
3166                     'st': 0,
3167                     'et': video_length,
3168                 })
3169
3170             url = urllib.parse.urlunparse(
3171                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
3172
3173             self._download_webpage(
3174                 url, video_id, f'Marking {label}watched',
3175                 'Unable to mark watched', fatal=False)
3176
3177     @classmethod
3178     def _extract_from_webpage(cls, url, webpage):
3179         # Invidious Instances
3180         # https://github.com/yt-dlp/yt-dlp/issues/195
3181         # https://github.com/iv-org/invidious/pull/1730
3182         mobj = re.search(
3183             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
3184             webpage)
3185         if mobj:
3186             yield cls.url_result(mobj.group('url'), cls)
3187             raise cls.StopExtraction()
3188
3189         yield from super()._extract_from_webpage(url, webpage)
3190
3191         # lazyYT YouTube embed
3192         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
3193             yield cls.url_result(unescapeHTML(id_), cls, id_)
3194
3195         # Wordpress "YouTube Video Importer" plugin
3196         for m in re.findall(r'''(?x)<div[^>]+
3197                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
3198                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
3199             yield cls.url_result(m[-1], cls, m[-1])
3200
3201     @classmethod
3202     def extract_id(cls, url):
3203         video_id = cls.get_temp_id(url)
3204         if not video_id:
3205             raise ExtractorError(f'Invalid URL: {url}')
3206         return video_id
3207
3208     def _extract_chapters_from_json(self, data, duration):
3209         chapter_list = traverse_obj(
3210             data, (
3211                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
3212                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3213             ), expected_type=list)
3214
3215         return self._extract_chapters_helper(
3216             chapter_list,
3217             start_function=lambda chapter: float_or_none(
3218                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3219             title_function=lambda chapter: traverse_obj(
3220                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3221             duration=duration)
3222
3223     def _extract_chapters_from_engagement_panel(self, data, duration):
3224         content_list = traverse_obj(
3225             data,
3226             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3227             expected_type=list)
3228         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3229         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3230
3231         return next(filter(None, (
3232             self._extract_chapters_helper(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3233                                           chapter_time, chapter_title, duration)
3234             for contents in content_list)), [])
3235
3236     def _extract_comment(self, comment_renderer, parent=None):
3237         comment_id = comment_renderer.get('commentId')
3238         if not comment_id:
3239             return
3240
3241         text = self._get_text(comment_renderer, 'contentText')
3242
3243         # Timestamp is an estimate calculated from the current time and time_text
3244         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3245         timestamp = self._parse_time_text(time_text)
3246
3247         author = self._get_text(comment_renderer, 'authorText')
3248         author_id = try_get(comment_renderer,
3249                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3250
3251         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3252                                                        lambda x: x['likeCount']), str)) or 0
3253         author_thumbnail = try_get(comment_renderer,
3254                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3255
3256         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3257         is_favorited = 'creatorHeart' in (try_get(
3258             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3259         return {
3260             'id': comment_id,
3261             'text': text,
3262             'timestamp': timestamp,
3263             'time_text': time_text,
3264             'like_count': votes,
3265             'is_favorited': is_favorited,
3266             'author': author,
3267             'author_id': author_id,
3268             'author_thumbnail': author_thumbnail,
3269             'author_is_uploader': author_is_uploader,
3270             'parent': parent or 'root'
3271         }
3272
3273     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3274
3275         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3276
3277         def extract_header(contents):
3278             _continuation = None
3279             for content in contents:
3280                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3281                 expected_comment_count = self._get_count(
3282                     comments_header_renderer, 'countText', 'commentsCount')
3283
3284                 if expected_comment_count:
3285                     tracker['est_total'] = expected_comment_count
3286                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3287                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3288
3289                 sort_menu_item = try_get(
3290                     comments_header_renderer,
3291                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3292                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3293
3294                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3295                 if not _continuation:
3296                     continue
3297
3298                 sort_text = str_or_none(sort_menu_item.get('title'))
3299                 if not sort_text:
3300                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3301                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3302                 break
3303             return _continuation
3304
3305         def extract_thread(contents):
3306             if not parent:
3307                 tracker['current_page_thread'] = 0
3308             for content in contents:
3309                 if not parent and tracker['total_parent_comments'] >= max_parents:
3310                     yield
3311                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3312                 comment_renderer = get_first(
3313                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3314                     expected_type=dict, default={})
3315
3316                 comment = self._extract_comment(comment_renderer, parent)
3317                 if not comment:
3318                     continue
3319                 # Sometimes YouTube may break and give us infinite looping comments.
3320                 # See: https://github.com/yt-dlp/yt-dlp/issues/6290
3321                 if comment['id'] in tracker['seen_comment_ids']:
3322                     self.report_warning('Detected YouTube comments looping. Stopping comment extraction as we probably cannot get any more.')
3323                     yield
3324                 else:
3325                     tracker['seen_comment_ids'].add(comment['id'])
3326
3327                 tracker['running_total'] += 1
3328                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3329                 yield comment
3330
3331                 # Attempt to get the replies
3332                 comment_replies_renderer = try_get(
3333                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3334
3335                 if comment_replies_renderer:
3336                     tracker['current_page_thread'] += 1
3337                     comment_entries_iter = self._comment_entries(
3338                         comment_replies_renderer, ytcfg, video_id,
3339                         parent=comment.get('id'), tracker=tracker)
3340                     yield from itertools.islice(comment_entries_iter, min(
3341                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3342
3343         # Keeps track of counts across recursive calls
3344         if not tracker:
3345             tracker = dict(
3346                 running_total=0,
3347                 est_total=0,
3348                 current_page_thread=0,
3349                 total_parent_comments=0,
3350                 total_reply_comments=0,
3351                 seen_comment_ids=set())
3352
3353         # TODO: Deprecated
3354         # YouTube comments have a max depth of 2
3355         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3356         if max_depth:
3357             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3358                                                 'Set max replies in the max-comments extractor argument instead')
3359         if max_depth == 1 and parent:
3360             return
3361
3362         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3363             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3364
3365         continuation = self._extract_continuation(root_continuation_data)
3366
3367         response = None
3368         is_forced_continuation = False
3369         is_first_continuation = parent is None
3370         if is_first_continuation and not continuation:
3371             # Sometimes you can get comments by generating the continuation yourself,
3372             # even if YouTube initially reports them being disabled - e.g. stories comments.
3373             # Note: if the comment section is actually disabled, YouTube may return a response with
3374             # required check_get_keys missing. So we will disable that check initially in this case.
3375             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3376             is_forced_continuation = True
3377
3378         for page_num in itertools.count(0):
3379             if not continuation:
3380                 break
3381             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3382             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3383             if page_num == 0:
3384                 if is_first_continuation:
3385                     note_prefix = 'Downloading comment section API JSON'
3386                 else:
3387                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3388                         tracker['current_page_thread'], comment_prog_str)
3389             else:
3390                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3391                     '       ' if parent else '', ' replies' if parent else '',
3392                     page_num, comment_prog_str)
3393             try:
3394                 response = self._extract_response(
3395                     item_id=None, query=continuation,
3396                     ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3397                     check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3398             except ExtractorError as e:
3399                 # Ignore incomplete data error for replies if retries didn't work.
3400                 # This is to allow any other parent comments and comment threads to be downloaded.
3401                 # See: https://github.com/yt-dlp/yt-dlp/issues/4669
3402                 if 'incomplete data' in str(e).lower() and parent and self.get_param('ignoreerrors') is True:
3403                     self.report_warning(
3404                         'Received incomplete data for a comment reply thread and retrying did not help. '
3405                         'Ignoring to let other comments be downloaded.')
3406                 else:
3407                     raise
3408             is_forced_continuation = False
3409             continuation_contents = traverse_obj(
3410                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3411
3412             continuation = None
3413             for continuation_section in continuation_contents:
3414                 continuation_items = traverse_obj(
3415                     continuation_section,
3416                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3417                     get_all=False, expected_type=list) or []
3418                 if is_first_continuation:
3419                     continuation = extract_header(continuation_items)
3420                     is_first_continuation = False
3421                     if continuation:
3422                         break
3423                     continue
3424
3425                 for entry in extract_thread(continuation_items):
3426                     if not entry:
3427                         return
3428                     yield entry
3429                 continuation = self._extract_continuation({'contents': continuation_items})
3430                 if continuation:
3431                     break
3432
3433         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3434         if message and not parent and tracker['running_total'] == 0:
3435             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3436             raise self.CommentsDisabled
3437
3438     @staticmethod
3439     def _generate_comment_continuation(video_id):
3440         """
3441         Generates initial comment section continuation token from given video id
3442         """
3443         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3444         return base64.b64encode(token.encode()).decode()
3445
3446     def _get_comments(self, ytcfg, video_id, contents, webpage):
3447         """Entry for comment extraction"""
3448         def _real_comment_extract(contents):
3449             renderer = next((
3450                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3451                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3452             yield from self._comment_entries(renderer, ytcfg, video_id)
3453
3454         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3455         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3456
3457     @staticmethod
3458     def _get_checkok_params():
3459         return {'contentCheckOk': True, 'racyCheckOk': True}
3460
3461     @classmethod
3462     def _generate_player_context(cls, sts=None):
3463         context = {
3464             'html5Preference': 'HTML5_PREF_WANTS',
3465         }
3466         if sts is not None:
3467             context['signatureTimestamp'] = sts
3468         return {
3469             'playbackContext': {
3470                 'contentPlaybackContext': context
3471             },
3472             **cls._get_checkok_params()
3473         }
3474
3475     @staticmethod
3476     def _is_agegated(player_response):
3477         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3478             return True
3479
3480         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')))
3481         AGE_GATE_REASONS = (
3482             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3483             'age_verification_required', 'age_check_required',  # status
3484         )
3485         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3486
3487     @staticmethod
3488     def _is_unplayable(player_response):
3489         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3490
3491     _STORY_PLAYER_PARAMS = '8AEB'
3492
3493     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3494
3495         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3496         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3497         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3498         headers = self.generate_api_headers(
3499             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3500
3501         yt_query = {
3502             'videoId': video_id,
3503         }
3504         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3505             yt_query['params'] = self._STORY_PLAYER_PARAMS
3506
3507         yt_query.update(self._generate_player_context(sts))
3508         return self._extract_response(
3509             item_id=video_id, ep='player', query=yt_query,
3510             ytcfg=player_ytcfg, headers=headers, fatal=True,
3511             default_client=client,
3512             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3513         ) or None
3514
3515     def _get_requested_clients(self, url, smuggled_data):
3516         requested_clients = []
3517         default = ['android', 'web']
3518         allowed_clients = sorted(
3519             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3520             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3521         for client in self._configuration_arg('player_client'):
3522             if client in allowed_clients:
3523                 requested_clients.append(client)
3524             elif client == 'default':
3525                 requested_clients.extend(default)
3526             elif client == 'all':
3527                 requested_clients.extend(allowed_clients)
3528             else:
3529                 self.report_warning(f'Skipping unsupported client {client}')
3530         if not requested_clients:
3531             requested_clients = default
3532
3533         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3534             requested_clients.extend(
3535                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3536
3537         return orderedSet(requested_clients)
3538
3539     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3540         initial_pr = None
3541         if webpage:
3542             initial_pr = self._search_json(
3543                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3544
3545         all_clients = set(clients)
3546         clients = clients[::-1]
3547         prs = []
3548
3549         def append_client(*client_names):
3550             """ Append the first client name that exists but not already used """
3551             for client_name in client_names:
3552                 actual_client = _split_innertube_client(client_name)[0]
3553                 if actual_client in INNERTUBE_CLIENTS:
3554                     if actual_client not in all_clients:
3555                         clients.append(client_name)
3556                         all_clients.add(actual_client)
3557                         return
3558
3559         # Android player_response does not have microFormats which are needed for
3560         # extraction of some data. So we return the initial_pr with formats
3561         # stripped out even if not requested by the user
3562         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3563         if initial_pr:
3564             pr = dict(initial_pr)
3565             pr['streamingData'] = None
3566             prs.append(pr)
3567
3568         last_error = None
3569         tried_iframe_fallback = False
3570         player_url = None
3571         while clients:
3572             client, base_client, variant = _split_innertube_client(clients.pop())
3573             player_ytcfg = master_ytcfg if client == 'web' else {}
3574             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3575                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3576
3577             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3578             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3579             if 'js' in self._configuration_arg('player_skip'):
3580                 require_js_player = False
3581                 player_url = None
3582
3583             if not player_url and not tried_iframe_fallback and require_js_player:
3584                 player_url = self._download_player_url(video_id)
3585                 tried_iframe_fallback = True
3586
3587             try:
3588                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3589                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3590             except ExtractorError as e:
3591                 if last_error:
3592                     self.report_warning(last_error)
3593                 last_error = e
3594                 continue
3595
3596             if pr:
3597                 # YouTube may return a different video player response than expected.
3598                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3599                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3600                 if pr_video_id and pr_video_id != video_id:
3601                     self.report_warning(
3602                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3603                 else:
3604                     # Save client name for introspection later
3605                     name = short_client_name(client)
3606                     sd = traverse_obj(pr, ('streamingData', {dict})) or {}
3607                     sd[STREAMING_DATA_CLIENT_NAME] = name
3608                     for f in traverse_obj(sd, (('formats', 'adaptiveFormats'), ..., {dict})):
3609                         f[STREAMING_DATA_CLIENT_NAME] = name
3610                     prs.append(pr)
3611
3612             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3613             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3614                 append_client(f'{base_client}_creator')
3615             elif self._is_agegated(pr):
3616                 if variant == 'tv_embedded':
3617                     append_client(f'{base_client}_embedded')
3618                 elif not variant:
3619                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3620
3621         if last_error:
3622             if not len(prs):
3623                 raise last_error
3624             self.report_warning(last_error)
3625         return prs, player_url
3626
3627     def _needs_live_processing(self, live_status, duration):
3628         if (live_status == 'is_live' and self.get_param('live_from_start')
3629                 or live_status == 'post_live' and (duration or 0) > 4 * 3600):
3630             return live_status
3631
3632     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3633         itags, stream_ids = collections.defaultdict(set), []
3634         itag_qualities, res_qualities = {}, {0: None}
3635         q = qualities([
3636             # Normally tiny is the smallest video-only formats. But
3637             # audio-only formats with unknown quality may get tagged as tiny
3638             'tiny',
3639             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3640             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3641         ])
3642         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
3643
3644         for fmt in streaming_formats:
3645             if fmt.get('targetDurationSec'):
3646                 continue
3647
3648             itag = str_or_none(fmt.get('itag'))
3649             audio_track = fmt.get('audioTrack') or {}
3650             stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
3651             if stream_id in stream_ids:
3652                 continue
3653
3654             quality = fmt.get('quality')
3655             height = int_or_none(fmt.get('height'))
3656             if quality == 'tiny' or not quality:
3657                 quality = fmt.get('audioQuality', '').lower() or quality
3658             # The 3gp format (17) in android client has a quality of "small",
3659             # but is actually worse than other formats
3660             if itag == '17':
3661                 quality = 'tiny'
3662             if quality:
3663                 if itag:
3664                     itag_qualities[itag] = quality
3665                 if height:
3666                     res_qualities[height] = quality
3667             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3668             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3669             # number of fragment that would subsequently requested with (`&sq=N`)
3670             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3671                 continue
3672
3673             fmt_url = fmt.get('url')
3674             if not fmt_url:
3675                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3676                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3677                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3678                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3679                     continue
3680                 try:
3681                     fmt_url += '&%s=%s' % (
3682                         traverse_obj(sc, ('sp', -1)) or 'signature',
3683                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3684                     )
3685                 except ExtractorError as e:
3686                     self.report_warning('Signature extraction failed: Some formats may be missing',
3687                                         video_id=video_id, only_once=True)
3688                     self.write_debug(e, only_once=True)
3689                     continue
3690
3691             query = parse_qs(fmt_url)
3692             throttled = False
3693             if query.get('n'):
3694                 try:
3695                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3696                     fmt_url = update_url_query(fmt_url, {
3697                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3698                     })
3699                 except ExtractorError as e:
3700                     phantomjs_hint = ''
3701                     if isinstance(e, JSInterpreter.Exception):
3702                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3703                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3704                     if player_url:
3705                         self.report_warning(
3706                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3707                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3708                         self.write_debug(e, only_once=True)
3709                     else:
3710                         self.report_warning(
3711                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3712                             video_id=video_id, only_once=True)
3713                     throttled = True
3714
3715             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3716             language_preference = (
3717                 10 if audio_track.get('audioIsDefault') and 10
3718                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3719                 else -1)
3720             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3721             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3722             # Make sure to avoid false positives with small duration differences.
3723             # E.g. __2ABJjxzNo, ySuUZEjARPY
3724             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3725             if is_damaged:
3726                 self.report_warning(
3727                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3728
3729             client_name = fmt.get(STREAMING_DATA_CLIENT_NAME)
3730             dct = {
3731                 'asr': int_or_none(fmt.get('audioSampleRate')),
3732                 'filesize': int_or_none(fmt.get('contentLength')),
3733                 'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
3734                 'format_note': join_nonempty(
3735                     join_nonempty(audio_track.get('displayName'),
3736                                   language_preference > 0 and ' (default)', delim=''),
3737                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3738                     fmt.get('isDrc') and 'DRC',
3739                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3740                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3741                     throttled and 'THROTTLED', is_damaged and 'DAMAGED',
3742                     self.get_param('verbose') and client_name,
3743                     delim=', '),
3744                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3745                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3746                 'fps': int_or_none(fmt.get('fps')) or None,
3747                 'audio_channels': fmt.get('audioChannels'),
3748                 'height': height,
3749                 'quality': q(quality) - bool(fmt.get('isDrc')) / 2,
3750                 'has_drm': bool(fmt.get('drmFamilies')),
3751                 'tbr': tbr,
3752                 'url': fmt_url,
3753                 'width': int_or_none(fmt.get('width')),
3754                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3755                                           'desc' if language_preference < -1 else '') or None,
3756                 'language_preference': language_preference,
3757                 # Strictly de-prioritize damaged and 3gp formats
3758                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3759             }
3760             mime_mobj = re.match(
3761                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3762             if mime_mobj:
3763                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3764                 dct.update(parse_codecs(mime_mobj.group(2)))
3765
3766             single_stream = 'none' in (dct.get('acodec'), dct.get('vcodec'))
3767             if single_stream and dct.get('ext'):
3768                 dct['container'] = dct['ext'] + '_dash'
3769             if single_stream or itag == '17':
3770                 CHUNK_SIZE = 10 << 20
3771                 dct.update({
3772                     'protocol': 'http_dash_segments',
3773                     'fragments': [{
3774                         'url': update_url_query(dct['url'], {
3775                             'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}'
3776                         })
3777                     } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
3778                 } if itag != '17' and dct['filesize'] else {
3779                     'downloader_options': {'http_chunk_size': CHUNK_SIZE}
3780                 })
3781
3782             if itag:
3783                 itags[itag].add(('https', dct.get('language')))
3784                 stream_ids.append(stream_id)
3785             yield dct
3786
3787         needs_live_processing = self._needs_live_processing(live_status, duration)
3788         skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
3789
3790         skip_manifests = set(self._configuration_arg('skip'))
3791         if (not self.get_param('youtube_include_hls_manifest', True)
3792                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3793                 or needs_live_processing and skip_bad_formats):
3794             skip_manifests.add('hls')
3795
3796         if not self.get_param('youtube_include_dash_manifest', True):
3797             skip_manifests.add('dash')
3798         if self._configuration_arg('include_live_dash'):
3799             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3800                                                 'Use include_incomplete_formats extractor argument instead')
3801         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3802             skip_manifests.add('dash')
3803
3804         def process_manifest_format(f, proto, client_name, itag):
3805             key = (proto, f.get('language'))
3806             if key in itags[itag]:
3807                 return False
3808             itags[itag].add(key)
3809
3810             if any(p != proto for p, _ in itags[itag]):
3811                 f['format_id'] = f'{itag}-{proto}'
3812             elif itag:
3813                 f['format_id'] = itag
3814
3815             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3816             if f['quality'] == -1 and f.get('height'):
3817                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3818             if self.get_param('verbose'):
3819                 f['format_note'] = join_nonempty(f.get('format_note'), client_name, delim=', ')
3820             return True
3821
3822         subtitles = {}
3823         for sd in streaming_data:
3824             client_name = sd.get(STREAMING_DATA_CLIENT_NAME)
3825
3826             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3827             if hls_manifest_url:
3828                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3829                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3830                 subtitles = self._merge_subtitles(subs, subtitles)
3831                 for f in fmts:
3832                     if process_manifest_format(f, 'hls', client_name, self._search_regex(
3833                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3834                         yield f
3835
3836             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3837             if dash_manifest_url:
3838                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3839                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3840                 for f in formats:
3841                     if process_manifest_format(f, 'dash', client_name, f['format_id']):
3842                         f['filesize'] = int_or_none(self._search_regex(
3843                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3844                         if needs_live_processing:
3845                             f['is_from_start'] = True
3846
3847                         yield f
3848         yield subtitles
3849
3850     def _extract_storyboard(self, player_responses, duration):
3851         spec = get_first(
3852             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3853         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3854         if not base_url:
3855             return
3856         L = len(spec) - 1
3857         for i, args in enumerate(spec):
3858             args = args.split('#')
3859             counts = list(map(int_or_none, args[:5]))
3860             if len(args) != 8 or not all(counts):
3861                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3862                 continue
3863             width, height, frame_count, cols, rows = counts
3864             N, sigh = args[6:]
3865
3866             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3867             fragment_count = frame_count / (cols * rows)
3868             fragment_duration = duration / fragment_count
3869             yield {
3870                 'format_id': f'sb{i}',
3871                 'format_note': 'storyboard',
3872                 'ext': 'mhtml',
3873                 'protocol': 'mhtml',
3874                 'acodec': 'none',
3875                 'vcodec': 'none',
3876                 'url': url,
3877                 'width': width,
3878                 'height': height,
3879                 'fps': frame_count / duration,
3880                 'rows': rows,
3881                 'columns': cols,
3882                 'fragments': [{
3883                     'url': url.replace('$M', str(j)),
3884                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3885                 } for j in range(math.ceil(fragment_count))],
3886             }
3887
3888     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3889         webpage = None
3890         if 'webpage' not in self._configuration_arg('player_skip'):
3891             query = {'bpctr': '9999999999', 'has_verified': '1'}
3892             if smuggled_data.get('is_story'):
3893                 query['pp'] = self._STORY_PLAYER_PARAMS
3894             webpage = self._download_webpage(
3895                 webpage_url, video_id, fatal=False, query=query)
3896
3897         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3898
3899         player_responses, player_url = self._extract_player_responses(
3900             self._get_requested_clients(url, smuggled_data),
3901             video_id, webpage, master_ytcfg, smuggled_data)
3902
3903         return webpage, master_ytcfg, player_responses, player_url
3904
3905     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3906         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3907         is_live = get_first(video_details, 'isLive')
3908         if is_live is None:
3909             is_live = get_first(live_broadcast_details, 'isLiveNow')
3910         live_content = get_first(video_details, 'isLiveContent')
3911         is_upcoming = get_first(video_details, 'isUpcoming')
3912         post_live = get_first(video_details, 'isPostLiveDvr')
3913         live_status = ('post_live' if post_live
3914                        else 'is_live' if is_live
3915                        else 'is_upcoming' if is_upcoming
3916                        else 'was_live' if live_content
3917                        else 'not_live' if False in (is_live, live_content)
3918                        else None)
3919         streaming_data = traverse_obj(player_responses, (..., 'streamingData'))
3920         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
3921
3922         return live_broadcast_details, live_status, streaming_data, formats, subtitles
3923
3924     def _real_extract(self, url):
3925         url, smuggled_data = unsmuggle_url(url, {})
3926         video_id = self._match_id(url)
3927
3928         base_url = self.http_scheme() + '//www.youtube.com/'
3929         webpage_url = base_url + 'watch?v=' + video_id
3930
3931         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3932
3933         playability_statuses = traverse_obj(
3934             player_responses, (..., 'playabilityStatus'), expected_type=dict)
3935
3936         trailer_video_id = get_first(
3937             playability_statuses,
3938             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3939             expected_type=str)
3940         if trailer_video_id:
3941             return self.url_result(
3942                 trailer_video_id, self.ie_key(), trailer_video_id)
3943
3944         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3945                        if webpage else (lambda x: None))
3946
3947         video_details = traverse_obj(player_responses, (..., 'videoDetails'), expected_type=dict)
3948         microformats = traverse_obj(
3949             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3950             expected_type=dict)
3951
3952         translated_title = self._get_text(microformats, (..., 'title'))
3953         video_title = (self._preferred_lang and translated_title
3954                        or get_first(video_details, 'title')  # primary
3955                        or translated_title
3956                        or search_meta(['og:title', 'twitter:title', 'title']))
3957         translated_description = self._get_text(microformats, (..., 'description'))
3958         original_description = get_first(video_details, 'shortDescription')
3959         video_description = (
3960             self._preferred_lang and translated_description
3961             # If original description is blank, it will be an empty string.
3962             # Do not prefer translated description in this case.
3963             or original_description if original_description is not None else translated_description)
3964
3965         multifeed_metadata_list = get_first(
3966             player_responses,
3967             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3968             expected_type=str)
3969         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3970             if self.get_param('noplaylist'):
3971                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3972             else:
3973                 entries = []
3974                 feed_ids = []
3975                 for feed in multifeed_metadata_list.split(','):
3976                     # Unquote should take place before split on comma (,) since textual
3977                     # fields may contain comma as well (see
3978                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3979                     feed_data = urllib.parse.parse_qs(
3980                         urllib.parse.unquote_plus(feed))
3981
3982                     def feed_entry(name):
3983                         return try_get(
3984                             feed_data, lambda x: x[name][0], str)
3985
3986                     feed_id = feed_entry('id')
3987                     if not feed_id:
3988                         continue
3989                     feed_title = feed_entry('title')
3990                     title = video_title
3991                     if feed_title:
3992                         title += ' (%s)' % feed_title
3993                     entries.append({
3994                         '_type': 'url_transparent',
3995                         'ie_key': 'Youtube',
3996                         'url': smuggle_url(
3997                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
3998                             {'force_singlefeed': True}),
3999                         'title': title,
4000                     })
4001                     feed_ids.append(feed_id)
4002                 self.to_screen(
4003                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
4004                     % (', '.join(feed_ids), video_id))
4005                 return self.playlist_result(
4006                     entries, video_id, video_title, video_description)
4007
4008         duration = (int_or_none(get_first(video_details, 'lengthSeconds'))
4009                     or int_or_none(get_first(microformats, 'lengthSeconds'))
4010                     or parse_duration(search_meta('duration')) or None)
4011
4012         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
4013             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
4014         if live_status == 'post_live':
4015             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
4016
4017         if not formats:
4018             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
4019                 self.report_drm(video_id)
4020             pemr = get_first(
4021                 playability_statuses,
4022                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
4023             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
4024             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
4025             if subreason:
4026                 if subreason == 'The uploader has not made this video available in your country.':
4027                     countries = get_first(microformats, 'availableCountries')
4028                     if not countries:
4029                         regions_allowed = search_meta('regionsAllowed')
4030                         countries = regions_allowed.split(',') if regions_allowed else None
4031                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
4032                 reason += f'. {subreason}'
4033             if reason:
4034                 self.raise_no_formats(reason, expected=True)
4035
4036         keywords = get_first(video_details, 'keywords', expected_type=list) or []
4037         if not keywords and webpage:
4038             keywords = [
4039                 unescapeHTML(m.group('content'))
4040                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
4041         for keyword in keywords:
4042             if keyword.startswith('yt:stretch='):
4043                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
4044                 if mobj:
4045                     # NB: float is intentional for forcing float division
4046                     w, h = (float(v) for v in mobj.groups())
4047                     if w > 0 and h > 0:
4048                         ratio = w / h
4049                         for f in formats:
4050                             if f.get('vcodec') != 'none':
4051                                 f['stretched_ratio'] = ratio
4052                         break
4053         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
4054         thumbnail_url = search_meta(['og:image', 'twitter:image'])
4055         if thumbnail_url:
4056             thumbnails.append({
4057                 'url': thumbnail_url,
4058             })
4059         original_thumbnails = thumbnails.copy()
4060
4061         # The best resolution thumbnails sometimes does not appear in the webpage
4062         # See: https://github.com/yt-dlp/yt-dlp/issues/340
4063         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
4064         thumbnail_names = [
4065             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
4066             # in resolution, these are not the custom thumbnail. So de-prioritize them
4067             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
4068             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
4069         ]
4070         n_thumbnail_names = len(thumbnail_names)
4071         thumbnails.extend({
4072             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
4073                 video_id=video_id, name=name, ext=ext,
4074                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
4075         } for name in thumbnail_names for ext in ('webp', 'jpg'))
4076         for thumb in thumbnails:
4077             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
4078             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
4079         self._remove_duplicate_formats(thumbnails)
4080         self._downloader._sort_thumbnails(original_thumbnails)
4081
4082         category = get_first(microformats, 'category') or search_meta('genre')
4083         channel_id = str_or_none(
4084             get_first(video_details, 'channelId')
4085             or get_first(microformats, 'externalChannelId')
4086             or search_meta('channelId'))
4087         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
4088
4089         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
4090         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
4091         if not duration and live_end_time and live_start_time:
4092             duration = live_end_time - live_start_time
4093
4094         needs_live_processing = self._needs_live_processing(live_status, duration)
4095
4096         def is_bad_format(fmt):
4097             if needs_live_processing and not fmt.get('is_from_start'):
4098                 return True
4099             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
4100                     and fmt.get('protocol') == 'http_dash_segments'):
4101                 return True
4102
4103         for fmt in filter(is_bad_format, formats):
4104             fmt['preference'] = (fmt.get('preference') or -1) - 10
4105             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
4106
4107         if needs_live_processing:
4108             self._prepare_live_from_start_formats(
4109                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
4110
4111         formats.extend(self._extract_storyboard(player_responses, duration))
4112
4113         info = {
4114             'id': video_id,
4115             'title': video_title,
4116             'formats': formats,
4117             'thumbnails': thumbnails,
4118             # The best thumbnail that we are sure exists. Prevents unnecessary
4119             # URL checking if user don't care about getting the best possible thumbnail
4120             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
4121             'description': video_description,
4122             'uploader': get_first(video_details, 'author'),
4123             'uploader_id': self._search_regex(r'/(?:channel/|user/|(?=@))([^/?&#]+)', owner_profile_url, 'uploader id', default=None),
4124             'uploader_url': owner_profile_url,
4125             'channel_id': channel_id,
4126             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
4127             'duration': duration,
4128             'view_count': int_or_none(
4129                 get_first((video_details, microformats), (..., 'viewCount'))
4130                 or search_meta('interactionCount')),
4131             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
4132             'age_limit': 18 if (
4133                 get_first(microformats, 'isFamilySafe') is False
4134                 or search_meta('isFamilyFriendly') == 'false'
4135                 or search_meta('og:restrictions:age') == '18+') else 0,
4136             'webpage_url': webpage_url,
4137             'categories': [category] if category else None,
4138             'tags': keywords,
4139             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
4140             'live_status': live_status,
4141             'release_timestamp': live_start_time,
4142             '_format_sort_fields': (  # source_preference is lower for throttled/potentially damaged formats
4143                 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto')
4144         }
4145
4146         subtitles = {}
4147         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
4148         if pctr:
4149             def get_lang_code(track):
4150                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
4151                         or track.get('languageCode'))
4152
4153             # Converted into dicts to remove duplicates
4154             captions = {
4155                 get_lang_code(sub): sub
4156                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...))}
4157             translation_languages = {
4158                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
4159                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...))}
4160
4161             def process_language(container, base_url, lang_code, sub_name, query):
4162                 lang_subs = container.setdefault(lang_code, [])
4163                 for fmt in self._SUBTITLE_FORMATS:
4164                     query.update({
4165                         'fmt': fmt,
4166                     })
4167                     lang_subs.append({
4168                         'ext': fmt,
4169                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
4170                         'name': sub_name,
4171                     })
4172
4173             # NB: Constructing the full subtitle dictionary is slow
4174             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
4175                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
4176             for lang_code, caption_track in captions.items():
4177                 base_url = caption_track.get('baseUrl')
4178                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
4179                 if not base_url:
4180                     continue
4181                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
4182                 if caption_track.get('kind') != 'asr':
4183                     if not lang_code:
4184                         continue
4185                     process_language(
4186                         subtitles, base_url, lang_code, lang_name, {})
4187                     if not caption_track.get('isTranslatable'):
4188                         continue
4189                 for trans_code, trans_name in translation_languages.items():
4190                     if not trans_code:
4191                         continue
4192                     orig_trans_code = trans_code
4193                     if caption_track.get('kind') != 'asr' and trans_code != 'und':
4194                         if not get_translated_subs:
4195                             continue
4196                         trans_code += f'-{lang_code}'
4197                         trans_name += format_field(lang_name, None, ' from %s')
4198                     # Add an "-orig" label to the original language so that it can be distinguished.
4199                     # The subs are returned without "-orig" as well for compatibility
4200                     if lang_code == f'a-{orig_trans_code}':
4201                         process_language(
4202                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
4203                     # Setting tlang=lang returns damaged subtitles.
4204                     process_language(automatic_captions, base_url, trans_code, trans_name,
4205                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
4206
4207         info['automatic_captions'] = automatic_captions
4208         info['subtitles'] = subtitles
4209
4210         parsed_url = urllib.parse.urlparse(url)
4211         for component in [parsed_url.fragment, parsed_url.query]:
4212             query = urllib.parse.parse_qs(component)
4213             for k, v in query.items():
4214                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
4215                     d_k += '_time'
4216                     if d_k not in info and k in s_ks:
4217                         info[d_k] = parse_duration(query[k][0])
4218
4219         # Youtube Music Auto-generated description
4220         if video_description:
4221             mobj = re.search(
4222                 r'''(?xs)
4223                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
4224                     (?P<album>[^\n]+)
4225                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4226                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4227                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
4228                     .+\nAuto-generated\ by\ YouTube\.\s*$
4229                 ''', video_description)
4230             if mobj:
4231                 release_year = mobj.group('release_year')
4232                 release_date = mobj.group('release_date')
4233                 if release_date:
4234                     release_date = release_date.replace('-', '')
4235                     if not release_year:
4236                         release_year = release_date[:4]
4237                 info.update({
4238                     'album': mobj.group('album'.strip()),
4239                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
4240                     'track': mobj.group('track').strip(),
4241                     'release_date': release_date,
4242                     'release_year': int_or_none(release_year),
4243                 })
4244
4245         initial_data = None
4246         if webpage:
4247             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4248         if not initial_data:
4249             query = {'videoId': video_id}
4250             query.update(self._get_checkok_params())
4251             initial_data = self._extract_response(
4252                 item_id=video_id, ep='next', fatal=False,
4253                 ytcfg=master_ytcfg, query=query,
4254                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4255                 note='Downloading initial data API JSON')
4256
4257         info['comment_count'] = traverse_obj(initial_data, (
4258             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4259             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount', 'simpleText'
4260         ), (
4261             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4262             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo', 'runs', ..., 'text'
4263         ), expected_type=int_or_none, get_all=False)
4264
4265         try:  # This will error if there is no livechat
4266             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4267         except (KeyError, IndexError, TypeError):
4268             pass
4269         else:
4270             info.setdefault('subtitles', {})['live_chat'] = [{
4271                 # url is needed to set cookies
4272                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4273                 'video_id': video_id,
4274                 'ext': 'json',
4275                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4276                              else 'youtube_live_chat_replay'),
4277             }]
4278
4279         if initial_data:
4280             info['chapters'] = (
4281                 self._extract_chapters_from_json(initial_data, duration)
4282                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4283                 or self._extract_chapters_from_description(video_description, duration)
4284                 or None)
4285
4286         contents = traverse_obj(
4287             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4288             expected_type=list, default=[])
4289
4290         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4291         if vpir:
4292             stl = vpir.get('superTitleLink')
4293             if stl:
4294                 stl = self._get_text(stl)
4295                 if try_get(
4296                         vpir,
4297                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4298                     info['location'] = stl
4299                 else:
4300                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4301                     if mobj:
4302                         info.update({
4303                             'series': mobj.group(1),
4304                             'season_number': int(mobj.group(2)),
4305                             'episode_number': int(mobj.group(3)),
4306                         })
4307             for tlb in (try_get(
4308                     vpir,
4309                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4310                     list) or []):
4311                 tbrs = variadic(
4312                     traverse_obj(
4313                         tlb, ('toggleButtonRenderer', ...),
4314                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer')))
4315                 for tbr in tbrs:
4316                     for getter, regex in [(
4317                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4318                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4319                                 lambda x: x['accessibility'],
4320                                 lambda x: x['accessibilityData']['accessibilityData'],
4321                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4322                         label = (try_get(tbr, getter, dict) or {}).get('label')
4323                         if label:
4324                             mobj = re.match(regex, label)
4325                             if mobj:
4326                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4327                                 break
4328             sbr_tooltip = try_get(
4329                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4330             if sbr_tooltip:
4331                 like_count, dislike_count = sbr_tooltip.split(' / ')
4332                 info.update({
4333                     'like_count': str_to_int(like_count),
4334                     'dislike_count': str_to_int(dislike_count),
4335                 })
4336             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4337             if vcr:
4338                 vc = self._get_count(vcr, 'viewCount')
4339                 # Upcoming premieres with waiting count are treated as live here
4340                 if vcr.get('isLive'):
4341                     info['concurrent_view_count'] = vc
4342                 elif info.get('view_count') is None:
4343                     info['view_count'] = vc
4344
4345         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4346         if vsir:
4347             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4348             info.update({
4349                 'channel': self._get_text(vor, 'title'),
4350                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4351
4352             rows = try_get(
4353                 vsir,
4354                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4355                 list) or []
4356             multiple_songs = False
4357             for row in rows:
4358                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4359                     multiple_songs = True
4360                     break
4361             for row in rows:
4362                 mrr = row.get('metadataRowRenderer') or {}
4363                 mrr_title = mrr.get('title')
4364                 if not mrr_title:
4365                     continue
4366                 mrr_title = self._get_text(mrr, 'title')
4367                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4368                 if mrr_title == 'License':
4369                     info['license'] = mrr_contents_text
4370                 elif not multiple_songs:
4371                     if mrr_title == 'Album':
4372                         info['album'] = mrr_contents_text
4373                     elif mrr_title == 'Artist':
4374                         info['artist'] = mrr_contents_text
4375                     elif mrr_title == 'Song':
4376                         info['track'] = mrr_contents_text
4377
4378         fallbacks = {
4379             'channel': 'uploader',
4380             'channel_id': 'uploader_id',
4381             'channel_url': 'uploader_url',
4382         }
4383
4384         # The upload date for scheduled, live and past live streams / premieres in microformats
4385         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4386         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4387         upload_date = (
4388             unified_strdate(get_first(microformats, 'uploadDate'))
4389             or unified_strdate(search_meta('uploadDate')))
4390         if not upload_date or (
4391             live_status in ('not_live', None)
4392             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4393         ):
4394             upload_date = strftime_or_none(
4395                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4396         info['upload_date'] = upload_date
4397
4398         for to, frm in fallbacks.items():
4399             if not info.get(to):
4400                 info[to] = info.get(frm)
4401
4402         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4403             v = info.get(s_k)
4404             if v:
4405                 info[d_k] = v
4406
4407         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4408
4409         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4410                       or get_first(video_details, 'isPrivate', expected_type=bool))
4411
4412         info['availability'] = (
4413             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4414             else self._availability(
4415                 is_private=is_private,
4416                 needs_premium=(
4417                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4418                     or False if initial_data and is_private is not None else None),
4419                 needs_subscription=(
4420                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4421                     or False if initial_data and is_private is not None else None),
4422                 needs_auth=info['age_limit'] >= 18,
4423                 is_unlisted=None if is_private is None else (
4424                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4425                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4426
4427         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4428
4429         self.mark_watched(video_id, player_responses)
4430
4431         return info
4432
4433
4434 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4435     @staticmethod
4436     def passthrough_smuggled_data(func):
4437         def _smuggle(info, smuggled_data):
4438             if info.get('_type') not in ('url', 'url_transparent'):
4439                 return info
4440             if smuggled_data.get('is_music_url'):
4441                 parsed_url = urllib.parse.urlparse(info['url'])
4442                 if parsed_url.netloc in ('www.youtube.com', 'music.youtube.com'):
4443                     smuggled_data.pop('is_music_url')
4444                     info['url'] = urllib.parse.urlunparse(parsed_url._replace(netloc='music.youtube.com'))
4445             if smuggled_data:
4446                 info['url'] = smuggle_url(info['url'], smuggled_data)
4447             return info
4448
4449         @functools.wraps(func)
4450         def wrapper(self, url):
4451             url, smuggled_data = unsmuggle_url(url, {})
4452             if self.is_music_url(url):
4453                 smuggled_data['is_music_url'] = True
4454             info_dict = func(self, url, smuggled_data)
4455             if smuggled_data:
4456                 _smuggle(info_dict, smuggled_data)
4457                 if info_dict.get('entries'):
4458                     info_dict['entries'] = (_smuggle(i, smuggled_data.copy()) for i in info_dict['entries'])
4459             return info_dict
4460         return wrapper
4461
4462     @staticmethod
4463     def _extract_basic_item_renderer(item):
4464         # Modified from _extract_grid_item_renderer
4465         known_basic_renderers = (
4466             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4467         )
4468         for key, renderer in item.items():
4469             if not isinstance(renderer, dict):
4470                 continue
4471             elif key in known_basic_renderers:
4472                 return renderer
4473             elif key.startswith('grid') and key.endswith('Renderer'):
4474                 return renderer
4475
4476     def _extract_channel_renderer(self, renderer):
4477         channel_id = renderer['channelId']
4478         title = self._get_text(renderer, 'title')
4479         channel_url = f'https://www.youtube.com/channel/{channel_id}'
4480         return {
4481             '_type': 'url',
4482             'url': channel_url,
4483             'id': channel_id,
4484             'ie_key': YoutubeTabIE.ie_key(),
4485             'channel': title,
4486             'channel_id': channel_id,
4487             'channel_url': channel_url,
4488             'title': title,
4489             'channel_follower_count': self._get_count(renderer, 'subscriberCountText'),
4490             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
4491             'playlist_count': self._get_count(renderer, 'videoCountText'),
4492             'description': self._get_text(renderer, 'descriptionSnippet'),
4493         }
4494
4495     def _grid_entries(self, grid_renderer):
4496         for item in grid_renderer['items']:
4497             if not isinstance(item, dict):
4498                 continue
4499             renderer = self._extract_basic_item_renderer(item)
4500             if not isinstance(renderer, dict):
4501                 continue
4502             title = self._get_text(renderer, 'title')
4503
4504             # playlist
4505             playlist_id = renderer.get('playlistId')
4506             if playlist_id:
4507                 yield self.url_result(
4508                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4509                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4510                     video_title=title)
4511                 continue
4512             # video
4513             video_id = renderer.get('videoId')
4514             if video_id:
4515                 yield self._extract_video(renderer)
4516                 continue
4517             # channel
4518             channel_id = renderer.get('channelId')
4519             if channel_id:
4520                 yield self._extract_channel_renderer(renderer)
4521                 continue
4522             # generic endpoint URL support
4523             ep_url = urljoin('https://www.youtube.com/', try_get(
4524                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4525                 str))
4526             if ep_url:
4527                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4528                     if ie.suitable(ep_url):
4529                         yield self.url_result(
4530                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4531                         break
4532
4533     def _music_reponsive_list_entry(self, renderer):
4534         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4535         if video_id:
4536             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4537                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4538         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4539         if playlist_id:
4540             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4541             if video_id:
4542                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4543                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4544             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4545                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4546         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4547         if browse_id:
4548             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4549                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4550
4551     def _shelf_entries_from_content(self, shelf_renderer):
4552         content = shelf_renderer.get('content')
4553         if not isinstance(content, dict):
4554             return
4555         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4556         if renderer:
4557             # TODO: add support for nested playlists so each shelf is processed
4558             # as separate playlist
4559             # TODO: this includes only first N items
4560             yield from self._grid_entries(renderer)
4561         renderer = content.get('horizontalListRenderer')
4562         if renderer:
4563             # TODO
4564             pass
4565
4566     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4567         ep = try_get(
4568             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4569             str)
4570         shelf_url = urljoin('https://www.youtube.com', ep)
4571         if shelf_url:
4572             # Skipping links to another channels, note that checking for
4573             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4574             # will not work
4575             if skip_channels and '/channels?' in shelf_url:
4576                 return
4577             title = self._get_text(shelf_renderer, 'title')
4578             yield self.url_result(shelf_url, video_title=title)
4579         # Shelf may not contain shelf URL, fallback to extraction from content
4580         yield from self._shelf_entries_from_content(shelf_renderer)
4581
4582     def _playlist_entries(self, video_list_renderer):
4583         for content in video_list_renderer['contents']:
4584             if not isinstance(content, dict):
4585                 continue
4586             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4587             if not isinstance(renderer, dict):
4588                 continue
4589             video_id = renderer.get('videoId')
4590             if not video_id:
4591                 continue
4592             yield self._extract_video(renderer)
4593
4594     def _rich_entries(self, rich_grid_renderer):
4595         renderer = traverse_obj(
4596             rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
4597         video_id = renderer.get('videoId')
4598         if not video_id:
4599             return
4600         yield self._extract_video(renderer)
4601
4602     def _video_entry(self, video_renderer):
4603         video_id = video_renderer.get('videoId')
4604         if video_id:
4605             return self._extract_video(video_renderer)
4606
4607     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4608         url = urljoin('https://youtube.com', traverse_obj(
4609             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4610         if url:
4611             return self.url_result(
4612                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4613
4614     def _post_thread_entries(self, post_thread_renderer):
4615         post_renderer = try_get(
4616             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4617         if not post_renderer:
4618             return
4619         # video attachment
4620         video_renderer = try_get(
4621             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4622         video_id = video_renderer.get('videoId')
4623         if video_id:
4624             entry = self._extract_video(video_renderer)
4625             if entry:
4626                 yield entry
4627         # playlist attachment
4628         playlist_id = try_get(
4629             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4630         if playlist_id:
4631             yield self.url_result(
4632                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4633                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4634         # inline video links
4635         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4636         for run in runs:
4637             if not isinstance(run, dict):
4638                 continue
4639             ep_url = try_get(
4640                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4641             if not ep_url:
4642                 continue
4643             if not YoutubeIE.suitable(ep_url):
4644                 continue
4645             ep_video_id = YoutubeIE._match_id(ep_url)
4646             if video_id == ep_video_id:
4647                 continue
4648             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4649
4650     def _post_thread_continuation_entries(self, post_thread_continuation):
4651         contents = post_thread_continuation.get('contents')
4652         if not isinstance(contents, list):
4653             return
4654         for content in contents:
4655             renderer = content.get('backstagePostThreadRenderer')
4656             if isinstance(renderer, dict):
4657                 yield from self._post_thread_entries(renderer)
4658                 continue
4659             renderer = content.get('videoRenderer')
4660             if isinstance(renderer, dict):
4661                 yield self._video_entry(renderer)
4662
4663     r''' # unused
4664     def _rich_grid_entries(self, contents):
4665         for content in contents:
4666             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4667             if video_renderer:
4668                 entry = self._video_entry(video_renderer)
4669                 if entry:
4670                     yield entry
4671     '''
4672
4673     def _report_history_entries(self, renderer):
4674         for url in traverse_obj(renderer, (
4675                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4676                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4677                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4678             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4679
4680     def _extract_entries(self, parent_renderer, continuation_list):
4681         # continuation_list is modified in-place with continuation_list = [continuation_token]
4682         continuation_list[:] = [None]
4683         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4684         for content in contents:
4685             if not isinstance(content, dict):
4686                 continue
4687             is_renderer = traverse_obj(
4688                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4689                 expected_type=dict)
4690             if not is_renderer:
4691                 if content.get('richItemRenderer'):
4692                     for entry in self._rich_entries(content['richItemRenderer']):
4693                         yield entry
4694                     continuation_list[0] = self._extract_continuation(parent_renderer)
4695                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4696                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4697                     yield from self._report_history_entries(table)
4698                     continuation_list[0] = self._extract_continuation(table)
4699                 continue
4700
4701             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4702             for isr_content in isr_contents:
4703                 if not isinstance(isr_content, dict):
4704                     continue
4705
4706                 known_renderers = {
4707                     'playlistVideoListRenderer': self._playlist_entries,
4708                     'gridRenderer': self._grid_entries,
4709                     'reelShelfRenderer': self._grid_entries,
4710                     'shelfRenderer': self._shelf_entries,
4711                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4712                     'backstagePostThreadRenderer': self._post_thread_entries,
4713                     'videoRenderer': lambda x: [self._video_entry(x)],
4714                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4715                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4716                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4717                 }
4718                 for key, renderer in isr_content.items():
4719                     if key not in known_renderers:
4720                         continue
4721                     for entry in known_renderers[key](renderer):
4722                         if entry:
4723                             yield entry
4724                     continuation_list[0] = self._extract_continuation(renderer)
4725                     break
4726
4727             if not continuation_list[0]:
4728                 continuation_list[0] = self._extract_continuation(is_renderer)
4729
4730         if not continuation_list[0]:
4731             continuation_list[0] = self._extract_continuation(parent_renderer)
4732
4733     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4734         continuation_list = [None]
4735         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4736         tab_content = try_get(tab, lambda x: x['content'], dict)
4737         if not tab_content:
4738             return
4739         parent_renderer = (
4740             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4741             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4742         yield from extract_entries(parent_renderer)
4743         continuation = continuation_list[0]
4744
4745         for page_num in itertools.count(1):
4746             if not continuation:
4747                 break
4748             headers = self.generate_api_headers(
4749                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4750             response = self._extract_response(
4751                 item_id=f'{item_id} page {page_num}',
4752                 query=continuation, headers=headers, ytcfg=ytcfg,
4753                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4754
4755             if not response:
4756                 break
4757             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4758             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4759             visitor_data = self._extract_visitor_data(response) or visitor_data
4760
4761             known_renderers = {
4762                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4763                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4764                 'gridVideoRenderer': (self._grid_entries, 'items'),
4765                 'gridChannelRenderer': (self._grid_entries, 'items'),
4766                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4767                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4768                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4769                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4770                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4771                 'playlistVideoListContinuation': (self._playlist_entries, None),
4772                 'gridContinuation': (self._grid_entries, None),
4773                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4774                 'sectionListContinuation': (extract_entries, None),  # for feeds
4775             }
4776
4777             continuation_items = traverse_obj(response, (
4778                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4779                 'appendContinuationItemsAction', 'continuationItems'
4780             ), 'continuationContents', get_all=False)
4781             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4782
4783             video_items_renderer = None
4784             for key in continuation_item.keys():
4785                 if key not in known_renderers:
4786                     continue
4787                 func, parent_key = known_renderers[key]
4788                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4789                 continuation_list = [None]
4790                 yield from func(video_items_renderer)
4791                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4792
4793             if not video_items_renderer:
4794                 break
4795
4796     @staticmethod
4797     def _extract_selected_tab(tabs, fatal=True):
4798         for tab_renderer in tabs:
4799             if tab_renderer.get('selected'):
4800                 return tab_renderer
4801         if fatal:
4802             raise ExtractorError('Unable to find selected tab')
4803
4804     @staticmethod
4805     def _extract_tab_renderers(response):
4806         return traverse_obj(
4807             response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
4808
4809     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4810         metadata = self._extract_metadata_from_tabs(item_id, data)
4811
4812         selected_tab = self._extract_selected_tab(tabs)
4813         metadata['title'] += format_field(selected_tab, 'title', ' - %s')
4814         metadata['title'] += format_field(selected_tab, 'expandedText', ' - %s')
4815
4816         return self.playlist_result(
4817             self._entries(
4818                 selected_tab, metadata['id'], ytcfg,
4819                 self._extract_account_syncid(ytcfg, data),
4820                 self._extract_visitor_data(data, ytcfg)),
4821             **metadata)
4822
4823     def _extract_metadata_from_tabs(self, item_id, data):
4824         info = {'id': item_id}
4825
4826         metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
4827         if metadata_renderer:
4828             info.update({
4829                 'uploader': metadata_renderer.get('title'),
4830                 'uploader_id': metadata_renderer.get('externalId'),
4831                 'uploader_url': metadata_renderer.get('channelUrl'),
4832             })
4833             if info['uploader_id']:
4834                 info['id'] = info['uploader_id']
4835         else:
4836             metadata_renderer = traverse_obj(data, ('metadata', 'playlistMetadataRenderer'), expected_type=dict)
4837
4838         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4839         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4840         def _get_uncropped(url):
4841             return url_or_none((url or '').split('=')[0] + '=s0')
4842
4843         avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
4844         if avatar_thumbnails:
4845             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4846             if uncropped_avatar:
4847                 avatar_thumbnails.append({
4848                     'url': uncropped_avatar,
4849                     'id': 'avatar_uncropped',
4850                     'preference': 1
4851                 })
4852
4853         channel_banners = self._extract_thumbnails(
4854             data, ('header', ..., ('banner', 'mobileBanner', 'tvBanner')))
4855         for banner in channel_banners:
4856             banner['preference'] = -10
4857
4858         if channel_banners:
4859             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4860             if uncropped_banner:
4861                 channel_banners.append({
4862                     'url': uncropped_banner,
4863                     'id': 'banner_uncropped',
4864                     'preference': -5
4865                 })
4866
4867         # Deprecated - remove primary_sidebar_renderer when layout discontinued
4868         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4869         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
4870
4871         primary_thumbnails = self._extract_thumbnails(
4872             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4873         playlist_thumbnails = self._extract_thumbnails(
4874             playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
4875
4876         info.update({
4877             'title': (traverse_obj(metadata_renderer, 'title')
4878                       or self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag'))
4879                       or info['id']),
4880             'availability': self._extract_availability(data),
4881             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4882             'description': try_get(metadata_renderer, lambda x: x.get('description', '')),
4883             'tags': try_get(metadata_renderer or {}, lambda x: x.get('keywords', '').split()),
4884             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
4885         })
4886
4887         # Playlist stats is a text runs array containing [video count, view count, last updated].
4888         # last updated or (view count and last updated) may be missing.
4889         playlist_stats = get_first(
4890             (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'), ))
4891
4892         last_updated_unix = self._parse_time_text(
4893             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
4894             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
4895         info['modified_date'] = strftime_or_none(last_updated_unix, '%Y%m%d')
4896
4897         info['view_count'] = self._get_count(playlist_stats, 1)
4898         if info['view_count'] is None:  # 0 is allowed
4899             info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText')
4900         if info['view_count'] is None:
4901             info['view_count'] = self._get_count(data, (
4902                 'contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., 'tabRenderer', 'content', 'sectionListRenderer',
4903                 'contents', ..., 'itemSectionRenderer', 'contents', ..., 'channelAboutFullMetadataRenderer', 'viewCountText'))
4904
4905         info['playlist_count'] = self._get_count(playlist_stats, 0)
4906         if info['playlist_count'] is None:  # 0 is allowed
4907             info['playlist_count'] = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
4908
4909         if not info.get('uploader_id'):
4910             owner = traverse_obj(playlist_header_renderer, 'ownerText')
4911             if not owner:  # Deprecated
4912                 owner = traverse_obj(
4913                     self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
4914                     ('videoOwner', 'videoOwnerRenderer', 'title'))
4915             owner_text = self._get_text(owner)
4916             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
4917             info.update({
4918                 'uploader': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
4919                 'uploader_id': browse_ep.get('browseId'),
4920                 'uploader_url': urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))
4921             })
4922
4923         info.update({
4924             'channel': info['uploader'],
4925             'channel_id': info['uploader_id'],
4926             'channel_url': info['uploader_url']
4927         })
4928         return info
4929
4930     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4931         first_id = last_id = response = None
4932         for page_num in itertools.count(1):
4933             videos = list(self._playlist_entries(playlist))
4934             if not videos:
4935                 return
4936             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4937             if start >= len(videos):
4938                 return
4939             yield from videos[start:]
4940             first_id = first_id or videos[0]['id']
4941             last_id = videos[-1]['id']
4942             watch_endpoint = try_get(
4943                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4944             headers = self.generate_api_headers(
4945                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4946                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4947             query = {
4948                 'playlistId': playlist_id,
4949                 'videoId': watch_endpoint.get('videoId') or last_id,
4950                 'index': watch_endpoint.get('index') or len(videos),
4951                 'params': watch_endpoint.get('params') or 'OAE%3D'
4952             }
4953             response = self._extract_response(
4954                 item_id='%s page %d' % (playlist_id, page_num),
4955                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4956                 check_get_keys='contents'
4957             )
4958             playlist = try_get(
4959                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4960
4961     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4962         title = playlist.get('title') or try_get(
4963             data, lambda x: x['titleText']['simpleText'], str)
4964         playlist_id = playlist.get('playlistId') or item_id
4965
4966         # Delegating everything except mix playlists to regular tab-based playlist URL
4967         playlist_url = urljoin(url, try_get(
4968             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4969             str))
4970
4971         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4972         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4973         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4974
4975         if playlist_url and playlist_url != url and not is_known_unviewable:
4976             return self.url_result(
4977                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4978                 video_title=title)
4979
4980         return self.playlist_result(
4981             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
4982             playlist_id=playlist_id, playlist_title=title)
4983
4984     def _extract_availability(self, data):
4985         """
4986         Gets the availability of a given playlist/tab.
4987         Note: Unless YouTube tells us explicitly, we do not assume it is public
4988         @param data: response
4989         """
4990         sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
4991         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
4992         player_header_privacy = playlist_header_renderer.get('privacy')
4993
4994         badges = self._extract_badges(sidebar_renderer)
4995
4996         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
4997         privacy_setting_icon = get_first(
4998             (playlist_header_renderer, sidebar_renderer),
4999             ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
5000              lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
5001             expected_type=str)
5002
5003         microformats_is_unlisted = traverse_obj(
5004             data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
5005
5006         return (
5007             'public' if (
5008                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
5009                 or player_header_privacy == 'PUBLIC'
5010                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
5011             else self._availability(
5012                 is_private=(
5013                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
5014                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
5015                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
5016                 is_unlisted=(
5017                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
5018                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
5019                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
5020                     else microformats_is_unlisted if microformats_is_unlisted is not None else None),
5021                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
5022                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
5023                 needs_auth=False))
5024
5025     @staticmethod
5026     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
5027         sidebar_renderer = try_get(
5028             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
5029         for item in sidebar_renderer:
5030             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
5031             if renderer:
5032                 return renderer
5033
5034     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
5035         """
5036         Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
5037         """
5038         is_playlist = bool(traverse_obj(
5039             data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
5040         if not is_playlist:
5041             return
5042         headers = self.generate_api_headers(
5043             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
5044             visitor_data=self._extract_visitor_data(data, ytcfg))
5045         query = {
5046             'params': 'wgYCCAA=',
5047             'browseId': f'VL{item_id}'
5048         }
5049         return self._extract_response(
5050             item_id=item_id, headers=headers, query=query,
5051             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
5052             note='Redownloading playlist API JSON with unavailable videos')
5053
5054     @functools.cached_property
5055     def skip_webpage(self):
5056         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
5057
5058     def _extract_webpage(self, url, item_id, fatal=True):
5059         webpage, data = None, None
5060         for retry in self.RetryManager(fatal=fatal):
5061             try:
5062                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
5063                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
5064             except ExtractorError as e:
5065                 if isinstance(e.cause, network_exceptions):
5066                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
5067                         retry.error = e
5068                         continue
5069                 self._error_or_warning(e, fatal=fatal)
5070                 break
5071
5072             try:
5073                 self._extract_and_report_alerts(data)
5074             except ExtractorError as e:
5075                 self._error_or_warning(e, fatal=fatal)
5076                 break
5077
5078             # Sometimes youtube returns a webpage with incomplete ytInitialData
5079             # See: https://github.com/yt-dlp/yt-dlp/issues/116
5080             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
5081                 retry.error = ExtractorError('Incomplete yt initial data received')
5082                 continue
5083
5084         return webpage, data
5085
5086     def _report_playlist_authcheck(self, ytcfg, fatal=True):
5087         """Use if failed to extract ytcfg (and data) from initial webpage"""
5088         if not ytcfg and self.is_authenticated:
5089             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
5090             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
5091                 raise ExtractorError(
5092                     f'{msg}. If you are not downloading private content, or '
5093                     'your cookies are only for the first account and channel,'
5094                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
5095                     expected=True)
5096             self.report_warning(msg, only_once=True)
5097
5098     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
5099         data = None
5100         if not self.skip_webpage:
5101             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
5102             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
5103             # Reject webpage data if redirected to home page without explicitly requesting
5104             selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
5105             if (url != 'https://www.youtube.com/feed/recommended'
5106                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
5107                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
5108                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
5109                 if fatal:
5110                     raise ExtractorError(msg, expected=True)
5111                 self.report_warning(msg, only_once=True)
5112         if not data:
5113             self._report_playlist_authcheck(ytcfg, fatal=fatal)
5114             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
5115         return data, ytcfg
5116
5117     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
5118         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
5119         resolve_response = self._extract_response(
5120             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
5121             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
5122         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
5123         for ep_key, ep in endpoints.items():
5124             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
5125             if params:
5126                 return self._extract_response(
5127                     item_id=item_id, query=params, ep=ep, headers=headers,
5128                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
5129                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
5130         err_note = 'Failed to resolve url (does the playlist exist?)'
5131         if fatal:
5132             raise ExtractorError(err_note, expected=True)
5133         self.report_warning(err_note, item_id)
5134
5135     _SEARCH_PARAMS = None
5136
5137     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
5138         data = {'query': query}
5139         if params is NO_DEFAULT:
5140             params = self._SEARCH_PARAMS
5141         if params:
5142             data['params'] = params
5143
5144         content_keys = (
5145             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
5146             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
5147             # ytmusic search
5148             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
5149             ('continuationContents', ),
5150         )
5151         display_id = f'query "{query}"'
5152         check_get_keys = tuple({keys[0] for keys in content_keys})
5153         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
5154         self._report_playlist_authcheck(ytcfg, fatal=False)
5155
5156         continuation_list = [None]
5157         search = None
5158         for page_num in itertools.count(1):
5159             data.update(continuation_list[0] or {})
5160             headers = self.generate_api_headers(
5161                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
5162             search = self._extract_response(
5163                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
5164                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
5165             slr_contents = traverse_obj(search, *content_keys)
5166             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
5167             if not continuation_list[0]:
5168                 break
5169
5170
5171 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
5172     IE_DESC = 'YouTube Tabs'
5173     _VALID_URL = r'''(?x:
5174         https?://
5175             (?!consent\.)(?:\w+\.)?
5176             (?:
5177                 youtube(?:kids)?\.com|
5178                 %(invidious)s
5179             )/
5180             (?:
5181                 (?P<channel_type>channel|c|user|browse)/|
5182                 (?P<not_channel>
5183                     feed/|hashtag/|
5184                     (?:playlist|watch)\?.*?\blist=
5185                 )|
5186                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
5187             )
5188             (?P<id>[^/?\#&]+)
5189     )''' % {
5190         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
5191         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5192     }
5193     IE_NAME = 'youtube:tab'
5194
5195     _TESTS = [{
5196         'note': 'playlists, multipage',
5197         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
5198         'playlist_mincount': 94,
5199         'info_dict': {
5200             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5201             'title': 'Igor Kleiner - Playlists',
5202             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5203             'uploader': 'Igor Kleiner',
5204             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5205             'channel': 'Igor Kleiner',
5206             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5207             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5208             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5209             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5210             'channel_follower_count': int
5211         },
5212     }, {
5213         'note': 'playlists, multipage, different order',
5214         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
5215         'playlist_mincount': 94,
5216         'info_dict': {
5217             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5218             'title': 'Igor Kleiner - Playlists',
5219             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5220             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5221             'uploader': 'Igor Kleiner',
5222             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5223             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5224             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5225             'channel': 'Igor Kleiner',
5226             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5227             'channel_follower_count': int
5228         },
5229     }, {
5230         'note': 'playlists, series',
5231         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5232         'playlist_mincount': 5,
5233         'info_dict': {
5234             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5235             'title': '3Blue1Brown - Playlists',
5236             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5237             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5238             'uploader': '3Blue1Brown',
5239             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5240             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5241             'channel': '3Blue1Brown',
5242             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5243             'tags': ['Mathematics'],
5244             'channel_follower_count': int
5245         },
5246     }, {
5247         'note': 'playlists, singlepage',
5248         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5249         'playlist_mincount': 4,
5250         'info_dict': {
5251             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5252             'title': 'ThirstForScience - Playlists',
5253             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5254             'uploader': 'ThirstForScience',
5255             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5256             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5257             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5258             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5259             'tags': 'count:13',
5260             'channel': 'ThirstForScience',
5261             'channel_follower_count': int
5262         }
5263     }, {
5264         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5265         'only_matching': True,
5266     }, {
5267         'note': 'basic, single video playlist',
5268         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5269         'info_dict': {
5270             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5271             'uploader': 'Sergey M.',
5272             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5273             'title': 'youtube-dl public playlist',
5274             'description': '',
5275             'tags': [],
5276             'view_count': int,
5277             'modified_date': '20201130',
5278             'channel': 'Sergey M.',
5279             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5280             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5281             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5282             'availability': 'public',
5283         },
5284         'playlist_count': 1,
5285     }, {
5286         'note': 'empty playlist',
5287         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5288         'info_dict': {
5289             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5290             'uploader': 'Sergey M.',
5291             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5292             'title': 'youtube-dl empty playlist',
5293             'tags': [],
5294             'channel': 'Sergey M.',
5295             'description': '',
5296             'modified_date': '20160902',
5297             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5298             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5299             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5300             'availability': 'public',
5301         },
5302         'playlist_count': 0,
5303     }, {
5304         'note': 'Home tab',
5305         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5306         'info_dict': {
5307             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5308             'title': 'lex will - Home',
5309             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5310             'uploader': 'lex will',
5311             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5312             'channel': 'lex will',
5313             'tags': ['bible', 'history', 'prophesy'],
5314             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5315             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5316             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5317             'channel_follower_count': int
5318         },
5319         'playlist_mincount': 2,
5320     }, {
5321         'note': 'Videos tab',
5322         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5323         'info_dict': {
5324             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5325             'title': 'lex will - Videos',
5326             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5327             'uploader': 'lex will',
5328             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5329             'tags': ['bible', 'history', 'prophesy'],
5330             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5331             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5332             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5333             'channel': 'lex will',
5334             'channel_follower_count': int
5335         },
5336         'playlist_mincount': 975,
5337     }, {
5338         'note': 'Videos tab, sorted by popular',
5339         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5340         'info_dict': {
5341             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5342             'title': 'lex will - Videos',
5343             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5344             'uploader': 'lex will',
5345             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5346             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5347             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5348             'channel': 'lex will',
5349             'tags': ['bible', 'history', 'prophesy'],
5350             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5351             'channel_follower_count': int
5352         },
5353         'playlist_mincount': 199,
5354     }, {
5355         'note': 'Playlists tab',
5356         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5357         'info_dict': {
5358             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5359             'title': 'lex will - Playlists',
5360             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5361             'uploader': 'lex will',
5362             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5363             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5364             'channel': 'lex will',
5365             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5366             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5367             'tags': ['bible', 'history', 'prophesy'],
5368             'channel_follower_count': int
5369         },
5370         'playlist_mincount': 17,
5371     }, {
5372         'note': 'Community tab',
5373         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5374         'info_dict': {
5375             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5376             'title': 'lex will - Community',
5377             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5378             'uploader': 'lex will',
5379             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5380             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5381             'channel': 'lex will',
5382             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5383             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5384             'tags': ['bible', 'history', 'prophesy'],
5385             'channel_follower_count': int
5386         },
5387         'playlist_mincount': 18,
5388     }, {
5389         'note': 'Channels tab',
5390         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5391         'info_dict': {
5392             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5393             'title': 'lex will - Channels',
5394             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5395             'uploader': 'lex will',
5396             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5397             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5398             'channel': 'lex will',
5399             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5400             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5401             'tags': ['bible', 'history', 'prophesy'],
5402             'channel_follower_count': int
5403         },
5404         'playlist_mincount': 12,
5405     }, {
5406         'note': 'Search tab',
5407         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5408         'playlist_mincount': 40,
5409         'info_dict': {
5410             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5411             'title': '3Blue1Brown - Search - linear algebra',
5412             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5413             'uploader': '3Blue1Brown',
5414             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5415             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5416             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5417             'tags': ['Mathematics'],
5418             'channel': '3Blue1Brown',
5419             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5420             'channel_follower_count': int
5421         },
5422     }, {
5423         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5424         'only_matching': True,
5425     }, {
5426         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5427         'only_matching': True,
5428     }, {
5429         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5430         'only_matching': True,
5431     }, {
5432         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5433         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5434         'info_dict': {
5435             'title': '29C3: Not my department',
5436             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5437             'uploader': 'Christiaan008',
5438             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5439             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5440             'tags': [],
5441             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5442             'view_count': int,
5443             'modified_date': '20150605',
5444             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5445             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5446             'channel': 'Christiaan008',
5447             'availability': 'public',
5448         },
5449         'playlist_count': 96,
5450     }, {
5451         'note': 'Large playlist',
5452         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5453         'info_dict': {
5454             'title': 'Uploads from Cauchemar',
5455             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5456             'uploader': 'Cauchemar',
5457             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5458             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5459             'tags': [],
5460             'modified_date': r're:\d{8}',
5461             'channel': 'Cauchemar',
5462             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5463             'view_count': int,
5464             'description': '',
5465             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5466             'availability': 'public',
5467         },
5468         'playlist_mincount': 1123,
5469         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5470     }, {
5471         'note': 'even larger playlist, 8832 videos',
5472         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5473         'only_matching': True,
5474     }, {
5475         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5476         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5477         'info_dict': {
5478             'title': 'Uploads from Interstellar Movie',
5479             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5480             'uploader': 'Interstellar Movie',
5481             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5482             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5483             'tags': [],
5484             'view_count': int,
5485             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5486             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5487             'channel': 'Interstellar Movie',
5488             'description': '',
5489             'modified_date': r're:\d{8}',
5490             'availability': 'public',
5491         },
5492         'playlist_mincount': 21,
5493     }, {
5494         'note': 'Playlist with "show unavailable videos" button',
5495         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5496         'info_dict': {
5497             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5498             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5499             'uploader': 'Phim Siêu Nhân Nhật Bản',
5500             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5501             'view_count': int,
5502             'channel': 'Phim Siêu Nhân Nhật Bản',
5503             'tags': [],
5504             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5505             'description': '',
5506             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5507             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5508             'modified_date': r're:\d{8}',
5509             'availability': 'public',
5510         },
5511         'playlist_mincount': 200,
5512         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5513     }, {
5514         'note': 'Playlist with unavailable videos in page 7',
5515         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5516         'info_dict': {
5517             'title': 'Uploads from BlankTV',
5518             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5519             'uploader': 'BlankTV',
5520             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5521             'channel': 'BlankTV',
5522             'channel_url': 'https://www.youtube.com/c/blanktv',
5523             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5524             'view_count': int,
5525             'tags': [],
5526             'uploader_url': 'https://www.youtube.com/c/blanktv',
5527             'modified_date': r're:\d{8}',
5528             'description': '',
5529             'availability': 'public',
5530         },
5531         'playlist_mincount': 1000,
5532         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5533     }, {
5534         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5535         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5536         'info_dict': {
5537             'title': 'Data Analysis with Dr Mike Pound',
5538             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5539             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5540             'uploader': 'Computerphile',
5541             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5542             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5543             'tags': [],
5544             'view_count': int,
5545             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5546             'channel_url': 'https://www.youtube.com/user/Computerphile',
5547             'channel': 'Computerphile',
5548             'availability': 'public',
5549             'modified_date': '20190712',
5550         },
5551         'playlist_mincount': 11,
5552     }, {
5553         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5554         'only_matching': True,
5555     }, {
5556         'note': 'Playlist URL that does not actually serve a playlist',
5557         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5558         'info_dict': {
5559             'id': 'FqZTN594JQw',
5560             'ext': 'webm',
5561             'title': "Smiley's People 01 detective, Adventure Series, Action",
5562             'uploader': 'STREEM',
5563             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5564             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5565             'upload_date': '20150526',
5566             'license': 'Standard YouTube License',
5567             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5568             'categories': ['People & Blogs'],
5569             'tags': list,
5570             'view_count': int,
5571             'like_count': int,
5572         },
5573         'params': {
5574             'skip_download': True,
5575         },
5576         'skip': 'This video is not available.',
5577         'add_ie': [YoutubeIE.ie_key()],
5578     }, {
5579         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5580         'only_matching': True,
5581     }, {
5582         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5583         'only_matching': True,
5584     }, {
5585         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5586         'info_dict': {
5587             'id': 'Wq15eF5vCbI',  # This will keep changing
5588             'ext': 'mp4',
5589             'title': str,
5590             'uploader': 'Sky News',
5591             'uploader_id': 'skynews',
5592             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5593             'upload_date': r're:\d{8}',
5594             'description': str,
5595             'categories': ['News & Politics'],
5596             'tags': list,
5597             'like_count': int,
5598             'release_timestamp': int,
5599             'channel': 'Sky News',
5600             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5601             'age_limit': 0,
5602             'view_count': int,
5603             'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
5604             'playable_in_embed': True,
5605             'release_date': r're:\d+',
5606             'availability': 'public',
5607             'live_status': 'is_live',
5608             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5609             'channel_follower_count': int,
5610             'concurrent_view_count': int,
5611         },
5612         'params': {
5613             'skip_download': True,
5614         },
5615         'expected_warnings': ['Ignoring subtitle tracks found in '],
5616     }, {
5617         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5618         'info_dict': {
5619             'id': 'a48o2S1cPoo',
5620             'ext': 'mp4',
5621             'title': 'The Young Turks - Live Main Show',
5622             'uploader': 'The Young Turks',
5623             'uploader_id': 'TheYoungTurks',
5624             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5625             'upload_date': '20150715',
5626             'license': 'Standard YouTube License',
5627             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5628             'categories': ['News & Politics'],
5629             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5630             'like_count': int,
5631         },
5632         'params': {
5633             'skip_download': True,
5634         },
5635         'only_matching': True,
5636     }, {
5637         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5638         'only_matching': True,
5639     }, {
5640         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5641         'only_matching': True,
5642     }, {
5643         'note': 'A channel that is not live. Should raise error',
5644         'url': 'https://www.youtube.com/user/numberphile/live',
5645         'only_matching': True,
5646     }, {
5647         'url': 'https://www.youtube.com/feed/trending',
5648         'only_matching': True,
5649     }, {
5650         'url': 'https://www.youtube.com/feed/library',
5651         'only_matching': True,
5652     }, {
5653         'url': 'https://www.youtube.com/feed/history',
5654         'only_matching': True,
5655     }, {
5656         'url': 'https://www.youtube.com/feed/subscriptions',
5657         'only_matching': True,
5658     }, {
5659         'url': 'https://www.youtube.com/feed/watch_later',
5660         'only_matching': True,
5661     }, {
5662         'note': 'Recommended - redirects to home page.',
5663         'url': 'https://www.youtube.com/feed/recommended',
5664         'only_matching': True,
5665     }, {
5666         'note': 'inline playlist with not always working continuations',
5667         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5668         'only_matching': True,
5669     }, {
5670         'url': 'https://www.youtube.com/course',
5671         'only_matching': True,
5672     }, {
5673         'url': 'https://www.youtube.com/zsecurity',
5674         'only_matching': True,
5675     }, {
5676         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5677         'only_matching': True,
5678     }, {
5679         'url': 'https://www.youtube.com/TheYoungTurks/live',
5680         'only_matching': True,
5681     }, {
5682         'url': 'https://www.youtube.com/hashtag/cctv9',
5683         'info_dict': {
5684             'id': 'cctv9',
5685             'title': '#cctv9',
5686             'tags': [],
5687         },
5688         'playlist_mincount': 300,  # not consistent but should be over 300
5689     }, {
5690         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5691         'only_matching': True,
5692     }, {
5693         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5694         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5695         'only_matching': True
5696     }, {
5697         'note': '/browse/ should redirect to /channel/',
5698         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5699         'only_matching': True
5700     }, {
5701         'note': 'VLPL, should redirect to playlist?list=PL...',
5702         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5703         'info_dict': {
5704             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5705             'uploader': 'NoCopyrightSounds',
5706             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5707             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5708             'title': 'NCS : All Releases 💿',
5709             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5710             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5711             'modified_date': r're:\d{8}',
5712             'view_count': int,
5713             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5714             'tags': [],
5715             'channel': 'NoCopyrightSounds',
5716             'availability': 'public',
5717         },
5718         'playlist_mincount': 166,
5719         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5720     }, {
5721         'note': 'Topic, should redirect to playlist?list=UU...',
5722         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5723         'info_dict': {
5724             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5725             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5726             'title': 'Uploads from Royalty Free Music - Topic',
5727             'uploader': 'Royalty Free Music - Topic',
5728             'tags': [],
5729             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5730             'channel': 'Royalty Free Music - Topic',
5731             'view_count': int,
5732             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5733             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5734             'modified_date': r're:\d{8}',
5735             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5736             'description': '',
5737             'availability': 'public',
5738         },
5739         'playlist_mincount': 101,
5740     }, {
5741         # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
5742         # Treat as a general feed
5743         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5744         'info_dict': {
5745             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5746             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5747             'tags': [],
5748         },
5749         'playlist_mincount': 9,
5750     }, {
5751         'note': 'Youtube music Album',
5752         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5753         'info_dict': {
5754             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5755             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5756             'tags': [],
5757             'view_count': int,
5758             'description': '',
5759             'availability': 'unlisted',
5760             'modified_date': r're:\d{8}',
5761         },
5762         'playlist_count': 50,
5763     }, {
5764         'note': 'unlisted single video playlist',
5765         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5766         'info_dict': {
5767             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5768             'uploader': 'colethedj',
5769             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5770             'title': 'yt-dlp unlisted playlist test',
5771             'availability': 'unlisted',
5772             'tags': [],
5773             'modified_date': '20220418',
5774             'channel': 'colethedj',
5775             'view_count': int,
5776             'description': '',
5777             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5778             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5779             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5780         },
5781         'playlist_count': 1,
5782     }, {
5783         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5784         'url': 'https://www.youtube.com/feed/recommended',
5785         'info_dict': {
5786             'id': 'recommended',
5787             'title': 'recommended',
5788             'tags': [],
5789         },
5790         'playlist_mincount': 50,
5791         'params': {
5792             'skip_download': True,
5793             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5794         },
5795     }, {
5796         'note': 'API Fallback: /videos tab, sorted by oldest first',
5797         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5798         'info_dict': {
5799             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5800             'title': 'Cody\'sLab - Videos',
5801             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5802             'uploader': 'Cody\'sLab',
5803             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5804             'channel': 'Cody\'sLab',
5805             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5806             'tags': [],
5807             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5808             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5809             'channel_follower_count': int
5810         },
5811         'playlist_mincount': 650,
5812         'params': {
5813             'skip_download': True,
5814             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5815         },
5816         'skip': 'Query for sorting no longer works',
5817     }, {
5818         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5819         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5820         'info_dict': {
5821             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5822             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5823             'title': 'Uploads from Royalty Free Music - Topic',
5824             'uploader': 'Royalty Free Music - Topic',
5825             'modified_date': r're:\d{8}',
5826             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5827             'description': '',
5828             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5829             'tags': [],
5830             'channel': 'Royalty Free Music - Topic',
5831             'view_count': int,
5832             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5833             'availability': 'public',
5834         },
5835         'playlist_mincount': 101,
5836         'params': {
5837             'skip_download': True,
5838             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5839         },
5840     }, {
5841         'note': 'non-standard redirect to regional channel',
5842         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5843         'only_matching': True
5844     }, {
5845         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5846         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5847         'info_dict': {
5848             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5849             'modified_date': '20220407',
5850             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5851             'tags': [],
5852             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5853             'uploader': 'pukkandan',
5854             'availability': 'unlisted',
5855             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5856             'channel': 'pukkandan',
5857             'description': 'Test for collaborative playlist',
5858             'title': 'yt-dlp test - collaborative playlist',
5859             'view_count': int,
5860             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5861         },
5862         'playlist_mincount': 2
5863     }, {
5864         'note': 'translated tab name',
5865         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5866         'info_dict': {
5867             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5868             'tags': [],
5869             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5870             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5871             'description': 'test description',
5872             'title': 'cole-dlp-test-acc - 再生リスト',
5873             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5874             'uploader': 'cole-dlp-test-acc',
5875             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5876             'channel': 'cole-dlp-test-acc',
5877         },
5878         'playlist_mincount': 1,
5879         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5880         'expected_warnings': ['Preferring "ja"'],
5881     }, {
5882         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5883         'note': 'preferred lang set with playlist with translated video titles',
5884         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5885         'info_dict': {
5886             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5887             'tags': [],
5888             'view_count': int,
5889             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5890             'uploader': 'cole-dlp-test-acc',
5891             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5892             'channel': 'cole-dlp-test-acc',
5893             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5894             'description': 'test',
5895             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5896             'title': 'dlp test playlist',
5897             'availability': 'public',
5898         },
5899         'playlist_mincount': 1,
5900         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5901         'expected_warnings': ['Preferring "ja"'],
5902     }, {
5903         # shorts audio pivot for 2GtVksBMYFM.
5904         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
5905         'info_dict': {
5906             'id': 'sfv_audio_pivot',
5907             'title': 'sfv_audio_pivot',
5908             'tags': [],
5909         },
5910         'playlist_mincount': 50,
5911
5912     }, {
5913         # Channel with a real live tab (not to be mistaken with streams tab)
5914         # Do not treat like it should redirect to live stream
5915         'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
5916         'info_dict': {
5917             'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
5918             'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
5919             'tags': [],
5920         },
5921         'playlist_mincount': 20,
5922     }, {
5923         # Tab name is not the same as tab id
5924         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
5925         'info_dict': {
5926             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5927             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
5928             'tags': [],
5929         },
5930         'playlist_mincount': 8,
5931     }, {
5932         # Home tab id is literally home. Not to get mistaken with featured
5933         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
5934         'info_dict': {
5935             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5936             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
5937             'tags': [],
5938         },
5939         'playlist_mincount': 8,
5940     }, {
5941         # Should get three playlists for videos, shorts and streams tabs
5942         'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5943         'info_dict': {
5944             'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5945             'title': 'Polka Ch. 尾丸ポルカ',
5946             'channel_follower_count': int,
5947             'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5948             'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5949             'uploader': 'Polka Ch. 尾丸ポルカ',
5950             'description': 'md5:3b8df1ac5af337aa206e37ee3d181ec9',
5951             'channel': 'Polka Ch. 尾丸ポルカ',
5952             'tags': 'count:35',
5953             'uploader_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5954             'uploader_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5955         },
5956         'playlist_count': 3,
5957     }, {
5958         # Shorts tab with channel with handle
5959         'url': 'https://www.youtube.com/@NotJustBikes/shorts',
5960         'info_dict': {
5961             'id': 'UC0intLFzLaudFG-xAvUEO-A',
5962             'title': 'Not Just Bikes - Shorts',
5963             'tags': 'count:12',
5964             'uploader': 'Not Just Bikes',
5965             'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5966             'description': 'md5:7513148b1f02b924783157d84c4ea555',
5967             'channel_follower_count': int,
5968             'uploader_id': 'UC0intLFzLaudFG-xAvUEO-A',
5969             'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
5970             'uploader_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5971             'channel': 'Not Just Bikes',
5972         },
5973         'playlist_mincount': 10,
5974     }, {
5975         # Streams tab
5976         'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
5977         'info_dict': {
5978             'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5979             'title': '中村悠一 - Live',
5980             'tags': 'count:7',
5981             'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5982             'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
5983             'uploader_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
5984             'channel': '中村悠一',
5985             'uploader_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
5986             'channel_follower_count': int,
5987             'uploader': '中村悠一',
5988             'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
5989         },
5990         'playlist_mincount': 60,
5991     }, {
5992         # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
5993         # See test_youtube_lists
5994         'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
5995         'only_matching': True,
5996     }, {
5997         # No uploads and no UCID given. Should fail with no uploads error
5998         # See test_youtube_lists
5999         'url': 'https://www.youtube.com/news',
6000         'only_matching': True
6001     }, {
6002         # No videos tab but has a shorts tab
6003         'url': 'https://www.youtube.com/c/TKFShorts',
6004         'info_dict': {
6005             'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6006             'title': 'Shorts Break - Shorts',
6007             'tags': 'count:32',
6008             'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6009             'channel': 'Shorts Break',
6010             'description': 'md5:a6c234cf3d50d878ef8721e34457cd11',
6011             'uploader': 'Shorts Break',
6012             'channel_follower_count': int,
6013             'uploader_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6014             'uploader_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
6015             'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
6016         },
6017         'playlist_mincount': 30,
6018     }, {
6019         # Trending Now Tab. tab id is empty
6020         'url': 'https://www.youtube.com/feed/trending',
6021         'info_dict': {
6022             'id': 'trending',
6023             'title': 'trending - Now',
6024             'tags': [],
6025         },
6026         'playlist_mincount': 30,
6027     }, {
6028         # Trending Gaming Tab. tab id is empty
6029         'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
6030         'info_dict': {
6031             'id': 'trending',
6032             'title': 'trending - Gaming',
6033             'tags': [],
6034         },
6035         'playlist_mincount': 30,
6036     }, {
6037         # Shorts url result in shorts tab
6038         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
6039         'info_dict': {
6040             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6041             'title': 'cole-dlp-test-acc - Shorts',
6042             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
6043             'channel': 'cole-dlp-test-acc',
6044             'description': 'test description',
6045             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6046             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6047             'tags': [],
6048             'uploader': 'cole-dlp-test-acc',
6049             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6050
6051         },
6052         'playlist': [{
6053             'info_dict': {
6054                 '_type': 'url',
6055                 'ie_key': 'Youtube',
6056                 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
6057                 'id': 'sSM9J5YH_60',
6058                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6059                 'title': 'SHORT short',
6060                 'channel': 'cole-dlp-test-acc',
6061                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6062                 'view_count': int,
6063                 'thumbnails': list,
6064             }
6065         }],
6066         'params': {'extract_flat': True},
6067     }, {
6068         # Live video status should be extracted
6069         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
6070         'info_dict': {
6071             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6072             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live',  # TODO, should be Minecraft - Live or Minecraft - Topic - Live
6073             'tags': []
6074         },
6075         'playlist': [{
6076             'info_dict': {
6077                 '_type': 'url',
6078                 'ie_key': 'Youtube',
6079                 'url': 'startswith:https://www.youtube.com/watch?v=',
6080                 'id': str,
6081                 'title': str,
6082                 'live_status': 'is_live',
6083                 'channel_id': str,
6084                 'channel_url': str,
6085                 'concurrent_view_count': int,
6086                 'channel': str,
6087             }
6088         }],
6089         'params': {'extract_flat': True, 'playlist_items': '1'},
6090         'playlist_mincount': 1
6091     }, {
6092         # Channel renderer metadata. Contains number of videos on the channel
6093         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
6094         'info_dict': {
6095             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6096             'title': 'cole-dlp-test-acc - Channels',
6097             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
6098             'channel': 'cole-dlp-test-acc',
6099             'description': 'test description',
6100             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6101             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6102             'tags': [],
6103             'uploader': 'cole-dlp-test-acc',
6104             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6105
6106         },
6107         'playlist': [{
6108             'info_dict': {
6109                 '_type': 'url',
6110                 'ie_key': 'YoutubeTab',
6111                 'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6112                 'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6113                 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6114                 'title': 'PewDiePie',
6115                 'channel': 'PewDiePie',
6116                 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6117                 'thumbnails': list,
6118                 'channel_follower_count': int,
6119                 'playlist_count': int
6120             }
6121         }],
6122         'params': {'extract_flat': True},
6123     }, {
6124         'url': 'https://www.youtube.com/@3blue1brown/about',
6125         'info_dict': {
6126             'id': 'UCYO_jab_esuFRV4b17AJtAw',
6127             'tags': ['Mathematics'],
6128             'title': '3Blue1Brown - About',
6129             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
6130             'channel_follower_count': int,
6131             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
6132             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
6133             'channel': '3Blue1Brown',
6134             'uploader': '3Blue1Brown',
6135             'view_count': int,
6136             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
6137             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
6138         },
6139         'playlist_count': 0,
6140     }]
6141
6142     @classmethod
6143     def suitable(cls, url):
6144         return False if YoutubeIE.suitable(url) else super().suitable(url)
6145
6146     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
6147
6148     def _get_url_mobj(self, url):
6149         mobj = self._URL_RE.match(url).groupdict()
6150         mobj.update((k, '') for k, v in mobj.items() if v is None)
6151         return mobj
6152
6153     def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
6154         tab_name = (tab.get('title') or '').lower()
6155         tab_url = urljoin(base_url, traverse_obj(
6156             tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
6157
6158         tab_id = (tab_url and self._get_url_mobj(tab_url)['tab'][1:]
6159                   or traverse_obj(tab, 'tabIdentifier', expected_type=str))
6160         if tab_id:
6161             return {
6162                 'TAB_ID_SPONSORSHIPS': 'membership',
6163             }.get(tab_id, tab_id), tab_name
6164
6165         # Fallback to tab name if we cannot get the tab id.
6166         # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
6167         # Note that in the case of translated tab name this may result in an empty string, which we don't want.
6168         if tab_name:
6169             self.write_debug(f'Falling back to selected tab name: {tab_name}')
6170         return {
6171             'home': 'featured',
6172             'live': 'streams',
6173         }.get(tab_name, tab_name), tab_name
6174
6175     def _has_tab(self, tabs, tab_id):
6176         return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
6177
6178     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
6179     def _real_extract(self, url, smuggled_data):
6180         item_id = self._match_id(url)
6181         url = urllib.parse.urlunparse(
6182             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
6183         compat_opts = self.get_param('compat_opts', [])
6184
6185         mobj = self._get_url_mobj(url)
6186         pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
6187         if is_channel and smuggled_data.get('is_music_url'):
6188             if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
6189                 return self.url_result(
6190                     f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
6191             elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
6192                 mdata = self._extract_tab_endpoint(
6193                     f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
6194                 murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
6195                                     get_all=False, expected_type=str)
6196                 if not murl:
6197                     raise ExtractorError('Failed to resolve album to playlist')
6198                 return self.url_result(murl, YoutubeTabIE)
6199             elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
6200                 return self.url_result(
6201                     f'https://music.youtube.com/channel/{item_id}{tab}{post}', YoutubeTabIE, item_id)
6202
6203         original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
6204         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
6205             url = f'{pre}/videos{post}'
6206         if smuggled_data.get('is_music_url'):
6207             self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}')
6208
6209         # Handle both video/playlist URLs
6210         qs = parse_qs(url)
6211         video_id, playlist_id = [traverse_obj(qs, (key, 0)) for key in ('v', 'list')]
6212         if not video_id and mobj['not_channel'].startswith('watch'):
6213             if not playlist_id:
6214                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
6215                 raise ExtractorError('A video URL was given without video ID', expected=True)
6216             # Common mistake: https://www.youtube.com/watch?list=playlist_id
6217             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
6218             return self.url_result(
6219                 f'https://www.youtube.com/playlist?list={playlist_id}', YoutubeTabIE, playlist_id)
6220
6221         if not self._yes_playlist(playlist_id, video_id):
6222             return self.url_result(
6223                 f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6224
6225         data, ytcfg = self._extract_data(url, display_id)
6226
6227         # YouTube may provide a non-standard redirect to the regional channel
6228         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
6229         # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
6230         redirect_url = traverse_obj(
6231             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
6232         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
6233             redirect_url = ''.join((urljoin('https://www.youtube.com', redirect_url), tab, post))
6234             self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
6235             return self.url_result(redirect_url, YoutubeTabIE)
6236
6237         tabs, extra_tabs = self._extract_tab_renderers(data), []
6238         if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
6239             selected_tab = self._extract_selected_tab(tabs)
6240             selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url)  # NB: Name may be translated
6241             self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
6242
6243             if not original_tab_id and selected_tab_name:
6244                 self.to_screen('Downloading all uploads of the channel. '
6245                                'To download only the videos in a specific tab, pass the tab\'s URL')
6246                 if self._has_tab(tabs, 'streams'):
6247                     extra_tabs.append(''.join((pre, '/streams', post)))
6248                 if self._has_tab(tabs, 'shorts'):
6249                     extra_tabs.append(''.join((pre, '/shorts', post)))
6250                 # XXX: Members-only tab should also be extracted
6251
6252                 if not extra_tabs and selected_tab_id != 'videos':
6253                     # Channel does not have streams, shorts or videos tabs
6254                     if item_id[:2] != 'UC':
6255                         raise ExtractorError('This channel has no uploads', expected=True)
6256
6257                     # Topic channels don't have /videos. Use the equivalent playlist instead
6258                     pl_id = f'UU{item_id[2:]}'
6259                     pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
6260                     try:
6261                         data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
6262                     except ExtractorError:
6263                         raise ExtractorError('This channel has no uploads', expected=True)
6264                     else:
6265                         item_id, url = pl_id, pl_url
6266                         self.to_screen(
6267                             f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
6268
6269                 elif extra_tabs and selected_tab_id != 'videos':
6270                     # When there are shorts/live tabs but not videos tab
6271                     url, data = f'{pre}{post}', None
6272
6273             elif (original_tab_id or 'videos') != selected_tab_id:
6274                 if original_tab_id == 'live':
6275                     # Live tab should have redirected to the video
6276                     # Except in the case the channel has an actual live tab
6277                     # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
6278                     raise UserNotLive(video_id=item_id)
6279                 elif selected_tab_name:
6280                     raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
6281
6282                 # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
6283                 url = f'{pre}{post}'
6284
6285         # YouTube sometimes provides a button to reload playlist with unavailable videos.
6286         if 'no-youtube-unavailable-videos' not in compat_opts:
6287             data = self._reload_with_unavailable_videos(display_id, data, ytcfg) or data
6288         self._extract_and_report_alerts(data, only_once=True)
6289
6290         tabs, entries = self._extract_tab_renderers(data), []
6291         if tabs:
6292             entries = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
6293             entries[0].update({
6294                 'extractor_key': YoutubeTabIE.ie_key(),
6295                 'extractor': YoutubeTabIE.IE_NAME,
6296                 'webpage_url': url,
6297             })
6298         if self.get_param('playlist_items') == '0':
6299             entries.extend(self.url_result(u, YoutubeTabIE) for u in extra_tabs)
6300         else:  # Users expect to get all `video_id`s even with `--flat-playlist`. So don't return `url_result`
6301             entries.extend(map(self._real_extract, extra_tabs))
6302
6303         if len(entries) == 1:
6304             return entries[0]
6305         elif entries:
6306             metadata = self._extract_metadata_from_tabs(item_id, data)
6307             uploads_url = 'the Uploads (UU) playlist URL'
6308             if try_get(metadata, lambda x: x['channel_id'].startswith('UC')):
6309                 uploads_url = f'https://www.youtube.com/playlist?list=UU{metadata["channel_id"][2:]}'
6310             self.to_screen(
6311                 'Downloading as multiple playlists, separated by tabs. '
6312                 f'To download as a single playlist instead, pass {uploads_url}')
6313             return self.playlist_result(entries, item_id, **metadata)
6314
6315         # Inline playlist
6316         playlist = traverse_obj(
6317             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
6318         if playlist:
6319             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
6320
6321         video_id = traverse_obj(
6322             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
6323         if video_id:
6324             if tab != '/live':  # live tab is expected to redirect to video
6325                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
6326             return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6327
6328         raise ExtractorError('Unable to recognize tab page')
6329
6330
6331 class YoutubePlaylistIE(InfoExtractor):
6332     IE_DESC = 'YouTube playlists'
6333     _VALID_URL = r'''(?x)(?:
6334                         (?:https?://)?
6335                         (?:\w+\.)?
6336                         (?:
6337                             (?:
6338                                 youtube(?:kids)?\.com|
6339                                 %(invidious)s
6340                             )
6341                             /.*?\?.*?\blist=
6342                         )?
6343                         (?P<id>%(playlist_id)s)
6344                      )''' % {
6345         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
6346         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
6347     }
6348     IE_NAME = 'youtube:playlist'
6349     _TESTS = [{
6350         'note': 'issue #673',
6351         'url': 'PLBB231211A4F62143',
6352         'info_dict': {
6353             'title': '[OLD]Team Fortress 2 (Class-based LP)',
6354             'id': 'PLBB231211A4F62143',
6355             'uploader': 'Wickman',
6356             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6357             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
6358             'view_count': int,
6359             'uploader_url': 'https://www.youtube.com/c/WickmanVT',
6360             'modified_date': r're:\d{8}',
6361             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6362             'channel': 'Wickman',
6363             'tags': [],
6364             'channel_url': 'https://www.youtube.com/c/WickmanVT',
6365             'availability': 'public',
6366         },
6367         'playlist_mincount': 29,
6368     }, {
6369         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6370         'info_dict': {
6371             'title': 'YDL_safe_search',
6372             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6373         },
6374         'playlist_count': 2,
6375         'skip': 'This playlist is private',
6376     }, {
6377         'note': 'embedded',
6378         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6379         'playlist_count': 4,
6380         'info_dict': {
6381             'title': 'JODA15',
6382             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6383             'uploader': 'milan',
6384             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6385             'description': '',
6386             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6387             'tags': [],
6388             'modified_date': '20140919',
6389             'view_count': int,
6390             'channel': 'milan',
6391             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6392             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6393             'availability': 'public',
6394         },
6395         'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'],
6396     }, {
6397         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6398         'playlist_mincount': 455,
6399         'info_dict': {
6400             'title': '2018 Chinese New Singles (11/6 updated)',
6401             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6402             'uploader': 'LBK',
6403             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6404             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
6405             'channel': 'LBK',
6406             'view_count': int,
6407             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
6408             'tags': [],
6409             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
6410             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6411             'modified_date': r're:\d{8}',
6412             'availability': 'public',
6413         },
6414         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
6415     }, {
6416         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
6417         'only_matching': True,
6418     }, {
6419         # music album playlist
6420         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
6421         'only_matching': True,
6422     }]
6423
6424     @classmethod
6425     def suitable(cls, url):
6426         if YoutubeTabIE.suitable(url):
6427             return False
6428         from ..utils import parse_qs
6429         qs = parse_qs(url)
6430         if qs.get('v', [None])[0]:
6431             return False
6432         return super().suitable(url)
6433
6434     def _real_extract(self, url):
6435         playlist_id = self._match_id(url)
6436         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
6437         url = update_url_query(
6438             'https://www.youtube.com/playlist',
6439             parse_qs(url) or {'list': playlist_id})
6440         if is_music_url:
6441             url = smuggle_url(url, {'is_music_url': True})
6442         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6443
6444
6445 class YoutubeYtBeIE(InfoExtractor):
6446     IE_DESC = 'youtu.be'
6447     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
6448     _TESTS = [{
6449         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
6450         'info_dict': {
6451             'id': 'yeWKywCrFtk',
6452             'ext': 'mp4',
6453             'title': 'Small Scale Baler and Braiding Rugs',
6454             'uploader': 'Backus-Page House Museum',
6455             'uploader_id': 'backuspagemuseum',
6456             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
6457             'upload_date': '20161008',
6458             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
6459             'categories': ['Nonprofits & Activism'],
6460             'tags': list,
6461             'like_count': int,
6462             'age_limit': 0,
6463             'playable_in_embed': True,
6464             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
6465             'channel': 'Backus-Page House Museum',
6466             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
6467             'live_status': 'not_live',
6468             'view_count': int,
6469             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
6470             'availability': 'public',
6471             'duration': 59,
6472             'comment_count': int,
6473             'channel_follower_count': int
6474         },
6475         'params': {
6476             'noplaylist': True,
6477             'skip_download': True,
6478         },
6479     }, {
6480         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
6481         'only_matching': True,
6482     }]
6483
6484     def _real_extract(self, url):
6485         mobj = self._match_valid_url(url)
6486         video_id = mobj.group('id')
6487         playlist_id = mobj.group('playlist_id')
6488         return self.url_result(
6489             update_url_query('https://www.youtube.com/watch', {
6490                 'v': video_id,
6491                 'list': playlist_id,
6492                 'feature': 'youtu.be',
6493             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6494
6495
6496 class YoutubeLivestreamEmbedIE(InfoExtractor):
6497     IE_DESC = 'YouTube livestream embeds'
6498     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6499     _TESTS = [{
6500         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6501         'only_matching': True,
6502     }]
6503
6504     def _real_extract(self, url):
6505         channel_id = self._match_id(url)
6506         return self.url_result(
6507             f'https://www.youtube.com/channel/{channel_id}/live',
6508             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6509
6510
6511 class YoutubeYtUserIE(InfoExtractor):
6512     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6513     IE_NAME = 'youtube:user'
6514     _VALID_URL = r'ytuser:(?P<id>.+)'
6515     _TESTS = [{
6516         'url': 'ytuser:phihag',
6517         'only_matching': True,
6518     }]
6519
6520     def _real_extract(self, url):
6521         user_id = self._match_id(url)
6522         return self.url_result(f'https://www.youtube.com/user/{user_id}', YoutubeTabIE, user_id)
6523
6524
6525 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6526     IE_NAME = 'youtube:favorites'
6527     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6528     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6529     _LOGIN_REQUIRED = True
6530     _TESTS = [{
6531         'url': ':ytfav',
6532         'only_matching': True,
6533     }, {
6534         'url': ':ytfavorites',
6535         'only_matching': True,
6536     }]
6537
6538     def _real_extract(self, url):
6539         return self.url_result(
6540             'https://www.youtube.com/playlist?list=LL',
6541             ie=YoutubeTabIE.ie_key())
6542
6543
6544 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6545     IE_NAME = 'youtube:notif'
6546     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6547     _VALID_URL = r':ytnotif(?:ication)?s?'
6548     _LOGIN_REQUIRED = True
6549     _TESTS = [{
6550         'url': ':ytnotif',
6551         'only_matching': True,
6552     }, {
6553         'url': ':ytnotifications',
6554         'only_matching': True,
6555     }]
6556
6557     def _extract_notification_menu(self, response, continuation_list):
6558         notification_list = traverse_obj(
6559             response,
6560             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6561             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6562             expected_type=list) or []
6563         continuation_list[0] = None
6564         for item in notification_list:
6565             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6566             if entry:
6567                 yield entry
6568             continuation = item.get('continuationItemRenderer')
6569             if continuation:
6570                 continuation_list[0] = continuation
6571
6572     def _extract_notification_renderer(self, notification):
6573         video_id = traverse_obj(
6574             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6575         url = f'https://www.youtube.com/watch?v={video_id}'
6576         channel_id = None
6577         if not video_id:
6578             browse_ep = traverse_obj(
6579                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6580             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
6581             post_id = self._search_regex(
6582                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6583                 'post id', default=None)
6584             if not channel_id or not post_id:
6585                 return
6586             # The direct /post url redirects to this in the browser
6587             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6588
6589         channel = traverse_obj(
6590             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6591             expected_type=str)
6592         notification_title = self._get_text(notification, 'shortMessage')
6593         if notification_title:
6594             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6595         # TODO: handle recommended videos
6596         title = self._search_regex(
6597             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6598             'video title', default=None)
6599         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6600                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6601                      else None)
6602         return {
6603             '_type': 'url',
6604             'url': url,
6605             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6606             'video_id': video_id,
6607             'title': title,
6608             'channel_id': channel_id,
6609             'channel': channel,
6610             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6611             'timestamp': timestamp,
6612         }
6613
6614     def _notification_menu_entries(self, ytcfg):
6615         continuation_list = [None]
6616         response = None
6617         for page in itertools.count(1):
6618             ctoken = traverse_obj(
6619                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6620             response = self._extract_response(
6621                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6622                 ep='notification/get_notification_menu', check_get_keys='actions',
6623                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6624             yield from self._extract_notification_menu(response, continuation_list)
6625             if not continuation_list[0]:
6626                 break
6627
6628     def _real_extract(self, url):
6629         display_id = 'notifications'
6630         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6631         self._report_playlist_authcheck(ytcfg)
6632         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6633
6634
6635 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6636     IE_DESC = 'YouTube search'
6637     IE_NAME = 'youtube:search'
6638     _SEARCH_KEY = 'ytsearch'
6639     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6640     _TESTS = [{
6641         'url': 'ytsearch5:youtube-dl test video',
6642         'playlist_count': 5,
6643         'info_dict': {
6644             'id': 'youtube-dl test video',
6645             'title': 'youtube-dl test video',
6646         }
6647     }]
6648
6649
6650 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6651     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6652     _SEARCH_KEY = 'ytsearchdate'
6653     IE_DESC = 'YouTube search, newest videos first'
6654     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6655     _TESTS = [{
6656         'url': 'ytsearchdate5:youtube-dl test video',
6657         'playlist_count': 5,
6658         'info_dict': {
6659             'id': 'youtube-dl test video',
6660             'title': 'youtube-dl test video',
6661         }
6662     }]
6663
6664
6665 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6666     IE_DESC = 'YouTube search URLs with sorting and filter support'
6667     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6668     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6669     _TESTS = [{
6670         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6671         'playlist_mincount': 5,
6672         'info_dict': {
6673             'id': 'youtube-dl test video',
6674             'title': 'youtube-dl test video',
6675         }
6676     }, {
6677         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6678         'playlist_mincount': 5,
6679         'info_dict': {
6680             'id': 'python',
6681             'title': 'python',
6682         }
6683     }, {
6684         'url': 'https://www.youtube.com/results?search_query=%23cats',
6685         'playlist_mincount': 1,
6686         'info_dict': {
6687             'id': '#cats',
6688             'title': '#cats',
6689             # The test suite does not have support for nested playlists
6690             # 'entries': [{
6691             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6692             #     'title': '#cats',
6693             # }],
6694         },
6695     }, {
6696         # Channel results
6697         'url': 'https://www.youtube.com/results?search_query=kurzgesagt&sp=EgIQAg%253D%253D',
6698         'info_dict': {
6699             'id': 'kurzgesagt',
6700             'title': 'kurzgesagt',
6701         },
6702         'playlist': [{
6703             'info_dict': {
6704                 '_type': 'url',
6705                 'id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6706                 'url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6707                 'ie_key': 'YoutubeTab',
6708                 'channel': 'Kurzgesagt – In a Nutshell',
6709                 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
6710                 'title': 'Kurzgesagt – In a Nutshell',
6711                 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6712                 'playlist_count': int,  # XXX: should have a way of saying > 1
6713                 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6714                 'thumbnails': list
6715             }
6716         }],
6717         'params': {'extract_flat': True, 'playlist_items': '1'},
6718         'playlist_mincount': 1,
6719     }, {
6720         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6721         'only_matching': True,
6722     }]
6723
6724     def _real_extract(self, url):
6725         qs = parse_qs(url)
6726         query = (qs.get('search_query') or qs.get('q'))[0]
6727         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6728
6729
6730 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6731     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6732     IE_NAME = 'youtube:music:search_url'
6733     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6734     _TESTS = [{
6735         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6736         'playlist_count': 16,
6737         'info_dict': {
6738             'id': 'royalty free music',
6739             'title': 'royalty free music',
6740         }
6741     }, {
6742         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6743         'playlist_mincount': 30,
6744         'info_dict': {
6745             'id': 'royalty free music - songs',
6746             'title': 'royalty free music - songs',
6747         },
6748         'params': {'extract_flat': 'in_playlist'}
6749     }, {
6750         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6751         'playlist_mincount': 30,
6752         'info_dict': {
6753             'id': 'royalty free music - community playlists',
6754             'title': 'royalty free music - community playlists',
6755         },
6756         'params': {'extract_flat': 'in_playlist'}
6757     }]
6758
6759     _SECTIONS = {
6760         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6761         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6762         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6763         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6764         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6765         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6766     }
6767
6768     def _real_extract(self, url):
6769         qs = parse_qs(url)
6770         query = (qs.get('search_query') or qs.get('q'))[0]
6771         params = qs.get('sp', (None,))[0]
6772         if params:
6773             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6774         else:
6775             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6776             params = self._SECTIONS.get(section)
6777             if not params:
6778                 section = None
6779         title = join_nonempty(query, section, delim=' - ')
6780         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6781
6782
6783 class YoutubeFeedsInfoExtractor(InfoExtractor):
6784     """
6785     Base class for feed extractors
6786     Subclasses must re-define the _FEED_NAME property.
6787     """
6788     _LOGIN_REQUIRED = True
6789     _FEED_NAME = 'feeds'
6790
6791     def _real_initialize(self):
6792         YoutubeBaseInfoExtractor._check_login_required(self)
6793
6794     @classproperty
6795     def IE_NAME(self):
6796         return f'youtube:{self._FEED_NAME}'
6797
6798     def _real_extract(self, url):
6799         return self.url_result(
6800             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6801
6802
6803 class YoutubeWatchLaterIE(InfoExtractor):
6804     IE_NAME = 'youtube:watchlater'
6805     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6806     _VALID_URL = r':ytwatchlater'
6807     _TESTS = [{
6808         'url': ':ytwatchlater',
6809         'only_matching': True,
6810     }]
6811
6812     def _real_extract(self, url):
6813         return self.url_result(
6814             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6815
6816
6817 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6818     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6819     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6820     _FEED_NAME = 'recommended'
6821     _LOGIN_REQUIRED = False
6822     _TESTS = [{
6823         'url': ':ytrec',
6824         'only_matching': True,
6825     }, {
6826         'url': ':ytrecommended',
6827         'only_matching': True,
6828     }, {
6829         'url': 'https://youtube.com',
6830         'only_matching': True,
6831     }]
6832
6833
6834 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6835     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6836     _VALID_URL = r':ytsub(?:scription)?s?'
6837     _FEED_NAME = 'subscriptions'
6838     _TESTS = [{
6839         'url': ':ytsubs',
6840         'only_matching': True,
6841     }, {
6842         'url': ':ytsubscriptions',
6843         'only_matching': True,
6844     }]
6845
6846
6847 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6848     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6849     _VALID_URL = r':ythis(?:tory)?'
6850     _FEED_NAME = 'history'
6851     _TESTS = [{
6852         'url': ':ythistory',
6853         'only_matching': True,
6854     }]
6855
6856
6857 class YoutubeStoriesIE(InfoExtractor):
6858     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6859     IE_NAME = 'youtube:stories'
6860     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6861     _TESTS = [{
6862         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6863         'only_matching': True,
6864     }]
6865
6866     def _real_extract(self, url):
6867         playlist_id = f'RLTD{self._match_id(url)}'
6868         return self.url_result(
6869             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6870             ie=YoutubeTabIE, video_id=playlist_id)
6871
6872
6873 class YoutubeShortsAudioPivotIE(InfoExtractor):
6874     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
6875     IE_NAME = 'youtube:shorts:pivot:audio'
6876     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
6877     _TESTS = [{
6878         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
6879         'only_matching': True,
6880     }]
6881
6882     @staticmethod
6883     def _generate_audio_pivot_params(video_id):
6884         """
6885         Generates sfv_audio_pivot browse params for this video id
6886         """
6887         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
6888         return urllib.parse.quote(base64.b64encode(pb_params).decode())
6889
6890     def _real_extract(self, url):
6891         video_id = self._match_id(url)
6892         return self.url_result(
6893             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
6894             ie=YoutubeTabIE)
6895
6896
6897 class YoutubeTruncatedURLIE(InfoExtractor):
6898     IE_NAME = 'youtube:truncated_url'
6899     IE_DESC = False  # Do not list
6900     _VALID_URL = r'''(?x)
6901         (?:https?://)?
6902         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6903         (?:watch\?(?:
6904             feature=[a-z_]+|
6905             annotation_id=annotation_[^&]+|
6906             x-yt-cl=[0-9]+|
6907             hl=[^&]*|
6908             t=[0-9]+
6909         )?
6910         |
6911             attribution_link\?a=[^&]+
6912         )
6913         $
6914     '''
6915
6916     _TESTS = [{
6917         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6918         'only_matching': True,
6919     }, {
6920         'url': 'https://www.youtube.com/watch?',
6921         'only_matching': True,
6922     }, {
6923         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6924         'only_matching': True,
6925     }, {
6926         'url': 'https://www.youtube.com/watch?feature=foo',
6927         'only_matching': True,
6928     }, {
6929         'url': 'https://www.youtube.com/watch?hl=en-GB',
6930         'only_matching': True,
6931     }, {
6932         'url': 'https://www.youtube.com/watch?t=2372',
6933         'only_matching': True,
6934     }]
6935
6936     def _real_extract(self, url):
6937         raise ExtractorError(
6938             'Did you forget to quote the URL? Remember that & is a meta '
6939             'character in most shells, so you want to put the URL in quotes, '
6940             'like  youtube-dl '
6941             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6942             ' or simply  youtube-dl BaW_jenozKc  .',
6943             expected=True)
6944
6945
6946 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6947     IE_NAME = 'youtube:clip'
6948     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6949     _TESTS = [{
6950         # FIXME: Other metadata should be extracted from the clip, not from the base video
6951         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6952         'info_dict': {
6953             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6954             'ext': 'mp4',
6955             'section_start': 29.0,
6956             'section_end': 39.7,
6957             'duration': 10.7,
6958             'age_limit': 0,
6959             'availability': 'public',
6960             'categories': ['Gaming'],
6961             'channel': 'Scott The Woz',
6962             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6963             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6964             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6965             'like_count': int,
6966             'playable_in_embed': True,
6967             'tags': 'count:17',
6968             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6969             'title': 'Mobile Games on Console - Scott The Woz',
6970             'upload_date': '20210920',
6971             'uploader': 'Scott The Woz',
6972             'uploader_id': 'scottthewoz',
6973             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6974             'view_count': int,
6975             'live_status': 'not_live',
6976             'channel_follower_count': int
6977         }
6978     }]
6979
6980     def _real_extract(self, url):
6981         clip_id = self._match_id(url)
6982         _, data = self._extract_webpage(url, clip_id)
6983
6984         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
6985         if not video_id:
6986             raise ExtractorError('Unable to find video ID')
6987
6988         clip_data = traverse_obj(data, (
6989             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
6990             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
6991             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
6992             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
6993
6994         return {
6995             '_type': 'url_transparent',
6996             'url': f'https://www.youtube.com/watch?v={video_id}',
6997             'ie_key': YoutubeIE.ie_key(),
6998             'id': clip_id,
6999             'section_start': int(clip_data['startTimeMs']) / 1000,
7000             'section_end': int(clip_data['endTimeMs']) / 1000,
7001         }
7002
7003
7004 class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
7005     IE_NAME = 'youtube:consent'
7006     IE_DESC = False  # Do not list
7007     _VALID_URL = r'https?://consent\.youtube\.com/m\?'
7008     _TESTS = [{
7009         'url': 'https://consent.youtube.com/m?continue=https%3A%2F%2Fwww.youtube.com%2Flive%2FqVv6vCqciTM%3Fcbrd%3D1&gl=NL&m=0&pc=yt&hl=en&src=1',
7010         'info_dict': {
7011             'id': 'qVv6vCqciTM',
7012             'ext': 'mp4',
7013             'age_limit': 0,
7014             'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
7015             'comment_count': int,
7016             'chapters': 'count:13',
7017             'upload_date': '20221223',
7018             'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
7019             'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
7020             'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
7021             'like_count': int,
7022             'release_date': '20221223',
7023             'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
7024             'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
7025             'view_count': int,
7026             'playable_in_embed': True,
7027             'duration': 4438,
7028             'availability': 'public',
7029             'channel_follower_count': int,
7030             'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
7031             'categories': ['Entertainment'],
7032             'live_status': 'was_live',
7033             'release_timestamp': 1671793345,
7034             'channel': 'さなちゃんねる',
7035             'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
7036             'uploader': 'さなちゃんねる',
7037         },
7038         'add_ie': ['Youtube'],
7039         'params': {'skip_download': 'Youtube'},
7040     }]
7041
7042     def _real_extract(self, url):
7043         redirect_url = url_or_none(parse_qs(url).get('continue', [None])[-1])
7044         if not redirect_url:
7045             raise ExtractorError('Invalid cookie consent redirect URL', expected=True)
7046         return self.url_result(redirect_url)
7047
7048
7049 class YoutubeTruncatedIDIE(InfoExtractor):
7050     IE_NAME = 'youtube:truncated_id'
7051     IE_DESC = False  # Do not list
7052     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
7053
7054     _TESTS = [{
7055         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
7056         'only_matching': True,
7057     }]
7058
7059     def _real_extract(self, url):
7060         video_id = self._match_id(url)
7061         raise ExtractorError(
7062             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
7063             expected=True)