yt_dlp/extractor/youtube.py

   1 import base64
   2 import calendar
   3 import collections
   4 import copy
   5 import datetime
   6 import enum
   7 import hashlib
   8 import itertools
   9 import json
  10 import math
  11 import os.path
  12 import random
  13 import re
  14 import sys
  15 import threading
  16 import time
  17 import traceback
  18 import urllib.error
  19 import urllib.parse
  20
  21 from .common import InfoExtractor, SearchInfoExtractor
  22 from .openload import PhantomJSwrapper
  23 from ..compat import functools
  24 from ..jsinterp import JSInterpreter
  25 from ..utils import (
  26     NO_DEFAULT,
  27     ExtractorError,
  28     LazyList,
  29     UserNotLive,
  30     bug_reports_message,
  31     classproperty,
  32     clean_html,
  33     datetime_from_str,
  34     dict_get,
  35     filter_dict,
  36     float_or_none,
  37     format_field,
  38     get_first,
  39     int_or_none,
  40     is_html,
  41     join_nonempty,
  42     js_to_json,
  43     mimetype2ext,
  44     network_exceptions,
  45     orderedSet,
  46     parse_codecs,
  47     parse_count,
  48     parse_duration,
  49     parse_iso8601,
  50     parse_qs,
  51     qualities,
  52     remove_start,
  53     smuggle_url,
  54     str_or_none,
  55     str_to_int,
  56     strftime_or_none,
  57     traverse_obj,
  58     try_get,
  59     unescapeHTML,
  60     unified_strdate,
  61     unified_timestamp,
  62     unsmuggle_url,
  63     update_url_query,
  64     url_or_none,
  65     urljoin,
  66     variadic,
  67 )
  68
  69
  70 STREAMING_DATA_CLIENT_NAME = '__yt_dlp_client'
  71 # any clients starting with _ cannot be explicitly requested by the user
  72 INNERTUBE_CLIENTS = {
  73     'web': {
  74         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  75         'INNERTUBE_CONTEXT': {
  76             'client': {
  77                 'clientName': 'WEB',
  78                 'clientVersion': '2.20220801.00.00',
  79             }
  80         },
  81         'INNERTUBE_CONTEXT_CLIENT_NAME': 1
  82     },
  83     'web_embedded': {
  84         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  85         'INNERTUBE_CONTEXT': {
  86             'client': {
  87                 'clientName': 'WEB_EMBEDDED_PLAYER',
  88                 'clientVersion': '1.20220731.00.00',
  89             },
  90         },
  91         'INNERTUBE_CONTEXT_CLIENT_NAME': 56
  92     },
  93     'web_music': {
  94         'INNERTUBE_API_KEY': 'AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30',
  95         'INNERTUBE_HOST': 'music.youtube.com',
  96         'INNERTUBE_CONTEXT': {
  97             'client': {
  98                 'clientName': 'WEB_REMIX',
  99                 'clientVersion': '1.20220727.01.00',
 100             }
 101         },
 102         'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
 103     },
 104     'web_creator': {
 105         'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo',
 106         'INNERTUBE_CONTEXT': {
 107             'client': {
 108                 'clientName': 'WEB_CREATOR',
 109                 'clientVersion': '1.20220726.00.00',
 110             }
 111         },
 112         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
 113     },
 114     'android': {
 115         'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
 116         'INNERTUBE_CONTEXT': {
 117             'client': {
 118                 'clientName': 'ANDROID',
 119                 'clientVersion': '17.31.35',
 120                 'androidSdkVersion': 30,
 121                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 122             }
 123         },
 124         'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
 125         'REQUIRE_JS_PLAYER': False
 126     },
 127     'android_embedded': {
 128         'INNERTUBE_API_KEY': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
 129         'INNERTUBE_CONTEXT': {
 130             'client': {
 131                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
 132                 'clientVersion': '17.31.35',
 133                 'androidSdkVersion': 30,
 134                 'userAgent': 'com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip'
 135             },
 136         },
 137         'INNERTUBE_CONTEXT_CLIENT_NAME': 55,
 138         'REQUIRE_JS_PLAYER': False
 139     },
 140     'android_music': {
 141         'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
 142         'INNERTUBE_CONTEXT': {
 143             'client': {
 144                 'clientName': 'ANDROID_MUSIC',
 145                 'clientVersion': '5.16.51',
 146                 'androidSdkVersion': 30,
 147                 'userAgent': 'com.google.android.apps.youtube.music/5.16.51 (Linux; U; Android 11) gzip'
 148             }
 149         },
 150         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
 151         'REQUIRE_JS_PLAYER': False
 152     },
 153     'android_creator': {
 154         'INNERTUBE_API_KEY': 'AIzaSyD_qjV8zaaUMehtLkrKFgVeSX_Iqbtyws8',
 155         'INNERTUBE_CONTEXT': {
 156             'client': {
 157                 'clientName': 'ANDROID_CREATOR',
 158                 'clientVersion': '22.30.100',
 159                 'androidSdkVersion': 30,
 160                 'userAgent': 'com.google.android.apps.youtube.creator/22.30.100 (Linux; U; Android 11) gzip'
 161             },
 162         },
 163         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
 164         'REQUIRE_JS_PLAYER': False
 165     },
 166     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
 167     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
 168     'ios': {
 169         'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
 170         'INNERTUBE_CONTEXT': {
 171             'client': {
 172                 'clientName': 'IOS',
 173                 'clientVersion': '17.33.2',
 174                 'deviceModel': 'iPhone14,3',
 175                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 176             }
 177         },
 178         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
 179         'REQUIRE_JS_PLAYER': False
 180     },
 181     'ios_embedded': {
 182         'INNERTUBE_CONTEXT': {
 183             'client': {
 184                 'clientName': 'IOS_MESSAGES_EXTENSION',
 185                 'clientVersion': '17.33.2',
 186                 'deviceModel': 'iPhone14,3',
 187                 'userAgent': 'com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 188             },
 189         },
 190         'INNERTUBE_CONTEXT_CLIENT_NAME': 66,
 191         'REQUIRE_JS_PLAYER': False
 192     },
 193     'ios_music': {
 194         'INNERTUBE_API_KEY': 'AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s',
 195         'INNERTUBE_CONTEXT': {
 196             'client': {
 197                 'clientName': 'IOS_MUSIC',
 198                 'clientVersion': '5.21',
 199                 'deviceModel': 'iPhone14,3',
 200                 'userAgent': 'com.google.ios.youtubemusic/5.21 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 201             },
 202         },
 203         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
 204         'REQUIRE_JS_PLAYER': False
 205     },
 206     'ios_creator': {
 207         'INNERTUBE_CONTEXT': {
 208             'client': {
 209                 'clientName': 'IOS_CREATOR',
 210                 'clientVersion': '22.33.101',
 211                 'deviceModel': 'iPhone14,3',
 212                 'userAgent': 'com.google.ios.ytcreator/22.33.101 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
 213             },
 214         },
 215         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
 216         'REQUIRE_JS_PLAYER': False
 217     },
 218     # mweb has 'ultralow' formats
 219     # See: https://github.com/yt-dlp/yt-dlp/pull/557
 220     'mweb': {
 221         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 222         'INNERTUBE_CONTEXT': {
 223             'client': {
 224                 'clientName': 'MWEB',
 225                 'clientVersion': '2.20220801.00.00',
 226             }
 227         },
 228         'INNERTUBE_CONTEXT_CLIENT_NAME': 2
 229     },
 230     # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
 231     # See: https://github.com/zerodytrash/YouTube-Internal-Clients
 232     'tv_embedded': {
 233         'INNERTUBE_API_KEY': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
 234         'INNERTUBE_CONTEXT': {
 235             'client': {
 236                 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
 237                 'clientVersion': '2.0',
 238             },
 239         },
 240         'INNERTUBE_CONTEXT_CLIENT_NAME': 85
 241     },
 242 }
 243
 244
 245 def _split_innertube_client(client_name):
 246     variant, *base = client_name.rsplit('.', 1)
 247     if base:
 248         return variant, base[0], variant
 249     base, *variant = client_name.split('_', 1)
 250     return client_name, base, variant[0] if variant else None
 251
 252
 253 def short_client_name(client_name):
 254     main, *parts = _split_innertube_client(client_name)[0].replace('embedscreen', 'e_s').split('_')
 255     return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
 256
 257
 258 def build_innertube_clients():
 259     THIRD_PARTY = {
 260         'embedUrl': 'https://www.youtube.com/',  # Can be any valid URL
 261     }
 262     BASE_CLIENTS = ('android', 'web', 'tv', 'ios', 'mweb')
 263     priority = qualities(BASE_CLIENTS[::-1])
 264
 265     for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
 266         ytcfg.setdefault('INNERTUBE_API_KEY', 'AIzaSyDCU8hByM-4DrUqRUYnGn-3llEO78bcxq8')
 267         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
 268         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
 269         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 270
 271         _, base_client, variant = _split_innertube_client(client)
 272         ytcfg['priority'] = 10 * priority(base_client)
 273
 274         if not variant:
 275             INNERTUBE_CLIENTS[f'{client}_embedscreen'] = embedscreen = copy.deepcopy(ytcfg)
 276             embedscreen['INNERTUBE_CONTEXT']['client']['clientScreen'] = 'EMBED'
 277             embedscreen['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 278             embedscreen['priority'] -= 3
 279         elif variant == 'embedded':
 280             ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
 281             ytcfg['priority'] -= 2
 282         else:
 283             ytcfg['priority'] -= 3
 284
 285
 286 build_innertube_clients()
 287
 288
 289 class BadgeType(enum.Enum):
 290     AVAILABILITY_UNLISTED = enum.auto()
 291     AVAILABILITY_PRIVATE = enum.auto()
 292     AVAILABILITY_PUBLIC = enum.auto()
 293     AVAILABILITY_PREMIUM = enum.auto()
 294     AVAILABILITY_SUBSCRIPTION = enum.auto()
 295     LIVE_NOW = enum.auto()
 296
 297
 298 class YoutubeBaseInfoExtractor(InfoExtractor):
 299     """Provide base functions for Youtube extractors"""
 300
 301     _RESERVED_NAMES = (
 302         r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
 303         r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
 304         r'browse|oembed|get_video_info|iframe_api|s/player|source|'
 305         r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
 306
 307     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
 308
 309     # _NETRC_MACHINE = 'youtube'
 310
 311     # If True it will raise an error if no login info is provided
 312     _LOGIN_REQUIRED = False
 313
 314     _INVIDIOUS_SITES = (
 315         # invidious-redirect websites
 316         r'(?:www\.)?redirect\.invidious\.io',
 317         r'(?:(?:www|dev)\.)?invidio\.us',
 318         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
 319         r'(?:www\.)?invidious\.pussthecat\.org',
 320         r'(?:www\.)?invidious\.zee\.li',
 321         r'(?:www\.)?invidious\.ethibox\.fr',
 322         r'(?:www\.)?iv\.ggtyler\.dev',
 323         r'(?:www\.)?inv\.vern\.i2p',
 324         r'(?:www\.)?am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd\.onion',
 325         r'(?:www\.)?inv\.riverside\.rocks',
 326         r'(?:www\.)?invidious\.silur\.me',
 327         r'(?:www\.)?inv\.bp\.projectsegfau\.lt',
 328         r'(?:www\.)?invidious\.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid\.onion',
 329         r'(?:www\.)?invidious\.slipfox\.xyz',
 330         r'(?:www\.)?invidious\.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd\.onion',
 331         r'(?:www\.)?inv\.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad\.onion',
 332         r'(?:www\.)?invidious\.tiekoetter\.com',
 333         r'(?:www\.)?iv\.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd\.onion',
 334         r'(?:www\.)?invidious\.nerdvpn\.de',
 335         r'(?:www\.)?invidious\.weblibre\.org',
 336         r'(?:www\.)?inv\.odyssey346\.dev',
 337         r'(?:www\.)?invidious\.dhusch\.de',
 338         r'(?:www\.)?iv\.melmac\.space',
 339         r'(?:www\.)?watch\.thekitty\.zone',
 340         r'(?:www\.)?invidious\.privacydev\.net',
 341         r'(?:www\.)?ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid\.onion',
 342         r'(?:www\.)?invidious\.drivet\.xyz',
 343         r'(?:www\.)?vid\.priv\.au',
 344         r'(?:www\.)?euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd\.onion',
 345         r'(?:www\.)?inv\.vern\.cc',
 346         r'(?:www\.)?invidious\.esmailelbob\.xyz',
 347         r'(?:www\.)?invidious\.sethforprivacy\.com',
 348         r'(?:www\.)?yt\.oelrichsgarcia\.de',
 349         r'(?:www\.)?yt\.artemislena\.eu',
 350         r'(?:www\.)?invidious\.flokinet\.to',
 351         r'(?:www\.)?invidious\.baczek\.me',
 352         r'(?:www\.)?y\.com\.sb',
 353         r'(?:www\.)?invidious\.epicsite\.xyz',
 354         r'(?:www\.)?invidious\.lidarshield\.cloud',
 355         r'(?:www\.)?yt\.funami\.tech',
 356         r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
 357         r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
 358         r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
 359         # youtube-dl invidious instances list
 360         r'(?:(?:www|no)\.)?invidiou\.sh',
 361         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 362         r'(?:www\.)?invidious\.kabi\.tk',
 363         r'(?:www\.)?invidious\.mastodon\.host',
 364         r'(?:www\.)?invidious\.zapashcanon\.fr',
 365         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
 366         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 367         r'(?:www\.)?invidious\.himiko\.cloud',
 368         r'(?:www\.)?invidious\.reallyancient\.tech',
 369         r'(?:www\.)?invidious\.tube',
 370         r'(?:www\.)?invidiou\.site',
 371         r'(?:www\.)?invidious\.site',
 372         r'(?:www\.)?invidious\.xyz',
 373         r'(?:www\.)?invidious\.nixnet\.xyz',
 374         r'(?:www\.)?invidious\.048596\.xyz',
 375         r'(?:www\.)?invidious\.drycat\.fr',
 376         r'(?:www\.)?inv\.skyn3t\.in',
 377         r'(?:www\.)?tube\.poal\.co',
 378         r'(?:www\.)?tube\.connect\.cafe',
 379         r'(?:www\.)?vid\.wxzm\.sx',
 380         r'(?:www\.)?vid\.mint\.lgbt',
 381         r'(?:www\.)?vid\.puffyan\.us',
 382         r'(?:www\.)?yewtu\.be',
 383         r'(?:www\.)?yt\.elukerio\.org',
 384         r'(?:www\.)?yt\.lelux\.fi',
 385         r'(?:www\.)?invidious\.ggc-project\.de',
 386         r'(?:www\.)?yt\.maisputain\.ovh',
 387         r'(?:www\.)?ytprivate\.com',
 388         r'(?:www\.)?invidious\.13ad\.de',
 389         r'(?:www\.)?invidious\.toot\.koeln',
 390         r'(?:www\.)?invidious\.fdn\.fr',
 391         r'(?:www\.)?watch\.nettohikari\.com',
 392         r'(?:www\.)?invidious\.namazso\.eu',
 393         r'(?:www\.)?invidious\.silkky\.cloud',
 394         r'(?:www\.)?invidious\.exonip\.de',
 395         r'(?:www\.)?invidious\.riverside\.rocks',
 396         r'(?:www\.)?invidious\.blamefran\.net',
 397         r'(?:www\.)?invidious\.moomoo\.de',
 398         r'(?:www\.)?ytb\.trom\.tf',
 399         r'(?:www\.)?yt\.cyberhost\.uk',
 400         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 401         r'(?:www\.)?qklhadlycap4cnod\.onion',
 402         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 403         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 404         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 405         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 406         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 407         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 408         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
 409         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
 410         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
 411         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
 412         # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
 413         r'(?:www\.)?piped\.kavin\.rocks',
 414         r'(?:www\.)?piped\.tokhmi\.xyz',
 415         r'(?:www\.)?piped\.syncpundit\.io',
 416         r'(?:www\.)?piped\.mha\.fi',
 417         r'(?:www\.)?watch\.whatever\.social',
 418         r'(?:www\.)?piped\.garudalinux\.org',
 419         r'(?:www\.)?piped\.rivo\.lol',
 420         r'(?:www\.)?piped-libre\.kavin\.rocks',
 421         r'(?:www\.)?yt\.jae\.fi',
 422         r'(?:www\.)?piped\.mint\.lgbt',
 423         r'(?:www\.)?il\.ax',
 424         r'(?:www\.)?piped\.esmailelbob\.xyz',
 425         r'(?:www\.)?piped\.projectsegfau\.lt',
 426         r'(?:www\.)?piped\.privacydev\.net',
 427         r'(?:www\.)?piped\.palveluntarjoaja\.eu',
 428         r'(?:www\.)?piped\.smnz\.de',
 429         r'(?:www\.)?piped\.adminforge\.de',
 430         r'(?:www\.)?watch\.whatevertinfoil\.de',
 431         r'(?:www\.)?piped\.qdi\.fi',
 432         r'(?:www\.)?piped\.video',
 433         r'(?:www\.)?piped\.aeong\.one',
 434         r'(?:www\.)?piped\.moomoo\.me',
 435         r'(?:www\.)?piped\.chauvet\.pro',
 436         r'(?:www\.)?watch\.leptons\.xyz',
 437         r'(?:www\.)?pd\.vern\.cc',
 438         r'(?:www\.)?piped\.hostux\.net',
 439         r'(?:www\.)?piped\.lunar\.icu',
 440         # Hyperpipe instances from https://hyperpipe.codeberg.page/
 441         r'(?:www\.)?hyperpipe\.surge\.sh',
 442         r'(?:www\.)?hyperpipe\.esmailelbob\.xyz',
 443         r'(?:www\.)?listen\.whatever\.social',
 444         r'(?:www\.)?music\.adminforge\.de',
 445     )
 446
 447     # extracted from account/account_menu ep
 448     # XXX: These are the supported YouTube UI and API languages,
 449     # which is slightly different from languages supported for translation in YouTube studio
 450     _SUPPORTED_LANG_CODES = [
 451         'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
 452         'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
 453         'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
 454         'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
 455         'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
 456         'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko'
 457     ]
 458
 459     _IGNORED_WARNINGS = {'Unavailable videos will be hidden during playback'}
 460
 461     @functools.cached_property
 462     def _preferred_lang(self):
 463         """
 464         Returns a language code supported by YouTube for the user preferred language.
 465         Returns None if no preferred language set.
 466         """
 467         preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
 468         if not preferred_lang:
 469             return
 470         if preferred_lang not in self._SUPPORTED_LANG_CODES:
 471             raise ExtractorError(
 472                 f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
 473                 expected=True)
 474         elif preferred_lang != 'en':
 475             self.report_warning(
 476                 f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
 477         return preferred_lang
 478
 479     def _initialize_consent(self):
 480         cookies = self._get_cookies('https://www.youtube.com/')
 481         if cookies.get('__Secure-3PSID'):
 482             return
 483         consent_id = None
 484         consent = cookies.get('CONSENT')
 485         if consent:
 486             if 'YES' in consent.value:
 487                 return
 488             consent_id = self._search_regex(
 489                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 490         if not consent_id:
 491             consent_id = random.randint(100, 999)
 492         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 493
 494     def _initialize_pref(self):
 495         cookies = self._get_cookies('https://www.youtube.com/')
 496         pref_cookie = cookies.get('PREF')
 497         pref = {}
 498         if pref_cookie:
 499             try:
 500                 pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
 501             except ValueError:
 502                 self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
 503         pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
 504         self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
 505
 506     def _real_initialize(self):
 507         self._initialize_pref()
 508         self._initialize_consent()
 509         self._check_login_required()
 510
 511     def _check_login_required(self):
 512         if self._LOGIN_REQUIRED and not self._cookies_passed:
 513             self.raise_login_required('Login details are needed to download this content', method='cookies')
 514
 515     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
 516     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
 517
 518     def _get_default_ytcfg(self, client='web'):
 519         return copy.deepcopy(INNERTUBE_CLIENTS[client])
 520
 521     def _get_innertube_host(self, client='web'):
 522         return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
 523
 524     def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
 525         # try_get but with fallback to default ytcfg client values when present
 526         _func = lambda y: try_get(y, getter, expected_type)
 527         return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
 528
 529     def _extract_client_name(self, ytcfg, default_client='web'):
 530         return self._ytcfg_get_safe(
 531             ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
 532                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
 533
 534     def _extract_client_version(self, ytcfg, default_client='web'):
 535         return self._ytcfg_get_safe(
 536             ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
 537                     lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
 538
 539     def _select_api_hostname(self, req_api_hostname, default_client=None):
 540         return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
 541                 or req_api_hostname or self._get_innertube_host(default_client or 'web'))
 542
 543     def _extract_api_key(self, ytcfg=None, default_client='web'):
 544         return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], str, default_client)
 545
 546     def _extract_context(self, ytcfg=None, default_client='web'):
 547         context = get_first(
 548             (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
 549         # Enforce language and tz for extraction
 550         client_context = traverse_obj(context, 'client', expected_type=dict, default={})
 551         client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
 552         return context
 553
 554     _SAPISID = None
 555
 556     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
 557         time_now = round(time.time())
 558         if self._SAPISID is None:
 559             yt_cookies = self._get_cookies('https://www.youtube.com')
 560             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
 561             # See: https://github.com/yt-dlp/yt-dlp/issues/393
 562             sapisid_cookie = dict_get(
 563                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
 564             if sapisid_cookie and sapisid_cookie.value:
 565                 self._SAPISID = sapisid_cookie.value
 566                 self.write_debug('Extracted SAPISID cookie')
 567                 # SAPISID cookie is required if not already present
 568                 if not yt_cookies.get('SAPISID'):
 569                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
 570                     self._set_cookie(
 571                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
 572             else:
 573                 self._SAPISID = False
 574         if not self._SAPISID:
 575             return None
 576         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
 577         sapisidhash = hashlib.sha1(
 578             f'{time_now} {self._SAPISID} {origin}'.encode()).hexdigest()
 579         return f'SAPISIDHASH {time_now}_{sapisidhash}'
 580
 581     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
 582                   note='Downloading API JSON', errnote='Unable to download API page',
 583                   context=None, api_key=None, api_hostname=None, default_client='web'):
 584
 585         data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
 586         data.update(query)
 587         real_headers = self.generate_api_headers(default_client=default_client)
 588         real_headers.update({'content-type': 'application/json'})
 589         if headers:
 590             real_headers.update(headers)
 591         api_key = (self._configuration_arg('innertube_key', [''], ie_key=YoutubeIE.ie_key(), casesense=True)[0]
 592                    or api_key or self._extract_api_key(default_client=default_client))
 593         return self._download_json(
 594             f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
 595             video_id=video_id, fatal=fatal, note=note, errnote=errnote,
 596             data=json.dumps(data).encode('utf8'), headers=real_headers,
 597             query={'key': api_key, 'prettyPrint': 'false'})
 598
 599     def extract_yt_initial_data(self, item_id, webpage, fatal=True):
 600         return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
 601
 602     @staticmethod
 603     def _extract_session_index(*data):
 604         """
 605         Index of current account in account list.
 606         See: https://github.com/yt-dlp/yt-dlp/pull/519
 607         """
 608         for ytcfg in data:
 609             session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
 610             if session_index is not None:
 611                 return session_index
 612
 613     # Deprecated?
 614     def _extract_identity_token(self, ytcfg=None, webpage=None):
 615         if ytcfg:
 616             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], str)
 617             if token:
 618                 return token
 619         if webpage:
 620             return self._search_regex(
 621                 r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
 622                 'identity token', default=None, fatal=False)
 623
 624     @staticmethod
 625     def _extract_account_syncid(*args):
 626         """
 627         Extract syncId required to download private playlists of secondary channels
 628         @params response and/or ytcfg
 629         """
 630         for data in args:
 631             # ytcfg includes channel_syncid if on secondary channel
 632             delegated_sid = try_get(data, lambda x: x['DELEGATED_SESSION_ID'], str)
 633             if delegated_sid:
 634                 return delegated_sid
 635             sync_ids = (try_get(
 636                 data, (lambda x: x['responseContext']['mainAppWebResponseContext']['datasyncId'],
 637                        lambda x: x['DATASYNC_ID']), str) or '').split('||')
 638             if len(sync_ids) >= 2 and sync_ids[1]:
 639                 # datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
 640                 # and just "user_syncid||" for primary channel. We only want the channel_syncid
 641                 return sync_ids[0]
 642
 643     @staticmethod
 644     def _extract_visitor_data(*args):
 645         """
 646         Extracts visitorData from an API response or ytcfg
 647         Appears to be used to track session state
 648         """
 649         return get_first(
 650             args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
 651             expected_type=str)
 652
 653     @functools.cached_property
 654     def is_authenticated(self):
 655         return bool(self._generate_sapisidhash_header())
 656
 657     def extract_ytcfg(self, video_id, webpage):
 658         if not webpage:
 659             return {}
 660         return self._parse_json(
 661             self._search_regex(
 662                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 663                 default='{}'), video_id, fatal=False) or {}
 664
 665     def generate_api_headers(
 666             self, *, ytcfg=None, account_syncid=None, session_index=None,
 667             visitor_data=None, identity_token=None, api_hostname=None, default_client='web'):
 668
 669         origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
 670         headers = {
 671             'X-YouTube-Client-Name': str(
 672                 self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
 673             'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
 674             'Origin': origin,
 675             'X-Youtube-Identity-Token': identity_token or self._extract_identity_token(ytcfg),
 676             'X-Goog-PageId': account_syncid or self._extract_account_syncid(ytcfg),
 677             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
 678             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client)
 679         }
 680         if session_index is None:
 681             session_index = self._extract_session_index(ytcfg)
 682         if account_syncid or session_index is not None:
 683             headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
 684
 685         auth = self._generate_sapisidhash_header(origin)
 686         if auth is not None:
 687             headers['Authorization'] = auth
 688             headers['X-Origin'] = origin
 689         return filter_dict(headers)
 690
 691     def _download_ytcfg(self, client, video_id):
 692         url = {
 693             'web': 'https://www.youtube.com',
 694             'web_music': 'https://music.youtube.com',
 695             'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1'
 696         }.get(client)
 697         if not url:
 698             return {}
 699         webpage = self._download_webpage(
 700             url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
 701         return self.extract_ytcfg(video_id, webpage) or {}
 702
 703     @staticmethod
 704     def _build_api_continuation_query(continuation, ctp=None):
 705         query = {
 706             'continuation': continuation
 707         }
 708         # TODO: Inconsistency with clickTrackingParams.
 709         # Currently we have a fixed ctp contained within context (from ytcfg)
 710         # and a ctp in root query for continuation.
 711         if ctp:
 712             query['clickTracking'] = {'clickTrackingParams': ctp}
 713         return query
 714
 715     @classmethod
 716     def _extract_next_continuation_data(cls, renderer):
 717         next_continuation = try_get(
 718             renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
 719                        lambda x: x['continuation']['reloadContinuationData']), dict)
 720         if not next_continuation:
 721             return
 722         continuation = next_continuation.get('continuation')
 723         if not continuation:
 724             return
 725         ctp = next_continuation.get('clickTrackingParams')
 726         return cls._build_api_continuation_query(continuation, ctp)
 727
 728     @classmethod
 729     def _extract_continuation_ep_data(cls, continuation_ep: dict):
 730         if isinstance(continuation_ep, dict):
 731             continuation = try_get(
 732                 continuation_ep, lambda x: x['continuationCommand']['token'], str)
 733             if not continuation:
 734                 return
 735             ctp = continuation_ep.get('clickTrackingParams')
 736             return cls._build_api_continuation_query(continuation, ctp)
 737
 738     @classmethod
 739     def _extract_continuation(cls, renderer):
 740         next_continuation = cls._extract_next_continuation_data(renderer)
 741         if next_continuation:
 742             return next_continuation
 743
 744         return traverse_obj(renderer, (
 745             ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
 746             ('continuationEndpoint', ('button', 'buttonRenderer', 'command'))
 747         ), get_all=False, expected_type=cls._extract_continuation_ep_data)
 748
 749     @classmethod
 750     def _extract_alerts(cls, data):
 751         for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
 752             if not isinstance(alert_dict, dict):
 753                 continue
 754             for alert in alert_dict.values():
 755                 alert_type = alert.get('type')
 756                 if not alert_type:
 757                     continue
 758                 message = cls._get_text(alert, 'text')
 759                 if message:
 760                     yield alert_type, message
 761
 762     def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
 763         errors, warnings = [], []
 764         for alert_type, alert_message in alerts:
 765             if alert_type.lower() == 'error' and fatal:
 766                 errors.append([alert_type, alert_message])
 767             elif alert_message not in self._IGNORED_WARNINGS:
 768                 warnings.append([alert_type, alert_message])
 769
 770         for alert_type, alert_message in (warnings + errors[:-1]):
 771             self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
 772         if errors:
 773             raise ExtractorError('YouTube said: %s' % errors[-1][1], expected=expected)
 774
 775     def _extract_and_report_alerts(self, data, *args, **kwargs):
 776         return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
 777
 778     def _extract_badges(self, renderer: dict):
 779         privacy_icon_map = {
 780             'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
 781             'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
 782             'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC
 783         }
 784
 785         badge_style_map = {
 786             'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
 787             'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
 788             'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW
 789         }
 790
 791         label_map = {
 792             'unlisted': BadgeType.AVAILABILITY_UNLISTED,
 793             'private': BadgeType.AVAILABILITY_PRIVATE,
 794             'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
 795             'live': BadgeType.LIVE_NOW,
 796             'premium': BadgeType.AVAILABILITY_PREMIUM
 797         }
 798
 799         badges = []
 800         for badge in traverse_obj(renderer, ('badges', ..., 'metadataBadgeRenderer')):
 801             badge_type = (
 802                 privacy_icon_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
 803                 or badge_style_map.get(traverse_obj(badge, 'style'))
 804             )
 805             if badge_type:
 806                 badges.append({'type': badge_type})
 807                 continue
 808
 809             # fallback, won't work in some languages
 810             label = traverse_obj(badge, 'label', expected_type=str, default='')
 811             for match, label_badge_type in label_map.items():
 812                 if match in label.lower():
 813                     badges.append({'type': badge_type})
 814                     continue
 815
 816         return badges
 817
 818     @staticmethod
 819     def _has_badge(badges, badge_type):
 820         return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
 821
 822     @staticmethod
 823     def _get_text(data, *path_list, max_runs=None):
 824         for path in path_list or [None]:
 825             if path is None:
 826                 obj = [data]
 827             else:
 828                 obj = traverse_obj(data, path, default=[])
 829                 if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
 830                     obj = [obj]
 831             for item in obj:
 832                 text = try_get(item, lambda x: x['simpleText'], str)
 833                 if text:
 834                     return text
 835                 runs = try_get(item, lambda x: x['runs'], list) or []
 836                 if not runs and isinstance(item, list):
 837                     runs = item
 838
 839                 runs = runs[:min(len(runs), max_runs or len(runs))]
 840                 text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str))
 841                 if text:
 842                     return text
 843
 844     def _get_count(self, data, *path_list):
 845         count_text = self._get_text(data, *path_list) or ''
 846         count = parse_count(count_text)
 847         if count is None:
 848             count = str_to_int(
 849                 self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
 850         return count
 851
 852     @staticmethod
 853     def _extract_thumbnails(data, *path_list):
 854         """
 855         Extract thumbnails from thumbnails dict
 856         @param path_list: path list to level that contains 'thumbnails' key
 857         """
 858         thumbnails = []
 859         for path in path_list or [()]:
 860             for thumbnail in traverse_obj(data, (*variadic(path), 'thumbnails', ...)):
 861                 thumbnail_url = url_or_none(thumbnail.get('url'))
 862                 if not thumbnail_url:
 863                     continue
 864                 # Sometimes youtube gives a wrong thumbnail URL. See:
 865                 # https://github.com/yt-dlp/yt-dlp/issues/233
 866                 # https://github.com/ytdl-org/youtube-dl/issues/28023
 867                 if 'maxresdefault' in thumbnail_url:
 868                     thumbnail_url = thumbnail_url.split('?')[0]
 869                 thumbnails.append({
 870                     'url': thumbnail_url,
 871                     'height': int_or_none(thumbnail.get('height')),
 872                     'width': int_or_none(thumbnail.get('width')),
 873                 })
 874         return thumbnails
 875
 876     @staticmethod
 877     def extract_relative_time(relative_time_text):
 878         """
 879         Extracts a relative time from string and converts to dt object
 880         e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today'
 881         """
 882         mobj = re.search(r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
 883         if mobj:
 884             start = mobj.group('start')
 885             if start:
 886                 return datetime_from_str(start)
 887             try:
 888                 return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')))
 889             except ValueError:
 890                 return None
 891
 892     def _parse_time_text(self, text):
 893         if not text:
 894             return
 895         dt = self.extract_relative_time(text)
 896         timestamp = None
 897         if isinstance(dt, datetime.datetime):
 898             timestamp = calendar.timegm(dt.timetuple())
 899
 900         if timestamp is None:
 901             timestamp = (
 902                 unified_timestamp(text) or unified_timestamp(
 903                     self._search_regex(
 904                         (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
 905                         text.lower(), 'time text', default=None)))
 906
 907         if text and timestamp is None and self._preferred_lang in (None, 'en'):
 908             self.report_warning(
 909                 f'Cannot parse localized time text "{text}"', only_once=True)
 910         return timestamp
 911
 912     def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
 913                           ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
 914                           default_client='web'):
 915         for retry in self.RetryManager():
 916             try:
 917                 response = self._call_api(
 918                     ep=ep, fatal=True, headers=headers,
 919                     video_id=item_id, query=query, note=note,
 920                     context=self._extract_context(ytcfg, default_client),
 921                     api_key=self._extract_api_key(ytcfg, default_client),
 922                     api_hostname=api_hostname, default_client=default_client)
 923             except ExtractorError as e:
 924                 if not isinstance(e.cause, network_exceptions):
 925                     return self._error_or_warning(e, fatal=fatal)
 926                 elif not isinstance(e.cause, urllib.error.HTTPError):
 927                     retry.error = e
 928                     continue
 929
 930                 first_bytes = e.cause.read(512)
 931                 if not is_html(first_bytes):
 932                     yt_error = try_get(
 933                         self._parse_json(
 934                             self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
 935                         lambda x: x['error']['message'], str)
 936                     if yt_error:
 937                         self._report_alerts([('ERROR', yt_error)], fatal=False)
 938                 # Downloading page may result in intermittent 5xx HTTP error
 939                 # Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
 940                 # We also want to catch all other network exceptions since errors in later pages can be troublesome
 941                 # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
 942                 if e.cause.code not in (403, 429):
 943                     retry.error = e
 944                     continue
 945                 return self._error_or_warning(e, fatal=fatal)
 946
 947             try:
 948                 self._extract_and_report_alerts(response, only_once=True)
 949             except ExtractorError as e:
 950                 # YouTube servers may return errors we want to retry on in a 200 OK response
 951                 # See: https://github.com/yt-dlp/yt-dlp/issues/839
 952                 if 'unknown error' in e.msg.lower():
 953                     retry.error = e
 954                     continue
 955                 return self._error_or_warning(e, fatal=fatal)
 956             # Youtube sometimes sends incomplete data
 957             # See: https://github.com/ytdl-org/youtube-dl/issues/28194
 958             if not traverse_obj(response, *variadic(check_get_keys)):
 959                 retry.error = ExtractorError('Incomplete data received', expected=True)
 960                 continue
 961
 962             return response
 963
 964     @staticmethod
 965     def is_music_url(url):
 966         return re.match(r'(https?://)?music\.youtube\.com/', url) is not None
 967
 968     def _extract_video(self, renderer):
 969         video_id = renderer.get('videoId')
 970
 971         reel_header_renderer = traverse_obj(renderer, (
 972             'navigationEndpoint', 'reelWatchEndpoint', 'overlay', 'reelPlayerOverlayRenderer',
 973             'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
 974
 975         title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
 976         description = self._get_text(renderer, 'descriptionSnippet')
 977
 978         duration = int_or_none(renderer.get('lengthSeconds'))
 979         if duration is None:
 980             duration = parse_duration(self._get_text(
 981                 renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
 982         if duration is None:
 983             # XXX: should write a parser to be more general to support more cases (e.g. shorts in shorts tab)
 984             duration = parse_duration(self._search_regex(
 985                 r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
 986                 traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
 987                 video_id, default=None, group='duration'))
 988
 989         channel_id = traverse_obj(
 990             renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
 991             expected_type=str, get_all=False)
 992         if not channel_id:
 993             channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
 994
 995         overlay_style = traverse_obj(
 996             renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
 997             get_all=False, expected_type=str)
 998         badges = self._extract_badges(renderer)
 999
1000         navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
1001             renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
1002             expected_type=str)) or ''
1003         url = f'https://www.youtube.com/watch?v={video_id}'
1004         if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
1005             url = f'https://www.youtube.com/shorts/{video_id}'
1006
1007         time_text = (self._get_text(renderer, 'publishedTimeText', 'videoInfo')
1008                      or self._get_text(reel_header_renderer, 'timestampText') or '')
1009         scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
1010
1011         live_status = (
1012             'is_upcoming' if scheduled_timestamp is not None
1013             else 'was_live' if 'streamed' in time_text.lower()
1014             else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
1015             else None)
1016
1017         # videoInfo is a string like '50K views • 10 years ago'.
1018         view_count_text = self._get_text(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo') or ''
1019         view_count = (0 if 'no views' in view_count_text.lower()
1020                       else self._get_count({'simpleText': view_count_text}))
1021         view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
1022
1023         return {
1024             '_type': 'url',
1025             'ie_key': YoutubeIE.ie_key(),
1026             'id': video_id,
1027             'url': url,
1028             'title': title,
1029             'description': description,
1030             'duration': duration,
1031             'channel_id': channel_id,
1032             'channel': (self._get_text(renderer, 'ownerText', 'shortBylineText')
1033                         or self._get_text(reel_header_renderer, 'channelTitleText')),
1034             'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
1035             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
1036             'timestamp': (self._parse_time_text(time_text)
1037                           if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
1038                           else None),
1039             'release_timestamp': scheduled_timestamp,
1040             'availability':
1041                 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
1042                 else self._availability(
1043                     is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
1044                     needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
1045                     needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
1046                     is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
1047             view_count_field: view_count,
1048             'live_status': live_status
1049         }
1050
1051
1052 class YoutubeIE(YoutubeBaseInfoExtractor):
1053     IE_DESC = 'YouTube'
1054     _VALID_URL = r"""(?x)^
1055                      (
1056                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
1057                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
1058                             (?:www\.)?deturl\.com/www\.youtube\.com|
1059                             (?:www\.)?pwnyoutube\.com|
1060                             (?:www\.)?hooktube\.com|
1061                             (?:www\.)?yourepeat\.com|
1062                             tube\.majestyc\.net|
1063                             %(invidious)s|
1064                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
1065                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
1066                          (?:                                                  # the various things that can precede the ID:
1067                              (?:(?:v|embed|e|shorts|live)/(?!videoseries|live_stream))  # v/ or embed/ or e/ or shorts/
1068                              |(?:                                             # or the v= param in all its forms
1069                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
1070                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
1071                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
1072                                  v=
1073                              )
1074                          ))
1075                          |(?:
1076                             youtu\.be|                                        # just youtu.be/xxxx
1077                             vid\.plus|                                        # or vid.plus/xxxx
1078                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
1079                             %(invidious)s
1080                          )/
1081                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
1082                          )
1083                      )?                                                       # all until now is optional -> you can pass the naked ID
1084                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
1085                      (?(1).+)?                                                # if we found the ID, everything can follow
1086                      (?:\#|$)""" % {
1087         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
1088     }
1089     _EMBED_REGEX = [
1090         r'''(?x)
1091             (?:
1092                 <(?:[0-9A-Za-z-]+?)?iframe[^>]+?src=|
1093                 data-video-url=|
1094                 <embed[^>]+?src=|
1095                 embedSWF\(?:\s*|
1096                 <object[^>]+data=|
1097                 new\s+SWFObject\(
1098             )
1099             (["\'])
1100                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1101                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1102             \1''',
1103         # https://wordpress.org/plugins/lazy-load-for-videos/
1104         r'''(?xs)
1105             <a\s[^>]*\bhref="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"
1106             \s[^>]*\bclass="[^"]*\blazy-load-youtube''',
1107     ]
1108     _RETURN_TYPE = 'video'  # XXX: How to handle multifeed?
1109
1110     _PLAYER_INFO_RE = (
1111         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
1112         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
1113         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
1114     )
1115     _formats = {
1116         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1117         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1118         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1119         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1120         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1121         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1122         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1123         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1124         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1125         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1126         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1127         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1128         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1129         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1130         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1131         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1132         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1133         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1134
1135
1136         # 3D videos
1137         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1138         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1139         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1140         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1141         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1142         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1143         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1144
1145         # Apple HTTP Live Streaming
1146         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1147         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1148         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1149         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1150         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1151         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1152         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1153         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1154
1155         # DASH mp4 video
1156         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1157         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1158         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1159         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1160         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1161         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1162         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1163         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1164         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1165         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1166         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1167         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1168
1169         # Dash mp4 audio
1170         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1171         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1172         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1173         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1174         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1175         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1176         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1177
1178         # Dash webm
1179         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1180         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1181         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1182         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1183         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1184         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1185         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1186         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1187         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1188         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1189         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1190         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1191         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1192         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1193         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1194         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1195         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1196         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1197         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1198         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1199         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1200         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1201
1202         # Dash webm audio
1203         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1204         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1205
1206         # Dash webm audio with opus inside
1207         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1208         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1209         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1210
1211         # RTMP (unnamed)
1212         '_rtmp': {'protocol': 'rtmp'},
1213
1214         # av01 video only formats sometimes served with "unknown" codecs
1215         '394': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1216         '395': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'av01.0.00M.08'},
1217         '396': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'av01.0.01M.08'},
1218         '397': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'av01.0.04M.08'},
1219         '398': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'av01.0.05M.08'},
1220         '399': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'av01.0.08M.08'},
1221         '400': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1222         '401': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'av01.0.12M.08'},
1223     }
1224     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
1225
1226     _GEO_BYPASS = False
1227
1228     IE_NAME = 'youtube'
1229     _TESTS = [
1230         {
1231             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
1232             'info_dict': {
1233                 'id': 'BaW_jenozKc',
1234                 'ext': 'mp4',
1235                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1236                 'uploader': 'Philipp Hagemeister',
1237                 'uploader_id': 'phihag',
1238                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1239                 'channel': 'Philipp Hagemeister',
1240                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1241                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1242                 'upload_date': '20121002',
1243                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1244                 'categories': ['Science & Technology'],
1245                 'tags': ['youtube-dl'],
1246                 'duration': 10,
1247                 'view_count': int,
1248                 'like_count': int,
1249                 'availability': 'public',
1250                 'playable_in_embed': True,
1251                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1252                 'live_status': 'not_live',
1253                 'age_limit': 0,
1254                 'start_time': 1,
1255                 'end_time': 9,
1256                 'comment_count': int,
1257                 'channel_follower_count': int
1258             }
1259         },
1260         {
1261             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
1262             'note': 'Embed-only video (#1746)',
1263             'info_dict': {
1264                 'id': 'yZIXLfi8CZQ',
1265                 'ext': 'mp4',
1266                 'upload_date': '20120608',
1267                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
1268                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
1269                 'uploader': 'SET India',
1270                 'uploader_id': 'setindia',
1271                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
1272                 'age_limit': 18,
1273             },
1274             'skip': 'Private video',
1275         },
1276         {
1277             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
1278             'note': 'Use the first video ID in the URL',
1279             'info_dict': {
1280                 'id': 'BaW_jenozKc',
1281                 'ext': 'mp4',
1282                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
1283                 'uploader': 'Philipp Hagemeister',
1284                 'uploader_id': 'phihag',
1285                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
1286                 'channel': 'Philipp Hagemeister',
1287                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
1288                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
1289                 'upload_date': '20121002',
1290                 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
1291                 'categories': ['Science & Technology'],
1292                 'tags': ['youtube-dl'],
1293                 'duration': 10,
1294                 'view_count': int,
1295                 'like_count': int,
1296                 'availability': 'public',
1297                 'playable_in_embed': True,
1298                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
1299                 'live_status': 'not_live',
1300                 'age_limit': 0,
1301                 'comment_count': int,
1302                 'channel_follower_count': int
1303             },
1304             'params': {
1305                 'skip_download': True,
1306             },
1307         },
1308         {
1309             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
1310             'note': '256k DASH audio (format 141) via DASH manifest',
1311             'info_dict': {
1312                 'id': 'a9LDPn-MO4I',
1313                 'ext': 'm4a',
1314                 'upload_date': '20121002',
1315                 'uploader_id': '8KVIDEO',
1316                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
1317                 'description': '',
1318                 'uploader': '8KVIDEO',
1319                 'title': 'UHDTV TEST 8K VIDEO.mp4'
1320             },
1321             'params': {
1322                 'youtube_include_dash_manifest': True,
1323                 'format': '141',
1324             },
1325             'skip': 'format 141 not served anymore',
1326         },
1327         # DASH manifest with encrypted signature
1328         {
1329             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
1330             'info_dict': {
1331                 'id': 'IB3lcPjvWLA',
1332                 'ext': 'm4a',
1333                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
1334                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
1335                 'duration': 244,
1336                 'uploader': 'AfrojackVEVO',
1337                 'uploader_id': 'AfrojackVEVO',
1338                 'upload_date': '20131011',
1339                 'abr': 129.495,
1340                 'like_count': int,
1341                 'channel_id': 'UChuZAo1RKL85gev3Eal9_zg',
1342                 'playable_in_embed': True,
1343                 'channel_url': 'https://www.youtube.com/channel/UChuZAo1RKL85gev3Eal9_zg',
1344                 'view_count': int,
1345                 'track': 'The Spark',
1346                 'live_status': 'not_live',
1347                 'thumbnail': 'https://i.ytimg.com/vi_webp/IB3lcPjvWLA/maxresdefault.webp',
1348                 'channel': 'Afrojack',
1349                 'uploader_url': 'http://www.youtube.com/user/AfrojackVEVO',
1350                 'tags': 'count:19',
1351                 'availability': 'public',
1352                 'categories': ['Music'],
1353                 'age_limit': 0,
1354                 'alt_title': 'The Spark',
1355                 'channel_follower_count': int
1356             },
1357             'params': {
1358                 'youtube_include_dash_manifest': True,
1359                 'format': '141/bestaudio[ext=m4a]',
1360             },
1361         },
1362         # Age-gate videos. See https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-888837000
1363         {
1364             'note': 'Embed allowed age-gate video',
1365             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
1366             'info_dict': {
1367                 'id': 'HtVdAasjOgU',
1368                 'ext': 'mp4',
1369                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
1370                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
1371                 'duration': 142,
1372                 'uploader': 'The Witcher',
1373                 'uploader_id': 'WitcherGame',
1374                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
1375                 'upload_date': '20140605',
1376                 'age_limit': 18,
1377                 'categories': ['Gaming'],
1378                 'thumbnail': 'https://i.ytimg.com/vi_webp/HtVdAasjOgU/maxresdefault.webp',
1379                 'availability': 'needs_auth',
1380                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
1381                 'like_count': int,
1382                 'channel': 'The Witcher',
1383                 'live_status': 'not_live',
1384                 'tags': 'count:17',
1385                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
1386                 'playable_in_embed': True,
1387                 'view_count': int,
1388                 'channel_follower_count': int
1389             },
1390         },
1391         {
1392             'note': 'Age-gate video with embed allowed in public site',
1393             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
1394             'info_dict': {
1395                 'id': 'HsUATh_Nc2U',
1396                 'ext': 'mp4',
1397                 'title': 'Godzilla 2 (Official Video)',
1398                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
1399                 'upload_date': '20200408',
1400                 'uploader_id': 'FlyingKitty900',
1401                 'uploader': 'FlyingKitty',
1402                 'age_limit': 18,
1403                 'availability': 'needs_auth',
1404                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
1405                 'uploader_url': 'http://www.youtube.com/user/FlyingKitty900',
1406                 'channel': 'FlyingKitty',
1407                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
1408                 'view_count': int,
1409                 'categories': ['Entertainment'],
1410                 'live_status': 'not_live',
1411                 'tags': ['Flyingkitty', 'godzilla 2'],
1412                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
1413                 'like_count': int,
1414                 'duration': 177,
1415                 'playable_in_embed': True,
1416                 'channel_follower_count': int
1417             },
1418         },
1419         {
1420             'note': 'Age-gate video embedable only with clientScreen=EMBED',
1421             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
1422             'info_dict': {
1423                 'id': 'Tq92D6wQ1mg',
1424                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
1425                 'ext': 'mp4',
1426                 'upload_date': '20191228',
1427                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1428                 'uploader': 'Projekt Melody',
1429                 'description': 'md5:17eccca93a786d51bc67646756894066',
1430                 'age_limit': 18,
1431                 'like_count': int,
1432                 'availability': 'needs_auth',
1433                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1434                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
1435                 'view_count': int,
1436                 'thumbnail': 'https://i.ytimg.com/vi_webp/Tq92D6wQ1mg/sddefault.webp',
1437                 'channel': 'Projekt Melody',
1438                 'live_status': 'not_live',
1439                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
1440                 'playable_in_embed': True,
1441                 'categories': ['Entertainment'],
1442                 'duration': 106,
1443                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
1444                 'comment_count': int,
1445                 'channel_follower_count': int
1446             },
1447         },
1448         {
1449             'note': 'Non-Agegated non-embeddable video',
1450             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
1451             'info_dict': {
1452                 'id': 'MeJVWBSsPAY',
1453                 'ext': 'mp4',
1454                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
1455                 'uploader': 'Herr Lurik',
1456                 'uploader_id': 'st3in234',
1457                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
1458                 'upload_date': '20130730',
1459                 'track': 'Such mich find mich',
1460                 'age_limit': 0,
1461                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
1462                 'like_count': int,
1463                 'playable_in_embed': False,
1464                 'creator': 'OOMPH!',
1465                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/sddefault.jpg',
1466                 'view_count': int,
1467                 'alt_title': 'Such mich find mich',
1468                 'duration': 210,
1469                 'channel': 'Herr Lurik',
1470                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
1471                 'categories': ['Music'],
1472                 'availability': 'public',
1473                 'uploader_url': 'http://www.youtube.com/user/st3in234',
1474                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
1475                 'live_status': 'not_live',
1476                 'artist': 'OOMPH!',
1477                 'channel_follower_count': int
1478             },
1479         },
1480         {
1481             'note': 'Non-bypassable age-gated video',
1482             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
1483             'only_matching': True,
1484         },
1485         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
1486         # YouTube Red ad is not captured for creator
1487         {
1488             'url': '__2ABJjxzNo',
1489             'info_dict': {
1490                 'id': '__2ABJjxzNo',
1491                 'ext': 'mp4',
1492                 'duration': 266,
1493                 'upload_date': '20100430',
1494                 'uploader_id': 'deadmau5',
1495                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
1496                 'creator': 'deadmau5',
1497                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
1498                 'uploader': 'deadmau5',
1499                 'title': 'Deadmau5 - Some Chords (HD)',
1500                 'alt_title': 'Some Chords',
1501                 'availability': 'public',
1502                 'tags': 'count:14',
1503                 'channel_id': 'UCYEK6xds6eo-3tr4xRdflmQ',
1504                 'view_count': int,
1505                 'live_status': 'not_live',
1506                 'channel': 'deadmau5',
1507                 'thumbnail': 'https://i.ytimg.com/vi_webp/__2ABJjxzNo/maxresdefault.webp',
1508                 'like_count': int,
1509                 'track': 'Some Chords',
1510                 'artist': 'deadmau5',
1511                 'playable_in_embed': True,
1512                 'age_limit': 0,
1513                 'channel_url': 'https://www.youtube.com/channel/UCYEK6xds6eo-3tr4xRdflmQ',
1514                 'categories': ['Music'],
1515                 'album': 'Some Chords',
1516                 'channel_follower_count': int
1517             },
1518             'expected_warnings': [
1519                 'DASH manifest missing',
1520             ]
1521         },
1522         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
1523         {
1524             'url': 'lqQg6PlCWgI',
1525             'info_dict': {
1526                 'id': 'lqQg6PlCWgI',
1527                 'ext': 'mp4',
1528                 'duration': 6085,
1529                 'upload_date': '20150827',
1530                 'uploader_id': 'olympic',
1531                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
1532                 'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
1533                 'uploader': 'Olympics',
1534                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
1535                 'like_count': int,
1536                 'release_timestamp': 1343767800,
1537                 'playable_in_embed': True,
1538                 'categories': ['Sports'],
1539                 'release_date': '20120731',
1540                 'channel': 'Olympics',
1541                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
1542                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
1543                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
1544                 'age_limit': 0,
1545                 'availability': 'public',
1546                 'live_status': 'was_live',
1547                 'view_count': int,
1548                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
1549                 'channel_follower_count': int
1550             },
1551             'params': {
1552                 'skip_download': 'requires avconv',
1553             }
1554         },
1555         # Non-square pixels
1556         {
1557             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
1558             'info_dict': {
1559                 'id': '_b-2C3KPAM0',
1560                 'ext': 'mp4',
1561                 'stretched_ratio': 16 / 9.,
1562                 'duration': 85,
1563                 'upload_date': '20110310',
1564                 'uploader_id': 'AllenMeow',
1565                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
1566                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
1567                 'uploader': '孫ᄋᄅ',
1568                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
1569                 'playable_in_embed': True,
1570                 'channel': '孫ᄋᄅ',
1571                 'age_limit': 0,
1572                 'tags': 'count:11',
1573                 'channel_url': 'https://www.youtube.com/channel/UCS-xxCmRaA6BFdmgDPA_BIw',
1574                 'channel_id': 'UCS-xxCmRaA6BFdmgDPA_BIw',
1575                 'thumbnail': 'https://i.ytimg.com/vi/_b-2C3KPAM0/maxresdefault.jpg',
1576                 'view_count': int,
1577                 'categories': ['People & Blogs'],
1578                 'like_count': int,
1579                 'live_status': 'not_live',
1580                 'availability': 'unlisted',
1581                 'comment_count': int,
1582                 'channel_follower_count': int
1583             },
1584         },
1585         # url_encoded_fmt_stream_map is empty string
1586         {
1587             'url': 'qEJwOuvDf7I',
1588             'info_dict': {
1589                 'id': 'qEJwOuvDf7I',
1590                 'ext': 'webm',
1591                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
1592                 'description': '',
1593                 'upload_date': '20150404',
1594                 'uploader_id': 'spbelect',
1595                 'uploader': 'Наблюдатели Петербурга',
1596             },
1597             'params': {
1598                 'skip_download': 'requires avconv',
1599             },
1600             'skip': 'This live event has ended.',
1601         },
1602         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
1603         {
1604             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
1605             'info_dict': {
1606                 'id': 'FIl7x6_3R5Y',
1607                 'ext': 'webm',
1608                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
1609                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
1610                 'duration': 220,
1611                 'upload_date': '20150625',
1612                 'uploader_id': 'dorappi2000',
1613                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
1614                 'uploader': 'dorappi2000',
1615                 'formats': 'mincount:31',
1616             },
1617             'skip': 'not actual anymore',
1618         },
1619         # DASH manifest with segment_list
1620         {
1621             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
1622             'md5': '8ce563a1d667b599d21064e982ab9e31',
1623             'info_dict': {
1624                 'id': 'CsmdDsKjzN8',
1625                 'ext': 'mp4',
1626                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
1627                 'uploader': 'Airtek',
1628                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
1629                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
1630                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
1631             },
1632             'params': {
1633                 'youtube_include_dash_manifest': True,
1634                 'format': '135',  # bestvideo
1635             },
1636             'skip': 'This live event has ended.',
1637         },
1638         {
1639             # Multifeed videos (multiple cameras), URL can be of any Camera
1640             'url': 'https://www.youtube.com/watch?v=zaPI8MvL8pg',
1641             'info_dict': {
1642                 'id': 'zaPI8MvL8pg',
1643                 'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04',
1644                 'description': 'md5:563ccbc698b39298481ca3c571169519',
1645             },
1646             'playlist': [{
1647                 'info_dict': {
1648                     'id': 'j5yGuxZ8lLU',
1649                     'ext': 'mp4',
1650                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Chris)',
1651                     'uploader': 'WiiLikeToPlay',
1652                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1653                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1654                     'duration': 10120,
1655                     'channel_follower_count': int,
1656                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1657                     'availability': 'public',
1658                     'playable_in_embed': True,
1659                     'upload_date': '20131105',
1660                     'uploader_id': 'WiiRikeToPray',
1661                     'categories': ['Gaming'],
1662                     'live_status': 'was_live',
1663                     'tags': 'count:24',
1664                     'release_timestamp': 1383701910,
1665                     'thumbnail': 'https://i.ytimg.com/vi/j5yGuxZ8lLU/maxresdefault.jpg',
1666                     'comment_count': int,
1667                     'age_limit': 0,
1668                     'like_count': int,
1669                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1670                     'channel': 'WiiLikeToPlay',
1671                     'view_count': int,
1672                     'release_date': '20131106',
1673                 },
1674             }, {
1675                 'info_dict': {
1676                     'id': 'zaPI8MvL8pg',
1677                     'ext': 'mp4',
1678                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Tyson)',
1679                     'uploader_id': 'WiiRikeToPray',
1680                     'availability': 'public',
1681                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1682                     'channel': 'WiiLikeToPlay',
1683                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1684                     'channel_follower_count': int,
1685                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1686                     'duration': 10108,
1687                     'age_limit': 0,
1688                     'like_count': int,
1689                     'tags': 'count:24',
1690                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1691                     'uploader': 'WiiLikeToPlay',
1692                     'release_timestamp': 1383701915,
1693                     'comment_count': int,
1694                     'upload_date': '20131105',
1695                     'thumbnail': 'https://i.ytimg.com/vi/zaPI8MvL8pg/maxresdefault.jpg',
1696                     'release_date': '20131106',
1697                     'playable_in_embed': True,
1698                     'live_status': 'was_live',
1699                     'categories': ['Gaming'],
1700                     'view_count': int,
1701                 },
1702             }, {
1703                 'info_dict': {
1704                     'id': 'R7r3vfO7Hao',
1705                     'ext': 'mp4',
1706                     'title': 'Terraria 1.2 Live Stream | Let\'s Play - Part 04 (Spencer)',
1707                     'thumbnail': 'https://i.ytimg.com/vi/R7r3vfO7Hao/maxresdefault.jpg',
1708                     'channel_id': 'UCN2XePorRokPB9TEgRZpddg',
1709                     'like_count': int,
1710                     'availability': 'public',
1711                     'playable_in_embed': True,
1712                     'upload_date': '20131105',
1713                     'description': 'md5:563ccbc698b39298481ca3c571169519',
1714                     'uploader_id': 'WiiRikeToPray',
1715                     'uploader_url': 'http://www.youtube.com/user/WiiRikeToPray',
1716                     'channel_follower_count': int,
1717                     'tags': 'count:24',
1718                     'release_date': '20131106',
1719                     'uploader': 'WiiLikeToPlay',
1720                     'comment_count': int,
1721                     'channel_url': 'https://www.youtube.com/channel/UCN2XePorRokPB9TEgRZpddg',
1722                     'channel': 'WiiLikeToPlay',
1723                     'categories': ['Gaming'],
1724                     'release_timestamp': 1383701914,
1725                     'live_status': 'was_live',
1726                     'age_limit': 0,
1727                     'duration': 10128,
1728                     'view_count': int,
1729                 },
1730             }],
1731             'params': {'skip_download': True},
1732         },
1733         {
1734             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
1735             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
1736             'info_dict': {
1737                 'id': 'gVfLd0zydlo',
1738                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
1739             },
1740             'playlist_count': 2,
1741             'skip': 'Not multifeed anymore',
1742         },
1743         {
1744             'url': 'https://vid.plus/FlRa-iH7PGw',
1745             'only_matching': True,
1746         },
1747         {
1748             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
1749             'only_matching': True,
1750         },
1751         {
1752             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1753             # Also tests cut-off URL expansion in video description (see
1754             # https://github.com/ytdl-org/youtube-dl/issues/1892,
1755             # https://github.com/ytdl-org/youtube-dl/issues/8164)
1756             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
1757             'info_dict': {
1758                 'id': 'lsguqyKfVQg',
1759                 'ext': 'mp4',
1760                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
1761                 'alt_title': 'Dark Walk',
1762                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
1763                 'duration': 133,
1764                 'upload_date': '20151119',
1765                 'uploader_id': 'IronSoulElf',
1766                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
1767                 'uploader': 'IronSoulElf',
1768                 'creator': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1769                 'track': 'Dark Walk',
1770                 'artist': 'Todd Haberman;\nDaniel Law Heath and Aaron Kaplan',
1771                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
1772                 'thumbnail': 'https://i.ytimg.com/vi_webp/lsguqyKfVQg/maxresdefault.webp',
1773                 'categories': ['Film & Animation'],
1774                 'view_count': int,
1775                 'live_status': 'not_live',
1776                 'channel_url': 'https://www.youtube.com/channel/UCTSRgz5jylBvFt_S7wnsqLQ',
1777                 'channel_id': 'UCTSRgz5jylBvFt_S7wnsqLQ',
1778                 'tags': 'count:13',
1779                 'availability': 'public',
1780                 'channel': 'IronSoulElf',
1781                 'playable_in_embed': True,
1782                 'like_count': int,
1783                 'age_limit': 0,
1784                 'channel_follower_count': int
1785             },
1786             'params': {
1787                 'skip_download': True,
1788             },
1789         },
1790         {
1791             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
1792             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
1793             'only_matching': True,
1794         },
1795         {
1796             # Video with yt:stretch=17:0
1797             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
1798             'info_dict': {
1799                 'id': 'Q39EVAstoRM',
1800                 'ext': 'mp4',
1801                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
1802                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
1803                 'upload_date': '20151107',
1804                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
1805                 'uploader': 'CH GAMER DROID',
1806             },
1807             'params': {
1808                 'skip_download': True,
1809             },
1810             'skip': 'This video does not exist.',
1811         },
1812         {
1813             # Video with incomplete 'yt:stretch=16:'
1814             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
1815             'only_matching': True,
1816         },
1817         {
1818             # Video licensed under Creative Commons
1819             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1820             'info_dict': {
1821                 'id': 'M4gD1WSo5mA',
1822                 'ext': 'mp4',
1823                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1824                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1825                 'duration': 721,
1826                 'upload_date': '20150128',
1827                 'uploader_id': 'BerkmanCenter',
1828                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1829                 'uploader': 'The Berkman Klein Center for Internet & Society',
1830                 'license': 'Creative Commons Attribution license (reuse allowed)',
1831                 'channel_id': 'UCuLGmD72gJDBwmLw06X58SA',
1832                 'channel_url': 'https://www.youtube.com/channel/UCuLGmD72gJDBwmLw06X58SA',
1833                 'like_count': int,
1834                 'age_limit': 0,
1835                 'tags': ['Copyright (Legal Subject)', 'Law (Industry)', 'William W. Fisher (Author)'],
1836                 'channel': 'The Berkman Klein Center for Internet & Society',
1837                 'availability': 'public',
1838                 'view_count': int,
1839                 'categories': ['Education'],
1840                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
1841                 'live_status': 'not_live',
1842                 'playable_in_embed': True,
1843                 'comment_count': int,
1844                 'channel_follower_count': int,
1845                 'chapters': list,
1846             },
1847             'params': {
1848                 'skip_download': True,
1849             },
1850         },
1851         {
1852             # Channel-like uploader_url
1853             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1854             'info_dict': {
1855                 'id': 'eQcmzGIKrzg',
1856                 'ext': 'mp4',
1857                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1858                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
1859                 'duration': 4060,
1860                 'upload_date': '20151120',
1861                 'uploader': 'Bernie Sanders',
1862                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1863                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1864                 'license': 'Creative Commons Attribution license (reuse allowed)',
1865                 'playable_in_embed': True,
1866                 'tags': 'count:12',
1867                 'like_count': int,
1868                 'channel_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1869                 'age_limit': 0,
1870                 'availability': 'public',
1871                 'categories': ['News & Politics'],
1872                 'channel': 'Bernie Sanders',
1873                 'thumbnail': 'https://i.ytimg.com/vi_webp/eQcmzGIKrzg/maxresdefault.webp',
1874                 'view_count': int,
1875                 'live_status': 'not_live',
1876                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1877                 'comment_count': int,
1878                 'channel_follower_count': int,
1879                 'chapters': list,
1880             },
1881             'params': {
1882                 'skip_download': True,
1883             },
1884         },
1885         {
1886             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1887             'only_matching': True,
1888         },
1889         {
1890             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1891             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1892             'only_matching': True,
1893         },
1894         {
1895             # Rental video preview
1896             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1897             'info_dict': {
1898                 'id': 'uGpuVWrhIzE',
1899                 'ext': 'mp4',
1900                 'title': 'Piku - Trailer',
1901                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1902                 'upload_date': '20150811',
1903                 'uploader': 'FlixMatrix',
1904                 'uploader_id': 'FlixMatrixKaravan',
1905                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1906                 'license': 'Standard YouTube License',
1907             },
1908             'params': {
1909                 'skip_download': True,
1910             },
1911             'skip': 'This video is not available.',
1912         },
1913         {
1914             # YouTube Red video with episode data
1915             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1916             'info_dict': {
1917                 'id': 'iqKdEhx-dD4',
1918                 'ext': 'mp4',
1919                 'title': 'Isolation - Mind Field (Ep 1)',
1920                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
1921                 'duration': 2085,
1922                 'upload_date': '20170118',
1923                 'uploader': 'Vsauce',
1924                 'uploader_id': 'Vsauce',
1925                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1926                 'series': 'Mind Field',
1927                 'season_number': 1,
1928                 'episode_number': 1,
1929                 'thumbnail': 'https://i.ytimg.com/vi_webp/iqKdEhx-dD4/maxresdefault.webp',
1930                 'tags': 'count:12',
1931                 'view_count': int,
1932                 'availability': 'public',
1933                 'age_limit': 0,
1934                 'channel': 'Vsauce',
1935                 'episode': 'Episode 1',
1936                 'categories': ['Entertainment'],
1937                 'season': 'Season 1',
1938                 'channel_id': 'UC6nSFpj9HTCZ5t-N3Rm3-HA',
1939                 'channel_url': 'https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA',
1940                 'like_count': int,
1941                 'playable_in_embed': True,
1942                 'live_status': 'not_live',
1943                 'channel_follower_count': int
1944             },
1945             'params': {
1946                 'skip_download': True,
1947             },
1948             'expected_warnings': [
1949                 'Skipping DASH manifest',
1950             ],
1951         },
1952         {
1953             # The following content has been identified by the YouTube community
1954             # as inappropriate or offensive to some audiences.
1955             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1956             'info_dict': {
1957                 'id': '6SJNVb0GnPI',
1958                 'ext': 'mp4',
1959                 'title': 'Race Differences in Intelligence',
1960                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1961                 'duration': 965,
1962                 'upload_date': '20140124',
1963                 'uploader': 'New Century Foundation',
1964                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1965                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1966             },
1967             'params': {
1968                 'skip_download': True,
1969             },
1970             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
1971         },
1972         {
1973             # itag 212
1974             'url': '1t24XAntNCY',
1975             'only_matching': True,
1976         },
1977         {
1978             # geo restricted to JP
1979             'url': 'sJL6WA-aGkQ',
1980             'only_matching': True,
1981         },
1982         {
1983             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1984             'only_matching': True,
1985         },
1986         {
1987             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
1988             'only_matching': True,
1989         },
1990         {
1991             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
1992             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
1993             'only_matching': True,
1994         },
1995         {
1996             # DRM protected
1997             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1998             'only_matching': True,
1999         },
2000         {
2001             # Video with unsupported adaptive stream type formats
2002             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
2003             'info_dict': {
2004                 'id': 'Z4Vy8R84T1U',
2005                 'ext': 'mp4',
2006                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
2007                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
2008                 'duration': 433,
2009                 'upload_date': '20130923',
2010                 'uploader': 'Amelia Putri Harwita',
2011                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
2012                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
2013                 'formats': 'maxcount:10',
2014             },
2015             'params': {
2016                 'skip_download': True,
2017                 'youtube_include_dash_manifest': False,
2018             },
2019             'skip': 'not actual anymore',
2020         },
2021         {
2022             # Youtube Music Auto-generated description
2023             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2024             'info_dict': {
2025                 'id': 'MgNrAu2pzNs',
2026                 'ext': 'mp4',
2027                 'title': 'Voyeur Girl',
2028                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
2029                 'upload_date': '20190312',
2030                 'uploader': 'Stephen - Topic',
2031                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
2032                 'artist': 'Stephen',
2033                 'track': 'Voyeur Girl',
2034                 'album': 'it\'s too much love to know my dear',
2035                 'release_date': '20190313',
2036                 'release_year': 2019,
2037                 'alt_title': 'Voyeur Girl',
2038                 'view_count': int,
2039                 'uploader_url': 'http://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
2040                 'playable_in_embed': True,
2041                 'like_count': int,
2042                 'categories': ['Music'],
2043                 'channel_url': 'https://www.youtube.com/channel/UC-pWHpBjdGG69N9mM2auIAA',
2044                 'channel': 'Stephen',
2045                 'availability': 'public',
2046                 'creator': 'Stephen',
2047                 'duration': 169,
2048                 'thumbnail': 'https://i.ytimg.com/vi_webp/MgNrAu2pzNs/maxresdefault.webp',
2049                 'age_limit': 0,
2050                 'channel_id': 'UC-pWHpBjdGG69N9mM2auIAA',
2051                 'tags': 'count:11',
2052                 'live_status': 'not_live',
2053                 'channel_follower_count': int
2054             },
2055             'params': {
2056                 'skip_download': True,
2057             },
2058         },
2059         {
2060             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
2061             'only_matching': True,
2062         },
2063         {
2064             # invalid -> valid video id redirection
2065             'url': 'DJztXj2GPfl',
2066             'info_dict': {
2067                 'id': 'DJztXj2GPfk',
2068                 'ext': 'mp4',
2069                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
2070                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
2071                 'upload_date': '20090125',
2072                 'uploader': 'Prochorowka',
2073                 'uploader_id': 'Prochorowka',
2074                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
2075                 'artist': 'Panjabi MC',
2076                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
2077                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
2078             },
2079             'params': {
2080                 'skip_download': True,
2081             },
2082             'skip': 'Video unavailable',
2083         },
2084         {
2085             # empty description results in an empty string
2086             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
2087             'info_dict': {
2088                 'id': 'x41yOUIvK2k',
2089                 'ext': 'mp4',
2090                 'title': 'IMG 3456',
2091                 'description': '',
2092                 'upload_date': '20170613',
2093                 'uploader_id': 'ElevageOrVert',
2094                 'uploader': 'ElevageOrVert',
2095                 'view_count': int,
2096                 'thumbnail': 'https://i.ytimg.com/vi_webp/x41yOUIvK2k/maxresdefault.webp',
2097                 'uploader_url': 'http://www.youtube.com/user/ElevageOrVert',
2098                 'like_count': int,
2099                 'channel_id': 'UCo03ZQPBW5U4UC3regpt1nw',
2100                 'tags': [],
2101                 'channel_url': 'https://www.youtube.com/channel/UCo03ZQPBW5U4UC3regpt1nw',
2102                 'availability': 'public',
2103                 'age_limit': 0,
2104                 'categories': ['Pets & Animals'],
2105                 'duration': 7,
2106                 'playable_in_embed': True,
2107                 'live_status': 'not_live',
2108                 'channel': 'ElevageOrVert',
2109                 'channel_follower_count': int
2110             },
2111             'params': {
2112                 'skip_download': True,
2113             },
2114         },
2115         {
2116             # with '};' inside yt initial data (see [1])
2117             # see [2] for an example with '};' inside ytInitialPlayerResponse
2118             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
2119             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2120             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
2121             'info_dict': {
2122                 'id': 'CHqg6qOn4no',
2123                 'ext': 'mp4',
2124                 'title': 'Part 77   Sort a list of simple types in c#',
2125                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
2126                 'upload_date': '20130831',
2127                 'uploader_id': 'kudvenkat',
2128                 'uploader': 'kudvenkat',
2129                 'channel_id': 'UCCTVrRB5KpIiK6V2GGVsR1Q',
2130                 'like_count': int,
2131                 'uploader_url': 'http://www.youtube.com/user/kudvenkat',
2132                 'channel_url': 'https://www.youtube.com/channel/UCCTVrRB5KpIiK6V2GGVsR1Q',
2133                 'live_status': 'not_live',
2134                 'categories': ['Education'],
2135                 'availability': 'public',
2136                 'thumbnail': 'https://i.ytimg.com/vi/CHqg6qOn4no/sddefault.jpg',
2137                 'tags': 'count:12',
2138                 'playable_in_embed': True,
2139                 'age_limit': 0,
2140                 'view_count': int,
2141                 'duration': 522,
2142                 'channel': 'kudvenkat',
2143                 'comment_count': int,
2144                 'channel_follower_count': int,
2145                 'chapters': list,
2146             },
2147             'params': {
2148                 'skip_download': True,
2149             },
2150         },
2151         {
2152             # another example of '};' in ytInitialData
2153             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
2154             'only_matching': True,
2155         },
2156         {
2157             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
2158             'only_matching': True,
2159         },
2160         {
2161             # https://github.com/ytdl-org/youtube-dl/pull/28094
2162             'url': 'OtqTfy26tG0',
2163             'info_dict': {
2164                 'id': 'OtqTfy26tG0',
2165                 'ext': 'mp4',
2166                 'title': 'Burn Out',
2167                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
2168                 'upload_date': '20141120',
2169                 'uploader': 'The Cinematic Orchestra - Topic',
2170                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2171                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2172                 'artist': 'The Cinematic Orchestra',
2173                 'track': 'Burn Out',
2174                 'album': 'Every Day',
2175                 'like_count': int,
2176                 'live_status': 'not_live',
2177                 'alt_title': 'Burn Out',
2178                 'duration': 614,
2179                 'age_limit': 0,
2180                 'view_count': int,
2181                 'channel_url': 'https://www.youtube.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
2182                 'creator': 'The Cinematic Orchestra',
2183                 'channel': 'The Cinematic Orchestra',
2184                 'tags': ['The Cinematic Orchestra', 'Every Day', 'Burn Out'],
2185                 'channel_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
2186                 'availability': 'public',
2187                 'thumbnail': 'https://i.ytimg.com/vi/OtqTfy26tG0/maxresdefault.jpg',
2188                 'categories': ['Music'],
2189                 'playable_in_embed': True,
2190                 'channel_follower_count': int
2191             },
2192             'params': {
2193                 'skip_download': True,
2194             },
2195         },
2196         {
2197             # controversial video, only works with bpctr when authenticated with cookies
2198             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
2199             'only_matching': True,
2200         },
2201         {
2202             # controversial video, requires bpctr/contentCheckOk
2203             'url': 'https://www.youtube.com/watch?v=SZJvDhaSDnc',
2204             'info_dict': {
2205                 'id': 'SZJvDhaSDnc',
2206                 'ext': 'mp4',
2207                 'title': 'San Diego teen commits suicide after bullying over embarrassing video',
2208                 'channel_id': 'UC-SJ6nODDmufqBzPBwCvYvQ',
2209                 'uploader': 'CBS Mornings',
2210                 'uploader_id': 'CBSThisMorning',
2211                 'upload_date': '20140716',
2212                 'description': 'md5:acde3a73d3f133fc97e837a9f76b53b7',
2213                 'duration': 170,
2214                 'categories': ['News & Politics'],
2215                 'uploader_url': 'http://www.youtube.com/user/CBSThisMorning',
2216                 'view_count': int,
2217                 'channel': 'CBS Mornings',
2218                 'tags': ['suicide', 'bullying', 'video', 'cbs', 'news'],
2219                 'thumbnail': 'https://i.ytimg.com/vi/SZJvDhaSDnc/hqdefault.jpg',
2220                 'age_limit': 18,
2221                 'availability': 'needs_auth',
2222                 'channel_url': 'https://www.youtube.com/channel/UC-SJ6nODDmufqBzPBwCvYvQ',
2223                 'like_count': int,
2224                 'live_status': 'not_live',
2225                 'playable_in_embed': True,
2226                 'channel_follower_count': int
2227             }
2228         },
2229         {
2230             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
2231             'url': 'cBvYw8_A0vQ',
2232             'info_dict': {
2233                 'id': 'cBvYw8_A0vQ',
2234                 'ext': 'mp4',
2235                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
2236                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
2237                 'upload_date': '20201120',
2238                 'uploader': 'Walk around Japan',
2239                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2240                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2241                 'duration': 1456,
2242                 'categories': ['Travel & Events'],
2243                 'channel_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
2244                 'view_count': int,
2245                 'channel': 'Walk around Japan',
2246                 'tags': ['Ueno Tokyo', 'Okachimachi Tokyo', 'Ameyoko Street', 'Tokyo attraction', 'Travel in Tokyo'],
2247                 'thumbnail': 'https://i.ytimg.com/vi_webp/cBvYw8_A0vQ/hqdefault.webp',
2248                 'age_limit': 0,
2249                 'availability': 'public',
2250                 'channel_url': 'https://www.youtube.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
2251                 'live_status': 'not_live',
2252                 'playable_in_embed': True,
2253                 'channel_follower_count': int
2254             },
2255             'params': {
2256                 'skip_download': True,
2257             },
2258         }, {
2259             # Has multiple audio streams
2260             'url': 'WaOKSUlf4TM',
2261             'only_matching': True
2262         }, {
2263             # Requires Premium: has format 141 when requested using YTM url
2264             'url': 'https://music.youtube.com/watch?v=XclachpHxis',
2265             'only_matching': True
2266         }, {
2267             # multiple subtitles with same lang_code
2268             'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
2269             'only_matching': True,
2270         }, {
2271             # Force use android client fallback
2272             'url': 'https://www.youtube.com/watch?v=YOelRv7fMxY',
2273             'info_dict': {
2274                 'id': 'YOelRv7fMxY',
2275                 'title': 'DIGGING A SECRET TUNNEL Part 1',
2276                 'ext': '3gp',
2277                 'upload_date': '20210624',
2278                 'channel_id': 'UCp68_FLety0O-n9QU6phsgw',
2279                 'uploader': 'colinfurze',
2280                 'uploader_id': 'colinfurze',
2281                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCp68_FLety0O-n9QU6phsgw',
2282                 'description': 'md5:5d5991195d599b56cd0c4148907eec50',
2283                 'duration': 596,
2284                 'categories': ['Entertainment'],
2285                 'uploader_url': 'http://www.youtube.com/user/colinfurze',
2286                 'view_count': int,
2287                 'channel': 'colinfurze',
2288                 'tags': ['Colin', 'furze', 'Terry', 'tunnel', 'underground', 'bunker'],
2289                 'thumbnail': 'https://i.ytimg.com/vi/YOelRv7fMxY/maxresdefault.jpg',
2290                 'age_limit': 0,
2291                 'availability': 'public',
2292                 'like_count': int,
2293                 'live_status': 'not_live',
2294                 'playable_in_embed': True,
2295                 'channel_follower_count': int,
2296                 'chapters': list,
2297             },
2298             'params': {
2299                 'format': '17',  # 3gp format available on android
2300                 'extractor_args': {'youtube': {'player_client': ['android']}},
2301             },
2302         },
2303         {
2304             # Skip download of additional client configs (remix client config in this case)
2305             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
2306             'only_matching': True,
2307             'params': {
2308                 'extractor_args': {'youtube': {'player_skip': ['configs']}},
2309             },
2310         }, {
2311             # shorts
2312             'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
2313             'only_matching': True,
2314         }, {
2315             'note': 'Storyboards',
2316             'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
2317             'info_dict': {
2318                 'id': '5KLPxDtMqe8',
2319                 'ext': 'mhtml',
2320                 'format_id': 'sb0',
2321                 'title': 'Your Brain is Plastic',
2322                 'uploader_id': 'scishow',
2323                 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
2324                 'upload_date': '20140324',
2325                 'uploader': 'SciShow',
2326                 'like_count': int,
2327                 'channel_id': 'UCZYTClx2T1of7BRZ86-8fow',
2328                 'channel_url': 'https://www.youtube.com/channel/UCZYTClx2T1of7BRZ86-8fow',
2329                 'view_count': int,
2330                 'thumbnail': 'https://i.ytimg.com/vi/5KLPxDtMqe8/maxresdefault.jpg',
2331                 'playable_in_embed': True,
2332                 'tags': 'count:12',
2333                 'uploader_url': 'http://www.youtube.com/user/scishow',
2334                 'availability': 'public',
2335                 'channel': 'SciShow',
2336                 'live_status': 'not_live',
2337                 'duration': 248,
2338                 'categories': ['Education'],
2339                 'age_limit': 0,
2340                 'channel_follower_count': int,
2341                 'chapters': list,
2342             }, 'params': {'format': 'mhtml', 'skip_download': True}
2343         }, {
2344             # Ensure video upload_date is in UTC timezone (video was uploaded 1641170939)
2345             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2346             'info_dict': {
2347                 'id': '2NUZ8W2llS4',
2348                 'ext': 'mp4',
2349                 'title': 'The NP that test your phone performance 🙂',
2350                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2351                 'uploader': 'Leon Nguyen',
2352                 'uploader_id': 'VNSXIII',
2353                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2354                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2355                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2356                 'duration': 21,
2357                 'view_count': int,
2358                 'age_limit': 0,
2359                 'categories': ['Gaming'],
2360                 'tags': 'count:23',
2361                 'playable_in_embed': True,
2362                 'live_status': 'not_live',
2363                 'upload_date': '20220103',
2364                 'like_count': int,
2365                 'availability': 'public',
2366                 'channel': 'Leon Nguyen',
2367                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2368                 'comment_count': int,
2369                 'channel_follower_count': int
2370             }
2371         }, {
2372             # Same video as above, but with --compat-opt no-youtube-prefer-utc-upload-date
2373             'url': 'https://www.youtube.com/watch?v=2NUZ8W2llS4',
2374             'info_dict': {
2375                 'id': '2NUZ8W2llS4',
2376                 'ext': 'mp4',
2377                 'title': 'The NP that test your phone performance 🙂',
2378                 'description': 'md5:144494b24d4f9dfacb97c1bbef5de84d',
2379                 'uploader': 'Leon Nguyen',
2380                 'uploader_id': 'VNSXIII',
2381                 'uploader_url': 'http://www.youtube.com/user/VNSXIII',
2382                 'channel_id': 'UCRqNBSOHgilHfAczlUmlWHA',
2383                 'channel_url': 'https://www.youtube.com/channel/UCRqNBSOHgilHfAczlUmlWHA',
2384                 'duration': 21,
2385                 'view_count': int,
2386                 'age_limit': 0,
2387                 'categories': ['Gaming'],
2388                 'tags': 'count:23',
2389                 'playable_in_embed': True,
2390                 'live_status': 'not_live',
2391                 'upload_date': '20220102',
2392                 'like_count': int,
2393                 'availability': 'public',
2394                 'channel': 'Leon Nguyen',
2395                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
2396                 'comment_count': int,
2397                 'channel_follower_count': int
2398             },
2399             'params': {'compat_opts': ['no-youtube-prefer-utc-upload-date']}
2400         }, {
2401             # date text is premiered video, ensure upload date in UTC (published 1641172509)
2402             'url': 'https://www.youtube.com/watch?v=mzZzzBU6lrM',
2403             'info_dict': {
2404                 'id': 'mzZzzBU6lrM',
2405                 'ext': 'mp4',
2406                 'title': 'I Met GeorgeNotFound In Real Life...',
2407                 'description': 'md5:cca98a355c7184e750f711f3a1b22c84',
2408                 'uploader': 'Quackity',
2409                 'uploader_id': 'QuackityHQ',
2410                 'uploader_url': 'http://www.youtube.com/user/QuackityHQ',
2411                 'channel_id': 'UC_8NknAFiyhOUaZqHR3lq3Q',
2412                 'channel_url': 'https://www.youtube.com/channel/UC_8NknAFiyhOUaZqHR3lq3Q',
2413                 'duration': 955,
2414                 'view_count': int,
2415                 'age_limit': 0,
2416                 'categories': ['Entertainment'],
2417                 'tags': 'count:26',
2418                 'playable_in_embed': True,
2419                 'live_status': 'not_live',
2420                 'release_timestamp': 1641172509,
2421                 'release_date': '20220103',
2422                 'upload_date': '20220103',
2423                 'like_count': int,
2424                 'availability': 'public',
2425                 'channel': 'Quackity',
2426                 'thumbnail': 'https://i.ytimg.com/vi/mzZzzBU6lrM/maxresdefault.jpg',
2427                 'channel_follower_count': int
2428             }
2429         },
2430         {   # continuous livestream. Microformat upload date should be preferred.
2431             # Upload date was 2021-06-19 (not UTC), while stream start is 2021-11-27
2432             'url': 'https://www.youtube.com/watch?v=kgx4WGK0oNU',
2433             'info_dict': {
2434                 'id': 'kgx4WGK0oNU',
2435                 'title': r're:jazz\/lofi hip hop radio🌱chill beats to relax\/study to \[LIVE 24\/7\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
2436                 'ext': 'mp4',
2437                 'channel_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2438                 'availability': 'public',
2439                 'age_limit': 0,
2440                 'release_timestamp': 1637975704,
2441                 'upload_date': '20210619',
2442                 'channel_url': 'https://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2443                 'live_status': 'is_live',
2444                 'thumbnail': 'https://i.ytimg.com/vi/kgx4WGK0oNU/maxresdefault.jpg',
2445                 'uploader': '阿鲍Abao',
2446                 'uploader_url': 'http://www.youtube.com/channel/UC84whx2xxsiA1gXHXXqKGOA',
2447                 'channel': 'Abao in Tokyo',
2448                 'channel_follower_count': int,
2449                 'release_date': '20211127',
2450                 'tags': 'count:39',
2451                 'categories': ['People & Blogs'],
2452                 'like_count': int,
2453                 'uploader_id': 'UC84whx2xxsiA1gXHXXqKGOA',
2454                 'view_count': int,
2455                 'playable_in_embed': True,
2456                 'description': 'md5:2ef1d002cad520f65825346e2084e49d',
2457                 'concurrent_view_count': int,
2458             },
2459             'params': {'skip_download': True}
2460         }, {
2461             # Story. Requires specific player params to work.
2462             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
2463             'info_dict': {
2464                 'id': 'vv8qTUWmulI',
2465                 'ext': 'mp4',
2466                 'availability': 'unlisted',
2467                 'view_count': int,
2468                 'channel_id': 'UCzIZ8HrzDgc-pNQDUG6avBA',
2469                 'upload_date': '20220526',
2470                 'categories': ['Education'],
2471                 'title': 'Story',
2472                 'channel': 'IT\'S HISTORY',
2473                 'description': '',
2474                 'uploader_id': 'BlastfromthePast',
2475                 'duration': 12,
2476                 'uploader': 'IT\'S HISTORY',
2477                 'playable_in_embed': True,
2478                 'age_limit': 0,
2479                 'live_status': 'not_live',
2480                 'tags': [],
2481                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
2482                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
2483                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
2484             },
2485             'skip': 'stories get removed after some period of time',
2486         }, {
2487             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
2488             'info_dict': {
2489                 'id': 'tjjjtzRLHvA',
2490                 'ext': 'mp4',
2491                 'title': 'ハッシュタグ無し };if window.ytcsi',
2492                 'upload_date': '20220323',
2493                 'like_count': int,
2494                 'availability': 'unlisted',
2495                 'channel': 'nao20010128nao',
2496                 'thumbnail': 'https://i.ytimg.com/vi_webp/tjjjtzRLHvA/maxresdefault.webp',
2497                 'age_limit': 0,
2498                 'uploader': 'nao20010128nao',
2499                 'uploader_id': 'nao20010128nao',
2500                 'categories': ['Music'],
2501                 'view_count': int,
2502                 'description': '',
2503                 'channel_url': 'https://www.youtube.com/channel/UCdqltm_7iv1Vs6kp6Syke5A',
2504                 'channel_id': 'UCdqltm_7iv1Vs6kp6Syke5A',
2505                 'live_status': 'not_live',
2506                 'playable_in_embed': True,
2507                 'channel_follower_count': int,
2508                 'duration': 6,
2509                 'tags': [],
2510                 'uploader_url': 'http://www.youtube.com/user/nao20010128nao',
2511             }
2512         }, {
2513             # Prefer primary title+description language metadata by default
2514             # Do not prefer translated description if primary is empty
2515             'url': 'https://www.youtube.com/watch?v=el3E4MbxRqQ',
2516             'info_dict': {
2517                 'id': 'el3E4MbxRqQ',
2518                 'ext': 'mp4',
2519                 'title': 'dlp test video 2 - primary sv no desc',
2520                 'description': '',
2521                 'channel': 'cole-dlp-test-acc',
2522                 'tags': [],
2523                 'view_count': int,
2524                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2525                 'like_count': int,
2526                 'playable_in_embed': True,
2527                 'availability': 'unlisted',
2528                 'thumbnail': 'https://i.ytimg.com/vi_webp/el3E4MbxRqQ/maxresdefault.webp',
2529                 'age_limit': 0,
2530                 'duration': 5,
2531                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2532                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2533                 'live_status': 'not_live',
2534                 'upload_date': '20220908',
2535                 'categories': ['People & Blogs'],
2536                 'uploader': 'cole-dlp-test-acc',
2537                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2538             },
2539             'params': {'skip_download': True}
2540         }, {
2541             # Extractor argument: prefer translated title+description
2542             'url': 'https://www.youtube.com/watch?v=gHKT4uU8Zng',
2543             'info_dict': {
2544                 'id': 'gHKT4uU8Zng',
2545                 'ext': 'mp4',
2546                 'channel': 'cole-dlp-test-acc',
2547                 'tags': [],
2548                 'duration': 5,
2549                 'live_status': 'not_live',
2550                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
2551                 'upload_date': '20220728',
2552                 'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
2553                 'view_count': int,
2554                 'categories': ['People & Blogs'],
2555                 'thumbnail': 'https://i.ytimg.com/vi_webp/gHKT4uU8Zng/maxresdefault.webp',
2556                 'title': 'dlp test video title translated (fr)',
2557                 'availability': 'public',
2558                 'uploader': 'cole-dlp-test-acc',
2559                 'age_limit': 0,
2560                 'description': 'dlp test video description translated (fr)',
2561                 'playable_in_embed': True,
2562                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2563                 'uploader_url': 'http://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
2564             },
2565             'params': {'skip_download': True, 'extractor_args': {'youtube': {'lang': ['fr']}}},
2566             'expected_warnings': [r'Preferring "fr" translated fields'],
2567         }, {
2568             'note': '6 channel audio',
2569             'url': 'https://www.youtube.com/watch?v=zgdo7-RRjgo',
2570             'only_matching': True,
2571         }, {
2572             'note': 'Multiple HLS formats with same itag',
2573             'url': 'https://www.youtube.com/watch?v=kX3nB4PpJko',
2574             'info_dict': {
2575                 'id': 'kX3nB4PpJko',
2576                 'ext': 'mp4',
2577                 'categories': ['Entertainment'],
2578                 'description': 'md5:e8031ff6e426cdb6a77670c9b81f6fa6',
2579                 'uploader_url': 'http://www.youtube.com/user/MrBeast6000',
2580                 'live_status': 'not_live',
2581                 'duration': 937,
2582                 'channel_follower_count': int,
2583                 'thumbnail': 'https://i.ytimg.com/vi_webp/kX3nB4PpJko/maxresdefault.webp',
2584                 'title': 'Last To Take Hand Off Jet, Keeps It!',
2585                 'channel': 'MrBeast',
2586                 'playable_in_embed': True,
2587                 'view_count': int,
2588                 'upload_date': '20221112',
2589                 'uploader': 'MrBeast',
2590                 'uploader_id': 'MrBeast6000',
2591                 'channel_url': 'https://www.youtube.com/channel/UCX6OQ3DkcsbYNE6H8uQQuVA',
2592                 'age_limit': 0,
2593                 'availability': 'public',
2594                 'channel_id': 'UCX6OQ3DkcsbYNE6H8uQQuVA',
2595                 'like_count': int,
2596                 'tags': [],
2597             },
2598             'params': {'extractor_args': {'youtube': {'player_client': ['ios']}}, 'format': '233-1'},
2599         }, {
2600             'note': 'Audio formats with Dynamic Range Compression',
2601             'url': 'https://www.youtube.com/watch?v=Tq92D6wQ1mg',
2602             'info_dict': {
2603                 'id': 'Tq92D6wQ1mg',
2604                 'ext': 'weba',
2605                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
2606                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2607                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2608                 'channel_follower_count': int,
2609                 'description': 'md5:17eccca93a786d51bc67646756894066',
2610                 'upload_date': '20191228',
2611                 'uploader_url': 'http://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
2612                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
2613                 'playable_in_embed': True,
2614                 'like_count': int,
2615                 'categories': ['Entertainment'],
2616                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
2617                 'age_limit': 18,
2618                 'channel': 'Projekt Melody',
2619                 'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
2620                 'view_count': int,
2621                 'availability': 'needs_auth',
2622                 'comment_count': int,
2623                 'live_status': 'not_live',
2624                 'uploader': 'Projekt Melody',
2625                 'duration': 106,
2626             },
2627             'params': {'extractor_args': {'youtube': {'player_client': ['tv_embedded']}}, 'format': '251-drc'},
2628         },
2629         {
2630             'url': 'https://www.youtube.com/live/qVv6vCqciTM',
2631             'info_dict': {
2632                 'id': 'qVv6vCqciTM',
2633                 'ext': 'mp4',
2634                 'age_limit': 0,
2635                 'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
2636                 'comment_count': int,
2637                 'chapters': 'count:13',
2638                 'upload_date': '20221223',
2639                 'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
2640                 'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
2641                 'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
2642                 'like_count': int,
2643                 'release_date': '20221223',
2644                 'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
2645                 'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
2646                 'view_count': int,
2647                 'playable_in_embed': True,
2648                 'duration': 4438,
2649                 'availability': 'public',
2650                 'channel_follower_count': int,
2651                 'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
2652                 'categories': ['Entertainment'],
2653                 'live_status': 'was_live',
2654                 'release_timestamp': 1671793345,
2655                 'channel': 'さなちゃんねる',
2656                 'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
2657                 'uploader': 'さなちゃんねる',
2658             },
2659         },
2660     ]
2661
2662     _WEBPAGE_TESTS = [
2663         # YouTube <object> embed
2664         {
2665             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
2666             'md5': '873c81d308b979f0e23ee7e620b312a3',
2667             'info_dict': {
2668                 'id': 'msN87y-iEx0',
2669                 'ext': 'mp4',
2670                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
2671                 'upload_date': '20080526',
2672                 'description': 'md5:873c81d308b979f0e23ee7e620b312a3',
2673                 'uploader': 'Christopher Sykes',
2674                 'uploader_id': 'ChristopherJSykes',
2675                 'age_limit': 0,
2676                 'tags': ['feynman', 'mirror', 'science', 'physics', 'imagination', 'fun', 'cool', 'puzzle'],
2677                 'channel_id': 'UCCeo--lls1vna5YJABWAcVA',
2678                 'playable_in_embed': True,
2679                 'thumbnail': 'https://i.ytimg.com/vi/msN87y-iEx0/hqdefault.jpg',
2680                 'like_count': int,
2681                 'comment_count': int,
2682                 'channel': 'Christopher Sykes',
2683                 'live_status': 'not_live',
2684                 'channel_url': 'https://www.youtube.com/channel/UCCeo--lls1vna5YJABWAcVA',
2685                 'availability': 'public',
2686                 'duration': 195,
2687                 'view_count': int,
2688                 'categories': ['Science & Technology'],
2689                 'channel_follower_count': int,
2690                 'uploader_url': 'http://www.youtube.com/user/ChristopherJSykes',
2691             },
2692             'params': {
2693                 'skip_download': True,
2694             }
2695         },
2696     ]
2697
2698     @classmethod
2699     def suitable(cls, url):
2700         from ..utils import parse_qs
2701
2702         qs = parse_qs(url)
2703         if qs.get('list', [None])[0]:
2704             return False
2705         return super().suitable(url)
2706
2707     def __init__(self, *args, **kwargs):
2708         super().__init__(*args, **kwargs)
2709         self._code_cache = {}
2710         self._player_cache = {}
2711
2712     def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live):
2713         lock = threading.Lock()
2714         start_time = time.time()
2715         formats = [f for f in formats if f.get('is_from_start')]
2716
2717         def refetch_manifest(format_id, delay):
2718             nonlocal formats, start_time, is_live
2719             if time.time() <= start_time + delay:
2720                 return
2721
2722             _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
2723             video_details = traverse_obj(prs, (..., 'videoDetails'), expected_type=dict)
2724             microformats = traverse_obj(
2725                 prs, (..., 'microformat', 'playerMicroformatRenderer'),
2726                 expected_type=dict)
2727             _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
2728             is_live = live_status == 'is_live'
2729             start_time = time.time()
2730
2731         def mpd_feed(format_id, delay):
2732             """
2733             @returns (manifest_url, manifest_stream_number, is_live) or None
2734             """
2735             for retry in self.RetryManager(fatal=False):
2736                 with lock:
2737                     refetch_manifest(format_id, delay)
2738
2739                 f = next((f for f in formats if f['format_id'] == format_id), None)
2740                 if not f:
2741                     if not is_live:
2742                         retry.error = f'{video_id}: Video is no longer live'
2743                     else:
2744                         retry.error = f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}'
2745                     continue
2746                 return f['manifest_url'], f['manifest_stream_number'], is_live
2747             return None
2748
2749         for f in formats:
2750             f['is_live'] = is_live
2751             gen = functools.partial(self._live_dash_fragments, video_id, f['format_id'],
2752                                     live_start_time, mpd_feed, not is_live and f.copy())
2753             if is_live:
2754                 f['fragments'] = gen
2755                 f['protocol'] = 'http_dash_segments_generator'
2756             else:
2757                 f['fragments'] = LazyList(gen({}))
2758                 del f['is_from_start']
2759
2760     def _live_dash_fragments(self, video_id, format_id, live_start_time, mpd_feed, manifestless_orig_fmt, ctx):
2761         FETCH_SPAN, MAX_DURATION = 5, 432000
2762
2763         mpd_url, stream_number, is_live = None, None, True
2764
2765         begin_index = 0
2766         download_start_time = ctx.get('start') or time.time()
2767
2768         lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
2769         if lack_early_segments:
2770             self.report_warning(bug_reports_message(
2771                 'Starting download from the last 120 hours of the live stream since '
2772                 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
2773             lack_early_segments = True
2774
2775         known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
2776         fragments, fragment_base_url = None, None
2777
2778         def _extract_sequence_from_mpd(refresh_sequence, immediate):
2779             nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
2780             # Obtain from MPD's maximum seq value
2781             old_mpd_url = mpd_url
2782             last_error = ctx.pop('last_error', None)
2783             expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
2784             mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
2785                                                or (mpd_url, stream_number, False))
2786             if not refresh_sequence:
2787                 if expire_fast and not is_live:
2788                     return False, last_seq
2789                 elif old_mpd_url == mpd_url:
2790                     return True, last_seq
2791             if manifestless_orig_fmt:
2792                 fmt_info = manifestless_orig_fmt
2793             else:
2794                 try:
2795                     fmts, _ = self._extract_mpd_formats_and_subtitles(
2796                         mpd_url, None, note=False, errnote=False, fatal=False)
2797                 except ExtractorError:
2798                     fmts = None
2799                 if not fmts:
2800                     no_fragment_score += 2
2801                     return False, last_seq
2802                 fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
2803             fragments = fmt_info['fragments']
2804             fragment_base_url = fmt_info['fragment_base_url']
2805             assert fragment_base_url
2806
2807             _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
2808             return True, _last_seq
2809
2810         self.write_debug(f'[{video_id}] Generating fragments for format {format_id}')
2811         while is_live:
2812             fetch_time = time.time()
2813             if no_fragment_score > 30:
2814                 return
2815             if last_segment_url:
2816                 # Obtain from "X-Head-Seqnum" header value from each segment
2817                 try:
2818                     urlh = self._request_webpage(
2819                         last_segment_url, None, note=False, errnote=False, fatal=False)
2820                 except ExtractorError:
2821                     urlh = None
2822                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
2823                 if last_seq is None:
2824                     no_fragment_score += 2
2825                     last_segment_url = None
2826                     continue
2827             else:
2828                 should_continue, last_seq = _extract_sequence_from_mpd(True, no_fragment_score > 15)
2829                 no_fragment_score += 2
2830                 if not should_continue:
2831                     continue
2832
2833             if known_idx > last_seq:
2834                 last_segment_url = None
2835                 continue
2836
2837             last_seq += 1
2838
2839             if begin_index < 0 and known_idx < 0:
2840                 # skip from the start when it's negative value
2841                 known_idx = last_seq + begin_index
2842             if lack_early_segments:
2843                 known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
2844             try:
2845                 for idx in range(known_idx, last_seq):
2846                     # do not update sequence here or you'll get skipped some part of it
2847                     should_continue, _ = _extract_sequence_from_mpd(False, False)
2848                     if not should_continue:
2849                         known_idx = idx - 1
2850                         raise ExtractorError('breaking out of outer loop')
2851                     last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
2852                     yield {
2853                         'url': last_segment_url,
2854                         'fragment_count': last_seq,
2855                     }
2856                 if known_idx == last_seq:
2857                     no_fragment_score += 5
2858                 else:
2859                     no_fragment_score = 0
2860                 known_idx = last_seq
2861             except ExtractorError:
2862                 continue
2863
2864             if manifestless_orig_fmt:
2865                 # Stop at the first iteration if running for post-live manifestless;
2866                 # fragment count no longer increase since it starts
2867                 break
2868
2869             time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
2870
2871     def _extract_player_url(self, *ytcfgs, webpage=None):
2872         player_url = traverse_obj(
2873             ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
2874             get_all=False, expected_type=str)
2875         if not player_url:
2876             return
2877         return urljoin('https://www.youtube.com', player_url)
2878
2879     def _download_player_url(self, video_id, fatal=False):
2880         res = self._download_webpage(
2881             'https://www.youtube.com/iframe_api',
2882             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
2883         if res:
2884             player_version = self._search_regex(
2885                 r'player\\?/([0-9a-fA-F]{8})\\?/', res, 'player version', fatal=fatal)
2886             if player_version:
2887                 return f'https://www.youtube.com/s/player/{player_version}/player_ias.vflset/en_US/base.js'
2888
2889     def _signature_cache_id(self, example_sig):
2890         """ Return a string representation of a signature """
2891         return '.'.join(str(len(part)) for part in example_sig.split('.'))
2892
2893     @classmethod
2894     def _extract_player_info(cls, player_url):
2895         for player_re in cls._PLAYER_INFO_RE:
2896             id_m = re.search(player_re, player_url)
2897             if id_m:
2898                 break
2899         else:
2900             raise ExtractorError('Cannot identify player %r' % player_url)
2901         return id_m.group('id')
2902
2903     def _load_player(self, video_id, player_url, fatal=True):
2904         player_id = self._extract_player_info(player_url)
2905         if player_id not in self._code_cache:
2906             code = self._download_webpage(
2907                 player_url, video_id, fatal=fatal,
2908                 note='Downloading player ' + player_id,
2909                 errnote='Download of %s failed' % player_url)
2910             if code:
2911                 self._code_cache[player_id] = code
2912         return self._code_cache.get(player_id)
2913
2914     def _extract_signature_function(self, video_id, player_url, example_sig):
2915         player_id = self._extract_player_info(player_url)
2916
2917         # Read from filesystem cache
2918         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
2919         assert os.path.basename(func_id) == func_id
2920
2921         self.write_debug(f'Extracting signature function {func_id}')
2922         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
2923
2924         if not cache_spec:
2925             code = self._load_player(video_id, player_url)
2926         if code:
2927             res = self._parse_sig_js(code)
2928             test_string = ''.join(map(chr, range(len(example_sig))))
2929             cache_spec = [ord(c) for c in res(test_string)]
2930             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
2931
2932         return lambda s: ''.join(s[i] for i in cache_spec)
2933
2934     def _print_sig_code(self, func, example_sig):
2935         if not self.get_param('youtube_print_sig_code'):
2936             return
2937
2938         def gen_sig_code(idxs):
2939             def _genslice(start, end, step):
2940                 starts = '' if start == 0 else str(start)
2941                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
2942                 steps = '' if step == 1 else (':%d' % step)
2943                 return f's[{starts}{ends}{steps}]'
2944
2945             step = None
2946             # Quelch pyflakes warnings - start will be set when step is set
2947             start = '(Never used)'
2948             for i, prev in zip(idxs[1:], idxs[:-1]):
2949                 if step is not None:
2950                     if i - prev == step:
2951                         continue
2952                     yield _genslice(start, prev, step)
2953                     step = None
2954                     continue
2955                 if i - prev in [-1, 1]:
2956                     step = i - prev
2957                     start = prev
2958                     continue
2959                 else:
2960                     yield 's[%d]' % prev
2961             if step is None:
2962                 yield 's[%d]' % i
2963             else:
2964                 yield _genslice(start, i, step)
2965
2966         test_string = ''.join(map(chr, range(len(example_sig))))
2967         cache_res = func(test_string)
2968         cache_spec = [ord(c) for c in cache_res]
2969         expr_code = ' + '.join(gen_sig_code(cache_spec))
2970         signature_id_tuple = '(%s)' % (
2971             ', '.join(str(len(p)) for p in example_sig.split('.')))
2972         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
2973                 '    return %s\n') % (signature_id_tuple, expr_code)
2974         self.to_screen('Extracted signature function:\n' + code)
2975
2976     def _parse_sig_js(self, jscode):
2977         funcname = self._search_regex(
2978             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2979              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2980              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
2981              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
2982              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
2983              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2984              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
2985              # Obsolete patterns
2986              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2987              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
2988              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2989              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2990              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2991              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2992              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
2993              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
2994             jscode, 'Initial JS player signature function name', group='sig')
2995
2996         jsi = JSInterpreter(jscode)
2997         initial_function = jsi.extract_function(funcname)
2998         return lambda s: initial_function([s])
2999
3000     def _cached(self, func, *cache_id):
3001         def inner(*args, **kwargs):
3002             if cache_id not in self._player_cache:
3003                 try:
3004                     self._player_cache[cache_id] = func(*args, **kwargs)
3005                 except ExtractorError as e:
3006                     self._player_cache[cache_id] = e
3007                 except Exception as e:
3008                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
3009
3010             ret = self._player_cache[cache_id]
3011             if isinstance(ret, Exception):
3012                 raise ret
3013             return ret
3014         return inner
3015
3016     def _decrypt_signature(self, s, video_id, player_url):
3017         """Turn the encrypted s field into a working signature"""
3018         extract_sig = self._cached(
3019             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
3020         func = extract_sig(video_id, player_url, s)
3021         self._print_sig_code(func, s)
3022         return func(s)
3023
3024     def _decrypt_nsig(self, s, video_id, player_url):
3025         """Turn the encrypted n field into a working signature"""
3026         if player_url is None:
3027             raise ExtractorError('Cannot decrypt nsig without player_url')
3028         player_url = urljoin('https://www.youtube.com', player_url)
3029
3030         try:
3031             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
3032         except ExtractorError as e:
3033             raise ExtractorError('Unable to extract nsig function code', cause=e)
3034         if self.get_param('youtube_print_sig_code'):
3035             self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
3036
3037         try:
3038             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
3039             ret = extract_nsig(jsi, func_code)(s)
3040         except JSInterpreter.Exception as e:
3041             try:
3042                 jsi = PhantomJSwrapper(self, timeout=5000)
3043             except ExtractorError:
3044                 raise e
3045             self.report_warning(
3046                 f'Native nsig extraction failed: Trying with PhantomJS\n'
3047                 f'         n = {s} ; player = {player_url}', video_id)
3048             self.write_debug(e, only_once=True)
3049
3050             args, func_body = func_code
3051             ret = jsi.execute(
3052                 f'console.log(function({", ".join(args)}) {{ {func_body} }}({s!r}));',
3053                 video_id=video_id, note='Executing signature code').strip()
3054
3055         self.write_debug(f'Decrypted nsig {s} => {ret}')
3056         return ret
3057
3058     def _extract_n_function_name(self, jscode):
3059         funcname, idx = self._search_regex(
3060             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]+)(?:\[(?P<idx>\d+)\])?\([a-zA-Z0-9]\)',
3061             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
3062         if not idx:
3063             return funcname
3064
3065         return json.loads(js_to_json(self._search_regex(
3066             rf'var {re.escape(funcname)}\s*=\s*(\[.+?\]);', jscode,
3067             f'Initial JS player n function list ({funcname}.{idx})')))[int(idx)]
3068
3069     def _extract_n_function_code(self, video_id, player_url):
3070         player_id = self._extract_player_info(player_url)
3071         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2022.09.1')
3072         jscode = func_code or self._load_player(video_id, player_url)
3073         jsi = JSInterpreter(jscode)
3074
3075         if func_code:
3076             return jsi, player_id, func_code
3077
3078         func_name = self._extract_n_function_name(jscode)
3079
3080         # For redundancy
3081         func_code = self._search_regex(
3082             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
3083                      # NB: The end of the regex is intentionally kept strict
3084                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
3085             jscode, 'nsig function', group=('var', 'code'), default=None)
3086         if func_code:
3087             func_code = ([func_code[0]], func_code[1])
3088         else:
3089             self.write_debug('Extracting nsig function with jsinterp')
3090             func_code = jsi.extract_function_code(func_name)
3091
3092         self.cache.store('youtube-nsig', player_id, func_code)
3093         return jsi, player_id, func_code
3094
3095     def _extract_n_function_from_code(self, jsi, func_code):
3096         func = jsi.extract_function_from_code(*func_code)
3097
3098         def extract_nsig(s):
3099             try:
3100                 ret = func([s])
3101             except JSInterpreter.Exception:
3102                 raise
3103             except Exception as e:
3104                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
3105
3106             if ret.startswith('enhanced_except_'):
3107                 raise JSInterpreter.Exception('Signature function returned an exception')
3108             return ret
3109
3110         return extract_nsig
3111
3112     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
3113         """
3114         Extract signatureTimestamp (sts)
3115         Required to tell API what sig/player version is in use.
3116         """
3117         sts = None
3118         if isinstance(ytcfg, dict):
3119             sts = int_or_none(ytcfg.get('STS'))
3120
3121         if not sts:
3122             # Attempt to extract from player
3123             if player_url is None:
3124                 error_msg = 'Cannot extract signature timestamp without player_url.'
3125                 if fatal:
3126                     raise ExtractorError(error_msg)
3127                 self.report_warning(error_msg)
3128                 return
3129             code = self._load_player(video_id, player_url, fatal=fatal)
3130             if code:
3131                 sts = int_or_none(self._search_regex(
3132                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
3133                     'JS player signature timestamp', group='sts', fatal=fatal))
3134         return sts
3135
3136     def _mark_watched(self, video_id, player_responses):
3137         for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')):
3138             label = 'fully ' if is_full else ''
3139             url = get_first(player_responses, ('playbackTracking', key, 'baseUrl'),
3140                             expected_type=url_or_none)
3141             if not url:
3142                 self.report_warning(f'Unable to mark {label}watched')
3143                 return
3144             parsed_url = urllib.parse.urlparse(url)
3145             qs = urllib.parse.parse_qs(parsed_url.query)
3146
3147             # cpn generation algorithm is reverse engineered from base.js.
3148             # In fact it works even with dummy cpn.
3149             CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
3150             cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
3151
3152             # # more consistent results setting it to right before the end
3153             video_length = [str(float((qs.get('len') or ['1.5'])[0]) - 1)]
3154
3155             qs.update({
3156                 'ver': ['2'],
3157                 'cpn': [cpn],
3158                 'cmt': video_length,
3159                 'el': 'detailpage',  # otherwise defaults to "shorts"
3160             })
3161
3162             if is_full:
3163                 # these seem to mark watchtime "history" in the real world
3164                 # they're required, so send in a single value
3165                 qs.update({
3166                     'st': 0,
3167                     'et': video_length,
3168                 })
3169
3170             url = urllib.parse.urlunparse(
3171                 parsed_url._replace(query=urllib.parse.urlencode(qs, True)))
3172
3173             self._download_webpage(
3174                 url, video_id, f'Marking {label}watched',
3175                 'Unable to mark watched', fatal=False)
3176
3177     @classmethod
3178     def _extract_from_webpage(cls, url, webpage):
3179         # Invidious Instances
3180         # https://github.com/yt-dlp/yt-dlp/issues/195
3181         # https://github.com/iv-org/invidious/pull/1730
3182         mobj = re.search(
3183             r'<link rel="alternate" href="(?P<url>https://www\.youtube\.com/watch\?v=[0-9A-Za-z_-]{11})"',
3184             webpage)
3185         if mobj:
3186             yield cls.url_result(mobj.group('url'), cls)
3187             raise cls.StopExtraction()
3188
3189         yield from super()._extract_from_webpage(url, webpage)
3190
3191         # lazyYT YouTube embed
3192         for id_ in re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage):
3193             yield cls.url_result(unescapeHTML(id_), cls, id_)
3194
3195         # Wordpress "YouTube Video Importer" plugin
3196         for m in re.findall(r'''(?x)<div[^>]+
3197                 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
3198                 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage):
3199             yield cls.url_result(m[-1], cls, m[-1])
3200
3201     @classmethod
3202     def extract_id(cls, url):
3203         video_id = cls.get_temp_id(url)
3204         if not video_id:
3205             raise ExtractorError(f'Invalid URL: {url}')
3206         return video_id
3207
3208     def _extract_chapters_from_json(self, data, duration):
3209         chapter_list = traverse_obj(
3210             data, (
3211                 'playerOverlays', 'playerOverlayRenderer', 'decoratedPlayerBarRenderer',
3212                 'decoratedPlayerBarRenderer', 'playerBar', 'chapteredPlayerBarRenderer', 'chapters'
3213             ), expected_type=list)
3214
3215         return self._extract_chapters_helper(
3216             chapter_list,
3217             start_function=lambda chapter: float_or_none(
3218                 traverse_obj(chapter, ('chapterRenderer', 'timeRangeStartMillis')), scale=1000),
3219             title_function=lambda chapter: traverse_obj(
3220                 chapter, ('chapterRenderer', 'title', 'simpleText'), expected_type=str),
3221             duration=duration)
3222
3223     def _extract_chapters_from_engagement_panel(self, data, duration):
3224         content_list = traverse_obj(
3225             data,
3226             ('engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'macroMarkersListRenderer', 'contents'),
3227             expected_type=list)
3228         chapter_time = lambda chapter: parse_duration(self._get_text(chapter, 'timeDescription'))
3229         chapter_title = lambda chapter: self._get_text(chapter, 'title')
3230
3231         return next(filter(None, (
3232             self._extract_chapters_helper(traverse_obj(contents, (..., 'macroMarkersListItemRenderer')),
3233                                           chapter_time, chapter_title, duration)
3234             for contents in content_list)), [])
3235
3236     def _extract_comment(self, comment_renderer, parent=None):
3237         comment_id = comment_renderer.get('commentId')
3238         if not comment_id:
3239             return
3240
3241         text = self._get_text(comment_renderer, 'contentText')
3242
3243         # Timestamp is an estimate calculated from the current time and time_text
3244         time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
3245         timestamp = self._parse_time_text(time_text)
3246
3247         author = self._get_text(comment_renderer, 'authorText')
3248         author_id = try_get(comment_renderer,
3249                             lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], str)
3250
3251         votes = parse_count(try_get(comment_renderer, (lambda x: x['voteCount']['simpleText'],
3252                                                        lambda x: x['likeCount']), str)) or 0
3253         author_thumbnail = try_get(comment_renderer,
3254                                    lambda x: x['authorThumbnail']['thumbnails'][-1]['url'], str)
3255
3256         author_is_uploader = try_get(comment_renderer, lambda x: x['authorIsChannelOwner'], bool)
3257         is_favorited = 'creatorHeart' in (try_get(
3258             comment_renderer, lambda x: x['actionButtons']['commentActionButtonsRenderer'], dict) or {})
3259         return {
3260             'id': comment_id,
3261             'text': text,
3262             'timestamp': timestamp,
3263             'time_text': time_text,
3264             'like_count': votes,
3265             'is_favorited': is_favorited,
3266             'author': author,
3267             'author_id': author_id,
3268             'author_thumbnail': author_thumbnail,
3269             'author_is_uploader': author_is_uploader,
3270             'parent': parent or 'root'
3271         }
3272
3273     def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
3274
3275         get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
3276
3277         def extract_header(contents):
3278             _continuation = None
3279             for content in contents:
3280                 comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
3281                 expected_comment_count = self._get_count(
3282                     comments_header_renderer, 'countText', 'commentsCount')
3283
3284                 if expected_comment_count:
3285                     tracker['est_total'] = expected_comment_count
3286                     self.to_screen(f'Downloading ~{expected_comment_count} comments')
3287                 comment_sort_index = int(get_single_config_arg('comment_sort') != 'top')  # 1 = new, 0 = top
3288
3289                 sort_menu_item = try_get(
3290                     comments_header_renderer,
3291                     lambda x: x['sortMenu']['sortFilterSubMenuRenderer']['subMenuItems'][comment_sort_index], dict) or {}
3292                 sort_continuation_ep = sort_menu_item.get('serviceEndpoint') or {}
3293
3294                 _continuation = self._extract_continuation_ep_data(sort_continuation_ep) or self._extract_continuation(sort_menu_item)
3295                 if not _continuation:
3296                     continue
3297
3298                 sort_text = str_or_none(sort_menu_item.get('title'))
3299                 if not sort_text:
3300                     sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
3301                 self.to_screen('Sorting comments by %s' % sort_text.lower())
3302                 break
3303             return _continuation
3304
3305         def extract_thread(contents):
3306             if not parent:
3307                 tracker['current_page_thread'] = 0
3308             for content in contents:
3309                 if not parent and tracker['total_parent_comments'] >= max_parents:
3310                     yield
3311                 comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
3312                 comment_renderer = get_first(
3313                     (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
3314                     expected_type=dict, default={})
3315
3316                 comment = self._extract_comment(comment_renderer, parent)
3317                 if not comment:
3318                     continue
3319                 is_pinned = bool(traverse_obj(comment_renderer, 'pinnedCommentBadge'))
3320                 comment_id = comment['id']
3321                 if is_pinned:
3322                     tracker['pinned_comment_ids'].add(comment_id)
3323                 # Sometimes YouTube may break and give us infinite looping comments.
3324                 # See: https://github.com/yt-dlp/yt-dlp/issues/6290
3325                 if comment_id in tracker['seen_comment_ids']:
3326                     if comment_id in tracker['pinned_comment_ids'] and not is_pinned:
3327                         # Pinned comments may appear a second time in newest first sort
3328                         # See: https://github.com/yt-dlp/yt-dlp/issues/6712
3329                         continue
3330                     self.report_warning('Detected YouTube comments looping. Stopping comment extraction as we probably cannot get any more.')
3331                     yield
3332                 else:
3333                     tracker['seen_comment_ids'].add(comment['id'])
3334
3335                 tracker['running_total'] += 1
3336                 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
3337                 yield comment
3338
3339                 # Attempt to get the replies
3340                 comment_replies_renderer = try_get(
3341                     comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
3342
3343                 if comment_replies_renderer:
3344                     tracker['current_page_thread'] += 1
3345                     comment_entries_iter = self._comment_entries(
3346                         comment_replies_renderer, ytcfg, video_id,
3347                         parent=comment.get('id'), tracker=tracker)
3348                     yield from itertools.islice(comment_entries_iter, min(
3349                         max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments'])))
3350
3351         # Keeps track of counts across recursive calls
3352         if not tracker:
3353             tracker = dict(
3354                 running_total=0,
3355                 est_total=0,
3356                 current_page_thread=0,
3357                 total_parent_comments=0,
3358                 total_reply_comments=0,
3359                 seen_comment_ids=set(),
3360                 pinned_comment_ids=set()
3361             )
3362
3363         # TODO: Deprecated
3364         # YouTube comments have a max depth of 2
3365         max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
3366         if max_depth:
3367             self._downloader.deprecated_feature('[youtube] max_comment_depth extractor argument is deprecated. '
3368                                                 'Set max replies in the max-comments extractor argument instead')
3369         if max_depth == 1 and parent:
3370             return
3371
3372         max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
3373             lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
3374
3375         continuation = self._extract_continuation(root_continuation_data)
3376
3377         response = None
3378         is_forced_continuation = False
3379         is_first_continuation = parent is None
3380         if is_first_continuation and not continuation:
3381             # Sometimes you can get comments by generating the continuation yourself,
3382             # even if YouTube initially reports them being disabled - e.g. stories comments.
3383             # Note: if the comment section is actually disabled, YouTube may return a response with
3384             # required check_get_keys missing. So we will disable that check initially in this case.
3385             continuation = self._build_api_continuation_query(self._generate_comment_continuation(video_id))
3386             is_forced_continuation = True
3387
3388         for page_num in itertools.count(0):
3389             if not continuation:
3390                 break
3391             headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
3392             comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
3393             if page_num == 0:
3394                 if is_first_continuation:
3395                     note_prefix = 'Downloading comment section API JSON'
3396                 else:
3397                     note_prefix = '    Downloading comment API JSON reply thread %d %s' % (
3398                         tracker['current_page_thread'], comment_prog_str)
3399             else:
3400                 note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
3401                     '       ' if parent else '', ' replies' if parent else '',
3402                     page_num, comment_prog_str)
3403             try:
3404                 response = self._extract_response(
3405                     item_id=None, query=continuation,
3406                     ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
3407                     check_get_keys='onResponseReceivedEndpoints' if not is_forced_continuation else None)
3408             except ExtractorError as e:
3409                 # Ignore incomplete data error for replies if retries didn't work.
3410                 # This is to allow any other parent comments and comment threads to be downloaded.
3411                 # See: https://github.com/yt-dlp/yt-dlp/issues/4669
3412                 if 'incomplete data' in str(e).lower() and parent and self.get_param('ignoreerrors') is True:
3413                     self.report_warning(
3414                         'Received incomplete data for a comment reply thread and retrying did not help. '
3415                         'Ignoring to let other comments be downloaded.')
3416                 else:
3417                     raise
3418             is_forced_continuation = False
3419             continuation_contents = traverse_obj(
3420                 response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
3421
3422             continuation = None
3423             for continuation_section in continuation_contents:
3424                 continuation_items = traverse_obj(
3425                     continuation_section,
3426                     (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
3427                     get_all=False, expected_type=list) or []
3428                 if is_first_continuation:
3429                     continuation = extract_header(continuation_items)
3430                     is_first_continuation = False
3431                     if continuation:
3432                         break
3433                     continue
3434
3435                 for entry in extract_thread(continuation_items):
3436                     if not entry:
3437                         return
3438                     yield entry
3439                 continuation = self._extract_continuation({'contents': continuation_items})
3440                 if continuation:
3441                     break
3442
3443         message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
3444         if message and not parent and tracker['running_total'] == 0:
3445             self.report_warning(f'Youtube said: {message}', video_id=video_id, only_once=True)
3446             raise self.CommentsDisabled
3447
3448     @staticmethod
3449     def _generate_comment_continuation(video_id):
3450         """
3451         Generates initial comment section continuation token from given video id
3452         """
3453         token = f'\x12\r\x12\x0b{video_id}\x18\x062\'"\x11"\x0b{video_id}0\x00x\x020\x00B\x10comments-section'
3454         return base64.b64encode(token.encode()).decode()
3455
3456     def _get_comments(self, ytcfg, video_id, contents, webpage):
3457         """Entry for comment extraction"""
3458         def _real_comment_extract(contents):
3459             renderer = next((
3460                 item for item in traverse_obj(contents, (..., 'itemSectionRenderer'), default={})
3461                 if item.get('sectionIdentifier') == 'comment-item-section'), None)
3462             yield from self._comment_entries(renderer, ytcfg, video_id)
3463
3464         max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
3465         return itertools.islice(_real_comment_extract(contents), 0, max_comments)
3466
3467     @staticmethod
3468     def _get_checkok_params():
3469         return {'contentCheckOk': True, 'racyCheckOk': True}
3470
3471     @classmethod
3472     def _generate_player_context(cls, sts=None):
3473         context = {
3474             'html5Preference': 'HTML5_PREF_WANTS',
3475         }
3476         if sts is not None:
3477             context['signatureTimestamp'] = sts
3478         return {
3479             'playbackContext': {
3480                 'contentPlaybackContext': context
3481             },
3482             **cls._get_checkok_params()
3483         }
3484
3485     @staticmethod
3486     def _is_agegated(player_response):
3487         if traverse_obj(player_response, ('playabilityStatus', 'desktopLegacyAgeGateReason')):
3488             return True
3489
3490         reasons = traverse_obj(player_response, ('playabilityStatus', ('status', 'reason')))
3491         AGE_GATE_REASONS = (
3492             'confirm your age', 'age-restricted', 'inappropriate',  # reason
3493             'age_verification_required', 'age_check_required',  # status
3494         )
3495         return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
3496
3497     @staticmethod
3498     def _is_unplayable(player_response):
3499         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
3500
3501     _STORY_PLAYER_PARAMS = '8AEB'
3502
3503     def _extract_player_response(self, client, video_id, master_ytcfg, player_ytcfg, player_url, initial_pr, smuggled_data):
3504
3505         session_index = self._extract_session_index(player_ytcfg, master_ytcfg)
3506         syncid = self._extract_account_syncid(player_ytcfg, master_ytcfg, initial_pr)
3507         sts = self._extract_signature_timestamp(video_id, player_url, master_ytcfg, fatal=False) if player_url else None
3508         headers = self.generate_api_headers(
3509             ytcfg=player_ytcfg, account_syncid=syncid, session_index=session_index, default_client=client)
3510
3511         yt_query = {
3512             'videoId': video_id,
3513         }
3514         if smuggled_data.get('is_story') or _split_innertube_client(client)[0] == 'android':
3515             yt_query['params'] = self._STORY_PLAYER_PARAMS
3516
3517         yt_query.update(self._generate_player_context(sts))
3518         return self._extract_response(
3519             item_id=video_id, ep='player', query=yt_query,
3520             ytcfg=player_ytcfg, headers=headers, fatal=True,
3521             default_client=client,
3522             note='Downloading %s player API JSON' % client.replace('_', ' ').strip()
3523         ) or None
3524
3525     def _get_requested_clients(self, url, smuggled_data):
3526         requested_clients = []
3527         default = ['android', 'web']
3528         allowed_clients = sorted(
3529             (client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'),
3530             key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
3531         for client in self._configuration_arg('player_client'):
3532             if client in allowed_clients:
3533                 requested_clients.append(client)
3534             elif client == 'default':
3535                 requested_clients.extend(default)
3536             elif client == 'all':
3537                 requested_clients.extend(allowed_clients)
3538             else:
3539                 self.report_warning(f'Skipping unsupported client {client}')
3540         if not requested_clients:
3541             requested_clients = default
3542
3543         if smuggled_data.get('is_music_url') or self.is_music_url(url):
3544             requested_clients.extend(
3545                 f'{client}_music' for client in requested_clients if f'{client}_music' in INNERTUBE_CLIENTS)
3546
3547         return orderedSet(requested_clients)
3548
3549     def _extract_player_responses(self, clients, video_id, webpage, master_ytcfg, smuggled_data):
3550         initial_pr = None
3551         if webpage:
3552             initial_pr = self._search_json(
3553                 self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response', video_id, fatal=False)
3554
3555         all_clients = set(clients)
3556         clients = clients[::-1]
3557         prs = []
3558
3559         def append_client(*client_names):
3560             """ Append the first client name that exists but not already used """
3561             for client_name in client_names:
3562                 actual_client = _split_innertube_client(client_name)[0]
3563                 if actual_client in INNERTUBE_CLIENTS:
3564                     if actual_client not in all_clients:
3565                         clients.append(client_name)
3566                         all_clients.add(actual_client)
3567                         return
3568
3569         # Android player_response does not have microFormats which are needed for
3570         # extraction of some data. So we return the initial_pr with formats
3571         # stripped out even if not requested by the user
3572         # See: https://github.com/yt-dlp/yt-dlp/issues/501
3573         if initial_pr:
3574             pr = dict(initial_pr)
3575             pr['streamingData'] = None
3576             prs.append(pr)
3577
3578         last_error = None
3579         tried_iframe_fallback = False
3580         player_url = None
3581         while clients:
3582             client, base_client, variant = _split_innertube_client(clients.pop())
3583             player_ytcfg = master_ytcfg if client == 'web' else {}
3584             if 'configs' not in self._configuration_arg('player_skip') and client != 'web':
3585                 player_ytcfg = self._download_ytcfg(client, video_id) or player_ytcfg
3586
3587             player_url = player_url or self._extract_player_url(master_ytcfg, player_ytcfg, webpage=webpage)
3588             require_js_player = self._get_default_ytcfg(client).get('REQUIRE_JS_PLAYER')
3589             if 'js' in self._configuration_arg('player_skip'):
3590                 require_js_player = False
3591                 player_url = None
3592
3593             if not player_url and not tried_iframe_fallback and require_js_player:
3594                 player_url = self._download_player_url(video_id)
3595                 tried_iframe_fallback = True
3596
3597             try:
3598                 pr = initial_pr if client == 'web' and initial_pr else self._extract_player_response(
3599                     client, video_id, player_ytcfg or master_ytcfg, player_ytcfg, player_url if require_js_player else None, initial_pr, smuggled_data)
3600             except ExtractorError as e:
3601                 if last_error:
3602                     self.report_warning(last_error)
3603                 last_error = e
3604                 continue
3605
3606             if pr:
3607                 # YouTube may return a different video player response than expected.
3608                 # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
3609                 pr_video_id = traverse_obj(pr, ('videoDetails', 'videoId'))
3610                 if pr_video_id and pr_video_id != video_id:
3611                     self.report_warning(
3612                         f'Skipping player response from {client} client (got player response for video "{pr_video_id}" instead of "{video_id}")' + bug_reports_message())
3613                 else:
3614                     # Save client name for introspection later
3615                     name = short_client_name(client)
3616                     sd = traverse_obj(pr, ('streamingData', {dict})) or {}
3617                     sd[STREAMING_DATA_CLIENT_NAME] = name
3618                     for f in traverse_obj(sd, (('formats', 'adaptiveFormats'), ..., {dict})):
3619                         f[STREAMING_DATA_CLIENT_NAME] = name
3620                     prs.append(pr)
3621
3622             # creator clients can bypass AGE_VERIFICATION_REQUIRED if logged in
3623             if variant == 'embedded' and self._is_unplayable(pr) and self.is_authenticated:
3624                 append_client(f'{base_client}_creator')
3625             elif self._is_agegated(pr):
3626                 if variant == 'tv_embedded':
3627                     append_client(f'{base_client}_embedded')
3628                 elif not variant:
3629                     append_client(f'tv_embedded.{base_client}', f'{base_client}_embedded')
3630
3631         if last_error:
3632             if not len(prs):
3633                 raise last_error
3634             self.report_warning(last_error)
3635         return prs, player_url
3636
3637     def _needs_live_processing(self, live_status, duration):
3638         if (live_status == 'is_live' and self.get_param('live_from_start')
3639                 or live_status == 'post_live' and (duration or 0) > 4 * 3600):
3640             return live_status
3641
3642     def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, live_status, duration):
3643         CHUNK_SIZE = 10 << 20
3644         itags, stream_ids = collections.defaultdict(set), []
3645         itag_qualities, res_qualities = {}, {0: None}
3646         q = qualities([
3647             # Normally tiny is the smallest video-only formats. But
3648             # audio-only formats with unknown quality may get tagged as tiny
3649             'tiny',
3650             'audio_quality_ultralow', 'audio_quality_low', 'audio_quality_medium', 'audio_quality_high',  # Audio only formats
3651             'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'
3652         ])
3653         streaming_formats = traverse_obj(streaming_data, (..., ('formats', 'adaptiveFormats'), ...))
3654         all_formats = self._configuration_arg('include_duplicate_formats')
3655
3656         def build_fragments(f):
3657             return LazyList({
3658                 'url': update_url_query(f['url'], {
3659                     'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, f["filesize"])}'
3660                 })
3661             } for range_start in range(0, f['filesize'], CHUNK_SIZE))
3662
3663         for fmt in streaming_formats:
3664             if fmt.get('targetDurationSec'):
3665                 continue
3666
3667             itag = str_or_none(fmt.get('itag'))
3668             audio_track = fmt.get('audioTrack') or {}
3669             stream_id = (itag, audio_track.get('id'), fmt.get('isDrc'))
3670             if not all_formats:
3671                 if stream_id in stream_ids:
3672                     continue
3673
3674             quality = fmt.get('quality')
3675             height = int_or_none(fmt.get('height'))
3676             if quality == 'tiny' or not quality:
3677                 quality = fmt.get('audioQuality', '').lower() or quality
3678             # The 3gp format (17) in android client has a quality of "small",
3679             # but is actually worse than other formats
3680             if itag == '17':
3681                 quality = 'tiny'
3682             if quality:
3683                 if itag:
3684                     itag_qualities[itag] = quality
3685                 if height:
3686                     res_qualities[height] = quality
3687             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
3688             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
3689             # number of fragment that would subsequently requested with (`&sq=N`)
3690             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
3691                 continue
3692
3693             fmt_url = fmt.get('url')
3694             if not fmt_url:
3695                 sc = urllib.parse.parse_qs(fmt.get('signatureCipher'))
3696                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
3697                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
3698                 if not all((sc, fmt_url, player_url, encrypted_sig)):
3699                     continue
3700                 try:
3701                     fmt_url += '&%s=%s' % (
3702                         traverse_obj(sc, ('sp', -1)) or 'signature',
3703                         self._decrypt_signature(encrypted_sig, video_id, player_url)
3704                     )
3705                 except ExtractorError as e:
3706                     self.report_warning('Signature extraction failed: Some formats may be missing',
3707                                         video_id=video_id, only_once=True)
3708                     self.write_debug(e, only_once=True)
3709                     continue
3710
3711             query = parse_qs(fmt_url)
3712             throttled = False
3713             if query.get('n'):
3714                 try:
3715                     decrypt_nsig = self._cached(self._decrypt_nsig, 'nsig', query['n'][0])
3716                     fmt_url = update_url_query(fmt_url, {
3717                         'n': decrypt_nsig(query['n'][0], video_id, player_url)
3718                     })
3719                 except ExtractorError as e:
3720                     phantomjs_hint = ''
3721                     if isinstance(e, JSInterpreter.Exception):
3722                         phantomjs_hint = (f'         Install {self._downloader._format_err("PhantomJS", self._downloader.Styles.EMPHASIS)} '
3723                                           f'to workaround the issue. {PhantomJSwrapper.INSTALL_HINT}\n')
3724                     if player_url:
3725                         self.report_warning(
3726                             f'nsig extraction failed: You may experience throttling for some formats\n{phantomjs_hint}'
3727                             f'         n = {query["n"][0]} ; player = {player_url}', video_id=video_id, only_once=True)
3728                         self.write_debug(e, only_once=True)
3729                     else:
3730                         self.report_warning(
3731                             'Cannot decrypt nsig without player_url: You may experience throttling for some formats',
3732                             video_id=video_id, only_once=True)
3733                     throttled = True
3734
3735             tbr = float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
3736             language_preference = (
3737                 10 if audio_track.get('audioIsDefault') and 10
3738                 else -10 if 'descriptive' in (audio_track.get('displayName') or '').lower() and -10
3739                 else -1)
3740             # Some formats may have much smaller duration than others (possibly damaged during encoding)
3741             # E.g. 2-nOtRESiUc Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
3742             # Make sure to avoid false positives with small duration differences.
3743             # E.g. __2ABJjxzNo, ySuUZEjARPY
3744             is_damaged = try_get(fmt, lambda x: float(x['approxDurationMs']) / duration < 500)
3745             if is_damaged:
3746                 self.report_warning(
3747                     f'{video_id}: Some formats are possibly damaged. They will be deprioritized', only_once=True)
3748
3749             client_name = fmt.get(STREAMING_DATA_CLIENT_NAME)
3750             dct = {
3751                 'asr': int_or_none(fmt.get('audioSampleRate')),
3752                 'filesize': int_or_none(fmt.get('contentLength')),
3753                 'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
3754                 'format_note': join_nonempty(
3755                     join_nonempty(audio_track.get('displayName'),
3756                                   language_preference > 0 and ' (default)', delim=''),
3757                     fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
3758                     fmt.get('isDrc') and 'DRC',
3759                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
3760                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
3761                     throttled and 'THROTTLED', is_damaged and 'DAMAGED',
3762                     (self.get_param('verbose') or all_formats) and client_name,
3763                     delim=', '),
3764                 # Format 22 is likely to be damaged. See https://github.com/yt-dlp/yt-dlp/issues/3372
3765                 'source_preference': -10 if throttled else -5 if itag == '22' else -1,
3766                 'fps': int_or_none(fmt.get('fps')) or None,
3767                 'audio_channels': fmt.get('audioChannels'),
3768                 'height': height,
3769                 'quality': q(quality) - bool(fmt.get('isDrc')) / 2,
3770                 'has_drm': bool(fmt.get('drmFamilies')),
3771                 'tbr': tbr,
3772                 'url': fmt_url,
3773                 'width': int_or_none(fmt.get('width')),
3774                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
3775                                           'desc' if language_preference < -1 else '') or None,
3776                 'language_preference': language_preference,
3777                 # Strictly de-prioritize damaged and 3gp formats
3778                 'preference': -10 if is_damaged else -2 if itag == '17' else None,
3779             }
3780             mime_mobj = re.match(
3781                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
3782             if mime_mobj:
3783                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
3784                 dct.update(parse_codecs(mime_mobj.group(2)))
3785             if itag:
3786                 itags[itag].add(('https', dct.get('language')))
3787                 stream_ids.append(stream_id)
3788             single_stream = 'none' in (dct.get('acodec'), dct.get('vcodec'))
3789             if single_stream and dct.get('ext'):
3790                 dct['container'] = dct['ext'] + '_dash'
3791
3792             if dct['filesize']:
3793                 yield {
3794                     **dct,
3795                     'format_id': f'{dct["format_id"]}-dashy' if all_formats else dct['format_id'],
3796                     'protocol': 'http_dash_segments',
3797                     'fragments': build_fragments(dct),
3798                 }
3799                 if not all_formats:
3800                     continue
3801             dct['downloader_options'] = {'http_chunk_size': CHUNK_SIZE}
3802             yield dct
3803
3804         needs_live_processing = self._needs_live_processing(live_status, duration)
3805         skip_bad_formats = not self._configuration_arg('include_incomplete_formats')
3806
3807         skip_manifests = set(self._configuration_arg('skip'))
3808         if (not self.get_param('youtube_include_hls_manifest', True)
3809                 or needs_live_processing == 'is_live'  # These will be filtered out by YoutubeDL anyway
3810                 or needs_live_processing and skip_bad_formats):
3811             skip_manifests.add('hls')
3812
3813         if not self.get_param('youtube_include_dash_manifest', True):
3814             skip_manifests.add('dash')
3815         if self._configuration_arg('include_live_dash'):
3816             self._downloader.deprecated_feature('[youtube] include_live_dash extractor argument is deprecated. '
3817                                                 'Use include_incomplete_formats extractor argument instead')
3818         elif skip_bad_formats and live_status == 'is_live' and needs_live_processing != 'is_live':
3819             skip_manifests.add('dash')
3820
3821         def process_manifest_format(f, proto, client_name, itag):
3822             key = (proto, f.get('language'))
3823             if not all_formats and key in itags[itag]:
3824                 return False
3825             itags[itag].add(key)
3826
3827             if itag and all_formats:
3828                 f['format_id'] = f'{itag}-{proto}'
3829             elif any(p != proto for p, _ in itags[itag]):
3830                 f['format_id'] = f'{itag}-{proto}'
3831             elif itag:
3832                 f['format_id'] = itag
3833
3834             f['quality'] = q(itag_qualities.get(try_get(f, lambda f: f['format_id'].split('-')[0]), -1))
3835             if f['quality'] == -1 and f.get('height'):
3836                 f['quality'] = q(res_qualities[min(res_qualities, key=lambda x: abs(x - f['height']))])
3837             if self.get_param('verbose'):
3838                 f['format_note'] = join_nonempty(f.get('format_note'), client_name, delim=', ')
3839             return True
3840
3841         subtitles = {}
3842         for sd in streaming_data:
3843             client_name = sd.get(STREAMING_DATA_CLIENT_NAME)
3844
3845             hls_manifest_url = 'hls' not in skip_manifests and sd.get('hlsManifestUrl')
3846             if hls_manifest_url:
3847                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
3848                     hls_manifest_url, video_id, 'mp4', fatal=False, live=live_status == 'is_live')
3849                 subtitles = self._merge_subtitles(subs, subtitles)
3850                 for f in fmts:
3851                     if process_manifest_format(f, 'hls', client_name, self._search_regex(
3852                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
3853                         yield f
3854
3855             dash_manifest_url = 'dash' not in skip_manifests and sd.get('dashManifestUrl')
3856             if dash_manifest_url:
3857                 formats, subs = self._extract_mpd_formats_and_subtitles(dash_manifest_url, video_id, fatal=False)
3858                 subtitles = self._merge_subtitles(subs, subtitles)  # Prioritize HLS subs over DASH
3859                 for f in formats:
3860                     if process_manifest_format(f, 'dash', client_name, f['format_id']):
3861                         f['filesize'] = int_or_none(self._search_regex(
3862                             r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
3863                         if needs_live_processing:
3864                             f['is_from_start'] = True
3865
3866                         yield f
3867         yield subtitles
3868
3869     def _extract_storyboard(self, player_responses, duration):
3870         spec = get_first(
3871             player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
3872         base_url = url_or_none(urljoin('https://i.ytimg.com/', spec.pop() or None))
3873         if not base_url:
3874             return
3875         L = len(spec) - 1
3876         for i, args in enumerate(spec):
3877             args = args.split('#')
3878             counts = list(map(int_or_none, args[:5]))
3879             if len(args) != 8 or not all(counts):
3880                 self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
3881                 continue
3882             width, height, frame_count, cols, rows = counts
3883             N, sigh = args[6:]
3884
3885             url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
3886             fragment_count = frame_count / (cols * rows)
3887             fragment_duration = duration / fragment_count
3888             yield {
3889                 'format_id': f'sb{i}',
3890                 'format_note': 'storyboard',
3891                 'ext': 'mhtml',
3892                 'protocol': 'mhtml',
3893                 'acodec': 'none',
3894                 'vcodec': 'none',
3895                 'url': url,
3896                 'width': width,
3897                 'height': height,
3898                 'fps': frame_count / duration,
3899                 'rows': rows,
3900                 'columns': cols,
3901                 'fragments': [{
3902                     'url': url.replace('$M', str(j)),
3903                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
3904                 } for j in range(math.ceil(fragment_count))],
3905             }
3906
3907     def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
3908         webpage = None
3909         if 'webpage' not in self._configuration_arg('player_skip'):
3910             query = {'bpctr': '9999999999', 'has_verified': '1'}
3911             if smuggled_data.get('is_story'):
3912                 query['pp'] = self._STORY_PLAYER_PARAMS
3913             webpage = self._download_webpage(
3914                 webpage_url, video_id, fatal=False, query=query)
3915
3916         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
3917
3918         player_responses, player_url = self._extract_player_responses(
3919             self._get_requested_clients(url, smuggled_data),
3920             video_id, webpage, master_ytcfg, smuggled_data)
3921
3922         return webpage, master_ytcfg, player_responses, player_url
3923
3924     def _list_formats(self, video_id, microformats, video_details, player_responses, player_url, duration=None):
3925         live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
3926         is_live = get_first(video_details, 'isLive')
3927         if is_live is None:
3928             is_live = get_first(live_broadcast_details, 'isLiveNow')
3929         live_content = get_first(video_details, 'isLiveContent')
3930         is_upcoming = get_first(video_details, 'isUpcoming')
3931         post_live = get_first(video_details, 'isPostLiveDvr')
3932         live_status = ('post_live' if post_live
3933                        else 'is_live' if is_live
3934                        else 'is_upcoming' if is_upcoming
3935                        else 'was_live' if live_content
3936                        else 'not_live' if False in (is_live, live_content)
3937                        else None)
3938         streaming_data = traverse_obj(player_responses, (..., 'streamingData'))
3939         *formats, subtitles = self._extract_formats_and_subtitles(streaming_data, video_id, player_url, live_status, duration)
3940
3941         return live_broadcast_details, live_status, streaming_data, formats, subtitles
3942
3943     def _real_extract(self, url):
3944         url, smuggled_data = unsmuggle_url(url, {})
3945         video_id = self._match_id(url)
3946
3947         base_url = self.http_scheme() + '//www.youtube.com/'
3948         webpage_url = base_url + 'watch?v=' + video_id
3949
3950         webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
3951
3952         playability_statuses = traverse_obj(
3953             player_responses, (..., 'playabilityStatus'), expected_type=dict)
3954
3955         trailer_video_id = get_first(
3956             playability_statuses,
3957             ('errorScreen', 'playerLegacyDesktopYpcTrailerRenderer', 'trailerVideoId'),
3958             expected_type=str)
3959         if trailer_video_id:
3960             return self.url_result(
3961                 trailer_video_id, self.ie_key(), trailer_video_id)
3962
3963         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None))
3964                        if webpage else (lambda x: None))
3965
3966         video_details = traverse_obj(player_responses, (..., 'videoDetails'), expected_type=dict)
3967         microformats = traverse_obj(
3968             player_responses, (..., 'microformat', 'playerMicroformatRenderer'),
3969             expected_type=dict)
3970
3971         translated_title = self._get_text(microformats, (..., 'title'))
3972         video_title = (self._preferred_lang and translated_title
3973                        or get_first(video_details, 'title')  # primary
3974                        or translated_title
3975                        or search_meta(['og:title', 'twitter:title', 'title']))
3976         translated_description = self._get_text(microformats, (..., 'description'))
3977         original_description = get_first(video_details, 'shortDescription')
3978         video_description = (
3979             self._preferred_lang and translated_description
3980             # If original description is blank, it will be an empty string.
3981             # Do not prefer translated description in this case.
3982             or original_description if original_description is not None else translated_description)
3983
3984         multifeed_metadata_list = get_first(
3985             player_responses,
3986             ('multicamera', 'playerLegacyMulticameraRenderer', 'metadataList'),
3987             expected_type=str)
3988         if multifeed_metadata_list and not smuggled_data.get('force_singlefeed'):
3989             if self.get_param('noplaylist'):
3990                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
3991             else:
3992                 entries = []
3993                 feed_ids = []
3994                 for feed in multifeed_metadata_list.split(','):
3995                     # Unquote should take place before split on comma (,) since textual
3996                     # fields may contain comma as well (see
3997                     # https://github.com/ytdl-org/youtube-dl/issues/8536)
3998                     feed_data = urllib.parse.parse_qs(
3999                         urllib.parse.unquote_plus(feed))
4000
4001                     def feed_entry(name):
4002                         return try_get(
4003                             feed_data, lambda x: x[name][0], str)
4004
4005                     feed_id = feed_entry('id')
4006                     if not feed_id:
4007                         continue
4008                     feed_title = feed_entry('title')
4009                     title = video_title
4010                     if feed_title:
4011                         title += ' (%s)' % feed_title
4012                     entries.append({
4013                         '_type': 'url_transparent',
4014                         'ie_key': 'Youtube',
4015                         'url': smuggle_url(
4016                             '%swatch?v=%s' % (base_url, feed_data['id'][0]),
4017                             {'force_singlefeed': True}),
4018                         'title': title,
4019                     })
4020                     feed_ids.append(feed_id)
4021                 self.to_screen(
4022                     'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
4023                     % (', '.join(feed_ids), video_id))
4024                 return self.playlist_result(
4025                     entries, video_id, video_title, video_description)
4026
4027         duration = (int_or_none(get_first(video_details, 'lengthSeconds'))
4028                     or int_or_none(get_first(microformats, 'lengthSeconds'))
4029                     or parse_duration(search_meta('duration')) or None)
4030
4031         live_broadcast_details, live_status, streaming_data, formats, automatic_captions = \
4032             self._list_formats(video_id, microformats, video_details, player_responses, player_url, duration)
4033         if live_status == 'post_live':
4034             self.write_debug(f'{video_id}: Video is in Post-Live Manifestless mode')
4035
4036         if not formats:
4037             if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
4038                 self.report_drm(video_id)
4039             pemr = get_first(
4040                 playability_statuses,
4041                 ('errorScreen', 'playerErrorMessageRenderer'), expected_type=dict) or {}
4042             reason = self._get_text(pemr, 'reason') or get_first(playability_statuses, 'reason')
4043             subreason = clean_html(self._get_text(pemr, 'subreason') or '')
4044             if subreason:
4045                 if subreason == 'The uploader has not made this video available in your country.':
4046                     countries = get_first(microformats, 'availableCountries')
4047                     if not countries:
4048                         regions_allowed = search_meta('regionsAllowed')
4049                         countries = regions_allowed.split(',') if regions_allowed else None
4050                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
4051                 reason += f'. {subreason}'
4052             if reason:
4053                 self.raise_no_formats(reason, expected=True)
4054
4055         keywords = get_first(video_details, 'keywords', expected_type=list) or []
4056         if not keywords and webpage:
4057             keywords = [
4058                 unescapeHTML(m.group('content'))
4059                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
4060         for keyword in keywords:
4061             if keyword.startswith('yt:stretch='):
4062                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
4063                 if mobj:
4064                     # NB: float is intentional for forcing float division
4065                     w, h = (float(v) for v in mobj.groups())
4066                     if w > 0 and h > 0:
4067                         ratio = w / h
4068                         for f in formats:
4069                             if f.get('vcodec') != 'none':
4070                                 f['stretched_ratio'] = ratio
4071                         break
4072         thumbnails = self._extract_thumbnails((video_details, microformats), (..., ..., 'thumbnail'))
4073         thumbnail_url = search_meta(['og:image', 'twitter:image'])
4074         if thumbnail_url:
4075             thumbnails.append({
4076                 'url': thumbnail_url,
4077             })
4078         original_thumbnails = thumbnails.copy()
4079
4080         # The best resolution thumbnails sometimes does not appear in the webpage
4081         # See: https://github.com/yt-dlp/yt-dlp/issues/340
4082         # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029>
4083         thumbnail_names = [
4084             # While the *1,*2,*3 thumbnails are just below their corresponding "*default" variants
4085             # in resolution, these are not the custom thumbnail. So de-prioritize them
4086             'maxresdefault', 'hq720', 'sddefault', 'hqdefault', '0', 'mqdefault', 'default',
4087             'sd1', 'sd2', 'sd3', 'hq1', 'hq2', 'hq3', 'mq1', 'mq2', 'mq3', '1', '2', '3'
4088         ]
4089         n_thumbnail_names = len(thumbnail_names)
4090         thumbnails.extend({
4091             'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format(
4092                 video_id=video_id, name=name, ext=ext,
4093                 webp='_webp' if ext == 'webp' else '', live='_live' if live_status == 'is_live' else ''),
4094         } for name in thumbnail_names for ext in ('webp', 'jpg'))
4095         for thumb in thumbnails:
4096             i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names)
4097             thumb['preference'] = (0 if '.webp' in thumb['url'] else -1) - (2 * i)
4098         self._remove_duplicate_formats(thumbnails)
4099         self._downloader._sort_thumbnails(original_thumbnails)
4100
4101         category = get_first(microformats, 'category') or search_meta('genre')
4102         channel_id = str_or_none(
4103             get_first(video_details, 'channelId')
4104             or get_first(microformats, 'externalChannelId')
4105             or search_meta('channelId'))
4106         owner_profile_url = get_first(microformats, 'ownerProfileUrl')
4107
4108         live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
4109         live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
4110         if not duration and live_end_time and live_start_time:
4111             duration = live_end_time - live_start_time
4112
4113         needs_live_processing = self._needs_live_processing(live_status, duration)
4114
4115         def is_bad_format(fmt):
4116             if needs_live_processing and not fmt.get('is_from_start'):
4117                 return True
4118             elif (live_status == 'is_live' and needs_live_processing != 'is_live'
4119                     and fmt.get('protocol') == 'http_dash_segments'):
4120                 return True
4121
4122         for fmt in filter(is_bad_format, formats):
4123             fmt['preference'] = (fmt.get('preference') or -1) - 10
4124             fmt['format_note'] = join_nonempty(fmt.get('format_note'), '(Last 4 hours)', delim=' ')
4125
4126         if needs_live_processing:
4127             self._prepare_live_from_start_formats(
4128                 formats, video_id, live_start_time, url, webpage_url, smuggled_data, live_status == 'is_live')
4129
4130         formats.extend(self._extract_storyboard(player_responses, duration))
4131
4132         info = {
4133             'id': video_id,
4134             'title': video_title,
4135             'formats': formats,
4136             'thumbnails': thumbnails,
4137             # The best thumbnail that we are sure exists. Prevents unnecessary
4138             # URL checking if user don't care about getting the best possible thumbnail
4139             'thumbnail': traverse_obj(original_thumbnails, (-1, 'url')),
4140             'description': video_description,
4141             'uploader': get_first(video_details, 'author'),
4142             'uploader_id': self._search_regex(r'/(?:channel/|user/|(?=@))([^/?&#]+)', owner_profile_url, 'uploader id', default=None),
4143             'uploader_url': owner_profile_url,
4144             'channel_id': channel_id,
4145             'channel_url': format_field(channel_id, None, 'https://www.youtube.com/channel/%s'),
4146             'duration': duration,
4147             'view_count': int_or_none(
4148                 get_first((video_details, microformats), (..., 'viewCount'))
4149                 or search_meta('interactionCount')),
4150             'average_rating': float_or_none(get_first(video_details, 'averageRating')),
4151             'age_limit': 18 if (
4152                 get_first(microformats, 'isFamilySafe') is False
4153                 or search_meta('isFamilyFriendly') == 'false'
4154                 or search_meta('og:restrictions:age') == '18+') else 0,
4155             'webpage_url': webpage_url,
4156             'categories': [category] if category else None,
4157             'tags': keywords,
4158             'playable_in_embed': get_first(playability_statuses, 'playableInEmbed'),
4159             'live_status': live_status,
4160             'release_timestamp': live_start_time,
4161             '_format_sort_fields': (  # source_preference is lower for throttled/potentially damaged formats
4162                 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec:vp9.2', 'channels', 'acodec', 'lang', 'proto')
4163         }
4164
4165         subtitles = {}
4166         pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
4167         if pctr:
4168             def get_lang_code(track):
4169                 return (remove_start(track.get('vssId') or '', '.').replace('.', '-')
4170                         or track.get('languageCode'))
4171
4172             # Converted into dicts to remove duplicates
4173             captions = {
4174                 get_lang_code(sub): sub
4175                 for sub in traverse_obj(pctr, (..., 'captionTracks', ...))}
4176             translation_languages = {
4177                 lang.get('languageCode'): self._get_text(lang.get('languageName'), max_runs=1)
4178                 for lang in traverse_obj(pctr, (..., 'translationLanguages', ...))}
4179
4180             def process_language(container, base_url, lang_code, sub_name, query):
4181                 lang_subs = container.setdefault(lang_code, [])
4182                 for fmt in self._SUBTITLE_FORMATS:
4183                     query.update({
4184                         'fmt': fmt,
4185                     })
4186                     lang_subs.append({
4187                         'ext': fmt,
4188                         'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)),
4189                         'name': sub_name,
4190                     })
4191
4192             # NB: Constructing the full subtitle dictionary is slow
4193             get_translated_subs = 'translated_subs' not in self._configuration_arg('skip') and (
4194                 self.get_param('writeautomaticsub', False) or self.get_param('listsubtitles'))
4195             for lang_code, caption_track in captions.items():
4196                 base_url = caption_track.get('baseUrl')
4197                 orig_lang = parse_qs(base_url).get('lang', [None])[-1]
4198                 if not base_url:
4199                     continue
4200                 lang_name = self._get_text(caption_track, 'name', max_runs=1)
4201                 if caption_track.get('kind') != 'asr':
4202                     if not lang_code:
4203                         continue
4204                     process_language(
4205                         subtitles, base_url, lang_code, lang_name, {})
4206                     if not caption_track.get('isTranslatable'):
4207                         continue
4208                 for trans_code, trans_name in translation_languages.items():
4209                     if not trans_code:
4210                         continue
4211                     orig_trans_code = trans_code
4212                     if caption_track.get('kind') != 'asr' and trans_code != 'und':
4213                         if not get_translated_subs:
4214                             continue
4215                         trans_code += f'-{lang_code}'
4216                         trans_name += format_field(lang_name, None, ' from %s')
4217                     # Add an "-orig" label to the original language so that it can be distinguished.
4218                     # The subs are returned without "-orig" as well for compatibility
4219                     if lang_code == f'a-{orig_trans_code}':
4220                         process_language(
4221                             automatic_captions, base_url, f'{trans_code}-orig', f'{trans_name} (Original)', {})
4222                     # Setting tlang=lang returns damaged subtitles.
4223                     process_language(automatic_captions, base_url, trans_code, trans_name,
4224                                      {} if orig_lang == orig_trans_code else {'tlang': trans_code})
4225
4226         info['automatic_captions'] = automatic_captions
4227         info['subtitles'] = subtitles
4228
4229         parsed_url = urllib.parse.urlparse(url)
4230         for component in [parsed_url.fragment, parsed_url.query]:
4231             query = urllib.parse.parse_qs(component)
4232             for k, v in query.items():
4233                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
4234                     d_k += '_time'
4235                     if d_k not in info and k in s_ks:
4236                         info[d_k] = parse_duration(query[k][0])
4237
4238         # Youtube Music Auto-generated description
4239         if video_description:
4240             mobj = re.search(
4241                 r'''(?xs)
4242                     (?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+
4243                     (?P<album>[^\n]+)
4244                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
4245                     (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
4246                     (.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?
4247                     .+\nAuto-generated\ by\ YouTube\.\s*$
4248                 ''', video_description)
4249             if mobj:
4250                 release_year = mobj.group('release_year')
4251                 release_date = mobj.group('release_date')
4252                 if release_date:
4253                     release_date = release_date.replace('-', '')
4254                     if not release_year:
4255                         release_year = release_date[:4]
4256                 info.update({
4257                     'album': mobj.group('album'.strip()),
4258                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
4259                     'track': mobj.group('track').strip(),
4260                     'release_date': release_date,
4261                     'release_year': int_or_none(release_year),
4262                 })
4263
4264         initial_data = None
4265         if webpage:
4266             initial_data = self.extract_yt_initial_data(video_id, webpage, fatal=False)
4267             if not traverse_obj(initial_data, 'contents'):
4268                 self.report_warning('Incomplete data received in embedded initial data; re-fetching using API.')
4269                 initial_data = None
4270         if not initial_data:
4271             query = {'videoId': video_id}
4272             query.update(self._get_checkok_params())
4273             initial_data = self._extract_response(
4274                 item_id=video_id, ep='next', fatal=False,
4275                 ytcfg=master_ytcfg, query=query, check_get_keys='contents',
4276                 headers=self.generate_api_headers(ytcfg=master_ytcfg),
4277                 note='Downloading initial data API JSON')
4278
4279         info['comment_count'] = traverse_obj(initial_data, (
4280             'contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'itemSectionRenderer',
4281             'contents', ..., 'commentsEntryPointHeaderRenderer', 'commentCount'
4282         ), (
4283             'engagementPanels', lambda _, v: v['engagementPanelSectionListRenderer']['panelIdentifier'] == 'comment-item-section',
4284             'engagementPanelSectionListRenderer', 'header', 'engagementPanelTitleHeaderRenderer', 'contextualInfo'
4285         ), expected_type=self._get_count, get_all=False)
4286
4287         try:  # This will error if there is no livechat
4288             initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
4289         except (KeyError, IndexError, TypeError):
4290             pass
4291         else:
4292             info.setdefault('subtitles', {})['live_chat'] = [{
4293                 # url is needed to set cookies
4294                 'url': f'https://www.youtube.com/watch?v={video_id}&bpctr=9999999999&has_verified=1',
4295                 'video_id': video_id,
4296                 'ext': 'json',
4297                 'protocol': ('youtube_live_chat' if live_status in ('is_live', 'is_upcoming')
4298                              else 'youtube_live_chat_replay'),
4299             }]
4300
4301         if initial_data:
4302             info['chapters'] = (
4303                 self._extract_chapters_from_json(initial_data, duration)
4304                 or self._extract_chapters_from_engagement_panel(initial_data, duration)
4305                 or self._extract_chapters_from_description(video_description, duration)
4306                 or None)
4307
4308         contents = traverse_obj(
4309             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents'),
4310             expected_type=list, default=[])
4311
4312         vpir = get_first(contents, 'videoPrimaryInfoRenderer')
4313         if vpir:
4314             stl = vpir.get('superTitleLink')
4315             if stl:
4316                 stl = self._get_text(stl)
4317                 if try_get(
4318                         vpir,
4319                         lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
4320                     info['location'] = stl
4321                 else:
4322                     mobj = re.search(r'(.+?)\s*S(\d+)\s*•?\s*E(\d+)', stl)
4323                     if mobj:
4324                         info.update({
4325                             'series': mobj.group(1),
4326                             'season_number': int(mobj.group(2)),
4327                             'episode_number': int(mobj.group(3)),
4328                         })
4329             for tlb in (try_get(
4330                     vpir,
4331                     lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
4332                     list) or []):
4333                 tbrs = variadic(
4334                     traverse_obj(
4335                         tlb, ('toggleButtonRenderer', ...),
4336                         ('segmentedLikeDislikeButtonRenderer', ..., 'toggleButtonRenderer')))
4337                 for tbr in tbrs:
4338                     for getter, regex in [(
4339                             lambda x: x['defaultText']['accessibility']['accessibilityData'],
4340                             r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
4341                                 lambda x: x['accessibility'],
4342                                 lambda x: x['accessibilityData']['accessibilityData'],
4343                             ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
4344                         label = (try_get(tbr, getter, dict) or {}).get('label')
4345                         if label:
4346                             mobj = re.match(regex, label)
4347                             if mobj:
4348                                 info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
4349                                 break
4350             sbr_tooltip = try_get(
4351                 vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
4352             if sbr_tooltip:
4353                 like_count, dislike_count = sbr_tooltip.split(' / ')
4354                 info.update({
4355                     'like_count': str_to_int(like_count),
4356                     'dislike_count': str_to_int(dislike_count),
4357                 })
4358             vcr = traverse_obj(vpir, ('viewCount', 'videoViewCountRenderer'))
4359             if vcr:
4360                 vc = self._get_count(vcr, 'viewCount')
4361                 # Upcoming premieres with waiting count are treated as live here
4362                 if vcr.get('isLive'):
4363                     info['concurrent_view_count'] = vc
4364                 elif info.get('view_count') is None:
4365                     info['view_count'] = vc
4366
4367         vsir = get_first(contents, 'videoSecondaryInfoRenderer')
4368         if vsir:
4369             vor = traverse_obj(vsir, ('owner', 'videoOwnerRenderer'))
4370             info.update({
4371                 'channel': self._get_text(vor, 'title'),
4372                 'channel_follower_count': self._get_count(vor, 'subscriberCountText')})
4373
4374             rows = try_get(
4375                 vsir,
4376                 lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
4377                 list) or []
4378             multiple_songs = False
4379             for row in rows:
4380                 if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
4381                     multiple_songs = True
4382                     break
4383             for row in rows:
4384                 mrr = row.get('metadataRowRenderer') or {}
4385                 mrr_title = mrr.get('title')
4386                 if not mrr_title:
4387                     continue
4388                 mrr_title = self._get_text(mrr, 'title')
4389                 mrr_contents_text = self._get_text(mrr, ('contents', 0))
4390                 if mrr_title == 'License':
4391                     info['license'] = mrr_contents_text
4392                 elif not multiple_songs:
4393                     if mrr_title == 'Album':
4394                         info['album'] = mrr_contents_text
4395                     elif mrr_title == 'Artist':
4396                         info['artist'] = mrr_contents_text
4397                     elif mrr_title == 'Song':
4398                         info['track'] = mrr_contents_text
4399
4400         fallbacks = {
4401             'channel': 'uploader',
4402             'channel_id': 'uploader_id',
4403             'channel_url': 'uploader_url',
4404         }
4405
4406         # The upload date for scheduled, live and past live streams / premieres in microformats
4407         # may be different from the stream date. Although not in UTC, we will prefer it in this case.
4408         # See: https://github.com/yt-dlp/yt-dlp/pull/2223#issuecomment-1008485139
4409         upload_date = (
4410             unified_strdate(get_first(microformats, 'uploadDate'))
4411             or unified_strdate(search_meta('uploadDate')))
4412         if not upload_date or (
4413             live_status in ('not_live', None)
4414             and 'no-youtube-prefer-utc-upload-date' not in self.get_param('compat_opts', [])
4415         ):
4416             upload_date = strftime_or_none(
4417                 self._parse_time_text(self._get_text(vpir, 'dateText')), '%Y%m%d') or upload_date
4418         info['upload_date'] = upload_date
4419
4420         for to, frm in fallbacks.items():
4421             if not info.get(to):
4422                 info[to] = info.get(frm)
4423
4424         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
4425             v = info.get(s_k)
4426             if v:
4427                 info[d_k] = v
4428
4429         badges = self._extract_badges(traverse_obj(contents, (..., 'videoPrimaryInfoRenderer'), get_all=False))
4430
4431         is_private = (self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
4432                       or get_first(video_details, 'isPrivate', expected_type=bool))
4433
4434         info['availability'] = (
4435             'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
4436             else self._availability(
4437                 is_private=is_private,
4438                 needs_premium=(
4439                     self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM)
4440                     or False if initial_data and is_private is not None else None),
4441                 needs_subscription=(
4442                     self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION)
4443                     or False if initial_data and is_private is not None else None),
4444                 needs_auth=info['age_limit'] >= 18,
4445                 is_unlisted=None if is_private is None else (
4446                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
4447                     or get_first(microformats, 'isUnlisted', expected_type=bool))))
4448
4449         info['__post_extractor'] = self.extract_comments(master_ytcfg, video_id, contents, webpage)
4450
4451         self.mark_watched(video_id, player_responses)
4452
4453         return info
4454
4455
4456 class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
4457     @staticmethod
4458     def passthrough_smuggled_data(func):
4459         def _smuggle(info, smuggled_data):
4460             if info.get('_type') not in ('url', 'url_transparent'):
4461                 return info
4462             if smuggled_data.get('is_music_url'):
4463                 parsed_url = urllib.parse.urlparse(info['url'])
4464                 if parsed_url.netloc in ('www.youtube.com', 'music.youtube.com'):
4465                     smuggled_data.pop('is_music_url')
4466                     info['url'] = urllib.parse.urlunparse(parsed_url._replace(netloc='music.youtube.com'))
4467             if smuggled_data:
4468                 info['url'] = smuggle_url(info['url'], smuggled_data)
4469             return info
4470
4471         @functools.wraps(func)
4472         def wrapper(self, url):
4473             url, smuggled_data = unsmuggle_url(url, {})
4474             if self.is_music_url(url):
4475                 smuggled_data['is_music_url'] = True
4476             info_dict = func(self, url, smuggled_data)
4477             if smuggled_data:
4478                 _smuggle(info_dict, smuggled_data)
4479                 if info_dict.get('entries'):
4480                     info_dict['entries'] = (_smuggle(i, smuggled_data.copy()) for i in info_dict['entries'])
4481             return info_dict
4482         return wrapper
4483
4484     @staticmethod
4485     def _extract_basic_item_renderer(item):
4486         # Modified from _extract_grid_item_renderer
4487         known_basic_renderers = (
4488             'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer'
4489         )
4490         for key, renderer in item.items():
4491             if not isinstance(renderer, dict):
4492                 continue
4493             elif key in known_basic_renderers:
4494                 return renderer
4495             elif key.startswith('grid') and key.endswith('Renderer'):
4496                 return renderer
4497
4498     def _extract_channel_renderer(self, renderer):
4499         channel_id = renderer['channelId']
4500         title = self._get_text(renderer, 'title')
4501         channel_url = f'https://www.youtube.com/channel/{channel_id}'
4502         return {
4503             '_type': 'url',
4504             'url': channel_url,
4505             'id': channel_id,
4506             'ie_key': YoutubeTabIE.ie_key(),
4507             'channel': title,
4508             'channel_id': channel_id,
4509             'channel_url': channel_url,
4510             'title': title,
4511             'channel_follower_count': self._get_count(renderer, 'subscriberCountText'),
4512             'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
4513             'playlist_count': self._get_count(renderer, 'videoCountText'),
4514             'description': self._get_text(renderer, 'descriptionSnippet'),
4515         }
4516
4517     def _grid_entries(self, grid_renderer):
4518         for item in grid_renderer['items']:
4519             if not isinstance(item, dict):
4520                 continue
4521             renderer = self._extract_basic_item_renderer(item)
4522             if not isinstance(renderer, dict):
4523                 continue
4524             title = self._get_text(renderer, 'title')
4525
4526             # playlist
4527             playlist_id = renderer.get('playlistId')
4528             if playlist_id:
4529                 yield self.url_result(
4530                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
4531                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
4532                     video_title=title)
4533                 continue
4534             # video
4535             video_id = renderer.get('videoId')
4536             if video_id:
4537                 yield self._extract_video(renderer)
4538                 continue
4539             # channel
4540             channel_id = renderer.get('channelId')
4541             if channel_id:
4542                 yield self._extract_channel_renderer(renderer)
4543                 continue
4544             # generic endpoint URL support
4545             ep_url = urljoin('https://www.youtube.com/', try_get(
4546                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
4547                 str))
4548             if ep_url:
4549                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
4550                     if ie.suitable(ep_url):
4551                         yield self.url_result(
4552                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
4553                         break
4554
4555     def _music_reponsive_list_entry(self, renderer):
4556         video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
4557         if video_id:
4558             return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
4559                                    ie=YoutubeIE.ie_key(), video_id=video_id)
4560         playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
4561         if playlist_id:
4562             video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
4563             if video_id:
4564                 return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
4565                                        ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4566             return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
4567                                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4568         browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
4569         if browse_id:
4570             return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
4571                                    ie=YoutubeTabIE.ie_key(), video_id=browse_id)
4572
4573     def _shelf_entries_from_content(self, shelf_renderer):
4574         content = shelf_renderer.get('content')
4575         if not isinstance(content, dict):
4576             return
4577         renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
4578         if renderer:
4579             # TODO: add support for nested playlists so each shelf is processed
4580             # as separate playlist
4581             # TODO: this includes only first N items
4582             yield from self._grid_entries(renderer)
4583         renderer = content.get('horizontalListRenderer')
4584         if renderer:
4585             # TODO
4586             pass
4587
4588     def _shelf_entries(self, shelf_renderer, skip_channels=False):
4589         ep = try_get(
4590             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4591             str)
4592         shelf_url = urljoin('https://www.youtube.com', ep)
4593         if shelf_url:
4594             # Skipping links to another channels, note that checking for
4595             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
4596             # will not work
4597             if skip_channels and '/channels?' in shelf_url:
4598                 return
4599             title = self._get_text(shelf_renderer, 'title')
4600             yield self.url_result(shelf_url, video_title=title)
4601         # Shelf may not contain shelf URL, fallback to extraction from content
4602         yield from self._shelf_entries_from_content(shelf_renderer)
4603
4604     def _playlist_entries(self, video_list_renderer):
4605         for content in video_list_renderer['contents']:
4606             if not isinstance(content, dict):
4607                 continue
4608             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
4609             if not isinstance(renderer, dict):
4610                 continue
4611             video_id = renderer.get('videoId')
4612             if not video_id:
4613                 continue
4614             yield self._extract_video(renderer)
4615
4616     def _rich_entries(self, rich_grid_renderer):
4617         renderer = traverse_obj(
4618             rich_grid_renderer, ('content', ('videoRenderer', 'reelItemRenderer')), get_all=False) or {}
4619         video_id = renderer.get('videoId')
4620         if not video_id:
4621             return
4622         yield self._extract_video(renderer)
4623
4624     def _video_entry(self, video_renderer):
4625         video_id = video_renderer.get('videoId')
4626         if video_id:
4627             return self._extract_video(video_renderer)
4628
4629     def _hashtag_tile_entry(self, hashtag_tile_renderer):
4630         url = urljoin('https://youtube.com', traverse_obj(
4631             hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
4632         if url:
4633             return self.url_result(
4634                 url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
4635
4636     def _post_thread_entries(self, post_thread_renderer):
4637         post_renderer = try_get(
4638             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
4639         if not post_renderer:
4640             return
4641         # video attachment
4642         video_renderer = try_get(
4643             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
4644         video_id = video_renderer.get('videoId')
4645         if video_id:
4646             entry = self._extract_video(video_renderer)
4647             if entry:
4648                 yield entry
4649         # playlist attachment
4650         playlist_id = try_get(
4651             post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
4652         if playlist_id:
4653             yield self.url_result(
4654                 'https://www.youtube.com/playlist?list=%s' % playlist_id,
4655                 ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
4656         # inline video links
4657         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
4658         for run in runs:
4659             if not isinstance(run, dict):
4660                 continue
4661             ep_url = try_get(
4662                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
4663             if not ep_url:
4664                 continue
4665             if not YoutubeIE.suitable(ep_url):
4666                 continue
4667             ep_video_id = YoutubeIE._match_id(ep_url)
4668             if video_id == ep_video_id:
4669                 continue
4670             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
4671
4672     def _post_thread_continuation_entries(self, post_thread_continuation):
4673         contents = post_thread_continuation.get('contents')
4674         if not isinstance(contents, list):
4675             return
4676         for content in contents:
4677             renderer = content.get('backstagePostThreadRenderer')
4678             if isinstance(renderer, dict):
4679                 yield from self._post_thread_entries(renderer)
4680                 continue
4681             renderer = content.get('videoRenderer')
4682             if isinstance(renderer, dict):
4683                 yield self._video_entry(renderer)
4684
4685     r''' # unused
4686     def _rich_grid_entries(self, contents):
4687         for content in contents:
4688             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
4689             if video_renderer:
4690                 entry = self._video_entry(video_renderer)
4691                 if entry:
4692                     yield entry
4693     '''
4694
4695     def _report_history_entries(self, renderer):
4696         for url in traverse_obj(renderer, (
4697                 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
4698                 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
4699                 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
4700             yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
4701
4702     def _extract_entries(self, parent_renderer, continuation_list):
4703         # continuation_list is modified in-place with continuation_list = [continuation_token]
4704         continuation_list[:] = [None]
4705         contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
4706         for content in contents:
4707             if not isinstance(content, dict):
4708                 continue
4709             is_renderer = traverse_obj(
4710                 content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
4711                 expected_type=dict)
4712             if not is_renderer:
4713                 if content.get('richItemRenderer'):
4714                     for entry in self._rich_entries(content['richItemRenderer']):
4715                         yield entry
4716                     continuation_list[0] = self._extract_continuation(parent_renderer)
4717                 elif content.get('reportHistorySectionRenderer'):  # https://www.youtube.com/reporthistory
4718                     table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
4719                     yield from self._report_history_entries(table)
4720                     continuation_list[0] = self._extract_continuation(table)
4721                 continue
4722
4723             isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
4724             for isr_content in isr_contents:
4725                 if not isinstance(isr_content, dict):
4726                     continue
4727
4728                 known_renderers = {
4729                     'playlistVideoListRenderer': self._playlist_entries,
4730                     'gridRenderer': self._grid_entries,
4731                     'reelShelfRenderer': self._grid_entries,
4732                     'shelfRenderer': self._shelf_entries,
4733                     'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
4734                     'backstagePostThreadRenderer': self._post_thread_entries,
4735                     'videoRenderer': lambda x: [self._video_entry(x)],
4736                     'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
4737                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
4738                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)]
4739                 }
4740                 for key, renderer in isr_content.items():
4741                     if key not in known_renderers:
4742                         continue
4743                     for entry in known_renderers[key](renderer):
4744                         if entry:
4745                             yield entry
4746                     continuation_list[0] = self._extract_continuation(renderer)
4747                     break
4748
4749             if not continuation_list[0]:
4750                 continuation_list[0] = self._extract_continuation(is_renderer)
4751
4752         if not continuation_list[0]:
4753             continuation_list[0] = self._extract_continuation(parent_renderer)
4754
4755     def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
4756         continuation_list = [None]
4757         extract_entries = lambda x: self._extract_entries(x, continuation_list)
4758         tab_content = try_get(tab, lambda x: x['content'], dict)
4759         if not tab_content:
4760             return
4761         parent_renderer = (
4762             try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
4763             or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
4764         yield from extract_entries(parent_renderer)
4765         continuation = continuation_list[0]
4766
4767         for page_num in itertools.count(1):
4768             if not continuation:
4769                 break
4770             headers = self.generate_api_headers(
4771                 ytcfg=ytcfg, account_syncid=account_syncid, visitor_data=visitor_data)
4772             response = self._extract_response(
4773                 item_id=f'{item_id} page {page_num}',
4774                 query=continuation, headers=headers, ytcfg=ytcfg,
4775                 check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
4776
4777             if not response:
4778                 break
4779             # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
4780             # See: https://github.com/ytdl-org/youtube-dl/issues/28702
4781             visitor_data = self._extract_visitor_data(response) or visitor_data
4782
4783             known_renderers = {
4784                 'videoRenderer': (self._grid_entries, 'items'),  # for membership tab
4785                 'gridPlaylistRenderer': (self._grid_entries, 'items'),
4786                 'gridVideoRenderer': (self._grid_entries, 'items'),
4787                 'gridChannelRenderer': (self._grid_entries, 'items'),
4788                 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
4789                 'itemSectionRenderer': (extract_entries, 'contents'),  # for feeds
4790                 'richItemRenderer': (extract_entries, 'contents'),  # for hashtag
4791                 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
4792                 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
4793                 'playlistVideoListContinuation': (self._playlist_entries, None),
4794                 'gridContinuation': (self._grid_entries, None),
4795                 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
4796                 'sectionListContinuation': (extract_entries, None),  # for feeds
4797             }
4798
4799             continuation_items = traverse_obj(response, (
4800                 ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
4801                 'appendContinuationItemsAction', 'continuationItems'
4802             ), 'continuationContents', get_all=False)
4803             continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
4804
4805             video_items_renderer = None
4806             for key in continuation_item.keys():
4807                 if key not in known_renderers:
4808                     continue
4809                 func, parent_key = known_renderers[key]
4810                 video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
4811                 continuation_list = [None]
4812                 yield from func(video_items_renderer)
4813                 continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
4814
4815             if not video_items_renderer:
4816                 break
4817
4818     @staticmethod
4819     def _extract_selected_tab(tabs, fatal=True):
4820         for tab_renderer in tabs:
4821             if tab_renderer.get('selected'):
4822                 return tab_renderer
4823         if fatal:
4824             raise ExtractorError('Unable to find selected tab')
4825
4826     @staticmethod
4827     def _extract_tab_renderers(response):
4828         return traverse_obj(
4829             response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
4830
4831     def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
4832         metadata = self._extract_metadata_from_tabs(item_id, data)
4833
4834         selected_tab = self._extract_selected_tab(tabs)
4835         metadata['title'] += format_field(selected_tab, 'title', ' - %s')
4836         metadata['title'] += format_field(selected_tab, 'expandedText', ' - %s')
4837
4838         return self.playlist_result(
4839             self._entries(
4840                 selected_tab, metadata['id'], ytcfg,
4841                 self._extract_account_syncid(ytcfg, data),
4842                 self._extract_visitor_data(data, ytcfg)),
4843             **metadata)
4844
4845     def _extract_metadata_from_tabs(self, item_id, data):
4846         info = {'id': item_id}
4847
4848         metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
4849         if metadata_renderer:
4850             info.update({
4851                 'uploader': metadata_renderer.get('title'),
4852                 'uploader_id': metadata_renderer.get('externalId'),
4853                 'uploader_url': metadata_renderer.get('channelUrl'),
4854             })
4855             if info['uploader_id']:
4856                 info['id'] = info['uploader_id']
4857         else:
4858             metadata_renderer = traverse_obj(data, ('metadata', 'playlistMetadataRenderer'), expected_type=dict)
4859
4860         # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
4861         # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
4862         def _get_uncropped(url):
4863             return url_or_none((url or '').split('=')[0] + '=s0')
4864
4865         avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
4866         if avatar_thumbnails:
4867             uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
4868             if uncropped_avatar:
4869                 avatar_thumbnails.append({
4870                     'url': uncropped_avatar,
4871                     'id': 'avatar_uncropped',
4872                     'preference': 1
4873                 })
4874
4875         channel_banners = self._extract_thumbnails(
4876             data, ('header', ..., ('banner', 'mobileBanner', 'tvBanner')))
4877         for banner in channel_banners:
4878             banner['preference'] = -10
4879
4880         if channel_banners:
4881             uncropped_banner = _get_uncropped(channel_banners[0]['url'])
4882             if uncropped_banner:
4883                 channel_banners.append({
4884                     'url': uncropped_banner,
4885                     'id': 'banner_uncropped',
4886                     'preference': -5
4887                 })
4888
4889         # Deprecated - remove primary_sidebar_renderer when layout discontinued
4890         primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
4891         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
4892
4893         primary_thumbnails = self._extract_thumbnails(
4894             primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
4895         playlist_thumbnails = self._extract_thumbnails(
4896             playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
4897
4898         info.update({
4899             'title': (traverse_obj(metadata_renderer, 'title')
4900                       or self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag'))
4901                       or info['id']),
4902             'availability': self._extract_availability(data),
4903             'channel_follower_count': self._get_count(data, ('header', ..., 'subscriberCountText')),
4904             'description': try_get(metadata_renderer, lambda x: x.get('description', '')),
4905             'tags': try_get(metadata_renderer or {}, lambda x: x.get('keywords', '').split()),
4906             'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
4907         })
4908
4909         # Playlist stats is a text runs array containing [video count, view count, last updated].
4910         # last updated or (view count and last updated) may be missing.
4911         playlist_stats = get_first(
4912             (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'), ))
4913
4914         last_updated_unix = self._parse_time_text(
4915             self._get_text(playlist_stats, 2)  # deprecated, remove when old layout discontinued
4916             or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
4917         info['modified_date'] = strftime_or_none(last_updated_unix, '%Y%m%d')
4918
4919         info['view_count'] = self._get_count(playlist_stats, 1)
4920         if info['view_count'] is None:  # 0 is allowed
4921             info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText')
4922         if info['view_count'] is None:
4923             info['view_count'] = self._get_count(data, (
4924                 'contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., 'tabRenderer', 'content', 'sectionListRenderer',
4925                 'contents', ..., 'itemSectionRenderer', 'contents', ..., 'channelAboutFullMetadataRenderer', 'viewCountText'))
4926
4927         info['playlist_count'] = self._get_count(playlist_stats, 0)
4928         if info['playlist_count'] is None:  # 0 is allowed
4929             info['playlist_count'] = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
4930
4931         if not info.get('uploader_id'):
4932             owner = traverse_obj(playlist_header_renderer, 'ownerText')
4933             if not owner:  # Deprecated
4934                 owner = traverse_obj(
4935                     self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
4936                     ('videoOwner', 'videoOwnerRenderer', 'title'))
4937             owner_text = self._get_text(owner)
4938             browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
4939             info.update({
4940                 'uploader': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
4941                 'uploader_id': browse_ep.get('browseId'),
4942                 'uploader_url': urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))
4943             })
4944
4945         info.update({
4946             'channel': info['uploader'],
4947             'channel_id': info['uploader_id'],
4948             'channel_url': info['uploader_url']
4949         })
4950         return info
4951
4952     def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
4953         first_id = last_id = response = None
4954         for page_num in itertools.count(1):
4955             videos = list(self._playlist_entries(playlist))
4956             if not videos:
4957                 return
4958             start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
4959             if start >= len(videos):
4960                 return
4961             yield from videos[start:]
4962             first_id = first_id or videos[0]['id']
4963             last_id = videos[-1]['id']
4964             watch_endpoint = try_get(
4965                 playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
4966             headers = self.generate_api_headers(
4967                 ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
4968                 visitor_data=self._extract_visitor_data(response, data, ytcfg))
4969             query = {
4970                 'playlistId': playlist_id,
4971                 'videoId': watch_endpoint.get('videoId') or last_id,
4972                 'index': watch_endpoint.get('index') or len(videos),
4973                 'params': watch_endpoint.get('params') or 'OAE%3D'
4974             }
4975             response = self._extract_response(
4976                 item_id='%s page %d' % (playlist_id, page_num),
4977                 query=query, ep='next', headers=headers, ytcfg=ytcfg,
4978                 check_get_keys='contents'
4979             )
4980             playlist = try_get(
4981                 response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
4982
4983     def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
4984         title = playlist.get('title') or try_get(
4985             data, lambda x: x['titleText']['simpleText'], str)
4986         playlist_id = playlist.get('playlistId') or item_id
4987
4988         # Delegating everything except mix playlists to regular tab-based playlist URL
4989         playlist_url = urljoin(url, try_get(
4990             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
4991             str))
4992
4993         # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
4994         # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
4995         is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
4996
4997         if playlist_url and playlist_url != url and not is_known_unviewable:
4998             return self.url_result(
4999                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
5000                 video_title=title)
5001
5002         return self.playlist_result(
5003             self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
5004             playlist_id=playlist_id, playlist_title=title)
5005
5006     def _extract_availability(self, data):
5007         """
5008         Gets the availability of a given playlist/tab.
5009         Note: Unless YouTube tells us explicitly, we do not assume it is public
5010         @param data: response
5011         """
5012         sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
5013         playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
5014         player_header_privacy = playlist_header_renderer.get('privacy')
5015
5016         badges = self._extract_badges(sidebar_renderer)
5017
5018         # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
5019         privacy_setting_icon = get_first(
5020             (playlist_header_renderer, sidebar_renderer),
5021             ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
5022              lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
5023             expected_type=str)
5024
5025         microformats_is_unlisted = traverse_obj(
5026             data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
5027
5028         return (
5029             'public' if (
5030                 self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
5031                 or player_header_privacy == 'PUBLIC'
5032                 or privacy_setting_icon == 'PRIVACY_PUBLIC')
5033             else self._availability(
5034                 is_private=(
5035                     self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
5036                     or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
5037                     else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
5038                 is_unlisted=(
5039                     self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
5040                     or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
5041                     else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
5042                     else microformats_is_unlisted if microformats_is_unlisted is not None else None),
5043                 needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
5044                 needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
5045                 needs_auth=False))
5046
5047     @staticmethod
5048     def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
5049         sidebar_renderer = try_get(
5050             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
5051         for item in sidebar_renderer:
5052             renderer = try_get(item, lambda x: x[info_renderer], expected_type)
5053             if renderer:
5054                 return renderer
5055
5056     def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
5057         """
5058         Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
5059         """
5060         is_playlist = bool(traverse_obj(
5061             data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
5062         if not is_playlist:
5063             return
5064         headers = self.generate_api_headers(
5065             ytcfg=ytcfg, account_syncid=self._extract_account_syncid(ytcfg, data),
5066             visitor_data=self._extract_visitor_data(data, ytcfg))
5067         query = {
5068             'params': 'wgYCCAA=',
5069             'browseId': f'VL{item_id}'
5070         }
5071         return self._extract_response(
5072             item_id=item_id, headers=headers, query=query,
5073             check_get_keys='contents', fatal=False, ytcfg=ytcfg,
5074             note='Redownloading playlist API JSON with unavailable videos')
5075
5076     @functools.cached_property
5077     def skip_webpage(self):
5078         return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
5079
5080     def _extract_webpage(self, url, item_id, fatal=True):
5081         webpage, data = None, None
5082         for retry in self.RetryManager(fatal=fatal):
5083             try:
5084                 webpage = self._download_webpage(url, item_id, note='Downloading webpage')
5085                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
5086             except ExtractorError as e:
5087                 if isinstance(e.cause, network_exceptions):
5088                     if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
5089                         retry.error = e
5090                         continue
5091                 self._error_or_warning(e, fatal=fatal)
5092                 break
5093
5094             try:
5095                 self._extract_and_report_alerts(data)
5096             except ExtractorError as e:
5097                 self._error_or_warning(e, fatal=fatal)
5098                 break
5099
5100             # Sometimes youtube returns a webpage with incomplete ytInitialData
5101             # See: https://github.com/yt-dlp/yt-dlp/issues/116
5102             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
5103                 retry.error = ExtractorError('Incomplete yt initial data received')
5104                 continue
5105
5106         return webpage, data
5107
5108     def _report_playlist_authcheck(self, ytcfg, fatal=True):
5109         """Use if failed to extract ytcfg (and data) from initial webpage"""
5110         if not ytcfg and self.is_authenticated:
5111             msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
5112             if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
5113                 raise ExtractorError(
5114                     f'{msg}. If you are not downloading private content, or '
5115                     'your cookies are only for the first account and channel,'
5116                     ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
5117                     expected=True)
5118             self.report_warning(msg, only_once=True)
5119
5120     def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
5121         data = None
5122         if not self.skip_webpage:
5123             webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
5124             ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
5125             # Reject webpage data if redirected to home page without explicitly requesting
5126             selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
5127             if (url != 'https://www.youtube.com/feed/recommended'
5128                     and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch'  # Home page
5129                     and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
5130                 msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
5131                 if fatal:
5132                     raise ExtractorError(msg, expected=True)
5133                 self.report_warning(msg, only_once=True)
5134         if not data:
5135             self._report_playlist_authcheck(ytcfg, fatal=fatal)
5136             data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
5137         return data, ytcfg
5138
5139     def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
5140         headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
5141         resolve_response = self._extract_response(
5142             item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
5143             ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
5144         endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
5145         for ep_key, ep in endpoints.items():
5146             params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
5147             if params:
5148                 return self._extract_response(
5149                     item_id=item_id, query=params, ep=ep, headers=headers,
5150                     ytcfg=ytcfg, fatal=fatal, default_client=default_client,
5151                     check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
5152         err_note = 'Failed to resolve url (does the playlist exist?)'
5153         if fatal:
5154             raise ExtractorError(err_note, expected=True)
5155         self.report_warning(err_note, item_id)
5156
5157     _SEARCH_PARAMS = None
5158
5159     def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
5160         data = {'query': query}
5161         if params is NO_DEFAULT:
5162             params = self._SEARCH_PARAMS
5163         if params:
5164             data['params'] = params
5165
5166         content_keys = (
5167             ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
5168             ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
5169             # ytmusic search
5170             ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
5171             ('continuationContents', ),
5172         )
5173         display_id = f'query "{query}"'
5174         check_get_keys = tuple({keys[0] for keys in content_keys})
5175         ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
5176         self._report_playlist_authcheck(ytcfg, fatal=False)
5177
5178         continuation_list = [None]
5179         search = None
5180         for page_num in itertools.count(1):
5181             data.update(continuation_list[0] or {})
5182             headers = self.generate_api_headers(
5183                 ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
5184             search = self._extract_response(
5185                 item_id=f'{display_id} page {page_num}', ep='search', query=data,
5186                 default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
5187             slr_contents = traverse_obj(search, *content_keys)
5188             yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
5189             if not continuation_list[0]:
5190                 break
5191
5192
5193 class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
5194     IE_DESC = 'YouTube Tabs'
5195     _VALID_URL = r'''(?x:
5196         https?://
5197             (?!consent\.)(?:\w+\.)?
5198             (?:
5199                 youtube(?:kids)?\.com|
5200                 %(invidious)s
5201             )/
5202             (?:
5203                 (?P<channel_type>channel|c|user|browse)/|
5204                 (?P<not_channel>
5205                     feed/|hashtag/|
5206                     (?:playlist|watch)\?.*?\blist=
5207                 )|
5208                 (?!(?:%(reserved_names)s)\b)  # Direct URLs
5209             )
5210             (?P<id>[^/?\#&]+)
5211     )''' % {
5212         'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
5213         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
5214     }
5215     IE_NAME = 'youtube:tab'
5216
5217     _TESTS = [{
5218         'note': 'playlists, multipage',
5219         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
5220         'playlist_mincount': 94,
5221         'info_dict': {
5222             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5223             'title': 'Igor Kleiner - Playlists',
5224             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5225             'uploader': 'Igor Kleiner',
5226             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5227             'channel': 'Igor Kleiner',
5228             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5229             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5230             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5231             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5232             'channel_follower_count': int
5233         },
5234     }, {
5235         'note': 'playlists, multipage, different order',
5236         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
5237         'playlist_mincount': 94,
5238         'info_dict': {
5239             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
5240             'title': 'Igor Kleiner - Playlists',
5241             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
5242             'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5243             'uploader': 'Igor Kleiner',
5244             'uploader_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5245             'tags': ['"критическое', 'мышление"', '"наука', 'просто"', 'математика', '"анализ', 'данных"'],
5246             'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
5247             'channel': 'Igor Kleiner',
5248             'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
5249             'channel_follower_count': int
5250         },
5251     }, {
5252         'note': 'playlists, series',
5253         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
5254         'playlist_mincount': 5,
5255         'info_dict': {
5256             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5257             'title': '3Blue1Brown - Playlists',
5258             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5259             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5260             'uploader': '3Blue1Brown',
5261             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5262             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5263             'channel': '3Blue1Brown',
5264             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5265             'tags': ['Mathematics'],
5266             'channel_follower_count': int
5267         },
5268     }, {
5269         'note': 'playlists, singlepage',
5270         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
5271         'playlist_mincount': 4,
5272         'info_dict': {
5273             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5274             'title': 'ThirstForScience - Playlists',
5275             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
5276             'uploader': 'ThirstForScience',
5277             'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5278             'uploader_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5279             'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
5280             'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
5281             'tags': 'count:13',
5282             'channel': 'ThirstForScience',
5283             'channel_follower_count': int
5284         }
5285     }, {
5286         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
5287         'only_matching': True,
5288     }, {
5289         'note': 'basic, single video playlist',
5290         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5291         'info_dict': {
5292             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5293             'uploader': 'Sergey M.',
5294             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5295             'title': 'youtube-dl public playlist',
5296             'description': '',
5297             'tags': [],
5298             'view_count': int,
5299             'modified_date': '20201130',
5300             'channel': 'Sergey M.',
5301             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5302             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5303             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5304             'availability': 'public',
5305         },
5306         'playlist_count': 1,
5307     }, {
5308         'note': 'empty playlist',
5309         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5310         'info_dict': {
5311             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5312             'uploader': 'Sergey M.',
5313             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
5314             'title': 'youtube-dl empty playlist',
5315             'tags': [],
5316             'channel': 'Sergey M.',
5317             'description': '',
5318             'modified_date': '20160902',
5319             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
5320             'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5321             'uploader_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5322             'availability': 'public',
5323         },
5324         'playlist_count': 0,
5325     }, {
5326         'note': 'Home tab',
5327         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
5328         'info_dict': {
5329             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5330             'title': 'lex will - Home',
5331             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5332             'uploader': 'lex will',
5333             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5334             'channel': 'lex will',
5335             'tags': ['bible', 'history', 'prophesy'],
5336             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5337             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5338             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5339             'channel_follower_count': int
5340         },
5341         'playlist_mincount': 2,
5342     }, {
5343         'note': 'Videos tab',
5344         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
5345         'info_dict': {
5346             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5347             'title': 'lex will - Videos',
5348             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5349             'uploader': 'lex will',
5350             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5351             'tags': ['bible', 'history', 'prophesy'],
5352             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5353             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5354             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5355             'channel': 'lex will',
5356             'channel_follower_count': int
5357         },
5358         'playlist_mincount': 975,
5359     }, {
5360         'note': 'Videos tab, sorted by popular',
5361         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
5362         'info_dict': {
5363             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5364             'title': 'lex will - Videos',
5365             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5366             'uploader': 'lex will',
5367             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5368             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5369             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5370             'channel': 'lex will',
5371             'tags': ['bible', 'history', 'prophesy'],
5372             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5373             'channel_follower_count': int
5374         },
5375         'playlist_mincount': 199,
5376     }, {
5377         'note': 'Playlists tab',
5378         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
5379         'info_dict': {
5380             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5381             'title': 'lex will - Playlists',
5382             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5383             'uploader': 'lex will',
5384             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5385             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5386             'channel': 'lex will',
5387             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5388             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5389             'tags': ['bible', 'history', 'prophesy'],
5390             'channel_follower_count': int
5391         },
5392         'playlist_mincount': 17,
5393     }, {
5394         'note': 'Community tab',
5395         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
5396         'info_dict': {
5397             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5398             'title': 'lex will - Community',
5399             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5400             'uploader': 'lex will',
5401             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5402             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5403             'channel': 'lex will',
5404             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5405             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5406             'tags': ['bible', 'history', 'prophesy'],
5407             'channel_follower_count': int
5408         },
5409         'playlist_mincount': 18,
5410     }, {
5411         'note': 'Channels tab',
5412         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
5413         'info_dict': {
5414             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5415             'title': 'lex will - Channels',
5416             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
5417             'uploader': 'lex will',
5418             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5419             'uploader_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5420             'channel': 'lex will',
5421             'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
5422             'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
5423             'tags': ['bible', 'history', 'prophesy'],
5424             'channel_follower_count': int
5425         },
5426         'playlist_mincount': 12,
5427     }, {
5428         'note': 'Search tab',
5429         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
5430         'playlist_mincount': 40,
5431         'info_dict': {
5432             'id': 'UCYO_jab_esuFRV4b17AJtAw',
5433             'title': '3Blue1Brown - Search - linear algebra',
5434             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
5435             'uploader': '3Blue1Brown',
5436             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
5437             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5438             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
5439             'tags': ['Mathematics'],
5440             'channel': '3Blue1Brown',
5441             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
5442             'channel_follower_count': int
5443         },
5444     }, {
5445         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5446         'only_matching': True,
5447     }, {
5448         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5449         'only_matching': True,
5450     }, {
5451         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
5452         'only_matching': True,
5453     }, {
5454         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
5455         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5456         'info_dict': {
5457             'title': '29C3: Not my department',
5458             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
5459             'uploader': 'Christiaan008',
5460             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5461             'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
5462             'tags': [],
5463             'uploader_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5464             'view_count': int,
5465             'modified_date': '20150605',
5466             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
5467             'channel_url': 'https://www.youtube.com/c/ChRiStIaAn008',
5468             'channel': 'Christiaan008',
5469             'availability': 'public',
5470         },
5471         'playlist_count': 96,
5472     }, {
5473         'note': 'Large playlist',
5474         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
5475         'info_dict': {
5476             'title': 'Uploads from Cauchemar',
5477             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
5478             'uploader': 'Cauchemar',
5479             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5480             'channel_url': 'https://www.youtube.com/c/Cauchemar89',
5481             'tags': [],
5482             'modified_date': r're:\d{8}',
5483             'channel': 'Cauchemar',
5484             'uploader_url': 'https://www.youtube.com/c/Cauchemar89',
5485             'view_count': int,
5486             'description': '',
5487             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
5488             'availability': 'public',
5489         },
5490         'playlist_mincount': 1123,
5491         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5492     }, {
5493         'note': 'even larger playlist, 8832 videos',
5494         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
5495         'only_matching': True,
5496     }, {
5497         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
5498         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
5499         'info_dict': {
5500             'title': 'Uploads from Interstellar Movie',
5501             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
5502             'uploader': 'Interstellar Movie',
5503             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5504             'uploader_url': 'https://www.youtube.com/c/InterstellarMovie',
5505             'tags': [],
5506             'view_count': int,
5507             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
5508             'channel_url': 'https://www.youtube.com/c/InterstellarMovie',
5509             'channel': 'Interstellar Movie',
5510             'description': '',
5511             'modified_date': r're:\d{8}',
5512             'availability': 'public',
5513         },
5514         'playlist_mincount': 21,
5515     }, {
5516         'note': 'Playlist with "show unavailable videos" button',
5517         'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
5518         'info_dict': {
5519             'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
5520             'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
5521             'uploader': 'Phim Siêu Nhân Nhật Bản',
5522             'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5523             'view_count': int,
5524             'channel': 'Phim Siêu Nhân Nhật Bản',
5525             'tags': [],
5526             'uploader_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5527             'description': '',
5528             'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
5529             'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
5530             'modified_date': r're:\d{8}',
5531             'availability': 'public',
5532         },
5533         'playlist_mincount': 200,
5534         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5535     }, {
5536         'note': 'Playlist with unavailable videos in page 7',
5537         'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
5538         'info_dict': {
5539             'title': 'Uploads from BlankTV',
5540             'id': 'UU8l9frL61Yl5KFOl87nIm2w',
5541             'uploader': 'BlankTV',
5542             'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5543             'channel': 'BlankTV',
5544             'channel_url': 'https://www.youtube.com/c/blanktv',
5545             'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
5546             'view_count': int,
5547             'tags': [],
5548             'uploader_url': 'https://www.youtube.com/c/blanktv',
5549             'modified_date': r're:\d{8}',
5550             'description': '',
5551             'availability': 'public',
5552         },
5553         'playlist_mincount': 1000,
5554         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5555     }, {
5556         'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
5557         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5558         'info_dict': {
5559             'title': 'Data Analysis with Dr Mike Pound',
5560             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
5561             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5562             'uploader': 'Computerphile',
5563             'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
5564             'uploader_url': 'https://www.youtube.com/user/Computerphile',
5565             'tags': [],
5566             'view_count': int,
5567             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
5568             'channel_url': 'https://www.youtube.com/user/Computerphile',
5569             'channel': 'Computerphile',
5570             'availability': 'public',
5571             'modified_date': '20190712',
5572         },
5573         'playlist_mincount': 11,
5574     }, {
5575         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
5576         'only_matching': True,
5577     }, {
5578         'note': 'Playlist URL that does not actually serve a playlist',
5579         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
5580         'info_dict': {
5581             'id': 'FqZTN594JQw',
5582             'ext': 'webm',
5583             'title': "Smiley's People 01 detective, Adventure Series, Action",
5584             'uploader': 'STREEM',
5585             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
5586             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
5587             'upload_date': '20150526',
5588             'license': 'Standard YouTube License',
5589             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
5590             'categories': ['People & Blogs'],
5591             'tags': list,
5592             'view_count': int,
5593             'like_count': int,
5594         },
5595         'params': {
5596             'skip_download': True,
5597         },
5598         'skip': 'This video is not available.',
5599         'add_ie': [YoutubeIE.ie_key()],
5600     }, {
5601         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
5602         'only_matching': True,
5603     }, {
5604         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
5605         'only_matching': True,
5606     }, {
5607         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
5608         'info_dict': {
5609             'id': 'Wq15eF5vCbI',  # This will keep changing
5610             'ext': 'mp4',
5611             'title': str,
5612             'uploader': 'Sky News',
5613             'uploader_id': 'skynews',
5614             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
5615             'upload_date': r're:\d{8}',
5616             'description': str,
5617             'categories': ['News & Politics'],
5618             'tags': list,
5619             'like_count': int,
5620             'release_timestamp': int,
5621             'channel': 'Sky News',
5622             'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
5623             'age_limit': 0,
5624             'view_count': int,
5625             'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
5626             'playable_in_embed': True,
5627             'release_date': r're:\d+',
5628             'availability': 'public',
5629             'live_status': 'is_live',
5630             'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
5631             'channel_follower_count': int,
5632             'concurrent_view_count': int,
5633         },
5634         'params': {
5635             'skip_download': True,
5636         },
5637         'expected_warnings': ['Ignoring subtitle tracks found in '],
5638     }, {
5639         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
5640         'info_dict': {
5641             'id': 'a48o2S1cPoo',
5642             'ext': 'mp4',
5643             'title': 'The Young Turks - Live Main Show',
5644             'uploader': 'The Young Turks',
5645             'uploader_id': 'TheYoungTurks',
5646             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
5647             'upload_date': '20150715',
5648             'license': 'Standard YouTube License',
5649             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
5650             'categories': ['News & Politics'],
5651             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
5652             'like_count': int,
5653         },
5654         'params': {
5655             'skip_download': True,
5656         },
5657         'only_matching': True,
5658     }, {
5659         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
5660         'only_matching': True,
5661     }, {
5662         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
5663         'only_matching': True,
5664     }, {
5665         'note': 'A channel that is not live. Should raise error',
5666         'url': 'https://www.youtube.com/user/numberphile/live',
5667         'only_matching': True,
5668     }, {
5669         'url': 'https://www.youtube.com/feed/trending',
5670         'only_matching': True,
5671     }, {
5672         'url': 'https://www.youtube.com/feed/library',
5673         'only_matching': True,
5674     }, {
5675         'url': 'https://www.youtube.com/feed/history',
5676         'only_matching': True,
5677     }, {
5678         'url': 'https://www.youtube.com/feed/subscriptions',
5679         'only_matching': True,
5680     }, {
5681         'url': 'https://www.youtube.com/feed/watch_later',
5682         'only_matching': True,
5683     }, {
5684         'note': 'Recommended - redirects to home page.',
5685         'url': 'https://www.youtube.com/feed/recommended',
5686         'only_matching': True,
5687     }, {
5688         'note': 'inline playlist with not always working continuations',
5689         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
5690         'only_matching': True,
5691     }, {
5692         'url': 'https://www.youtube.com/course',
5693         'only_matching': True,
5694     }, {
5695         'url': 'https://www.youtube.com/zsecurity',
5696         'only_matching': True,
5697     }, {
5698         'url': 'http://www.youtube.com/NASAgovVideo/videos',
5699         'only_matching': True,
5700     }, {
5701         'url': 'https://www.youtube.com/TheYoungTurks/live',
5702         'only_matching': True,
5703     }, {
5704         'url': 'https://www.youtube.com/hashtag/cctv9',
5705         'info_dict': {
5706             'id': 'cctv9',
5707             'title': '#cctv9',
5708             'tags': [],
5709         },
5710         'playlist_mincount': 300,  # not consistent but should be over 300
5711     }, {
5712         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
5713         'only_matching': True,
5714     }, {
5715         'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
5716         'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5717         'only_matching': True
5718     }, {
5719         'note': '/browse/ should redirect to /channel/',
5720         'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
5721         'only_matching': True
5722     }, {
5723         'note': 'VLPL, should redirect to playlist?list=PL...',
5724         'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5725         'info_dict': {
5726             'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
5727             'uploader': 'NoCopyrightSounds',
5728             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
5729             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5730             'title': 'NCS : All Releases 💿',
5731             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5732             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
5733             'modified_date': r're:\d{8}',
5734             'view_count': int,
5735             'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
5736             'tags': [],
5737             'channel': 'NoCopyrightSounds',
5738             'availability': 'public',
5739         },
5740         'playlist_mincount': 166,
5741         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
5742     }, {
5743         'note': 'Topic, should redirect to playlist?list=UU...',
5744         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5745         'info_dict': {
5746             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5747             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5748             'title': 'Uploads from Royalty Free Music - Topic',
5749             'uploader': 'Royalty Free Music - Topic',
5750             'tags': [],
5751             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5752             'channel': 'Royalty Free Music - Topic',
5753             'view_count': int,
5754             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5755             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5756             'modified_date': r're:\d{8}',
5757             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5758             'description': '',
5759             'availability': 'public',
5760         },
5761         'playlist_mincount': 101,
5762     }, {
5763         # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
5764         # Treat as a general feed
5765         'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
5766         'info_dict': {
5767             'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
5768             'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
5769             'tags': [],
5770         },
5771         'playlist_mincount': 9,
5772     }, {
5773         'note': 'Youtube music Album',
5774         'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
5775         'info_dict': {
5776             'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
5777             'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
5778             'tags': [],
5779             'view_count': int,
5780             'description': '',
5781             'availability': 'unlisted',
5782             'modified_date': r're:\d{8}',
5783         },
5784         'playlist_count': 50,
5785     }, {
5786         'note': 'unlisted single video playlist',
5787         'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5788         'info_dict': {
5789             'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5790             'uploader': 'colethedj',
5791             'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
5792             'title': 'yt-dlp unlisted playlist test',
5793             'availability': 'unlisted',
5794             'tags': [],
5795             'modified_date': '20220418',
5796             'channel': 'colethedj',
5797             'view_count': int,
5798             'description': '',
5799             'uploader_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5800             'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
5801             'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
5802         },
5803         'playlist_count': 1,
5804     }, {
5805         'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
5806         'url': 'https://www.youtube.com/feed/recommended',
5807         'info_dict': {
5808             'id': 'recommended',
5809             'title': 'recommended',
5810             'tags': [],
5811         },
5812         'playlist_mincount': 50,
5813         'params': {
5814             'skip_download': True,
5815             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5816         },
5817     }, {
5818         'note': 'API Fallback: /videos tab, sorted by oldest first',
5819         'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
5820         'info_dict': {
5821             'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5822             'title': 'Cody\'sLab - Videos',
5823             'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
5824             'uploader': 'Cody\'sLab',
5825             'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5826             'channel': 'Cody\'sLab',
5827             'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
5828             'tags': [],
5829             'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5830             'uploader_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
5831             'channel_follower_count': int
5832         },
5833         'playlist_mincount': 650,
5834         'params': {
5835             'skip_download': True,
5836             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5837         },
5838         'skip': 'Query for sorting no longer works',
5839     }, {
5840         'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
5841         'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
5842         'info_dict': {
5843             'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
5844             'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5845             'title': 'Uploads from Royalty Free Music - Topic',
5846             'uploader': 'Royalty Free Music - Topic',
5847             'modified_date': r're:\d{8}',
5848             'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
5849             'description': '',
5850             'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5851             'tags': [],
5852             'channel': 'Royalty Free Music - Topic',
5853             'view_count': int,
5854             'uploader_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
5855             'availability': 'public',
5856         },
5857         'playlist_mincount': 101,
5858         'params': {
5859             'skip_download': True,
5860             'extractor_args': {'youtubetab': {'skip': ['webpage']}}
5861         },
5862     }, {
5863         'note': 'non-standard redirect to regional channel',
5864         'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
5865         'only_matching': True
5866     }, {
5867         'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
5868         'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5869         'info_dict': {
5870             'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
5871             'modified_date': '20220407',
5872             'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5873             'tags': [],
5874             'uploader_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5875             'uploader': 'pukkandan',
5876             'availability': 'unlisted',
5877             'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
5878             'channel': 'pukkandan',
5879             'description': 'Test for collaborative playlist',
5880             'title': 'yt-dlp test - collaborative playlist',
5881             'view_count': int,
5882             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
5883         },
5884         'playlist_mincount': 2
5885     }, {
5886         'note': 'translated tab name',
5887         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
5888         'info_dict': {
5889             'id': 'UCiu-3thuViMebBjw_5nWYrA',
5890             'tags': [],
5891             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5892             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5893             'description': 'test description',
5894             'title': 'cole-dlp-test-acc - 再生リスト',
5895             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5896             'uploader': 'cole-dlp-test-acc',
5897             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5898             'channel': 'cole-dlp-test-acc',
5899         },
5900         'playlist_mincount': 1,
5901         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5902         'expected_warnings': ['Preferring "ja"'],
5903     }, {
5904         # XXX: this should really check flat playlist entries, but the test suite doesn't support that
5905         'note': 'preferred lang set with playlist with translated video titles',
5906         'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5907         'info_dict': {
5908             'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
5909             'tags': [],
5910             'view_count': int,
5911             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5912             'uploader': 'cole-dlp-test-acc',
5913             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
5914             'channel': 'cole-dlp-test-acc',
5915             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
5916             'description': 'test',
5917             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
5918             'title': 'dlp test playlist',
5919             'availability': 'public',
5920         },
5921         'playlist_mincount': 1,
5922         'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
5923         'expected_warnings': ['Preferring "ja"'],
5924     }, {
5925         # shorts audio pivot for 2GtVksBMYFM.
5926         'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
5927         'info_dict': {
5928             'id': 'sfv_audio_pivot',
5929             'title': 'sfv_audio_pivot',
5930             'tags': [],
5931         },
5932         'playlist_mincount': 50,
5933
5934     }, {
5935         # Channel with a real live tab (not to be mistaken with streams tab)
5936         # Do not treat like it should redirect to live stream
5937         'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
5938         'info_dict': {
5939             'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
5940             'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
5941             'tags': [],
5942         },
5943         'playlist_mincount': 20,
5944     }, {
5945         # Tab name is not the same as tab id
5946         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
5947         'info_dict': {
5948             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5949             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
5950             'tags': [],
5951         },
5952         'playlist_mincount': 8,
5953     }, {
5954         # Home tab id is literally home. Not to get mistaken with featured
5955         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
5956         'info_dict': {
5957             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
5958             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
5959             'tags': [],
5960         },
5961         'playlist_mincount': 8,
5962     }, {
5963         # Should get three playlists for videos, shorts and streams tabs
5964         'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5965         'info_dict': {
5966             'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5967             'title': 'Polka Ch. 尾丸ポルカ',
5968             'channel_follower_count': int,
5969             'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5970             'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5971             'uploader': 'Polka Ch. 尾丸ポルカ',
5972             'description': 'md5:3b8df1ac5af337aa206e37ee3d181ec9',
5973             'channel': 'Polka Ch. 尾丸ポルカ',
5974             'tags': 'count:35',
5975             'uploader_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
5976             'uploader_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
5977         },
5978         'playlist_count': 3,
5979     }, {
5980         # Shorts tab with channel with handle
5981         'url': 'https://www.youtube.com/@NotJustBikes/shorts',
5982         'info_dict': {
5983             'id': 'UC0intLFzLaudFG-xAvUEO-A',
5984             'title': 'Not Just Bikes - Shorts',
5985             'tags': 'count:12',
5986             'uploader': 'Not Just Bikes',
5987             'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5988             'description': 'md5:7513148b1f02b924783157d84c4ea555',
5989             'channel_follower_count': int,
5990             'uploader_id': 'UC0intLFzLaudFG-xAvUEO-A',
5991             'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
5992             'uploader_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
5993             'channel': 'Not Just Bikes',
5994         },
5995         'playlist_mincount': 10,
5996     }, {
5997         # Streams tab
5998         'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
5999         'info_dict': {
6000             'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
6001             'title': '中村悠一 - Live',
6002             'tags': 'count:7',
6003             'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
6004             'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
6005             'uploader_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
6006             'channel': '中村悠一',
6007             'uploader_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
6008             'channel_follower_count': int,
6009             'uploader': '中村悠一',
6010             'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
6011         },
6012         'playlist_mincount': 60,
6013     }, {
6014         # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
6015         # See test_youtube_lists
6016         'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
6017         'only_matching': True,
6018     }, {
6019         # No uploads and no UCID given. Should fail with no uploads error
6020         # See test_youtube_lists
6021         'url': 'https://www.youtube.com/news',
6022         'only_matching': True
6023     }, {
6024         # No videos tab but has a shorts tab
6025         'url': 'https://www.youtube.com/c/TKFShorts',
6026         'info_dict': {
6027             'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6028             'title': 'Shorts Break - Shorts',
6029             'tags': 'count:32',
6030             'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6031             'channel': 'Shorts Break',
6032             'description': 'md5:a6c234cf3d50d878ef8721e34457cd11',
6033             'uploader': 'Shorts Break',
6034             'channel_follower_count': int,
6035             'uploader_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
6036             'uploader_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
6037             'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
6038         },
6039         'playlist_mincount': 30,
6040     }, {
6041         # Trending Now Tab. tab id is empty
6042         'url': 'https://www.youtube.com/feed/trending',
6043         'info_dict': {
6044             'id': 'trending',
6045             'title': 'trending - Now',
6046             'tags': [],
6047         },
6048         'playlist_mincount': 30,
6049     }, {
6050         # Trending Gaming Tab. tab id is empty
6051         'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
6052         'info_dict': {
6053             'id': 'trending',
6054             'title': 'trending - Gaming',
6055             'tags': [],
6056         },
6057         'playlist_mincount': 30,
6058     }, {
6059         # Shorts url result in shorts tab
6060         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
6061         'info_dict': {
6062             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6063             'title': 'cole-dlp-test-acc - Shorts',
6064             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
6065             'channel': 'cole-dlp-test-acc',
6066             'description': 'test description',
6067             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6068             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6069             'tags': [],
6070             'uploader': 'cole-dlp-test-acc',
6071             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6072
6073         },
6074         'playlist': [{
6075             'info_dict': {
6076                 '_type': 'url',
6077                 'ie_key': 'Youtube',
6078                 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
6079                 'id': 'sSM9J5YH_60',
6080                 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6081                 'title': 'SHORT short',
6082                 'channel': 'cole-dlp-test-acc',
6083                 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6084                 'view_count': int,
6085                 'thumbnails': list,
6086             }
6087         }],
6088         'params': {'extract_flat': True},
6089     }, {
6090         # Live video status should be extracted
6091         'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
6092         'info_dict': {
6093             'id': 'UCQvWX73GQygcwXOTSf_VDVg',
6094             'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live',  # TODO, should be Minecraft - Live or Minecraft - Topic - Live
6095             'tags': []
6096         },
6097         'playlist': [{
6098             'info_dict': {
6099                 '_type': 'url',
6100                 'ie_key': 'Youtube',
6101                 'url': 'startswith:https://www.youtube.com/watch?v=',
6102                 'id': str,
6103                 'title': str,
6104                 'live_status': 'is_live',
6105                 'channel_id': str,
6106                 'channel_url': str,
6107                 'concurrent_view_count': int,
6108                 'channel': str,
6109             }
6110         }],
6111         'params': {'extract_flat': True, 'playlist_items': '1'},
6112         'playlist_mincount': 1
6113     }, {
6114         # Channel renderer metadata. Contains number of videos on the channel
6115         'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
6116         'info_dict': {
6117             'id': 'UCiu-3thuViMebBjw_5nWYrA',
6118             'title': 'cole-dlp-test-acc - Channels',
6119             'uploader_id': 'UCiu-3thuViMebBjw_5nWYrA',
6120             'channel': 'cole-dlp-test-acc',
6121             'description': 'test description',
6122             'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
6123             'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6124             'tags': [],
6125             'uploader': 'cole-dlp-test-acc',
6126             'uploader_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
6127
6128         },
6129         'playlist': [{
6130             'info_dict': {
6131                 '_type': 'url',
6132                 'ie_key': 'YoutubeTab',
6133                 'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6134                 'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6135                 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6136                 'title': 'PewDiePie',
6137                 'channel': 'PewDiePie',
6138                 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
6139                 'thumbnails': list,
6140                 'channel_follower_count': int,
6141                 'playlist_count': int
6142             }
6143         }],
6144         'params': {'extract_flat': True},
6145     }, {
6146         'url': 'https://www.youtube.com/@3blue1brown/about',
6147         'info_dict': {
6148             'id': 'UCYO_jab_esuFRV4b17AJtAw',
6149             'tags': ['Mathematics'],
6150             'title': '3Blue1Brown - About',
6151             'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
6152             'channel_follower_count': int,
6153             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
6154             'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
6155             'channel': '3Blue1Brown',
6156             'uploader': '3Blue1Brown',
6157             'view_count': int,
6158             'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
6159             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
6160         },
6161         'playlist_count': 0,
6162     }]
6163
6164     @classmethod
6165     def suitable(cls, url):
6166         return False if YoutubeIE.suitable(url) else super().suitable(url)
6167
6168     _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
6169
6170     def _get_url_mobj(self, url):
6171         mobj = self._URL_RE.match(url).groupdict()
6172         mobj.update((k, '') for k, v in mobj.items() if v is None)
6173         return mobj
6174
6175     def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
6176         tab_name = (tab.get('title') or '').lower()
6177         tab_url = urljoin(base_url, traverse_obj(
6178             tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
6179
6180         tab_id = (tab_url and self._get_url_mobj(tab_url)['tab'][1:]
6181                   or traverse_obj(tab, 'tabIdentifier', expected_type=str))
6182         if tab_id:
6183             return {
6184                 'TAB_ID_SPONSORSHIPS': 'membership',
6185             }.get(tab_id, tab_id), tab_name
6186
6187         # Fallback to tab name if we cannot get the tab id.
6188         # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
6189         # Note that in the case of translated tab name this may result in an empty string, which we don't want.
6190         if tab_name:
6191             self.write_debug(f'Falling back to selected tab name: {tab_name}')
6192         return {
6193             'home': 'featured',
6194             'live': 'streams',
6195         }.get(tab_name, tab_name), tab_name
6196
6197     def _has_tab(self, tabs, tab_id):
6198         return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
6199
6200     @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
6201     def _real_extract(self, url, smuggled_data):
6202         item_id = self._match_id(url)
6203         url = urllib.parse.urlunparse(
6204             urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
6205         compat_opts = self.get_param('compat_opts', [])
6206
6207         mobj = self._get_url_mobj(url)
6208         pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
6209         if is_channel and smuggled_data.get('is_music_url'):
6210             if item_id[:2] == 'VL':  # Youtube music VL channels have an equivalent playlist
6211                 return self.url_result(
6212                     f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
6213             elif item_id[:2] == 'MP':  # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
6214                 mdata = self._extract_tab_endpoint(
6215                     f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
6216                 murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
6217                                     get_all=False, expected_type=str)
6218                 if not murl:
6219                     raise ExtractorError('Failed to resolve album to playlist')
6220                 return self.url_result(murl, YoutubeTabIE)
6221             elif mobj['channel_type'] == 'browse':  # Youtube music /browse/ should be changed to /channel/
6222                 return self.url_result(
6223                     f'https://music.youtube.com/channel/{item_id}{tab}{post}', YoutubeTabIE, item_id)
6224
6225         original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
6226         if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
6227             url = f'{pre}/videos{post}'
6228         if smuggled_data.get('is_music_url'):
6229             self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}')
6230
6231         # Handle both video/playlist URLs
6232         qs = parse_qs(url)
6233         video_id, playlist_id = [traverse_obj(qs, (key, 0)) for key in ('v', 'list')]
6234         if not video_id and mobj['not_channel'].startswith('watch'):
6235             if not playlist_id:
6236                 # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
6237                 raise ExtractorError('A video URL was given without video ID', expected=True)
6238             # Common mistake: https://www.youtube.com/watch?list=playlist_id
6239             self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
6240             return self.url_result(
6241                 f'https://www.youtube.com/playlist?list={playlist_id}', YoutubeTabIE, playlist_id)
6242
6243         if not self._yes_playlist(playlist_id, video_id):
6244             return self.url_result(
6245                 f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6246
6247         data, ytcfg = self._extract_data(url, display_id)
6248
6249         # YouTube may provide a non-standard redirect to the regional channel
6250         # See: https://github.com/yt-dlp/yt-dlp/issues/2694
6251         # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
6252         redirect_url = traverse_obj(
6253             data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
6254         if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
6255             redirect_url = ''.join((urljoin('https://www.youtube.com', redirect_url), tab, post))
6256             self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
6257             return self.url_result(redirect_url, YoutubeTabIE)
6258
6259         tabs, extra_tabs = self._extract_tab_renderers(data), []
6260         if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
6261             selected_tab = self._extract_selected_tab(tabs)
6262             selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url)  # NB: Name may be translated
6263             self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
6264
6265             if not original_tab_id and selected_tab_name:
6266                 self.to_screen('Downloading all uploads of the channel. '
6267                                'To download only the videos in a specific tab, pass the tab\'s URL')
6268                 if self._has_tab(tabs, 'streams'):
6269                     extra_tabs.append(''.join((pre, '/streams', post)))
6270                 if self._has_tab(tabs, 'shorts'):
6271                     extra_tabs.append(''.join((pre, '/shorts', post)))
6272                 # XXX: Members-only tab should also be extracted
6273
6274                 if not extra_tabs and selected_tab_id != 'videos':
6275                     # Channel does not have streams, shorts or videos tabs
6276                     if item_id[:2] != 'UC':
6277                         raise ExtractorError('This channel has no uploads', expected=True)
6278
6279                     # Topic channels don't have /videos. Use the equivalent playlist instead
6280                     pl_id = f'UU{item_id[2:]}'
6281                     pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
6282                     try:
6283                         data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
6284                     except ExtractorError:
6285                         raise ExtractorError('This channel has no uploads', expected=True)
6286                     else:
6287                         item_id, url = pl_id, pl_url
6288                         self.to_screen(
6289                             f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
6290
6291                 elif extra_tabs and selected_tab_id != 'videos':
6292                     # When there are shorts/live tabs but not videos tab
6293                     url, data = f'{pre}{post}', None
6294
6295             elif (original_tab_id or 'videos') != selected_tab_id:
6296                 if original_tab_id == 'live':
6297                     # Live tab should have redirected to the video
6298                     # Except in the case the channel has an actual live tab
6299                     # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
6300                     raise UserNotLive(video_id=item_id)
6301                 elif selected_tab_name:
6302                     raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
6303
6304                 # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
6305                 url = f'{pre}{post}'
6306
6307         # YouTube sometimes provides a button to reload playlist with unavailable videos.
6308         if 'no-youtube-unavailable-videos' not in compat_opts:
6309             data = self._reload_with_unavailable_videos(display_id, data, ytcfg) or data
6310         self._extract_and_report_alerts(data, only_once=True)
6311
6312         tabs, entries = self._extract_tab_renderers(data), []
6313         if tabs:
6314             entries = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
6315             entries[0].update({
6316                 'extractor_key': YoutubeTabIE.ie_key(),
6317                 'extractor': YoutubeTabIE.IE_NAME,
6318                 'webpage_url': url,
6319             })
6320         if self.get_param('playlist_items') == '0':
6321             entries.extend(self.url_result(u, YoutubeTabIE) for u in extra_tabs)
6322         else:  # Users expect to get all `video_id`s even with `--flat-playlist`. So don't return `url_result`
6323             entries.extend(map(self._real_extract, extra_tabs))
6324
6325         if len(entries) == 1:
6326             return entries[0]
6327         elif entries:
6328             metadata = self._extract_metadata_from_tabs(item_id, data)
6329             uploads_url = 'the Uploads (UU) playlist URL'
6330             if try_get(metadata, lambda x: x['channel_id'].startswith('UC')):
6331                 uploads_url = f'https://www.youtube.com/playlist?list=UU{metadata["channel_id"][2:]}'
6332             self.to_screen(
6333                 'Downloading as multiple playlists, separated by tabs. '
6334                 f'To download as a single playlist instead, pass {uploads_url}')
6335             return self.playlist_result(entries, item_id, **metadata)
6336
6337         # Inline playlist
6338         playlist = traverse_obj(
6339             data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
6340         if playlist:
6341             return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
6342
6343         video_id = traverse_obj(
6344             data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
6345         if video_id:
6346             if tab != '/live':  # live tab is expected to redirect to video
6347                 self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
6348             return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
6349
6350         raise ExtractorError('Unable to recognize tab page')
6351
6352
6353 class YoutubePlaylistIE(InfoExtractor):
6354     IE_DESC = 'YouTube playlists'
6355     _VALID_URL = r'''(?x)(?:
6356                         (?:https?://)?
6357                         (?:\w+\.)?
6358                         (?:
6359                             (?:
6360                                 youtube(?:kids)?\.com|
6361                                 %(invidious)s
6362                             )
6363                             /.*?\?.*?\blist=
6364                         )?
6365                         (?P<id>%(playlist_id)s)
6366                      )''' % {
6367         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
6368         'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
6369     }
6370     IE_NAME = 'youtube:playlist'
6371     _TESTS = [{
6372         'note': 'issue #673',
6373         'url': 'PLBB231211A4F62143',
6374         'info_dict': {
6375             'title': '[OLD]Team Fortress 2 (Class-based LP)',
6376             'id': 'PLBB231211A4F62143',
6377             'uploader': 'Wickman',
6378             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6379             'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
6380             'view_count': int,
6381             'uploader_url': 'https://www.youtube.com/c/WickmanVT',
6382             'modified_date': r're:\d{8}',
6383             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
6384             'channel': 'Wickman',
6385             'tags': [],
6386             'channel_url': 'https://www.youtube.com/c/WickmanVT',
6387             'availability': 'public',
6388         },
6389         'playlist_mincount': 29,
6390     }, {
6391         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6392         'info_dict': {
6393             'title': 'YDL_safe_search',
6394             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
6395         },
6396         'playlist_count': 2,
6397         'skip': 'This playlist is private',
6398     }, {
6399         'note': 'embedded',
6400         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6401         'playlist_count': 4,
6402         'info_dict': {
6403             'title': 'JODA15',
6404             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
6405             'uploader': 'milan',
6406             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6407             'description': '',
6408             'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6409             'tags': [],
6410             'modified_date': '20140919',
6411             'view_count': int,
6412             'channel': 'milan',
6413             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
6414             'uploader_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
6415             'availability': 'public',
6416         },
6417         'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden'],
6418     }, {
6419         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6420         'playlist_mincount': 455,
6421         'info_dict': {
6422             'title': '2018 Chinese New Singles (11/6 updated)',
6423             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
6424             'uploader': 'LBK',
6425             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6426             'description': 'md5:da521864744d60a198e3a88af4db0d9d',
6427             'channel': 'LBK',
6428             'view_count': int,
6429             'channel_url': 'https://www.youtube.com/c/愛低音的國王',
6430             'tags': [],
6431             'uploader_url': 'https://www.youtube.com/c/愛低音的國王',
6432             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
6433             'modified_date': r're:\d{8}',
6434             'availability': 'public',
6435         },
6436         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
6437     }, {
6438         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
6439         'only_matching': True,
6440     }, {
6441         # music album playlist
6442         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
6443         'only_matching': True,
6444     }]
6445
6446     @classmethod
6447     def suitable(cls, url):
6448         if YoutubeTabIE.suitable(url):
6449             return False
6450         from ..utils import parse_qs
6451         qs = parse_qs(url)
6452         if qs.get('v', [None])[0]:
6453             return False
6454         return super().suitable(url)
6455
6456     def _real_extract(self, url):
6457         playlist_id = self._match_id(url)
6458         is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
6459         url = update_url_query(
6460             'https://www.youtube.com/playlist',
6461             parse_qs(url) or {'list': playlist_id})
6462         if is_music_url:
6463             url = smuggle_url(url, {'is_music_url': True})
6464         return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6465
6466
6467 class YoutubeYtBeIE(InfoExtractor):
6468     IE_DESC = 'youtu.be'
6469     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
6470     _TESTS = [{
6471         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
6472         'info_dict': {
6473             'id': 'yeWKywCrFtk',
6474             'ext': 'mp4',
6475             'title': 'Small Scale Baler and Braiding Rugs',
6476             'uploader': 'Backus-Page House Museum',
6477             'uploader_id': 'backuspagemuseum',
6478             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
6479             'upload_date': '20161008',
6480             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
6481             'categories': ['Nonprofits & Activism'],
6482             'tags': list,
6483             'like_count': int,
6484             'age_limit': 0,
6485             'playable_in_embed': True,
6486             'thumbnail': 'https://i.ytimg.com/vi_webp/yeWKywCrFtk/maxresdefault.webp',
6487             'channel': 'Backus-Page House Museum',
6488             'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
6489             'live_status': 'not_live',
6490             'view_count': int,
6491             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
6492             'availability': 'public',
6493             'duration': 59,
6494             'comment_count': int,
6495             'channel_follower_count': int
6496         },
6497         'params': {
6498             'noplaylist': True,
6499             'skip_download': True,
6500         },
6501     }, {
6502         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
6503         'only_matching': True,
6504     }]
6505
6506     def _real_extract(self, url):
6507         mobj = self._match_valid_url(url)
6508         video_id = mobj.group('id')
6509         playlist_id = mobj.group('playlist_id')
6510         return self.url_result(
6511             update_url_query('https://www.youtube.com/watch', {
6512                 'v': video_id,
6513                 'list': playlist_id,
6514                 'feature': 'youtu.be',
6515             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
6516
6517
6518 class YoutubeLivestreamEmbedIE(InfoExtractor):
6519     IE_DESC = 'YouTube livestream embeds'
6520     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
6521     _TESTS = [{
6522         'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
6523         'only_matching': True,
6524     }]
6525
6526     def _real_extract(self, url):
6527         channel_id = self._match_id(url)
6528         return self.url_result(
6529             f'https://www.youtube.com/channel/{channel_id}/live',
6530             ie=YoutubeTabIE.ie_key(), video_id=channel_id)
6531
6532
6533 class YoutubeYtUserIE(InfoExtractor):
6534     IE_DESC = 'YouTube user videos; "ytuser:" prefix'
6535     IE_NAME = 'youtube:user'
6536     _VALID_URL = r'ytuser:(?P<id>.+)'
6537     _TESTS = [{
6538         'url': 'ytuser:phihag',
6539         'only_matching': True,
6540     }]
6541
6542     def _real_extract(self, url):
6543         user_id = self._match_id(url)
6544         return self.url_result(f'https://www.youtube.com/user/{user_id}', YoutubeTabIE, user_id)
6545
6546
6547 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
6548     IE_NAME = 'youtube:favorites'
6549     IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
6550     _VALID_URL = r':ytfav(?:ou?rite)?s?'
6551     _LOGIN_REQUIRED = True
6552     _TESTS = [{
6553         'url': ':ytfav',
6554         'only_matching': True,
6555     }, {
6556         'url': ':ytfavorites',
6557         'only_matching': True,
6558     }]
6559
6560     def _real_extract(self, url):
6561         return self.url_result(
6562             'https://www.youtube.com/playlist?list=LL',
6563             ie=YoutubeTabIE.ie_key())
6564
6565
6566 class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
6567     IE_NAME = 'youtube:notif'
6568     IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
6569     _VALID_URL = r':ytnotif(?:ication)?s?'
6570     _LOGIN_REQUIRED = True
6571     _TESTS = [{
6572         'url': ':ytnotif',
6573         'only_matching': True,
6574     }, {
6575         'url': ':ytnotifications',
6576         'only_matching': True,
6577     }]
6578
6579     def _extract_notification_menu(self, response, continuation_list):
6580         notification_list = traverse_obj(
6581             response,
6582             ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
6583             ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
6584             expected_type=list) or []
6585         continuation_list[0] = None
6586         for item in notification_list:
6587             entry = self._extract_notification_renderer(item.get('notificationRenderer'))
6588             if entry:
6589                 yield entry
6590             continuation = item.get('continuationItemRenderer')
6591             if continuation:
6592                 continuation_list[0] = continuation
6593
6594     def _extract_notification_renderer(self, notification):
6595         video_id = traverse_obj(
6596             notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
6597         url = f'https://www.youtube.com/watch?v={video_id}'
6598         channel_id = None
6599         if not video_id:
6600             browse_ep = traverse_obj(
6601                 notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
6602             channel_id = traverse_obj(browse_ep, 'browseId', expected_type=str)
6603             post_id = self._search_regex(
6604                 r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
6605                 'post id', default=None)
6606             if not channel_id or not post_id:
6607                 return
6608             # The direct /post url redirects to this in the browser
6609             url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
6610
6611         channel = traverse_obj(
6612             notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
6613             expected_type=str)
6614         notification_title = self._get_text(notification, 'shortMessage')
6615         if notification_title:
6616             notification_title = notification_title.replace('\xad', '')  # remove soft hyphens
6617         # TODO: handle recommended videos
6618         title = self._search_regex(
6619             rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
6620             'video title', default=None)
6621         timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
6622                      if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
6623                      else None)
6624         return {
6625             '_type': 'url',
6626             'url': url,
6627             'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
6628             'video_id': video_id,
6629             'title': title,
6630             'channel_id': channel_id,
6631             'channel': channel,
6632             'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
6633             'timestamp': timestamp,
6634         }
6635
6636     def _notification_menu_entries(self, ytcfg):
6637         continuation_list = [None]
6638         response = None
6639         for page in itertools.count(1):
6640             ctoken = traverse_obj(
6641                 continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
6642             response = self._extract_response(
6643                 item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
6644                 ep='notification/get_notification_menu', check_get_keys='actions',
6645                 headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
6646             yield from self._extract_notification_menu(response, continuation_list)
6647             if not continuation_list[0]:
6648                 break
6649
6650     def _real_extract(self, url):
6651         display_id = 'notifications'
6652         ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
6653         self._report_playlist_authcheck(ytcfg)
6654         return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
6655
6656
6657 class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6658     IE_DESC = 'YouTube search'
6659     IE_NAME = 'youtube:search'
6660     _SEARCH_KEY = 'ytsearch'
6661     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
6662     _TESTS = [{
6663         'url': 'ytsearch5:youtube-dl test video',
6664         'playlist_count': 5,
6665         'info_dict': {
6666             'id': 'youtube-dl test video',
6667             'title': 'youtube-dl test video',
6668         }
6669     }]
6670
6671
6672 class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
6673     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
6674     _SEARCH_KEY = 'ytsearchdate'
6675     IE_DESC = 'YouTube search, newest videos first'
6676     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
6677     _TESTS = [{
6678         'url': 'ytsearchdate5:youtube-dl test video',
6679         'playlist_count': 5,
6680         'info_dict': {
6681             'id': 'youtube-dl test video',
6682             'title': 'youtube-dl test video',
6683         }
6684     }]
6685
6686
6687 class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
6688     IE_DESC = 'YouTube search URLs with sorting and filter support'
6689     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
6690     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6691     _TESTS = [{
6692         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
6693         'playlist_mincount': 5,
6694         'info_dict': {
6695             'id': 'youtube-dl test video',
6696             'title': 'youtube-dl test video',
6697         }
6698     }, {
6699         'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
6700         'playlist_mincount': 5,
6701         'info_dict': {
6702             'id': 'python',
6703             'title': 'python',
6704         }
6705     }, {
6706         'url': 'https://www.youtube.com/results?search_query=%23cats',
6707         'playlist_mincount': 1,
6708         'info_dict': {
6709             'id': '#cats',
6710             'title': '#cats',
6711             # The test suite does not have support for nested playlists
6712             # 'entries': [{
6713             #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
6714             #     'title': '#cats',
6715             # }],
6716         },
6717     }, {
6718         # Channel results
6719         'url': 'https://www.youtube.com/results?search_query=kurzgesagt&sp=EgIQAg%253D%253D',
6720         'info_dict': {
6721             'id': 'kurzgesagt',
6722             'title': 'kurzgesagt',
6723         },
6724         'playlist': [{
6725             'info_dict': {
6726                 '_type': 'url',
6727                 'id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6728                 'url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6729                 'ie_key': 'YoutubeTab',
6730                 'channel': 'Kurzgesagt – In a Nutshell',
6731                 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
6732                 'title': 'Kurzgesagt – In a Nutshell',
6733                 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
6734                 'playlist_count': int,  # XXX: should have a way of saying > 1
6735                 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
6736                 'thumbnails': list
6737             }
6738         }],
6739         'params': {'extract_flat': True, 'playlist_items': '1'},
6740         'playlist_mincount': 1,
6741     }, {
6742         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
6743         'only_matching': True,
6744     }]
6745
6746     def _real_extract(self, url):
6747         qs = parse_qs(url)
6748         query = (qs.get('search_query') or qs.get('q'))[0]
6749         return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
6750
6751
6752 class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
6753     IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
6754     IE_NAME = 'youtube:music:search_url'
6755     _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
6756     _TESTS = [{
6757         'url': 'https://music.youtube.com/search?q=royalty+free+music',
6758         'playlist_count': 16,
6759         'info_dict': {
6760             'id': 'royalty free music',
6761             'title': 'royalty free music',
6762         }
6763     }, {
6764         'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
6765         'playlist_mincount': 30,
6766         'info_dict': {
6767             'id': 'royalty free music - songs',
6768             'title': 'royalty free music - songs',
6769         },
6770         'params': {'extract_flat': 'in_playlist'}
6771     }, {
6772         'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
6773         'playlist_mincount': 30,
6774         'info_dict': {
6775             'id': 'royalty free music - community playlists',
6776             'title': 'royalty free music - community playlists',
6777         },
6778         'params': {'extract_flat': 'in_playlist'}
6779     }]
6780
6781     _SECTIONS = {
6782         'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
6783         'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
6784         'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
6785         'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
6786         'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
6787         'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
6788     }
6789
6790     def _real_extract(self, url):
6791         qs = parse_qs(url)
6792         query = (qs.get('search_query') or qs.get('q'))[0]
6793         params = qs.get('sp', (None,))[0]
6794         if params:
6795             section = next((k for k, v in self._SECTIONS.items() if v == params), params)
6796         else:
6797             section = urllib.parse.unquote_plus((url.split('#') + [''])[1]).lower()
6798             params = self._SECTIONS.get(section)
6799             if not params:
6800                 section = None
6801         title = join_nonempty(query, section, delim=' - ')
6802         return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
6803
6804
6805 class YoutubeFeedsInfoExtractor(InfoExtractor):
6806     """
6807     Base class for feed extractors
6808     Subclasses must re-define the _FEED_NAME property.
6809     """
6810     _LOGIN_REQUIRED = True
6811     _FEED_NAME = 'feeds'
6812
6813     def _real_initialize(self):
6814         YoutubeBaseInfoExtractor._check_login_required(self)
6815
6816     @classproperty
6817     def IE_NAME(self):
6818         return f'youtube:{self._FEED_NAME}'
6819
6820     def _real_extract(self, url):
6821         return self.url_result(
6822             f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
6823
6824
6825 class YoutubeWatchLaterIE(InfoExtractor):
6826     IE_NAME = 'youtube:watchlater'
6827     IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
6828     _VALID_URL = r':ytwatchlater'
6829     _TESTS = [{
6830         'url': ':ytwatchlater',
6831         'only_matching': True,
6832     }]
6833
6834     def _real_extract(self, url):
6835         return self.url_result(
6836             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
6837
6838
6839 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
6840     IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
6841     _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
6842     _FEED_NAME = 'recommended'
6843     _LOGIN_REQUIRED = False
6844     _TESTS = [{
6845         'url': ':ytrec',
6846         'only_matching': True,
6847     }, {
6848         'url': ':ytrecommended',
6849         'only_matching': True,
6850     }, {
6851         'url': 'https://youtube.com',
6852         'only_matching': True,
6853     }]
6854
6855
6856 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
6857     IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
6858     _VALID_URL = r':ytsub(?:scription)?s?'
6859     _FEED_NAME = 'subscriptions'
6860     _TESTS = [{
6861         'url': ':ytsubs',
6862         'only_matching': True,
6863     }, {
6864         'url': ':ytsubscriptions',
6865         'only_matching': True,
6866     }]
6867
6868
6869 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
6870     IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
6871     _VALID_URL = r':ythis(?:tory)?'
6872     _FEED_NAME = 'history'
6873     _TESTS = [{
6874         'url': ':ythistory',
6875         'only_matching': True,
6876     }]
6877
6878
6879 class YoutubeStoriesIE(InfoExtractor):
6880     IE_DESC = 'YouTube channel stories; "ytstories:" prefix'
6881     IE_NAME = 'youtube:stories'
6882     _VALID_URL = r'ytstories:UC(?P<id>[A-Za-z0-9_-]{21}[AQgw])$'
6883     _TESTS = [{
6884         'url': 'ytstories:UCwFCb4jeqaKWnciAYM-ZVHg',
6885         'only_matching': True,
6886     }]
6887
6888     def _real_extract(self, url):
6889         playlist_id = f'RLTD{self._match_id(url)}'
6890         return self.url_result(
6891             smuggle_url(f'https://www.youtube.com/playlist?list={playlist_id}&playnext=1', {'is_story': True}),
6892             ie=YoutubeTabIE, video_id=playlist_id)
6893
6894
6895 class YoutubeShortsAudioPivotIE(InfoExtractor):
6896     IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
6897     IE_NAME = 'youtube:shorts:pivot:audio'
6898     _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
6899     _TESTS = [{
6900         'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
6901         'only_matching': True,
6902     }]
6903
6904     @staticmethod
6905     def _generate_audio_pivot_params(video_id):
6906         """
6907         Generates sfv_audio_pivot browse params for this video id
6908         """
6909         pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
6910         return urllib.parse.quote(base64.b64encode(pb_params).decode())
6911
6912     def _real_extract(self, url):
6913         video_id = self._match_id(url)
6914         return self.url_result(
6915             f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
6916             ie=YoutubeTabIE)
6917
6918
6919 class YoutubeTruncatedURLIE(InfoExtractor):
6920     IE_NAME = 'youtube:truncated_url'
6921     IE_DESC = False  # Do not list
6922     _VALID_URL = r'''(?x)
6923         (?:https?://)?
6924         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
6925         (?:watch\?(?:
6926             feature=[a-z_]+|
6927             annotation_id=annotation_[^&]+|
6928             x-yt-cl=[0-9]+|
6929             hl=[^&]*|
6930             t=[0-9]+
6931         )?
6932         |
6933             attribution_link\?a=[^&]+
6934         )
6935         $
6936     '''
6937
6938     _TESTS = [{
6939         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
6940         'only_matching': True,
6941     }, {
6942         'url': 'https://www.youtube.com/watch?',
6943         'only_matching': True,
6944     }, {
6945         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
6946         'only_matching': True,
6947     }, {
6948         'url': 'https://www.youtube.com/watch?feature=foo',
6949         'only_matching': True,
6950     }, {
6951         'url': 'https://www.youtube.com/watch?hl=en-GB',
6952         'only_matching': True,
6953     }, {
6954         'url': 'https://www.youtube.com/watch?t=2372',
6955         'only_matching': True,
6956     }]
6957
6958     def _real_extract(self, url):
6959         raise ExtractorError(
6960             'Did you forget to quote the URL? Remember that & is a meta '
6961             'character in most shells, so you want to put the URL in quotes, '
6962             'like  youtube-dl '
6963             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
6964             ' or simply  youtube-dl BaW_jenozKc  .',
6965             expected=True)
6966
6967
6968 class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
6969     IE_NAME = 'youtube:clip'
6970     _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
6971     _TESTS = [{
6972         # FIXME: Other metadata should be extracted from the clip, not from the base video
6973         'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
6974         'info_dict': {
6975             'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
6976             'ext': 'mp4',
6977             'section_start': 29.0,
6978             'section_end': 39.7,
6979             'duration': 10.7,
6980             'age_limit': 0,
6981             'availability': 'public',
6982             'categories': ['Gaming'],
6983             'channel': 'Scott The Woz',
6984             'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
6985             'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
6986             'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
6987             'like_count': int,
6988             'playable_in_embed': True,
6989             'tags': 'count:17',
6990             'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
6991             'title': 'Mobile Games on Console - Scott The Woz',
6992             'upload_date': '20210920',
6993             'uploader': 'Scott The Woz',
6994             'uploader_id': 'scottthewoz',
6995             'uploader_url': 'http://www.youtube.com/user/scottthewoz',
6996             'view_count': int,
6997             'live_status': 'not_live',
6998             'channel_follower_count': int
6999         }
7000     }]
7001
7002     def _real_extract(self, url):
7003         clip_id = self._match_id(url)
7004         _, data = self._extract_webpage(url, clip_id)
7005
7006         video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
7007         if not video_id:
7008             raise ExtractorError('Unable to find video ID')
7009
7010         clip_data = traverse_obj(data, (
7011             'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
7012             'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
7013             'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
7014             'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
7015
7016         return {
7017             '_type': 'url_transparent',
7018             'url': f'https://www.youtube.com/watch?v={video_id}',
7019             'ie_key': YoutubeIE.ie_key(),
7020             'id': clip_id,
7021             'section_start': int(clip_data['startTimeMs']) / 1000,
7022             'section_end': int(clip_data['endTimeMs']) / 1000,
7023         }
7024
7025
7026 class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
7027     IE_NAME = 'youtube:consent'
7028     IE_DESC = False  # Do not list
7029     _VALID_URL = r'https?://consent\.youtube\.com/m\?'
7030     _TESTS = [{
7031         'url': 'https://consent.youtube.com/m?continue=https%3A%2F%2Fwww.youtube.com%2Flive%2FqVv6vCqciTM%3Fcbrd%3D1&gl=NL&m=0&pc=yt&hl=en&src=1',
7032         'info_dict': {
7033             'id': 'qVv6vCqciTM',
7034             'ext': 'mp4',
7035             'age_limit': 0,
7036             'uploader_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
7037             'comment_count': int,
7038             'chapters': 'count:13',
7039             'upload_date': '20221223',
7040             'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
7041             'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
7042             'uploader_url': 'http://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
7043             'like_count': int,
7044             'release_date': '20221223',
7045             'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
7046             'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
7047             'view_count': int,
7048             'playable_in_embed': True,
7049             'duration': 4438,
7050             'availability': 'public',
7051             'channel_follower_count': int,
7052             'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
7053             'categories': ['Entertainment'],
7054             'live_status': 'was_live',
7055             'release_timestamp': 1671793345,
7056             'channel': 'さなちゃんねる',
7057             'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
7058             'uploader': 'さなちゃんねる',
7059         },
7060         'add_ie': ['Youtube'],
7061         'params': {'skip_download': 'Youtube'},
7062     }]
7063
7064     def _real_extract(self, url):
7065         redirect_url = url_or_none(parse_qs(url).get('continue', [None])[-1])
7066         if not redirect_url:
7067             raise ExtractorError('Invalid cookie consent redirect URL', expected=True)
7068         return self.url_result(redirect_url)
7069
7070
7071 class YoutubeTruncatedIDIE(InfoExtractor):
7072     IE_NAME = 'youtube:truncated_id'
7073     IE_DESC = False  # Do not list
7074     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
7075
7076     _TESTS = [{
7077         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
7078         'only_matching': True,
7079     }]
7080
7081     def _real_extract(self, url):
7082         video_id = self._match_id(url)
7083         raise ExtractorError(
7084             f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
7085             expected=True)