from __future__ import unicode_literals
-import base64
import calendar
import copy
import datetime
+import functools
import hashlib
import itertools
import json
+import math
import os.path
import random
import re
+import sys
import time
import traceback
+import threading
from .common import InfoExtractor, SearchInfoExtractor
from ..compat import (
)
from ..jsinterp import JSInterpreter
from ..utils import (
- bytes_to_intlist,
+ bug_reports_message,
clean_html,
datetime_from_str,
dict_get,
float_or_none,
format_field,
int_or_none,
- intlist_to_bytes,
is_html,
join_nonempty,
mimetype2ext,
network_exceptions,
+ NO_DEFAULT,
orderedSet,
parse_codecs,
parse_count,
smuggle_url,
str_or_none,
str_to_int,
+ strftime_or_none,
traverse_obj,
try_get,
unescapeHTML,
)
+def get_first(obj, keys, **kwargs):
+ return traverse_obj(obj, (..., *variadic(keys)), **kwargs, get_all=False)
+
+
# any clients starting with _ cannot be explicity requested by the user
INNERTUBE_CLIENTS = {
'web': {
# If True it will raise an error if no login info is provided
_LOGIN_REQUIRED = False
+ _INVIDIOUS_SITES = (
+ # invidious-redirect websites
+ r'(?:www\.)?redirect\.invidious\.io',
+ r'(?:(?:www|dev)\.)?invidio\.us',
+ # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
+ r'(?:www\.)?invidious\.pussthecat\.org',
+ r'(?:www\.)?invidious\.zee\.li',
+ r'(?:www\.)?invidious\.ethibox\.fr',
+ r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
+ # youtube-dl invidious instances list
+ r'(?:(?:www|no)\.)?invidiou\.sh',
+ r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
+ r'(?:www\.)?invidious\.kabi\.tk',
+ r'(?:www\.)?invidious\.mastodon\.host',
+ r'(?:www\.)?invidious\.zapashcanon\.fr',
+ r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
+ r'(?:www\.)?invidious\.tinfoil-hat\.net',
+ r'(?:www\.)?invidious\.himiko\.cloud',
+ r'(?:www\.)?invidious\.reallyancient\.tech',
+ r'(?:www\.)?invidious\.tube',
+ r'(?:www\.)?invidiou\.site',
+ r'(?:www\.)?invidious\.site',
+ r'(?:www\.)?invidious\.xyz',
+ r'(?:www\.)?invidious\.nixnet\.xyz',
+ r'(?:www\.)?invidious\.048596\.xyz',
+ r'(?:www\.)?invidious\.drycat\.fr',
+ r'(?:www\.)?inv\.skyn3t\.in',
+ r'(?:www\.)?tube\.poal\.co',
+ r'(?:www\.)?tube\.connect\.cafe',
+ r'(?:www\.)?vid\.wxzm\.sx',
+ r'(?:www\.)?vid\.mint\.lgbt',
+ r'(?:www\.)?vid\.puffyan\.us',
+ r'(?:www\.)?yewtu\.be',
+ r'(?:www\.)?yt\.elukerio\.org',
+ r'(?:www\.)?yt\.lelux\.fi',
+ r'(?:www\.)?invidious\.ggc-project\.de',
+ r'(?:www\.)?yt\.maisputain\.ovh',
+ r'(?:www\.)?ytprivate\.com',
+ r'(?:www\.)?invidious\.13ad\.de',
+ r'(?:www\.)?invidious\.toot\.koeln',
+ r'(?:www\.)?invidious\.fdn\.fr',
+ r'(?:www\.)?watch\.nettohikari\.com',
+ r'(?:www\.)?invidious\.namazso\.eu',
+ r'(?:www\.)?invidious\.silkky\.cloud',
+ r'(?:www\.)?invidious\.exonip\.de',
+ r'(?:www\.)?invidious\.riverside\.rocks',
+ r'(?:www\.)?invidious\.blamefran\.net',
+ r'(?:www\.)?invidious\.moomoo\.de',
+ r'(?:www\.)?ytb\.trom\.tf',
+ r'(?:www\.)?yt\.cyberhost\.uk',
+ r'(?:www\.)?kgg2m7yk5aybusll\.onion',
+ r'(?:www\.)?qklhadlycap4cnod\.onion',
+ r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
+ r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
+ r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
+ r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
+ r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
+ r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
+ r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
+ r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
+ r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
+ r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
+ )
+
def _login(self):
"""
Attempt to log in to YouTube.
consent_id = random.randint(100, 999)
self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
+ def _initialize_pref(self):
+ cookies = self._get_cookies('https://www.youtube.com/')
+ pref_cookie = cookies.get('PREF')
+ pref = {}
+ if pref_cookie:
+ try:
+ pref = dict(compat_urlparse.parse_qsl(pref_cookie.value))
+ except ValueError:
+ self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
+ pref.update({'hl': 'en'})
+ self._set_cookie('.youtube.com', name='PREF', value=compat_urllib_parse_urlencode(pref))
+
def _real_initialize(self):
+ self._initialize_pref()
self._initialize_consent()
self._login()
return self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_API_KEY'], compat_str, default_client)
def _extract_context(self, ytcfg=None, default_client='web'):
- _get_context = lambda y: try_get(y, lambda x: x['INNERTUBE_CONTEXT'], dict)
- context = _get_context(ytcfg)
- if context:
- return context
-
- context = _get_context(self._get_default_ytcfg(default_client))
- if not ytcfg:
- return context
-
- # Recreate the client context (required)
- context['client'].update({
- 'clientVersion': self._extract_client_version(ytcfg, default_client),
- 'clientName': self._extract_client_name(ytcfg, default_client),
- })
- visitor_data = try_get(ytcfg, lambda x: x['VISITOR_DATA'], compat_str)
- if visitor_data:
- context['client']['visitorData'] = visitor_data
+ context = get_first(
+ (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
+ # Enforce language for extraction
+ traverse_obj(context, 'client', expected_type=dict, default={})['hl'] = 'en'
return context
_SAPISID = None
Extracts visitorData from an API response or ytcfg
Appears to be used to track session state
"""
- return traverse_obj(
- args, (..., ('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))),
- expected_type=compat_str, get_all=False)
+ return get_first(
+ args, (('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))),
+ expected_type=str)
@property
def is_authenticated(self):
if text:
return text
+ @staticmethod
+ def extract_relative_time(relative_time_text):
+ """
+ Extracts a relative time from string and converts to dt object
+ e.g. 'streamed 6 days ago', '5 seconds ago (edited)'
+ """
+ mobj = re.search(r'(?P<time>\d+)\s*(?P<unit>microsecond|second|minute|hour|day|week|month|year)s?\s*ago', relative_time_text)
+ if mobj:
+ try:
+ return datetime_from_str('now-%s%s' % (mobj.group('time'), mobj.group('unit')), precision='auto')
+ except ValueError:
+ return None
+
+ def _extract_time_text(self, renderer, *path_list):
+ text = self._get_text(renderer, *path_list) or ''
+ dt = self.extract_relative_time(text)
+ timestamp = None
+ if isinstance(dt, datetime.datetime):
+ timestamp = calendar.timegm(dt.timetuple())
+ if text and timestamp is None:
+ self.report_warning('Cannot parse localized time text' + bug_reports_message(), only_once=True)
+ return timestamp, text
+
def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
default_client='web'):
'view count', default=None))
uploader = self._get_text(renderer, 'ownerText', 'shortBylineText')
-
+ channel_id = traverse_obj(
+ renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'), expected_type=str, get_all=False)
+ timestamp, time_text = self._extract_time_text(renderer, 'publishedTimeText')
+ scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
+ overlay_style = traverse_obj(
+ renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'), get_all=False, expected_type=str)
+ badges = self._extract_badges(renderer)
return {
'_type': 'url',
'ie_key': YoutubeIE.ie_key(),
'duration': duration,
'view_count': view_count,
'uploader': uploader,
+ 'channel_id': channel_id,
+ 'upload_date': strftime_or_none(timestamp, '%Y%m%d'),
+ 'live_status': ('is_upcoming' if scheduled_timestamp is not None
+ else 'was_live' if 'streamed' in time_text.lower()
+ else 'is_live' if overlay_style is not None and overlay_style == 'LIVE' or 'live now' in badges
+ else None),
+ 'release_timestamp': scheduled_timestamp,
+ 'availability': self._availability(needs_premium='premium' in badges, needs_subscription='members only' in badges)
}
class YoutubeIE(YoutubeBaseInfoExtractor):
IE_DESC = 'YouTube'
- _INVIDIOUS_SITES = (
- # invidious-redirect websites
- r'(?:www\.)?redirect\.invidious\.io',
- r'(?:(?:www|dev)\.)?invidio\.us',
- # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
- r'(?:www\.)?invidious\.pussthecat\.org',
- r'(?:www\.)?invidious\.zee\.li',
- r'(?:www\.)?invidious\.ethibox\.fr',
- r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
- # youtube-dl invidious instances list
- r'(?:(?:www|no)\.)?invidiou\.sh',
- r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
- r'(?:www\.)?invidious\.kabi\.tk',
- r'(?:www\.)?invidious\.mastodon\.host',
- r'(?:www\.)?invidious\.zapashcanon\.fr',
- r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
- r'(?:www\.)?invidious\.tinfoil-hat\.net',
- r'(?:www\.)?invidious\.himiko\.cloud',
- r'(?:www\.)?invidious\.reallyancient\.tech',
- r'(?:www\.)?invidious\.tube',
- r'(?:www\.)?invidiou\.site',
- r'(?:www\.)?invidious\.site',
- r'(?:www\.)?invidious\.xyz',
- r'(?:www\.)?invidious\.nixnet\.xyz',
- r'(?:www\.)?invidious\.048596\.xyz',
- r'(?:www\.)?invidious\.drycat\.fr',
- r'(?:www\.)?inv\.skyn3t\.in',
- r'(?:www\.)?tube\.poal\.co',
- r'(?:www\.)?tube\.connect\.cafe',
- r'(?:www\.)?vid\.wxzm\.sx',
- r'(?:www\.)?vid\.mint\.lgbt',
- r'(?:www\.)?vid\.puffyan\.us',
- r'(?:www\.)?yewtu\.be',
- r'(?:www\.)?yt\.elukerio\.org',
- r'(?:www\.)?yt\.lelux\.fi',
- r'(?:www\.)?invidious\.ggc-project\.de',
- r'(?:www\.)?yt\.maisputain\.ovh',
- r'(?:www\.)?ytprivate\.com',
- r'(?:www\.)?invidious\.13ad\.de',
- r'(?:www\.)?invidious\.toot\.koeln',
- r'(?:www\.)?invidious\.fdn\.fr',
- r'(?:www\.)?watch\.nettohikari\.com',
- r'(?:www\.)?invidious\.namazso\.eu',
- r'(?:www\.)?invidious\.silkky\.cloud',
- r'(?:www\.)?invidious\.exonip\.de',
- r'(?:www\.)?invidious\.riverside\.rocks',
- r'(?:www\.)?invidious\.blamefran\.net',
- r'(?:www\.)?invidious\.moomoo\.de',
- r'(?:www\.)?ytb\.trom\.tf',
- r'(?:www\.)?yt\.cyberhost\.uk',
- r'(?:www\.)?kgg2m7yk5aybusll\.onion',
- r'(?:www\.)?qklhadlycap4cnod\.onion',
- r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
- r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
- r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
- r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
- r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
- r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
- r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
- r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
- r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
- r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
- )
_VALID_URL = r"""(?x)^
(
(?:https?://|//) # http(s):// or protocol-independent URL
(?P<id>[0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
(?(1).+)? # if we found the ID, everything can follow
(?:\#|$)""" % {
- 'invidious': '|'.join(_INVIDIOUS_SITES),
+ 'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
}
_PLAYER_INFO_RE = (
r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
'uploader': 'Philipp Hagemeister',
'uploader_id': 'phihag',
'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
+ 'channel': 'Philipp Hagemeister',
'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
'upload_date': '20121002',
- 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
+ 'description': 'md5:8fb536f4877b8a7455c2ec23794dbc22',
'categories': ['Science & Technology'],
'tags': ['youtube-dl'],
'duration': 10,
'view_count': int,
'like_count': int,
- 'dislike_count': int,
+ # 'dislike_count': int,
+ 'availability': 'public',
+ 'playable_in_embed': True,
+ 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
+ 'live_status': 'not_live',
+ 'age_limit': 0,
'start_time': 1,
'end_time': 9,
}
# shorts
'url': 'https://www.youtube.com/shorts/BGQWPY4IigY',
'only_matching': True,
- },
+ }, {
+ 'note': 'Storyboards',
+ 'url': 'https://www.youtube.com/watch?v=5KLPxDtMqe8',
+ 'info_dict': {
+ 'id': '5KLPxDtMqe8',
+ 'ext': 'mhtml',
+ 'format_id': 'sb0',
+ 'title': 'Your Brain is Plastic',
+ 'uploader_id': 'scishow',
+ 'description': 'md5:89cd86034bdb5466cd87c6ba206cd2bc',
+ 'upload_date': '20140324',
+ 'uploader': 'SciShow',
+ }, 'params': {'format': 'mhtml', 'skip_download': True}
+ }
]
@classmethod
self._code_cache = {}
self._player_cache = {}
+ def _prepare_live_from_start_formats(self, formats, video_id, live_start_time, url, webpage_url, smuggled_data):
+ EXPIRATION_DURATION = 18_000
+ lock = threading.Lock()
+
+ is_live = True
+ expiration_time = time.time() + EXPIRATION_DURATION
+ formats = [f for f in formats if f.get('is_from_start')]
+
+ def refetch_manifest(format_id):
+ nonlocal formats, expiration_time, is_live
+ if time.time() <= expiration_time:
+ return
+
+ _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
+ video_details = traverse_obj(
+ prs, (..., 'videoDetails'), expected_type=dict, default=[])
+ microformats = traverse_obj(
+ prs, (..., 'microformat', 'playerMicroformatRenderer'),
+ expected_type=dict, default=[])
+ _, is_live, _, formats = self._list_formats(video_id, microformats, video_details, prs, player_url)
+ expiration_time = time.time() + EXPIRATION_DURATION
+
+ def mpd_feed(format_id):
+ """
+ @returns (manifest_url, manifest_stream_number, is_live) or None
+ """
+ with lock:
+ refetch_manifest(format_id)
+
+ f = next((f for f in formats if f['format_id'] == format_id), None)
+ if not f:
+ self.report_warning(
+ f'Cannot find refreshed manifest for format {format_id}{bug_reports_message()}')
+ return None
+ return f['manifest_url'], f['manifest_stream_number'], is_live
+
+ for f in formats:
+ f['protocol'] = 'http_dash_segments_generator'
+ f['fragments'] = functools.partial(
+ self._live_dash_fragments, f['format_id'], live_start_time, mpd_feed)
+
+ def _live_dash_fragments(self, format_id, live_start_time, mpd_feed, ctx):
+ FETCH_SPAN, MAX_DURATION = 5, 432000
+
+ mpd_url, stream_number, is_live = None, None, True
+
+ begin_index = 0
+ download_start_time = ctx.get('start') or time.time()
+
+ lack_early_segments = download_start_time - (live_start_time or download_start_time) > MAX_DURATION
+ if lack_early_segments:
+ self.report_warning(bug_reports_message(
+ 'Starting download from the last 120 hours of the live stream since '
+ 'YouTube does not have data before that. If you think this is wrong,'), only_once=True)
+ lack_early_segments = True
+
+ known_idx, no_fragment_score, last_segment_url = begin_index, 0, None
+ fragments, fragment_base_url = None, None
+
+ def _extract_sequence_from_mpd(refresh_sequence):
+ nonlocal mpd_url, stream_number, is_live, no_fragment_score, fragments, fragment_base_url
+ # Obtain from MPD's maximum seq value
+ old_mpd_url = mpd_url
+ mpd_url, stream_number, is_live = mpd_feed(format_id) or (mpd_url, stream_number, False)
+ if old_mpd_url == mpd_url and not refresh_sequence:
+ return True, last_seq
+ try:
+ fmts, _ = self._extract_mpd_formats_and_subtitles(
+ mpd_url, None, note=False, errnote=False, fatal=False)
+ except ExtractorError:
+ fmts = None
+ if not fmts:
+ no_fragment_score += 1
+ return False, last_seq
+ fmt_info = next(x for x in fmts if x['manifest_stream_number'] == stream_number)
+ fragments = fmt_info['fragments']
+ fragment_base_url = fmt_info['fragment_base_url']
+ assert fragment_base_url
+
+ _last_seq = int(re.search(r'(?:/|^)sq/(\d+)', fragments[-1]['path']).group(1))
+ return True, _last_seq
+
+ while is_live:
+ fetch_time = time.time()
+ if no_fragment_score > 30:
+ return
+ if last_segment_url:
+ # Obtain from "X-Head-Seqnum" header value from each segment
+ try:
+ urlh = self._request_webpage(
+ last_segment_url, None, note=False, errnote=False, fatal=False)
+ except ExtractorError:
+ urlh = None
+ last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
+ if last_seq is None:
+ no_fragment_score += 1
+ last_segment_url = None
+ continue
+ else:
+ should_retry, last_seq = _extract_sequence_from_mpd(True)
+ if not should_retry:
+ continue
+
+ if known_idx > last_seq:
+ last_segment_url = None
+ continue
+
+ last_seq += 1
+
+ if begin_index < 0 and known_idx < 0:
+ # skip from the start when it's negative value
+ known_idx = last_seq + begin_index
+ if lack_early_segments:
+ known_idx = max(known_idx, last_seq - int(MAX_DURATION // fragments[-1]['duration']))
+ try:
+ for idx in range(known_idx, last_seq):
+ # do not update sequence here or you'll get skipped some part of it
+ should_retry, _ = _extract_sequence_from_mpd(False)
+ if not should_retry:
+ # retry when it gets weird state
+ known_idx = idx - 1
+ raise ExtractorError('breaking out of outer loop')
+ last_segment_url = urljoin(fragment_base_url, 'sq/%d' % idx)
+ yield {
+ 'url': last_segment_url,
+ }
+ if known_idx == last_seq:
+ no_fragment_score += 5
+ else:
+ no_fragment_score = 0
+ known_idx = last_seq
+ except ExtractorError:
+ continue
+
+ time.sleep(max(0, FETCH_SPAN + fetch_time - time.time()))
+
def _extract_player_url(self, *ytcfgs, webpage=None):
player_url = traverse_obj(
ytcfgs, (..., 'PLAYER_JS_URL'), (..., 'WEB_PLAYER_CONTEXT_CONFIGS', ..., 'jsUrl'),
self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
return self._player_cache[sig_id]
except Exception as e:
- raise ExtractorError(traceback.format_exc(), cause=e)
+ raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
def _extract_n_function_name(self, jscode):
return self._search_regex(
return sts
def _mark_watched(self, video_id, player_responses):
- playback_url = traverse_obj(
- player_responses, (..., 'playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
- expected_type=url_or_none, get_all=False)
+ playback_url = get_first(
+ player_responses, ('playbackTracking', 'videostatsPlaybackUrl', 'baseUrl'),
+ expected_type=url_or_none)
if not playback_url:
self.report_warning('Unable to mark watched')
return
(r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
regex), webpage, name, default='{}'), video_id, fatal=False)
- @staticmethod
- def parse_time_text(time_text):
- """
- Parse the comment time text
- time_text is in the format 'X units ago (edited)'
- """
- time_text_split = time_text.split(' ')
- if len(time_text_split) >= 3:
- try:
- return datetime_from_str('now-%s%s' % (time_text_split[0], time_text_split[1]), precision='auto')
- except ValueError:
- return None
-
def _extract_comment(self, comment_renderer, parent=None):
comment_id = comment_renderer.get('commentId')
if not comment_id:
text = self._get_text(comment_renderer, 'contentText')
# note: timestamp is an estimate calculated from the current time and time_text
- time_text = self._get_text(comment_renderer, 'publishedTimeText') or ''
- time_text_dt = self.parse_time_text(time_text)
- if isinstance(time_text_dt, datetime.datetime):
- timestamp = calendar.timegm(time_text_dt.timetuple())
+ timestamp, time_text = self._extract_time_text(comment_renderer, 'publishedTimeText')
author = self._get_text(comment_renderer, 'authorText')
author_id = try_get(comment_renderer,
lambda x: x['authorEndpoint']['browseEndpoint']['browseId'], compat_str)
'parent': parent or 'root'
}
- def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, comment_counts=None):
+ def _comment_entries(self, root_continuation_data, ytcfg, video_id, parent=None, tracker=None):
+
+ get_single_config_arg = lambda c: self._configuration_arg(c, [''])[0]
def extract_header(contents):
_continuation = None
for content in contents:
- comments_header_renderer = try_get(content, lambda x: x['commentsHeaderRenderer'])
+ comments_header_renderer = traverse_obj(content, 'commentsHeaderRenderer')
expected_comment_count = parse_count(self._get_text(
comments_header_renderer, 'countText', 'commentsCount', max_runs=1))
if expected_comment_count:
- comment_counts[1] = expected_comment_count
- self.to_screen('Downloading ~%d comments' % expected_comment_count)
- sort_mode_str = self._configuration_arg('comment_sort', [''])[0]
- comment_sort_index = int(sort_mode_str != 'top') # 1 = new, 0 = top
+ tracker['est_total'] = expected_comment_count
+ self.to_screen(f'Downloading ~{expected_comment_count} comments')
+ comment_sort_index = int(get_single_config_arg('comment_sort') != 'top') # 1 = new, 0 = top
sort_menu_item = try_get(
comments_header_renderer,
if not _continuation:
continue
- sort_text = sort_menu_item.get('title')
- if isinstance(sort_text, compat_str):
- sort_text = sort_text.lower()
- else:
+ sort_text = str_or_none(sort_menu_item.get('title'))
+ if not sort_text:
sort_text = 'top comments' if comment_sort_index == 0 else 'newest first'
- self.to_screen('Sorting comments by %s' % sort_text)
+ self.to_screen('Sorting comments by %s' % sort_text.lower())
break
return _continuation
def extract_thread(contents):
if not parent:
- comment_counts[2] = 0
+ tracker['current_page_thread'] = 0
for content in contents:
+ if not parent and tracker['total_parent_comments'] >= max_parents:
+ yield
comment_thread_renderer = try_get(content, lambda x: x['commentThreadRenderer'])
- comment_renderer = try_get(
- comment_thread_renderer, (lambda x: x['comment']['commentRenderer'], dict)) or try_get(
- content, (lambda x: x['commentRenderer'], dict))
+ comment_renderer = get_first(
+ (comment_thread_renderer, content), [['commentRenderer', ('comment', 'commentRenderer')]],
+ expected_type=dict, default={})
- if not comment_renderer:
- continue
comment = self._extract_comment(comment_renderer, parent)
if not comment:
continue
- comment_counts[0] += 1
+
+ tracker['running_total'] += 1
+ tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1
yield comment
+
# Attempt to get the replies
comment_replies_renderer = try_get(
comment_thread_renderer, lambda x: x['replies']['commentRepliesRenderer'], dict)
if comment_replies_renderer:
- comment_counts[2] += 1
+ tracker['current_page_thread'] += 1
comment_entries_iter = self._comment_entries(
comment_replies_renderer, ytcfg, video_id,
- parent=comment.get('id'), comment_counts=comment_counts)
-
- for reply_comment in comment_entries_iter:
+ parent=comment.get('id'), tracker=tracker)
+ for reply_comment in itertools.islice(comment_entries_iter, min(max_replies_per_thread, max(0, max_replies - tracker['total_reply_comments']))):
yield reply_comment
+ # Keeps track of counts across recursive calls
+ if not tracker:
+ tracker = dict(
+ running_total=0,
+ est_total=0,
+ current_page_thread=0,
+ total_parent_comments=0,
+ total_reply_comments=0)
+
+ # TODO: Deprecated
# YouTube comments have a max depth of 2
- max_depth = int_or_none(self._configuration_arg('max_comment_depth', [''])[0]) or float('inf')
+ max_depth = int_or_none(get_single_config_arg('max_comment_depth'))
+ if max_depth:
+ self._downloader.deprecation_warning(
+ '[youtube] max_comment_depth extractor argument is deprecated. Set max replies in the max-comments extractor argument instead.')
if max_depth == 1 and parent:
return
- if not comment_counts:
- # comment so far, est. total comments, current comment thread #
- comment_counts = [0, 0, 0]
- continuation = self._extract_continuation(root_continuation_data)
- if continuation and len(continuation['continuation']) < 27:
- self.write_debug('Detected old API continuation token. Generating new API compatible token.')
- continuation_token = self._generate_comment_continuation(video_id)
- continuation = self._build_api_continuation_query(continuation_token, None)
+ max_comments, max_parents, max_replies, max_replies_per_thread, *_ = map(
+ lambda p: int_or_none(p, default=sys.maxsize), self._configuration_arg('max_comments', ) + [''] * 4)
+ continuation = self._extract_continuation(root_continuation_data)
message = self._get_text(root_continuation_data, ('contents', ..., 'messageRenderer', 'text'), max_runs=1)
if message and not parent:
self.report_warning(message, video_id=video_id)
- visitor_data = None
+ response = None
is_first_continuation = parent is None
for page_num in itertools.count(0):
if not continuation:
break
- headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=visitor_data)
- comment_prog_str = '(%d/%d)' % (comment_counts[0], comment_counts[1])
+ headers = self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response))
+ comment_prog_str = f"({tracker['running_total']}/{tracker['est_total']})"
if page_num == 0:
if is_first_continuation:
note_prefix = 'Downloading comment section API JSON'
else:
note_prefix = ' Downloading comment API JSON reply thread %d %s' % (
- comment_counts[2], comment_prog_str)
+ tracker['current_page_thread'], comment_prog_str)
else:
note_prefix = '%sDownloading comment%s API JSON page %d %s' % (
' ' if parent else '', ' replies' if parent else '',
response = self._extract_response(
item_id=None, query=continuation,
ep='next', ytcfg=ytcfg, headers=headers, note=note_prefix,
- check_get_keys=('onResponseReceivedEndpoints', 'continuationContents'))
- if not response:
- break
- visitor_data = try_get(
- response,
- lambda x: x['responseContext']['webResponseContextExtensionData']['ytConfigData']['visitorData'],
- compat_str) or visitor_data
+ check_get_keys='onResponseReceivedEndpoints')
- continuation_contents = dict_get(response, ('onResponseReceivedEndpoints', 'continuationContents'))
+ continuation_contents = traverse_obj(
+ response, 'onResponseReceivedEndpoints', expected_type=list, default=[])
continuation = None
- if isinstance(continuation_contents, list):
- for continuation_section in continuation_contents:
- if not isinstance(continuation_section, dict):
- continue
- continuation_items = try_get(
- continuation_section,
- (lambda x: x['reloadContinuationItemsCommand']['continuationItems'],
- lambda x: x['appendContinuationItemsAction']['continuationItems']),
- list) or []
- if is_first_continuation:
- continuation = extract_header(continuation_items)
- is_first_continuation = False
- if continuation:
- break
- continue
- count = 0
- for count, entry in enumerate(extract_thread(continuation_items)):
- yield entry
- continuation = self._extract_continuation({'contents': continuation_items})
+ for continuation_section in continuation_contents:
+ continuation_items = traverse_obj(
+ continuation_section,
+ (('reloadContinuationItemsCommand', 'appendContinuationItemsAction'), 'continuationItems'),
+ get_all=False, expected_type=list) or []
+ if is_first_continuation:
+ continuation = extract_header(continuation_items)
+ is_first_continuation = False
if continuation:
- # Sometimes YouTube provides a continuation without any comments
- # In most cases we end up just downloading these with very little comments to come.
- if count == 0:
- if not parent:
- self.report_warning('No comments received - assuming end of comments')
- continuation = None
break
+ continue
- # Deprecated response structure
- elif isinstance(continuation_contents, dict):
- known_continuation_renderers = ('itemSectionContinuation', 'commentRepliesContinuation')
- for key, continuation_renderer in continuation_contents.items():
- if key not in known_continuation_renderers:
- continue
- if not isinstance(continuation_renderer, dict):
- continue
- if is_first_continuation:
- header_continuation_items = [continuation_renderer.get('header') or {}]
- continuation = extract_header(header_continuation_items)
- is_first_continuation = False
- if continuation:
- break
-
- # Sometimes YouTube provides a continuation without any comments
- # In most cases we end up just downloading these with very little comments to come.
- count = 0
- for count, entry in enumerate(extract_thread(continuation_renderer.get('contents') or {})):
- yield entry
- continuation = self._extract_continuation(continuation_renderer)
- if count == 0:
- if not parent:
- self.report_warning('No comments received - assuming end of comments')
- continuation = None
+ for entry in extract_thread(continuation_items):
+ if not entry:
+ return
+ yield entry
+ continuation = self._extract_continuation({'contents': continuation_items})
+ if continuation:
break
- @staticmethod
- def _generate_comment_continuation(video_id):
- """
- Generates initial comment section continuation token from given video id
- """
- b64_vid_id = base64.b64encode(bytes(video_id.encode('utf-8')))
- parts = ('Eg0SCw==', b64_vid_id, 'GAYyJyIRIgs=', b64_vid_id, 'MAB4AjAAQhBjb21tZW50cy1zZWN0aW9u')
- new_continuation_intlist = list(itertools.chain.from_iterable(
- [bytes_to_intlist(base64.b64decode(part)) for part in parts]))
- return base64.b64encode(intlist_to_bytes(new_continuation_intlist)).decode('utf-8')
-
def _get_comments(self, ytcfg, video_id, contents, webpage):
"""Entry for comment extraction"""
def _real_comment_extract(contents):
yield from self._comment_entries(renderer, ytcfg, video_id)
max_comments = int_or_none(self._configuration_arg('max_comments', [''])[0])
- # Force English regardless of account setting to prevent parsing issues
- # See: https://github.com/yt-dlp/yt-dlp/issues/532
- ytcfg = copy.deepcopy(ytcfg)
- traverse_obj(
- ytcfg, ('INNERTUBE_CONTEXT', 'client'), expected_type=dict, default={})['hl'] = 'en'
return itertools.islice(_real_comment_extract(contents), 0, max_comments)
@staticmethod
def _get_requested_clients(self, url, smuggled_data):
requested_clients = []
+ default = ['android', 'web']
allowed_clients = sorted(
[client for client in INNERTUBE_CLIENTS.keys() if client[:1] != '_'],
key=lambda client: INNERTUBE_CLIENTS[client]['priority'], reverse=True)
for client in self._configuration_arg('player_client'):
if client in allowed_clients:
requested_clients.append(client)
+ elif client == 'default':
+ requested_clients.extend(default)
elif client == 'all':
requested_clients.extend(allowed_clients)
else:
self.report_warning(f'Skipping unsupported client {client}')
if not requested_clients:
- requested_clients = ['android', 'web']
+ requested_clients = default
if smuggled_data.get('is_music_url') or self.is_music_url(url):
requested_clients.extend(
query = parse_qs(fmt_url)
throttled = False
- if query.get('ratebypass') != ['yes'] and query.get('n'):
+ if query.get('n'):
try:
fmt_url = update_url_query(fmt_url, {
'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
except ExtractorError as e:
- self.report_warning(f'nsig extraction failed: You may experience throttling for some formats\n{e}', only_once=True)
+ self.report_warning(
+ f'nsig extraction failed: You may experience throttling for some formats\n'
+ f'n = {query["n"][0]} ; player = {player_url}\n{e}', only_once=True)
throttled = True
if itag:
dct['container'] = dct['ext'] + '_dash'
yield dct
+ live_from_start = is_live and self.get_param('live_from_start')
skip_manifests = self._configuration_arg('skip')
- get_dash = (
- (not is_live or self._configuration_arg('include_live_dash'))
- and 'dash' not in skip_manifests and self.get_param('youtube_include_dash_manifest', True))
- get_hls = 'hls' not in skip_manifests and self.get_param('youtube_include_hls_manifest', True)
+ if not self.get_param('youtube_include_hls_manifest', True):
+ skip_manifests.append('hls')
+ get_dash = 'dash' not in skip_manifests and (
+ not is_live or live_from_start or self._configuration_arg('include_live_dash'))
+ get_hls = not live_from_start and 'hls' not in skip_manifests
def process_manifest_format(f, proto, itag):
if itag in itags:
f['quality'] = next((
q(qdict[val])
- for val, qdict in ((f.get('format_id'), itag_qualities), (f.get('height'), res_qualities))
+ for val, qdict in ((f.get('format_id', '').split('-')[0], itag_qualities), (f.get('height'), res_qualities))
if val in qdict), -1)
return True
if process_manifest_format(f, 'dash', f['format_id']):
f['filesize'] = int_or_none(self._search_regex(
r'/clen/(\d+)', f.get('fragment_base_url') or f['url'], 'file size', default=None))
+ if live_from_start:
+ f['is_from_start'] = True
+
yield f
- def _real_extract(self, url):
- url, smuggled_data = unsmuggle_url(url, {})
- video_id = self._match_id(url)
+ def _extract_storyboard(self, player_responses, duration):
+ spec = get_first(
+ player_responses, ('storyboards', 'playerStoryboardSpecRenderer', 'spec'), default='').split('|')[::-1]
+ if not spec:
+ return
+ base_url = spec.pop()
+ L = len(spec) - 1
+ for i, args in enumerate(spec):
+ args = args.split('#')
+ counts = list(map(int_or_none, args[:5]))
+ if len(args) != 8 or not all(counts):
+ self.report_warning(f'Malformed storyboard {i}: {"#".join(args)}{bug_reports_message()}')
+ continue
+ width, height, frame_count, cols, rows = counts
+ N, sigh = args[6:]
+
+ url = base_url.replace('$L', str(L - i)).replace('$N', N) + f'&sigh={sigh}'
+ fragment_count = frame_count / (cols * rows)
+ fragment_duration = duration / fragment_count
+ yield {
+ 'format_id': f'sb{i}',
+ 'format_note': 'storyboard',
+ 'ext': 'mhtml',
+ 'protocol': 'mhtml',
+ 'acodec': 'none',
+ 'vcodec': 'none',
+ 'url': url,
+ 'width': width,
+ 'height': height,
+ 'fragments': [{
+ 'path': url.replace('$M', str(j)),
+ 'duration': min(fragment_duration, duration - (j * fragment_duration)),
+ } for j in range(math.ceil(fragment_count))],
+ }
- base_url = self.http_scheme() + '//www.youtube.com/'
- webpage_url = base_url + 'watch?v=' + video_id
+ def _download_player_responses(self, url, smuggled_data, video_id, webpage_url):
webpage = None
if 'webpage' not in self._configuration_arg('player_skip'):
webpage = self._download_webpage(
self._get_requested_clients(url, smuggled_data),
video_id, webpage, master_ytcfg)
- get_first = lambda obj, keys, **kwargs: traverse_obj(obj, (..., *variadic(keys)), **kwargs, get_all=False)
+ return webpage, master_ytcfg, player_responses, player_url
+
+ def _list_formats(self, video_id, microformats, video_details, player_responses, player_url):
+ live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
+ is_live = get_first(video_details, 'isLive')
+ if is_live is None:
+ is_live = get_first(live_broadcast_details, 'isLiveNow')
+
+ streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
+ formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live))
+
+ return live_broadcast_details, is_live, streaming_data, formats
+
+ def _real_extract(self, url):
+ url, smuggled_data = unsmuggle_url(url, {})
+ video_id = self._match_id(url)
+
+ base_url = self.http_scheme() + '//www.youtube.com/'
+ webpage_url = base_url + 'watch?v=' + video_id
+
+ webpage, master_ytcfg, player_responses, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
playability_statuses = traverse_obj(
player_responses, (..., 'playabilityStatus'), expected_type=dict, default=[])
return self.playlist_result(
entries, video_id, video_title, video_description)
- live_broadcast_details = traverse_obj(microformats, (..., 'liveBroadcastDetails'))
- is_live = get_first(video_details, 'isLive')
- if is_live is None:
- is_live = get_first(live_broadcast_details, 'isLiveNow')
-
- streaming_data = traverse_obj(player_responses, (..., 'streamingData'), default=[])
- formats = list(self._extract_formats(streaming_data, video_id, player_url, is_live))
+ live_broadcast_details, is_live, streaming_data, formats = self._list_formats(video_id, microformats, video_details, player_responses, player_url)
if not formats:
if not self.get_param('allow_unplayable_formats') and traverse_obj(streaming_data, (..., 'licenseInfos')):
if reason:
self.raise_no_formats(reason, expected=True)
- # Source is given priority since formats that throttle are given lower source_preference
- # When throttling issue is fully fixed, remove this
- self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
-
keywords = get_first(video_details, 'keywords', expected_type=list) or []
if not keywords and webpage:
keywords = [
is_live = False
if is_upcoming is None and (live_content or is_live):
is_upcoming = False
- live_starttime = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
- live_endtime = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
- if not duration and live_endtime and live_starttime:
- duration = live_endtime - live_starttime
+ live_start_time = parse_iso8601(get_first(live_broadcast_details, 'startTimestamp'))
+ live_end_time = parse_iso8601(get_first(live_broadcast_details, 'endTimestamp'))
+ if not duration and live_end_time and live_start_time:
+ duration = live_end_time - live_start_time
+
+ if is_live and self.get_param('live_from_start'):
+ self._prepare_live_from_start_formats(formats, video_id, live_start_time, url, webpage_url, smuggled_data)
+
+ formats.extend(self._extract_storyboard(player_responses, duration))
+
+ # Source is given priority since formats that throttle are given lower source_preference
+ # When throttling issue is fully fixed, remove this
+ self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang', 'proto'))
info = {
'id': video_id,
- 'title': self._live_title(video_title) if is_live else video_title,
+ 'title': video_title,
'formats': formats,
'thumbnails': thumbnails,
# The best thumbnail that we are sure exists. Prevents unnecessary
else None if is_live is None or is_upcoming is None
else live_content),
'live_status': 'is_upcoming' if is_upcoming else None, # rest will be set by YoutubeDL
- 'release_timestamp': live_starttime,
+ 'release_timestamp': live_start_time,
}
pctr = traverse_obj(player_responses, (..., 'captions', 'playerCaptionsTracklistRenderer'), expected_type=dict)
return info
-class YoutubeTabIE(YoutubeBaseInfoExtractor):
- IE_DESC = 'YouTube Tabs'
- _VALID_URL = r'''(?x)
- https?://
- (?:\w+\.)?
- (?:
- youtube(?:kids)?\.com|
- invidio\.us
- )/
- (?:
- (?P<channel_type>channel|c|user|browse)/|
- (?P<not_channel>
- feed/|hashtag/|
- (?:playlist|watch)\?.*?\blist=
- )|
- (?!(?:%s)\b) # Direct URLs
- )
- (?P<id>[^/?\#&]+)
- ''' % YoutubeBaseInfoExtractor._RESERVED_NAMES
- IE_NAME = 'youtube:tab'
-
- _TESTS = [{
- 'note': 'playlists, multipage',
- 'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
- 'playlist_mincount': 94,
- 'info_dict': {
- 'id': 'UCqj7Cz7revf5maW9g5pgNcg',
- 'title': 'Игорь Клейнер - Playlists',
- 'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
- 'uploader': 'Игорь Клейнер',
- 'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
- },
- }, {
- 'note': 'playlists, multipage, different order',
- 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
- 'playlist_mincount': 94,
- 'info_dict': {
- 'id': 'UCqj7Cz7revf5maW9g5pgNcg',
- 'title': 'Игорь Клейнер - Playlists',
- 'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
- 'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
- 'uploader': 'Игорь Клейнер',
- },
- }, {
- 'note': 'playlists, series',
- 'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
- 'playlist_mincount': 5,
- 'info_dict': {
- 'id': 'UCYO_jab_esuFRV4b17AJtAw',
- 'title': '3Blue1Brown - Playlists',
- 'description': 'md5:e1384e8a133307dd10edee76e875d62f',
- 'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
- 'uploader': '3Blue1Brown',
- },
- }, {
- 'note': 'playlists, singlepage',
- 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
- 'playlist_mincount': 4,
- 'info_dict': {
- 'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
- 'title': 'ThirstForScience - Playlists',
- 'description': 'md5:609399d937ea957b0f53cbffb747a14c',
- 'uploader': 'ThirstForScience',
- 'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
- }
- }, {
- 'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
- 'only_matching': True,
- }, {
- 'note': 'basic, single video playlist',
- 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
- 'info_dict': {
- 'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
- 'uploader': 'Sergey M.',
- 'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
- 'title': 'youtube-dl public playlist',
- },
- 'playlist_count': 1,
- }, {
- 'note': 'empty playlist',
- 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
- 'info_dict': {
- 'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
- 'uploader': 'Sergey M.',
- 'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
- 'title': 'youtube-dl empty playlist',
- },
- 'playlist_count': 0,
- }, {
- 'note': 'Home tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Home',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- },
- 'playlist_mincount': 2,
- }, {
- 'note': 'Videos tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Videos',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- },
- 'playlist_mincount': 975,
- }, {
- 'note': 'Videos tab, sorted by popular',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Videos',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- },
- 'playlist_mincount': 199,
- }, {
- 'note': 'Playlists tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Playlists',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- },
- 'playlist_mincount': 17,
- }, {
- 'note': 'Community tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Community',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- },
- 'playlist_mincount': 18,
- }, {
- 'note': 'Channels tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Channels',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- },
- 'playlist_mincount': 12,
- }, {
- 'note': 'Search tab',
- 'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
- 'playlist_mincount': 40,
- 'info_dict': {
- 'id': 'UCYO_jab_esuFRV4b17AJtAw',
- 'title': '3Blue1Brown - Search - linear algebra',
- 'description': 'md5:e1384e8a133307dd10edee76e875d62f',
- 'uploader': '3Blue1Brown',
- 'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
- },
- }, {
- 'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
- 'only_matching': True,
- }, {
- 'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
- 'only_matching': True,
- }, {
- 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
- 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
- 'info_dict': {
- 'title': '29C3: Not my department',
- 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
- 'uploader': 'Christiaan008',
- 'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
- 'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
- },
- 'playlist_count': 96,
- }, {
- 'note': 'Large playlist',
- 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
- 'info_dict': {
- 'title': 'Uploads from Cauchemar',
- 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
- 'uploader': 'Cauchemar',
- 'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
- },
- 'playlist_mincount': 1123,
- }, {
- 'note': 'even larger playlist, 8832 videos',
- 'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
- 'only_matching': True,
- }, {
- 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
- 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
- 'info_dict': {
- 'title': 'Uploads from Interstellar Movie',
- 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
- 'uploader': 'Interstellar Movie',
- 'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
- },
- 'playlist_mincount': 21,
- }, {
- 'note': 'Playlist with "show unavailable videos" button',
- 'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
- 'info_dict': {
- 'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
- 'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
- 'uploader': 'Phim Siêu Nhân Nhật Bản',
- 'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
- },
- 'playlist_mincount': 200,
- }, {
- 'note': 'Playlist with unavailable videos in page 7',
- 'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
- 'info_dict': {
- 'title': 'Uploads from BlankTV',
- 'id': 'UU8l9frL61Yl5KFOl87nIm2w',
- 'uploader': 'BlankTV',
- 'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
- },
- 'playlist_mincount': 1000,
- }, {
- 'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
- 'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
- 'info_dict': {
- 'title': 'Data Analysis with Dr Mike Pound',
- 'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
- 'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
- 'uploader': 'Computerphile',
- 'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
- },
- 'playlist_mincount': 11,
- }, {
- 'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
- 'only_matching': True,
- }, {
- 'note': 'Playlist URL that does not actually serve a playlist',
- 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
- 'info_dict': {
- 'id': 'FqZTN594JQw',
- 'ext': 'webm',
- 'title': "Smiley's People 01 detective, Adventure Series, Action",
- 'uploader': 'STREEM',
- 'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
- 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
- 'upload_date': '20150526',
- 'license': 'Standard YouTube License',
- 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
- 'categories': ['People & Blogs'],
- 'tags': list,
- 'view_count': int,
- 'like_count': int,
- 'dislike_count': int,
- },
- 'params': {
- 'skip_download': True,
- },
- 'skip': 'This video is not available.',
- 'add_ie': [YoutubeIE.ie_key()],
- }, {
- 'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
- 'info_dict': {
- 'id': '3yImotZU3tw', # This will keep changing
- 'ext': 'mp4',
- 'title': compat_str,
- 'uploader': 'Sky News',
- 'uploader_id': 'skynews',
- 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
- 'upload_date': r're:\d{8}',
- 'description': compat_str,
- 'categories': ['News & Politics'],
- 'tags': list,
- 'like_count': int,
- 'dislike_count': int,
- },
- 'params': {
- 'skip_download': True,
- },
- 'expected_warnings': ['Downloading just video ', 'Ignoring subtitle tracks found in '],
- }, {
- 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
- 'info_dict': {
- 'id': 'a48o2S1cPoo',
- 'ext': 'mp4',
- 'title': 'The Young Turks - Live Main Show',
- 'uploader': 'The Young Turks',
- 'uploader_id': 'TheYoungTurks',
- 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
- 'upload_date': '20150715',
- 'license': 'Standard YouTube License',
- 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
- 'categories': ['News & Politics'],
- 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
- 'like_count': int,
- 'dislike_count': int,
- },
- 'params': {
- 'skip_download': True,
- },
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
- 'only_matching': True,
- }, {
- 'note': 'A channel that is not live. Should raise error',
- 'url': 'https://www.youtube.com/user/numberphile/live',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/trending',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/library',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/history',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/subscriptions',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/watch_later',
- 'only_matching': True,
- }, {
- 'note': 'Recommended - redirects to home page.',
- 'url': 'https://www.youtube.com/feed/recommended',
- 'only_matching': True,
- }, {
- 'note': 'inline playlist with not always working continuations',
- 'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/course',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/zsecurity',
- 'only_matching': True,
- }, {
- 'url': 'http://www.youtube.com/NASAgovVideo/videos',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/TheYoungTurks/live',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/hashtag/cctv9',
- 'info_dict': {
- 'id': 'cctv9',
- 'title': '#cctv9',
- },
- 'playlist_mincount': 350,
- }, {
- 'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
- 'only_matching': True,
- }, {
- 'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
- 'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
- 'only_matching': True
- }, {
- 'note': '/browse/ should redirect to /channel/',
- 'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
- 'only_matching': True
- }, {
- 'note': 'VLPL, should redirect to playlist?list=PL...',
- 'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
- 'info_dict': {
- 'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
- 'uploader': 'NoCopyrightSounds',
- 'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
- 'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
- 'title': 'NCS Releases',
- },
- 'playlist_mincount': 166,
- }, {
- 'note': 'Topic, should redirect to playlist?list=UU...',
- 'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
- 'info_dict': {
- 'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
- 'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
- 'title': 'Uploads from Royalty Free Music - Topic',
- 'uploader': 'Royalty Free Music - Topic',
- },
- 'expected_warnings': [
- 'A channel/user page was given',
- 'The URL does not have a videos tab',
- ],
- 'playlist_mincount': 101,
- }, {
- 'note': 'Topic without a UU playlist',
- 'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
- 'info_dict': {
- 'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
- 'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
- },
- 'expected_warnings': [
- 'A channel/user page was given',
- 'The URL does not have a videos tab',
- 'Falling back to channel URL',
- ],
- 'playlist_mincount': 9,
- }, {
- 'note': 'Youtube music Album',
- 'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
- 'info_dict': {
- 'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
- 'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
- },
- 'playlist_count': 50,
- }, {
- 'note': 'unlisted single video playlist',
- 'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
- 'info_dict': {
- 'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
- 'uploader': 'colethedj',
- 'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
- 'title': 'yt-dlp unlisted playlist test',
- 'availability': 'unlisted'
- },
- 'playlist_count': 1,
- }, {
- 'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
- 'url': 'https://www.youtube.com/feed/recommended',
- 'info_dict': {
- 'id': 'recommended',
- 'title': 'recommended',
- },
- 'playlist_mincount': 50,
- 'params': {
- 'skip_download': True,
- 'extractor_args': {'youtubetab': {'skip': ['webpage']}}
- },
- }, {
- 'note': 'API Fallback: /videos tab, sorted by oldest first',
- 'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
- 'info_dict': {
- 'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
- 'title': 'Cody\'sLab - Videos',
- 'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
- 'uploader': 'Cody\'sLab',
- 'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
- },
- 'playlist_mincount': 650,
- 'params': {
- 'skip_download': True,
- 'extractor_args': {'youtubetab': {'skip': ['webpage']}}
- },
- }, {
- 'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
- 'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
- 'info_dict': {
- 'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
- 'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
- 'title': 'Uploads from Royalty Free Music - Topic',
- 'uploader': 'Royalty Free Music - Topic',
- },
- 'expected_warnings': [
- 'A channel/user page was given',
- 'The URL does not have a videos tab',
- ],
- 'playlist_mincount': 101,
- 'params': {
- 'skip_download': True,
- 'extractor_args': {'youtubetab': {'skip': ['webpage']}}
- },
- }]
-
- @classmethod
- def suitable(cls, url):
- return False if YoutubeIE.suitable(url) else super(
- YoutubeTabIE, cls).suitable(url)
+class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
def _extract_channel_id(self, webpage):
channel_id = self._html_search_meta(
if entry:
yield entry
'''
- def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
-
- def extract_entries(parent_renderer): # this needs to called again for continuation to work with feeds
- contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
- for content in contents:
- if not isinstance(content, dict):
- continue
- is_renderer = try_get(content, lambda x: x['itemSectionRenderer'], dict)
- if not is_renderer:
- renderer = content.get('richItemRenderer')
- if renderer:
- for entry in self._rich_entries(renderer):
- yield entry
- continuation_list[0] = self._extract_continuation(parent_renderer)
+ def _extract_entries(self, parent_renderer, continuation_list):
+ # continuation_list is modified in-place with continuation_list = [continuation_token]
+ continuation_list[:] = [None]
+ contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
+ for content in contents:
+ if not isinstance(content, dict):
+ continue
+ is_renderer = try_get(content, lambda x: x['itemSectionRenderer'], dict)
+ if not is_renderer:
+ renderer = content.get('richItemRenderer')
+ if renderer:
+ for entry in self._rich_entries(renderer):
+ yield entry
+ continuation_list[0] = self._extract_continuation(parent_renderer)
+ continue
+ isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
+ for isr_content in isr_contents:
+ if not isinstance(isr_content, dict):
continue
- isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
- for isr_content in isr_contents:
- if not isinstance(isr_content, dict):
- continue
-
- known_renderers = {
- 'playlistVideoListRenderer': self._playlist_entries,
- 'gridRenderer': self._grid_entries,
- 'shelfRenderer': lambda x: self._shelf_entries(x, tab.get('title') != 'Channels'),
- 'backstagePostThreadRenderer': self._post_thread_entries,
- 'videoRenderer': lambda x: [self._video_entry(x)],
- }
- for key, renderer in isr_content.items():
- if key not in known_renderers:
- continue
- for entry in known_renderers[key](renderer):
- if entry:
- yield entry
- continuation_list[0] = self._extract_continuation(renderer)
- break
- if not continuation_list[0]:
- continuation_list[0] = self._extract_continuation(is_renderer)
+ known_renderers = {
+ 'playlistVideoListRenderer': self._playlist_entries,
+ 'gridRenderer': self._grid_entries,
+ 'shelfRenderer': lambda x: self._shelf_entries(x),
+ 'backstagePostThreadRenderer': self._post_thread_entries,
+ 'videoRenderer': lambda x: [self._video_entry(x)],
+ 'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
+ 'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
+ }
+ for key, renderer in isr_content.items():
+ if key not in known_renderers:
+ continue
+ for entry in known_renderers[key](renderer):
+ if entry:
+ yield entry
+ continuation_list[0] = self._extract_continuation(renderer)
+ break
if not continuation_list[0]:
- continuation_list[0] = self._extract_continuation(parent_renderer)
+ continuation_list[0] = self._extract_continuation(is_renderer)
- continuation_list = [None] # Python 2 does not support nonlocal
+ if not continuation_list[0]:
+ continuation_list[0] = self._extract_continuation(parent_renderer)
+
+ def _entries(self, tab, item_id, ytcfg, account_syncid, visitor_data):
+ continuation_list = [None]
+ extract_entries = lambda x: self._extract_entries(x, continuation_list)
tab_content = try_get(tab, lambda x: x['content'], dict)
if not tab_content:
return
raise ExtractorError(err_note, expected=True)
self.report_warning(err_note, item_id)
- @staticmethod
- def _smuggle_data(entries, data):
- for entry in entries:
- if data:
- entry['url'] = smuggle_url(entry['url'], data)
- yield entry
+ @staticmethod
+ def _smuggle_data(entries, data):
+ for entry in entries:
+ if data:
+ entry['url'] = smuggle_url(entry['url'], data)
+ yield entry
+
+ _SEARCH_PARAMS = None
+
+ def _search_results(self, query, params=NO_DEFAULT):
+ data = {'query': query}
+ if params is NO_DEFAULT:
+ params = self._SEARCH_PARAMS
+ if params:
+ data['params'] = params
+ continuation_list = [None]
+ for page_num in itertools.count(1):
+ data.update(continuation_list[0] or {})
+ search = self._extract_response(
+ item_id='query "%s" page %s' % (query, page_num), ep='search', query=data,
+ check_get_keys=('contents', 'onResponseReceivedCommands'))
+ slr_contents = try_get(
+ search,
+ (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
+ lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
+ list)
+ yield from self._extract_entries({'contents': slr_contents}, continuation_list)
+ if not continuation_list[0]:
+ break
+
+
+class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
+ IE_DESC = 'YouTube Tabs'
+ _VALID_URL = r'''(?x:
+ https?://
+ (?:\w+\.)?
+ (?:
+ youtube(?:kids)?\.com|
+ %(invidious)s
+ )/
+ (?:
+ (?P<channel_type>channel|c|user|browse)/|
+ (?P<not_channel>
+ feed/|hashtag/|
+ (?:playlist|watch)\?.*?\blist=
+ )|
+ (?!(?:%(reserved_names)s)\b) # Direct URLs
+ )
+ (?P<id>[^/?\#&]+)
+ )''' % {
+ 'reserved_names': YoutubeBaseInfoExtractor._RESERVED_NAMES,
+ 'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
+ }
+ IE_NAME = 'youtube:tab'
+
+ _TESTS = [{
+ 'note': 'playlists, multipage',
+ 'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
+ 'playlist_mincount': 94,
+ 'info_dict': {
+ 'id': 'UCqj7Cz7revf5maW9g5pgNcg',
+ 'title': 'Игорь Клейнер - Playlists',
+ 'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
+ 'uploader': 'Игорь Клейнер',
+ 'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
+ },
+ }, {
+ 'note': 'playlists, multipage, different order',
+ 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
+ 'playlist_mincount': 94,
+ 'info_dict': {
+ 'id': 'UCqj7Cz7revf5maW9g5pgNcg',
+ 'title': 'Игорь Клейнер - Playlists',
+ 'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
+ 'uploader_id': 'UCqj7Cz7revf5maW9g5pgNcg',
+ 'uploader': 'Игорь Клейнер',
+ },
+ }, {
+ 'note': 'playlists, series',
+ 'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
+ 'playlist_mincount': 5,
+ 'info_dict': {
+ 'id': 'UCYO_jab_esuFRV4b17AJtAw',
+ 'title': '3Blue1Brown - Playlists',
+ 'description': 'md5:e1384e8a133307dd10edee76e875d62f',
+ 'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
+ 'uploader': '3Blue1Brown',
+ },
+ }, {
+ 'note': 'playlists, singlepage',
+ 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
+ 'playlist_mincount': 4,
+ 'info_dict': {
+ 'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
+ 'title': 'ThirstForScience - Playlists',
+ 'description': 'md5:609399d937ea957b0f53cbffb747a14c',
+ 'uploader': 'ThirstForScience',
+ 'uploader_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
+ }
+ }, {
+ 'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
+ 'only_matching': True,
+ }, {
+ 'note': 'basic, single video playlist',
+ 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
+ 'info_dict': {
+ 'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
+ 'uploader': 'Sergey M.',
+ 'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
+ 'title': 'youtube-dl public playlist',
+ },
+ 'playlist_count': 1,
+ }, {
+ 'note': 'empty playlist',
+ 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
+ 'info_dict': {
+ 'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
+ 'uploader': 'Sergey M.',
+ 'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
+ 'title': 'youtube-dl empty playlist',
+ },
+ 'playlist_count': 0,
+ }, {
+ 'note': 'Home tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Home',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ },
+ 'playlist_mincount': 2,
+ }, {
+ 'note': 'Videos tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Videos',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ },
+ 'playlist_mincount': 975,
+ }, {
+ 'note': 'Videos tab, sorted by popular',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Videos',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ },
+ 'playlist_mincount': 199,
+ }, {
+ 'note': 'Playlists tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Playlists',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ },
+ 'playlist_mincount': 17,
+ }, {
+ 'note': 'Community tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Community',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ },
+ 'playlist_mincount': 18,
+ }, {
+ 'note': 'Channels tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Channels',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ },
+ 'playlist_mincount': 12,
+ }, {
+ 'note': 'Search tab',
+ 'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
+ 'playlist_mincount': 40,
+ 'info_dict': {
+ 'id': 'UCYO_jab_esuFRV4b17AJtAw',
+ 'title': '3Blue1Brown - Search - linear algebra',
+ 'description': 'md5:e1384e8a133307dd10edee76e875d62f',
+ 'uploader': '3Blue1Brown',
+ 'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
+ },
+ }, {
+ 'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
+ 'only_matching': True,
+ }, {
+ 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
+ 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
+ 'info_dict': {
+ 'title': '29C3: Not my department',
+ 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
+ 'uploader': 'Christiaan008',
+ 'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
+ 'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
+ },
+ 'playlist_count': 96,
+ }, {
+ 'note': 'Large playlist',
+ 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
+ 'info_dict': {
+ 'title': 'Uploads from Cauchemar',
+ 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
+ 'uploader': 'Cauchemar',
+ 'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
+ },
+ 'playlist_mincount': 1123,
+ }, {
+ 'note': 'even larger playlist, 8832 videos',
+ 'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
+ 'only_matching': True,
+ }, {
+ 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
+ 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
+ 'info_dict': {
+ 'title': 'Uploads from Interstellar Movie',
+ 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
+ 'uploader': 'Interstellar Movie',
+ 'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
+ },
+ 'playlist_mincount': 21,
+ }, {
+ 'note': 'Playlist with "show unavailable videos" button',
+ 'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
+ 'info_dict': {
+ 'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
+ 'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
+ 'uploader': 'Phim Siêu Nhân Nhật Bản',
+ 'uploader_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
+ },
+ 'playlist_mincount': 200,
+ }, {
+ 'note': 'Playlist with unavailable videos in page 7',
+ 'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
+ 'info_dict': {
+ 'title': 'Uploads from BlankTV',
+ 'id': 'UU8l9frL61Yl5KFOl87nIm2w',
+ 'uploader': 'BlankTV',
+ 'uploader_id': 'UC8l9frL61Yl5KFOl87nIm2w',
+ },
+ 'playlist_mincount': 1000,
+ }, {
+ 'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
+ 'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
+ 'info_dict': {
+ 'title': 'Data Analysis with Dr Mike Pound',
+ 'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
+ 'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
+ 'uploader': 'Computerphile',
+ 'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
+ },
+ 'playlist_mincount': 11,
+ }, {
+ 'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
+ 'only_matching': True,
+ }, {
+ 'note': 'Playlist URL that does not actually serve a playlist',
+ 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
+ 'info_dict': {
+ 'id': 'FqZTN594JQw',
+ 'ext': 'webm',
+ 'title': "Smiley's People 01 detective, Adventure Series, Action",
+ 'uploader': 'STREEM',
+ 'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
+ 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
+ 'upload_date': '20150526',
+ 'license': 'Standard YouTube License',
+ 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
+ 'categories': ['People & Blogs'],
+ 'tags': list,
+ 'view_count': int,
+ 'like_count': int,
+ 'dislike_count': int,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'skip': 'This video is not available.',
+ 'add_ie': [YoutubeIE.ie_key()],
+ }, {
+ 'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
+ 'info_dict': {
+ 'id': '3yImotZU3tw', # This will keep changing
+ 'ext': 'mp4',
+ 'title': compat_str,
+ 'uploader': 'Sky News',
+ 'uploader_id': 'skynews',
+ 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
+ 'upload_date': r're:\d{8}',
+ 'description': compat_str,
+ 'categories': ['News & Politics'],
+ 'tags': list,
+ 'like_count': int,
+ 'dislike_count': int,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'expected_warnings': ['Downloading just video ', 'Ignoring subtitle tracks found in '],
+ }, {
+ 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
+ 'info_dict': {
+ 'id': 'a48o2S1cPoo',
+ 'ext': 'mp4',
+ 'title': 'The Young Turks - Live Main Show',
+ 'uploader': 'The Young Turks',
+ 'uploader_id': 'TheYoungTurks',
+ 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
+ 'upload_date': '20150715',
+ 'license': 'Standard YouTube License',
+ 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
+ 'categories': ['News & Politics'],
+ 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
+ 'like_count': int,
+ 'dislike_count': int,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
+ 'only_matching': True,
+ }, {
+ 'note': 'A channel that is not live. Should raise error',
+ 'url': 'https://www.youtube.com/user/numberphile/live',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/trending',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/library',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/history',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/subscriptions',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/watch_later',
+ 'only_matching': True,
+ }, {
+ 'note': 'Recommended - redirects to home page.',
+ 'url': 'https://www.youtube.com/feed/recommended',
+ 'only_matching': True,
+ }, {
+ 'note': 'inline playlist with not always working continuations',
+ 'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/course',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/zsecurity',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.youtube.com/NASAgovVideo/videos',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/TheYoungTurks/live',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/hashtag/cctv9',
+ 'info_dict': {
+ 'id': 'cctv9',
+ 'title': '#cctv9',
+ },
+ 'playlist_mincount': 350,
+ }, {
+ 'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
+ 'only_matching': True,
+ }, {
+ 'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
+ 'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
+ 'only_matching': True
+ }, {
+ 'note': '/browse/ should redirect to /channel/',
+ 'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
+ 'only_matching': True
+ }, {
+ 'note': 'VLPL, should redirect to playlist?list=PL...',
+ 'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
+ 'info_dict': {
+ 'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
+ 'uploader': 'NoCopyrightSounds',
+ 'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
+ 'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
+ 'title': 'NCS Releases',
+ },
+ 'playlist_mincount': 166,
+ }, {
+ 'note': 'Topic, should redirect to playlist?list=UU...',
+ 'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'info_dict': {
+ 'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
+ 'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'title': 'Uploads from Royalty Free Music - Topic',
+ 'uploader': 'Royalty Free Music - Topic',
+ },
+ 'expected_warnings': [
+ 'A channel/user page was given',
+ 'The URL does not have a videos tab',
+ ],
+ 'playlist_mincount': 101,
+ }, {
+ 'note': 'Topic without a UU playlist',
+ 'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
+ 'info_dict': {
+ 'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
+ 'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
+ },
+ 'expected_warnings': [
+ 'A channel/user page was given',
+ 'The URL does not have a videos tab',
+ 'Falling back to channel URL',
+ ],
+ 'playlist_mincount': 9,
+ }, {
+ 'note': 'Youtube music Album',
+ 'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
+ 'info_dict': {
+ 'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
+ 'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
+ },
+ 'playlist_count': 50,
+ }, {
+ 'note': 'unlisted single video playlist',
+ 'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
+ 'info_dict': {
+ 'uploader_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
+ 'uploader': 'colethedj',
+ 'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
+ 'title': 'yt-dlp unlisted playlist test',
+ 'availability': 'unlisted'
+ },
+ 'playlist_count': 1,
+ }, {
+ 'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
+ 'url': 'https://www.youtube.com/feed/recommended',
+ 'info_dict': {
+ 'id': 'recommended',
+ 'title': 'recommended',
+ },
+ 'playlist_mincount': 50,
+ 'params': {
+ 'skip_download': True,
+ 'extractor_args': {'youtubetab': {'skip': ['webpage']}}
+ },
+ }, {
+ 'note': 'API Fallback: /videos tab, sorted by oldest first',
+ 'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
+ 'info_dict': {
+ 'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
+ 'title': 'Cody\'sLab - Videos',
+ 'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
+ 'uploader': 'Cody\'sLab',
+ 'uploader_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
+ },
+ 'playlist_mincount': 650,
+ 'params': {
+ 'skip_download': True,
+ 'extractor_args': {'youtubetab': {'skip': ['webpage']}}
+ },
+ }, {
+ 'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
+ 'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'info_dict': {
+ 'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
+ 'uploader_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'title': 'Uploads from Royalty Free Music - Topic',
+ 'uploader': 'Royalty Free Music - Topic',
+ },
+ 'expected_warnings': [
+ 'A channel/user page was given',
+ 'The URL does not have a videos tab',
+ ],
+ 'playlist_mincount': 101,
+ 'params': {
+ 'skip_download': True,
+ 'extractor_args': {'youtubetab': {'skip': ['webpage']}}
+ },
+ }]
+
+ @classmethod
+ def suitable(cls, url):
+ return False if YoutubeIE.suitable(url) else super(
+ YoutubeTabIE, cls).suitable(url)
def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {})
info_dict['entries'] = self._smuggle_data(info_dict['entries'], smuggled_data)
return info_dict
- _url_re = re.compile(r'(?P<pre>%s)(?(channel_type)(?P<tab>/\w+))?(?P<post>.*)$' % _VALID_URL)
+ _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(channel_type)(?P<tab>/\w+))?(?P<post>.*)$')
def __real_extract(self, url, smuggled_data):
item_id = self._match_id(url)
compat_opts = self.get_param('compat_opts', [])
def get_mobj(url):
- mobj = self._url_re.match(url).groupdict()
+ mobj = self._URL_RE.match(url).groupdict()
mobj.update((k, '') for k, v in mobj.items() if v is None)
return mobj
- mobj = get_mobj(url)
+ mobj, redirect_warning = get_mobj(url), None
# Youtube returns incomplete data if tabname is not lower case
pre, tab, post, is_channel = mobj['pre'], mobj['tab'].lower(), mobj['post'], not mobj['not_channel']
if is_channel:
if smuggled_data.get('is_music_url'):
- if item_id[:2] == 'VL':
- # Youtube music VL channels have an equivalent playlist
+ if item_id[:2] == 'VL': # Youtube music VL channels have an equivalent playlist
item_id = item_id[2:]
- pre, tab, post, is_channel = 'https://www.youtube.com/playlist?list=%s' % item_id, '', '', False
- elif item_id[:2] == 'MP':
- # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
+ pre, tab, post, is_channel = f'https://www.youtube.com/playlist?list={item_id}', '', '', False
+ elif item_id[:2] == 'MP': # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
mdata = self._extract_tab_endpoint(
- 'https://music.youtube.com/channel/%s' % item_id, item_id, default_client='web_music')
- murl = traverse_obj(
- mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'), get_all=False, expected_type=compat_str)
+ f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
+ murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
+ get_all=False, expected_type=compat_str)
if not murl:
- raise ExtractorError('Failed to resolve album to playlist.')
+ raise ExtractorError('Failed to resolve album to playlist')
return self.url_result(murl, ie=YoutubeTabIE.ie_key())
- elif mobj['channel_type'] == 'browse':
- # Youtube music /browse/ should be changed to /channel/
- pre = 'https://www.youtube.com/channel/%s' % item_id
+ elif mobj['channel_type'] == 'browse': # Youtube music /browse/ should be changed to /channel/
+ pre = f'https://www.youtube.com/channel/{item_id}'
+
if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
# Home URLs should redirect to /videos/
- self.report_warning(
- 'A channel/user page was given. All the channel\'s videos will be downloaded. '
- 'To download only the videos in the home page, add a "/featured" to the URL')
+ redirect_warning = ('A channel/user page was given. All the channel\'s videos will be downloaded. '
+ 'To download only the videos in the home page, add a "/featured" to the URL')
tab = '/videos'
url = ''.join((pre, tab, post))
# Handle both video/playlist URLs
qs = parse_qs(url)
- video_id = qs.get('v', [None])[0]
- playlist_id = qs.get('list', [None])[0]
+ video_id, playlist_id = [qs.get(key, [None])[0] for key in ('v', 'list')]
if not video_id and mobj['not_channel'].startswith('watch'):
if not playlist_id:
# If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
raise ExtractorError('Unable to recognize tab page')
# Common mistake: https://www.youtube.com/watch?list=playlist_id
- self.report_warning('A video URL was given without video ID. Trying to download playlist %s' % playlist_id)
- url = 'https://www.youtube.com/playlist?list=%s' % playlist_id
+ self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
+ url = f'https://www.youtube.com/playlist?list={playlist_id}'
mobj = get_mobj(url)
if video_id and playlist_id:
if self.get_param('noplaylist'):
- self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
- return self.url_result(f'https://www.youtube.com/watch?v={video_id}', ie=YoutubeIE.ie_key(), video_id=video_id)
- self.to_screen('Downloading playlist %s; add --no-playlist to just download video %s' % (playlist_id, video_id))
+ self.to_screen(f'Downloading just video {video_id} because of --no-playlist')
+ return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
+ ie=YoutubeIE.ie_key(), video_id=video_id)
+ self.to_screen(f'Downloading playlist {playlist_id}; add --no-playlist to just download video {video_id}')
data, ytcfg = self._extract_data(url, item_id)
- tabs = try_get(
- data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
+ tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
if tabs:
selected_tab = self._extract_selected_tab(tabs)
tab_name = selected_tab.get('title', '')
# Live tab should have redirected to the video
raise ExtractorError('The channel is not currently live', expected=True)
if mobj['tab'] == '/videos' and tab_name.lower() != mobj['tab'][1:]:
+ redirect_warning = f'The URL does not have a {mobj["tab"][1:]} tab'
if not mobj['not_channel'] and item_id[:2] == 'UC':
# Topic channels don't have /videos. Use the equivalent playlist instead
- self.report_warning('The URL does not have a %s tab. Trying to redirect to playlist UU%s instead' % (mobj['tab'][1:], item_id[2:]))
- pl_id = 'UU%s' % item_id[2:]
- pl_url = 'https://www.youtube.com/playlist?list=%s%s' % (pl_id, mobj['post'])
+ pl_id = f'UU{item_id[2:]}'
+ pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
try:
- data, ytcfg, item_id, url = *self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True), pl_id, pl_url
+ data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True)
except ExtractorError:
- self.report_warning('The playlist gave error. Falling back to channel URL')
- else:
- self.report_warning('The URL does not have a %s tab. %s is being downloaded instead' % (mobj['tab'][1:], tab_name))
+ redirect_warning += ' and the playlist redirect gave error'
+ else:
+ item_id, url, tab_name = pl_id, pl_url, mobj['tab'][1:]
+ redirect_warning += f'. Redirecting to playlist {pl_id} instead'
+ if tab_name.lower() != mobj['tab'][1:]:
+ redirect_warning += f'. {tab_name} tab is being downloaded instead'
- self.write_debug('Final URL: %s' % url)
+ if redirect_warning:
+ self.report_warning(redirect_warning)
+ self.write_debug(f'Final URL: {url}')
# YouTube sometimes provides a button to reload playlist with unavailable videos.
if 'no-youtube-unavailable-videos' not in compat_opts:
data = self._reload_with_unavailable_videos(item_id, data, ytcfg) or data
self._extract_and_report_alerts(data, only_once=True)
- tabs = try_get(
- data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
+ tabs = traverse_obj(data, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs'), expected_type=list)
if tabs:
return self._extract_from_tabs(item_id, ytcfg, data, tabs)
- playlist = try_get(
- data, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
+ playlist = traverse_obj(
+ data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
if playlist:
return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
- video_id = try_get(
- data, lambda x: x['currentVideoEndpoint']['watchEndpoint']['videoId'],
- compat_str) or video_id
+ video_id = traverse_obj(
+ data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
if video_id:
if mobj['tab'] != '/live': # live tab is expected to redirect to video
- self.report_warning('Unable to recognize playlist. Downloading just video %s' % video_id)
- return self.url_result(f'https://www.youtube.com/watch?v={video_id}', ie=YoutubeIE.ie_key(), video_id=video_id)
+ self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
+ return self.url_result(f'https://www.youtube.com/watch?v={video_id}',
+ ie=YoutubeIE.ie_key(), video_id=video_id)
raise ExtractorError('Unable to recognize tab page')
(?:
(?:
youtube(?:kids)?\.com|
- invidio\.us
+ %(invidious)s
)
/.*?\?.*?\blist=
)?
(?P<id>%(playlist_id)s)
- )''' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
+ )''' % {
+ 'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
+ 'invidious': '|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
+ }
IE_NAME = 'youtube:playlist'
_TESTS = [{
'note': 'issue #673',
def _real_extract(self, url):
user_id = self._match_id(url)
return self.url_result(
- 'https://www.youtube.com/user/%s' % user_id,
+ 'https://www.youtube.com/user/%s/videos' % user_id,
ie=YoutubeTabIE.ie_key(), video_id=user_id)
ie=YoutubeTabIE.ie_key())
-class YoutubeSearchIE(SearchInfoExtractor, YoutubeTabIE):
- IE_DESC = 'YouTube searches'
+class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
+ IE_DESC = 'YouTube search'
IE_NAME = 'youtube:search'
_SEARCH_KEY = 'ytsearch'
- _SEARCH_PARAMS = None
+ _SEARCH_PARAMS = 'EgIQAQ%3D%3D' # Videos only
_TESTS = []
- def _search_results(self, query):
- data = {'query': query}
- if self._SEARCH_PARAMS:
- data['params'] = self._SEARCH_PARAMS
- continuation = {}
- for page_num in itertools.count(1):
- data.update(continuation)
- search = self._extract_response(
- item_id='query "%s" page %s' % (query, page_num), ep='search', query=data,
- check_get_keys=('contents', 'onResponseReceivedCommands')
- )
- if not search:
- break
- slr_contents = try_get(
- search,
- (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
- lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
- list)
- if not slr_contents:
- break
-
- # Youtube sometimes adds promoted content to searches,
- # changing the index location of videos and token.
- # So we search through all entries till we find them.
- continuation = None
- for slr_content in slr_contents:
- if not continuation:
- continuation = self._extract_continuation({'contents': [slr_content]})
-
- isr_contents = try_get(
- slr_content,
- lambda x: x['itemSectionRenderer']['contents'],
- list)
- if not isr_contents:
- continue
- for content in isr_contents:
- if not isinstance(content, dict):
- continue
- video = content.get('videoRenderer')
- if not isinstance(video, dict):
- continue
- video_id = video.get('videoId')
- if not video_id:
- continue
-
- yield self._extract_video(video)
-
- if not continuation:
- break
-
-class YoutubeSearchDateIE(YoutubeSearchIE):
+class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
_SEARCH_KEY = 'ytsearchdate'
- IE_DESC = 'YouTube searches, newest videos first'
- _SEARCH_PARAMS = 'CAI%3D'
+ IE_DESC = 'YouTube search, newest videos first'
+ _SEARCH_PARAMS = 'CAISAhAB' # Videos only, sorted by date
-class YoutubeSearchURLIE(YoutubeSearchIE):
+class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
IE_DESC = 'YouTube search URLs with sorting and filter support'
IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
- _SEARCH_KEY = None
_VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?:[^&]+)(?:[&]|$)'
- # _MAX_RESULTS = 100
_TESTS = [{
'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
'playlist_mincount': 5,
'id': 'youtube-dl test video',
'title': 'youtube-dl test video',
}
+ }, {
+ 'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
+ 'playlist_mincount': 5,
+ 'info_dict': {
+ 'id': 'python',
+ 'title': 'python',
+ }
+
}, {
'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
'only_matching': True,
}]
- @classmethod
- def _make_valid_url(cls):
- return cls._VALID_URL
-
def _real_extract(self, url):
qs = parse_qs(url)
query = (qs.get('search_query') or qs.get('q'))[0]
- self._SEARCH_PARAMS = qs.get('sp', ('',))[0]
- return self._get_n_results(query, self._MAX_RESULTS)
+ return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
class YoutubeFeedsInfoExtractor(YoutubeTabIE):
projects / yt-dlp.git / blobdiff