_TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
_RESERVED_NAMES = (
- r'course|embed|channel|c|user|playlist|watch|w|results|storefront|'
+ r'course|embed|channel|c|user|playlist|watch|w|results|storefront|oops|'
r'shared|index|account|reporthistory|t/terms|about|upload|signin|logout|'
r'feed/(watch_later|history|subscriptions|library|trending|recommended)')
'396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
'397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
}
- _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
+ _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt') # TODO 'json3' raising issues with automatic captions
_GEO_BYPASS = False
return self._parse_json(
uppercase_escape(config), video_id, fatal=False)
- def _get_music_metadata_from_yt_initial(self, yt_initial):
- music_metadata = []
- key_map = {
- 'Album': 'album',
- 'Artist': 'artist',
- 'Song': 'track'
- }
- contents = try_get(yt_initial, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'])
- if type(contents) is list:
- for content in contents:
- music_track = {}
- if type(content) is not dict:
- continue
- videoSecondaryInfoRenderer = try_get(content, lambda x: x['videoSecondaryInfoRenderer'])
- if type(videoSecondaryInfoRenderer) is not dict:
- continue
- rows = try_get(videoSecondaryInfoRenderer, lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'])
- if type(rows) is not list:
- continue
- for row in rows:
- metadataRowRenderer = try_get(row, lambda x: x['metadataRowRenderer'])
- if type(metadataRowRenderer) is not dict:
- continue
- key = try_get(metadataRowRenderer, lambda x: x['title']['simpleText'])
- value = try_get(metadataRowRenderer, lambda x: x['contents'][0]['simpleText']) or \
- try_get(metadataRowRenderer, lambda x: x['contents'][0]['runs'][0]['text'])
- if type(key) is not str or type(value) is not str:
- continue
- if key in key_map:
- if key_map[key] in music_track:
- # we've started on a new track
- music_metadata.append(music_track)
- music_track = {}
- music_track[key_map[key]] = value
- if len(music_track.keys()):
- music_metadata.append(music_track)
- return music_metadata
-
def _get_automatic_captions(self, video_id, webpage):
"""We need the webpage for getting the captions url, pass it as an
argument to speed up the process."""
# Youtube Music Auto-generated description
release_date = release_year = None
if video_description:
- mobj = re.search(r'(?s)Provided to YouTube by [^\n]+\n+(?P<track>[^·]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?', video_description)
+ mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
if mobj:
if not track:
track = mobj.group('track').strip()
if release_year:
release_year = int(release_year)
- yt_initial = self._get_yt_initial_data(video_id, video_webpage)
- if yt_initial:
- music_metadata = self._get_music_metadata_from_yt_initial(yt_initial)
- if len(music_metadata):
- album = music_metadata[0].get('album')
- artist = music_metadata[0].get('artist')
- track = music_metadata[0].get('track')
+ yt_initial_data = self._extract_yt_initial_data(video_id, video_webpage)
+ contents = try_get(yt_initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
+ for content in contents:
+ rows = try_get(content, lambda x: x['videoSecondaryInfoRenderer']['metadataRowContainer']['metadataRowContainerRenderer']['rows'], list) or []
+ multiple_songs = False
+ for row in rows:
+ if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
+ multiple_songs = True
+ break
+ for row in rows:
+ mrr = row.get('metadataRowRenderer') or {}
+ mrr_title = try_get(
+ mrr, lambda x: x['title']['simpleText'], compat_str)
+ mrr_contents = try_get(
+ mrr, lambda x: x['contents'][0], dict) or {}
+ mrr_contents_text = try_get(mrr_contents, [lambda x: x['simpleText'], lambda x: x['runs'][0]['text']], compat_str)
+ if not (mrr_title and mrr_contents_text):
+ continue
+ if mrr_title == 'License':
+ video_license = mrr_contents_text
+ elif not multiple_songs:
+ if mrr_title == 'Album':
+ album = mrr_contents_text
+ elif mrr_title == 'Artist':
+ artist = mrr_contents_text
+ elif mrr_title == 'Song':
+ track = mrr_contents_text
m_episode = re.search(
r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*•\s*E(?P<episode>\d+)</span>',
(?:
(?:channel|c|user)/|
(?P<not_channel>
+ feed/|
(?:playlist|watch)\?.*?\blist=
)|
(?!(%s)([/#?]|$)) # Direct URLs
}, {
'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
'only_matching': True,
- },
+ }, {
+ 'url': 'https://www.youtube.com/feed/trending',
+ 'only_matching': True,
+ }, {
+ # needs auth
+ 'url': 'https://www.youtube.com/feed/library',
+ 'only_matching': True,
+ }, {
+ # needs auth
+ 'url': 'https://www.youtube.com/feed/history',
+ 'only_matching': True,
+ }, {
+ # needs auth
+ 'url': 'https://www.youtube.com/feed/subscriptions',
+ 'only_matching': True,
+ }, {
+ # needs auth
+ 'url': 'https://www.youtube.com/feed/watch_later',
+ 'only_matching': True,
+ }, {
+ # no longer available?
+ 'url': 'https://www.youtube.com/feed/recommended',
+ 'only_matching': True,
+ }
# TODO
# {
# 'url': 'https://www.youtube.com/TheYoungTurks/live',
'https://www.youtube.com/channel/%s' % channel_id,
ie=YoutubeTabIE.ie_key(), video_title=title)
- def _shelf_entries_trimmed(self, shelf_renderer):
- renderer = try_get(
- shelf_renderer, lambda x: x['content']['horizontalListRenderer'], dict)
- if not renderer:
+ def _shelf_entries_from_content(self, shelf_renderer):
+ content = shelf_renderer.get('content')
+ if not isinstance(content, dict):
return
- # TODO: add support for nested playlists so each shelf is processed
- # as separate playlist
- # TODO: this includes only first N items
- for entry in self._grid_entries(renderer):
- yield entry
+ renderer = content.get('gridRenderer')
+ if renderer:
+ # TODO: add support for nested playlists so each shelf is processed
+ # as separate playlist
+ # TODO: this includes only first N items
+ for entry in self._grid_entries(renderer):
+ yield entry
+ renderer = content.get('horizontalListRenderer')
+ if renderer:
+ # TODO
+ pass
def _shelf_entries(self, shelf_renderer):
ep = try_get(
shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
compat_str)
shelf_url = urljoin('https://www.youtube.com', ep)
- if not shelf_url:
- return
- title = try_get(
- shelf_renderer, lambda x: x['title']['runs'][0]['text'], compat_str)
- yield self.url_result(shelf_url, video_title=title)
+ if shelf_url:
+ title = try_get(
+ shelf_renderer, lambda x: x['title']['runs'][0]['text'], compat_str)
+ yield self.url_result(shelf_url, video_title=title)
+ # Shelf may not contain shelf URL, fallback to extraction from content
+ for entry in self._shelf_entries_from_content(shelf_renderer):
+ yield entry
def _playlist_entries(self, video_list_renderer):
for content in video_list_renderer['contents']:
continue
yield self._extract_video(renderer)
+ r""" # Not needed in the new implementation
def _itemSection_entries(self, item_sect_renderer):
for content in item_sect_renderer['contents']:
if not isinstance(content, dict):
if not video_id:
continue
yield self._extract_video(renderer)
+ """
def _rich_entries(self, rich_grid_renderer):
renderer = try_get(
self._playlist_entries(playlist), playlist_id=playlist_id,
playlist_title=title)
+ def _extract_alerts(self, data):
+ for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
+ for renderer in alert_dict:
+ alert = alert_dict[renderer]
+ alert_type = alert.get('type')
+ if not alert_type:
+ continue
+ message = try_get(alert, lambda x: x['text']['simpleText'], compat_str)
+ if message:
+ yield alert_type, message
+ for run in try_get(alert, lambda x: x['text']['runs'], list) or []:
+ message = try_get(run, lambda x: x['text'], compat_str)
+ if message:
+ yield alert_type, message
+
def _real_extract(self, url):
item_id = self._match_id(url)
url = compat_urlparse.urlunparse(
qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
video_id = qs.get('v', [None])[0]
playlist_id = qs.get('list', [None])[0]
+
+ if is_home.group('not_channel') is not None and is_home.group('not_channel').startswith('watch') and not video_id:
+ if playlist_id:
+ self._downloader.report_warning('%s is not a valid Youtube URL. Trying to download playlist %s' % (url, playlist_id))
+ url = 'https://www.youtube.com/playlist?list=%s' % playlist_id
+ # return self.url_result(playlist_id, ie=YoutubePlaylistIE.ie_key())
+ else:
+ raise ExtractorError('Unable to recognize tab page')
if video_id and playlist_id:
if self._downloader.params.get('noplaylist'):
self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
+
webpage = self._download_webpage(url, item_id)
identity_token = self._search_regex(
r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
'identity token', default=None)
data = self._extract_yt_initial_data(item_id, webpage)
+ for alert_type, alert_message in self._extract_alerts(data):
+ self._downloader.report_warning('YouTube said: %s - %s' % (alert_type, alert_message))
tabs = try_get(
data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
if tabs:
ie=YoutubeTabIE.ie_key(), video_id=user_id)
-class YoutubeFavouritesIE(InfoExtractor):
+class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
IE_NAME = 'youtube:favorites'
IE_DESC = 'YouTube.com liked videos, ":ytfav" for short (requires authentication)'
_VALID_URL = r':ytfav(?:ou?rite)?s?'
list)
if not slr_contents:
break
- isr_contents = try_get(
- slr_contents,
- lambda x: x[0]['itemSectionRenderer']['contents'],
- list)
+
+ isr_contents = []
+ continuation_token = None
+ # Youtube sometimes adds promoted content to searches,
+ # changing the index location of videos and token.
+ # So we search through all entries till we find them.
+ for index, isr in enumerate(slr_contents):
+ if not isr_contents:
+ isr_contents = try_get(
+ slr_contents,
+ (lambda x: x[index]['itemSectionRenderer']['contents']),
+ list)
+ for content in isr_contents:
+ if content.get('videoRenderer') is not None:
+ break
+ else:
+ isr_contents = []
+
+ if continuation_token is None:
+ continuation_token = try_get(
+ slr_contents,
+ lambda x: x[index]['continuationItemRenderer']['continuationEndpoint']['continuationCommand'][
+ 'token'],
+ compat_str)
+ if continuation_token is not None and isr_contents:
+ break
+
if not isr_contents:
break
for content in isr_contents:
}
if total == n:
return
- token = try_get(
- slr_contents,
- lambda x: x[1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
- compat_str)
- if not token:
+ if not continuation_token:
break
- data['continuation'] = token
+ data['continuation'] = continuation_token
def _get_n_results(self, query, n):
"""Get a specified number of results for a query"""
class YoutubeFeedsInfoExtractor(YoutubeTabIE):
"""
Base class for feed extractors
- Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
+ Subclasses must define the _FEED_NAME property.
"""
_LOGIN_REQUIRED = True
# _MAX_PAGES = 5
def _real_initialize(self):
self._login()
- def _shelf_entries(self, shelf_renderer):
- renderer = try_get(shelf_renderer, lambda x: x['content']['gridRenderer'], dict)
- if not renderer:
- return
- for entry in self._grid_entries(renderer):
- yield entry
-
- def _extract_from_tabs(self, item_id, webpage, data, tabs, identity_token):
- selected_tab = self._extract_selected_tab(tabs)
- return self.playlist_result(
- self._entries(selected_tab['content'], identity_token),
- playlist_title=self._PLAYLIST_TITLE)
-
def _real_extract(self, url):
- item_id = self._FEED_NAME
- url = 'https://www.youtube.com/feed/%s' % self._FEED_NAME
- webpage = self._download_webpage(url, item_id)
- identity_token = self._search_regex(
- r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
- 'identity token', default=None)
- data = self._extract_yt_initial_data(item_id, webpage)
- tabs = try_get(
- data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
- if tabs:
- return self._extract_from_tabs(item_id, webpage, data, tabs, identity_token)
- # Failed to recognize
- raise ExtractorError('Unable to recognize feed page')
+ return self.url_result(
+ 'https://www.youtube.com/feed/%s' % self._FEED_NAME,
+ ie=YoutubeTabIE.ie_key())
class YoutubeWatchLaterIE(InfoExtractor):
IE_NAME = 'youtube:watchlater'
- _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/watch_later|:ytwatchlater|WL'
-
IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
+ _VALID_URL = r':ytwatchlater'
_TESTS = [{
- 'url': 'https://www.youtube.com/feed/watch_later',
- 'only_matching': True,
- }, {
'url': ':ytwatchlater',
'only_matching': True,
}]
class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
- _VALID_URL = r'https?://(?:www\.)?youtube\.com(?:/feed/recommended|/?[?#]|/?$)|:ytrec(?:ommended)?'
+ _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
_FEED_NAME = 'recommended'
- _PLAYLIST_TITLE = 'Youtube Recommended videos'
+ _TESTS = [{
+ 'url': ':ytrec',
+ 'only_matching': True,
+ }, {
+ 'url': ':ytrecommended',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://youtube.com',
+ 'only_matching': True,
+ }]
class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsub(?:scription)?s?'
IE_DESC = 'YouTube.com subscriptions feed, ":ytsubs" for short (requires authentication)'
+ _VALID_URL = r':ytsub(?:scription)?s?'
_FEED_NAME = 'subscriptions'
- _PLAYLIST_TITLE = 'Youtube Subscriptions'
+ _TESTS = [{
+ 'url': ':ytsubs',
+ 'only_matching': True,
+ }, {
+ 'url': ':ytsubscriptions',
+ 'only_matching': True,
+ }]
class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
- _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
+ _VALID_URL = r':ythistory'
_FEED_NAME = 'history'
- _PLAYLIST_TITLE = 'Youtube History'
+ _TESTS = [{
+ 'url': ':ythistory',
+ 'only_matching': True,
+ }]
class YoutubeTruncatedURLIE(InfoExtractor):
projects / yt-dlp.git / blobdiff