-# coding: utf-8
-from __future__ import unicode_literals
-
import collections
import itertools
import json
from ..compat import (
compat_parse_qs,
compat_str,
- compat_urlparse,
compat_urllib_parse_urlencode,
compat_urllib_parse_urlparse,
)
from ..utils import (
+ ExtractorError,
+ UserNotLive,
+ base_url,
clean_html,
dict_get,
- ExtractorError,
float_or_none,
int_or_none,
+ make_archive_id,
parse_duration,
parse_iso8601,
+ parse_qs,
qualities,
+ str_or_none,
+ traverse_obj,
try_get,
unified_timestamp,
update_url_query,
_USHER_BASE = 'https://usher.ttvnw.net'
_LOGIN_FORM_URL = 'https://www.twitch.tv/login'
_LOGIN_POST_URL = 'https://passport.twitch.tv/login'
- _CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
_NETRC_MACHINE = 'twitch'
_OPERATION_HASHES = {
'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
- 'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
- 'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
+ 'ChannelCollectionsContent': '447aec6a0cc1e8d0a8d7732d47eb0762c336a2294fdb009e9c9d854e49d484b9',
+ 'StreamMetadata': 'a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962',
'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
+ 'VideoAccessToken_Clip': '36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11',
'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
- 'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
+ 'VideoMetadata': '49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad',
+ 'VideoPlayer_ChapterSelectButtonVideo': '8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41',
+ 'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
}
- def _real_initialize(self):
- self._login()
-
- def _login(self):
- username, password = self._get_login_info()
- if username is None:
- return
+ @property
+ def _CLIENT_ID(self):
+ return self._configuration_arg(
+ 'client_id', ['ue6666qo983tsx6so1t0vnawi233wa'], ie_key='Twitch', casesense=True)[0]
+ def _perform_login(self, username, password):
def fail(message):
raise ExtractorError(
'Unable to login. Twitch said: %s' % message, expected=True)
'quality': 10,
'format_note': 'Source',
})
- self._sort_formats(formats)
def _download_base_gql(self, video_id, ops, note, fatal=True):
headers = {
video_id, ops,
'Downloading %s access token GraphQL' % token_kind)['data'][method]
+ def _get_thumbnails(self, thumbnail):
+ return [{
+ 'url': re.sub(r'\d+x\d+(\.\w+)($|(?=[?#]))', r'0x0\g<1>', thumbnail),
+ 'preference': 1,
+ }, {
+ 'url': thumbnail,
+ }] if thumbnail else None
+
class TwitchVodIE(TwitchBaseIE):
IE_NAME = 'twitch:vod'
https?://
(?:
(?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/v(?:ideo)?|videos)/|
- player\.twitch\.tv/\?.*?\bvideo=v?
+ player\.twitch\.tv/\?.*?\bvideo=v?|
+ www\.twitch\.tv/[^/]+/schedule\?vodID=
)
(?P<id>\d+)
'''
'uploader_id': 'riotgames',
'view_count': int,
'start_time': 310,
+ 'chapters': [
+ {
+ 'start_time': 0,
+ 'end_time': 17208,
+ 'title': 'League of Legends'
+ }
+ ],
+ 'live_status': 'was_live',
},
'params': {
# m3u8 download
}, {
'url': 'https://player.twitch.tv/?video=480452374',
'only_matching': True,
+ }, {
+ 'url': 'https://www.twitch.tv/videos/635475444',
+ 'info_dict': {
+ 'id': 'v635475444',
+ 'ext': 'mp4',
+ 'title': 'Riot Games',
+ 'duration': 11643,
+ 'uploader': 'Riot Games',
+ 'uploader_id': 'riotgames',
+ 'timestamp': 1590770569,
+ 'upload_date': '20200529',
+ 'chapters': [
+ {
+ 'start_time': 0,
+ 'end_time': 573,
+ 'title': 'League of Legends'
+ },
+ {
+ 'start_time': 573,
+ 'end_time': 3922,
+ 'title': 'Legends of Runeterra'
+ },
+ {
+ 'start_time': 3922,
+ 'end_time': 11643,
+ 'title': 'Art'
+ }
+ ],
+ 'live_status': 'was_live',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'view_count': int,
+ },
+ 'params': {
+ 'skip_download': True
+ },
+ }, {
+ 'note': 'Storyboards',
+ 'url': 'https://www.twitch.tv/videos/635475444',
+ 'info_dict': {
+ 'id': 'v635475444',
+ 'format_id': 'sb0',
+ 'ext': 'mhtml',
+ 'title': 'Riot Games',
+ 'duration': 11643,
+ 'uploader': 'Riot Games',
+ 'uploader_id': 'riotgames',
+ 'timestamp': 1590770569,
+ 'upload_date': '20200529',
+ 'chapters': [
+ {
+ 'start_time': 0,
+ 'end_time': 573,
+ 'title': 'League of Legends'
+ },
+ {
+ 'start_time': 573,
+ 'end_time': 3922,
+ 'title': 'Legends of Runeterra'
+ },
+ {
+ 'start_time': 3922,
+ 'end_time': 11643,
+ 'title': 'Art'
+ }
+ ],
+ 'live_status': 'was_live',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'view_count': int,
+ 'columns': int,
+ 'rows': int,
+ },
+ 'params': {
+ 'format': 'mhtml',
+ 'skip_download': True
+ }
+ }, {
+ 'note': 'VOD with single chapter',
+ 'url': 'https://www.twitch.tv/videos/1536751224',
+ 'info_dict': {
+ 'id': 'v1536751224',
+ 'ext': 'mp4',
+ 'title': 'Porter Robinson Star Guardian Stream Tour with LilyPichu',
+ 'duration': 8353,
+ 'uploader': 'Riot Games',
+ 'uploader_id': 'riotgames',
+ 'timestamp': 1658267731,
+ 'upload_date': '20220719',
+ 'chapters': [
+ {
+ 'start_time': 0,
+ 'end_time': 8353,
+ 'title': 'League of Legends'
+ }
+ ],
+ 'live_status': 'was_live',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'view_count': int,
+ },
+ 'params': {
+ 'skip_download': True
+ },
+ 'expected_warnings': ['Unable to download JSON metadata: HTTP Error 403: Forbidden']
+ }, {
+ 'url': 'https://www.twitch.tv/tangotek/schedule?vodID=1822395420',
+ 'only_matching': True,
}]
def _download_info(self, item_id):
'channelLogin': '',
'videoID': item_id,
},
+ }, {
+ 'operationName': 'VideoPlayer_ChapterSelectButtonVideo',
+ 'variables': {
+ 'includePrivate': False,
+ 'videoID': item_id,
+ },
+ }, {
+ 'operationName': 'VideoPlayer_VODSeekbarPreviewVideo',
+ 'variables': {
+ 'includePrivate': False,
+ 'videoID': item_id,
+ },
}],
- 'Downloading stream metadata GraphQL')[0]['data']
- video = data.get('video')
+ 'Downloading stream metadata GraphQL')
+
+ video = traverse_obj(data, (..., 'data', 'video'), get_all=False)
if video is None:
- raise ExtractorError(
- 'Video %s does not exist' % item_id, expected=True)
- return self._extract_info_gql(video, item_id)
+ raise ExtractorError(f'Video {item_id} does not exist', expected=True)
- @staticmethod
- def _extract_info(info):
+ video['moments'] = traverse_obj(data, (..., 'data', 'video', 'moments', 'edges', ..., 'node'))
+ video['storyboard'] = traverse_obj(
+ data, (..., 'data', 'video', 'seekPreviewsURL', {url_or_none}), get_all=False)
+
+ return video
+
+ def _extract_info(self, info):
status = info.get('status')
if status == 'recording':
is_live = True
'timestamp': parse_iso8601(info.get('recorded_at')),
'view_count': int_or_none(info.get('views')),
'is_live': is_live,
+ 'was_live': True,
}
- @staticmethod
- def _extract_info_gql(info, item_id):
+ def _extract_chapters(self, info, item_id):
+ if not info.get('moments'):
+ game = traverse_obj(info, ('game', 'displayName'))
+ if game:
+ yield {'title': game}
+ return
+
+ for moment in info['moments']:
+ start_time = int_or_none(moment.get('positionMilliseconds'), 1000)
+ duration = int_or_none(moment.get('durationMilliseconds'), 1000)
+ name = str_or_none(moment.get('description'))
+
+ if start_time is None or duration is None:
+ self.report_warning(f'Important chapter information missing for chapter {name}', item_id)
+ continue
+ yield {
+ 'start_time': start_time,
+ 'end_time': start_time + duration,
+ 'title': name,
+ }
+
+ def _extract_info_gql(self, info, item_id):
vod_id = info.get('id') or item_id
# id backward compatibility for download archives
if vod_id[0] != 'v':
vod_id = 'v%s' % vod_id
thumbnail = url_or_none(info.get('previewThumbnailURL'))
+ is_live = None
if thumbnail:
- for p in ('width', 'height'):
- thumbnail = thumbnail.replace('{%s}' % p, '0')
+ if re.findall(r'/404_processing_[^.?#]+\.png', thumbnail):
+ is_live, thumbnail = True, None
+ else:
+ is_live = False
+
return {
'id': vod_id,
'title': info.get('title') or 'Untitled Broadcast',
'description': info.get('description'),
'duration': int_or_none(info.get('lengthSeconds')),
- 'thumbnail': thumbnail,
+ 'thumbnails': self._get_thumbnails(thumbnail),
'uploader': try_get(info, lambda x: x['owner']['displayName'], compat_str),
'uploader_id': try_get(info, lambda x: x['owner']['login'], compat_str),
'timestamp': unified_timestamp(info.get('publishedAt')),
'view_count': int_or_none(info.get('viewCount')),
+ 'chapters': list(self._extract_chapters(info, item_id)),
+ 'is_live': is_live,
+ 'was_live': True,
}
+ def _extract_storyboard(self, item_id, storyboard_json_url, duration):
+ if not duration or not storyboard_json_url:
+ return
+ spec = self._download_json(storyboard_json_url, item_id, 'Downloading storyboard metadata JSON', fatal=False) or []
+ # sort from highest quality to lowest
+ # This makes sb0 the highest-quality format, sb1 - lower, etc which is consistent with youtube sb ordering
+ spec.sort(key=lambda x: int_or_none(x.get('width')) or 0, reverse=True)
+ base = base_url(storyboard_json_url)
+ for i, s in enumerate(spec):
+ count = int_or_none(s.get('count'))
+ images = s.get('images')
+ if not (images and count):
+ continue
+ fragment_duration = duration / len(images)
+ yield {
+ 'format_id': f'sb{i}',
+ 'format_note': 'storyboard',
+ 'ext': 'mhtml',
+ 'protocol': 'mhtml',
+ 'acodec': 'none',
+ 'vcodec': 'none',
+ 'url': urljoin(base, images[0]),
+ 'width': int_or_none(s.get('width')),
+ 'height': int_or_none(s.get('height')),
+ 'fps': count / duration,
+ 'rows': int_or_none(s.get('rows')),
+ 'columns': int_or_none(s.get('cols')),
+ 'fragments': [{
+ 'url': urljoin(base, path),
+ 'duration': fragment_duration,
+ } for path in images],
+ }
+
def _real_extract(self, url):
vod_id = self._match_id(url)
- info = self._download_info(vod_id)
+ video = self._download_info(vod_id)
+ info = self._extract_info_gql(video, vod_id)
access_token = self._download_access_token(vod_id, 'video', 'id')
formats = self._extract_m3u8_formats(
})),
vod_id, 'mp4', entry_protocol='m3u8_native')
+ formats.extend(self._extract_storyboard(vod_id, video.get('storyboard'), info.get('duration')))
+
self._prefer_source(formats)
info['formats'] = formats
def _real_extract(self, url):
channel_name = self._match_id(url)
- qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+ qs = parse_qs(url)
filter = qs.get('filter', ['all'])[0]
sort = qs.get('sort', ['time'])[0]
broadcast = self._BROADCASTS.get(filter, self._DEFAULT_BROADCAST)
def _real_extract(self, url):
channel_name = self._match_id(url)
- qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+ qs = parse_qs(url)
range = qs.get('range', ['7d'])[0]
clip = self._RANGE.get(range, self._DEFAULT_CLIP)
return self.playlist_result(
'title': 'spamfish - Collections',
},
'playlist_mincount': 3,
+ }, {
+ 'url': 'https://www.twitch.tv/monstercat/videos?filter=collections',
+ 'info_dict': {
+ 'id': 'monstercat',
+ 'title': 'monstercat - Collections',
+ },
+ 'playlist_mincount': 13,
}]
_OPERATION_NAME = 'ChannelCollectionsContent'
# m3u8 download
'skip_download': True,
},
+ 'skip': 'User does not exist',
}, {
'url': 'http://www.twitch.tv/miracle_doto#profile-0',
'only_matching': True,
}, {
'url': 'https://m.twitch.tv/food',
'only_matching': True,
+ }, {
+ 'url': 'https://www.twitch.tv/monstercat',
+ 'info_dict': {
+ 'id': '40500071752',
+ 'display_id': 'monstercat',
+ 'title': 're:Monstercat',
+ 'description': 'md5:0945ad625e615bc8f0469396537d87d9',
+ 'is_live': True,
+ 'timestamp': 1677107190,
+ 'upload_date': '20230222',
+ 'uploader': 'Monstercat',
+ 'uploader_id': 'monstercat',
+ 'live_status': 'is_live',
+ 'thumbnail': 're:https://.*.jpg',
+ 'ext': 'mp4',
+ },
+ 'params': {
+ 'skip_download': 'Livestream',
+ },
}]
@classmethod
stream = user['stream']
if not stream:
- raise ExtractorError('%s is offline' % channel_name, expected=True)
+ raise UserNotLive(video_id=channel_name)
access_token = self._download_access_token(
channel_name, 'stream', 'channelName')
return {
'id': stream_id,
'display_id': channel_name,
- 'title': self._live_title(title),
+ 'title': title,
'description': description,
- 'thumbnail': thumbnail,
+ 'thumbnails': self._get_thumbnails(thumbnail),
'uploader': uploader,
'uploader_id': channel_name,
'timestamp': timestamp,
https?://
(?:
clips\.twitch\.tv/(?:embed\?.*?\bclip=|(?:[^/]+/)*)|
- (?:(?:www|go|m)\.)?twitch\.tv/[^/]+/clip/
+ (?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/)?clip/
)
(?P<id>[^/?#&]+)
'''
'md5': '761769e1eafce0ffebfb4089cb3847cd',
'info_dict': {
'id': '42850523',
+ 'display_id': 'FaintLightGullWholeWheat',
'ext': 'mp4',
'title': 'EA Play 2016 Live from the Novo Theatre',
'thumbnail': r're:^https?://.*\.jpg',
}, {
'url': 'https://go.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
'only_matching': True,
+ }, {
+ 'url': 'https://m.twitch.tv/clip/FaintLightGullWholeWheat',
+ 'only_matching': True,
}]
def _real_extract(self, url):
video_id = self._match_id(url)
- clip = self._download_base_gql(
+ clip = self._download_gql(
+ video_id, [{
+ 'operationName': 'VideoAccessToken_Clip',
+ 'variables': {
+ 'slug': video_id,
+ },
+ }],
+ 'Downloading clip access token GraphQL')[0]['data']['clip']
+
+ if not clip:
+ raise ExtractorError(
+ 'This clip is no longer available', expected=True)
+
+ access_query = {
+ 'sig': clip['playbackAccessToken']['signature'],
+ 'token': clip['playbackAccessToken']['value'],
+ }
+
+ data = self._download_base_gql(
video_id, {
'query': '''{
clip(slug: "%s") {
}
viewCount
}
-}''' % video_id}, 'Downloading clip GraphQL')['data']['clip']
+}''' % video_id}, 'Downloading clip GraphQL', fatal=False)
- if not clip:
- raise ExtractorError(
- 'This clip is no longer available', expected=True)
+ if data:
+ clip = try_get(data, lambda x: x['data']['clip'], dict) or clip
formats = []
for option in clip.get('videoQualities', []):
if not source:
continue
formats.append({
- 'url': source,
+ 'url': update_url_query(source, access_query),
'format_id': option.get('quality'),
'height': int_or_none(option.get('quality')),
'fps': int_or_none(option.get('frameRate')),
})
- self._sort_formats(formats)
thumbnails = []
for thumbnail_id in ('tiny', 'small', 'medium'):
})
thumbnails.append(thumb)
+ old_id = self._search_regex(r'%7C(\d+)(?:-\d+)?.mp4', formats[-1]['url'], 'old id', default=None)
+
return {
'id': clip.get('id') or video_id,
- 'title': clip.get('title') or video_id,
+ '_old_archive_ids': [make_archive_id(self, old_id)] if old_id else None,
+ 'display_id': video_id,
+ 'title': clip.get('title'),
'formats': formats,
'duration': int_or_none(clip.get('durationSeconds')),
- 'views': int_or_none(clip.get('viewCount')),
+ 'view_count': int_or_none(clip.get('viewCount')),
'timestamp': unified_timestamp(clip.get('createdAt')),
'thumbnails': thumbnails,
'creator': try_get(clip, lambda x: x['broadcaster']['displayName'], compat_str),
projects / yt-dlp.git / blobdiff