from .common import InfoExtractor
from ..compat import (
- compat_kwargs,
compat_parse_qs,
compat_str,
compat_urlparse,
parse_duration,
parse_iso8601,
qualities,
- str_or_none,
try_get,
unified_timestamp,
update_url_query,
_CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
_NETRC_MACHINE = 'twitch'
- def _handle_error(self, response):
- if not isinstance(response, dict):
- return
- error = response.get('error')
- if error:
- raise ExtractorError(
- '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
- expected=True)
-
- def _call_api(self, path, item_id, *args, **kwargs):
- headers = kwargs.get('headers', {}).copy()
- headers.update({
- 'Accept': 'application/vnd.twitchtv.v5+json; charset=UTF-8',
- 'Client-ID': self._CLIENT_ID,
- })
- kwargs.update({
- 'headers': headers,
- 'expected_status': (400, 410),
- })
- response = self._download_json(
- '%s/%s' % (self._API_BASE, path), item_id,
- *args, **compat_kwargs(kwargs))
- self._handle_error(response)
- return response
+ _OPERATION_HASHES = {
+ 'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
+ 'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
+ 'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
+ 'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
+ 'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
+ 'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
+ 'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
+ 'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
+ }
def _real_initialize(self):
self._login()
})
self._sort_formats(formats)
- def _download_access_token(self, channel_name):
- return self._call_api(
- 'api/channels/%s/access_token' % channel_name, channel_name,
- 'Downloading access token JSON')
+ def _download_base_gql(self, video_id, ops, note, fatal=True):
+ return self._download_json(
+ 'https://gql.twitch.tv/gql', video_id, note,
+ data=json.dumps(ops).encode(),
+ headers={
+ 'Content-Type': 'text/plain;charset=UTF-8',
+ 'Client-ID': self._CLIENT_ID,
+ }, fatal=fatal)
- def _extract_channel_id(self, token, channel_name):
- return compat_str(self._parse_json(token, channel_name)['channel_id'])
+ def _download_gql(self, video_id, ops, note, fatal=True):
+ for op in ops:
+ op['extensions'] = {
+ 'persistedQuery': {
+ 'version': 1,
+ 'sha256Hash': self._OPERATION_HASHES[op['operationName']],
+ }
+ }
+ return self._download_base_gql(video_id, ops, note)
+
+ def _download_access_token(self, video_id, token_kind, param_name):
+ method = '%sPlaybackAccessToken' % token_kind
+ ops = {
+ 'query': '''{
+ %s(
+ %s: "%s",
+ params: {
+ platform: "web",
+ playerBackend: "mediaplayer",
+ playerType: "site"
+ }
+ )
+ {
+ value
+ signature
+ }
+ }''' % (method, param_name, video_id),
+ }
+ return self._download_base_gql(
+ video_id, ops,
+ 'Downloading %s access token GraphQL' % token_kind)['data'][method]
class TwitchVodIE(TwitchBaseIE):
)
(?P<id>\d+)
'''
- _ITEM_TYPE = 'vod'
- _ITEM_SHORTCUT = 'v'
_TESTS = [{
'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
'title': 'LCK Summer Split - Week 6 Day 1',
'thumbnail': r're:^https?://.*\.jpg$',
'duration': 17208,
- 'timestamp': 1435131709,
+ 'timestamp': 1435131734,
'upload_date': '20150624',
'uploader': 'Riot Games',
'uploader_id': 'riotgames',
}]
def _download_info(self, item_id):
- return self._extract_info(
- self._call_api(
- 'kraken/videos/%s' % item_id, item_id,
- 'Downloading video info JSON'))
+ data = self._download_gql(
+ item_id, [{
+ 'operationName': 'VideoMetadata',
+ 'variables': {
+ 'channelLogin': '',
+ 'videoID': item_id,
+ },
+ }],
+ 'Downloading stream metadata GraphQL')[0]['data']
+ video = data.get('video')
+ if video is None:
+ raise ExtractorError(
+ 'Video %s does not exist' % item_id, expected=True)
+ return self._extract_info_gql(video, item_id)
@staticmethod
def _extract_info(info):
'is_live': is_live,
}
+ @staticmethod
+ def _extract_info_gql(info, item_id):
+ vod_id = info.get('id') or item_id
+ # id backward compatibility for download archives
+ if vod_id[0] != 'v':
+ vod_id = 'v%s' % vod_id
+ thumbnail = url_or_none(info.get('previewThumbnailURL'))
+ if thumbnail:
+ for p in ('width', 'height'):
+ thumbnail = thumbnail.replace('{%s}' % p, '0')
+ return {
+ 'id': vod_id,
+ 'title': info.get('title') or 'Untitled Broadcast',
+ 'description': info.get('description'),
+ 'duration': int_or_none(info.get('lengthSeconds')),
+ 'thumbnail': thumbnail,
+ 'uploader': try_get(info, lambda x: x['owner']['displayName'], compat_str),
+ 'uploader_id': try_get(info, lambda x: x['owner']['login'], compat_str),
+ 'timestamp': unified_timestamp(info.get('publishedAt')),
+ 'view_count': int_or_none(info.get('viewCount')),
+ }
+
def _real_extract(self, url):
vod_id = self._match_id(url)
info = self._download_info(vod_id)
- access_token = self._call_api(
- 'api/vods/%s/access_token' % vod_id, vod_id,
- 'Downloading %s access token' % self._ITEM_TYPE)
+ access_token = self._download_access_token(vod_id, 'video', 'id')
formats = self._extract_m3u8_formats(
'%s/vod/%s.m3u8?%s' % (
'allow_spectre': 'true',
'player': 'twitchweb',
'playlist_include_framerate': 'true',
- 'nauth': access_token['token'],
- 'nauthsig': access_token['sig'],
+ 'nauth': access_token['value'],
+ 'nauthsig': access_token['signature'],
})),
vod_id, 'mp4', entry_protocol='m3u8_native')
return {
'_type': 'url_transparent',
'ie_key': TwitchVodIE.ie_key(),
- 'id': video_id,
+ 'id': 'v' + video_id,
'url': 'https://www.twitch.tv/videos/%s' % video_id,
'title': node.get('title'),
'thumbnail': node.get('previewThumbnailURL'),
}
-class TwitchGraphQLBaseIE(TwitchBaseIE):
- _PAGE_LIMIT = 100
-
- def _download_gql(self, video_id, op, variables, sha256_hash, note, fatal=True):
- return self._download_json(
- 'https://gql.twitch.tv/gql', video_id, note,
- data=json.dumps({
- 'operationName': op,
- 'variables': variables,
- 'extensions': {
- 'persistedQuery': {
- 'version': 1,
- 'sha256Hash': sha256_hash,
- }
- }
- }).encode(),
- headers={
- 'Content-Type': 'text/plain;charset=UTF-8',
- 'Client-ID': self._CLIENT_ID,
- }, fatal=fatal)
-
-
-class TwitchCollectionIE(TwitchGraphQLBaseIE):
+class TwitchCollectionIE(TwitchBaseIE):
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/collections/(?P<id>[^/]+)'
_TESTS = [{
}]
_OPERATION_NAME = 'CollectionSideBar'
- _SHA256_HASH = '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14'
def _real_extract(self, url):
collection_id = self._match_id(url)
collection = self._download_gql(
- collection_id, self._OPERATION_NAME,
- {'collectionID': collection_id}, self._SHA256_HASH,
- 'Downloading collection GraphQL')['data']['collection']
+ collection_id, [{
+ 'operationName': self._OPERATION_NAME,
+ 'variables': {'collectionID': collection_id},
+ }],
+ 'Downloading collection GraphQL')[0]['data']['collection']
title = collection.get('title')
entries = []
for edge in collection['items']['edges']:
entries, playlist_id=collection_id, playlist_title=title)
-class TwitchPlaylistBaseIE(TwitchGraphQLBaseIE):
+class TwitchPlaylistBaseIE(TwitchBaseIE):
+ _PAGE_LIMIT = 100
+
def _entries(self, channel_name, *args):
cursor = None
variables_common = self._make_variables(channel_name, *args)
if cursor:
variables['cursor'] = cursor
page = self._download_gql(
- channel_name, self._OPERATION_NAME, variables,
- self._SHA256_HASH,
+ channel_name, [{
+ 'operationName': self._OPERATION_NAME,
+ 'variables': variables,
+ }],
'Downloading %ss GraphQL page %s' % (self._NODE_KIND, page_num),
fatal=False)
if not page:
break
edges = try_get(
- page, lambda x: x['data']['user'][entries_key]['edges'], list)
+ page, lambda x: x[0]['data']['user'][entries_key]['edges'], list)
if not edges:
break
for edge in edges:
if not cursor or not isinstance(cursor, compat_str):
break
- # Deprecated kraken v5 API
- def _entries_kraken(self, channel_name, broadcast_type, sort):
- access_token = self._download_access_token(channel_name)
- channel_id = self._extract_channel_id(access_token['token'], channel_name)
- offset = 0
- counter_override = None
- for counter in itertools.count(1):
- response = self._call_api(
- 'kraken/channels/%s/videos/' % channel_id,
- channel_id,
- 'Downloading video JSON page %s' % (counter_override or counter),
- query={
- 'offset': offset,
- 'limit': self._PAGE_LIMIT,
- 'broadcast_type': broadcast_type,
- 'sort': sort,
- })
- videos = response.get('videos')
- if not isinstance(videos, list):
- break
- for video in videos:
- if not isinstance(video, dict):
- continue
- video_url = url_or_none(video.get('url'))
- if not video_url:
- continue
- yield {
- '_type': 'url_transparent',
- 'ie_key': TwitchVodIE.ie_key(),
- 'id': video.get('_id'),
- 'url': video_url,
- 'title': video.get('title'),
- 'description': video.get('description'),
- 'timestamp': unified_timestamp(video.get('published_at')),
- 'duration': float_or_none(video.get('length')),
- 'view_count': int_or_none(video.get('views')),
- 'language': video.get('language'),
- }
- offset += self._PAGE_LIMIT
- total = int_or_none(response.get('_total'))
- if total and offset >= total:
- break
-
class TwitchVideosIE(TwitchPlaylistBaseIE):
_VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:videos|profile)'
'views': 'Popular',
}
- _SHA256_HASH = 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb'
_OPERATION_NAME = 'FilterableVideoTower_Videos'
_ENTRY_KIND = 'video'
_EDGE_KIND = 'VideoEdge'
# NB: values other than 20 result in skipped videos
_PAGE_LIMIT = 20
- _SHA256_HASH = 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777'
_OPERATION_NAME = 'ClipsCards__User'
_ENTRY_KIND = 'clip'
_EDGE_KIND = 'ClipEdge'
'playlist_mincount': 3,
}]
- _SHA256_HASH = '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84'
_OPERATION_NAME = 'ChannelCollectionsContent'
_ENTRY_KIND = 'collection'
_EDGE_KIND = 'CollectionsItemEdge'
else super(TwitchStreamIE, cls).suitable(url))
def _real_extract(self, url):
- channel_name = self._match_id(url)
-
- access_token = self._download_access_token(channel_name)
-
- token = access_token['token']
- channel_id = self._extract_channel_id(token, channel_name)
+ channel_name = self._match_id(url).lower()
+
+ gql = self._download_gql(
+ channel_name, [{
+ 'operationName': 'StreamMetadata',
+ 'variables': {'channelLogin': channel_name},
+ }, {
+ 'operationName': 'ComscoreStreamingQuery',
+ 'variables': {
+ 'channel': channel_name,
+ 'clipSlug': '',
+ 'isClip': False,
+ 'isLive': True,
+ 'isVodOrCollection': False,
+ 'vodID': '',
+ },
+ }, {
+ 'operationName': 'VideoPreviewOverlay',
+ 'variables': {'login': channel_name},
+ }],
+ 'Downloading stream GraphQL')
+
+ user = gql[0]['data']['user']
+
+ if not user:
+ raise ExtractorError(
+ '%s does not exist' % channel_name, expected=True)
- stream = self._call_api(
- 'kraken/streams/%s?stream_type=all' % channel_id,
- channel_id, 'Downloading stream JSON').get('stream')
+ stream = user['stream']
if not stream:
- raise ExtractorError('%s is offline' % channel_id, expected=True)
+ raise ExtractorError('%s is offline' % channel_name, expected=True)
- # Channel name may be typed if different case than the original channel name
- # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
- # an invalid m3u8 URL. Working around by use of original channel name from stream
- # JSON and fallback to lowercase if it's not available.
- channel_name = try_get(
- stream, lambda x: x['channel']['name'],
- compat_str) or channel_name.lower()
+ access_token = self._download_access_token(
+ channel_name, 'stream', 'channelName')
+ token = access_token['value']
+ stream_id = stream.get('id') or channel_name
query = {
'allow_source': 'true',
'allow_audio_only': 'true',
'player': 'twitchweb',
'playlist_include_framerate': 'true',
'segment_preference': '4',
- 'sig': access_token['sig'].encode('utf-8'),
+ 'sig': access_token['signature'].encode('utf-8'),
'token': token.encode('utf-8'),
}
formats = self._extract_m3u8_formats(
- '%s/api/channel/hls/%s.m3u8?%s'
- % (self._USHER_BASE, channel_name, compat_urllib_parse_urlencode(query)),
- channel_id, 'mp4')
+ '%s/api/channel/hls/%s.m3u8' % (self._USHER_BASE, channel_name),
+ stream_id, 'mp4', query=query)
self._prefer_source(formats)
view_count = stream.get('viewers')
- timestamp = parse_iso8601(stream.get('created_at'))
+ timestamp = unified_timestamp(stream.get('createdAt'))
- channel = stream['channel']
- title = self._live_title(channel.get('display_name') or channel.get('name'))
- description = channel.get('status')
+ sq_user = try_get(gql, lambda x: x[1]['data']['user'], dict) or {}
+ uploader = sq_user.get('displayName')
+ description = try_get(
+ sq_user, lambda x: x['broadcastSettings']['title'], compat_str)
- thumbnails = []
- for thumbnail_key, thumbnail_url in stream['preview'].items():
- m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
- if not m:
- continue
- thumbnails.append({
- 'url': thumbnail_url,
- 'width': int(m.group('width')),
- 'height': int(m.group('height')),
- })
+ thumbnail = url_or_none(try_get(
+ gql, lambda x: x[2]['data']['user']['stream']['previewImageURL'],
+ compat_str))
+
+ title = uploader or channel_name
+ stream_type = stream.get('type')
+ if stream_type in ['rerun', 'live']:
+ title += ' (%s)' % stream_type
return {
- 'id': str_or_none(stream.get('_id')) or channel_id,
+ 'id': stream_id,
'display_id': channel_name,
- 'title': title,
+ 'title': self._live_title(title),
'description': description,
- 'thumbnails': thumbnails,
- 'uploader': channel.get('display_name'),
- 'uploader_id': channel.get('name'),
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'uploader_id': channel_name,
'timestamp': timestamp,
'view_count': view_count,
'formats': formats,
- 'is_live': True,
+ 'is_live': stream_type == 'live',
}
def _real_extract(self, url):
video_id = self._match_id(url)
- clip = self._download_json(
- 'https://gql.twitch.tv/gql', video_id, data=json.dumps({
+ clip = self._download_base_gql(
+ video_id, {
'query': '''{
clip(slug: "%s") {
broadcaster {
}
viewCount
}
-}''' % video_id,
- }).encode(), headers={
- 'Client-ID': self._CLIENT_ID,
- })['data']['clip']
+}''' % video_id}, 'Downloading clip GraphQL')['data']['clip']
if not clip:
raise ExtractorError(
projects / yt-dlp.git / blobdiff