[yt-dlp.git] / yt_dlp / extractor / trovo.py

# coding: utf-8
from __future__ import unicode_literals

import itertools
import json

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    format_field,
    int_or_none,
    str_or_none,
    try_get,
)


class TrovoBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?trovo\.live/'
    _HEADERS = {'Origin': 'https://trovo.live'}

    def _call_api(self, video_id, query=None, data=None):
        return self._download_json(
            'https://gql.trovo.live/', video_id, query=query, data=data,
            headers={'Accept': 'application/json'})

    def _extract_streamer_info(self, data):
        streamer_info = data.get('streamerInfo') or {}
        username = streamer_info.get('userName')
        return {
            'uploader': streamer_info.get('nickName'),
            'uploader_id': str_or_none(streamer_info.get('uid')),
            'uploader_url': format_field(username, template='https://trovo.live/%s'),
        }


class TrovoIE(TrovoBaseIE):
    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?!(?:clip|video)/)(?P<id>[^/?&#]+)'

    def _real_extract(self, url):
        username = self._match_id(url)
        live_info = self._call_api(username, query={
            'query': '''{
  getLiveInfo(params: {userName: "%s"}) {
    isLive
    programInfo {
      coverUrl
      id
      streamInfo {
        desc
        playUrl
      }
      title
    }
    streamerInfo {
        nickName
        uid
        userName
    }
  }
}''' % username,
        })['data']['getLiveInfo']
        if live_info.get('isLive') == 0:
            raise ExtractorError('%s is offline' % username, expected=True)
        program_info = live_info['programInfo']
        program_id = program_info['id']
        title = program_info['title']

        formats = []
        for stream_info in (program_info.get('streamInfo') or []):
            play_url = stream_info.get('playUrl')
            if not play_url:
                continue
            format_id = stream_info.get('desc')
            formats.append({
                'format_id': format_id,
                'height': int_or_none(format_id[:-1]) if format_id else None,
                'url': play_url,
                'http_headers': self._HEADERS,
            })
        self._sort_formats(formats)

        info = {
            'id': program_id,
            'title': title,
            'formats': formats,
            'thumbnail': program_info.get('coverUrl'),
            'is_live': True,
        }
        info.update(self._extract_streamer_info(live_info))
        return info


class TrovoVodIE(TrovoBaseIE):
    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:clip|video)/(?P<id>[^/?&#]+)'
    _TESTS = [{
        'url': 'https://trovo.live/video/ltv-100095501_100095501_1609596043',
        'info_dict': {
            'id': 'ltv-100095501_100095501_1609596043',
            'ext': 'mp4',
            'title': 'Spontaner 12 Stunden Stream! - Ok Boomer!',
            'uploader': 'Exsl',
            'timestamp': 1609640305,
            'upload_date': '20210103',
            'uploader_id': '100095501',
            'duration': 43977,
            'view_count': int,
            'like_count': int,
            'comment_count': int,
            'comments': 'mincount:8',
            'categories': ['Grand Theft Auto V'],
        },
        'skip': '404'
    }, {
        'url': 'https://trovo.live/clip/lc-5285890810184026005',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        vid = self._match_id(url)
        resp = self._call_api(vid, data=json.dumps([{
            'query': '''{
  batchGetVodDetailInfo(params: {vids: ["%s"]}) {
    VodDetailInfos
  }
}''' % vid,
        }, {
            'query': '''{
  getCommentList(params: {appInfo: {postID: "%s"}, pageSize: 1000000000, preview: {}}) {
    commentList {
      author {
        nickName
        uid
      }
      commentID
      content
      createdAt
      parentID
    }
  }
}''' % vid,
        }]).encode())
        vod_detail_info = resp[0]['data']['batchGetVodDetailInfo']['VodDetailInfos'][vid]
        vod_info = vod_detail_info['vodInfo']
        title = vod_info['title']

        language = vod_info.get('languageName')
        formats = []
        for play_info in (vod_info.get('playInfos') or []):
            play_url = play_info.get('playUrl')
            if not play_url:
                continue
            format_id = play_info.get('desc')
            formats.append({
                'ext': 'mp4',
                'filesize': int_or_none(play_info.get('fileSize')),
                'format_id': format_id,
                'height': int_or_none(format_id[:-1]) if format_id else None,
                'language': language,
                'protocol': 'm3u8_native',
                'tbr': int_or_none(play_info.get('bitrate')),
                'url': play_url,
                'http_headers': self._HEADERS,
            })
        self._sort_formats(formats)

        category = vod_info.get('categoryName')
        get_count = lambda x: int_or_none(vod_info.get(x + 'Num'))

        comment_list = try_get(resp, lambda x: x[1]['data']['getCommentList']['commentList'], list) or []
        comments = []
        for comment in comment_list:
            content = comment.get('content')
            if not content:
                continue
            author = comment.get('author') or {}
            parent = comment.get('parentID')
            comments.append({
                'author': author.get('nickName'),
                'author_id': str_or_none(author.get('uid')),
                'id': str_or_none(comment.get('commentID')),
                'text': content,
                'timestamp': int_or_none(comment.get('createdAt')),
                'parent': 'root' if parent == 0 else str_or_none(parent),
            })

        info = {
            'id': vid,
            'title': title,
            'formats': formats,
            'thumbnail': vod_info.get('coverUrl'),
            'timestamp': int_or_none(vod_info.get('publishTs')),
            'duration': int_or_none(vod_info.get('duration')),
            'view_count': get_count('watch'),
            'like_count': get_count('like'),
            'comment_count': get_count('comment'),
            'comments': comments,
            'categories': [category] if category else None,
        }
        info.update(self._extract_streamer_info(vod_detail_info))
        return info


class TrovoChannelBaseIE(TrovoBaseIE):
    def _get_vod_json(self, page, uid):
        raise NotImplementedError('This method must be implemented by subclasses')

    def _entries(self, uid):
        for page in itertools.count(1):
            vod_json = self._get_vod_json(page, uid)
            vods = vod_json.get('vodInfos', [])
            for vod in vods:
                yield self.url_result(
                    'https://trovo.live/%s/%s' % (self._TYPE, vod.get('vid')),
                    ie=TrovoVodIE.ie_key())
            has_more = vod_json['hasMore']
            if not has_more:
                break

    def _real_extract(self, url):
        id = self._match_id(url)
        uid = str(self._call_api(id, query={
            'query': '{getLiveInfo(params:{userName:"%s"}){streamerInfo{uid}}}' % id
        })['data']['getLiveInfo']['streamerInfo']['uid'])
        return self.playlist_result(self._entries(uid), playlist_id=uid)


class TrovoChannelVodIE(TrovoChannelBaseIE):
    _VALID_URL = r'trovovod:(?P<id>[^\s]+)'
    IE_DESC = 'All VODs of a trovo.live channel; "trovovod:" prefix'

    _TESTS = [{
        'url': 'trovovod:OneTappedYou',
        'playlist_mincount': 24,
        'info_dict': {
            'id': '100719456',
        },
    }]

    _QUERY = '{getChannelLtvVideoInfos(params:{pageSize:99,currPage:%d,channelID:%s}){hasMore,vodInfos{vid}}}'
    _TYPE = 'video'

    def _get_vod_json(self, page, uid):
        return self._call_api(uid, query={
            'query': self._QUERY % (page, uid)
        })['data']['getChannelLtvVideoInfos']


class TrovoChannelClipIE(TrovoChannelBaseIE):
    _VALID_URL = r'trovoclip:(?P<id>[^\s]+)'
    IE_DESC = 'All Clips of a trovo.live channel; "trovoclip:" prefix'

    _TESTS = [{
        'url': 'trovoclip:OneTappedYou',
        'playlist_mincount': 29,
        'info_dict': {
            'id': '100719456',
        },
    }]

    _QUERY = '{getChannelClipVideoInfos(params:{pageSize:99,currPage:%d,channelID:%s,albumType:VOD_CLIP_ALBUM_TYPE_LATEST}){hasMore,vodInfos{vid}}}'
    _TYPE = 'clip'

    def _get_vod_json(self, page, uid):
        return self._call_api(uid, query={
            'query': self._QUERY % (page, uid)
        })['data']['getChannelClipVideoInfos']
Commit	Line	Data
a820dc72 RA	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
974208e1	4	import itertools
a820dc72 RA	5	import json
	6
	7	from .common import InfoExtractor
	8	from ..utils import (
	9	ExtractorError,
e0ddbd02	10	format_field,
a820dc72 RA	11	int_or_none,
	12	str_or_none,
	13	try_get,
	14	)
	15
	16
	17	class TrovoBaseIE(InfoExtractor):
	18	_VALID_URL_BASE = r'https?://(?:www\.)?trovo\.live/'
36147a63	19	_HEADERS = {'Origin': 'https://trovo.live'}
a820dc72	20
0cbed930	21	def _call_api(self, video_id, query=None, data=None):
	22	return self._download_json(
	23	'https://gql.trovo.live/', video_id, query=query, data=data,
	24	headers={'Accept': 'application/json'})
	25
a820dc72 RA	26	def _extract_streamer_info(self, data):
	27	streamer_info = data.get('streamerInfo') or {}
	28	username = streamer_info.get('userName')
	29	return {
	30	'uploader': streamer_info.get('nickName'),
	31	'uploader_id': str_or_none(streamer_info.get('uid')),
e0ddbd02	32	'uploader_url': format_field(username, template='https://trovo.live/%s'),
a820dc72 RA	33	}
	34
	35
	36	class TrovoIE(TrovoBaseIE):
	37	_VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?!(?:clip\|video)/)(?P<id>[^/?&#]+)'
	38
	39	def _real_extract(self, url):
	40	username = self._match_id(url)
0cbed930	41	live_info = self._call_api(username, query={
0cbed930	42	'query': '''{
a820dc72 RA	43	getLiveInfo(params: {userName: "%s"}) {
a820dc72 RA	44	isLive
5d62709b	45	programInfo {
a820dc72 RA	46	coverUrl
	47	id
	48	streamInfo {
	49	desc
	50	playUrl
	51	}
	52	title
	53	}
	54	streamerInfo {
	55	nickName
	56	uid
	57	userName
	58	}
	59	}
	60	}''' % username,
0cbed930	61	})['data']['getLiveInfo']
a820dc72 RA	62	if live_info.get('isLive') == 0:
	63	raise ExtractorError('%s is offline' % username, expected=True)
	64	program_info = live_info['programInfo']
	65	program_id = program_info['id']
39ca3b5c	66	title = program_info['title']
a820dc72 RA	67
	68	formats = []
	69	for stream_info in (program_info.get('streamInfo') or []):
	70	play_url = stream_info.get('playUrl')
	71	if not play_url:
	72	continue
	73	format_id = stream_info.get('desc')
	74	formats.append({
	75	'format_id': format_id,
	76	'height': int_or_none(format_id[:-1]) if format_id else None,
	77	'url': play_url,
36147a63	78	'http_headers': self._HEADERS,
a820dc72 RA	79	})
	80	self._sort_formats(formats)
	81
	82	info = {
	83	'id': program_id,
	84	'title': title,
	85	'formats': formats,
	86	'thumbnail': program_info.get('coverUrl'),
	87	'is_live': True,
	88	}
	89	info.update(self._extract_streamer_info(live_info))
	90	return info
	91
	92
	93	class TrovoVodIE(TrovoBaseIE):
	94	_VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:clip\|video)/(?P<id>[^/?&#]+)'
	95	_TESTS = [{
	96	'url': 'https://trovo.live/video/ltv-100095501_100095501_1609596043',
	97	'info_dict': {
	98	'id': 'ltv-100095501_100095501_1609596043',
	99	'ext': 'mp4',
	100	'title': 'Spontaner 12 Stunden Stream! - Ok Boomer!',
	101	'uploader': 'Exsl',
	102	'timestamp': 1609640305,
	103	'upload_date': '20210103',
	104	'uploader_id': '100095501',
	105	'duration': 43977,
	106	'view_count': int,
	107	'like_count': int,
	108	'comment_count': int,
	109	'comments': 'mincount:8',
	110	'categories': ['Grand Theft Auto V'],
	111	},
3262f8ab	112	'skip': '404'
a820dc72 RA	113	}, {
	114	'url': 'https://trovo.live/clip/lc-5285890810184026005',
	115	'only_matching': True,
	116	}]
	117
	118	def _real_extract(self, url):
	119	vid = self._match_id(url)
0cbed930	120	resp = self._call_api(vid, data=json.dumps([{
0cbed930	121	'query': '''{
a820dc72 RA	122	batchGetVodDetailInfo(params: {vids: ["%s"]}) {
	123	VodDetailInfos
	124	}
	125	}''' % vid,
0cbed930	126	}, {
0cbed930	127	'query': '''{
a820dc72 RA	128	getCommentList(params: {appInfo: {postID: "%s"}, pageSize: 1000000000, preview: {}}) {
	129	commentList {
	130	author {
	131	nickName
	132	uid
	133	}
	134	commentID
	135	content
	136	createdAt
	137	parentID
	138	}
	139	}
	140	}''' % vid,
0cbed930	141	}]).encode())
a820dc72 RA	142	vod_detail_info = resp[0]['data']['batchGetVodDetailInfo']['VodDetailInfos'][vid]
	143	vod_info = vod_detail_info['vodInfo']
	144	title = vod_info['title']
	145
	146	language = vod_info.get('languageName')
	147	formats = []
	148	for play_info in (vod_info.get('playInfos') or []):
	149	play_url = play_info.get('playUrl')
	150	if not play_url:
	151	continue
	152	format_id = play_info.get('desc')
	153	formats.append({
	154	'ext': 'mp4',
	155	'filesize': int_or_none(play_info.get('fileSize')),
	156	'format_id': format_id,
	157	'height': int_or_none(format_id[:-1]) if format_id else None,
	158	'language': language,
	159	'protocol': 'm3u8_native',
	160	'tbr': int_or_none(play_info.get('bitrate')),
	161	'url': play_url,
36147a63	162	'http_headers': self._HEADERS,
a820dc72 RA	163	})
	164	self._sort_formats(formats)
	165
	166	category = vod_info.get('categoryName')
	167	get_count = lambda x: int_or_none(vod_info.get(x + 'Num'))
	168
	169	comment_list = try_get(resp, lambda x: x[1]['data']['getCommentList']['commentList'], list) or []
	170	comments = []
	171	for comment in comment_list:
	172	content = comment.get('content')
	173	if not content:
	174	continue
	175	author = comment.get('author') or {}
	176	parent = comment.get('parentID')
	177	comments.append({
	178	'author': author.get('nickName'),
	179	'author_id': str_or_none(author.get('uid')),
	180	'id': str_or_none(comment.get('commentID')),
	181	'text': content,
	182	'timestamp': int_or_none(comment.get('createdAt')),
	183	'parent': 'root' if parent == 0 else str_or_none(parent),
	184	})
	185
	186	info = {
	187	'id': vid,
	188	'title': title,
	189	'formats': formats,
	190	'thumbnail': vod_info.get('coverUrl'),
	191	'timestamp': int_or_none(vod_info.get('publishTs')),
	192	'duration': int_or_none(vod_info.get('duration')),
	193	'view_count': get_count('watch'),
	194	'like_count': get_count('like'),
	195	'comment_count': get_count('comment'),
	196	'comments': comments,
	197	'categories': [category] if category else None,
	198	}
	199	info.update(self._extract_streamer_info(vod_detail_info))
	200	return info
974208e1 AG	201
974208e1 AG	202
3262f8ab	203	class TrovoChannelBaseIE(TrovoBaseIE):
974208e1 AG	204	def _get_vod_json(self, page, uid):
	205	raise NotImplementedError('This method must be implemented by subclasses')
	206
	207	def _entries(self, uid):
	208	for page in itertools.count(1):
	209	vod_json = self._get_vod_json(page, uid)
	210	vods = vod_json.get('vodInfos', [])
	211	for vod in vods:
	212	yield self.url_result(
	213	'https://trovo.live/%s/%s' % (self._TYPE, vod.get('vid')),
	214	ie=TrovoVodIE.ie_key())
	215	has_more = vod_json['hasMore']
	216	if not has_more:
	217	break
	218
	219	def _real_extract(self, url):
	220	id = self._match_id(url)
0cbed930	221	uid = str(self._call_api(id, query={
974208e1 AG	222	'query': '{getLiveInfo(params:{userName:"%s"}){streamerInfo{uid}}}' % id
	223	})['data']['getLiveInfo']['streamerInfo']['uid'])
	224	return self.playlist_result(self._entries(uid), playlist_id=uid)
	225
	226
	227	class TrovoChannelVodIE(TrovoChannelBaseIE):
	228	_VALID_URL = r'trovovod:(?P<id>[^\s]+)'
96565c7e	229	IE_DESC = 'All VODs of a trovo.live channel; "trovovod:" prefix'
974208e1 AG	230
	231	_TESTS = [{
	232	'url': 'trovovod:OneTappedYou',
	233	'playlist_mincount': 24,
	234	'info_dict': {
	235	'id': '100719456',
	236	},
	237	}]
	238
	239	_QUERY = '{getChannelLtvVideoInfos(params:{pageSize:99,currPage:%d,channelID:%s}){hasMore,vodInfos{vid}}}'
	240	_TYPE = 'video'
	241
	242	def _get_vod_json(self, page, uid):
0cbed930	243	return self._call_api(uid, query={
974208e1 AG	244	'query': self._QUERY % (page, uid)
	245	})['data']['getChannelLtvVideoInfos']
	246
	247
	248	class TrovoChannelClipIE(TrovoChannelBaseIE):
	249	_VALID_URL = r'trovoclip:(?P<id>[^\s]+)'
96565c7e	250	IE_DESC = 'All Clips of a trovo.live channel; "trovoclip:" prefix'
974208e1 AG	251
	252	_TESTS = [{
	253	'url': 'trovoclip:OneTappedYou',
	254	'playlist_mincount': 29,
	255	'info_dict': {
	256	'id': '100719456',
	257	},
	258	}]
	259
	260	_QUERY = '{getChannelClipVideoInfos(params:{pageSize:99,currPage:%d,channelID:%s,albumType:VOD_CLIP_ALBUM_TYPE_LATEST}){hasMore,vodInfos{vid}}}'
	261	_TYPE = 'clip'
	262
	263	def _get_vod_json(self, page, uid):
0cbed930	264	return self._call_api(uid, query={
974208e1 AG	265	'query': self._QUERY % (page, uid)
974208e1 AG	266	})['data']['getChannelClipVideoInfos']