[yt-dlp.git] / yt_dlp / extractor / trovo.py

import itertools
import json

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    format_field,
    int_or_none,
    str_or_none,
    try_get,
)


class TrovoBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?trovo\.live/'
    _HEADERS = {'Origin': 'https://trovo.live'}

    def _call_api(self, video_id, query=None, data=None):
        return self._download_json(
            'https://gql.trovo.live/', video_id, query=query, data=data,
            headers={'Accept': 'application/json'})

    def _extract_streamer_info(self, data):
        streamer_info = data.get('streamerInfo') or {}
        username = streamer_info.get('userName')
        return {
            'uploader': streamer_info.get('nickName'),
            'uploader_id': str_or_none(streamer_info.get('uid')),
            'uploader_url': format_field(username, template='https://trovo.live/%s'),
        }


class TrovoIE(TrovoBaseIE):
    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?!(?:clip|video)/)(?P<id>[^/?&#]+)'

    def _real_extract(self, url):
        username = self._match_id(url)
        live_info = self._call_api(username, query={
            'query': '''{
  getLiveInfo(params: {userName: "%s"}) {
    isLive
    programInfo {
      coverUrl
      id
      streamInfo {
        desc
        playUrl
      }
      title
    }
    streamerInfo {
        nickName
        uid
        userName
    }
  }
}''' % username,
        })['data']['getLiveInfo']
        if live_info.get('isLive') == 0:
            raise ExtractorError('%s is offline' % username, expected=True)
        program_info = live_info['programInfo']
        program_id = program_info['id']
        title = program_info['title']

        formats = []
        for stream_info in (program_info.get('streamInfo') or []):
            play_url = stream_info.get('playUrl')
            if not play_url:
                continue
            format_id = stream_info.get('desc')
            formats.append({
                'format_id': format_id,
                'height': int_or_none(format_id[:-1]) if format_id else None,
                'url': play_url,
                'http_headers': self._HEADERS,
            })
        self._sort_formats(formats)

        info = {
            'id': program_id,
            'title': title,
            'formats': formats,
            'thumbnail': program_info.get('coverUrl'),
            'is_live': True,
        }
        info.update(self._extract_streamer_info(live_info))
        return info


class TrovoVodIE(TrovoBaseIE):
    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:clip|video)/(?P<id>[^/?&#]+)'
    _TESTS = [{
        'url': 'https://trovo.live/video/ltv-100095501_100095501_1609596043',
        'info_dict': {
            'id': 'ltv-100095501_100095501_1609596043',
            'ext': 'mp4',
            'title': 'Spontaner 12 Stunden Stream! - Ok Boomer!',
            'uploader': 'Exsl',
            'timestamp': 1609640305,
            'upload_date': '20210103',
            'uploader_id': '100095501',
            'duration': 43977,
            'view_count': int,
            'like_count': int,
            'comment_count': int,
            'comments': 'mincount:8',
            'categories': ['Grand Theft Auto V'],
        },
        'skip': '404'
    }, {
        'url': 'https://trovo.live/clip/lc-5285890810184026005',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        vid = self._match_id(url)
        resp = self._call_api(vid, data=json.dumps([{
            'query': '''{
  batchGetVodDetailInfo(params: {vids: ["%s"]}) {
    VodDetailInfos
  }
}''' % vid,
        }, {
            'query': '''{
  getCommentList(params: {appInfo: {postID: "%s"}, pageSize: 1000000000, preview: {}}) {
    commentList {
      author {
        nickName
        uid
      }
      commentID
      content
      createdAt
      parentID
    }
  }
}''' % vid,
        }]).encode())
        vod_detail_info = resp[0]['data']['batchGetVodDetailInfo']['VodDetailInfos'][vid]
        vod_info = vod_detail_info['vodInfo']
        title = vod_info['title']

        language = vod_info.get('languageName')
        formats = []
        for play_info in (vod_info.get('playInfos') or []):
            play_url = play_info.get('playUrl')
            if not play_url:
                continue
            format_id = play_info.get('desc')
            formats.append({
                'ext': 'mp4',
                'filesize': int_or_none(play_info.get('fileSize')),
                'format_id': format_id,
                'height': int_or_none(format_id[:-1]) if format_id else None,
                'language': language,
                'protocol': 'm3u8_native',
                'tbr': int_or_none(play_info.get('bitrate')),
                'url': play_url,
                'http_headers': self._HEADERS,
            })
        self._sort_formats(formats)

        category = vod_info.get('categoryName')
        get_count = lambda x: int_or_none(vod_info.get(x + 'Num'))

        comment_list = try_get(resp, lambda x: x[1]['data']['getCommentList']['commentList'], list) or []
        comments = []
        for comment in comment_list:
            content = comment.get('content')
            if not content:
                continue
            author = comment.get('author') or {}
            parent = comment.get('parentID')
            comments.append({
                'author': author.get('nickName'),
                'author_id': str_or_none(author.get('uid')),
                'id': str_or_none(comment.get('commentID')),
                'text': content,
                'timestamp': int_or_none(comment.get('createdAt')),
                'parent': 'root' if parent == 0 else str_or_none(parent),
            })

        info = {
            'id': vid,
            'title': title,
            'formats': formats,
            'thumbnail': vod_info.get('coverUrl'),
            'timestamp': int_or_none(vod_info.get('publishTs')),
            'duration': int_or_none(vod_info.get('duration')),
            'view_count': get_count('watch'),
            'like_count': get_count('like'),
            'comment_count': get_count('comment'),
            'comments': comments,
            'categories': [category] if category else None,
        }
        info.update(self._extract_streamer_info(vod_detail_info))
        return info


class TrovoChannelBaseIE(TrovoBaseIE):
    def _get_vod_json(self, page, uid):
        raise NotImplementedError('This method must be implemented by subclasses')

    def _entries(self, uid):
        for page in itertools.count(1):
            vod_json = self._get_vod_json(page, uid)
            vods = vod_json.get('vodInfos', [])
            for vod in vods:
                yield self.url_result(
                    'https://trovo.live/%s/%s' % (self._TYPE, vod.get('vid')),
                    ie=TrovoVodIE.ie_key())
            has_more = vod_json['hasMore']
            if not has_more:
                break

    def _real_extract(self, url):
        id = self._match_id(url)
        uid = str(self._call_api(id, query={
            'query': '{getLiveInfo(params:{userName:"%s"}){streamerInfo{uid}}}' % id
        })['data']['getLiveInfo']['streamerInfo']['uid'])
        return self.playlist_result(self._entries(uid), playlist_id=uid)


class TrovoChannelVodIE(TrovoChannelBaseIE):
    _VALID_URL = r'trovovod:(?P<id>[^\s]+)'
    IE_DESC = 'All VODs of a trovo.live channel; "trovovod:" prefix'

    _TESTS = [{
        'url': 'trovovod:OneTappedYou',
        'playlist_mincount': 24,
        'info_dict': {
            'id': '100719456',
        },
    }]

    _QUERY = '{getChannelLtvVideoInfos(params:{pageSize:99,currPage:%d,channelID:%s}){hasMore,vodInfos{vid}}}'
    _TYPE = 'video'

    def _get_vod_json(self, page, uid):
        return self._call_api(uid, query={
            'query': self._QUERY % (page, uid)
        })['data']['getChannelLtvVideoInfos']


class TrovoChannelClipIE(TrovoChannelBaseIE):
    _VALID_URL = r'trovoclip:(?P<id>[^\s]+)'
    IE_DESC = 'All Clips of a trovo.live channel; "trovoclip:" prefix'

    _TESTS = [{
        'url': 'trovoclip:OneTappedYou',
        'playlist_mincount': 29,
        'info_dict': {
            'id': '100719456',
        },
    }]

    _QUERY = '{getChannelClipVideoInfos(params:{pageSize:99,currPage:%d,channelID:%s,albumType:VOD_CLIP_ALBUM_TYPE_LATEST}){hasMore,vodInfos{vid}}}'
    _TYPE = 'clip'

    def _get_vod_json(self, page, uid):
        return self._call_api(uid, query={
            'query': self._QUERY % (page, uid)
        })['data']['getChannelClipVideoInfos']
Commit	Line	Data
974208e1	1	import itertools
a820dc72 RA	2	import json
	3
	4	from .common import InfoExtractor
	5	from ..utils import (
	6	ExtractorError,
e0ddbd02	7	format_field,
a820dc72 RA	8	int_or_none,
	9	str_or_none,
	10	try_get,
	11	)
	12
	13
	14	class TrovoBaseIE(InfoExtractor):
	15	_VALID_URL_BASE = r'https?://(?:www\.)?trovo\.live/'
36147a63	16	_HEADERS = {'Origin': 'https://trovo.live'}
a820dc72	17
0cbed930	18	def _call_api(self, video_id, query=None, data=None):
	19	return self._download_json(
	20	'https://gql.trovo.live/', video_id, query=query, data=data,
	21	headers={'Accept': 'application/json'})
	22
a820dc72 RA	23	def _extract_streamer_info(self, data):
	24	streamer_info = data.get('streamerInfo') or {}
	25	username = streamer_info.get('userName')
	26	return {
	27	'uploader': streamer_info.get('nickName'),
	28	'uploader_id': str_or_none(streamer_info.get('uid')),
e0ddbd02	29	'uploader_url': format_field(username, template='https://trovo.live/%s'),
a820dc72 RA	30	}
	31
	32
	33	class TrovoIE(TrovoBaseIE):
	34	_VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?!(?:clip\|video)/)(?P<id>[^/?&#]+)'
	35
	36	def _real_extract(self, url):
	37	username = self._match_id(url)
0cbed930	38	live_info = self._call_api(username, query={
0cbed930	39	'query': '''{
a820dc72 RA	40	getLiveInfo(params: {userName: "%s"}) {
a820dc72 RA	41	isLive
5d62709b	42	programInfo {
a820dc72 RA	43	coverUrl
	44	id
	45	streamInfo {
	46	desc
	47	playUrl
	48	}
	49	title
	50	}
	51	streamerInfo {
	52	nickName
	53	uid
	54	userName
	55	}
	56	}
	57	}''' % username,
0cbed930	58	})['data']['getLiveInfo']
a820dc72 RA	59	if live_info.get('isLive') == 0:
	60	raise ExtractorError('%s is offline' % username, expected=True)
	61	program_info = live_info['programInfo']
	62	program_id = program_info['id']
39ca3b5c	63	title = program_info['title']
a820dc72 RA	64
	65	formats = []
	66	for stream_info in (program_info.get('streamInfo') or []):
	67	play_url = stream_info.get('playUrl')
	68	if not play_url:
	69	continue
	70	format_id = stream_info.get('desc')
	71	formats.append({
	72	'format_id': format_id,
	73	'height': int_or_none(format_id[:-1]) if format_id else None,
	74	'url': play_url,
36147a63	75	'http_headers': self._HEADERS,
a820dc72 RA	76	})
	77	self._sort_formats(formats)
	78
	79	info = {
	80	'id': program_id,
	81	'title': title,
	82	'formats': formats,
	83	'thumbnail': program_info.get('coverUrl'),
	84	'is_live': True,
	85	}
	86	info.update(self._extract_streamer_info(live_info))
	87	return info
	88
	89
	90	class TrovoVodIE(TrovoBaseIE):
	91	_VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:clip\|video)/(?P<id>[^/?&#]+)'
	92	_TESTS = [{
	93	'url': 'https://trovo.live/video/ltv-100095501_100095501_1609596043',
	94	'info_dict': {
	95	'id': 'ltv-100095501_100095501_1609596043',
	96	'ext': 'mp4',
	97	'title': 'Spontaner 12 Stunden Stream! - Ok Boomer!',
	98	'uploader': 'Exsl',
	99	'timestamp': 1609640305,
	100	'upload_date': '20210103',
	101	'uploader_id': '100095501',
	102	'duration': 43977,
	103	'view_count': int,
	104	'like_count': int,
	105	'comment_count': int,
	106	'comments': 'mincount:8',
	107	'categories': ['Grand Theft Auto V'],
	108	},
3262f8ab	109	'skip': '404'
a820dc72 RA	110	}, {
	111	'url': 'https://trovo.live/clip/lc-5285890810184026005',
	112	'only_matching': True,
	113	}]
	114
	115	def _real_extract(self, url):
	116	vid = self._match_id(url)
0cbed930	117	resp = self._call_api(vid, data=json.dumps([{
0cbed930	118	'query': '''{
a820dc72 RA	119	batchGetVodDetailInfo(params: {vids: ["%s"]}) {
	120	VodDetailInfos
	121	}
	122	}''' % vid,
0cbed930	123	}, {
0cbed930	124	'query': '''{
a820dc72 RA	125	getCommentList(params: {appInfo: {postID: "%s"}, pageSize: 1000000000, preview: {}}) {
	126	commentList {
	127	author {
	128	nickName
	129	uid
	130	}
	131	commentID
	132	content
	133	createdAt
	134	parentID
	135	}
	136	}
	137	}''' % vid,
0cbed930	138	}]).encode())
a820dc72 RA	139	vod_detail_info = resp[0]['data']['batchGetVodDetailInfo']['VodDetailInfos'][vid]
	140	vod_info = vod_detail_info['vodInfo']
	141	title = vod_info['title']
	142
	143	language = vod_info.get('languageName')
	144	formats = []
	145	for play_info in (vod_info.get('playInfos') or []):
	146	play_url = play_info.get('playUrl')
	147	if not play_url:
	148	continue
	149	format_id = play_info.get('desc')
	150	formats.append({
	151	'ext': 'mp4',
	152	'filesize': int_or_none(play_info.get('fileSize')),
	153	'format_id': format_id,
	154	'height': int_or_none(format_id[:-1]) if format_id else None,
	155	'language': language,
	156	'protocol': 'm3u8_native',
	157	'tbr': int_or_none(play_info.get('bitrate')),
	158	'url': play_url,
36147a63	159	'http_headers': self._HEADERS,
a820dc72 RA	160	})
	161	self._sort_formats(formats)
	162
	163	category = vod_info.get('categoryName')
	164	get_count = lambda x: int_or_none(vod_info.get(x + 'Num'))
	165
	166	comment_list = try_get(resp, lambda x: x[1]['data']['getCommentList']['commentList'], list) or []
	167	comments = []
	168	for comment in comment_list:
	169	content = comment.get('content')
	170	if not content:
	171	continue
	172	author = comment.get('author') or {}
	173	parent = comment.get('parentID')
	174	comments.append({
	175	'author': author.get('nickName'),
	176	'author_id': str_or_none(author.get('uid')),
	177	'id': str_or_none(comment.get('commentID')),
	178	'text': content,
	179	'timestamp': int_or_none(comment.get('createdAt')),
	180	'parent': 'root' if parent == 0 else str_or_none(parent),
	181	})
	182
	183	info = {
	184	'id': vid,
	185	'title': title,
	186	'formats': formats,
	187	'thumbnail': vod_info.get('coverUrl'),
	188	'timestamp': int_or_none(vod_info.get('publishTs')),
	189	'duration': int_or_none(vod_info.get('duration')),
	190	'view_count': get_count('watch'),
	191	'like_count': get_count('like'),
	192	'comment_count': get_count('comment'),
	193	'comments': comments,
	194	'categories': [category] if category else None,
	195	}
	196	info.update(self._extract_streamer_info(vod_detail_info))
	197	return info
974208e1 AG	198
974208e1 AG	199
3262f8ab	200	class TrovoChannelBaseIE(TrovoBaseIE):
974208e1 AG	201	def _get_vod_json(self, page, uid):
	202	raise NotImplementedError('This method must be implemented by subclasses')
	203
	204	def _entries(self, uid):
	205	for page in itertools.count(1):
	206	vod_json = self._get_vod_json(page, uid)
	207	vods = vod_json.get('vodInfos', [])
	208	for vod in vods:
	209	yield self.url_result(
	210	'https://trovo.live/%s/%s' % (self._TYPE, vod.get('vid')),
	211	ie=TrovoVodIE.ie_key())
	212	has_more = vod_json['hasMore']
	213	if not has_more:
	214	break
	215
	216	def _real_extract(self, url):
	217	id = self._match_id(url)
0cbed930	218	uid = str(self._call_api(id, query={
974208e1 AG	219	'query': '{getLiveInfo(params:{userName:"%s"}){streamerInfo{uid}}}' % id
	220	})['data']['getLiveInfo']['streamerInfo']['uid'])
	221	return self.playlist_result(self._entries(uid), playlist_id=uid)
	222
	223
	224	class TrovoChannelVodIE(TrovoChannelBaseIE):
	225	_VALID_URL = r'trovovod:(?P<id>[^\s]+)'
96565c7e	226	IE_DESC = 'All VODs of a trovo.live channel; "trovovod:" prefix'
974208e1 AG	227
	228	_TESTS = [{
	229	'url': 'trovovod:OneTappedYou',
	230	'playlist_mincount': 24,
	231	'info_dict': {
	232	'id': '100719456',
	233	},
	234	}]
	235
	236	_QUERY = '{getChannelLtvVideoInfos(params:{pageSize:99,currPage:%d,channelID:%s}){hasMore,vodInfos{vid}}}'
	237	_TYPE = 'video'
	238
	239	def _get_vod_json(self, page, uid):
0cbed930	240	return self._call_api(uid, query={
974208e1 AG	241	'query': self._QUERY % (page, uid)
	242	})['data']['getChannelLtvVideoInfos']
	243
	244
	245	class TrovoChannelClipIE(TrovoChannelBaseIE):
	246	_VALID_URL = r'trovoclip:(?P<id>[^\s]+)'
96565c7e	247	IE_DESC = 'All Clips of a trovo.live channel; "trovoclip:" prefix'
974208e1 AG	248
	249	_TESTS = [{
	250	'url': 'trovoclip:OneTappedYou',
	251	'playlist_mincount': 29,
	252	'info_dict': {
	253	'id': '100719456',
	254	},
	255	}]
	256
	257	_QUERY = '{getChannelClipVideoInfos(params:{pageSize:99,currPage:%d,channelID:%s,albumType:VOD_CLIP_ALBUM_TYPE_LATEST}){hasMore,vodInfos{vid}}}'
	258	_TYPE = 'clip'
	259
	260	def _get_vod_json(self, page, uid):
0cbed930	261	return self._call_api(uid, query={
974208e1 AG	262	'query': self._QUERY % (page, uid)
974208e1 AG	263	})['data']['getChannelClipVideoInfos']