[yt-dlp.git] / yt_dlp / extractor / gettr.py

# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    bool_or_none,
    ExtractorError,
    dict_get,
    float_or_none,
    int_or_none,
    remove_end,
    str_or_none,
    try_get,
    url_or_none,
    urljoin,
)


class GettrBaseIE(InfoExtractor):
    _BASE_REGEX = r'https?://(www\.)?gettr\.com/'
    _MEDIA_BASE_URL = 'https://media.gettr.com/'

    def _call_api(self, path, video_id, *args, **kwargs):
        return self._download_json(urljoin('https://api.gettr.com/u/', path), video_id, *args, **kwargs)['result']


class GettrIE(GettrBaseIE):
    _VALID_URL = GettrBaseIE._BASE_REGEX + r'post/(?P<id>[a-z0-9]+)'

    _TESTS = [{
        'url': 'https://www.gettr.com/post/pcf6uv838f',
        'info_dict': {
            'id': 'pcf6uv838f',
            'title': 'md5:9086a646bbd06c41c4fe8e52b3c93454',
            'description': 'md5:be0577f1e4caadc06de4a002da2bf287',
            'ext': 'mp4',
            'uploader': 'EpochTV',
            'uploader_id': 'epochtv',
            'thumbnail': r're:^https?://.+/out\.jpg',
            'timestamp': 1632782451058,
            'duration': 58.5585,
        }
    }, {
        'url': 'https://gettr.com/post/p4iahp',
        'info_dict': {
            'id': 'p4iahp',
            'title': 'md5:b03c07883db6fbc1aab88877a6c3b149',
            'description': 'md5:741b7419d991c403196ed2ea7749a39d',
            'ext': 'mp4',
            'uploader': 'Neues Forum Freiheit',
            'uploader_id': 'nf_freiheit',
            'thumbnail': r're:^https?://.+/out\.jpg',
            'timestamp': 1626594455017,
            'duration': 23,
        }
    }]

    def _real_extract(self, url):
        post_id = self._match_id(url)
        webpage = self._download_webpage(url, post_id)

        api_data = self._call_api('post/%s?incl="poststats|userinfo"' % post_id, post_id)

        post_data = api_data.get('data')
        user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']]) or {}

        if post_data.get('nfound'):
            raise ExtractorError(post_data.get('txt'), expected=True)

        title = description = str_or_none(
            post_data.get('txt') or self._og_search_description(webpage))

        uploader = str_or_none(
            user_data.get('nickname')
            or remove_end(self._og_search_title(webpage), ' on GETTR'))
        if uploader:
            title = '%s - %s' % (uploader, title)

        if not dict_get(post_data, ['vid', 'ovid']):
            raise ExtractorError('There\'s no video in this post.')

        vid = post_data.get('vid')
        ovid = post_data.get('ovid')

        formats = self._extract_m3u8_formats(
            urljoin(self._MEDIA_BASE_URL, vid), post_id, 'mp4',
            entry_protocol='m3u8_native', m3u8_id='hls') if vid else []

        if ovid:
            formats.append({
                'url': urljoin(self._MEDIA_BASE_URL, ovid),
                'format_id': 'ovid',
                'ext': 'mp4',
                'width': int_or_none(post_data.get('vid_wid')),
                'height': int_or_none(post_data.get('vid_hgt')),
                'source_preference': 1,
                'quality': 1,
            })

        self._sort_formats(formats)

        return {
            'id': post_id,
            'title': title,
            'description': description,
            'thumbnail': url_or_none(
                urljoin(self._MEDIA_BASE_URL, post_data.get('main'))
                or self._og_search_thumbnail(webpage)),
            'timestamp': int_or_none(post_data.get('cdate')),
            'uploader_id': str_or_none(
                dict_get(user_data, ['_id', 'username'])
                or post_data.get('uid')),
            'uploader': uploader,
            'formats': formats,
            'duration': float_or_none(post_data.get('vid_dur')),
            'tags': post_data.get('htgs'),
        }


class GettrStreamingIE(GettrBaseIE):
    _VALID_URL = GettrBaseIE._BASE_REGEX + r'streaming/(?P<id>[a-z0-9]+)'

    _TESTS = [{
        'url': 'https://gettr.com/streaming/psoiulc122',
        'info_dict': {
            'id': 'psoiulc122',
            'ext': 'mp4',
            'description': 'md5:56bca4b8f48f1743d9fd03d49c723017',
            'view_count': int,
            'uploader': 'Corona Investigative Committee',
            'uploader_id': 'coronacommittee',
            'duration': 5180.184,
            'thumbnail': r're:^https?://.+',
            'title': 'Day 1: Opening Session of the Grand Jury Proceeding',
            'timestamp': 1644080997.164,
            'upload_date': '20220205',
        }
    }, {
        'url': 'https://gettr.com/streaming/psfmeefcc1',
        'info_dict': {
            'id': 'psfmeefcc1',
            'ext': 'mp4',
            'title': 'Session 90: "The Virus Of Power"',
            'view_count': int,
            'uploader_id': 'coronacommittee',
            'description': 'md5:98986acdf656aa836bf36f9c9704c65b',
            'uploader': 'Corona Investigative Committee',
            'thumbnail': r're:^https?://.+',
            'duration': 21872.507,
            'timestamp': 1643976662.858,
            'upload_date': '20220204',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_info = self._call_api('live/join/%s' % video_id, video_id, data={})

        live_info = video_info['broadcast']
        live_url = url_or_none(live_info.get('url'))

        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
            live_url, video_id, ext='mp4',
            entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if live_url else ([], {})

        thumbnails = [{
            'url': urljoin(self._MEDIA_BASE_URL, thumbnail),
        } for thumbnail in try_get(video_info, lambda x: x['postData']['imgs']) or []]

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': try_get(video_info, lambda x: x['postData']['ttl']),
            'description': try_get(video_info, lambda x: x['postData']['dsc']),
            'formats': formats,
            'subtitles': subtitles,
            'thumbnails': thumbnails,
            'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname']),
            'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id']),
            'view_count': int_or_none(live_info.get('viewsCount')),
            'timestamp': float_or_none(live_info.get('startAt'), scale=1000),
            'duration': float_or_none(live_info.get('duration'), scale=1000),
            'is_live': bool_or_none(live_info.get('isLive')),
        }
Commit	Line	Data
85187609	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
	4	from .common import InfoExtractor
	5	from ..utils import (
82b51767	6	bool_or_none,
85187609	7	ExtractorError,
	8	dict_get,
	9	float_or_none,
	10	int_or_none,
	11	remove_end,
	12	str_or_none,
	13	try_get,
	14	url_or_none,
	15	urljoin,
	16	)
	17
	18
82b51767	19	class GettrBaseIE(InfoExtractor):
82b51767	20	_BASE_REGEX = r'https?://(www\.)?gettr\.com/'
85187609	21	_MEDIA_BASE_URL = 'https://media.gettr.com/'
85187609	22
82b51767	23	def _call_api(self, path, video_id, args, *kwargs):
	24	return self._download_json(urljoin('https://api.gettr.com/u/', path), video_id, args, *kwargs)['result']
	25
	26
	27	class GettrIE(GettrBaseIE):
	28	_VALID_URL = GettrBaseIE._BASE_REGEX + r'post/(?P<id>[a-z0-9]+)'
	29
85187609	30	_TESTS = [{
	31	'url': 'https://www.gettr.com/post/pcf6uv838f',
	32	'info_dict': {
	33	'id': 'pcf6uv838f',
	34	'title': 'md5:9086a646bbd06c41c4fe8e52b3c93454',
	35	'description': 'md5:be0577f1e4caadc06de4a002da2bf287',
	36	'ext': 'mp4',
	37	'uploader': 'EpochTV',
	38	'uploader_id': 'epochtv',
	39	'thumbnail': r're:^https?://.+/out\.jpg',
	40	'timestamp': 1632782451058,
	41	'duration': 58.5585,
	42	}
	43	}, {
	44	'url': 'https://gettr.com/post/p4iahp',
	45	'info_dict': {
	46	'id': 'p4iahp',
	47	'title': 'md5:b03c07883db6fbc1aab88877a6c3b149',
	48	'description': 'md5:741b7419d991c403196ed2ea7749a39d',
	49	'ext': 'mp4',
	50	'uploader': 'Neues Forum Freiheit',
	51	'uploader_id': 'nf_freiheit',
	52	'thumbnail': r're:^https?://.+/out\.jpg',
	53	'timestamp': 1626594455017,
	54	'duration': 23,
	55	}
	56	}]
	57
	58	def _real_extract(self, url):
	59	post_id = self._match_id(url)
	60	webpage = self._download_webpage(url, post_id)
	61
82b51767	62	api_data = self._call_api('post/%s?incl="poststats\|userinfo"' % post_id, post_id)
85187609	63
82b51767	64	post_data = api_data.get('data')
82b51767	65	user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']]) or {}
85187609	66
	67	if post_data.get('nfound'):
	68	raise ExtractorError(post_data.get('txt'), expected=True)
	69
	70	title = description = str_or_none(
	71	post_data.get('txt') or self._og_search_description(webpage))
	72
	73	uploader = str_or_none(
	74	user_data.get('nickname')
	75	or remove_end(self._og_search_title(webpage), ' on GETTR'))
	76	if uploader:
	77	title = '%s - %s' % (uploader, title)
	78
	79	if not dict_get(post_data, ['vid', 'ovid']):
	80	raise ExtractorError('There\'s no video in this post.')
	81
	82	vid = post_data.get('vid')
	83	ovid = post_data.get('ovid')
	84
	85	formats = self._extract_m3u8_formats(
	86	urljoin(self._MEDIA_BASE_URL, vid), post_id, 'mp4',
	87	entry_protocol='m3u8_native', m3u8_id='hls') if vid else []
	88
	89	if ovid:
	90	formats.append({
	91	'url': urljoin(self._MEDIA_BASE_URL, ovid),
	92	'format_id': 'ovid',
	93	'ext': 'mp4',
	94	'width': int_or_none(post_data.get('vid_wid')),
	95	'height': int_or_none(post_data.get('vid_hgt')),
	96	'source_preference': 1,
	97	'quality': 1,
	98	})
	99
	100	self._sort_formats(formats)
	101
	102	return {
	103	'id': post_id,
	104	'title': title,
	105	'description': description,
	106	'thumbnail': url_or_none(
	107	urljoin(self._MEDIA_BASE_URL, post_data.get('main'))
	108	or self._og_search_thumbnail(webpage)),
	109	'timestamp': int_or_none(post_data.get('cdate')),
	110	'uploader_id': str_or_none(
	111	dict_get(user_data, ['_id', 'username'])
	112	or post_data.get('uid')),
	113	'uploader': uploader,
	114	'formats': formats,
	115	'duration': float_or_none(post_data.get('vid_dur')),
	116	'tags': post_data.get('htgs'),
	117	}
82b51767	118
	119
	120	class GettrStreamingIE(GettrBaseIE):
	121	_VALID_URL = GettrBaseIE._BASE_REGEX + r'streaming/(?P<id>[a-z0-9]+)'
	122
	123	_TESTS = [{
	124	'url': 'https://gettr.com/streaming/psoiulc122',
	125	'info_dict': {
	126	'id': 'psoiulc122',
	127	'ext': 'mp4',
	128	'description': 'md5:56bca4b8f48f1743d9fd03d49c723017',
	129	'view_count': int,
	130	'uploader': 'Corona Investigative Committee',
	131	'uploader_id': 'coronacommittee',
	132	'duration': 5180.184,
	133	'thumbnail': r're:^https?://.+',
	134	'title': 'Day 1: Opening Session of the Grand Jury Proceeding',
	135	'timestamp': 1644080997.164,
	136	'upload_date': '20220205',
	137	}
	138	}, {
	139	'url': 'https://gettr.com/streaming/psfmeefcc1',
	140	'info_dict': {
	141	'id': 'psfmeefcc1',
	142	'ext': 'mp4',
	143	'title': 'Session 90: "The Virus Of Power"',
	144	'view_count': int,
	145	'uploader_id': 'coronacommittee',
	146	'description': 'md5:98986acdf656aa836bf36f9c9704c65b',
	147	'uploader': 'Corona Investigative Committee',
	148	'thumbnail': r're:^https?://.+',
	149	'duration': 21872.507,
	150	'timestamp': 1643976662.858,
	151	'upload_date': '20220204',
	152	}
	153	}]
	154
	155	def _real_extract(self, url):
	156	video_id = self._match_id(url)
	157	video_info = self._call_api('live/join/%s' % video_id, video_id, data={})
	158
	159	live_info = video_info['broadcast']
	160	live_url = url_or_none(live_info.get('url'))
	161
	162	formats, subtitles = self._extract_m3u8_formats_and_subtitles(
	163	live_url, video_id, ext='mp4',
	164	entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if live_url else ([], {})
	165
	166	thumbnails = [{
	167	'url': urljoin(self._MEDIA_BASE_URL, thumbnail),
	168	} for thumbnail in try_get(video_info, lambda x: x['postData']['imgs']) or []]
	169
	170	self._sort_formats(formats)
	171
	172	return {
	173	'id': video_id,
	174	'title': try_get(video_info, lambda x: x['postData']['ttl']),
	175	'description': try_get(video_info, lambda x: x['postData']['dsc']),
	176	'formats': formats,
	177	'subtitles': subtitles,
	178	'thumbnails': thumbnails,
	179	'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname']),
	180	'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id']),
	181	'view_count': int_or_none(live_info.get('viewsCount')),
182	'timestamp': float_or_none(live_info.get('startAt'), scale=1000),
183	'duration': float_or_none(live_info.get('duration'), scale=1000),
184	'is_live': bool_or_none(live_info.get('isLive')),
185	}