yt_dlp/extractor/gettr.py

   1 from .common import InfoExtractor
   2 from ..utils import (
   3     bool_or_none,
   4     ExtractorError,
   5     dict_get,
   6     float_or_none,
   7     int_or_none,
   8     str_or_none,
   9     traverse_obj,
  10     try_get,
  11     url_or_none,
  12     urljoin,
  13 )
  14
  15
  16 class GettrBaseIE(InfoExtractor):
  17     _BASE_REGEX = r'https?://(www\.)?gettr\.com/'
  18     _MEDIA_BASE_URL = 'https://media.gettr.com/'
  19
  20     def _call_api(self, path, video_id, *args, **kwargs):
  21         return self._download_json(urljoin('https://api.gettr.com/u/', path), video_id, *args, **kwargs)['result']
  22
  23
  24 class GettrIE(GettrBaseIE):
  25     _VALID_URL = GettrBaseIE._BASE_REGEX + r'post/(?P<id>[a-z0-9]+)'
  26
  27     _TESTS = [{
  28         'url': 'https://www.gettr.com/post/pcf6uv838f',
  29         'info_dict': {
  30             'id': 'pcf6uv838f',
  31             'title': 'md5:9086a646bbd06c41c4fe8e52b3c93454',
  32             'description': 'md5:be0577f1e4caadc06de4a002da2bf287',
  33             'ext': 'mp4',
  34             'uploader': 'EpochTV',
  35             'uploader_id': 'epochtv',
  36             'upload_date': '20210927',
  37             'thumbnail': r're:^https?://.+/out\.jpg',
  38             'timestamp': 1632782451.058,
  39             'duration': 58.5585,
  40             'tags': ['hornofafrica', 'explorations'],
  41         }
  42     }, {
  43         'url': 'https://gettr.com/post/p4iahp',
  44         'info_dict': {
  45             'id': 'p4iahp',
  46             'title': 'md5:b03c07883db6fbc1aab88877a6c3b149',
  47             'description': 'md5:741b7419d991c403196ed2ea7749a39d',
  48             'ext': 'mp4',
  49             'uploader': 'Neues Forum Freiheit',
  50             'uploader_id': 'nf_freiheit',
  51             'upload_date': '20210718',
  52             'thumbnail': r're:^https?://.+/out\.jpg',
  53             'timestamp': 1626594455.017,
  54             'duration': 23,
  55             'tags': 'count:12',
  56         }
  57     }, {
  58         # quote post
  59         'url': 'https://gettr.com/post/pxn5b743a9',
  60         'only_matching': True,
  61     }, {
  62         # quote with video
  63         'url': 'https://gettr.com/post/pxtiiz5ca2',
  64         'only_matching': True,
  65     }, {
  66         # streaming embed
  67         'url': 'https://gettr.com/post/pxlu8p3b13',
  68         'only_matching': True,
  69     }, {
  70         # youtube embed
  71         'url': 'https://gettr.com/post/pv6wp9e24c',
  72         'only_matching': True,
  73         'add_ie': ['Youtube'],
  74     }]
  75
  76     def _real_extract(self, url):
  77         post_id = self._match_id(url)
  78         webpage = self._download_webpage(url, post_id)
  79         api_data = self._call_api('post/%s?incl="poststats|userinfo"' % post_id, post_id)
  80
  81         post_data = api_data.get('data')
  82         user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']], dict) or {}
  83
  84         vid = post_data.get('vid')
  85         ovid = post_data.get('ovid')
  86
  87         if post_data.get('p_type') == 'stream':
  88             return self.url_result(f'https://gettr.com/streaming/{post_id}', ie='GettrStreaming', video_id=post_id)
  89
  90         if not (ovid or vid):
  91             embed_url = url_or_none(post_data.get('prevsrc'))
  92             shared_post_id = traverse_obj(api_data, ('aux', 'shrdpst', '_id'), ('data', 'rpstIds', 0), expected_type=str)
  93
  94             if embed_url:
  95                 return self.url_result(embed_url)
  96             elif shared_post_id:
  97                 return self.url_result(f'https://gettr.com/post/{shared_post_id}', ie='Gettr', video_id=shared_post_id)
  98             else:
  99                 raise ExtractorError('There\'s no video in this post.')
 100
 101         title = description = str_or_none(
 102             post_data.get('txt') or self._og_search_description(webpage))
 103
 104         uploader = str_or_none(
 105             user_data.get('nickname')
 106             or self._search_regex(r'^(.+?) on GETTR', self._og_search_title(webpage, default=''), 'uploader', fatal=False))
 107
 108         if uploader:
 109             title = '%s - %s' % (uploader, title)
 110
 111         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
 112             urljoin(self._MEDIA_BASE_URL, vid), post_id, 'mp4',
 113             entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if vid else ([], {})
 114
 115         if ovid:
 116             formats.append({
 117                 'url': urljoin(self._MEDIA_BASE_URL, ovid),
 118                 'format_id': 'ovid',
 119                 'ext': 'mp4',
 120                 'width': int_or_none(post_data.get('vid_wid')),
 121                 'height': int_or_none(post_data.get('vid_hgt')),
 122             })
 123
 124         self._sort_formats(formats)
 125
 126         return {
 127             'id': post_id,
 128             'title': title,
 129             'description': description,
 130             'formats': formats,
 131             'subtitles': subtitles,
 132             'uploader': uploader,
 133             'uploader_id': str_or_none(
 134                 dict_get(user_data, ['_id', 'username'])
 135                 or post_data.get('uid')),
 136             'thumbnail': url_or_none(
 137                 urljoin(self._MEDIA_BASE_URL, post_data.get('main'))
 138                 or self._html_search_meta(['og:image', 'image'], webpage, 'thumbnail', fatal=False)),
 139             'timestamp': float_or_none(dict_get(post_data, ['cdate', 'udate']), scale=1000),
 140             'duration': float_or_none(post_data.get('vid_dur')),
 141             'tags': post_data.get('htgs'),
 142         }
 143
 144
 145 class GettrStreamingIE(GettrBaseIE):
 146     _VALID_URL = GettrBaseIE._BASE_REGEX + r'streaming/(?P<id>[a-z0-9]+)'
 147
 148     _TESTS = [{
 149         'url': 'https://gettr.com/streaming/psoiulc122',
 150         'info_dict': {
 151             'id': 'psoiulc122',
 152             'ext': 'mp4',
 153             'description': 'md5:56bca4b8f48f1743d9fd03d49c723017',
 154             'view_count': int,
 155             'uploader': 'Corona Investigative Committee',
 156             'uploader_id': 'coronacommittee',
 157             'duration': 5180.184,
 158             'thumbnail': r're:^https?://.+',
 159             'title': 'Day 1: Opening Session of the Grand Jury Proceeding',
 160             'timestamp': 1644080997.164,
 161             'upload_date': '20220205',
 162         }
 163     }, {
 164         'url': 'https://gettr.com/streaming/psfmeefcc1',
 165         'info_dict': {
 166             'id': 'psfmeefcc1',
 167             'ext': 'mp4',
 168             'title': 'Session 90: "The Virus Of Power"',
 169             'view_count': int,
 170             'uploader_id': 'coronacommittee',
 171             'description': 'md5:98986acdf656aa836bf36f9c9704c65b',
 172             'uploader': 'Corona Investigative Committee',
 173             'thumbnail': r're:^https?://.+',
 174             'duration': 21872.507,
 175             'timestamp': 1643976662.858,
 176             'upload_date': '20220204',
 177         }
 178     }]
 179
 180     def _real_extract(self, url):
 181         video_id = self._match_id(url)
 182         video_info = self._call_api('live/join/%s' % video_id, video_id, data={})
 183
 184         live_info = video_info['broadcast']
 185         live_url = url_or_none(live_info.get('url'))
 186
 187         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
 188             live_url, video_id, ext='mp4',
 189             entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if live_url else ([], {})
 190
 191         thumbnails = [{
 192             'url': urljoin(self._MEDIA_BASE_URL, thumbnail),
 193         } for thumbnail in try_get(video_info, lambda x: x['postData']['imgs'], list) or []]
 194
 195         self._sort_formats(formats)
 196
 197         return {
 198             'id': video_id,
 199             'title': try_get(video_info, lambda x: x['postData']['ttl'], str),
 200             'description': try_get(video_info, lambda x: x['postData']['dsc'], str),
 201             'formats': formats,
 202             'subtitles': subtitles,
 203             'thumbnails': thumbnails,
 204             'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname'], str),
 205             'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id'], str),
 206             'view_count': int_or_none(live_info.get('viewsCount')),
 207             'timestamp': float_or_none(live_info.get('startAt'), scale=1000),
 208             'duration': float_or_none(live_info.get('duration'), scale=1000),
 209             'is_live': bool_or_none(live_info.get('isLive')),
 210         }