]> jfr.im git - yt-dlp.git/blobdiff - yt_dlp/extractor/gettr.py
[ie/mlbtv] Fix extraction (#10296)
[yt-dlp.git] / yt_dlp / extractor / gettr.py
index 179bd7c47f0d01bfd756b24c48329f25ee6ce7fa..2a9d5e7e98c9566a43dce1bb6fae1c182312474c 100644 (file)
@@ -1,15 +1,12 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
 from .common import InfoExtractor
 from ..utils import (
-    bool_or_none,
     ExtractorError,
+    bool_or_none,
     dict_get,
     float_or_none,
     int_or_none,
-    remove_end,
     str_or_none,
+    traverse_obj,
     try_get,
     url_or_none,
     urljoin,
@@ -36,10 +33,12 @@ class GettrIE(GettrBaseIE):
             'ext': 'mp4',
             'uploader': 'EpochTV',
             'uploader_id': 'epochtv',
+            'upload_date': '20210927',
             'thumbnail': r're:^https?://.+/out\.jpg',
-            'timestamp': 1632782451058,
+            'timestamp': 1632782451.058,
             'duration': 58.5585,
-        }
+            'tags': ['hornofafrica', 'explorations'],
+        },
     }, {
         'url': 'https://gettr.com/post/p4iahp',
         'info_dict': {
@@ -49,42 +48,69 @@ class GettrIE(GettrBaseIE):
             'ext': 'mp4',
             'uploader': 'Neues Forum Freiheit',
             'uploader_id': 'nf_freiheit',
+            'upload_date': '20210718',
             'thumbnail': r're:^https?://.+/out\.jpg',
-            'timestamp': 1626594455017,
+            'timestamp': 1626594455.017,
             'duration': 23,
-        }
+            'tags': 'count:12',
+        },
+    }, {
+        # quote post
+        'url': 'https://gettr.com/post/pxn5b743a9',
+        'only_matching': True,
+    }, {
+        # quote with video
+        'url': 'https://gettr.com/post/pxtiiz5ca2',
+        'only_matching': True,
+    }, {
+        # streaming embed
+        'url': 'https://gettr.com/post/pxlu8p3b13',
+        'only_matching': True,
+    }, {
+        # youtube embed
+        'url': 'https://gettr.com/post/pv6wp9e24c',
+        'only_matching': True,
+        'add_ie': ['Youtube'],
     }]
 
     def _real_extract(self, url):
         post_id = self._match_id(url)
         webpage = self._download_webpage(url, post_id)
-
-        api_data = self._call_api('post/%s?incl="poststats|userinfo"' % post_id, post_id)
+        api_data = self._call_api(f'post/{post_id}?incl="poststats|userinfo"', post_id)
 
         post_data = api_data.get('data')
-        user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']]) or {}
+        user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']], dict) or {}
+
+        vid = post_data.get('vid')
+        ovid = post_data.get('ovid')
 
-        if post_data.get('nfound'):
-            raise ExtractorError(post_data.get('txt'), expected=True)
+        if post_data.get('p_type') == 'stream':
+            return self.url_result(f'https://gettr.com/streaming/{post_id}', ie='GettrStreaming', video_id=post_id)
+
+        if not (ovid or vid):
+            embed_url = url_or_none(post_data.get('prevsrc'))
+            shared_post_id = traverse_obj(api_data, ('aux', 'shrdpst', '_id'), ('data', 'rpstIds', 0), expected_type=str)
+
+            if embed_url:
+                return self.url_result(embed_url)
+            elif shared_post_id:
+                return self.url_result(f'https://gettr.com/post/{shared_post_id}', ie='Gettr', video_id=shared_post_id)
+            else:
+                raise ExtractorError('There\'s no video in this post.')
 
         title = description = str_or_none(
             post_data.get('txt') or self._og_search_description(webpage))
 
         uploader = str_or_none(
             user_data.get('nickname')
-            or remove_end(self._og_search_title(webpage), ' on GETTR'))
-        if uploader:
-            title = '%s - %s' % (uploader, title)
-
-        if not dict_get(post_data, ['vid', 'ovid']):
-            raise ExtractorError('There\'s no video in this post.')
+            or self._search_regex(r'^(.+?) on GETTR', self._og_search_title(webpage, default=''), 'uploader', fatal=False))
 
-        vid = post_data.get('vid')
-        ovid = post_data.get('ovid')
+        if uploader:
+            title = f'{uploader} - {title}'
 
-        formats = self._extract_m3u8_formats(
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
             urljoin(self._MEDIA_BASE_URL, vid), post_id, 'mp4',
-            entry_protocol='m3u8_native', m3u8_id='hls') if vid else []
+            entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if vid else ([], {})
 
         if ovid:
             formats.append({
@@ -93,25 +119,22 @@ def _real_extract(self, url):
                 'ext': 'mp4',
                 'width': int_or_none(post_data.get('vid_wid')),
                 'height': int_or_none(post_data.get('vid_hgt')),
-                'source_preference': 1,
-                'quality': 1,
             })
 
-        self._sort_formats(formats)
-
         return {
             'id': post_id,
             'title': title,
             'description': description,
-            'thumbnail': url_or_none(
-                urljoin(self._MEDIA_BASE_URL, post_data.get('main'))
-                or self._og_search_thumbnail(webpage)),
-            'timestamp': int_or_none(post_data.get('cdate')),
+            'formats': formats,
+            'subtitles': subtitles,
+            'uploader': uploader,
             'uploader_id': str_or_none(
                 dict_get(user_data, ['_id', 'username'])
                 or post_data.get('uid')),
-            'uploader': uploader,
-            'formats': formats,
+            'thumbnail': url_or_none(
+                urljoin(self._MEDIA_BASE_URL, post_data.get('main'))
+                or self._html_search_meta(['og:image', 'image'], webpage, 'thumbnail', fatal=False)),
+            'timestamp': float_or_none(dict_get(post_data, ['cdate', 'udate']), scale=1000),
             'duration': float_or_none(post_data.get('vid_dur')),
             'tags': post_data.get('htgs'),
         }
@@ -134,7 +157,7 @@ class GettrStreamingIE(GettrBaseIE):
             'title': 'Day 1: Opening Session of the Grand Jury Proceeding',
             'timestamp': 1644080997.164,
             'upload_date': '20220205',
-        }
+        },
     }, {
         'url': 'https://gettr.com/streaming/psfmeefcc1',
         'info_dict': {
@@ -149,12 +172,12 @@ class GettrStreamingIE(GettrBaseIE):
             'duration': 21872.507,
             'timestamp': 1643976662.858,
             'upload_date': '20220204',
-        }
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_info = self._call_api('live/join/%s' % video_id, video_id, data={})
+        video_info = self._call_api(f'live/join/{video_id}', video_id, data={})
 
         live_info = video_info['broadcast']
         live_url = url_or_none(live_info.get('url'))
@@ -165,19 +188,17 @@ def _real_extract(self, url):
 
         thumbnails = [{
             'url': urljoin(self._MEDIA_BASE_URL, thumbnail),
-        } for thumbnail in try_get(video_info, lambda x: x['postData']['imgs']) or []]
-
-        self._sort_formats(formats)
+        } for thumbnail in try_get(video_info, lambda x: x['postData']['imgs'], list) or []]
 
         return {
             'id': video_id,
-            'title': try_get(video_info, lambda x: x['postData']['ttl']),
-            'description': try_get(video_info, lambda x: x['postData']['dsc']),
+            'title': try_get(video_info, lambda x: x['postData']['ttl'], str),
+            'description': try_get(video_info, lambda x: x['postData']['dsc'], str),
             'formats': formats,
             'subtitles': subtitles,
             'thumbnails': thumbnails,
-            'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname']),
-            'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id']),
+            'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname'], str),
+            'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id'], str),
             'view_count': int_or_none(live_info.get('viewsCount')),
             'timestamp': float_or_none(live_info.get('startAt'), scale=1000),
             'duration': float_or_none(live_info.get('duration'), scale=1000),