]>
jfr.im git - yt-dlp.git/blob - yt_dlp/extractor/rokfin.py
3 from datetime
import datetime
5 from .common
import InfoExtractor
19 _API_BASE_URL
= 'https://prod-api-v2.production.rokfin.com/api/v2/public/'
22 class RokfinIE(InfoExtractor
):
23 _VALID_URL
= r
'https?://(?:www\.)?rokfin\.com/(?P<id>(?P<type>post|stream)/\d+)'
25 'url': 'https://www.rokfin.com/post/57548/Mitt-Romneys-Crazy-Solution-To-Climate-Change',
29 'title': 'Mitt Romney\'s Crazy Solution To Climate Change',
30 'thumbnail': r
're:https://img\.production\.rokfin\.com/.+',
31 'upload_date': '20211023',
32 'timestamp': 1634998029,
33 'channel': 'Jimmy Dore',
35 'channel_url': 'https://rokfin.com/TheJimmyDoreShow',
37 'availability': 'public',
38 'live_status': 'not_live',
43 'url': 'https://rokfin.com/post/223/Julian-Assange-Arrested-Streaming-In-Real-Time',
47 'title': 'Julian Assange Arrested: Streaming In Real Time',
48 'thumbnail': r
're:https://img\.production\.rokfin\.com/.+',
49 'upload_date': '20190412',
50 'timestamp': 1555052644,
51 'channel': 'Ron Placone',
53 'channel_url': 'https://rokfin.com/RonPlacone',
54 'availability': 'public',
55 'live_status': 'not_live',
58 'tags': ['FreeThinkingMedia^', 'RealProgressives^'],
61 'url': 'https://www.rokfin.com/stream/10543/Its-A-Crazy-Mess-Regional-Director-Blows-Whistle-On-Pfizers-Vaccine-Trial-Data',
65 'title': '"It\'s A Crazy Mess" Regional Director Blows Whistle On Pfizer\'s Vaccine Trial Data',
66 'thumbnail': r
're:https://img\.production\.rokfin\.com/.+',
67 'description': 'md5:324ce2d3e3b62e659506409e458b9d8e',
68 'channel': 'Ryan Cristián',
70 'channel_url': 'https://rokfin.com/TLAVagabond',
71 'availability': 'public',
74 'live_status': 'was_live',
75 'timestamp': 1635874720,
76 'release_timestamp': 1635874720,
77 'release_date': '20211102',
78 'upload_date': '20211102',
81 'tags': ['FreeThinkingMedia^'],
85 def _real_extract(self
, url
):
86 video_id
, video_type
= self
._match
_valid
_url
(url
).group('id', 'type')
88 metadata
= self
._download
_json
(f
'{_API_BASE_URL}{video_id}', video_id
)
90 scheduled
= unified_timestamp(metadata
.get('scheduledAt'))
91 live_status
= ('was_live' if metadata
.get('stoppedAt')
92 else 'is_upcoming' if scheduled
93 else 'is_live' if video_type
== 'stream'
96 video_url
= traverse_obj(metadata
, 'url', ('content', 'contentUrl'), expected_type
=url_or_none
)
97 formats
, subtitles
= [{'url': video_url}
] if video_url
else [], {}
98 if determine_ext(video_url
) == 'm3u8':
99 formats
, subtitles
= self
._extract
_m
3u8_formats
_and
_subtitles
(
100 video_url
, video_id
, fatal
=False, live
=live_status
== 'is_live')
103 if traverse_obj(metadata
, 'premiumPlan', 'premium'):
104 self
.raise_login_required('This video is only available to premium users', True, method
='cookies')
106 self
.raise_no_formats(
107 f
'Stream is offline; sheduled for {datetime.fromtimestamp(scheduled).strftime("%Y-%m-%d %H:%M:%S")}',
108 video_id
=video_id
, expected
=True)
109 self
._sort
_formats
(formats
)
111 uploader
= traverse_obj(metadata
, ('createdBy', 'username'), ('creator', 'username'))
112 timestamp
= (scheduled
or float_or_none(metadata
.get('postedAtMilli'), 1000)
113 or unified_timestamp(metadata
.get('creationDateTime')))
117 'subtitles': subtitles
,
118 'title': str_or_none(traverse_obj(metadata
, 'title', ('content', 'contentTitle'))),
119 'duration': float_or_none(traverse_obj(metadata
, ('content', 'duration'))),
120 'thumbnail': url_or_none(traverse_obj(metadata
, 'thumbnail', ('content', 'thumbnailUrl1'))),
121 'description': str_or_none(traverse_obj(metadata
, 'description', ('content', 'contentDescription'))),
122 'like_count': int_or_none(metadata
.get('likeCount')),
123 'dislike_count': int_or_none(metadata
.get('dislikeCount')),
124 'channel': str_or_none(traverse_obj(metadata
, ('createdBy', 'name'), ('creator', 'name'))),
125 'channel_id': traverse_obj(metadata
, ('createdBy', 'id'), ('creator', 'id')),
126 'channel_url': url_or_none(f
'https://rokfin.com/{uploader}') if uploader
else None,
127 'timestamp': timestamp
,
128 'release_timestamp': timestamp
if live_status
!= 'not_live' else None,
129 'tags': traverse_obj(metadata
, ('tags', ..., 'title'), expected_type
=str_or_none
),
130 'live_status': live_status
,
131 'availability': self
._availability
(
132 needs_premium
=bool(traverse_obj(metadata
, 'premiumPlan', 'premium')),
133 is_private
=False, needs_subscription
=False, needs_auth
=False, is_unlisted
=False),
134 # 'comment_count': metadata.get('numComments'), # Data provided by website is wrong
135 '__post_extractor': self
.extract_comments(video_id
) if video_type
== 'post' else None,
138 def _get_comments(self
, video_id
):
140 for page_n
in itertools
.count():
141 raw_comments
= self
._download
_json
(
142 f
'{_API_BASE_URL}comment?postId={video_id[5:]}&page={page_n}&size=50',
143 video_id
, note
=f
'Downloading viewer comments page {page_n + 1}{format_field(pages_total, template=" of %s")}',
146 for comment
in raw_comments
.get('content') or []:
148 'text': str_or_none(comment
.get('comment')),
149 'author': str_or_none(comment
.get('name')),
150 'id': comment
.get('commentId'),
151 'author_id': comment
.get('userId'),
153 'like_count': int_or_none(comment
.get('numLikes')),
154 'dislike_count': int_or_none(comment
.get('numDislikes')),
155 'timestamp': unified_timestamp(comment
.get('postedAt'))
158 pages_total
= int_or_none(raw_comments
.get('totalPages')) or None
159 is_last
= raw_comments
.get('last')
160 if not raw_comments
.get('content') or is_last
or (page_n
> pages_total
if pages_total
else is_last
is not False):
164 class RokfinPlaylistBaseIE(InfoExtractor
):
169 'dead_stream': 'stream',
173 def _get_video_data(self
, metadata
):
174 for content
in metadata
.get('content') or []:
175 media_type
= self
._TYPES
.get(content
.get('mediaType'))
176 video_id
= content
.get('id') if media_type
== 'post' else content
.get('mediaId')
177 if not media_type
or not video_id
:
180 yield self
.url_result(f
'https://rokfin.com/{media_type}/{video_id}', video_id
=f
'{media_type}/{video_id}',
181 video_title
=str_or_none(traverse_obj(content
, ('content', 'contentTitle'))))
184 class RokfinStackIE(RokfinPlaylistBaseIE
):
185 IE_NAME
= 'rokfin:stack'
186 _VALID_URL
= r
'https?://(?:www\.)?rokfin\.com/stack/(?P<id>[^/]+)'
188 'url': 'https://www.rokfin.com/stack/271/Tulsi-Gabbard-Portsmouth-Townhall-FULL--Feb-9-2020',
195 def _real_extract(self
, url
):
196 list_id
= self
._match
_id
(url
)
197 return self
.playlist_result(self
._get
_video
_data
(
198 self
._download
_json
(f
'{_API_BASE_URL}stack/{list_id}', list_id
)), list_id
)
201 class RokfinChannelIE(RokfinPlaylistBaseIE
):
202 IE_NAME
= 'rokfin:channel'
203 _VALID_URL
= r
'https?://(?:www\.)?rokfin\.com/(?!((feed/?)|(discover/?)|(channels/?))$)(?P<id>[^/]+)/?$'
205 'url': 'https://rokfin.com/TheConvoCouch',
206 'playlist_mincount': 100,
209 'title': 'TheConvoCouch - New',
210 'description': 'md5:bb622b1bca100209b91cd685f7847f06',
223 def _real_initialize(self
):
224 self
._validate
_extractor
_args
()
226 def _validate_extractor_args(self
):
227 requested_tabs
= self
._configuration
_arg
('tab', None)
228 if requested_tabs
is not None and (len(requested_tabs
) > 1 or requested_tabs
[0] not in self
._TABS
):
229 raise ExtractorError(f
'Invalid extractor-arg "tab". Must be one of {", ".join(self._TABS)}', expected
=True)
231 def _entries(self
, channel_id
, channel_name
, tab
):
233 for page_n
in itertools
.count(0):
234 if tab
in ('posts', 'top'):
235 data_url
= f
'{_API_BASE_URL}user/{channel_name}/{tab}?page={page_n}&size=50'
237 data_url
= f
'{_API_BASE_URL}post/search/{tab}?page={page_n}&size=50&creator={channel_id}'
238 metadata
= self
._download
_json
(
239 data_url
, channel_name
,
240 note
=f
'Downloading video metadata page {page_n + 1}{format_field(pages_total, template=" of %s")}')
242 yield from self
._get
_video
_data
(metadata
)
243 pages_total
= int_or_none(metadata
.get('totalPages')) or None
244 is_last
= metadata
.get('last')
245 if is_last
or (page_n
> pages_total
if pages_total
else is_last
is not False):
248 def _real_extract(self
, url
):
249 channel_name
= self
._match
_id
(url
)
250 channel_info
= self
._download
_json
(f
'{_API_BASE_URL}user/{channel_name}', channel_name
)
251 channel_id
= channel_info
['id']
252 tab
= self
._configuration
_arg
('tab', default
=['new'])[0]
254 return self
.playlist_result(
255 self
._entries
(channel_id
, channel_name
, self
._TABS
[tab
]),
256 f
'{channel_id}-{tab}', f
'{channel_name} - {tab.title()}', str_or_none(channel_info
.get('description')))