5 from datetime
import datetime
7 from .common
import InfoExtractor
, SearchInfoExtractor
23 _API_BASE_URL
= 'https://prod-api-v2.production.rokfin.com/api/v2/public/'
26 class RokfinIE(InfoExtractor
):
27 _VALID_URL
= r
'https?://(?:www\.)?rokfin\.com/(?P<id>(?P<type>post|stream)/\d+)'
28 _NETRC_MACHINE
= 'rokfin'
29 _AUTH_BASE
= 'https://secure.rokfin.com/auth/realms/rokfin-web/protocol/openid-connect'
30 _access_mgmt_tokens
= {} # OAuth 2.0: RFC 6749, Sec. 1.4-5
32 'url': 'https://www.rokfin.com/post/57548/Mitt-Romneys-Crazy-Solution-To-Climate-Change',
36 'title': 'Mitt Romney\'s Crazy Solution To Climate Change',
37 'thumbnail': r
're:https://img\.production\.rokfin\.com/.+',
38 'upload_date': '20211023',
39 'timestamp': 1634998029,
40 'channel': 'Jimmy Dore',
42 'channel_url': 'https://rokfin.com/TheJimmyDoreShow',
44 'availability': 'public',
45 'live_status': 'not_live',
51 'url': 'https://rokfin.com/post/223/Julian-Assange-Arrested-Streaming-In-Real-Time',
55 'title': 'Julian Assange Arrested: Streaming In Real Time',
56 'thumbnail': r
're:https://img\.production\.rokfin\.com/.+',
57 'upload_date': '20190412',
58 'timestamp': 1555052644,
59 'channel': 'Ron Placone',
61 'channel_url': 'https://rokfin.com/RonPlacone',
62 'availability': 'public',
63 'live_status': 'not_live',
66 'tags': ['FreeThinkingMedia^', 'RealProgressives^'],
69 'url': 'https://www.rokfin.com/stream/10543/Its-A-Crazy-Mess-Regional-Director-Blows-Whistle-On-Pfizers-Vaccine-Trial-Data',
73 'title': '"It\'s A Crazy Mess" Regional Director Blows Whistle On Pfizer\'s Vaccine Trial Data',
74 'thumbnail': r
're:https://img\.production\.rokfin\.com/.+',
75 'description': 'md5:324ce2d3e3b62e659506409e458b9d8e',
76 'channel': 'TLAVagabond',
78 'channel_url': 'https://rokfin.com/TLAVagabond',
79 'availability': 'public',
82 'live_status': 'was_live',
83 'timestamp': 1635874720,
84 'release_timestamp': 1635874720,
85 'release_date': '20211102',
86 'upload_date': '20211102',
89 'tags': ['FreeThinkingMedia^'],
93 'url': 'https://rokfin.com/post/126703/Brave-New-World--Aldous-Huxley-DEEPDIVE--Chpts-13--Quite-Frankly--Jay-Dyer',
97 'title': 'Brave New World - Aldous Huxley DEEPDIVE! (Chpts 1-3) - Quite Frankly & Jay Dyer',
98 'thumbnail': r
're:https://img\.production\.rokfin\.com/.+',
99 'channel': 'Jay Dyer',
100 'channel_id': 186881,
101 'channel_url': 'https://rokfin.com/jaydyer',
102 'availability': 'premium_only',
103 'live_status': 'not_live',
104 'dislike_count': int,
106 'timestamp': 1678213357,
107 'upload_date': '20230307',
108 'tags': ['FreeThinkingMedia^', 'OpenMind^'],
109 'description': 'md5:cb04e32e68326c9b2b251b297bacff35',
113 'url': 'https://rokfin.com/stream/31332/The-Grayzone-live-on-Nordstream-blame-game',
115 'id': 'stream/31332',
117 'title': 'The Grayzone live on Nordstream blame game',
118 'thumbnail': r
're:https://image\.v\.rokfin\.com/.+',
119 'channel': 'Max Blumenthal',
120 'channel_id': 248902,
121 'channel_url': 'https://rokfin.com/MaxBlumenthal',
122 'availability': 'premium_only',
123 'live_status': 'was_live',
124 'dislike_count': int,
126 'timestamp': 1678475166,
127 'release_timestamp': 1678475166.0,
128 'release_date': '20230310',
129 'upload_date': '20230310',
130 'tags': ['FreeThinkingMedia^'],
134 def _real_extract(self
, url
):
135 video_id
, video_type
= self
._match
_valid
_url
(url
).group('id', 'type')
136 metadata
= self
._download
_json
_using
_access
_token
(f
'{_API_BASE_URL}{video_id}', video_id
)
138 scheduled
= unified_timestamp(metadata
.get('scheduledAt'))
139 live_status
= ('was_live' if metadata
.get('stoppedAt')
140 else 'is_upcoming' if scheduled
141 else 'is_live' if video_type
== 'stream'
144 video_url
= traverse_obj(metadata
, 'url', ('content', 'contentUrl'), expected_type
=url_or_none
)
145 if video_url
in (None, 'fake.m3u8'):
146 video_url
= format_field(self
._search
_regex
(
147 r
'https?://[^/]+/([^/]+)/storyboard.vtt',
148 traverse_obj(metadata
, 'timelineUrl', ('content', 'timelineUrl'), expected_type
=url_or_none
),
149 video_id
, default
=None), None, 'https://stream.v.rokfin.com/%s.m3u8')
151 formats
, subtitles
= [{'url': video_url}
] if video_url
else [], {}
152 if determine_ext(video_url
) == 'm3u8':
153 formats
, subtitles
= self
._extract
_m
3u8_formats
_and
_subtitles
(
154 video_url
, video_id
, fatal
=False, live
=live_status
== 'is_live')
157 if traverse_obj(metadata
, 'premiumPlan', 'premium'):
158 self
.raise_login_required('This video is only available to premium users', True, method
='cookies')
160 self
.raise_no_formats(
161 f
'Stream is offline; scheduled for {datetime.fromtimestamp(scheduled).strftime("%Y-%m-%d %H:%M:%S")}',
162 video_id
=video_id
, expected
=True)
164 uploader
= traverse_obj(metadata
, ('createdBy', 'username'), ('creator', 'username'))
165 timestamp
= (scheduled
or float_or_none(metadata
.get('postedAtMilli'), 1000)
166 or unified_timestamp(metadata
.get('creationDateTime')))
170 'subtitles': subtitles
,
171 'title': str_or_none(traverse_obj(metadata
, 'title', ('content', 'contentTitle'))),
172 'duration': float_or_none(traverse_obj(metadata
, ('content', 'duration'))),
173 'thumbnail': url_or_none(traverse_obj(metadata
, 'thumbnail', ('content', 'thumbnailUrl1'))),
174 'description': str_or_none(traverse_obj(metadata
, 'description', ('content', 'contentDescription'))),
175 'like_count': int_or_none(metadata
.get('likeCount')),
176 'dislike_count': int_or_none(metadata
.get('dislikeCount')),
177 'channel': str_or_none(traverse_obj(metadata
, ('createdBy', 'name'), ('creator', 'name'))),
178 'channel_id': traverse_obj(metadata
, ('createdBy', 'id'), ('creator', 'id')),
179 'channel_url': url_or_none(f
'https://rokfin.com/{uploader}') if uploader
else None,
180 'timestamp': timestamp
,
181 'release_timestamp': timestamp
if live_status
!= 'not_live' else None,
182 'tags': traverse_obj(metadata
, ('tags', ..., 'title'), expected_type
=str_or_none
),
183 'live_status': live_status
,
184 'availability': self
._availability
(
185 needs_premium
=bool(traverse_obj(metadata
, 'premiumPlan', 'premium')),
186 is_private
=False, needs_subscription
=False, needs_auth
=False, is_unlisted
=False),
187 # 'comment_count': metadata.get('numComments'), # Data provided by website is wrong
188 '__post_extractor': self
.extract_comments(video_id
) if video_type
== 'post' else None,
191 def _get_comments(self
, video_id
):
193 for page_n
in itertools
.count():
194 raw_comments
= self
._download
_json
(
195 f
'{_API_BASE_URL}comment?postId={video_id[5:]}&page={page_n}&size=50',
196 video_id
, note
=f
'Downloading viewer comments page {page_n + 1}{format_field(pages_total, None, " of %s")}',
199 for comment
in raw_comments
.get('content') or []:
201 'text': str_or_none(comment
.get('comment')),
202 'author': str_or_none(comment
.get('name')),
203 'id': comment
.get('commentId'),
204 'author_id': comment
.get('userId'),
206 'like_count': int_or_none(comment
.get('numLikes')),
207 'dislike_count': int_or_none(comment
.get('numDislikes')),
208 'timestamp': unified_timestamp(comment
.get('postedAt'))
211 pages_total
= int_or_none(raw_comments
.get('totalPages')) or None
212 is_last
= raw_comments
.get('last')
213 if not raw_comments
.get('content') or is_last
or (page_n
> pages_total
if pages_total
else is_last
is not False):
216 def _perform_login(self
, username
, password
):
217 # https://openid.net/specs/openid-connect-core-1_0.html#CodeFlowAuth (Sec. 3.1)
218 login_page
= self
._download
_webpage
(
219 f
'{self._AUTH_BASE}/auth?client_id=web&redirect_uri=https%3A%2F%2Frokfin.com%2Ffeed&response_mode=fragment&response_type=code&scope=openid',
220 None, note
='loading login page', errnote
='error loading login page')
221 authentication_point_url
= unescapeHTML(self
._search
_regex
(
222 r
'<form\s+[^>]+action\s*=\s*"(https://secure\.rokfin\.com/auth/realms/rokfin-web/login-actions/authenticate\?[^"]+)"',
223 login_page
, name
='Authentication URL'))
225 resp_body
= self
._download
_webpage
(
226 authentication_point_url
, None, note
='logging in', fatal
=False, expected_status
=404,
227 data
=urlencode_postdata({'username': username, 'password': password, 'rememberMe': 'off', 'credentialId': ''}
))
228 if not self
._authentication
_active
():
229 if re
.search(r
'(?i)(invalid\s+username\s+or\s+password)', resp_body
or ''):
230 raise ExtractorError('invalid username/password', expected
=True)
231 raise ExtractorError('Login failed')
233 urlh
= self
._request
_webpage
(
234 f
'{self._AUTH_BASE}/auth', None,
235 note
='granting user authorization', errnote
='user authorization rejected by Rokfin',
239 'redirect_uri': 'https://rokfin.com/silent-check-sso.html',
240 'response_mode': 'fragment',
241 'response_type': 'code',
244 self
._access
_mgmt
_tokens
= self
._download
_json
(
245 f
'{self._AUTH_BASE}/token', None,
246 note
='getting access credentials', errnote
='error getting access credentials',
247 data
=urlencode_postdata({
248 'code': urllib
.parse
.parse_qs(urllib
.parse
.urldefrag(urlh
.geturl()).fragment
).get('code')[0],
250 'grant_type': 'authorization_code',
251 'redirect_uri': 'https://rokfin.com/silent-check-sso.html'
254 def _authentication_active(self
):
256 {'KEYCLOAK_IDENTITY', 'KEYCLOAK_IDENTITY_LEGACY', 'KEYCLOAK_SESSION', 'KEYCLOAK_SESSION_LEGACY'}
257 - set(self
._get
_cookies
(self
._AUTH
_BASE
)))
259 def _get_auth_token(self
):
260 return try_get(self
._access
_mgmt
_tokens
, lambda x
: ' '.join([x
['token_type'], x
['access_token']]))
262 def _download_json_using_access_token(self
, url_or_request
, video_id
, headers
={}, query={}
):
263 assert 'authorization' not in headers
264 headers
= headers
.copy()
265 auth_token
= self
._get
_auth
_token
()
266 refresh_token
= self
._access
_mgmt
_tokens
.get('refresh_token')
268 headers
['authorization'] = auth_token
270 json_string
, urlh
= self
._download
_webpage
_handle
(
271 url_or_request
, video_id
, headers
=headers
, query
=query
, expected_status
=401)
272 if not auth_token
or urlh
.code
!= 401 or refresh_token
is None:
273 return self
._parse
_json
(json_string
, video_id
)
275 self
._access
_mgmt
_tokens
= self
._download
_json
(
276 f
'{self._AUTH_BASE}/token', video_id
,
277 note
='User authorization expired or canceled by Rokfin. Re-authorizing ...', errnote
='Failed to re-authorize',
278 data
=urlencode_postdata({
279 'grant_type': 'refresh_token',
280 'refresh_token': refresh_token
,
283 headers
['authorization'] = self
._get
_auth
_token
()
284 if headers
['authorization'] is None:
285 raise ExtractorError('User authorization lost', expected
=True)
287 return self
._download
_json
(url_or_request
, video_id
, headers
=headers
, query
=query
)
290 class RokfinPlaylistBaseIE(InfoExtractor
):
295 'dead_stream': 'stream',
299 def _get_video_data(self
, metadata
):
300 for content
in metadata
.get('content') or []:
301 media_type
= self
._TYPES
.get(content
.get('mediaType'))
302 video_id
= content
.get('id') if media_type
== 'post' else content
.get('mediaId')
303 if not media_type
or not video_id
:
306 yield self
.url_result(f
'https://rokfin.com/{media_type}/{video_id}', video_id
=f
'{media_type}/{video_id}',
307 video_title
=str_or_none(traverse_obj(content
, ('content', 'contentTitle'))))
310 class RokfinStackIE(RokfinPlaylistBaseIE
):
311 IE_NAME
= 'rokfin:stack'
312 IE_DESC
= 'Rokfin Stacks'
313 _VALID_URL
= r
'https?://(?:www\.)?rokfin\.com/stack/(?P<id>[^/]+)'
315 'url': 'https://www.rokfin.com/stack/271/Tulsi-Gabbard-Portsmouth-Townhall-FULL--Feb-9-2020',
322 def _real_extract(self
, url
):
323 list_id
= self
._match
_id
(url
)
324 return self
.playlist_result(self
._get
_video
_data
(
325 self
._download
_json
(f
'{_API_BASE_URL}stack/{list_id}', list_id
)), list_id
)
328 class RokfinChannelIE(RokfinPlaylistBaseIE
):
329 IE_NAME
= 'rokfin:channel'
330 IE_DESC
= 'Rokfin Channels'
331 _VALID_URL
= r
'https?://(?:www\.)?rokfin\.com/(?!((feed/?)|(discover/?)|(channels/?))$)(?P<id>[^/]+)/?$'
333 'url': 'https://rokfin.com/TheConvoCouch',
334 'playlist_mincount': 100,
337 'title': 'TheConvoCouch - New',
338 'description': 'md5:bb622b1bca100209b91cd685f7847f06',
351 def _real_initialize(self
):
352 self
._validate
_extractor
_args
()
354 def _validate_extractor_args(self
):
355 requested_tabs
= self
._configuration
_arg
('tab', None)
356 if requested_tabs
is not None and (len(requested_tabs
) > 1 or requested_tabs
[0] not in self
._TABS
):
357 raise ExtractorError(f
'Invalid extractor-arg "tab". Must be one of {", ".join(self._TABS)}', expected
=True)
359 def _entries(self
, channel_id
, channel_name
, tab
):
361 for page_n
in itertools
.count(0):
362 if tab
in ('posts', 'top'):
363 data_url
= f
'{_API_BASE_URL}user/{channel_name}/{tab}?page={page_n}&size=50'
365 data_url
= f
'{_API_BASE_URL}post/search/{tab}?page={page_n}&size=50&creator={channel_id}'
366 metadata
= self
._download
_json
(
367 data_url
, channel_name
,
368 note
=f
'Downloading video metadata page {page_n + 1}{format_field(pages_total, None, " of %s")}')
370 yield from self
._get
_video
_data
(metadata
)
371 pages_total
= int_or_none(metadata
.get('totalPages')) or None
372 is_last
= metadata
.get('last')
373 if is_last
or (page_n
> pages_total
if pages_total
else is_last
is not False):
376 def _real_extract(self
, url
):
377 channel_name
= self
._match
_id
(url
)
378 channel_info
= self
._download
_json
(f
'{_API_BASE_URL}user/{channel_name}', channel_name
)
379 channel_id
= channel_info
['id']
380 tab
= self
._configuration
_arg
('tab', default
=['new'])[0]
382 return self
.playlist_result(
383 self
._entries
(channel_id
, channel_name
, self
._TABS
[tab
]),
384 f
'{channel_id}-{tab}', f
'{channel_name} - {tab.title()}', str_or_none(channel_info
.get('description')))
387 class RokfinSearchIE(SearchInfoExtractor
):
388 IE_NAME
= 'rokfin:search'
389 IE_DESC
= 'Rokfin Search'
390 _SEARCH_KEY
= 'rkfnsearch'
392 'video': (('id', 'raw'), 'post'),
393 'audio': (('id', 'raw'), 'post'),
394 'stream': (('content_id', 'raw'), 'stream'),
395 'dead_stream': (('content_id', 'raw'), 'stream'),
396 'stack': (('content_id', 'raw'), 'stack'),
399 'url': 'rkfnsearch5:"zelenko"',
403 'title': '"zelenko"',
407 _db_access_key
= None
409 def _real_initialize(self
):
410 self
._db
_url
, self
._db
_access
_key
= self
.cache
.load(self
.ie_key(), 'auth', default
=(None, None))
412 self
._get
_db
_access
_credentials
()
414 def _search_results(self
, query
):
416 for page_number
in itertools
.count(1):
417 search_results
= self
._run
_search
_query
(
418 query
, data
={'query': query, 'page': {'size': 100, 'current': page_number}
},
419 note
=f
'Downloading page {page_number}{format_field(total_pages, None, " of ~%s")}')
420 total_pages
= traverse_obj(search_results
, ('meta', 'page', 'total_pages'), expected_type
=int_or_none
)
422 for result
in search_results
.get('results') or []:
423 video_id_key
, video_type
= self
._TYPES
.get(traverse_obj(result
, ('content_type', 'raw')), (None, None))
424 video_id
= traverse_obj(result
, video_id_key
, expected_type
=int_or_none
)
425 if video_id
and video_type
:
426 yield self
.url_result(url
=f
'https://rokfin.com/{video_type}/{video_id}')
427 if not search_results
.get('results'):
430 def _run_search_query(self
, video_id
, data
, **kwargs
):
431 data
= json
.dumps(data
).encode()
432 for attempt
in range(2):
433 search_results
= self
._download
_json
(
434 self
._db
_url
, video_id
, data
=data
, fatal
=(attempt
== 1),
435 headers
={'authorization': self._db_access_key}
, **kwargs
)
437 return search_results
438 self
.write_debug('Updating access credentials')
439 self
._get
_db
_access
_credentials
(video_id
)
441 def _get_db_access_credentials(self
, video_id
=None):
442 auth_data
= {'SEARCH_KEY': None, 'ENDPOINT_BASE': None}
443 notfound_err_page
= self
._download
_webpage
(
444 'https://rokfin.com/discover', video_id
, expected_status
=404, note
='Downloading home page')
445 for js_file_path
in re
.findall(r
'<script\b[^>]*\ssrc\s*=\s*"(/static/js/[^">]+)"', notfound_err_page
):
446 js_content
= self
._download
_webpage
(
447 f
'https://rokfin.com{js_file_path}', video_id
, note
='Downloading JavaScript file', fatal
=False)
448 auth_data
.update(re
.findall(
449 rf
'REACT_APP_({"|".join(auth_data.keys())})\s*:\s*"([^"]+)"', js_content
or ''))
450 if not all(auth_data
.values()):
453 self
._db
_url
= url_or_none(f
'{auth_data["ENDPOINT_BASE"]}/api/as/v1/engines/rokfin-search/search.json')
454 self
._db
_access
_key
= f
'Bearer {auth_data["SEARCH_KEY"]}'
455 self
.cache
.store(self
.ie_key(), 'auth', (self
._db
_url
, self
._db
_access
_key
))
457 raise ExtractorError('Unable to extract access credentials')