]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/rcti.py
projects / yt-dlp.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[RCTIPlusSeries] Lazy extraction and video type selection (#2050)
[yt-dlp.git] / yt_dlp / extractor / rcti.py
CommitLineData
17f0eb66
M		 1# coding: utf-8
2from __future__ import unicode_literals
3
d9e6e948 4import json
3dbb2a9d 5import random
d9e6e948 6import time
17f0eb66
M		 7
8from .common import InfoExtractor
3dbb2a9d 9from ..compat import compat_HTTPError
17f0eb66 10from ..utils import (
3dbb2a9d 11 dict_get,
17f0eb66 12 ExtractorError,
17f0eb66 13 strip_or_none,
c8b80b96 14 traverse_obj,
17f0eb66
M		 15 try_get
16)
17
18
19class RCTIPlusBaseIE(InfoExtractor):
20 def _real_initialize(self):
21 self._AUTH_KEY = self._download_json(
22 'https://api.rctiplus.com/api/v1/visitor?platform=web', # platform can be web, mweb, android, ios
23 None, 'Fetching authorization key')['data']['access_token']
24
25 def _call_api(self, url, video_id, note=None):
26 json = self._download_json(
27 url, video_id, note=note, headers={'Authorization': self._AUTH_KEY})
28 if json.get('status', {}).get('code', 0) != 0:
c8b80b96 29 raise ExtractorError(f'{self.IE_NAME} said: {json["status"]["message_client"]}', cause=json)
17f0eb66
M		 30 return json.get('data'), json.get('meta')
31
32
33class RCTIPlusIE(RCTIPlusBaseIE):
3dbb2a9d 34 _VALID_URL = r'https://www\.rctiplus\.com/(?:programs/\d+?/.*?/)?(?P<type>episode|clip|extra|live-event|missed-event)/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
17f0eb66
M		 35 _TESTS = [{
36 'url': 'https://www.rctiplus.com/programs/1259/kiko-untuk-lola/episode/22124/untuk-lola',
37 'md5': '56ed45affad45fa18d5592a1bc199997',
38 'info_dict': {
39 'id': 'v_e22124',
40 'title': 'Untuk Lola',
41 'display_id': 'untuk-lola',
42 'description': 'md5:2b809075c0b1e071e228ad6d13e41deb',
43 'ext': 'mp4',
44 'duration': 1400,
45 'timestamp': 1615978800,
46 'upload_date': '20210317',
47 'series': 'Kiko : Untuk Lola',
48 'season_number': 1,
49 'episode_number': 1,
50 'channel': 'RCTI',
51 },
52 'params': {
53 'fixup': 'never',
54 },
55 }, { # Clip; Series title doesn't appear on metadata JSON
56 'url': 'https://www.rctiplus.com/programs/316/cahaya-terindah/clip/3921/make-a-wish',
57 'md5': 'd179b2ff356f0e91a53bcc6a4d8504f0',
58 'info_dict': {
59 'id': 'v_c3921',
60 'title': 'Make A Wish',
61 'display_id': 'make-a-wish',
62 'description': 'Make A Wish',
63 'ext': 'mp4',
64 'duration': 288,
65 'timestamp': 1571652600,
66 'upload_date': '20191021',
67 'series': 'Cahaya Terindah',
68 'channel': 'RCTI',
69 },
70 'params': {
71 'fixup': 'never',
72 },
73 }, { # Extra
74 'url': 'https://www.rctiplus.com/programs/616/inews-malam/extra/9438/diungkapkan-melalui-surat-terbuka-ceo-ruangguru-belva-devara-mundur-dari-staf-khusus-presiden',
75 'md5': 'c48106afdbce609749f5e0c007d9278a',
76 'info_dict': {
77 'id': 'v_ex9438',
78 'title': 'md5:2ede828c0f8bde249e0912be150314ca',
79 'display_id': 'md5:62b8d4e9ff096db527a1ad797e8a9933',
80 'description': 'md5:2ede828c0f8bde249e0912be150314ca',
81 'ext': 'mp4',
82 'duration': 93,
83 'timestamp': 1587561540,
84 'upload_date': '20200422',
85 'series': 'iNews Malam',
86 'channel': 'INews',
87 },
3dbb2a9d
M		 88 }, { # Missed event/replay
89 'url': 'https://www.rctiplus.com/missed-event/2507/mou-signing-ceremony-27-juli-2021-1400-wib',
90 'md5': '649c5f27250faed1452ca8b91e06922d',
91 'info_dict': {
92 'id': 'v_pe2507',
93 'title': 'MOU Signing Ceremony | 27 Juli 2021 | 14.00 WIB',
94 'display_id': 'mou-signing-ceremony-27-juli-2021-1400-wib',
95 'ext': 'mp4',
96 'timestamp': 1627142400,
97 'upload_date': '20210724',
98 'was_live': True,
99 'release_timestamp': 1627369200,
100 },
101 'params': {
102 'fixup': 'never',
103 },
104 }, { # Live event; Cloudfront CDN
105 'url': 'https://www.rctiplus.com/live-event/2530/dai-muda-charging-imun-dengan-iman-4-agustus-2021-1600-wib',
106 'info_dict': {
107 'id': 'v_le2530',
108 'title': 'Dai Muda : Charging Imun dengan Iman | 4 Agustus 2021 | 16.00 WIB',
109 'display_id': 'dai-muda-charging-imun-dengan-iman-4-agustus-2021-1600-wib',
110 'ext': 'mp4',
111 'timestamp': 1627898400,
112 'upload_date': '20210802',
113 'release_timestamp': 1628067600,
114 },
115 'params': {
116 'skip_download': True,
117 },
118 'skip': 'This live event has ended.',
119 }, { # TV; live_at is null
120 'url': 'https://www.rctiplus.com/live-event/1/rcti',
121 'info_dict': {
122 'id': 'v_lt1',
123 'title': 'RCTI',
124 'display_id': 'rcti',
125 'ext': 'mp4',
126 'timestamp': 1546344000,
127 'upload_date': '20190101',
128 'is_live': True,
129 },
130 'params': {
131 'skip_download': True,
3dbb2a9d 132 },
17f0eb66 133 }]
d9e6e948
M		 134 _CONVIVA_JSON_TEMPLATE = {
135 't': 'CwsSessionHb',
136 'cid': 'ff84ae928c3b33064b76dec08f12500465e59a6f',
137 'clid': '0',
138 'sid': 0,
139 'seq': 0,
140 'caps': 0,
141 'sf': 7,
142 'sdk': True,
143 }
17f0eb66 144
17f0eb66 145 def _real_extract(self, url):
5ad28e7f 146 match = self._match_valid_url(url).groupdict()
3dbb2a9d 147 video_type, video_id, display_id = match['type'], match['id'], match['display_id']
17f0eb66 148
3dbb2a9d
M		 149 url_api_version = 'v2' if video_type == 'missed-event' else 'v1'
150 appier_id = '23984824_' + str(random.randint(0, 10000000000)) # Based on the webpage's uuidRandom generator
17f0eb66 151 video_json = self._call_api(
3dbb2a9d 152 f'https://api.rctiplus.com/api/{url_api_version}/{video_type}/{video_id}/url?appierid={appier_id}', display_id, 'Downloading video URL JSON')[0]
17f0eb66 153 video_url = video_json['url']
3dbb2a9d
M		 154
155 is_upcoming = try_get(video_json, lambda x: x['current_date'] < x['live_at'])
156 if is_upcoming is None:
157 is_upcoming = try_get(video_json, lambda x: x['current_date'] < x['start_date'])
158 if is_upcoming:
159 self.raise_no_formats(
160 'This event will start at %s.' % video_json['live_label'] if video_json.get('live_label') else 'This event has not started yet.', expected=True)
17f0eb66 161 if 'akamaized' in video_url:
d9e6e948
M		 162 # For some videos hosted on Akamai's CDN (possibly AES-encrypted ones?), a session needs to at least be made via Conviva's API
163 conviva_json_data = {
164 **self._CONVIVA_JSON_TEMPLATE,
165 'url': video_url,
166 'sst': int(time.time())
167 }
168 conviva_json_res = self._download_json(
169 'https://ff84ae928c3b33064b76dec08f12500465e59a6f.cws.conviva.com/0/wsg', display_id,
170 'Creating Conviva session', 'Failed to create Conviva session',
171 fatal=False, data=json.dumps(conviva_json_data).encode('utf-8'))
172 if conviva_json_res and conviva_json_res.get('err') != 'ok':
173 self.report_warning('Conviva said: %s' % str(conviva_json_res.get('err')))
17f0eb66
M		 174
175 video_meta, meta_paths = self._call_api(
176 'https://api.rctiplus.com/api/v1/%s/%s' % (video_type, video_id), display_id, 'Downloading video metadata')
177
178 thumbnails, image_path = [], meta_paths.get('image_path', 'https://rstatic.akamaized.net/media/')
179 if video_meta.get('portrait_image'):
180 thumbnails.append({
181 'id': 'portrait_image',
182 'url': '%s%d%s' % (image_path, 2000, video_meta['portrait_image']) # 2000px seems to be the highest resolution that can be given
183 })
184 if video_meta.get('landscape_image'):
185 thumbnails.append({
186 'id': 'landscape_image',
187 'url': '%s%d%s' % (image_path, 2000, video_meta['landscape_image'])
188 })
3dbb2a9d
M		 189 try:
190 formats = self._extract_m3u8_formats(video_url, display_id, 'mp4', headers={'Referer': 'https://www.rctiplus.com/'})
191 except ExtractorError as e:
192 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
193 self.raise_geo_restricted(countries=['ID'], metadata_available=True)
194 else:
195 raise e
17f0eb66 196 for f in formats:
3dbb2a9d
M		 197 if 'akamaized' in f['url'] or 'cloudfront' in f['url']:
198 f.setdefault('http_headers', {})['Referer'] = 'https://www.rctiplus.com/' # Referer header is required for akamai/cloudfront CDNs
17f0eb66
M		 199
200 self._sort_formats(formats)
201
202 return {
203 'id': video_meta.get('product_id') or video_json.get('product_id'),
3dbb2a9d 204 'title': dict_get(video_meta, ('title', 'name')) or dict_get(video_json, ('content_name', 'assets_name')),
17f0eb66
M		 205 'display_id': display_id,
206 'description': video_meta.get('summary'),
3dbb2a9d 207 'timestamp': video_meta.get('release_date') or video_json.get('start_date'),
17f0eb66 208 'duration': video_meta.get('duration'),
3dbb2a9d 209 'categories': [video_meta['genre']] if video_meta.get('genre') else None,
17f0eb66
M		 210 'average_rating': video_meta.get('star_rating'),
211 'series': video_meta.get('program_title') or video_json.get('program_title'),
212 'season_number': video_meta.get('season'),
213 'episode_number': video_meta.get('episode'),
214 'channel': video_json.get('tv_name'),
215 'channel_id': video_json.get('tv_id'),
216 'formats': formats,
3dbb2a9d
M		 217 'thumbnails': thumbnails,
218 'is_live': video_type == 'live-event' and not is_upcoming,
219 'was_live': video_type == 'missed-event',
220 'live_status': 'is_upcoming' if is_upcoming else None,
221 'release_timestamp': video_json.get('live_at'),
17f0eb66
M		 222 }
223
224
225class RCTIPlusSeriesIE(RCTIPlusBaseIE):
c8b80b96 226 _VALID_URL = r'https://www\.rctiplus\.com/programs/(?P<id>\d+)/(?P<display_id>[^/?#&]+)(?:/(?P<type>episodes|extras|clips))?'
17f0eb66 227 _TESTS = [{
c8b80b96
M		 228 'url': 'https://www.rctiplus.com/programs/829/putri-untuk-pangeran',
229 'playlist_mincount': 1019,
17f0eb66 230 'info_dict': {
c8b80b96
M		 231 'id': '829',
232 'title': 'Putri Untuk Pangeran',
233 'description': 'md5:aca7b54d05bd95a67d4f4613cc1d622d',
234 'age_limit': 2,
235 'cast': ['Verrel Bramasta', 'Ranty Maria', 'Riza Syah', 'Ivan Fadilla', 'Nicole Parham', 'Dll', 'Aviv Elham'],
236 'display_id': 'putri-untuk-pangeran',
237 'tag': 'count:18',
17f0eb66 238 },
c8b80b96
M		 239 }, { # No episodes
240 'url': 'https://www.rctiplus.com/programs/615/inews-pagi',
241 'playlist_mincount': 388,
242 'info_dict': {
243 'id': '615',
244 'title': 'iNews Pagi',
245 'description': 'md5:f18ee3d4643cfb41c358e5a9b693ee04',
246 'age_limit': 2,
247 'tag': 'count:11',
248 'display_id': 'inews-pagi',
249 }
17f0eb66
M		 250 }]
251 _AGE_RATINGS = { # Based off https://id.wikipedia.org/wiki/Sistem_rating_konten_televisi with additional ratings
252 'S-SU': 2,
253 'SU': 2,
254 'P': 2,
255 'A': 7,
256 'R': 13,
257 'R-R/1': 17, # Labelled as 17+ despite being R
258 'D': 18,
259 }
260
3dbb2a9d
M		 261 @classmethod
262 def suitable(cls, url):
263 return False if RCTIPlusIE.suitable(url) else super(RCTIPlusSeriesIE, cls).suitable(url)
264
17f0eb66
M		 265 def _entries(self, url, display_id=None, note='Downloading entries JSON', metadata={}):
266 total_pages = 0
267 try:
268 total_pages = self._call_api(
269 '%s&length=20&page=0' % url,
270 display_id, note)[1]['pagination']['total_page']
271 except ExtractorError as e:
272 if 'not found' in str(e):
273 return []
274 raise e
275 if total_pages <= 0:
276 return []
277
278 for page_num in range(1, total_pages + 1):
279 episode_list = self._call_api(
280 '%s&length=20&page=%s' % (url, page_num),
281 display_id, '%s page %s' % (note, page_num))[0] or []
282
283 for video_json in episode_list:
c8b80b96
M		 284 yield {
285 '_type': 'url',
286 'url': video_json['share_link'],
287 'ie_key': RCTIPlusIE.ie_key(),
288 'id': video_json.get('product_id'),
289 'title': video_json.get('title'),
290 'display_id': video_json.get('title_code').replace('_', '-'),
291 'description': video_json.get('summary'),
292 'timestamp': video_json.get('release_date'),
293 'duration': video_json.get('duration'),
294 'season_number': video_json.get('season'),
295 'episode_number': video_json.get('episode'),
296 **metadata
297 }
298
299 def _series_entries(self, series_id, display_id=None, video_type=None, metadata={}):
300 if not video_type or video_type in 'episodes':
301 try:
302 seasons_list = self._call_api(
303 f'https://api.rctiplus.com/api/v1/program/{series_id}/season',
304 display_id, 'Downloading seasons list JSON')[0]
305 except ExtractorError as e:
306 if 'not found' not in str(e):
307 raise
308 seasons_list = []
309 for season in seasons_list:
310 yield from self._entries(
311 f'https://api.rctiplus.com/api/v2/program/{series_id}/episode?season={season["season"]}',
312 display_id, f'Downloading season {season["season"]} episode entries', metadata)
313 if not video_type or video_type in 'extras':
314 yield from self._entries(
315 f'https://api.rctiplus.com/api/v2/program/{series_id}/extra?content_id=0',
316 display_id, 'Downloading extra entries', metadata)
317 if not video_type or video_type in 'clips':
318 yield from self._entries(
319 f'https://api.rctiplus.com/api/v2/program/{series_id}/clip?content_id=0',
320 display_id, 'Downloading clip entries', metadata)
17f0eb66
M		 321
322 def _real_extract(self, url):
c8b80b96
M		 323 series_id, display_id, video_type = self._match_valid_url(url).group('id', 'display_id', 'type')
324 if video_type:
325 self.report_warning(
326 f'Only {video_type} will be downloaded. '
327 f'To download everything from the series, remove "/{video_type}" from the URL')
17f0eb66
M		 328
329 series_meta, meta_paths = self._call_api(
c8b80b96 330 f'https://api.rctiplus.com/api/v1/program/{series_id}/detail', display_id, 'Downloading series metadata')
17f0eb66 331 metadata = {
c8b80b96
M		 332 'age_limit': try_get(series_meta, lambda x: self._AGE_RATINGS[x['age_restriction'][0]['code']]),
333 'cast': traverse_obj(series_meta, (('starring', 'creator', 'writer'), ..., 'name'),
334 expected_type=lambda x: strip_or_none(x) or None),
335 'tag': traverse_obj(series_meta, ('tag', ..., 'name'),
336 expected_type=lambda x: strip_or_none(x) or None),
17f0eb66 337 }
c8b80b96
M		 338 return self.playlist_result(
339 self._series_entries(series_id, display_id, video_type, metadata), series_id,
340 series_meta.get('title'), series_meta.get('summary'), display_id=display_id, **metadata)
3dbb2a9d
M		 341
342
343class RCTIPlusTVIE(RCTIPlusBaseIE):
344 _VALID_URL = r'https://www\.rctiplus\.com/((tv/(?P<tvname>\w+))|(?P<eventname>live-event|missed-event))'
345 _TESTS = [{
346 'url': 'https://www.rctiplus.com/tv/rcti',
347 'info_dict': {
348 'id': 'v_lt1',
349 'title': 'RCTI',
350 'ext': 'mp4',
351 'timestamp': 1546344000,
352 'upload_date': '20190101',
353 },
354 'params': {
355 'skip_download': True,
3dbb2a9d
M		 356 }
357 }, {
358 # Returned video will always change
359 'url': 'https://www.rctiplus.com/live-event',
360 'only_matching': True,
361 }, {
362 # Returned video will also always change
363 'url': 'https://www.rctiplus.com/missed-event',
364 'only_matching': True,
365 }]
366
367 @classmethod
368 def suitable(cls, url):
369 return False if RCTIPlusIE.suitable(url) else super(RCTIPlusTVIE, cls).suitable(url)
370
371 def _real_extract(self, url):
5ad28e7f 372 match = self._match_valid_url(url).groupdict()
3dbb2a9d
M		 373 tv_id = match.get('tvname') or match.get('eventname')
374 webpage = self._download_webpage(url, tv_id)
375 video_type, video_id = self._search_regex(
c8b80b96
M		 376 r'url\s*:\s*["\']https://api\.rctiplus\.com/api/v./(?P<type>[^/]+)/(?P<id>\d+)/url',
377 webpage, 'video link', group=('type', 'id'))
3dbb2a9d 378 return self.url_result(f'https://www.rctiplus.com/{video_type}/{video_id}/{tv_id}', 'RCTIPlus')
fork of https://github.com/yt-dlp/yt-dlp/