yt_dlp/extractor/rcti.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import itertools
   5 import json
   6 import random
   7 import time
   8
   9 from .common import InfoExtractor
  10 from ..compat import compat_HTTPError
  11 from ..utils import (
  12     dict_get,
  13     ExtractorError,
  14     strip_or_none,
  15     try_get
  16 )
  17
  18
  19 class RCTIPlusBaseIE(InfoExtractor):
  20     def _real_initialize(self):
  21         self._AUTH_KEY = self._download_json(
  22             'https://api.rctiplus.com/api/v1/visitor?platform=web',  # platform can be web, mweb, android, ios
  23             None, 'Fetching authorization key')['data']['access_token']
  24
  25     def _call_api(self, url, video_id, note=None):
  26         json = self._download_json(
  27             url, video_id, note=note, headers={'Authorization': self._AUTH_KEY})
  28         if json.get('status', {}).get('code', 0) != 0:
  29             raise ExtractorError('%s said: %s' % (self.IE_NAME, json["status"]["message_client"]), cause=json)
  30         return json.get('data'), json.get('meta')
  31
  32
  33 class RCTIPlusIE(RCTIPlusBaseIE):
  34     _VALID_URL = r'https://www\.rctiplus\.com/(?:programs/\d+?/.*?/)?(?P<type>episode|clip|extra|live-event|missed-event)/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
  35     _TESTS = [{
  36         'url': 'https://www.rctiplus.com/programs/1259/kiko-untuk-lola/episode/22124/untuk-lola',
  37         'md5': '56ed45affad45fa18d5592a1bc199997',
  38         'info_dict': {
  39             'id': 'v_e22124',
  40             'title': 'Untuk Lola',
  41             'display_id': 'untuk-lola',
  42             'description': 'md5:2b809075c0b1e071e228ad6d13e41deb',
  43             'ext': 'mp4',
  44             'duration': 1400,
  45             'timestamp': 1615978800,
  46             'upload_date': '20210317',
  47             'series': 'Kiko : Untuk Lola',
  48             'season_number': 1,
  49             'episode_number': 1,
  50             'channel': 'RCTI',
  51         },
  52         'params': {
  53             'fixup': 'never',
  54         },
  55     }, {  # Clip; Series title doesn't appear on metadata JSON
  56         'url': 'https://www.rctiplus.com/programs/316/cahaya-terindah/clip/3921/make-a-wish',
  57         'md5': 'd179b2ff356f0e91a53bcc6a4d8504f0',
  58         'info_dict': {
  59             'id': 'v_c3921',
  60             'title': 'Make A Wish',
  61             'display_id': 'make-a-wish',
  62             'description': 'Make A Wish',
  63             'ext': 'mp4',
  64             'duration': 288,
  65             'timestamp': 1571652600,
  66             'upload_date': '20191021',
  67             'series': 'Cahaya Terindah',
  68             'channel': 'RCTI',
  69         },
  70         'params': {
  71             'fixup': 'never',
  72         },
  73     }, {  # Extra
  74         'url': 'https://www.rctiplus.com/programs/616/inews-malam/extra/9438/diungkapkan-melalui-surat-terbuka-ceo-ruangguru-belva-devara-mundur-dari-staf-khusus-presiden',
  75         'md5': 'c48106afdbce609749f5e0c007d9278a',
  76         'info_dict': {
  77             'id': 'v_ex9438',
  78             'title': 'md5:2ede828c0f8bde249e0912be150314ca',
  79             'display_id': 'md5:62b8d4e9ff096db527a1ad797e8a9933',
  80             'description': 'md5:2ede828c0f8bde249e0912be150314ca',
  81             'ext': 'mp4',
  82             'duration': 93,
  83             'timestamp': 1587561540,
  84             'upload_date': '20200422',
  85             'series': 'iNews Malam',
  86             'channel': 'INews',
  87         },
  88         'params': {
  89             'format': 'bestvideo',
  90         },
  91     }, {  # Missed event/replay
  92         'url': 'https://www.rctiplus.com/missed-event/2507/mou-signing-ceremony-27-juli-2021-1400-wib',
  93         'md5': '649c5f27250faed1452ca8b91e06922d',
  94         'info_dict': {
  95             'id': 'v_pe2507',
  96             'title': 'MOU Signing Ceremony | 27 Juli 2021 | 14.00 WIB',
  97             'display_id': 'mou-signing-ceremony-27-juli-2021-1400-wib',
  98             'ext': 'mp4',
  99             'timestamp': 1627142400,
 100             'upload_date': '20210724',
 101             'was_live': True,
 102             'release_timestamp': 1627369200,
 103         },
 104         'params': {
 105             'fixup': 'never',
 106         },
 107     }, {  # Live event; Cloudfront CDN
 108         'url': 'https://www.rctiplus.com/live-event/2530/dai-muda-charging-imun-dengan-iman-4-agustus-2021-1600-wib',
 109         'info_dict': {
 110             'id': 'v_le2530',
 111             'title': 'Dai Muda : Charging Imun dengan Iman | 4 Agustus 2021 | 16.00 WIB',
 112             'display_id': 'dai-muda-charging-imun-dengan-iman-4-agustus-2021-1600-wib',
 113             'ext': 'mp4',
 114             'timestamp': 1627898400,
 115             'upload_date': '20210802',
 116             'release_timestamp': 1628067600,
 117         },
 118         'params': {
 119             'skip_download': True,
 120         },
 121         'skip': 'This live event has ended.',
 122     }, {  # TV; live_at is null
 123         'url': 'https://www.rctiplus.com/live-event/1/rcti',
 124         'info_dict': {
 125             'id': 'v_lt1',
 126             'title': 'RCTI',
 127             'display_id': 'rcti',
 128             'ext': 'mp4',
 129             'timestamp': 1546344000,
 130             'upload_date': '20190101',
 131             'is_live': True,
 132         },
 133         'params': {
 134             'skip_download': True,
 135             'format': 'bestvideo',
 136         },
 137     }]
 138     _CONVIVA_JSON_TEMPLATE = {
 139         't': 'CwsSessionHb',
 140         'cid': 'ff84ae928c3b33064b76dec08f12500465e59a6f',
 141         'clid': '0',
 142         'sid': 0,
 143         'seq': 0,
 144         'caps': 0,
 145         'sf': 7,
 146         'sdk': True,
 147     }
 148
 149     def _real_extract(self, url):
 150         match = self._match_valid_url(url).groupdict()
 151         video_type, video_id, display_id = match['type'], match['id'], match['display_id']
 152
 153         url_api_version = 'v2' if video_type == 'missed-event' else 'v1'
 154         appier_id = '23984824_' + str(random.randint(0, 10000000000))  # Based on the webpage's uuidRandom generator
 155         video_json = self._call_api(
 156             f'https://api.rctiplus.com/api/{url_api_version}/{video_type}/{video_id}/url?appierid={appier_id}', display_id, 'Downloading video URL JSON')[0]
 157         video_url = video_json['url']
 158
 159         is_upcoming = try_get(video_json, lambda x: x['current_date'] < x['live_at'])
 160         if is_upcoming is None:
 161             is_upcoming = try_get(video_json, lambda x: x['current_date'] < x['start_date'])
 162         if is_upcoming:
 163             self.raise_no_formats(
 164                 'This event will start at %s.' % video_json['live_label'] if video_json.get('live_label') else 'This event has not started yet.', expected=True)
 165         if 'akamaized' in video_url:
 166             # For some videos hosted on Akamai's CDN (possibly AES-encrypted ones?), a session needs to at least be made via Conviva's API
 167             conviva_json_data = {
 168                 **self._CONVIVA_JSON_TEMPLATE,
 169                 'url': video_url,
 170                 'sst': int(time.time())
 171             }
 172             conviva_json_res = self._download_json(
 173                 'https://ff84ae928c3b33064b76dec08f12500465e59a6f.cws.conviva.com/0/wsg', display_id,
 174                 'Creating Conviva session', 'Failed to create Conviva session',
 175                 fatal=False, data=json.dumps(conviva_json_data).encode('utf-8'))
 176             if conviva_json_res and conviva_json_res.get('err') != 'ok':
 177                 self.report_warning('Conviva said: %s' % str(conviva_json_res.get('err')))
 178
 179         video_meta, meta_paths = self._call_api(
 180             'https://api.rctiplus.com/api/v1/%s/%s' % (video_type, video_id), display_id, 'Downloading video metadata')
 181
 182         thumbnails, image_path = [], meta_paths.get('image_path', 'https://rstatic.akamaized.net/media/')
 183         if video_meta.get('portrait_image'):
 184             thumbnails.append({
 185                 'id': 'portrait_image',
 186                 'url': '%s%d%s' % (image_path, 2000, video_meta['portrait_image'])  # 2000px seems to be the highest resolution that can be given
 187             })
 188         if video_meta.get('landscape_image'):
 189             thumbnails.append({
 190                 'id': 'landscape_image',
 191                 'url': '%s%d%s' % (image_path, 2000, video_meta['landscape_image'])
 192             })
 193         try:
 194             formats = self._extract_m3u8_formats(video_url, display_id, 'mp4', headers={'Referer': 'https://www.rctiplus.com/'})
 195         except ExtractorError as e:
 196             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
 197                 self.raise_geo_restricted(countries=['ID'], metadata_available=True)
 198             else:
 199                 raise e
 200         for f in formats:
 201             if 'akamaized' in f['url'] or 'cloudfront' in f['url']:
 202                 f.setdefault('http_headers', {})['Referer'] = 'https://www.rctiplus.com/'  # Referer header is required for akamai/cloudfront CDNs
 203
 204         self._sort_formats(formats)
 205
 206         return {
 207             'id': video_meta.get('product_id') or video_json.get('product_id'),
 208             'title': dict_get(video_meta, ('title', 'name')) or dict_get(video_json, ('content_name', 'assets_name')),
 209             'display_id': display_id,
 210             'description': video_meta.get('summary'),
 211             'timestamp': video_meta.get('release_date') or video_json.get('start_date'),
 212             'duration': video_meta.get('duration'),
 213             'categories': [video_meta['genre']] if video_meta.get('genre') else None,
 214             'average_rating': video_meta.get('star_rating'),
 215             'series': video_meta.get('program_title') or video_json.get('program_title'),
 216             'season_number': video_meta.get('season'),
 217             'episode_number': video_meta.get('episode'),
 218             'channel': video_json.get('tv_name'),
 219             'channel_id': video_json.get('tv_id'),
 220             'formats': formats,
 221             'thumbnails': thumbnails,
 222             'is_live': video_type == 'live-event' and not is_upcoming,
 223             'was_live': video_type == 'missed-event',
 224             'live_status': 'is_upcoming' if is_upcoming else None,
 225             'release_timestamp': video_json.get('live_at'),
 226         }
 227
 228
 229 class RCTIPlusSeriesIE(RCTIPlusBaseIE):
 230     _VALID_URL = r'https://www\.rctiplus\.com/programs/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
 231     _TESTS = [{
 232         'url': 'https://www.rctiplus.com/programs/540/upin-ipin',
 233         'playlist_mincount': 417,
 234         'info_dict': {
 235             'id': '540',
 236             'title': 'Upin & Ipin',
 237             'description': 'md5:22cc912381f389664416844e1ec4f86b',
 238         },
 239     }, {
 240         'url': 'https://www.rctiplus.com/programs/540/upin-ipin/episodes?utm_source=Rplusdweb&utm_medium=share_copy&utm_campaign=programsupin-ipin',
 241         'only_matching': True,
 242     }]
 243     _AGE_RATINGS = {  # Based off https://id.wikipedia.org/wiki/Sistem_rating_konten_televisi with additional ratings
 244         'S-SU': 2,
 245         'SU': 2,
 246         'P': 2,
 247         'A': 7,
 248         'R': 13,
 249         'R-R/1': 17,  # Labelled as 17+ despite being R
 250         'D': 18,
 251     }
 252
 253     @classmethod
 254     def suitable(cls, url):
 255         return False if RCTIPlusIE.suitable(url) else super(RCTIPlusSeriesIE, cls).suitable(url)
 256
 257     def _entries(self, url, display_id=None, note='Downloading entries JSON', metadata={}):
 258         total_pages = 0
 259         try:
 260             total_pages = self._call_api(
 261                 '%s&length=20&page=0' % url,
 262                 display_id, note)[1]['pagination']['total_page']
 263         except ExtractorError as e:
 264             if 'not found' in str(e):
 265                 return []
 266             raise e
 267         if total_pages <= 0:
 268             return []
 269
 270         for page_num in range(1, total_pages + 1):
 271             episode_list = self._call_api(
 272                 '%s&length=20&page=%s' % (url, page_num),
 273                 display_id, '%s page %s' % (note, page_num))[0] or []
 274
 275             for video_json in episode_list:
 276                 link = video_json['share_link']
 277                 url_res = self.url_result(link, 'RCTIPlus', video_json.get('product_id'), video_json.get('title'))
 278                 url_res.update(metadata)
 279                 yield url_res
 280
 281     def _real_extract(self, url):
 282         series_id, display_id = self._match_valid_url(url).groups()
 283
 284         series_meta, meta_paths = self._call_api(
 285             'https://api.rctiplus.com/api/v1/program/%s/detail' % series_id, display_id, 'Downloading series metadata')
 286         metadata = {
 287             'age_limit': try_get(series_meta, lambda x: self._AGE_RATINGS[x['age_restriction'][0]['code']])
 288         }
 289
 290         cast = []
 291         for star in series_meta.get('starring', []):
 292             cast.append(strip_or_none(star.get('name')))
 293         for star in series_meta.get('creator', []):
 294             cast.append(strip_or_none(star.get('name')))
 295         for star in series_meta.get('writer', []):
 296             cast.append(strip_or_none(star.get('name')))
 297         metadata['cast'] = cast
 298
 299         tags = []
 300         for tag in series_meta.get('tag', []):
 301             tags.append(strip_or_none(tag.get('name')))
 302         metadata['tag'] = tags
 303
 304         entries = []
 305         seasons_list = self._call_api(
 306             'https://api.rctiplus.com/api/v1/program/%s/season' % series_id, display_id, 'Downloading seasons list JSON')[0]
 307         for season in seasons_list:
 308             entries.append(self._entries('https://api.rctiplus.com/api/v2/program/%s/episode?season=%s' % (series_id, season['season']),
 309                                          display_id, 'Downloading season %s episode entries' % season['season'], metadata))
 310
 311         entries.append(self._entries('https://api.rctiplus.com/api/v2/program/%s/clip?content_id=0' % series_id,
 312                                      display_id, 'Downloading clip entries', metadata))
 313         entries.append(self._entries('https://api.rctiplus.com/api/v2/program/%s/extra?content_id=0' % series_id,
 314                                      display_id, 'Downloading extra entries', metadata))
 315
 316         return self.playlist_result(itertools.chain(*entries), series_id, series_meta.get('title'), series_meta.get('summary'), **metadata)
 317
 318
 319 class RCTIPlusTVIE(RCTIPlusBaseIE):
 320     _VALID_URL = r'https://www\.rctiplus\.com/((tv/(?P<tvname>\w+))|(?P<eventname>live-event|missed-event))'
 321     _TESTS = [{
 322         'url': 'https://www.rctiplus.com/tv/rcti',
 323         'info_dict': {
 324             'id': 'v_lt1',
 325             'title': 'RCTI',
 326             'ext': 'mp4',
 327             'timestamp': 1546344000,
 328             'upload_date': '20190101',
 329         },
 330         'params': {
 331             'skip_download': True,
 332             'format': 'bestvideo',
 333         }
 334     }, {
 335         # Returned video will always change
 336         'url': 'https://www.rctiplus.com/live-event',
 337         'only_matching': True,
 338     }, {
 339         # Returned video will also always change
 340         'url': 'https://www.rctiplus.com/missed-event',
 341         'only_matching': True,
 342     }]
 343
 344     @classmethod
 345     def suitable(cls, url):
 346         return False if RCTIPlusIE.suitable(url) else super(RCTIPlusTVIE, cls).suitable(url)
 347
 348     def _real_extract(self, url):
 349         match = self._match_valid_url(url).groupdict()
 350         tv_id = match.get('tvname') or match.get('eventname')
 351         webpage = self._download_webpage(url, tv_id)
 352         video_type, video_id = self._search_regex(
 353             r'url\s*:\s*["\']https://api\.rctiplus\.com/api/v./(?P<type>[^/]+)/(?P<id>\d+)/url', webpage, 'video link', group=('type', 'id'))
 354         return self.url_result(f'https://www.rctiplus.com/{video_type}/{video_id}/{tv_id}', 'RCTIPlus')