yt_dlp/extractor/hotstar.py

   1 import hashlib
   2 import hmac
   3 import re
   4 import time
   5 import uuid
   6 import json
   7
   8 from .common import InfoExtractor
   9 from ..compat import (
  10     compat_HTTPError,
  11     compat_str
  12 )
  13 from ..utils import (
  14     determine_ext,
  15     ExtractorError,
  16     int_or_none,
  17     join_nonempty,
  18     str_or_none,
  19     try_get,
  20     url_or_none,
  21 )
  22
  23
  24 class HotStarBaseIE(InfoExtractor):
  25     _BASE_URL = 'https://www.hotstar.com'
  26     _API_URL = 'https://api.hotstar.com'
  27     _AKAMAI_ENCRYPTION_KEY = b'\x05\xfc\x1a\x01\xca\xc9\x4b\xc4\x12\xfc\x53\x12\x07\x75\xf9\xee'
  28
  29     def _call_api_impl(self, path, video_id, query, st=None, cookies=None):
  30         st = int_or_none(st) or int(time.time())
  31         exp = st + 6000
  32         auth = 'st=%d~exp=%d~acl=/*' % (st, exp)
  33         auth += '~hmac=' + hmac.new(self._AKAMAI_ENCRYPTION_KEY, auth.encode(), hashlib.sha256).hexdigest()
  34
  35         if cookies and cookies.get('userUP'):
  36             token = cookies.get('userUP').value
  37         else:
  38             token = self._download_json(
  39                 f'{self._API_URL}/um/v3/users',
  40                 video_id, note='Downloading token',
  41                 data=json.dumps({"device_ids": [{"id": compat_str(uuid.uuid4()), "type": "device_id"}]}).encode('utf-8'),
  42                 headers={
  43                     'hotstarauth': auth,
  44                     'x-hs-platform': 'PCTV',  # or 'web'
  45                     'Content-Type': 'application/json',
  46                 })['user_identity']
  47
  48         response = self._download_json(
  49             f'{self._API_URL}/{path}', video_id, query=query,
  50             headers={
  51                 'hotstarauth': auth,
  52                 'x-hs-appversion': '6.72.2',
  53                 'x-hs-platform': 'web',
  54                 'x-hs-usertoken': token,
  55             })
  56
  57         if response['message'] != "Playback URL's fetched successfully":
  58             raise ExtractorError(
  59                 response['message'], expected=True)
  60         return response['data']
  61
  62     def _call_api(self, path, video_id, query_name='contentId'):
  63         return self._download_json(
  64             f'{self._API_URL}/{path}', video_id=video_id,
  65             query={
  66                 query_name: video_id,
  67                 'tas': 10000,
  68             }, headers={
  69                 'x-country-code': 'IN',
  70                 'x-platform-code': 'PCTV',
  71             })
  72
  73     def _call_api_v2(self, path, video_id, st=None, cookies=None):
  74         return self._call_api_impl(
  75             f'{path}/content/{video_id}', video_id, st=st, cookies=cookies, query={
  76                 'desired-config': 'audio_channel:stereo|container:fmp4|dynamic_range:hdr|encryption:plain|ladder:tv|package:dash|resolution:fhd|subs-tag:HotstarVIP|video_codec:h265',
  77                 'device-id': cookies.get('device_id').value if cookies.get('device_id') else compat_str(uuid.uuid4()),
  78                 'os-name': 'Windows',
  79                 'os-version': '10',
  80             })
  81
  82
  83 class HotStarIE(HotStarBaseIE):
  84     IE_NAME = 'hotstar'
  85     _VALID_URL = r'''(?x)
  86         https?://(?:www\.)?hotstar\.com(?:/in)?/(?!in/)
  87         (?:
  88             (?P<type>movies|sports|episode|(?P<tv>tv))/
  89             (?(tv)(?:[^/?#]+/){2}|[^?#]*)
  90         )?
  91         [^/?#]+/
  92         (?P<id>\d{10})
  93     '''
  94
  95     _TESTS = [{
  96         'url': 'https://www.hotstar.com/can-you-not-spread-rumours/1000076273',
  97         'info_dict': {
  98             'id': '1000076273',
  99             'ext': 'mp4',
 100             'title': 'Can You Not Spread Rumours?',
 101             'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
 102             'timestamp': 1447248600,
 103             'upload_date': '20151111',
 104             'duration': 381,
 105             'episode': 'Can You Not Spread Rumours?',
 106         },
 107     }, {
 108         'url': 'https://www.hotstar.com/tv/ek-bhram-sarvagun-sampanna/s-2116/janhvi-targets-suman/1000234847',
 109         'info_dict': {
 110             'id': '1000234847',
 111             'ext': 'mp4',
 112             'title': 'Janhvi Targets Suman',
 113             'description': 'md5:78a85509348910bd1ca31be898c5796b',
 114             'timestamp': 1556670600,
 115             'upload_date': '20190501',
 116             'duration': 1219,
 117             'channel': 'StarPlus',
 118             'channel_id': 3,
 119             'series': 'Ek Bhram - Sarvagun Sampanna',
 120             'season': 'Chapter 1',
 121             'season_number': 1,
 122             'season_id': 6771,
 123             'episode': 'Janhvi Targets Suman',
 124             'episode_number': 8,
 125         }
 126     }, {
 127         'url': 'https://www.hotstar.com/movies/radha-gopalam/1000057157',
 128         'only_matching': True,
 129     }, {
 130         'url': 'https://www.hotstar.com/in/sports/cricket/follow-the-blues-2021/recap-eng-fight-back-on-day-2/1260066104',
 131         'only_matching': True,
 132     }, {
 133         'url': 'https://www.hotstar.com/in/sports/football/most-costly-pl-transfers-ft-grealish/1260065956',
 134         'only_matching': True,
 135     }]
 136     _GEO_BYPASS = False
 137
 138     _TYPE = {
 139         'movies': 'movie',
 140         'sports': 'match',
 141         'episode': 'episode',
 142         'tv': 'episode',
 143         None: 'content',
 144     }
 145
 146     _IGNORE_MAP = {
 147         'res': 'resolution',
 148         'vcodec': 'video_codec',
 149         'dr': 'dynamic_range',
 150     }
 151
 152     @classmethod
 153     def _video_url(cls, video_id, video_type=None, *, slug='ignore_me', root=None):
 154         assert None in (video_type, root)
 155         if not root:
 156             root = join_nonempty(cls._BASE_URL, video_type, delim='/')
 157         return f'{root}/{slug}/{video_id}'
 158
 159     def _real_extract(self, url):
 160         video_id, video_type = self._match_valid_url(url).group('id', 'type')
 161         video_type = self._TYPE.get(video_type, video_type)
 162         cookies = self._get_cookies(url)  # Cookies before any request
 163
 164         video_data = self._call_api(f'o/v1/{video_type}/detail', video_id)['body']['results']['item']
 165         if not self.get_param('allow_unplayable_formats') and video_data.get('drmProtected'):
 166             self.report_drm(video_id)
 167
 168         # See https://github.com/yt-dlp/yt-dlp/issues/396
 169         st = self._download_webpage_handle(f'{self._BASE_URL}/in', video_id)[1].headers.get('x-origin-date')
 170
 171         geo_restricted = False
 172         formats, subs = [], {}
 173         headers = {'Referer': f'{self._BASE_URL}/in'}
 174
 175         # change to v2 in the future
 176         playback_sets = self._call_api_v2('play/v1/playback', video_id, st=st, cookies=cookies)['playBackSets']
 177         for playback_set in playback_sets:
 178             if not isinstance(playback_set, dict):
 179                 continue
 180             tags = str_or_none(playback_set.get('tagsCombination')) or ''
 181             if any(f'{prefix}:{ignore}' in tags
 182                    for key, prefix in self._IGNORE_MAP.items()
 183                    for ignore in self._configuration_arg(key)):
 184                 continue
 185
 186             format_url = url_or_none(playback_set.get('playbackUrl'))
 187             if not format_url:
 188                 continue
 189             format_url = re.sub(r'(?<=//staragvod)(\d)', r'web\1', format_url)
 190             dr = re.search(r'dynamic_range:(?P<dr>[a-z]+)', playback_set.get('tagsCombination')).group('dr')
 191             ext = determine_ext(format_url)
 192
 193             current_formats, current_subs = [], {}
 194             try:
 195                 if 'package:hls' in tags or ext == 'm3u8':
 196                     current_formats, current_subs = self._extract_m3u8_formats_and_subtitles(
 197                         format_url, video_id, 'mp4',
 198                         entry_protocol='m3u8_native',
 199                         m3u8_id=f'{dr}-hls', headers=headers)
 200                 elif 'package:dash' in tags or ext == 'mpd':
 201                     current_formats, current_subs = self._extract_mpd_formats_and_subtitles(
 202                         format_url, video_id, mpd_id=f'{dr}-dash', headers=headers)
 203                 elif ext == 'f4m':
 204                     pass  # XXX: produce broken files
 205                 else:
 206                     current_formats = [{
 207                         'url': format_url,
 208                         'width': int_or_none(playback_set.get('width')),
 209                         'height': int_or_none(playback_set.get('height')),
 210                     }]
 211             except ExtractorError as e:
 212                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
 213                     geo_restricted = True
 214                 continue
 215
 216             if tags and 'encryption:plain' not in tags:
 217                 for f in current_formats:
 218                     f['has_drm'] = True
 219             if tags and 'language' in tags:
 220                 lang = re.search(r'language:(?P<lang>[a-z]+)', tags).group('lang')
 221                 for f in current_formats:
 222                     if not f.get('langauge'):
 223                         f['language'] = lang
 224
 225             formats.extend(current_formats)
 226             subs = self._merge_subtitles(subs, current_subs)
 227
 228         if not formats and geo_restricted:
 229             self.raise_geo_restricted(countries=['IN'], metadata_available=True)
 230         self._sort_formats(formats)
 231         for f in formats:
 232             f.setdefault('http_headers', {}).update(headers)
 233
 234         return {
 235             'id': video_id,
 236             'title': video_data.get('title'),
 237             'description': video_data.get('description'),
 238             'duration': int_or_none(video_data.get('duration')),
 239             'timestamp': int_or_none(video_data.get('broadcastDate') or video_data.get('startDate')),
 240             'formats': formats,
 241             'subtitles': subs,
 242             'channel': video_data.get('channelName'),
 243             'channel_id': video_data.get('channelId'),
 244             'series': video_data.get('showName'),
 245             'season': video_data.get('seasonName'),
 246             'season_number': int_or_none(video_data.get('seasonNo')),
 247             'season_id': video_data.get('seasonId'),
 248             'episode': video_data.get('title'),
 249             'episode_number': int_or_none(video_data.get('episodeNo')),
 250         }
 251
 252
 253 class HotStarPrefixIE(InfoExtractor):
 254     """ The "hotstar:" prefix is no longer in use, but this is kept for backward compatibility """
 255     IE_DESC = False
 256     _VALID_URL = r'hotstar:(?:(?P<type>\w+):)?(?P<id>\d+)$'
 257     _TESTS = [{
 258         'url': 'hotstar:1000076273',
 259         'only_matching': True,
 260     }, {
 261         'url': 'hotstar:movies:1000057157',
 262         'info_dict': {
 263             'id': '1000057157',
 264             'ext': 'mp4',
 265             'title': 'Radha Gopalam',
 266             'description': 'md5:be3bc342cc120bbc95b3b0960e2b0d22',
 267             'timestamp': 1140805800,
 268             'upload_date': '20060224',
 269             'duration': 9182,
 270             'episode': 'Radha Gopalam',
 271         },
 272     }, {
 273         'url': 'hotstar:episode:1000234847',
 274         'only_matching': True,
 275     }, {
 276         # contentData
 277         'url': 'hotstar:sports:1260065956',
 278         'only_matching': True,
 279     }, {
 280         # contentData
 281         'url': 'hotstar:sports:1260066104',
 282         'only_matching': True,
 283     }]
 284
 285     def _real_extract(self, url):
 286         video_id, video_type = self._match_valid_url(url).group('id', 'type')
 287         return self.url_result(HotStarIE._video_url(video_id, video_type), HotStarIE, video_id)
 288
 289
 290 class HotStarPlaylistIE(HotStarBaseIE):
 291     IE_NAME = 'hotstar:playlist'
 292     _VALID_URL = r'https?://(?:www\.)?hotstar\.com/tv/[^/]+/s-\w+/list/[^/]+/t-(?P<id>\w+)'
 293     _TESTS = [{
 294         'url': 'https://www.hotstar.com/tv/savdhaan-india/s-26/list/popular-clips/t-3_2_26',
 295         'info_dict': {
 296             'id': '3_2_26',
 297         },
 298         'playlist_mincount': 20,
 299     }, {
 300         'url': 'https://www.hotstar.com/tv/savdhaan-india/s-26/list/extras/t-2480',
 301         'only_matching': True,
 302     }]
 303
 304     def _real_extract(self, url):
 305         playlist_id = self._match_id(url)
 306
 307         collection = self._call_api('o/v1/tray/find', playlist_id, 'uqId')['body']['results']
 308         entries = [
 309             self.url_result(HotStarIE._video_url(video['contentId']), HotStarIE, video['contentId'])
 310             for video in collection['assets']['items'] if video.get('contentId')]
 311
 312         return self.playlist_result(entries, playlist_id)
 313
 314
 315 class HotStarSeriesIE(HotStarBaseIE):
 316     IE_NAME = 'hotstar:series'
 317     _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com(?:/in)?/tv/[^/]+/(?P<id>\d+))'
 318     _TESTS = [{
 319         'url': 'https://www.hotstar.com/in/tv/radhakrishn/1260000646',
 320         'info_dict': {
 321             'id': '1260000646',
 322         },
 323         'playlist_mincount': 690,
 324     }, {
 325         'url': 'https://www.hotstar.com/tv/dancee-/1260050431',
 326         'info_dict': {
 327             'id': '1260050431',
 328         },
 329         'playlist_mincount': 43,
 330     }, {
 331         'url': 'https://www.hotstar.com/in/tv/mahabharat/435/',
 332         'info_dict': {
 333             'id': '435',
 334         },
 335         'playlist_mincount': 269,
 336     }]
 337
 338     def _real_extract(self, url):
 339         url, series_id = self._match_valid_url(url).groups()
 340         headers = {
 341             'x-country-code': 'IN',
 342             'x-platform-code': 'PCTV',
 343         }
 344         detail_json = self._download_json(
 345             f'{self._API_URL}/o/v1/show/detail?contentId={series_id}', series_id, headers=headers)
 346         id = try_get(detail_json, lambda x: x['body']['results']['item']['id'], int)
 347         item_json = self._download_json(
 348             f'{self._API_URL}/o/v1/tray/g/1/items?etid=0&tao=0&tas=10000&eid={id}', series_id, headers=headers)
 349
 350         return self.playlist_result([
 351             self.url_result(HotStarIE._video_url(video['contentId'], root=url), HotStarIE, video['contentId'])
 352             for video in item_json['body']['results']['items'] if video.get('contentId')
 353         ], series_id)