yt_dlp/extractor/triller.py

   1 import itertools
   2 import json
   3
   4 from .common import InfoExtractor
   5 from ..utils import (
   6     ExtractorError,
   7     int_or_none,
   8     str_or_none,
   9     traverse_obj,
  10     unified_strdate,
  11     unified_timestamp,
  12     url_basename,
  13 )
  14
  15
  16 class TrillerBaseIE(InfoExtractor):
  17     _NETRC_MACHINE = 'triller'
  18     _API_BASE_URL = 'https://social.triller.co/v1.5'
  19     _API_HEADERS = {'Origin': 'https://triller.co'}
  20
  21     def _perform_login(self, username, password):
  22         if self._API_HEADERS.get('Authorization'):
  23             return
  24
  25         user_check = self._download_json(
  26             f'{self._API_BASE_URL}/api/user/is-valid-username', None, note='Checking username',
  27             fatal=False, expected_status=400, headers={
  28                 'Content-Type': 'application/json',
  29                 'Origin': 'https://triller.co',
  30             }, data=json.dumps({'username': username}, separators=(',', ':')).encode('utf-8'))
  31         if user_check.get('status'):  # endpoint returns "status":false if username exists
  32             raise ExtractorError('Unable to login: Invalid username', expected=True)
  33
  34         credentials = {
  35             'username': username,
  36             'password': password,
  37         }
  38         login = self._download_json(
  39             f'{self._API_BASE_URL}/user/auth', None, note='Logging in',
  40             fatal=False, expected_status=400, headers={
  41                 'Content-Type': 'application/json',
  42                 'Origin': 'https://triller.co',
  43             }, data=json.dumps(credentials, separators=(',', ':')).encode('utf-8'))
  44         if not login.get('auth_token'):
  45             if login.get('error') == 1008:
  46                 raise ExtractorError('Unable to login: Incorrect password', expected=True)
  47             raise ExtractorError('Unable to login')
  48
  49         self._API_HEADERS['Authorization'] = f'Bearer {login["auth_token"]}'
  50
  51     def _get_comments(self, video_id, limit=15):
  52         comment_info = self._download_json(
  53             f'{self._API_BASE_URL}/api/videos/{video_id}/comments_v2',
  54             video_id, fatal=False, note='Downloading comments API JSON',
  55             headers=self._API_HEADERS, query={'limit': limit}) or {}
  56         if not comment_info.get('comments'):
  57             return
  58         for comment_dict in comment_info['comments']:
  59             yield {
  60                 'author': traverse_obj(comment_dict, ('author', 'username')),
  61                 'author_id': traverse_obj(comment_dict, ('author', 'user_id')),
  62                 'id': comment_dict.get('id'),
  63                 'text': comment_dict.get('body'),
  64                 'timestamp': unified_timestamp(comment_dict.get('timestamp')),
  65             }
  66
  67     def _check_user_info(self, user_info):
  68         if not user_info:
  69             self.report_warning('Unable to extract user info')
  70         elif user_info.get('private') and not user_info.get('followed_by_me'):
  71             raise ExtractorError('This video is private', expected=True)
  72         elif traverse_obj(user_info, 'blocked_by_user', 'blocking_user'):
  73             raise ExtractorError('The author of the video is blocked', expected=True)
  74         return user_info
  75
  76     def _parse_video_info(self, video_info, username, user_info=None):
  77         video_uuid = video_info.get('video_uuid')
  78         video_id = video_info.get('id')
  79
  80         formats = []
  81         video_url = traverse_obj(video_info, 'video_url', 'stream_url')
  82         if video_url:
  83             formats.append({
  84                 'url': video_url,
  85                 'ext': 'mp4',
  86                 'vcodec': 'h264',
  87                 'width': video_info.get('width'),
  88                 'height': video_info.get('height'),
  89                 'format_id': url_basename(video_url).split('.')[0],
  90                 'filesize': video_info.get('filesize'),
  91             })
  92         video_set = video_info.get('video_set') or []
  93         for video in video_set:
  94             resolution = video.get('resolution') or ''
  95             formats.append({
  96                 'url': video['url'],
  97                 'ext': 'mp4',
  98                 'vcodec': video.get('codec'),
  99                 'vbr': int_or_none(video.get('bitrate'), 1000),
 100                 'width': int_or_none(resolution.split('x')[0]),
 101                 'height': int_or_none(resolution.split('x')[1]),
 102                 'format_id': url_basename(video['url']).split('.')[0],
 103             })
 104         audio_url = video_info.get('audio_url')
 105         if audio_url:
 106             formats.append({
 107                 'url': audio_url,
 108                 'ext': 'm4a',
 109                 'format_id': url_basename(audio_url).split('.')[0],
 110             })
 111
 112         manifest_url = video_info.get('transcoded_url')
 113         if manifest_url:
 114             formats.extend(self._extract_m3u8_formats(
 115                 manifest_url, video_id, 'mp4', entry_protocol='m3u8_native',
 116                 m3u8_id='hls', fatal=False))
 117         self._sort_formats(formats)
 118
 119         comment_count = int_or_none(video_info.get('comment_count'))
 120
 121         user_info = user_info or traverse_obj(video_info, 'user', default={})
 122
 123         return {
 124             'id': str_or_none(video_id) or video_uuid,
 125             'title': video_info.get('description') or f'Video by {username}',
 126             'thumbnail': video_info.get('thumbnail_url'),
 127             'description': video_info.get('description'),
 128             'uploader': str_or_none(username),
 129             'uploader_id': str_or_none(user_info.get('user_id')),
 130             'creator': str_or_none(user_info.get('name')),
 131             'timestamp': unified_timestamp(video_info.get('timestamp')),
 132             'upload_date': unified_strdate(video_info.get('timestamp')),
 133             'duration': int_or_none(video_info.get('duration')),
 134             'view_count': int_or_none(video_info.get('play_count')),
 135             'like_count': int_or_none(video_info.get('likes_count')),
 136             'artist': str_or_none(video_info.get('song_artist')),
 137             'track': str_or_none(video_info.get('song_title')),
 138             'webpage_url': f'https://triller.co/@{username}/video/{video_uuid}',
 139             'uploader_url': f'https://triller.co/@{username}',
 140             'extractor_key': TrillerIE.ie_key(),
 141             'extractor': TrillerIE.IE_NAME,
 142             'formats': formats,
 143             'comment_count': comment_count,
 144             '__post_extractor': self.extract_comments(video_id, comment_count),
 145         }
 146
 147
 148 class TrillerIE(TrillerBaseIE):
 149     _VALID_URL = r'''(?x)
 150             https?://(?:www\.)?triller\.co/
 151             @(?P<username>[\w\._]+)/video/
 152             (?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})
 153         '''
 154     _TESTS = [{
 155         'url': 'https://triller.co/@theestallion/video/2358fcd7-3df2-4c77-84c8-1d091610a6cf',
 156         'md5': '228662d783923b60d78395fedddc0a20',
 157         'info_dict': {
 158             'id': '71595734',
 159             'ext': 'mp4',
 160             'title': 'md5:9a2bf9435c5c4292678996a464669416',
 161             'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
 162             'description': 'md5:9a2bf9435c5c4292678996a464669416',
 163             'uploader': 'theestallion',
 164             'uploader_id': '18992236',
 165             'creator': 'Megan Thee Stallion',
 166             'timestamp': 1660598222,
 167             'upload_date': '20220815',
 168             'duration': 47,
 169             'height': 3840,
 170             'width': 2160,
 171             'view_count': int,
 172             'like_count': int,
 173             'artist': 'Megan Thee Stallion',
 174             'track': 'Her',
 175             'webpage_url': 'https://triller.co/@theestallion/video/2358fcd7-3df2-4c77-84c8-1d091610a6cf',
 176             'uploader_url': 'https://triller.co/@theestallion',
 177             'comment_count': int,
 178         }
 179     }, {
 180         'url': 'https://triller.co/@charlidamelio/video/46c6fcfa-aa9e-4503-a50c-68444f44cddc',
 181         'md5': '874055f462af5b0699b9dbb527a505a0',
 182         'info_dict': {
 183             'id': '71621339',
 184             'ext': 'mp4',
 185             'title': 'md5:4c91ea82760fe0fffb71b8c3aa7295fc',
 186             'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
 187             'description': 'md5:4c91ea82760fe0fffb71b8c3aa7295fc',
 188             'uploader': 'charlidamelio',
 189             'uploader_id': '1875551',
 190             'creator': 'charli damelio',
 191             'timestamp': 1660773354,
 192             'upload_date': '20220817',
 193             'duration': 16,
 194             'height': 1920,
 195             'width': 1080,
 196             'view_count': int,
 197             'like_count': int,
 198             'artist': 'Dixie',
 199             'track': 'Someone to Blame',
 200             'webpage_url': 'https://triller.co/@charlidamelio/video/46c6fcfa-aa9e-4503-a50c-68444f44cddc',
 201             'uploader_url': 'https://triller.co/@charlidamelio',
 202             'comment_count': int,
 203         }
 204     }]
 205
 206     def _real_extract(self, url):
 207         username, video_uuid = self._match_valid_url(url).group('username', 'id')
 208
 209         video_info = traverse_obj(self._download_json(
 210             f'{self._API_BASE_URL}/api/videos/{video_uuid}',
 211             video_uuid, note='Downloading video info API JSON',
 212             errnote='Unable to download video info API JSON',
 213             headers=self._API_HEADERS), ('videos', 0))
 214         if not video_info:
 215             raise ExtractorError('No video info found in API response')
 216
 217         user_info = self._check_user_info(video_info.get('user') or {})
 218         return self._parse_video_info(video_info, username, user_info)
 219
 220
 221 class TrillerUserIE(TrillerBaseIE):
 222     _VALID_URL = r'https?://(?:www\.)?triller\.co/@(?P<id>[\w\._]+)/?(?:$|[#?])'
 223     _TESTS = [{
 224         # first videos request only returns 2 videos
 225         'url': 'https://triller.co/@theestallion',
 226         'playlist_mincount': 9,
 227         'info_dict': {
 228             'id': '18992236',
 229             'title': 'theestallion',
 230             'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
 231         }
 232     }, {
 233         'url': 'https://triller.co/@charlidamelio',
 234         'playlist_mincount': 25,
 235         'info_dict': {
 236             'id': '1875551',
 237             'title': 'charlidamelio',
 238             'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
 239         }
 240     }]
 241
 242     def _real_initialize(self):
 243         if not self._API_HEADERS.get('Authorization'):
 244             guest = self._download_json(
 245                 f'{self._API_BASE_URL}/user/create_guest',
 246                 None, note='Creating guest session', data=b'', headers=self._API_HEADERS, query={
 247                     'platform': 'Web',
 248                     'app_version': '',
 249                 })
 250             if not guest.get('auth_token'):
 251                 raise ExtractorError('Unable to fetch required auth token for user extraction')
 252
 253             self._API_HEADERS['Authorization'] = f'Bearer {guest["auth_token"]}'
 254
 255     def _extract_video_list(self, username, user_id, limit=6):
 256         query = {
 257             'limit': limit,
 258         }
 259         for page in itertools.count(1):
 260             for retry in self.RetryManager():
 261                 try:
 262                     video_list = self._download_json(
 263                         f'{self._API_BASE_URL}/api/users/{user_id}/videos',
 264                         username, note=f'Downloading user video list page {page}',
 265                         errnote='Unable to download user video list', headers=self._API_HEADERS,
 266                         query=query)
 267                 except ExtractorError as e:
 268                     if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0:
 269                         retry.error = e
 270                         continue
 271                     raise
 272             if not video_list.get('videos'):
 273                 break
 274             yield from video_list['videos']
 275             query['before_time'] = traverse_obj(video_list, ('videos', -1, 'timestamp'))
 276             if not query['before_time']:
 277                 break
 278
 279     def _entries(self, videos, username, user_info):
 280         for video in videos:
 281             yield self._parse_video_info(video, username, user_info)
 282
 283     def _real_extract(self, url):
 284         username = self._match_id(url)
 285         user_info = self._check_user_info(self._download_json(
 286             f'{self._API_BASE_URL}/api/users/by_username/{username}',
 287             username, note='Downloading user info',
 288             errnote='Failed to download user info', headers=self._API_HEADERS).get('user', {}))
 289
 290         user_id = str_or_none(user_info.get('user_id'))
 291         videos = self._extract_video_list(username, user_id)
 292         thumbnail = user_info.get('avatar_url')
 293
 294         return self.playlist_result(
 295             self._entries(videos, username, user_info), user_id, username, thumbnail=thumbnail)