]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/triller.py
[extractor/youtube] Ignore wrong fps of some formats
[yt-dlp.git] / yt_dlp / extractor / triller.py
CommitLineData
92aa6d68 1import itertools
2import json
33b737be 3import re
92aa6d68 4
5from .common import InfoExtractor
6from ..utils import (
d2c8aadf 7 ExtractorError,
33b737be 8 HEADRequest,
9 UnsupportedError,
10 determine_ext,
92aa6d68 11 int_or_none,
33b737be 12 parse_resolution,
92aa6d68 13 str_or_none,
14 traverse_obj,
92aa6d68 15 unified_timestamp,
16 url_basename,
33b737be 17 urljoin,
18 url_or_none,
92aa6d68 19)
20
21
22class TrillerBaseIE(InfoExtractor):
23 _NETRC_MACHINE = 'triller'
92aa6d68 24 _API_BASE_URL = 'https://social.triller.co/v1.5'
d6f88719 25 _API_HEADERS = {'Origin': 'https://triller.co'}
92aa6d68 26
27 def _perform_login(self, username, password):
d6f88719 28 if self._API_HEADERS.get('Authorization'):
92aa6d68 29 return
30
33b737be 31 headers = {**self._API_HEADERS, 'Content-Type': 'application/json'}
32 user_check = traverse_obj(self._download_json(
92aa6d68 33 f'{self._API_BASE_URL}/api/user/is-valid-username', None, note='Checking username',
33b737be 34 fatal=False, expected_status=400, headers=headers,
35 data=json.dumps({'username': username}, separators=(',', ':')).encode()), 'status')
36
37 if user_check: # endpoint returns `"status":false` if username exists
92aa6d68 38 raise ExtractorError('Unable to login: Invalid username', expected=True)
39
92aa6d68 40 login = self._download_json(
33b737be 41 f'{self._API_BASE_URL}/user/auth', None, note='Logging in', fatal=False,
42 expected_status=400, headers=headers, data=json.dumps({
43 'username': username,
44 'password': password,
45 }, separators=(',', ':')).encode()) or {}
46
92aa6d68 47 if not login.get('auth_token'):
48 if login.get('error') == 1008:
49 raise ExtractorError('Unable to login: Incorrect password', expected=True)
50 raise ExtractorError('Unable to login')
51
d6f88719 52 self._API_HEADERS['Authorization'] = f'Bearer {login["auth_token"]}'
92aa6d68 53
54 def _get_comments(self, video_id, limit=15):
55 comment_info = self._download_json(
56 f'{self._API_BASE_URL}/api/videos/{video_id}/comments_v2',
57 video_id, fatal=False, note='Downloading comments API JSON',
d6f88719 58 headers=self._API_HEADERS, query={'limit': limit}) or {}
92aa6d68 59 if not comment_info.get('comments'):
60 return
33b737be 61 yield from traverse_obj(comment_info, ('comments', ..., {
62 'id': ('id', {str_or_none}),
63 'text': 'body',
64 'author': ('author', 'username'),
65 'author_id': ('author', 'user_id'),
66 'timestamp': ('timestamp', {unified_timestamp}),
67 }))
92aa6d68 68
69 def _check_user_info(self, user_info):
33b737be 70 if user_info.get('private') and not user_info.get('followed_by_me'):
92aa6d68 71 raise ExtractorError('This video is private', expected=True)
72 elif traverse_obj(user_info, 'blocked_by_user', 'blocking_user'):
73 raise ExtractorError('The author of the video is blocked', expected=True)
74 return user_info
75
33b737be 76 def _parse_video_info(self, video_info, username, user_id, display_id=None):
77 video_id = str(video_info['id'])
78 display_id = display_id or video_info.get('video_uuid')
79
80 if traverse_obj(video_info, (
81 None, ('transcoded_url', 'video_url', 'stream_url', 'audio_url'),
82 {lambda x: re.search(r'/copyright/', x)}), get_all=False):
83 self.raise_no_formats('This video has been removed due to licensing restrictions', expected=True)
84
85 def format_info(url):
86 return {
87 'url': url,
88 'ext': determine_ext(url),
89 'format_id': url_basename(url).split('.')[0],
90 }
92aa6d68 91
92 formats = []
33b737be 93
94 if determine_ext(video_info.get('transcoded_url')) == 'm3u8':
95 formats.extend(self._extract_m3u8_formats(
96 video_info['transcoded_url'], video_id, 'mp4', m3u8_id='hls', fatal=False))
97
98 for video in traverse_obj(video_info, ('video_set', lambda _, v: url_or_none(v['url']))):
92aa6d68 99 formats.append({
33b737be 100 **format_info(video['url']),
101 **parse_resolution(video.get('resolution')),
92aa6d68 102 'vcodec': video.get('codec'),
103 'vbr': int_or_none(video.get('bitrate'), 1000),
92aa6d68 104 })
33b737be 105
106 video_url = traverse_obj(video_info, 'video_url', 'stream_url', expected_type=url_or_none)
107 if video_url:
92aa6d68 108 formats.append({
33b737be 109 **format_info(video_url),
110 'vcodec': 'h264',
111 **traverse_obj(video_info, {
112 'width': 'width',
113 'height': 'height',
114 'filesize': 'filesize',
115 }, expected_type=int_or_none),
92aa6d68 116 })
117
33b737be 118 audio_url = url_or_none(video_info.get('audio_url'))
119 if audio_url:
120 formats.append(format_info(audio_url))
92aa6d68 121
33b737be 122 comment_count = traverse_obj(video_info, ('comment_count', {int_or_none}))
92aa6d68 123
124 return {
33b737be 125 'id': video_id,
126 'display_id': display_id,
127 'uploader': username,
128 'uploader_id': user_id or traverse_obj(video_info, ('user', 'user_id', {str_or_none})),
129 'webpage_url': urljoin(f'https://triller.co/@{username}/video/', display_id),
92aa6d68 130 'uploader_url': f'https://triller.co/@{username}',
131 'extractor_key': TrillerIE.ie_key(),
132 'extractor': TrillerIE.IE_NAME,
133 'formats': formats,
134 'comment_count': comment_count,
135 '__post_extractor': self.extract_comments(video_id, comment_count),
33b737be 136 **traverse_obj(video_info, {
137 'title': ('description', {lambda x: x.replace('\r\n', ' ')}),
138 'description': 'description',
139 'creator': ((('user'), ('users', lambda _, v: str(v['user_id']) == user_id)), 'name'),
140 'thumbnail': ('thumbnail_url', {url_or_none}),
141 'timestamp': ('timestamp', {unified_timestamp}),
142 'duration': ('duration', {int_or_none}),
143 'view_count': ('play_count', {int_or_none}),
144 'like_count': ('likes_count', {int_or_none}),
145 'artist': 'song_artist',
146 'track': 'song_title',
147 }, get_all=False),
92aa6d68 148 }
149
150
151class TrillerIE(TrillerBaseIE):
152 _VALID_URL = r'''(?x)
153 https?://(?:www\.)?triller\.co/
33b737be 154 @(?P<username>[\w.]+)/video/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})
92aa6d68 155 '''
156 _TESTS = [{
157 'url': 'https://triller.co/@theestallion/video/2358fcd7-3df2-4c77-84c8-1d091610a6cf',
158 'md5': '228662d783923b60d78395fedddc0a20',
159 'info_dict': {
160 'id': '71595734',
161 'ext': 'mp4',
162 'title': 'md5:9a2bf9435c5c4292678996a464669416',
163 'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
164 'description': 'md5:9a2bf9435c5c4292678996a464669416',
165 'uploader': 'theestallion',
166 'uploader_id': '18992236',
167 'creator': 'Megan Thee Stallion',
168 'timestamp': 1660598222,
169 'upload_date': '20220815',
170 'duration': 47,
92aa6d68 171 'view_count': int,
172 'like_count': int,
173 'artist': 'Megan Thee Stallion',
174 'track': 'Her',
92aa6d68 175 'uploader_url': 'https://triller.co/@theestallion',
176 'comment_count': int,
33b737be 177 },
178 'skip': 'This video has been removed due to licensing restrictions',
92aa6d68 179 }, {
180 'url': 'https://triller.co/@charlidamelio/video/46c6fcfa-aa9e-4503-a50c-68444f44cddc',
181 'md5': '874055f462af5b0699b9dbb527a505a0',
182 'info_dict': {
183 'id': '71621339',
184 'ext': 'mp4',
185 'title': 'md5:4c91ea82760fe0fffb71b8c3aa7295fc',
33b737be 186 'display_id': '46c6fcfa-aa9e-4503-a50c-68444f44cddc',
92aa6d68 187 'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
188 'description': 'md5:4c91ea82760fe0fffb71b8c3aa7295fc',
189 'uploader': 'charlidamelio',
190 'uploader_id': '1875551',
191 'creator': 'charli damelio',
192 'timestamp': 1660773354,
193 'upload_date': '20220817',
194 'duration': 16,
92aa6d68 195 'view_count': int,
196 'like_count': int,
197 'artist': 'Dixie',
198 'track': 'Someone to Blame',
92aa6d68 199 'uploader_url': 'https://triller.co/@charlidamelio',
200 'comment_count': int,
33b737be 201 },
202 }, {
203 'url': 'https://triller.co/@theestallion/video/07f35f38-1f51-48e2-8c5f-f7a8e829988f',
204 'md5': 'af7b3553e4b8bfca507636471ee2eb41',
205 'info_dict': {
206 'id': '71837829',
207 'ext': 'mp4',
208 'title': 'UNGRATEFUL VIDEO OUT NOW ๐Ÿ‘๐Ÿพ๐Ÿ‘๐Ÿพ๐Ÿ‘๐Ÿพ ๐Ÿ’™๐Ÿ’™ link my bio #womeninhiphop',
209 'display_id': '07f35f38-1f51-48e2-8c5f-f7a8e829988f',
210 'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
211 'description': 'UNGRATEFUL VIDEO OUT NOW ๐Ÿ‘๐Ÿพ๐Ÿ‘๐Ÿพ๐Ÿ‘๐Ÿพ ๐Ÿ’™๐Ÿ’™ link my bio\r\n #womeninhiphop',
212 'uploader': 'theestallion',
213 'uploader_id': '18992236',
214 'creator': 'Megan Thee Stallion',
215 'timestamp': 1662486178,
216 'upload_date': '20220906',
217 'duration': 30,
218 'view_count': int,
219 'like_count': int,
220 'artist': 'Unknown',
221 'track': 'Unknown',
222 'uploader_url': 'https://triller.co/@theestallion',
223 'comment_count': int,
224 },
92aa6d68 225 }]
226
227 def _real_extract(self, url):
33b737be 228 username, display_id = self._match_valid_url(url).group('username', 'id')
229
230 video_info = self._download_json(
231 f'{self._API_BASE_URL}/api/videos/{display_id}', display_id,
232 headers=self._API_HEADERS)['videos'][0]
92aa6d68 233
33b737be 234 self._check_user_info(video_info.get('user') or {})
92aa6d68 235
33b737be 236 return self._parse_video_info(video_info, username, None, display_id)
92aa6d68 237
238
239class TrillerUserIE(TrillerBaseIE):
33b737be 240 _VALID_URL = r'https?://(?:www\.)?triller\.co/@(?P<id>[\w.]+)/?(?:$|[#?])'
92aa6d68 241 _TESTS = [{
92aa6d68 242 'url': 'https://triller.co/@theestallion',
33b737be 243 'playlist_mincount': 12,
92aa6d68 244 'info_dict': {
245 'id': '18992236',
246 'title': 'theestallion',
247 'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
33b737be 248 },
92aa6d68 249 }, {
250 'url': 'https://triller.co/@charlidamelio',
33b737be 251 'playlist_mincount': 150,
92aa6d68 252 'info_dict': {
253 'id': '1875551',
254 'title': 'charlidamelio',
255 'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
33b737be 256 },
92aa6d68 257 }]
258
259 def _real_initialize(self):
d6f88719 260 if not self._API_HEADERS.get('Authorization'):
92aa6d68 261 guest = self._download_json(
33b737be 262 f'{self._API_BASE_URL}/user/create_guest', None,
263 note='Creating guest session', data=b'', headers=self._API_HEADERS, query={
92aa6d68 264 'platform': 'Web',
265 'app_version': '',
266 })
267 if not guest.get('auth_token'):
268 raise ExtractorError('Unable to fetch required auth token for user extraction')
269
d6f88719 270 self._API_HEADERS['Authorization'] = f'Bearer {guest["auth_token"]}'
92aa6d68 271
33b737be 272 def _entries(self, username, user_id, limit=6):
273 query = {'limit': limit}
92aa6d68 274 for page in itertools.count(1):
33b737be 275 videos = self._download_json(
276 f'{self._API_BASE_URL}/api/users/{user_id}/videos',
277 username, note=f'Downloading user video list page {page}',
278 headers=self._API_HEADERS, query=query)
279
280 for video in traverse_obj(videos, ('videos', ...)):
281 yield self._parse_video_info(video, username, user_id)
282
283 query['before_time'] = traverse_obj(videos, ('videos', -1, 'timestamp'))
92aa6d68 284 if not query['before_time']:
285 break
286
92aa6d68 287 def _real_extract(self, url):
288 username = self._match_id(url)
33b737be 289
92aa6d68 290 user_info = self._check_user_info(self._download_json(
291 f'{self._API_BASE_URL}/api/users/by_username/{username}',
33b737be 292 username, note='Downloading user info', headers=self._API_HEADERS)['user'])
92aa6d68 293
294 user_id = str_or_none(user_info.get('user_id'))
33b737be 295 if not user_id:
296 raise ExtractorError('Unable to extract user ID')
92aa6d68 297
298 return self.playlist_result(
33b737be 299 self._entries(username, user_id), user_id, username, thumbnail=user_info.get('avatar_url'))
300
301
302class TrillerShortIE(InfoExtractor):
303 _VALID_URL = r'https?://v\.triller\.co/(?P<id>\w+)'
304 _TESTS = [{
305 'url': 'https://v.triller.co/WWZNWk',
306 'md5': '5eb8dc2c971bd8cd794ec9e8d5e9d101',
307 'info_dict': {
308 'id': '66210052',
309 'ext': 'mp4',
310 'title': 'md5:2dfc89d154cd91a4a18cd9582ba03e16',
311 'display_id': 'f4480e1f-fb4e-45b9-a44c-9e6c679ce7eb',
312 'thumbnail': r're:^https://uploads\.cdn\.triller\.co/.+\.jpg$',
313 'description': 'md5:2dfc89d154cd91a4a18cd9582ba03e16',
314 'uploader': 'statefairent',
315 'uploader_id': '487545193',
316 'creator': 'Officialย Summerย Fairย ofย LA',
317 'timestamp': 1629655457,
318 'upload_date': '20210822',
319 'duration': 19,
320 'view_count': int,
321 'like_count': int,
322 'artist': 'Unknown',
323 'track': 'Unknown',
324 'uploader_url': 'https://triller.co/@statefairent',
325 'comment_count': int,
326 },
327 }]
328
329 def _real_extract(self, url):
330 real_url = self._request_webpage(HEADRequest(url), self._match_id(url)).geturl()
331 if self.suitable(real_url): # Prevent infinite loop in case redirect fails
332 raise UnsupportedError(real_url)
333 return self.url_result(real_url)