]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/tiktok.py
projects / yt-dlp.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[version] update
[yt-dlp.git] / yt_dlp / extractor / tiktok.py
CommitLineData
1ead840d
KS		 1# coding: utf-8
2from __future__ import unicode_literals
f7f18f90
A		 3
4import itertools
bd9ff55b
M		 5import random
6import string
7import time
0fd6661e 8import json
1ead840d
KS		 9
10from .common import InfoExtractor
be1f331f
M		 11from ..compat import (
12 compat_urllib_parse_unquote,
13 compat_urllib_parse_urlparse
14)
1ead840d 15from ..utils import (
ce18a19b 16 ExtractorError,
88afe056 17 HEADRequest,
1ead840d 18 int_or_none,
34921b43 19 join_nonempty,
b3187433 20 LazyList,
e0585e65 21 srt_subtitles_timecode,
1ead840d 22 str_or_none,
bd9ff55b
M		 23 traverse_obj,
24 try_get,
943d5ab1 25 url_or_none,
bd9ff55b 26 qualities,
1ead840d
KS		 27)
28
29
0fd6661e 30class TikTokBaseIE(InfoExtractor):
046cab39
M		 31 _APP_VERSIONS = [('20.9.3', '293'), ('20.4.3', '243'), ('20.2.1', '221'), ('20.1.2', '212'), ('20.0.4', '204')]
32 _WORKING_APP_VERSION = None
943d5ab1
M		 33 _APP_NAME = 'trill'
34 _AID = 1180
53dad39e 35 _API_HOSTNAME = 'api-h2.tiktokv.com'
943d5ab1 36 _UPLOADER_URL_FORMAT = 'https://www.tiktok.com/@%s'
53dad39e 37 _WEBPAGE_HOST = 'https://www.tiktok.com/'
be1f331f 38 QUALITIES = ('360p', '540p', '720p', '1080p')
ce18a19b 39
046cab39
M		 40 def _call_api_impl(self, ep, query, manifest_app_version, video_id, fatal=True,
41 note='Downloading API JSON', errnote='Unable to download API page'):
42 self._set_cookie(self._API_HOSTNAME, 'odin_tt', ''.join(random.choice('0123456789abcdef') for _ in range(160)))
43 webpage_cookies = self._get_cookies(self._WEBPAGE_HOST)
44 if webpage_cookies.get('sid_tt'):
45 self._set_cookie(self._API_HOSTNAME, 'sid_tt', webpage_cookies['sid_tt'].value)
46 return self._download_json(
47 'https://%s/aweme/v1/%s/' % (self._API_HOSTNAME, ep), video_id=video_id,
48 fatal=fatal, note=note, errnote=errnote, headers={
49 'User-Agent': f'com.ss.android.ugc.trill/{manifest_app_version} (Linux; U; Android 10; en_US; Pixel 4; Build/QQ3A.200805.001; Cronet/58.0.2991.0)',
50 'Accept': 'application/json',
51 }, query=query)
52
53 def _build_api_query(self, query, app_version, manifest_app_version):
54 return {
0fd6661e 55 **query,
046cab39
M		 56 'version_name': app_version,
57 'version_code': manifest_app_version,
58 'build_number': app_version,
59 'manifest_version_code': manifest_app_version,
60 'update_version_code': manifest_app_version,
0930b11f 61 'openudid': ''.join(random.choice('0123456789abcdef') for _ in range(16)),
62 'uuid': ''.join([random.choice(string.digits) for _ in range(16)]),
bd9ff55b
M		 63 '_rticket': int(time.time() * 1000),
64 'ts': int(time.time()),
65 'device_brand': 'Google',
66 'device_type': 'Pixel 4',
67 'device_platform': 'android',
68 'resolution': '1080*1920',
69 'dpi': 420,
70 'os_version': '10',
71 'os_api': '29',
72 'carrier_region': 'US',
73 'sys_region': 'US',
74 'region': 'US',
943d5ab1 75 'app_name': self._APP_NAME,
bd9ff55b
M		 76 'app_language': 'en',
77 'language': 'en',
78 'timezone_name': 'America/New_York',
79 'timezone_offset': '-14400',
80 'channel': 'googleplay',
81 'ac': 'wifi',
82 'mcc_mnc': '310260',
83 'is_my_cn': 0,
943d5ab1 84 'aid': self._AID,
bd9ff55b
M		 85 'ssmix': 'a',
86 'as': 'a1qwert123',
87 'cp': 'cbfhckdckkde1',
88 }
046cab39
M		 89
90 def _call_api(self, ep, query, video_id, fatal=True,
91 note='Downloading API JSON', errnote='Unable to download API page'):
92 if not self._WORKING_APP_VERSION:
93 app_version = self._configuration_arg('app_version', [''], ie_key=TikTokIE.ie_key())[0]
94 manifest_app_version = self._configuration_arg('manifest_app_version', [''], ie_key=TikTokIE.ie_key())[0]
95 if app_version and manifest_app_version:
96 self._WORKING_APP_VERSION = (app_version, manifest_app_version)
97 self.write_debug('Imported app version combo from extractor arguments')
98 elif app_version or manifest_app_version:
99 self.report_warning('Only one of the two required version params are passed as extractor arguments', only_once=True)
100
101 if self._WORKING_APP_VERSION:
102 app_version, manifest_app_version = self._WORKING_APP_VERSION
103 real_query = self._build_api_query(query, app_version, manifest_app_version)
104 return self._call_api_impl(ep, real_query, manifest_app_version, video_id, fatal, note, errnote)
105
106 for count, (app_version, manifest_app_version) in enumerate(self._APP_VERSIONS, start=1):
107 real_query = self._build_api_query(query, app_version, manifest_app_version)
108 try:
109 res = self._call_api_impl(ep, real_query, manifest_app_version, video_id, fatal, note, errnote)
110 self._WORKING_APP_VERSION = (app_version, manifest_app_version)
111 return res
112 except ExtractorError as e:
113 if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0:
114 if count == len(self._APP_VERSIONS):
115 if fatal:
116 raise e
117 else:
118 self.report_warning(str(e.cause or e.msg))
119 return
120 self.report_warning('%s. Retrying... (attempt %s of %s)' % (str(e.cause or e.msg), count, len(self._APP_VERSIONS)))
121 continue
122 raise e
0fd6661e 123
e0585e65
M		 124 def _get_subtitles(self, aweme_detail, aweme_id):
125 # TODO: Extract text positioning info
126 subtitles = {}
127 captions_info = traverse_obj(
128 aweme_detail, ('interaction_stickers', ..., 'auto_video_caption_info', 'auto_captions', ...), expected_type=dict, default=[])
129 for caption in captions_info:
130 caption_url = traverse_obj(caption, ('url', 'url_list', ...), expected_type=url_or_none, get_all=False)
131 if not caption_url:
132 continue
133 caption_json = self._download_json(
134 caption_url, aweme_id, note='Downloading captions', errnote='Unable to download captions', fatal=False)
135 if not caption_json:
136 continue
137 subtitles.setdefault(caption.get('language', 'en'), []).append({
138 'ext': 'srt',
139 'data': '\n\n'.join(
140 f'{i + 1}\n{srt_subtitles_timecode(line["start_time"] / 1000)} --> {srt_subtitles_timecode(line["end_time"] / 1000)}\n{line["text"]}'
141 for i, line in enumerate(caption_json['utterances']) if line.get('text'))
142 })
143 return subtitles
144
943d5ab1 145 def _parse_aweme_video_app(self, aweme_detail):
0fd6661e 146 aweme_id = aweme_detail['aweme_id']
bd9ff55b
M		 147 video_info = aweme_detail['video']
148
149 def parse_url_key(url_key):
150 format_id, codec, res, bitrate = self._search_regex(
151 r'v[^_]+_(?P<id>(?P<codec>[^_]+)_(?P<res>\d+p)_(?P<bitrate>\d+))', url_key,
152 'url key', default=(None, None, None, None), group=('id', 'codec', 'res', 'bitrate'))
153 if not format_id:
154 return {}, None
155 return {
156 'format_id': format_id,
157 'vcodec': 'h265' if codec == 'bytevc1' else codec,
158 'tbr': int_or_none(bitrate, scale=1000) or None,
159 'quality': qualities(self.QUALITIES)(res),
160 }, res
161
162 known_resolutions = {}
163
164 def extract_addr(addr, add_meta={}):
165 parsed_meta, res = parse_url_key(addr.get('url_key', ''))
166 if res:
167 known_resolutions.setdefault(res, {}).setdefault('height', add_meta.get('height'))
168 known_resolutions[res].setdefault('width', add_meta.get('width'))
169 parsed_meta.update(known_resolutions.get(res, {}))
170 add_meta.setdefault('height', int_or_none(res[:-1]))
171 return [{
172 'url': url,
173 'filesize': int_or_none(addr.get('data_size')),
174 'ext': 'mp4',
175 'acodec': 'aac',
0fd6661e
M		 176 'source_preference': -2 if 'aweme/v1' in url else -1, # Downloads from API might get blocked
177 **add_meta, **parsed_meta,
34921b43 178 'format_note': join_nonempty(
179 add_meta.get('format_note'), '(API)' if 'aweme/v1' in url else None, delim=' ')
bd9ff55b
M		 180 } for url in addr.get('url_list') or []]
181
182 # Hack: Add direct video links first to prioritize them when removing duplicate formats
183 formats = []
184 if video_info.get('play_addr'):
185 formats.extend(extract_addr(video_info['play_addr'], {
186 'format_id': 'play_addr',
187 'format_note': 'Direct video',
188 'vcodec': 'h265' if traverse_obj(
be1f331f 189 video_info, 'is_bytevc1', 'is_h265') else 'h264', # TODO: Check for "direct iOS" videos, like https://www.tiktok.com/@cookierun_dev/video/7039716639834656002
bd9ff55b
M		 190 'width': video_info.get('width'),
191 'height': video_info.get('height'),
192 }))
193 if video_info.get('download_addr'):
194 formats.extend(extract_addr(video_info['download_addr'], {
195 'format_id': 'download_addr',
196 'format_note': 'Download video%s' % (', watermarked' if video_info.get('has_watermark') else ''),
197 'vcodec': 'h264',
198 'width': video_info.get('width'),
199 'height': video_info.get('height'),
0fd6661e 200 'preference': -2 if video_info.get('has_watermark') else -1,
bd9ff55b
M		 201 }))
202 if video_info.get('play_addr_h264'):
203 formats.extend(extract_addr(video_info['play_addr_h264'], {
204 'format_id': 'play_addr_h264',
205 'format_note': 'Direct video',
206 'vcodec': 'h264',
207 }))
208 if video_info.get('play_addr_bytevc1'):
209 formats.extend(extract_addr(video_info['play_addr_bytevc1'], {
210 'format_id': 'play_addr_bytevc1',
211 'format_note': 'Direct video',
212 'vcodec': 'h265',
213 }))
214
215 for bitrate in video_info.get('bit_rate', []):
216 if bitrate.get('play_addr'):
217 formats.extend(extract_addr(bitrate['play_addr'], {
218 'format_id': bitrate.get('gear_name'),
219 'format_note': 'Playback video',
220 'tbr': try_get(bitrate, lambda x: x['bit_rate'] / 1000),
221 'vcodec': 'h265' if traverse_obj(
222 bitrate, 'is_bytevc1', 'is_h265') else 'h264',
943d5ab1 223 'fps': bitrate.get('FPS'),
bd9ff55b
M		 224 }))
225
226 self._remove_duplicate_formats(formats)
6134fbeb
M		 227 auth_cookie = self._get_cookies(self._WEBPAGE_HOST).get('sid_tt')
228 if auth_cookie:
229 for f in formats:
be1f331f 230 self._set_cookie(compat_urllib_parse_urlparse(f['url']).hostname, 'sid_tt', auth_cookie.value)
0fd6661e 231 self._sort_formats(formats, ('quality', 'codec', 'size', 'br'))
bd9ff55b
M		 232
233 thumbnails = []
234 for cover_id in ('cover', 'ai_dynamic_cover', 'animated_cover', 'ai_dynamic_cover_bak',
235 'origin_cover', 'dynamic_cover'):
236 cover = video_info.get(cover_id)
237 if cover:
238 for cover_url in cover['url_list']:
239 thumbnails.append({
240 'id': cover_id,
241 'url': cover_url,
242 })
243
244 stats_info = aweme_detail.get('statistics', {})
245 author_info = aweme_detail.get('author', {})
246 music_info = aweme_detail.get('music', {})
943d5ab1
M		 247 user_url = self._UPLOADER_URL_FORMAT % (traverse_obj(author_info,
248 'sec_uid', 'id', 'uid', 'unique_id',
249 expected_type=str_or_none, get_all=False))
a8549f19 250 labels = traverse_obj(aweme_detail, ('hybrid_label', ..., 'text'), expected_type=str, default=[])
bd9ff55b
M		 251
252 contained_music_track = traverse_obj(
253 music_info, ('matched_song', 'title'), ('matched_pgc_sound', 'title'), expected_type=str)
254 contained_music_author = traverse_obj(
255 music_info, ('matched_song', 'author'), ('matched_pgc_sound', 'author'), 'author', expected_type=str)
256
257 is_generic_og_trackname = music_info.get('is_original_sound') and music_info.get('title') == 'original sound - %s' % music_info.get('owner_handle')
258 if is_generic_og_trackname:
259 music_track, music_author = contained_music_track or 'original sound', contained_music_author
260 else:
261 music_track, music_author = music_info.get('title'), music_info.get('author')
262
263 return {
264 'id': aweme_id,
265 'title': aweme_detail['desc'],
266 'description': aweme_detail['desc'],
267 'view_count': int_or_none(stats_info.get('play_count')),
268 'like_count': int_or_none(stats_info.get('digg_count')),
269 'repost_count': int_or_none(stats_info.get('share_count')),
270 'comment_count': int_or_none(stats_info.get('comment_count')),
271 'uploader': str_or_none(author_info.get('unique_id')),
943d5ab1 272 'creator': str_or_none(author_info.get('nickname')),
bd9ff55b 273 'uploader_id': str_or_none(author_info.get('uid')),
943d5ab1 274 'uploader_url': user_url,
bd9ff55b
M		 275 'track': music_track,
276 'album': str_or_none(music_info.get('album')) or None,
277 'artist': music_author,
278 'timestamp': int_or_none(aweme_detail.get('create_time')),
279 'formats': formats,
e0585e65 280 'subtitles': self.extract_subtitles(aweme_detail, aweme_id),
bd9ff55b 281 'thumbnails': thumbnails,
53dad39e
M		 282 'duration': int_or_none(traverse_obj(video_info, 'duration', ('download_addr', 'duration')), scale=1000),
283 'availability': self._availability(
284 is_private='Private' in labels,
285 needs_subscription='Friends only' in labels,
286 is_unlisted='Followers only' in labels)
bd9ff55b
M		 287 }
288
0481e266 289 def _parse_aweme_video_web(self, aweme_detail, webpage_url):
943d5ab1 290 video_info = aweme_detail['video']
11aa91a1 291 author_info = traverse_obj(aweme_detail, 'authorInfo', 'author', expected_type=dict, default={})
943d5ab1
M		 292 music_info = aweme_detail.get('music') or {}
293 stats_info = aweme_detail.get('stats') or {}
294 user_url = self._UPLOADER_URL_FORMAT % (traverse_obj(author_info,
295 'secUid', 'id', 'uid', 'uniqueId',
11aa91a1
M		 296 expected_type=str_or_none, get_all=False)
297 or aweme_detail.get('authorSecId'))
943d5ab1
M		 298
299 formats = []
300 play_url = video_info.get('playAddr')
301 width = video_info.get('width')
302 height = video_info.get('height')
303 if isinstance(play_url, str):
304 formats = [{
305 'url': self._proto_relative_url(play_url),
306 'ext': 'mp4',
307 'width': width,
308 'height': height,
309 }]
310 elif isinstance(play_url, list):
311 formats = [{
312 'url': self._proto_relative_url(url),
313 'ext': 'mp4',
314 'width': width,
315 'height': height,
316 } for url in traverse_obj(play_url, (..., 'src'), expected_type=url_or_none, default=[]) if url]
317
318 download_url = url_or_none(video_info.get('downloadAddr')) or traverse_obj(video_info, ('download', 'url'), expected_type=url_or_none)
319 if download_url:
320 formats.append({
321 'format_id': 'download',
322 'url': self._proto_relative_url(download_url),
323 'ext': 'mp4',
324 'width': width,
325 'height': height,
326 })
327 self._remove_duplicate_formats(formats)
328 self._sort_formats(formats)
329
330 thumbnails = []
331 for thumbnail_name in ('thumbnail', 'cover', 'dynamicCover', 'originCover'):
332 if aweme_detail.get(thumbnail_name):
333 thumbnails = [{
334 'url': self._proto_relative_url(aweme_detail[thumbnail_name]),
335 'width': width,
336 'height': height
337 }]
338
339 return {
340 'id': traverse_obj(aweme_detail, 'id', 'awemeId', expected_type=str_or_none),
341 'title': aweme_detail.get('desc'),
342 'duration': try_get(aweme_detail, lambda x: x['video']['duration'], int),
343 'view_count': int_or_none(stats_info.get('playCount')),
344 'like_count': int_or_none(stats_info.get('diggCount')),
345 'repost_count': int_or_none(stats_info.get('shareCount')),
346 'comment_count': int_or_none(stats_info.get('commentCount')),
347 'timestamp': int_or_none(aweme_detail.get('createTime')),
348 'creator': str_or_none(author_info.get('nickname')),
11aa91a1
M		 349 'uploader': str_or_none(author_info.get('uniqueId') or aweme_detail.get('author')),
350 'uploader_id': str_or_none(author_info.get('id') or aweme_detail.get('authorId')),
943d5ab1
M		 351 'uploader_url': user_url,
352 'track': str_or_none(music_info.get('title')),
353 'album': str_or_none(music_info.get('album')) or None,
354 'artist': str_or_none(music_info.get('authorName')),
355 'formats': formats,
356 'thumbnails': thumbnails,
357 'description': str_or_none(aweme_detail.get('desc')),
358 'http_headers': {
0481e266 359 'Referer': webpage_url
943d5ab1
M		 360 }
361 }
362
0fd6661e
M		 363
364class TikTokIE(TikTokBaseIE):
365 _VALID_URL = r'https?://www\.tiktok\.com/@[\w\.-]+/video/(?P<id>\d+)'
366
367 _TESTS = [{
368 'url': 'https://www.tiktok.com/@leenabhushan/video/6748451240264420610',
0481e266 369 'md5': '736bb7a466c6f0a6afeb597da1e6f5b7',
0fd6661e
M		 370 'info_dict': {
371 'id': '6748451240264420610',
372 'ext': 'mp4',
373 'title': '#jassmanak #lehanga #leenabhushan',
374 'description': '#jassmanak #lehanga #leenabhushan',
375 'duration': 13,
0481e266 376 'height': 1024,
377 'width': 576,
0fd6661e
M		 378 'uploader': 'leenabhushan',
379 'uploader_id': '6691488002098119685',
0481e266 380 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAA_Eb4t1vodM1IuTy_cvp9CY22RAb59xqrO0Xtz9CYQJvgXaDvZxYnZYRzDWhhgJmy',
0fd6661e
M		 381 'creator': 'facestoriesbyleenabh',
382 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?',
383 'upload_date': '20191016',
384 'timestamp': 1571246252,
385 'view_count': int,
386 'like_count': int,
387 'repost_count': int,
388 'comment_count': int,
389 }
390 }, {
391 'url': 'https://www.tiktok.com/@patroxofficial/video/6742501081818877190?langCountry=en',
0481e266 392 'md5': '6f3cf8cdd9b28cb8363fe0a9a160695b',
0fd6661e
M		 393 'info_dict': {
394 'id': '6742501081818877190',
395 'ext': 'mp4',
396 'title': 'md5:5e2a23877420bb85ce6521dbee39ba94',
397 'description': 'md5:5e2a23877420bb85ce6521dbee39ba94',
398 'duration': 27,
399 'height': 960,
400 'width': 540,
401 'uploader': 'patrox',
402 'uploader_id': '18702747',
0481e266 403 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAiFnldaILebi5heDoVU6bn4jBWWycX6-9U3xuNPqZ8Ws',
0fd6661e
M		 404 'creator': 'patroX',
405 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?',
406 'upload_date': '20190930',
407 'timestamp': 1569860870,
408 'view_count': int,
409 'like_count': int,
410 'repost_count': int,
411 'comment_count': int,
412 }
413 }, {
96f13f01
M		 414 # Banned audio, only available on the app
415 'url': 'https://www.tiktok.com/@barudakhb_/video/6984138651336838402',
416 'info_dict': {
417 'id': '6984138651336838402',
418 'ext': 'mp4',
419 'title': 'Balas @yolaaftwsr hayu yu ? #SquadRandom_ 🔥',
420 'description': 'Balas @yolaaftwsr hayu yu ? #SquadRandom_ 🔥',
421 'uploader': 'barudakhb_',
422 'creator': 'md5:29f238c49bc0c176cb3cef1a9cea9fa6',
423 'uploader_id': '6974687867511718913',
424 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAbhBwQC-R1iKoix6jDFsF-vBdfx2ABoDjaZrM9fX6arU3w71q3cOWgWuTXn1soZ7d',
425 'track': 'Boka Dance',
426 'artist': 'md5:29f238c49bc0c176cb3cef1a9cea9fa6',
427 'timestamp': 1626121503,
428 'duration': 18,
429 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?',
430 'upload_date': '20210712',
431 'view_count': int,
432 'like_count': int,
433 'repost_count': int,
434 'comment_count': int,
435 }
436 }, {
437 # Sponsored video, only available with feed workaround
438 'url': 'https://www.tiktok.com/@MS4wLjABAAAATh8Vewkn0LYM7Fo03iec3qKdeCUOcBIouRk1mkiag6h3o_pQu_dUXvZ2EZlGST7_/video/7042692929109986561',
439 'info_dict': {
440 'id': '7042692929109986561',
441 'ext': 'mp4',
442 'title': 'Slap and Run!',
443 'description': 'Slap and Run!',
444 'uploader': 'user440922249',
445 'creator': 'Slap And Run',
446 'uploader_id': '7036055384943690754',
447 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAATh8Vewkn0LYM7Fo03iec3qKdeCUOcBIouRk1mkiag6h3o_pQu_dUXvZ2EZlGST7_',
448 'track': 'Promoted Music',
449 'timestamp': 1639754738,
450 'duration': 30,
451 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?',
452 'upload_date': '20211217',
453 'view_count': int,
454 'like_count': int,
455 'repost_count': int,
456 'comment_count': int,
457 },
458 'expected_warnings': ['Video not available']
e0585e65
M		 459 }, {
460 # Auto-captions available
461 'url': 'https://www.tiktok.com/@hankgreen1/video/7047596209028074758',
462 'only_matching': True
0fd6661e
M		 463 }]
464
0fd6661e 465 def _extract_aweme_app(self, aweme_id):
e540c56f
M		 466 try:
467 aweme_detail = self._call_api('aweme/detail', {'aweme_id': aweme_id}, aweme_id,
468 note='Downloading video details', errnote='Unable to download video details').get('aweme_detail')
469 if not aweme_detail:
470 raise ExtractorError('Video not available', video_id=aweme_id)
471 except ExtractorError as e:
472 self.report_warning(f'{e}; Retrying with feed workaround')
473 feed_list = self._call_api('feed', {'aweme_id': aweme_id}, aweme_id,
474 note='Downloading video feed', errnote='Unable to download video feed').get('aweme_list') or []
be1f331f 475 aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None)
e540c56f
M		 476 if not aweme_detail:
477 raise ExtractorError('Unable to find video in feed', video_id=aweme_id)
943d5ab1 478 return self._parse_aweme_video_app(aweme_detail)
0fd6661e 479
ce18a19b
S		 480 def _real_extract(self, url):
481 video_id = self._match_id(url)
ce18a19b 482
bd9ff55b
M		 483 try:
484 return self._extract_aweme_app(video_id)
485 except ExtractorError as e:
486 self.report_warning(f'{e}; Retrying with webpage')
487
7bbc0bbc 488 # If we only call once, we get a 403 when downlaoding the video.
61e76c1e 489 self._download_webpage(url, video_id)
6fb11ca8 490 webpage = self._download_webpage(url, video_id, note='Downloading video webpage')
135dfa2c 491 next_data = self._search_nextjs_data(webpage, video_id, default='{}')
11aa91a1 492
135dfa2c 493 if next_data:
11aa91a1
M		 494 status = traverse_obj(next_data, ('props', 'pageProps', 'statusCode'), expected_type=int) or 0
495 video_data = traverse_obj(next_data, ('props', 'pageProps', 'itemInfo', 'itemStruct'), expected_type=dict)
496 else:
497 sigi_json = self._search_regex(
498 r'>\s*window\[[\'"]SIGI_STATE[\'"]\]\s*=\s*(?P<sigi_state>{.+});',
499 webpage, 'sigi data', group='sigi_state')
500 sigi_data = self._parse_json(sigi_json, video_id)
501 status = traverse_obj(sigi_data, ('VideoPage', 'statusCode'), expected_type=int) or 0
502 video_data = traverse_obj(sigi_data, ('ItemModule', video_id), expected_type=dict)
503
1418a043 504 if status == 0:
11aa91a1 505 return self._parse_aweme_video_web(video_data, url)
1418a043 506 elif status == 10216:
507 raise ExtractorError('This video is private', expected=True)
6fb11ca8 508 raise ExtractorError('Video not available', video_id=video_id)
f7f18f90
A		 509
510
0fd6661e 511class TikTokUserIE(TikTokBaseIE):
f7f18f90 512 IE_NAME = 'tiktok:user'
0fd6661e 513 _VALID_URL = r'https?://(?:www\.)?tiktok\.com/@(?P<id>[\w\.-]+)/?(?:$|[#?])'
f7f18f90 514 _TESTS = [{
526d74ec 515 'url': 'https://tiktok.com/@corgibobaa?lang=en',
f7f18f90
A		 516 'playlist_mincount': 45,
517 'info_dict': {
518 'id': '6935371178089399301',
0481e266 519 'title': 'corgibobaa',
b3187433 520 'thumbnail': r're:https://.+_1080x1080\.webp'
f7f18f90 521 },
0481e266 522 'expected_warnings': ['Retrying']
f7f18f90
A		 523 }, {
524 'url': 'https://www.tiktok.com/@meme',
525 'playlist_mincount': 593,
526 'info_dict': {
527 'id': '79005827461758976',
0481e266 528 'title': 'meme',
b3187433 529 'thumbnail': r're:https://.+_1080x1080\.webp'
f7f18f90 530 },
0481e266 531 'expected_warnings': ['Retrying']
f7f18f90
A		 532 }]
533
0fd6661e
M		 534 r''' # TODO: Fix by adding _signature to api_url
535 def _entries(self, webpage, user_id, username):
536 secuid = self._search_regex(r'\"secUid\":\"(?P<secUid>[^\"]+)', webpage, username)
f7f18f90
A		 537 verifyfp_cookie = self._get_cookies('https://www.tiktok.com').get('s_v_web_id')
538 if not verifyfp_cookie:
539 raise ExtractorError('Improper cookies (missing s_v_web_id).', expected=True)
540 api_url = f'https://m.tiktok.com/api/post/item_list/?aid=1988&cookie_enabled=true&count=30&verifyFp={verifyfp_cookie.value}&secUid={secuid}&cursor='
541 cursor = '0'
542 for page in itertools.count():
0fd6661e 543 data_json = self._download_json(api_url + cursor, username, note='Downloading Page %d' % page)
f7f18f90
A		 544 for video in data_json.get('itemList', []):
545 video_id = video['id']
546 video_url = f'https://www.tiktok.com/@{user_id}/video/{video_id}'
bd9ff55b 547 yield self._url_result(video_url, 'TikTok', video_id, str_or_none(video.get('desc')))
526d74ec 548 if not data_json.get('hasMore'):
f7f18f90
A		 549 break
550 cursor = data_json['cursor']
0fd6661e
M		 551 '''
552
b3187433 553 def _video_entries_api(self, webpage, user_id, username):
0fd6661e
M		 554 query = {
555 'user_id': user_id,
556 'count': 21,
557 'max_cursor': 0,
558 'min_cursor': 0,
559 'retry_type': 'no_retry',
0930b11f 560 'device_id': ''.join(random.choice(string.digits) for _ in range(19)), # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
0fd6661e
M		 561 }
562
563 max_retries = self.get_param('extractor_retries', 3)
564 for page in itertools.count(1):
565 for retries in itertools.count():
566 try:
567 post_list = self._call_api('aweme/post', query, username,
568 note='Downloading user video list page %d%s' % (page, f' (attempt {retries})' if retries != 0 else ''),
569 errnote='Unable to download user video list')
570 except ExtractorError as e:
571 if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0 and retries != max_retries:
572 self.report_warning('%s. Retrying...' % str(e.cause or e.msg))
573 continue
574 raise
575 break
b3187433 576 yield from post_list.get('aweme_list', [])
0fd6661e
M		 577 if not post_list.get('has_more'):
578 break
579 query['max_cursor'] = post_list['max_cursor']
f7f18f90 580
b3187433 581 def _entries_api(self, user_id, videos):
582 for video in videos:
583 yield {
584 **self._parse_aweme_video_app(video),
585 'extractor_key': TikTokIE.ie_key(),
586 'extractor': 'TikTok',
587 'webpage_url': f'https://tiktok.com/@{user_id}/video/{video["aweme_id"]}',
588 }
589
f7f18f90 590 def _real_extract(self, url):
0481e266 591 user_name = self._match_id(url)
592 webpage = self._download_webpage(url, user_name, headers={
0fd6661e
M		 593 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
594 })
0481e266 595 user_id = self._html_search_regex(r'snssdk\d*://user/profile/(\d+)', webpage, 'user ID')
b3187433 596
597 videos = LazyList(self._video_entries_api(webpage, user_id, user_name))
598 thumbnail = traverse_obj(videos, (0, 'author', 'avatar_larger', 'url_list', 0))
599
600 return self.playlist_result(self._entries_api(user_id, videos), user_id, user_name, thumbnail=thumbnail)
943d5ab1
M		 601
602
8126298c
M		 603class TikTokBaseListIE(TikTokBaseIE):
604 def _entries(self, list_id, display_id):
605 query = {
606 self._QUERY_NAME: list_id,
607 'cursor': 0,
608 'count': 20,
609 'type': 5,
610 'device_id': ''.join(random.choice(string.digits) for i in range(19))
611 }
612
613 max_retries = self.get_param('extractor_retries', 3)
614 for page in itertools.count(1):
615 for retries in itertools.count():
616 try:
617 post_list = self._call_api(self._API_ENDPOINT, query, display_id,
618 note='Downloading video list page %d%s' % (page, f' (attempt {retries})' if retries != 0 else ''),
619 errnote='Unable to download video list')
620 except ExtractorError as e:
621 if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0 and retries != max_retries:
622 self.report_warning('%s. Retrying...' % str(e.cause or e.msg))
623 continue
624 raise
625 break
626 for video in post_list.get('aweme_list', []):
627 yield {
628 **self._parse_aweme_video_app(video),
0b77924a 629 'extractor_key': TikTokIE.ie_key(),
8126298c
M		 630 'extractor': 'TikTok',
631 'webpage_url': f'https://tiktok.com/@_/video/{video["aweme_id"]}',
632 }
633 if not post_list.get('has_more'):
634 break
635 query['cursor'] = post_list['cursor']
636
637 def _real_extract(self, url):
638 list_id = self._match_id(url)
639 return self.playlist_result(self._entries(list_id, list_id), list_id)
640
641
642class TikTokSoundIE(TikTokBaseListIE):
643 IE_NAME = 'tiktok:sound'
644 _VALID_URL = r'https?://(?:www\.)?tiktok\.com/music/[\w\.-]+-(?P<id>[\d]+)[/?#&]?'
645 _QUERY_NAME = 'music_id'
646 _API_ENDPOINT = 'music/aweme'
647 _TESTS = [{
648 'url': 'https://www.tiktok.com/music/Build-a-Btch-6956990112127585029?lang=en',
649 'playlist_mincount': 100,
650 'info_dict': {
651 'id': '6956990112127585029'
652 },
653 'expected_warnings': ['Retrying']
654 }, {
655 # Actual entries are less than listed video count
656 'url': 'https://www.tiktok.com/music/jiefei-soap-remix-7036843036118469381',
657 'playlist_mincount': 2182,
658 'info_dict': {
659 'id': '7036843036118469381'
660 },
661 'expected_warnings': ['Retrying']
662 }]
663
664
665class TikTokEffectIE(TikTokBaseListIE):
666 IE_NAME = 'tiktok:effect'
667 _VALID_URL = r'https?://(?:www\.)?tiktok\.com/sticker/[\w\.-]+-(?P<id>[\d]+)[/?#&]?'
668 _QUERY_NAME = 'sticker_id'
669 _API_ENDPOINT = 'sticker/aweme'
670 _TESTS = [{
671 'url': 'https://www.tiktok.com/sticker/MATERIAL-GWOOORL-1258156',
672 'playlist_mincount': 100,
673 'info_dict': {
674 'id': '1258156',
675 },
676 'expected_warnings': ['Retrying']
677 }, {
678 # Different entries between mobile and web, depending on region
679 'url': 'https://www.tiktok.com/sticker/Elf-Friend-479565',
680 'only_matching': True
681 }]
682
683
684class TikTokTagIE(TikTokBaseListIE):
685 IE_NAME = 'tiktok:tag'
686 _VALID_URL = r'https?://(?:www\.)?tiktok\.com/tag/(?P<id>[^/?#&]+)'
687 _QUERY_NAME = 'ch_id'
688 _API_ENDPOINT = 'challenge/aweme'
689 _TESTS = [{
690 'url': 'https://tiktok.com/tag/hello2018',
691 'playlist_mincount': 39,
692 'info_dict': {
693 'id': '46294678',
694 'title': 'hello2018',
695 },
696 'expected_warnings': ['Retrying']
697 }, {
698 'url': 'https://tiktok.com/tag/fypシ?is_copy_url=0&is_from_webapp=v1',
699 'only_matching': True
700 }]
701
702 def _real_extract(self, url):
703 display_id = self._match_id(url)
704 webpage = self._download_webpage(url, display_id, headers={
705 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
706 })
707 tag_id = self._html_search_regex(r'snssdk\d*://challenge/detail/(\d+)', webpage, 'tag ID')
708 return self.playlist_result(self._entries(tag_id, display_id), tag_id, display_id)
709
710
943d5ab1
M		 711class DouyinIE(TikTokIE):
712 _VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)'
713 _TESTS = [{
714 'url': 'https://www.douyin.com/video/6961737553342991651',
715 'md5': '10523312c8b8100f353620ac9dc8f067',
716 'info_dict': {
717 'id': '6961737553342991651',
718 'ext': 'mp4',
719 'title': '#杨超越 小小水手带你去远航❤️',
720 'uploader': '杨超越',
721 'upload_date': '20210513',
722 'timestamp': 1620905839,
723 'uploader_id': '110403406559',
724 'view_count': int,
725 'like_count': int,
726 'repost_count': int,
727 'comment_count': int,
728 }
729 }, {
730 'url': 'https://www.douyin.com/video/6982497745948921092',
731 'md5': 'd78408c984b9b5102904cf6b6bc2d712',
732 'info_dict': {
733 'id': '6982497745948921092',
734 'ext': 'mp4',
735 'title': '这个夏日和小羊@杨超越 一起遇见白色幻想',
736 'uploader': '杨超越工作室',
737 'upload_date': '20210708',
738 'timestamp': 1625739481,
739 'uploader_id': '408654318141572',
740 'view_count': int,
741 'like_count': int,
742 'repost_count': int,
743 'comment_count': int,
744 }
745 }, {
746 'url': 'https://www.douyin.com/video/6953975910773099811',
747 'md5': '72e882e24f75064c218b76c8b713c185',
748 'info_dict': {
749 'id': '6953975910773099811',
750 'ext': 'mp4',
751 'title': '#一起看海 出现在你的夏日里',
752 'uploader': '杨超越',
753 'upload_date': '20210422',
754 'timestamp': 1619098692,
755 'uploader_id': '110403406559',
756 'view_count': int,
757 'like_count': int,
758 'repost_count': int,
759 'comment_count': int,
760 }
761 }, {
762 'url': 'https://www.douyin.com/video/6950251282489675042',
763 'md5': 'b4db86aec367ef810ddd38b1737d2fed',
764 'info_dict': {
765 'id': '6950251282489675042',
766 'ext': 'mp4',
767 'title': '哈哈哈，成功了哈哈哈哈哈哈',
768 'uploader': '杨超越',
769 'upload_date': '20210412',
770 'timestamp': 1618231483,
771 'uploader_id': '110403406559',
772 'view_count': int,
773 'like_count': int,
774 'repost_count': int,
775 'comment_count': int,
776 }
777 }, {
778 'url': 'https://www.douyin.com/video/6963263655114722595',
779 'md5': '1abe1c477d05ee62efb40bf2329957cf',
780 'info_dict': {
781 'id': '6963263655114722595',
782 'ext': 'mp4',
783 'title': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
784 'uploader': '杨超越',
785 'upload_date': '20210517',
786 'timestamp': 1621261163,
787 'uploader_id': '110403406559',
788 'view_count': int,
789 'like_count': int,
790 'repost_count': int,
791 'comment_count': int,
792 }
793 }]
046cab39 794 _APP_VERSIONS = [('9.6.0', '960')]
943d5ab1
M		 795 _APP_NAME = 'aweme'
796 _AID = 1128
797 _API_HOSTNAME = 'aweme.snssdk.com'
798 _UPLOADER_URL_FORMAT = 'https://www.douyin.com/user/%s'
53dad39e 799 _WEBPAGE_HOST = 'https://www.douyin.com/'
943d5ab1
M		 800
801 def _real_extract(self, url):
802 video_id = self._match_id(url)
803
804 try:
805 return self._extract_aweme_app(video_id)
806 except ExtractorError as e:
807 self.report_warning(f'{e}; Retrying with webpage')
808
809 webpage = self._download_webpage(url, video_id)
810 render_data_json = self._search_regex(
811 r'<script [^>]*\bid=[\'"]RENDER_DATA[\'"][^>]*>(%7B.+%7D)</script>',
812 webpage, 'render data', default=None)
813 if not render_data_json:
814 # TODO: Run verification challenge code to generate signature cookies
815 raise ExtractorError('Fresh cookies (not necessarily logged in) are needed')
816
817 render_data = self._parse_json(
818 render_data_json, video_id, transform_source=compat_urllib_parse_unquote)
819 return self._parse_aweme_video_web(
0481e266 820 traverse_obj(render_data, (..., 'aweme', 'detail'), get_all=False), url)
88afe056 821
822
49895f06 823class TikTokVMIE(InfoExtractor):
824 _VALID_URL = r'https?://(?:vm|vt)\.tiktok\.com/(?P<id>\w+)'
88afe056 825 IE_NAME = 'vm.tiktok'
826
49895f06 827 _TESTS = [{
828 'url': 'https://vm.tiktok.com/ZSe4FqkKd',
829 'info_dict': {
830 'id': '7023491746608712966',
831 'ext': 'mp4',
832 'title': 'md5:5607564db90271abbbf8294cca77eddd',
833 'description': 'md5:5607564db90271abbbf8294cca77eddd',
834 'duration': 11,
835 'upload_date': '20211026',
836 'uploader_id': '7007385080558846981',
837 'creator': 'Memes',
838 'artist': 'Memes',
839 'track': 'original sound',
840 'uploader': 'susmandem',
841 'timestamp': 1635284105,
842 'thumbnail': r're:https://.+\.webp.*',
843 'like_count': int,
844 'view_count': int,
845 'comment_count': int,
846 'repost_count': int,
847 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAXcNoOEOxVyBzuII_E--T0MeCrLP0ay1Sm6x_n3dluiWEoWZD0VlQOytwad4W0i0n',
848 }
849 }, {
850 'url': 'https://vt.tiktok.com/ZSe4FqkKd',
851 'only_matching': True,
852 }]
853
88afe056 854 def _real_extract(self, url):
855 return self.url_result(self._request_webpage(
856 HEADRequest(url), self._match_id(url), headers={'User-Agent': 'facebookexternalhit/1.1'}).geturl(), TikTokIE)
fork of https://github.com/yt-dlp/yt-dlp/