]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/tiktok.py
projects / yt-dlp.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[Drooble] Add extractor (#1547)
[yt-dlp.git] / yt_dlp / extractor / tiktok.py
CommitLineData
1ead840d
KS		 1# coding: utf-8
2from __future__ import unicode_literals
f7f18f90
A		 3
4import itertools
bd9ff55b
M		 5import random
6import string
7import time
0fd6661e 8import json
1ead840d
KS		 9
10from .common import InfoExtractor
943d5ab1 11from ..compat import compat_urllib_parse_unquote
1ead840d 12from ..utils import (
ce18a19b 13 ExtractorError,
1ead840d 14 int_or_none,
34921b43 15 join_nonempty,
1ead840d 16 str_or_none,
bd9ff55b
M		 17 traverse_obj,
18 try_get,
943d5ab1 19 url_or_none,
bd9ff55b 20 qualities,
1ead840d
KS		 21)
22
23
0fd6661e 24class TikTokBaseIE(InfoExtractor):
bd9ff55b
M		 25 _APP_VERSION = '20.9.3'
26 _MANIFEST_APP_VERSION = '291'
943d5ab1
M		 27 _APP_NAME = 'trill'
28 _AID = 1180
53dad39e 29 _API_HOSTNAME = 'api-h2.tiktokv.com'
943d5ab1 30 _UPLOADER_URL_FORMAT = 'https://www.tiktok.com/@%s'
53dad39e 31 _WEBPAGE_HOST = 'https://www.tiktok.com/'
bd9ff55b 32 QUALITIES = ('360p', '540p', '720p')
ce18a19b 33
0fd6661e
M		 34 def _call_api(self, ep, query, video_id, fatal=True,
35 note='Downloading API JSON', errnote='Unable to download API page'):
36 real_query = {
37 **query,
bd9ff55b
M		 38 'version_name': self._APP_VERSION,
39 'version_code': self._MANIFEST_APP_VERSION,
40 'build_number': self._APP_VERSION,
41 'manifest_version_code': self._MANIFEST_APP_VERSION,
42 'update_version_code': self._MANIFEST_APP_VERSION,
0930b11f 43 'openudid': ''.join(random.choice('0123456789abcdef') for _ in range(16)),
44 'uuid': ''.join([random.choice(string.digits) for _ in range(16)]),
bd9ff55b
M		 45 '_rticket': int(time.time() * 1000),
46 'ts': int(time.time()),
47 'device_brand': 'Google',
48 'device_type': 'Pixel 4',
49 'device_platform': 'android',
50 'resolution': '1080*1920',
51 'dpi': 420,
52 'os_version': '10',
53 'os_api': '29',
54 'carrier_region': 'US',
55 'sys_region': 'US',
56 'region': 'US',
943d5ab1 57 'app_name': self._APP_NAME,
bd9ff55b
M		 58 'app_language': 'en',
59 'language': 'en',
60 'timezone_name': 'America/New_York',
61 'timezone_offset': '-14400',
62 'channel': 'googleplay',
63 'ac': 'wifi',
64 'mcc_mnc': '310260',
65 'is_my_cn': 0,
943d5ab1 66 'aid': self._AID,
bd9ff55b
M		 67 'ssmix': 'a',
68 'as': 'a1qwert123',
69 'cp': 'cbfhckdckkde1',
70 }
0930b11f 71 self._set_cookie(self._API_HOSTNAME, 'odin_tt', ''.join(random.choice('0123456789abcdef') for _ in range(160)))
53dad39e
M		 72 webpage_cookies = self._get_cookies(self._WEBPAGE_HOST)
73 if webpage_cookies.get('sid_tt'):
74 self._set_cookie(self._API_HOSTNAME, 'sid_tt', webpage_cookies['sid_tt'].value)
0fd6661e 75 return self._download_json(
943d5ab1 76 'https://%s/aweme/v1/%s/' % (self._API_HOSTNAME, ep), video_id=video_id,
0fd6661e 77 fatal=fatal, note=note, errnote=errnote, headers={
bd9ff55b 78 'User-Agent': f'com.ss.android.ugc.trill/{self._MANIFEST_APP_VERSION} (Linux; U; Android 10; en_US; Pixel 4; Build/QQ3A.200805.001; Cronet/58.0.2991.0)',
0fd6661e
M		 79 'Accept': 'application/json',
80 }, query=real_query)
81
943d5ab1 82 def _parse_aweme_video_app(self, aweme_detail):
0fd6661e 83 aweme_id = aweme_detail['aweme_id']
bd9ff55b
M		 84 video_info = aweme_detail['video']
85
86 def parse_url_key(url_key):
87 format_id, codec, res, bitrate = self._search_regex(
88 r'v[^_]+_(?P<id>(?P<codec>[^_]+)_(?P<res>\d+p)_(?P<bitrate>\d+))', url_key,
89 'url key', default=(None, None, None, None), group=('id', 'codec', 'res', 'bitrate'))
90 if not format_id:
91 return {}, None
92 return {
93 'format_id': format_id,
94 'vcodec': 'h265' if codec == 'bytevc1' else codec,
95 'tbr': int_or_none(bitrate, scale=1000) or None,
96 'quality': qualities(self.QUALITIES)(res),
97 }, res
98
99 known_resolutions = {}
100
101 def extract_addr(addr, add_meta={}):
102 parsed_meta, res = parse_url_key(addr.get('url_key', ''))
103 if res:
104 known_resolutions.setdefault(res, {}).setdefault('height', add_meta.get('height'))
105 known_resolutions[res].setdefault('width', add_meta.get('width'))
106 parsed_meta.update(known_resolutions.get(res, {}))
107 add_meta.setdefault('height', int_or_none(res[:-1]))
108 return [{
109 'url': url,
110 'filesize': int_or_none(addr.get('data_size')),
111 'ext': 'mp4',
112 'acodec': 'aac',
0fd6661e
M		 113 'source_preference': -2 if 'aweme/v1' in url else -1, # Downloads from API might get blocked
114 **add_meta, **parsed_meta,
34921b43 115 'format_note': join_nonempty(
116 add_meta.get('format_note'), '(API)' if 'aweme/v1' in url else None, delim=' ')
bd9ff55b
M		 117 } for url in addr.get('url_list') or []]
118
119 # Hack: Add direct video links first to prioritize them when removing duplicate formats
120 formats = []
121 if video_info.get('play_addr'):
122 formats.extend(extract_addr(video_info['play_addr'], {
123 'format_id': 'play_addr',
124 'format_note': 'Direct video',
125 'vcodec': 'h265' if traverse_obj(
126 video_info, 'is_bytevc1', 'is_h265') else 'h264', # Always h264?
127 'width': video_info.get('width'),
128 'height': video_info.get('height'),
129 }))
130 if video_info.get('download_addr'):
131 formats.extend(extract_addr(video_info['download_addr'], {
132 'format_id': 'download_addr',
133 'format_note': 'Download video%s' % (', watermarked' if video_info.get('has_watermark') else ''),
134 'vcodec': 'h264',
135 'width': video_info.get('width'),
136 'height': video_info.get('height'),
0fd6661e 137 'preference': -2 if video_info.get('has_watermark') else -1,
bd9ff55b
M		 138 }))
139 if video_info.get('play_addr_h264'):
140 formats.extend(extract_addr(video_info['play_addr_h264'], {
141 'format_id': 'play_addr_h264',
142 'format_note': 'Direct video',
143 'vcodec': 'h264',
144 }))
145 if video_info.get('play_addr_bytevc1'):
146 formats.extend(extract_addr(video_info['play_addr_bytevc1'], {
147 'format_id': 'play_addr_bytevc1',
148 'format_note': 'Direct video',
149 'vcodec': 'h265',
150 }))
151
152 for bitrate in video_info.get('bit_rate', []):
153 if bitrate.get('play_addr'):
154 formats.extend(extract_addr(bitrate['play_addr'], {
155 'format_id': bitrate.get('gear_name'),
156 'format_note': 'Playback video',
157 'tbr': try_get(bitrate, lambda x: x['bit_rate'] / 1000),
158 'vcodec': 'h265' if traverse_obj(
159 bitrate, 'is_bytevc1', 'is_h265') else 'h264',
943d5ab1 160 'fps': bitrate.get('FPS'),
bd9ff55b
M		 161 }))
162
163 self._remove_duplicate_formats(formats)
0fd6661e 164 self._sort_formats(formats, ('quality', 'codec', 'size', 'br'))
bd9ff55b
M		 165
166 thumbnails = []
167 for cover_id in ('cover', 'ai_dynamic_cover', 'animated_cover', 'ai_dynamic_cover_bak',
168 'origin_cover', 'dynamic_cover'):
169 cover = video_info.get(cover_id)
170 if cover:
171 for cover_url in cover['url_list']:
172 thumbnails.append({
173 'id': cover_id,
174 'url': cover_url,
175 })
176
177 stats_info = aweme_detail.get('statistics', {})
178 author_info = aweme_detail.get('author', {})
179 music_info = aweme_detail.get('music', {})
943d5ab1
M		 180 user_url = self._UPLOADER_URL_FORMAT % (traverse_obj(author_info,
181 'sec_uid', 'id', 'uid', 'unique_id',
182 expected_type=str_or_none, get_all=False))
a8549f19 183 labels = traverse_obj(aweme_detail, ('hybrid_label', ..., 'text'), expected_type=str, default=[])
bd9ff55b
M		 184
185 contained_music_track = traverse_obj(
186 music_info, ('matched_song', 'title'), ('matched_pgc_sound', 'title'), expected_type=str)
187 contained_music_author = traverse_obj(
188 music_info, ('matched_song', 'author'), ('matched_pgc_sound', 'author'), 'author', expected_type=str)
189
190 is_generic_og_trackname = music_info.get('is_original_sound') and music_info.get('title') == 'original sound - %s' % music_info.get('owner_handle')
191 if is_generic_og_trackname:
192 music_track, music_author = contained_music_track or 'original sound', contained_music_author
193 else:
194 music_track, music_author = music_info.get('title'), music_info.get('author')
195
196 return {
197 'id': aweme_id,
198 'title': aweme_detail['desc'],
199 'description': aweme_detail['desc'],
200 'view_count': int_or_none(stats_info.get('play_count')),
201 'like_count': int_or_none(stats_info.get('digg_count')),
202 'repost_count': int_or_none(stats_info.get('share_count')),
203 'comment_count': int_or_none(stats_info.get('comment_count')),
204 'uploader': str_or_none(author_info.get('unique_id')),
943d5ab1 205 'creator': str_or_none(author_info.get('nickname')),
bd9ff55b 206 'uploader_id': str_or_none(author_info.get('uid')),
943d5ab1 207 'uploader_url': user_url,
bd9ff55b
M		 208 'track': music_track,
209 'album': str_or_none(music_info.get('album')) or None,
210 'artist': music_author,
211 'timestamp': int_or_none(aweme_detail.get('create_time')),
212 'formats': formats,
213 'thumbnails': thumbnails,
53dad39e
M		 214 'duration': int_or_none(traverse_obj(video_info, 'duration', ('download_addr', 'duration')), scale=1000),
215 'availability': self._availability(
216 is_private='Private' in labels,
217 needs_subscription='Friends only' in labels,
218 is_unlisted='Followers only' in labels)
bd9ff55b
M		 219 }
220
0481e266 221 def _parse_aweme_video_web(self, aweme_detail, webpage_url):
943d5ab1
M		 222 video_info = aweme_detail['video']
223 author_info = traverse_obj(aweme_detail, 'author', 'authorInfo', default={})
224 music_info = aweme_detail.get('music') or {}
225 stats_info = aweme_detail.get('stats') or {}
226 user_url = self._UPLOADER_URL_FORMAT % (traverse_obj(author_info,
227 'secUid', 'id', 'uid', 'uniqueId',
228 expected_type=str_or_none, get_all=False))
229
230 formats = []
231 play_url = video_info.get('playAddr')
232 width = video_info.get('width')
233 height = video_info.get('height')
234 if isinstance(play_url, str):
235 formats = [{
236 'url': self._proto_relative_url(play_url),
237 'ext': 'mp4',
238 'width': width,
239 'height': height,
240 }]
241 elif isinstance(play_url, list):
242 formats = [{
243 'url': self._proto_relative_url(url),
244 'ext': 'mp4',
245 'width': width,
246 'height': height,
247 } for url in traverse_obj(play_url, (..., 'src'), expected_type=url_or_none, default=[]) if url]
248
249 download_url = url_or_none(video_info.get('downloadAddr')) or traverse_obj(video_info, ('download', 'url'), expected_type=url_or_none)
250 if download_url:
251 formats.append({
252 'format_id': 'download',
253 'url': self._proto_relative_url(download_url),
254 'ext': 'mp4',
255 'width': width,
256 'height': height,
257 })
258 self._remove_duplicate_formats(formats)
259 self._sort_formats(formats)
260
261 thumbnails = []
262 for thumbnail_name in ('thumbnail', 'cover', 'dynamicCover', 'originCover'):
263 if aweme_detail.get(thumbnail_name):
264 thumbnails = [{
265 'url': self._proto_relative_url(aweme_detail[thumbnail_name]),
266 'width': width,
267 'height': height
268 }]
269
270 return {
271 'id': traverse_obj(aweme_detail, 'id', 'awemeId', expected_type=str_or_none),
272 'title': aweme_detail.get('desc'),
273 'duration': try_get(aweme_detail, lambda x: x['video']['duration'], int),
274 'view_count': int_or_none(stats_info.get('playCount')),
275 'like_count': int_or_none(stats_info.get('diggCount')),
276 'repost_count': int_or_none(stats_info.get('shareCount')),
277 'comment_count': int_or_none(stats_info.get('commentCount')),
278 'timestamp': int_or_none(aweme_detail.get('createTime')),
279 'creator': str_or_none(author_info.get('nickname')),
280 'uploader': str_or_none(author_info.get('uniqueId')),
281 'uploader_id': str_or_none(author_info.get('id')),
282 'uploader_url': user_url,
283 'track': str_or_none(music_info.get('title')),
284 'album': str_or_none(music_info.get('album')) or None,
285 'artist': str_or_none(music_info.get('authorName')),
286 'formats': formats,
287 'thumbnails': thumbnails,
288 'description': str_or_none(aweme_detail.get('desc')),
289 'http_headers': {
0481e266 290 'Referer': webpage_url
943d5ab1
M		 291 }
292 }
293
0fd6661e
M		 294
295class TikTokIE(TikTokBaseIE):
296 _VALID_URL = r'https?://www\.tiktok\.com/@[\w\.-]+/video/(?P<id>\d+)'
297
298 _TESTS = [{
299 'url': 'https://www.tiktok.com/@leenabhushan/video/6748451240264420610',
0481e266 300 'md5': '736bb7a466c6f0a6afeb597da1e6f5b7',
0fd6661e
M		 301 'info_dict': {
302 'id': '6748451240264420610',
303 'ext': 'mp4',
304 'title': '#jassmanak #lehanga #leenabhushan',
305 'description': '#jassmanak #lehanga #leenabhushan',
306 'duration': 13,
0481e266 307 'height': 1024,
308 'width': 576,
0fd6661e
M		 309 'uploader': 'leenabhushan',
310 'uploader_id': '6691488002098119685',
0481e266 311 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAA_Eb4t1vodM1IuTy_cvp9CY22RAb59xqrO0Xtz9CYQJvgXaDvZxYnZYRzDWhhgJmy',
0fd6661e
M		 312 'creator': 'facestoriesbyleenabh',
313 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?',
314 'upload_date': '20191016',
315 'timestamp': 1571246252,
316 'view_count': int,
317 'like_count': int,
318 'repost_count': int,
319 'comment_count': int,
320 }
321 }, {
322 'url': 'https://www.tiktok.com/@patroxofficial/video/6742501081818877190?langCountry=en',
0481e266 323 'md5': '6f3cf8cdd9b28cb8363fe0a9a160695b',
0fd6661e
M		 324 'info_dict': {
325 'id': '6742501081818877190',
326 'ext': 'mp4',
327 'title': 'md5:5e2a23877420bb85ce6521dbee39ba94',
328 'description': 'md5:5e2a23877420bb85ce6521dbee39ba94',
329 'duration': 27,
330 'height': 960,
331 'width': 540,
332 'uploader': 'patrox',
333 'uploader_id': '18702747',
0481e266 334 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAiFnldaILebi5heDoVU6bn4jBWWycX6-9U3xuNPqZ8Ws',
0fd6661e
M		 335 'creator': 'patroX',
336 'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?',
337 'upload_date': '20190930',
338 'timestamp': 1569860870,
339 'view_count': int,
340 'like_count': int,
341 'repost_count': int,
342 'comment_count': int,
343 }
344 }, {
345 # Promoted content/ad
346 'url': 'https://www.tiktok.com/@MS4wLjABAAAAAR29F6J2Ktu0Daw03BJyXPNoRQ-W7U5a0Mn3lVCq2rQhjOd_WNLclHUoFgwX8Eno/video/6932675057474981122',
347 'only_matching': True,
348 }]
349
0fd6661e 350 def _extract_aweme_app(self, aweme_id):
e540c56f
M		 351 try:
352 aweme_detail = self._call_api('aweme/detail', {'aweme_id': aweme_id}, aweme_id,
353 note='Downloading video details', errnote='Unable to download video details').get('aweme_detail')
354 if not aweme_detail:
355 raise ExtractorError('Video not available', video_id=aweme_id)
356 except ExtractorError as e:
357 self.report_warning(f'{e}; Retrying with feed workaround')
358 feed_list = self._call_api('feed', {'aweme_id': aweme_id}, aweme_id,
359 note='Downloading video feed', errnote='Unable to download video feed').get('aweme_list') or []
360 aweme_detail = next(aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id)
361 if not aweme_detail:
362 raise ExtractorError('Unable to find video in feed', video_id=aweme_id)
943d5ab1 363 return self._parse_aweme_video_app(aweme_detail)
0fd6661e 364
ce18a19b
S		 365 def _real_extract(self, url):
366 video_id = self._match_id(url)
ce18a19b 367
bd9ff55b
M		 368 try:
369 return self._extract_aweme_app(video_id)
370 except ExtractorError as e:
371 self.report_warning(f'{e}; Retrying with webpage')
372
7bbc0bbc 373 # If we only call once, we get a 403 when downlaoding the video.
61e76c1e 374 self._download_webpage(url, video_id)
6fb11ca8 375 webpage = self._download_webpage(url, video_id, note='Downloading video webpage')
4b6d03ed 376 json_string = self._search_regex(
6255e567
AG		 377 r'id=\"__NEXT_DATA__\"\s+type=\"application\/json\"\s*[^>]+>\s*(?P<json_string_ld>[^<]+)',
378 webpage, 'json_string', group='json_string_ld')
4b6d03ed 379 json_data = self._parse_json(json_string, video_id)
4f5a0ad8 380 props_data = try_get(json_data, lambda x: x['props'], expected_type=dict)
ce18a19b 381
4b6d03ed 382 # Chech statusCode for success
1418a043 383 status = props_data.get('pageProps').get('statusCode')
384 if status == 0:
0481e266 385 return self._parse_aweme_video_web(props_data['pageProps']['itemInfo']['itemStruct'], url)
1418a043 386 elif status == 10216:
387 raise ExtractorError('This video is private', expected=True)
4b6d03ed 388
6fb11ca8 389 raise ExtractorError('Video not available', video_id=video_id)
f7f18f90
A		 390
391
0fd6661e 392class TikTokUserIE(TikTokBaseIE):
f7f18f90 393 IE_NAME = 'tiktok:user'
0fd6661e 394 _VALID_URL = r'https?://(?:www\.)?tiktok\.com/@(?P<id>[\w\.-]+)/?(?:$|[#?])'
f7f18f90 395 _TESTS = [{
526d74ec 396 'url': 'https://tiktok.com/@corgibobaa?lang=en',
f7f18f90
A		 397 'playlist_mincount': 45,
398 'info_dict': {
399 'id': '6935371178089399301',
0481e266 400 'title': 'corgibobaa',
f7f18f90 401 },
0481e266 402 'expected_warnings': ['Retrying']
f7f18f90
A		 403 }, {
404 'url': 'https://www.tiktok.com/@meme',
405 'playlist_mincount': 593,
406 'info_dict': {
407 'id': '79005827461758976',
0481e266 408 'title': 'meme',
f7f18f90 409 },
0481e266 410 'expected_warnings': ['Retrying']
f7f18f90
A		 411 }]
412
0fd6661e
M		 413 r''' # TODO: Fix by adding _signature to api_url
414 def _entries(self, webpage, user_id, username):
415 secuid = self._search_regex(r'\"secUid\":\"(?P<secUid>[^\"]+)', webpage, username)
f7f18f90
A		 416 verifyfp_cookie = self._get_cookies('https://www.tiktok.com').get('s_v_web_id')
417 if not verifyfp_cookie:
418 raise ExtractorError('Improper cookies (missing s_v_web_id).', expected=True)
419 api_url = f'https://m.tiktok.com/api/post/item_list/?aid=1988&cookie_enabled=true&count=30&verifyFp={verifyfp_cookie.value}&secUid={secuid}&cursor='
420 cursor = '0'
421 for page in itertools.count():
0fd6661e 422 data_json = self._download_json(api_url + cursor, username, note='Downloading Page %d' % page)
f7f18f90
A		 423 for video in data_json.get('itemList', []):
424 video_id = video['id']
425 video_url = f'https://www.tiktok.com/@{user_id}/video/{video_id}'
bd9ff55b 426 yield self._url_result(video_url, 'TikTok', video_id, str_or_none(video.get('desc')))
526d74ec 427 if not data_json.get('hasMore'):
f7f18f90
A		 428 break
429 cursor = data_json['cursor']
0fd6661e
M		 430 '''
431
432 def _entries_api(self, webpage, user_id, username):
433 query = {
434 'user_id': user_id,
435 'count': 21,
436 'max_cursor': 0,
437 'min_cursor': 0,
438 'retry_type': 'no_retry',
0930b11f 439 'device_id': ''.join(random.choice(string.digits) for _ in range(19)), # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
0fd6661e
M		 440 }
441
442 max_retries = self.get_param('extractor_retries', 3)
443 for page in itertools.count(1):
444 for retries in itertools.count():
445 try:
446 post_list = self._call_api('aweme/post', query, username,
447 note='Downloading user video list page %d%s' % (page, f' (attempt {retries})' if retries != 0 else ''),
448 errnote='Unable to download user video list')
449 except ExtractorError as e:
450 if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0 and retries != max_retries:
451 self.report_warning('%s. Retrying...' % str(e.cause or e.msg))
452 continue
453 raise
454 break
455 for video in post_list.get('aweme_list', []):
456 yield {
0481e266 457 **self._parse_aweme_video_app(video),
0b77924a 458 'extractor_key': TikTokIE.ie_key(),
0fd6661e 459 'extractor': 'TikTok',
b28cdcc0 460 'webpage_url': f'https://tiktok.com/@{user_id}/video/{video["aweme_id"]}',
0fd6661e
M		 461 }
462 if not post_list.get('has_more'):
463 break
464 query['max_cursor'] = post_list['max_cursor']
f7f18f90
A		 465
466 def _real_extract(self, url):
0481e266 467 user_name = self._match_id(url)
468 webpage = self._download_webpage(url, user_name, headers={
0fd6661e
M		 469 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
470 })
0481e266 471 user_id = self._html_search_regex(r'snssdk\d*://user/profile/(\d+)', webpage, 'user ID')
472 return self.playlist_result(self._entries_api(webpage, user_id, user_name), user_id, user_name)
943d5ab1
M		 473
474
8126298c
M		 475class TikTokBaseListIE(TikTokBaseIE):
476 def _entries(self, list_id, display_id):
477 query = {
478 self._QUERY_NAME: list_id,
479 'cursor': 0,
480 'count': 20,
481 'type': 5,
482 'device_id': ''.join(random.choice(string.digits) for i in range(19))
483 }
484
485 max_retries = self.get_param('extractor_retries', 3)
486 for page in itertools.count(1):
487 for retries in itertools.count():
488 try:
489 post_list = self._call_api(self._API_ENDPOINT, query, display_id,
490 note='Downloading video list page %d%s' % (page, f' (attempt {retries})' if retries != 0 else ''),
491 errnote='Unable to download video list')
492 except ExtractorError as e:
493 if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0 and retries != max_retries:
494 self.report_warning('%s. Retrying...' % str(e.cause or e.msg))
495 continue
496 raise
497 break
498 for video in post_list.get('aweme_list', []):
499 yield {
500 **self._parse_aweme_video_app(video),
0b77924a 501 'extractor_key': TikTokIE.ie_key(),
8126298c
M		 502 'extractor': 'TikTok',
503 'webpage_url': f'https://tiktok.com/@_/video/{video["aweme_id"]}',
504 }
505 if not post_list.get('has_more'):
506 break
507 query['cursor'] = post_list['cursor']
508
509 def _real_extract(self, url):
510 list_id = self._match_id(url)
511 return self.playlist_result(self._entries(list_id, list_id), list_id)
512
513
514class TikTokSoundIE(TikTokBaseListIE):
515 IE_NAME = 'tiktok:sound'
516 _VALID_URL = r'https?://(?:www\.)?tiktok\.com/music/[\w\.-]+-(?P<id>[\d]+)[/?#&]?'
517 _QUERY_NAME = 'music_id'
518 _API_ENDPOINT = 'music/aweme'
519 _TESTS = [{
520 'url': 'https://www.tiktok.com/music/Build-a-Btch-6956990112127585029?lang=en',
521 'playlist_mincount': 100,
522 'info_dict': {
523 'id': '6956990112127585029'
524 },
525 'expected_warnings': ['Retrying']
526 }, {
527 # Actual entries are less than listed video count
528 'url': 'https://www.tiktok.com/music/jiefei-soap-remix-7036843036118469381',
529 'playlist_mincount': 2182,
530 'info_dict': {
531 'id': '7036843036118469381'
532 },
533 'expected_warnings': ['Retrying']
534 }]
535
536
537class TikTokEffectIE(TikTokBaseListIE):
538 IE_NAME = 'tiktok:effect'
539 _VALID_URL = r'https?://(?:www\.)?tiktok\.com/sticker/[\w\.-]+-(?P<id>[\d]+)[/?#&]?'
540 _QUERY_NAME = 'sticker_id'
541 _API_ENDPOINT = 'sticker/aweme'
542 _TESTS = [{
543 'url': 'https://www.tiktok.com/sticker/MATERIAL-GWOOORL-1258156',
544 'playlist_mincount': 100,
545 'info_dict': {
546 'id': '1258156',
547 },
548 'expected_warnings': ['Retrying']
549 }, {
550 # Different entries between mobile and web, depending on region
551 'url': 'https://www.tiktok.com/sticker/Elf-Friend-479565',
552 'only_matching': True
553 }]
554
555
556class TikTokTagIE(TikTokBaseListIE):
557 IE_NAME = 'tiktok:tag'
558 _VALID_URL = r'https?://(?:www\.)?tiktok\.com/tag/(?P<id>[^/?#&]+)'
559 _QUERY_NAME = 'ch_id'
560 _API_ENDPOINT = 'challenge/aweme'
561 _TESTS = [{
562 'url': 'https://tiktok.com/tag/hello2018',
563 'playlist_mincount': 39,
564 'info_dict': {
565 'id': '46294678',
566 'title': 'hello2018',
567 },
568 'expected_warnings': ['Retrying']
569 }, {
570 'url': 'https://tiktok.com/tag/fypシ?is_copy_url=0&is_from_webapp=v1',
571 'only_matching': True
572 }]
573
574 def _real_extract(self, url):
575 display_id = self._match_id(url)
576 webpage = self._download_webpage(url, display_id, headers={
577 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
578 })
579 tag_id = self._html_search_regex(r'snssdk\d*://challenge/detail/(\d+)', webpage, 'tag ID')
580 return self.playlist_result(self._entries(tag_id, display_id), tag_id, display_id)
581
582
943d5ab1
M		 583class DouyinIE(TikTokIE):
584 _VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)'
585 _TESTS = [{
586 'url': 'https://www.douyin.com/video/6961737553342991651',
587 'md5': '10523312c8b8100f353620ac9dc8f067',
588 'info_dict': {
589 'id': '6961737553342991651',
590 'ext': 'mp4',
591 'title': '#杨超越 小小水手带你去远航❤️',
592 'uploader': '杨超越',
593 'upload_date': '20210513',
594 'timestamp': 1620905839,
595 'uploader_id': '110403406559',
596 'view_count': int,
597 'like_count': int,
598 'repost_count': int,
599 'comment_count': int,
600 }
601 }, {
602 'url': 'https://www.douyin.com/video/6982497745948921092',
603 'md5': 'd78408c984b9b5102904cf6b6bc2d712',
604 'info_dict': {
605 'id': '6982497745948921092',
606 'ext': 'mp4',
607 'title': '这个夏日和小羊@杨超越 一起遇见白色幻想',
608 'uploader': '杨超越工作室',
609 'upload_date': '20210708',
610 'timestamp': 1625739481,
611 'uploader_id': '408654318141572',
612 'view_count': int,
613 'like_count': int,
614 'repost_count': int,
615 'comment_count': int,
616 }
617 }, {
618 'url': 'https://www.douyin.com/video/6953975910773099811',
619 'md5': '72e882e24f75064c218b76c8b713c185',
620 'info_dict': {
621 'id': '6953975910773099811',
622 'ext': 'mp4',
623 'title': '#一起看海 出现在你的夏日里',
624 'uploader': '杨超越',
625 'upload_date': '20210422',
626 'timestamp': 1619098692,
627 'uploader_id': '110403406559',
628 'view_count': int,
629 'like_count': int,
630 'repost_count': int,
631 'comment_count': int,
632 }
633 }, {
634 'url': 'https://www.douyin.com/video/6950251282489675042',
635 'md5': 'b4db86aec367ef810ddd38b1737d2fed',
636 'info_dict': {
637 'id': '6950251282489675042',
638 'ext': 'mp4',
639 'title': '哈哈哈，成功了哈哈哈哈哈哈',
640 'uploader': '杨超越',
641 'upload_date': '20210412',
642 'timestamp': 1618231483,
643 'uploader_id': '110403406559',
644 'view_count': int,
645 'like_count': int,
646 'repost_count': int,
647 'comment_count': int,
648 }
649 }, {
650 'url': 'https://www.douyin.com/video/6963263655114722595',
651 'md5': '1abe1c477d05ee62efb40bf2329957cf',
652 'info_dict': {
653 'id': '6963263655114722595',
654 'ext': 'mp4',
655 'title': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
656 'uploader': '杨超越',
657 'upload_date': '20210517',
658 'timestamp': 1621261163,
659 'uploader_id': '110403406559',
660 'view_count': int,
661 'like_count': int,
662 'repost_count': int,
663 'comment_count': int,
664 }
665 }]
666 _APP_VERSION = '9.6.0'
667 _MANIFEST_APP_VERSION = '960'
668 _APP_NAME = 'aweme'
669 _AID = 1128
670 _API_HOSTNAME = 'aweme.snssdk.com'
671 _UPLOADER_URL_FORMAT = 'https://www.douyin.com/user/%s'
53dad39e 672 _WEBPAGE_HOST = 'https://www.douyin.com/'
943d5ab1
M		 673
674 def _real_extract(self, url):
675 video_id = self._match_id(url)
676
677 try:
678 return self._extract_aweme_app(video_id)
679 except ExtractorError as e:
680 self.report_warning(f'{e}; Retrying with webpage')
681
682 webpage = self._download_webpage(url, video_id)
683 render_data_json = self._search_regex(
684 r'<script [^>]*\bid=[\'"]RENDER_DATA[\'"][^>]*>(%7B.+%7D)</script>',
685 webpage, 'render data', default=None)
686 if not render_data_json:
687 # TODO: Run verification challenge code to generate signature cookies
688 raise ExtractorError('Fresh cookies (not necessarily logged in) are needed')
689
690 render_data = self._parse_json(
691 render_data_json, video_id, transform_source=compat_urllib_parse_unquote)
692 return self._parse_aweme_video_web(
0481e266 693 traverse_obj(render_data, (..., 'aweme', 'detail'), get_all=False), url)
fork of https://github.com/yt-dlp/yt-dlp/