+ user_name = self._match_id(url)
+ webpage = self._download_webpage(url, user_name, headers={
+ 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
+ })
+ user_id = self._html_search_regex(r'snssdk\d*://user/profile/(\d+)', webpage, 'user ID')
+ return self.playlist_result(self._entries_api(webpage, user_id, user_name), user_id, user_name)
+
+
+class TikTokBaseListIE(TikTokBaseIE):
+ def _entries(self, list_id, display_id):
+ query = {
+ self._QUERY_NAME: list_id,
+ 'cursor': 0,
+ 'count': 20,
+ 'type': 5,
+ 'device_id': ''.join(random.choice(string.digits) for i in range(19))
+ }
+
+ max_retries = self.get_param('extractor_retries', 3)
+ for page in itertools.count(1):
+ for retries in itertools.count():
+ try:
+ post_list = self._call_api(self._API_ENDPOINT, query, display_id,
+ note='Downloading video list page %d%s' % (page, f' (attempt {retries})' if retries != 0 else ''),
+ errnote='Unable to download video list')
+ except ExtractorError as e:
+ if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0 and retries != max_retries:
+ self.report_warning('%s. Retrying...' % str(e.cause or e.msg))
+ continue
+ raise
+ break
+ for video in post_list.get('aweme_list', []):
+ yield {
+ **self._parse_aweme_video_app(video),
+ 'extractor_key': TikTokIE.ie_key(),
+ 'extractor': 'TikTok',
+ 'webpage_url': f'https://tiktok.com/@_/video/{video["aweme_id"]}',
+ }
+ if not post_list.get('has_more'):
+ break
+ query['cursor'] = post_list['cursor']
+
+ def _real_extract(self, url):
+ list_id = self._match_id(url)
+ return self.playlist_result(self._entries(list_id, list_id), list_id)
+
+
+class TikTokSoundIE(TikTokBaseListIE):
+ IE_NAME = 'tiktok:sound'
+ _VALID_URL = r'https?://(?:www\.)?tiktok\.com/music/[\w\.-]+-(?P<id>[\d]+)[/?#&]?'
+ _QUERY_NAME = 'music_id'
+ _API_ENDPOINT = 'music/aweme'
+ _TESTS = [{
+ 'url': 'https://www.tiktok.com/music/Build-a-Btch-6956990112127585029?lang=en',
+ 'playlist_mincount': 100,
+ 'info_dict': {
+ 'id': '6956990112127585029'
+ },
+ 'expected_warnings': ['Retrying']
+ }, {
+ # Actual entries are less than listed video count
+ 'url': 'https://www.tiktok.com/music/jiefei-soap-remix-7036843036118469381',
+ 'playlist_mincount': 2182,
+ 'info_dict': {
+ 'id': '7036843036118469381'
+ },
+ 'expected_warnings': ['Retrying']
+ }]
+
+
+class TikTokEffectIE(TikTokBaseListIE):
+ IE_NAME = 'tiktok:effect'
+ _VALID_URL = r'https?://(?:www\.)?tiktok\.com/sticker/[\w\.-]+-(?P<id>[\d]+)[/?#&]?'
+ _QUERY_NAME = 'sticker_id'
+ _API_ENDPOINT = 'sticker/aweme'
+ _TESTS = [{
+ 'url': 'https://www.tiktok.com/sticker/MATERIAL-GWOOORL-1258156',
+ 'playlist_mincount': 100,
+ 'info_dict': {
+ 'id': '1258156',
+ },
+ 'expected_warnings': ['Retrying']
+ }, {
+ # Different entries between mobile and web, depending on region
+ 'url': 'https://www.tiktok.com/sticker/Elf-Friend-479565',
+ 'only_matching': True
+ }]
+
+
+class TikTokTagIE(TikTokBaseListIE):
+ IE_NAME = 'tiktok:tag'
+ _VALID_URL = r'https?://(?:www\.)?tiktok\.com/tag/(?P<id>[^/?#&]+)'
+ _QUERY_NAME = 'ch_id'
+ _API_ENDPOINT = 'challenge/aweme'
+ _TESTS = [{
+ 'url': 'https://tiktok.com/tag/hello2018',
+ 'playlist_mincount': 39,
+ 'info_dict': {
+ 'id': '46294678',
+ 'title': 'hello2018',
+ },
+ 'expected_warnings': ['Retrying']
+ }, {
+ 'url': 'https://tiktok.com/tag/fypシ?is_copy_url=0&is_from_webapp=v1',
+ 'only_matching': True
+ }]
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+ webpage = self._download_webpage(url, display_id, headers={
+ 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
+ })
+ tag_id = self._html_search_regex(r'snssdk\d*://challenge/detail/(\d+)', webpage, 'tag ID')
+ return self.playlist_result(self._entries(tag_id, display_id), tag_id, display_id)
+
+
+class DouyinIE(TikTokIE):
+ _VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)'
+ _TESTS = [{
+ 'url': 'https://www.douyin.com/video/6961737553342991651',
+ 'md5': '10523312c8b8100f353620ac9dc8f067',
+ 'info_dict': {
+ 'id': '6961737553342991651',
+ 'ext': 'mp4',
+ 'title': '#杨超越 小小水手带你去远航❤️',
+ 'uploader': '杨超越',
+ 'upload_date': '20210513',
+ 'timestamp': 1620905839,
+ 'uploader_id': '110403406559',
+ 'view_count': int,
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
+ }
+ }, {
+ 'url': 'https://www.douyin.com/video/6982497745948921092',
+ 'md5': 'd78408c984b9b5102904cf6b6bc2d712',
+ 'info_dict': {
+ 'id': '6982497745948921092',
+ 'ext': 'mp4',
+ 'title': '这个夏日和小羊@杨超越 一起遇见白色幻想',
+ 'uploader': '杨超越工作室',
+ 'upload_date': '20210708',
+ 'timestamp': 1625739481,
+ 'uploader_id': '408654318141572',
+ 'view_count': int,
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
+ }
+ }, {
+ 'url': 'https://www.douyin.com/video/6953975910773099811',
+ 'md5': '72e882e24f75064c218b76c8b713c185',
+ 'info_dict': {
+ 'id': '6953975910773099811',
+ 'ext': 'mp4',
+ 'title': '#一起看海 出现在你的夏日里',
+ 'uploader': '杨超越',
+ 'upload_date': '20210422',
+ 'timestamp': 1619098692,
+ 'uploader_id': '110403406559',
+ 'view_count': int,
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
+ }
+ }, {
+ 'url': 'https://www.douyin.com/video/6950251282489675042',
+ 'md5': 'b4db86aec367ef810ddd38b1737d2fed',
+ 'info_dict': {
+ 'id': '6950251282489675042',
+ 'ext': 'mp4',
+ 'title': '哈哈哈,成功了哈哈哈哈哈哈',
+ 'uploader': '杨超越',
+ 'upload_date': '20210412',
+ 'timestamp': 1618231483,
+ 'uploader_id': '110403406559',
+ 'view_count': int,
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
+ }
+ }, {
+ 'url': 'https://www.douyin.com/video/6963263655114722595',
+ 'md5': '1abe1c477d05ee62efb40bf2329957cf',
+ 'info_dict': {
+ 'id': '6963263655114722595',
+ 'ext': 'mp4',
+ 'title': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
+ 'uploader': '杨超越',
+ 'upload_date': '20210517',
+ 'timestamp': 1621261163,
+ 'uploader_id': '110403406559',
+ 'view_count': int,
+ 'like_count': int,
+ 'repost_count': int,
+ 'comment_count': int,
+ }
+ }]
+ _APP_VERSION = '9.6.0'
+ _MANIFEST_APP_VERSION = '960'
+ _APP_NAME = 'aweme'
+ _AID = 1128
+ _API_HOSTNAME = 'aweme.snssdk.com'
+ _UPLOADER_URL_FORMAT = 'https://www.douyin.com/user/%s'
+ _WEBPAGE_HOST = 'https://www.douyin.com/'
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ try:
+ return self._extract_aweme_app(video_id)
+ except ExtractorError as e:
+ self.report_warning(f'{e}; Retrying with webpage')
+
+ webpage = self._download_webpage(url, video_id)
+ render_data_json = self._search_regex(
+ r'<script [^>]*\bid=[\'"]RENDER_DATA[\'"][^>]*>(%7B.+%7D)</script>',
+ webpage, 'render data', default=None)
+ if not render_data_json:
+ # TODO: Run verification challenge code to generate signature cookies
+ raise ExtractorError('Fresh cookies (not necessarily logged in) are needed')
+
+ render_data = self._parse_json(
+ render_data_json, video_id, transform_source=compat_urllib_parse_unquote)
+ return self._parse_aweme_video_web(
+ traverse_obj(render_data, (..., 'aweme', 'detail'), get_all=False), url)