Merge branch 'tiktok' of https://github.com/skyme5/youtube-dl into skyme5-tiktok

diff --combined youtube_dlc/extractor/extractors.py
index e70e779ffcc04e844d8e1e7ac85150dcafc743c9,50f69f0b6c153d2be19250b829bf2a28af25cec0..af1bc6e31d0c25693faaa0cfed62f657cda03ee9
--- 1/youtube_dlc/extractor/extractors.py
--- 2/youtube_dl/extractor/extractors.py
+++ b/youtube_dlc/extractor/extractors.py
@@@ -36,10 -36,6 +36,10 @@@ from .afreecatv import AfreecaTVI
  from .airmozilla import AirMozillaIE
  from .aljazeera import AlJazeeraIE
  from .alphaporno import AlphaPornoIE
 +from .alura import (
 +    AluraIE,
 +    AluraCourseIE
 +)
  from .amcnetworks import AMCNetworksIE
  from .americastestkitchen import AmericasTestKitchenIE
  from .animeondemand import AnimeOnDemandIE
@@@ -109,7 -105,6 +109,7 @@@ from .bilibili import 
      BiliBiliBangumiIE,
      BilibiliAudioIE,
      BilibiliAudioAlbumIE,
 +    BiliBiliPlayerIE,
  )
  from .biobiochiletv import BioBioChileTVIE
  from .bitchute import (
@@@ -266,10 -261,7 +266,10 @@@ from .daum import 
  )
  from .dbtv import DBTVIE
  from .dctp import DctpTvIE
 -from .deezer import DeezerPlaylistIE
 +from .deezer import (
 +    DeezerPlaylistIE,
 +    DeezerAlbumIE,
 +)
  from .democracynow import DemocracynowIE
  from .dfb import DFBIE
  from .dhm import DHMIE
@@@ -280,6 -272,7 +280,6 @@@ from .douyutv import 
      DouyuTVIE,
  )
  from .dplay import DPlayIE
 -from .dreisat import DreiSatIE
  from .drbonanza import DRBonanzaIE
  from .drtuber import DrTuberIE
  from .drtv import (
@@@ -288,10 -281,6 +288,10 @@@
  )
  from .dtube import DTubeIE
  from .dvtv import DVTVIE
 +from .duboku import (
 +    DubokuIE,
 +    DubokuPlaylistIE
 +)
  from .dumpert import DumpertIE
  from .defense import DefenseGouvFrIE
  from .discovery import DiscoveryIE
@@@ -303,7 -292,6 +303,7 @@@ from .discoverynetworks import Discover
  from .discoveryvr import DiscoveryVRIE
  from .disney import DisneyIE
  from .dispeak import DigitallySpeakingIE
 +from .doodstream import DoodStreamIE
  from .dropbox import DropboxIE
  from .dw import (
      DWIE,
@@@ -451,7 -439,6 +451,7 @@@ from .hotstar import 
  )
  from .howcast import HowcastIE
  from .howstuffworks import HowStuffWorksIE
 +from .hrfensehen import HRFernsehenIE
  from .hrti import (
      HRTiIE,
      HRTiPlaylistIE,
@@@ -510,6 -497,7 +510,6 @@@ from .jeuxvideo import JeuxVideoI
  from .jove import JoveIE
  from .joj import JojIE
  from .jwplatform import JWPlatformIE
 -from .jpopsukitv import JpopsukiIE
  from .kakao import KakaoIE
  from .kaltura import KalturaIE
  from .kanalplay import KanalPlayIE
@@@ -597,7 -585,6 +597,7 @@@ from .lynda import 
      LyndaCourseIE
  )
  from .m6 import M6IE
 +from .magentamusik360 import MagentaMusik360IE
  from .mailru import (
      MailRuIE,
      MailRuMusicIE,
@@@ -649,10 -636,7 +649,10 @@@ from .mixcloud import 
  from .mlb import MLBIE
  from .mnet import MnetIE
  from .moevideo import MoeVideoIE
 -from .mofosex import MofosexIE
 +from .mofosex import (
 +    MofosexIE,
 +    MofosexEmbedIE,
 +)
  from .mojvideo import MojvideoIE
  from .morningstar import MorningstarIE
  from .motherless import (
@@@ -680,7 -664,6 +680,7 @@@ from .myvi import 
      MyviIE,
      MyviEmbedIE,
  )
 +from .myvideoge import MyVideoGeIE
  from .myvidster import MyVidsterIE
  from .nationalgeographic import (
      NationalGeographicVideoIE,
@@@ -818,16 -801,6 +818,16 @@@ from .orf import 
      ORFFM4IE,
      ORFFM4StoryIE,
      ORFOE1IE,
 +    ORFOE3IE,
 +    ORFNOEIE,
 +    ORFWIEIE,
 +    ORFBGLIE,
 +    ORFOOEIE,
 +    ORFSTMIE,
 +    ORFKTNIE,
 +    ORFSBGIE,
 +    ORFTIRIE,
 +    ORFVBGIE,
      ORFIPTVIE,
  )
  from .outsidetv import OutsideTVIE
@@@ -835,6 -808,7 +835,6 @@@ from .packtpub import 
      PacktPubIE,
      PacktPubCourseIE,
  )
 -from .pandatv import PandaTVIE
  from .pandoratv import PandoraTVIE
  from .parliamentliveuk import ParliamentLiveUKIE
  from .patreon import PatreonIE
@@@ -872,15 -846,11 +872,15 @@@ from .pluralsight import 
      PluralsightCourseIE,
  )
  from .podomatic import PodomaticIE
 -from .pokemon import PokemonIE
 +from .pokemon import (
 +    PokemonIE,
 +    PokemonWatchIE,
 +)
  from .polskieradio import (
      PolskieRadioIE,
      PolskieRadioCategoryIE,
  )
 +from .popcorntimes import PopcorntimesIE
  from .popcorntv import PopcornTVIE
  from .porn91 import Porn91IE
  from .porncom import PornComIE
@@@ -993,10 -963,7 +993,10 @@@ from .savefrom import SaveFromI
  from .sbs import SBSIE
  from .screencast import ScreencastIE
  from .screencastomatic import ScreencastOMaticIE
 -from .scrippsnetworks import ScrippsNetworksWatchIE
 +from .scrippsnetworks import (
 +    ScrippsNetworksWatchIE,
 +    ScrippsNetworksIE,
 +)
  from .scte import (
      SCTEIE,
      SCTECourseIE,
@@@ -1074,11 -1041,6 +1074,11 @@@ from .spike import 
      BellatorIE,
      ParamountNetworkIE,
  )
 +from .storyfire import (
 +    StoryFireIE,
 +    StoryFireUserIE,
 +    StoryFireSeriesIE,
 +)
  from .stitcher import StitcherIE
  from .sport5 import Sport5IE
  from .sportbox import SportBoxIE
@@@ -1165,10 -1127,7 +1165,7 @@@ from .thisamericanlife import ThisAmeri
  from .thisav import ThisAVIE
  from .thisoldhouse import ThisOldHouseIE
  from .threeqsdn import ThreeQSDNIE
- from .tiktok import (
-     TikTokIE,
-     TikTokUserIE,
- )
+ from .tiktok import TikTokIE
  from .tinypic import TinyPicIE
  from .tmz import (
      TMZIE,
@@@ -1229,7 -1188,6 +1226,7 @@@ from .tvnet import TVNetI
  from .tvnoe import TVNoeIE
  from .tvnow import (
      TVNowIE,
 +    TVNowFilmIE,
      TVNowNewIE,
      TVNowSeasonIE,
      TVNowAnnualIE,
@@@ -1252,11 -1210,14 +1249,11 @@@ from .twentymin import TwentyMinutenI
  from .twentythreevideo import TwentyThreeVideoIE
  from .twitcasting import TwitCastingIE
  from .twitch import (
 -    TwitchVideoIE,
 -    TwitchChapterIE,
      TwitchVodIE,
 -    TwitchProfileIE,
 -    TwitchAllVideosIE,
 -    TwitchUploadsIE,
 -    TwitchPastBroadcastsIE,
 -    TwitchHighlightsIE,
 +    TwitchCollectionIE,
 +    TwitchVideosIE,
 +    TwitchVideosClipsIE,
 +    TwitchVideosCollectionsIE,
      TwitchStreamIE,
      TwitchClipsIE,
  )

diff --combined youtube_dlc/extractor/tiktok.py
index 66088b9abd48905e0abe884d8d7af5787f990ff4,075a2cdf93a880efc984218a40217fc438a76bcb..075a2cdf93a880efc984218a40217fc438a76bcb
--- 1/youtube_dlc/extractor/tiktok.py
--- 2/youtube_dl/extractor/tiktok.py
+++ b/youtube_dlc/extractor/tiktok.py
@@@ -1,138 -1,136 +1,136 @@@
  # coding: utf-8
  from __future__ import unicode_literals
+ from datetime import datetime
  
  from .common import InfoExtractor
  from ..utils import (
-     compat_str,
      ExtractorError,
      int_or_none,
      str_or_none,
-     try_get,
-     url_or_none,
+     try_get
  )
  
  
  class TikTokBaseIE(InfoExtractor):
-     def _extract_aweme(self, data):
-         video = data['video']
-         description = str_or_none(try_get(data, lambda x: x['desc']))
-         width = int_or_none(try_get(data, lambda x: video['width']))
-         height = int_or_none(try_get(data, lambda x: video['height']))
+     def _extract_aweme(self, video_data, webpage):
+         video_info = try_get(
+             video_data, lambda x: x['videoData']['itemInfos'], dict)
+         author_info = try_get(
+             video_data, lambda x: x['videoData']['authorInfos'], dict)
+         share_info = try_get(video_data, lambda x: x['shareMeta'], dict)
  
-         format_urls = set()
-         formats = []
-         for format_id in (
-                 'play_addr_lowbr', 'play_addr', 'play_addr_h264',
-                 'download_addr'):
-             for format in try_get(
-                     video, lambda x: x[format_id]['url_list'], list) or []:
-                 format_url = url_or_none(format)
-                 if not format_url:
-                     continue
-                 if format_url in format_urls:
-                     continue
-                 format_urls.add(format_url)
-                 formats.append({
-                     'url': format_url,
-                     'ext': 'mp4',
-                     'height': height,
-                     'width': width,
-                 })
-         self._sort_formats(formats)
+         unique_id = str_or_none(author_info.get('uniqueId'))
+         timestamp = try_get(video_info, lambda x: int(x['createTime']), int)
+         date = datetime.fromtimestamp(timestamp).strftime('%Y%m%d')
  
-         thumbnail = url_or_none(try_get(
-             video, lambda x: x['cover']['url_list'][0], compat_str))
-         uploader = try_get(data, lambda x: x['author']['nickname'], compat_str)
-         timestamp = int_or_none(data.get('create_time'))
-         comment_count = int_or_none(data.get('comment_count')) or int_or_none(
-             try_get(data, lambda x: x['statistics']['comment_count']))
-         repost_count = int_or_none(try_get(
-             data, lambda x: x['statistics']['share_count']))
+         height = try_get(video_info, lambda x: x['video']['videoMeta']['height'], int)
+         width = try_get(video_info, lambda x: x['video']['videoMeta']['width'], int)
+         thumbnails = []
+         thumbnails.append({
+             'url': video_info.get('thumbnail') or self._og_search_thumbnail(webpage),
+             'width': width,
+             'height': height
+         })
  
-         aweme_id = data['aweme_id']
+         formats = []
+         formats.append({
+             'url': try_get(video_info, lambda x: x['video']['urls'][0]),
+             'ext': 'mp4',
+             'height': height,
+             'width': width
+         })
  
          return {
-             'id': aweme_id,
-             'title': uploader or aweme_id,
-             'description': description,
-             'thumbnail': thumbnail,
-             'uploader': uploader,
+             'comment_count': int_or_none(video_info.get('commentCount')),
+             'duration': try_get(video_info, lambda x: x['video']['videoMeta']['duration'], int),
+             'height': height,
+             'id': str_or_none(video_info.get('id')),
+             'like_count': int_or_none(video_info.get('diggCount')),
+             'repost_count': int_or_none(video_info.get('shareCount')),
+             'thumbnail': try_get(video_info, lambda x: x['covers'][0]),
              'timestamp': timestamp,
-             'comment_count': comment_count,
-             'repost_count': repost_count,
-             'formats': formats,
+             'width': width,
+             'title': str_or_none(share_info.get('title')) or self._og_search_title(webpage),
+             'creator': str_or_none(author_info.get('nickName')),
+             'uploader': unique_id,
+             'uploader_id': str_or_none(author_info.get('userId')),
+             'uploader_url': 'https://www.tiktok.com/@' + unique_id,
+             'thumbnails': thumbnails,
+             'upload_date': date,
+             'webpage_url': self._og_search_url(webpage),
+             'description': str_or_none(video_info.get('text')) or str_or_none(share_info.get('desc')),
+             'ext': 'mp4',
+             'formats': formats
          }
  
  
  class TikTokIE(TikTokBaseIE):
-     _VALID_URL = r'''(?x)
-                         https?://
-                             (?:
-                                 (?:m\.)?tiktok\.com/v|
-                                 (?:www\.)?tiktok\.com/share/video
-                             )
-                             /(?P<id>\d+)
-                     '''
+     _VALID_URL = r'https?://www\.tiktok\.com/@[\w\._]+/video/(?P<id>\d+)'
+ 
      _TESTS = [{
-         'url': 'https://m.tiktok.com/v/6606727368545406213.html',
-         'md5': 'd584b572e92fcd48888051f238022420',
+         'url': 'https://www.tiktok.com/@leenabhushan/video/6748451240264420610',
+         'md5': '34a7543afd5a151b0840ba6736fb633b',
          'info_dict': {
-             'id': '6606727368545406213',
-             'ext': 'mp4',
-             'title': 'Zureeal',
-             'description': '#bowsette#mario#cosplay#uk#lgbt#gaming#asian#bowsettecosplay',
-             'thumbnail': r're:^https?://.*~noop.image',
-             'uploader': 'Zureeal',
-             'timestamp': 1538248586,
-             'upload_date': '20180929',
              'comment_count': int,
+             'creator': 'facestoriesbyleenabh',
+             'description': 'md5:a9f6c0c44a1ff2249cae610372d0ae95',
+             'duration': 13,
+             'ext': 'mp4',
+             'formats': list,
+             'height': 1280,
+             'id': '6748451240264420610',
+             'like_count': int,
              'repost_count': int,
+             'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?',
+             'thumbnails': list,
+             'timestamp': 1571246252,
+             'title': 'facestoriesbyleenabh on TikTok',
+             'upload_date': '20191016',
+             'uploader': 'leenabhushan',
+             'uploader_id': '6691488002098119685',
+             'uploader_url': r're:https://www.tiktok.com/@leenabhushan',
+             'webpage_url': r're:https://www.tiktok.com/@leenabhushan/(video/)?6748451240264420610',
+             'width': 720,
          }
      }, {
-         'url': 'https://www.tiktok.com/share/video/6606727368545406213',
-         'only_matching': True,
+         'url': 'https://www.tiktok.com/@patroxofficial/video/6742501081818877190?langCountry=en',
+         'md5': '06b9800d47d5fe51a19e322dd86e61c9',
+         'info_dict': {
+             'comment_count': int,
+             'creator': 'patroX',
+             'description': 'md5:5e2a23877420bb85ce6521dbee39ba94',
+             'duration': 27,
+             'ext': 'mp4',
+             'formats': list,
+             'height': 960,
+             'id': '6742501081818877190',
+             'like_count': int,
+             'repost_count': int,
+             'thumbnail': r're:^https?://[\w\/\.\-]+(~[\w\-]+\.image)?',
+             'thumbnails': list,
+             'timestamp': 1569860870,
+             'title': 'patroX on TikTok',
+             'upload_date': '20190930',
+             'uploader': 'patroxofficial',
+             'uploader_id': '18702747',
+             'uploader_url': r're:https://www.tiktok.com/@patroxofficial',
+             'webpage_url': r're:https://www.tiktok.com/@patroxofficial/(video/)?6742501081818877190',
+             'width': 540,
+         }
      }]
  
      def _real_extract(self, url):
          video_id = self._match_id(url)
-         webpage = self._download_webpage(
-             'https://m.tiktok.com/v/%s.html' % video_id, video_id)
-         data = self._parse_json(self._search_regex(
-             r'\bdata\s*=\s*({.+?})\s*;', webpage, 'data'), video_id)
-         return self._extract_aweme(data)
  
+         webpage = self._download_webpage(url, video_id, note='Downloading video webpage')
+         json_string = self._search_regex(
+             r'id=\"__NEXT_DATA__\"\s+type=\"application\/json\"\s*[^>]+>\s*(?P<json_string_ld>[^<]+)',
+             webpage, 'json_string', group='json_string_ld')
+         json_data = self._parse_json(json_string, video_id)
+         video_data = try_get(json_data, lambda x: x['props']['pageProps'], expected_type=dict)
  
- class TikTokUserIE(TikTokBaseIE):
-     _VALID_URL = r'''(?x)
-                         https?://
-                             (?:
-                                 (?:m\.)?tiktok\.com/h5/share/usr|
-                                 (?:www\.)?tiktok\.com/share/user
-                             )
-                             /(?P<id>\d+)
-                     '''
-     _TESTS = [{
-         'url': 'https://m.tiktok.com/h5/share/usr/188294915489964032.html',
-         'info_dict': {
-             'id': '188294915489964032',
-         },
-         'playlist_mincount': 24,
-     }, {
-         'url': 'https://www.tiktok.com/share/user/188294915489964032',
-         'only_matching': True,
-     }]
+         # Chech statusCode for success
+         if video_data.get('statusCode') == 0:
+             return self._extract_aweme(video_data, webpage)
  
-     def _real_extract(self, url):
-         user_id = self._match_id(url)
-         data = self._download_json(
-             'https://m.tiktok.com/h5/share/usr/list/%s/' % user_id, user_id,
-             query={'_signature': '_'})
-         entries = []
-         for aweme in data['aweme_list']:
-             try:
-                 entry = self._extract_aweme(aweme)
-             except ExtractorError:
-                 continue
-             entry['extractor_key'] = TikTokIE.ie_key()
-             entries.append(entry)
-         return self.playlist_result(entries, user_id)
+         raise ExtractorError('Video not available', video_id=video_id)