youtube_dl/extractor/storyfire.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import itertools
   5 from .common import InfoExtractor
   6
   7
   8 class StoryFireIE(InfoExtractor):
   9     _VALID_URL = r'(?:(?:https?://(?:www\.)?storyfire\.com/video-details)|(?:https://storyfire.app.link))/(?P<id>[^/\s]+)'
  10     _TESTS = [{
  11         'url': 'https://storyfire.com/video-details/5df1d132b6378700117f9181',
  12         'md5': '560953bfca81a69003cfa5e53ac8a920',
  13         'info_dict': {
  14             'id': '5df1d132b6378700117f9181',
  15             'ext': 'mp4',
  16             'title': 'Buzzfeed Teaches You About Memes',
  17             'uploader_id': 'ntZAJFECERSgqHSxzonV5K2E89s1',
  18             'timestamp': 1576129028,
  19             'description': 'Mocking Buzzfeed\'s meme lesson. Reuploaded from YouTube because of their new policies',
  20             'uploader': 'whang!',
  21             'upload_date': '20191212',
  22         },
  23         'params': {'format': 'bestvideo'}  # There are no merged formats in the playlist.
  24     }, {
  25         'url': 'https://storyfire.app.link/5GxAvWOQr8',  # Alternate URL format, with unrelated short ID
  26         'md5': '7a2dc6d60c4889edfed459c620fe690d',
  27         'info_dict': {
  28             'id': '5f1e11ecd78a57b6c702001d',
  29             'ext': 'm4a',
  30             'title': 'Weird Nintendo Prototype Leaks',
  31             'description': 'A stream taking a look at some weird Nintendo Prototypes with Luigi in Mario 64 and weird Yoshis',
  32             'timestamp': 1595808576,
  33             'upload_date': '20200727',
  34             'uploader': 'whang!',
  35             'uploader_id': 'ntZAJFECERSgqHSxzonV5K2E89s1',
  36         },
  37         'params': {'format': 'bestaudio'}  # Verifying audio extraction
  38
  39     }]
  40
  41     _aformats = {
  42         'audio-medium-audio': {'acodec': 'aac', 'abr': 125, 'preference': -10},
  43         'audio-high-audio': {'acodec': 'aac', 'abr': 254, 'preference': -1},
  44     }
  45
  46     def _real_extract(self, url):
  47         video_id = self._match_id(url)
  48         webpage = self._download_webpage(url, video_id)
  49
  50         # Extracting the json blob is mandatory to proceed with extraction.
  51         jsontext = self._html_search_regex(
  52             r'<script id="__NEXT_DATA__" type="application/json">(.+?)</script>',
  53             webpage, 'json_data')
  54
  55         json = self._parse_json(jsontext, video_id)
  56
  57         # The currentVideo field in the json is mandatory
  58         # because it contains the only link to the m3u playlist
  59         video = json['props']['initialState']['video']['currentVideo']
  60         videourl = video['vimeoVideoURL']  # Video URL is mandatory
  61
  62         # Extract other fields from the json in an error tolerant fashion
  63         # ID may be incorrect (on short URL format), correct it.
  64         parsed_id = video.get('_id')
  65         if parsed_id:
  66             video_id = parsed_id
  67
  68         title = video.get('title')
  69         description = video.get('description')
  70
  71         thumbnail = video.get('storyImage')
  72         views = video.get('views')
  73         likes = video.get('likesCount')
  74         comments = video.get('commentsCount')
  75         duration = video.get('videoDuration')
  76         publishdate = video.get('publishDate')  # Apparently epoch time, day only
  77
  78         uploader = video.get('username')
  79         uploader_id = video.get('hostID')
  80         # Construct an uploader URL
  81         uploader_url = None
  82         if uploader_id:
  83             uploader_url = "https://storyfire.com/user/%s/video" % uploader_id
  84
  85         # Collect root playlist to determine formats
  86         formats = self._extract_m3u8_formats(
  87             videourl, video_id, 'mp4', 'm3u8_native')
  88
  89         # Modify formats to fill in missing information about audio codecs
  90         for format in formats:
  91             aformat = self._aformats.get(format['format_id'])
  92             if aformat:
  93                 format['acodec'] = aformat['acodec']
  94                 format['abr'] = aformat['abr']
  95                 format['preference'] = aformat['preference']
  96                 format['ext'] = 'm4a'
  97
  98         self._sort_formats(formats)
  99
 100         return {
 101             'id': video_id,
 102             'title': title,
 103             'description': description,
 104             'ext': "mp4",
 105             'url': videourl,
 106             'formats': formats,
 107
 108             'thumbnail': thumbnail,
 109             'view_count': views,
 110             'like_count': likes,
 111             'comment_count': comments,
 112             'duration': duration,
 113             'timestamp': publishdate,
 114
 115             'uploader': uploader,
 116             'uploader_id': uploader_id,
 117             'uploader_url': uploader_url,
 118
 119         }
 120
 121
 122 class StoryFireUserIE(InfoExtractor):
 123     _VALID_URL = r'https?://(?:www\.)?storyfire\.com/user/(?P<id>[^/\s]+)/video'
 124     _TESTS = [{
 125         'url': 'https://storyfire.com/user/ntZAJFECERSgqHSxzonV5K2E89s1/video',
 126         'info_dict': {
 127             'id': 'ntZAJFECERSgqHSxzonV5K2E89s1',
 128             'title': 'whang!',
 129         },
 130         'playlist_mincount': 18
 131     }, {
 132         'url': 'https://storyfire.com/user/UQ986nFxmAWIgnkZQ0ftVhq4nOk2/video',
 133         'info_dict': {
 134             'id': 'UQ986nFxmAWIgnkZQ0ftVhq4nOk2',
 135             'title': 'McJuggerNuggets',
 136         },
 137         'playlist_mincount': 143
 138
 139     }]
 140
 141     # Generator for fetching playlist items
 142     def _enum_videos(self, baseurl, user_id, firstjson):
 143         totalVideos = int(firstjson['videosCount'])
 144         haveVideos = 0
 145         json = firstjson
 146
 147         for page in itertools.count(1):
 148             for video in json['videos']:
 149                 id = video['_id']
 150                 url = "https://storyfire.com/video-details/%s" % id
 151                 haveVideos += 1
 152                 yield {
 153                     '_type': 'url',
 154                     'id': id,
 155                     'url': url,
 156                     'ie_key': 'StoryFire',
 157
 158                     'title': video.get('title'),
 159                     'description': video.get('description'),
 160                     'view_count': video.get('views'),
 161                     'comment_count': video.get('commentsCount'),
 162                     'duration': video.get('videoDuration'),
 163                     'timestamp': video.get('publishDate'),
 164                 }
 165             # Are there more pages we could fetch?
 166             if haveVideos < totalVideos:
 167                 pageurl = baseurl + ("%i" % haveVideos)
 168                 json = self._download_json(pageurl, user_id,
 169                                            note='Downloading page %s' % page)
 170
 171                 # Are there any videos in the new json?
 172                 videos = json.get('videos')
 173                 if not videos or len(videos) == 0:
 174                     break  # no videos
 175
 176             else:
 177                 break  # We have fetched all the videos, stop
 178
 179     def _real_extract(self, url):
 180         user_id = self._match_id(url)
 181
 182         baseurl = "https://storyfire.com/app/publicVideos/%s?skip=" % user_id
 183
 184         # Download first page to ensure it can be downloaded, and get user information if available.
 185         firstpage = baseurl + "0"
 186         firstjson = self._download_json(firstpage, user_id)
 187
 188         title = None
 189         videos = firstjson.get('videos')
 190         if videos and len(videos):
 191             title = videos[1].get('username')
 192
 193         return {
 194             '_type': 'playlist',
 195             'entries': self._enum_videos(baseurl, user_id, firstjson),
 196             'id': user_id,
 197             'title': title,
 198         }
 199
 200
 201 class StoryFireSeriesIE(InfoExtractor):
 202     _VALID_URL = r'https?://(?:www\.)?storyfire\.com/write/series/stories/(?P<id>[^/\s]+)'
 203     _TESTS = [{
 204         'url': 'https://storyfire.com/write/series/stories/-Lq6MsuIHLODO6d2dDkr/',
 205         'info_dict': {
 206             'id': '-Lq6MsuIHLODO6d2dDkr',
 207         },
 208         'playlist_mincount': 13
 209     }, {
 210         'url': 'https://storyfire.com/write/series/stories/the_mortal_one/',
 211         'info_dict': {
 212             'id': 'the_mortal_one',
 213         },
 214         'playlist_count': 0  # This playlist has entries, but no videos.
 215     }, {
 216         'url': 'https://storyfire.com/write/series/stories/story_time',
 217         'info_dict': {
 218             'id': 'story_time',
 219         },
 220         'playlist_mincount': 10
 221     }]
 222
 223     # Generator for returning playlist items
 224     # This object is substantially different than the one in the user videos page above
 225     def _enum_videos(self, jsonlist):
 226         for video in jsonlist:
 227             id = video['_id']
 228             if video.get('hasVideo'):  # Boolean element
 229                 url = "https://storyfire.com/video-details/%s" % id
 230                 yield {
 231                     '_type': 'url',
 232                     'id': id,
 233                     'url': url,
 234                     'ie_key': 'StoryFire',
 235
 236                     'title': video.get('title'),
 237                     'description': video.get('description'),
 238                     'view_count': video.get('views'),
 239                     'likes_count': video.get('likesCount'),
 240                     'comment_count': video.get('commentsCount'),
 241                     'duration': video.get('videoDuration'),
 242                     'timestamp': video.get('publishDate'),
 243                 }
 244
 245     def _real_extract(self, url):
 246         list_id = self._match_id(url)
 247
 248         listurl = "https://storyfire.com/app/seriesStories/%s/list" % list_id
 249         json = self._download_json(listurl, list_id)
 250
 251         return {
 252             '_type': 'playlist',
 253             'entries': self._enum_videos(json),
 254             'id': list_id
 255         }