yt_dlp/extractor/minds.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 from .common import InfoExtractor
   5 from ..compat import compat_str
   6 from ..utils import (
   7     clean_html,
   8     format_field,
   9     int_or_none,
  10     str_or_none,
  11     strip_or_none,
  12 )
  13
  14
  15 class MindsBaseIE(InfoExtractor):
  16     _VALID_URL_BASE = r'https?://(?:www\.)?minds\.com/'
  17
  18     def _call_api(self, path, video_id, resource, query=None):
  19         api_url = 'https://www.minds.com/api/' + path
  20         token = self._get_cookies(api_url).get('XSRF-TOKEN')
  21         return self._download_json(
  22             api_url, video_id, 'Downloading %s JSON metadata' % resource, headers={
  23                 'Referer': 'https://www.minds.com/',
  24                 'X-XSRF-TOKEN': token.value if token else '',
  25             }, query=query)
  26
  27
  28 class MindsIE(MindsBaseIE):
  29     IE_NAME = 'minds'
  30     _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'(?:media|newsfeed|archive/view)/(?P<id>[0-9]+)'
  31     _TESTS = [{
  32         'url': 'https://www.minds.com/media/100000000000086822',
  33         'md5': '215a658184a419764852239d4970b045',
  34         'info_dict': {
  35             'id': '100000000000086822',
  36             'ext': 'mp4',
  37             'title': 'Minds intro sequence',
  38             'thumbnail': r're:https?://.+\.png',
  39             'uploader_id': 'ottman',
  40             'upload_date': '20130524',
  41             'timestamp': 1369404826,
  42             'uploader': 'Bill Ottman',
  43             'view_count': int,
  44             'like_count': int,
  45             'dislike_count': int,
  46             'tags': ['animation'],
  47             'comment_count': int,
  48             'license': 'attribution-cc',
  49         },
  50     }, {
  51         # entity.type == 'activity' and empty title
  52         'url': 'https://www.minds.com/newsfeed/798025111988506624',
  53         'md5': 'b2733a74af78d7fd3f541c4cbbaa5950',
  54         'info_dict': {
  55             'id': '798022190320226304',
  56             'ext': 'mp4',
  57             'title': '798022190320226304',
  58             'uploader': 'ColinFlaherty',
  59             'upload_date': '20180111',
  60             'timestamp': 1515639316,
  61             'uploader_id': 'ColinFlaherty',
  62         },
  63     }, {
  64         'url': 'https://www.minds.com/archive/view/715172106794442752',
  65         'only_matching': True,
  66     }, {
  67         # youtube perma_url
  68         'url': 'https://www.minds.com/newsfeed/1197131838022602752',
  69         'only_matching': True,
  70     }]
  71
  72     def _real_extract(self, url):
  73         entity_id = self._match_id(url)
  74         entity = self._call_api(
  75             'v1/entities/entity/' + entity_id, entity_id, 'entity')['entity']
  76         if entity.get('type') == 'activity':
  77             if entity.get('custom_type') == 'video':
  78                 video_id = entity['entity_guid']
  79             else:
  80                 return self.url_result(entity['perma_url'])
  81         else:
  82             assert(entity['subtype'] == 'video')
  83             video_id = entity_id
  84         # 1080p and webm formats available only on the sources array
  85         video = self._call_api(
  86             'v2/media/video/' + video_id, video_id, 'video')
  87
  88         formats = []
  89         for source in (video.get('sources') or []):
  90             src = source.get('src')
  91             if not src:
  92                 continue
  93             formats.append({
  94                 'format_id': source.get('label'),
  95                 'height': int_or_none(source.get('size')),
  96                 'url': src,
  97             })
  98         self._sort_formats(formats)
  99
 100         entity = video.get('entity') or entity
 101         owner = entity.get('ownerObj') or {}
 102         uploader_id = owner.get('username')
 103
 104         tags = entity.get('tags')
 105         if tags and isinstance(tags, compat_str):
 106             tags = [tags]
 107
 108         thumbnail = None
 109         poster = video.get('poster') or entity.get('thumbnail_src')
 110         if poster:
 111             urlh = self._request_webpage(poster, video_id, fatal=False)
 112             if urlh:
 113                 thumbnail = urlh.geturl()
 114
 115         return {
 116             'id': video_id,
 117             'title': entity.get('title') or video_id,
 118             'formats': formats,
 119             'description': clean_html(entity.get('description')) or None,
 120             'license': str_or_none(entity.get('license')),
 121             'timestamp': int_or_none(entity.get('time_created')),
 122             'uploader': strip_or_none(owner.get('name')),
 123             'uploader_id': uploader_id,
 124             'uploader_url': format_field(uploader_id, template='https://www.minds.com/%s'),
 125             'view_count': int_or_none(entity.get('play:count')),
 126             'like_count': int_or_none(entity.get('thumbs:up:count')),
 127             'dislike_count': int_or_none(entity.get('thumbs:down:count')),
 128             'tags': tags,
 129             'comment_count': int_or_none(entity.get('comments:count')),
 130             'thumbnail': thumbnail,
 131         }
 132
 133
 134 class MindsFeedBaseIE(MindsBaseIE):
 135     _PAGE_SIZE = 150
 136
 137     def _entries(self, feed_id):
 138         query = {'limit': self._PAGE_SIZE, 'sync': 1}
 139         i = 1
 140         while True:
 141             data = self._call_api(
 142                 'v2/feeds/container/%s/videos' % feed_id,
 143                 feed_id, 'page %s' % i, query)
 144             entities = data.get('entities') or []
 145             for entity in entities:
 146                 guid = entity.get('guid')
 147                 if not guid:
 148                     continue
 149                 yield self.url_result(
 150                     'https://www.minds.com/newsfeed/' + guid,
 151                     MindsIE.ie_key(), guid)
 152             query['from_timestamp'] = data['load-next']
 153             if not (query['from_timestamp'] and len(entities) == self._PAGE_SIZE):
 154                 break
 155             i += 1
 156
 157     def _real_extract(self, url):
 158         feed_id = self._match_id(url)
 159         feed = self._call_api(
 160             'v1/%s/%s' % (self._FEED_PATH, feed_id),
 161             feed_id, self._FEED_TYPE)[self._FEED_TYPE]
 162
 163         return self.playlist_result(
 164             self._entries(feed['guid']), feed_id,
 165             strip_or_none(feed.get('name')),
 166             feed.get('briefdescription'))
 167
 168
 169 class MindsChannelIE(MindsFeedBaseIE):
 170     _FEED_TYPE = 'channel'
 171     IE_NAME = 'minds:' + _FEED_TYPE
 172     _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'(?!(?:newsfeed|media|api|archive|groups)/)(?P<id>[^/?&#]+)'
 173     _FEED_PATH = 'channel'
 174     _TEST = {
 175         'url': 'https://www.minds.com/ottman',
 176         'info_dict': {
 177             'id': 'ottman',
 178             'title': 'Bill Ottman',
 179             'description': 'Co-creator & CEO @minds',
 180         },
 181         'playlist_mincount': 54,
 182     }
 183
 184
 185 class MindsGroupIE(MindsFeedBaseIE):
 186     _FEED_TYPE = 'group'
 187     IE_NAME = 'minds:' + _FEED_TYPE
 188     _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'groups/profile/(?P<id>[0-9]+)'
 189     _FEED_PATH = 'groups/group'
 190     _TEST = {
 191         'url': 'https://www.minds.com/groups/profile/785582576369672204/feed/videos',
 192         'info_dict': {
 193             'id': '785582576369672204',
 194             'title': 'Cooking Videos',
 195         },
 196         'playlist_mincount': 1,
 197     }