yt_dlp/extractor/curiositystream.py

   1 import re
   2 import urllib.parse
   3
   4 from .common import InfoExtractor
   5 from ..compat import compat_str
   6 from ..utils import ExtractorError, int_or_none, urlencode_postdata
   7
   8
   9 class CuriosityStreamBaseIE(InfoExtractor):
  10     _NETRC_MACHINE = 'curiositystream'
  11     _auth_token = None
  12
  13     def _handle_errors(self, result):
  14         error = result.get('error', {}).get('message')
  15         if error:
  16             if isinstance(error, dict):
  17                 error = ', '.join(error.values())
  18             raise ExtractorError(
  19                 '%s said: %s' % (self.IE_NAME, error), expected=True)
  20
  21     def _call_api(self, path, video_id, query=None):
  22         headers = {}
  23         if not self._auth_token:
  24             auth_cookie = self._get_cookies('https://curiositystream.com').get('auth_token')
  25             if auth_cookie:
  26                 self.write_debug('Obtained auth_token cookie')
  27                 self._auth_token = urllib.parse.unquote(auth_cookie.value)
  28         if self._auth_token:
  29             headers['X-Auth-Token'] = self._auth_token
  30         result = self._download_json(
  31             self._API_BASE_URL + path, video_id, headers=headers, query=query)
  32         self._handle_errors(result)
  33         return result['data']
  34
  35     def _perform_login(self, username, password):
  36         result = self._download_json(
  37             'https://api.curiositystream.com/v1/login', None,
  38             note='Logging in', data=urlencode_postdata({
  39                 'email': username,
  40                 'password': password,
  41             }))
  42         self._handle_errors(result)
  43         CuriosityStreamBaseIE._auth_token = result['message']['auth_token']
  44
  45
  46 class CuriosityStreamIE(CuriosityStreamBaseIE):
  47     IE_NAME = 'curiositystream'
  48     _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/video/(?P<id>\d+)'
  49     _TESTS = [{
  50         'url': 'http://app.curiositystream.com/video/2',
  51         'info_dict': {
  52             'id': '2',
  53             'ext': 'mp4',
  54             'title': 'How Did You Develop The Internet?',
  55             'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
  56             'channel': 'Curiosity Stream',
  57             'categories': ['Technology', 'Interview'],
  58             'average_rating': float,
  59             'series_id': '2',
  60             'thumbnail': r're:https://img.curiositystream.com/.+\.jpg',
  61             'tags': [],
  62             'duration': 158
  63         },
  64         'params': {
  65             # m3u8 download
  66             'skip_download': True,
  67         },
  68     }]
  69
  70     _API_BASE_URL = 'https://api.curiositystream.com/v1/media/'
  71
  72     def _real_extract(self, url):
  73         video_id = self._match_id(url)
  74
  75         formats = []
  76         for encoding_format in ('m3u8', 'mpd'):
  77             media = self._call_api(video_id, video_id, query={
  78                 'encodingsNew': 'true',
  79                 'encodingsFormat': encoding_format,
  80             })
  81             for encoding in media.get('encodings', []):
  82                 playlist_url = encoding.get('master_playlist_url')
  83                 if encoding_format == 'm3u8':
  84                     # use `m3u8` entry_protocol until EXT-X-MAP is properly supported by `m3u8_native` entry_protocol
  85                     formats.extend(self._extract_m3u8_formats(
  86                         playlist_url, video_id, 'mp4',
  87                         m3u8_id='hls', fatal=False))
  88                 elif encoding_format == 'mpd':
  89                     formats.extend(self._extract_mpd_formats(
  90                         playlist_url, video_id, mpd_id='dash', fatal=False))
  91                 encoding_url = encoding.get('url')
  92                 file_url = encoding.get('file_url')
  93                 if not encoding_url and not file_url:
  94                     continue
  95                 f = {
  96                     'width': int_or_none(encoding.get('width')),
  97                     'height': int_or_none(encoding.get('height')),
  98                     'vbr': int_or_none(encoding.get('video_bitrate')),
  99                     'abr': int_or_none(encoding.get('audio_bitrate')),
 100                     'filesize': int_or_none(encoding.get('size_in_bytes')),
 101                     'vcodec': encoding.get('video_codec'),
 102                     'acodec': encoding.get('audio_codec'),
 103                     'container': encoding.get('container_type'),
 104                 }
 105                 for f_url in (encoding_url, file_url):
 106                     if not f_url:
 107                         continue
 108                     fmt = f.copy()
 109                     rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
 110                     if rtmp:
 111                         fmt.update({
 112                             'url': rtmp.group('url'),
 113                             'play_path': rtmp.group('playpath'),
 114                             'app': rtmp.group('app'),
 115                             'ext': 'flv',
 116                             'format_id': 'rtmp',
 117                         })
 118                     else:
 119                         fmt.update({
 120                             'url': f_url,
 121                             'format_id': 'http',
 122                         })
 123                     formats.append(fmt)
 124
 125         title = media['title']
 126
 127         subtitles = {}
 128         for closed_caption in media.get('closed_captions', []):
 129             sub_url = closed_caption.get('file')
 130             if not sub_url:
 131                 continue
 132             lang = closed_caption.get('code') or closed_caption.get('language') or 'en'
 133             subtitles.setdefault(lang, []).append({
 134                 'url': sub_url,
 135             })
 136
 137         return {
 138             'id': video_id,
 139             'formats': formats,
 140             'title': title,
 141             'description': media.get('description'),
 142             'thumbnail': media.get('image_large') or media.get('image_medium') or media.get('image_small'),
 143             'duration': int_or_none(media.get('duration')),
 144             'tags': media.get('tags'),
 145             'subtitles': subtitles,
 146             'channel': media.get('producer'),
 147             'categories': [media.get('primary_category'), media.get('type')],
 148             'average_rating': media.get('rating_percentage'),
 149             'series_id': str(media.get('collection_id') or '') or None,
 150         }
 151
 152
 153 class CuriosityStreamCollectionBaseIE(CuriosityStreamBaseIE):
 154
 155     def _real_extract(self, url):
 156         collection_id = self._match_id(url)
 157         collection = self._call_api(collection_id, collection_id)
 158         entries = []
 159         for media in collection.get('media', []):
 160             media_id = compat_str(media.get('id'))
 161             media_type, ie = ('series', CuriosityStreamSeriesIE) if media.get('is_collection') else ('video', CuriosityStreamIE)
 162             entries.append(self.url_result(
 163                 'https://curiositystream.com/%s/%s' % (media_type, media_id),
 164                 ie=ie.ie_key(), video_id=media_id))
 165         return self.playlist_result(
 166             entries, collection_id,
 167             collection.get('title'), collection.get('description'))
 168
 169
 170 class CuriosityStreamCollectionsIE(CuriosityStreamCollectionBaseIE):
 171     IE_NAME = 'curiositystream:collections'
 172     _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/collections/(?P<id>\d+)'
 173     _API_BASE_URL = 'https://api.curiositystream.com/v2/collections/'
 174     _TESTS = [{
 175         'url': 'https://curiositystream.com/collections/86',
 176         'info_dict': {
 177             'id': '86',
 178             'title': 'Staff Picks',
 179             'description': 'Wondering where to start? Here are a few of our favorite series and films... from our couch to yours.',
 180         },
 181         'playlist_mincount': 7,
 182     }, {
 183         'url': 'https://curiositystream.com/collections/36',
 184         'only_matching': True,
 185     }]
 186
 187
 188 class CuriosityStreamSeriesIE(CuriosityStreamCollectionBaseIE):
 189     IE_NAME = 'curiositystream:series'
 190     _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/(?:series|collection)/(?P<id>\d+)'
 191     _API_BASE_URL = 'https://api.curiositystream.com/v2/series/'
 192     _TESTS = [{
 193         'url': 'https://curiositystream.com/series/2',
 194         'info_dict': {
 195             'id': '2',
 196             'title': 'Curious Minds: The Internet',
 197             'description': 'How is the internet shaping our lives in the 21st Century?',
 198         },
 199         'playlist_mincount': 16,
 200     }, {
 201         'url': 'https://curiositystream.com/collection/2',
 202         'only_matching': True,
 203     }]