yt_dlp/extractor/curiositystream.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     int_or_none,
   9     urlencode_postdata,
  10     compat_str,
  11     ExtractorError,
  12 )
  13
  14
  15 class CuriosityStreamBaseIE(InfoExtractor):
  16     _NETRC_MACHINE = 'curiositystream'
  17     _auth_token = None
  18     _API_BASE_URL = 'https://api.curiositystream.com/v1/'
  19
  20     def _handle_errors(self, result):
  21         error = result.get('error', {}).get('message')
  22         if error:
  23             if isinstance(error, dict):
  24                 error = ', '.join(error.values())
  25             raise ExtractorError(
  26                 '%s said: %s' % (self.IE_NAME, error), expected=True)
  27
  28     def _call_api(self, path, video_id, query=None):
  29         headers = {}
  30         if self._auth_token:
  31             headers['X-Auth-Token'] = self._auth_token
  32         result = self._download_json(
  33             self._API_BASE_URL + path, video_id, headers=headers, query=query)
  34         self._handle_errors(result)
  35         return result['data']
  36
  37     def _real_initialize(self):
  38         email, password = self._get_login_info()
  39         if email is None:
  40             return
  41         result = self._download_json(
  42             self._API_BASE_URL + 'login', None, data=urlencode_postdata({
  43                 'email': email,
  44                 'password': password,
  45             }))
  46         self._handle_errors(result)
  47         self._auth_token = result['message']['auth_token']
  48
  49
  50 class CuriosityStreamIE(CuriosityStreamBaseIE):
  51     IE_NAME = 'curiositystream'
  52     _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/video/(?P<id>\d+)'
  53     _TEST = {
  54         'url': 'https://app.curiositystream.com/video/2',
  55         'info_dict': {
  56             'id': '2',
  57             'ext': 'mp4',
  58             'title': 'How Did You Develop The Internet?',
  59             'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
  60         },
  61         'params': {
  62             # m3u8 download
  63             'skip_download': True,
  64         },
  65     }
  66
  67     def _real_extract(self, url):
  68         video_id = self._match_id(url)
  69
  70         formats = []
  71         for encoding_format in ('m3u8', 'mpd'):
  72             media = self._call_api('media/' + video_id, video_id, query={
  73                 'encodingsNew': 'true',
  74                 'encodingsFormat': encoding_format,
  75             })
  76             for encoding in media.get('encodings', []):
  77                 playlist_url = encoding.get('master_playlist_url')
  78                 if encoding_format == 'm3u8':
  79                     # use `m3u8` entry_protocol until EXT-X-MAP is properly supported by `m3u8_native` entry_protocol
  80                     formats.extend(self._extract_m3u8_formats(
  81                         playlist_url, video_id, 'mp4',
  82                         m3u8_id='hls', fatal=False))
  83                 elif encoding_format == 'mpd':
  84                     formats.extend(self._extract_mpd_formats(
  85                         playlist_url, video_id, mpd_id='dash', fatal=False))
  86                 encoding_url = encoding.get('url')
  87                 file_url = encoding.get('file_url')
  88                 if not encoding_url and not file_url:
  89                     continue
  90                 f = {
  91                     'width': int_or_none(encoding.get('width')),
  92                     'height': int_or_none(encoding.get('height')),
  93                     'vbr': int_or_none(encoding.get('video_bitrate')),
  94                     'abr': int_or_none(encoding.get('audio_bitrate')),
  95                     'filesize': int_or_none(encoding.get('size_in_bytes')),
  96                     'vcodec': encoding.get('video_codec'),
  97                     'acodec': encoding.get('audio_codec'),
  98                     'container': encoding.get('container_type'),
  99                 }
 100                 for f_url in (encoding_url, file_url):
 101                     if not f_url:
 102                         continue
 103                     fmt = f.copy()
 104                     rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
 105                     if rtmp:
 106                         fmt.update({
 107                             'url': rtmp.group('url'),
 108                             'play_path': rtmp.group('playpath'),
 109                             'app': rtmp.group('app'),
 110                             'ext': 'flv',
 111                             'format_id': 'rtmp',
 112                         })
 113                     else:
 114                         fmt.update({
 115                             'url': f_url,
 116                             'format_id': 'http',
 117                         })
 118                     formats.append(fmt)
 119         self._sort_formats(formats)
 120
 121         title = media['title']
 122
 123         subtitles = {}
 124         for closed_caption in media.get('closed_captions', []):
 125             sub_url = closed_caption.get('file')
 126             if not sub_url:
 127                 continue
 128             lang = closed_caption.get('code') or closed_caption.get('language') or 'en'
 129             subtitles.setdefault(lang, []).append({
 130                 'url': sub_url,
 131             })
 132
 133         return {
 134             'id': video_id,
 135             'formats': formats,
 136             'title': title,
 137             'description': media.get('description'),
 138             'thumbnail': media.get('image_large') or media.get('image_medium') or media.get('image_small'),
 139             'duration': int_or_none(media.get('duration')),
 140             'tags': media.get('tags'),
 141             'subtitles': subtitles,
 142         }
 143
 144
 145 class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
 146     IE_NAME = 'curiositystream:collection'
 147     _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/(?:collections?|series)/(?P<id>\d+)'
 148     _API_BASE_URL = 'https://api.curiositystream.com/v2/collections/'
 149     _TESTS = [{
 150         'url': 'https://curiositystream.com/collections/86',
 151         'info_dict': {
 152             'id': '86',
 153             'title': 'Staff Picks',
 154             'description': 'Wondering where to start? Here are a few of our favorite series and films... from our couch to yours.',
 155         },
 156         'playlist_mincount': 7,
 157     }, {
 158         'url': 'https://app.curiositystream.com/collection/2',
 159         'info_dict': {
 160             'id': '2',
 161             'title': 'Curious Minds: The Internet',
 162             'description': 'How is the internet shaping our lives in the 21st Century?',
 163         },
 164         'playlist_mincount': 16,
 165     }, {
 166         'url': 'https://curiositystream.com/series/2',
 167         'only_matching': True,
 168     }, {
 169         'url': 'https://curiositystream.com/collections/36',
 170         'only_matching': True,
 171     }]
 172
 173     def _real_extract(self, url):
 174         collection_id = self._match_id(url)
 175         collection = self._call_api(collection_id, collection_id)
 176         entries = []
 177         for media in collection.get('media', []):
 178             media_id = compat_str(media.get('id'))
 179             media_type, ie = ('series', CuriosityStreamCollectionIE) if media.get('is_collection') else ('video', CuriosityStreamIE)
 180             entries.append(self.url_result(
 181                 'https://curiositystream.com/%s/%s' % (media_type, media_id),
 182                 ie=ie.ie_key(), video_id=media_id))
 183         return self.playlist_result(
 184             entries, collection_id,
 185             collection.get('title'), collection.get('description'))