yt_dlp/extractor/nebula.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import json
   5
   6 from .common import InfoExtractor
   7 from ..compat import compat_str
   8 from ..utils import (
   9     ExtractorError,
  10     parse_iso8601,
  11     try_get,
  12     urljoin,
  13 )
  14
  15
  16 class NebulaIE(InfoExtractor):
  17
  18     _VALID_URL = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app)/videos/(?P<id>[-\w]+)'
  19     _TESTS = [
  20         {
  21             'url': 'https://nebula.app/videos/that-time-disney-remade-beauty-and-the-beast',
  22             'md5': 'fe79c4df8b3aa2fea98a93d027465c7e',
  23             'info_dict': {
  24                 'id': '5c271b40b13fd613090034fd',
  25                 'ext': 'mp4',
  26                 'title': 'That Time Disney Remade Beauty and the Beast',
  27                 'description': 'Note: this video was originally posted on YouTube with the sponsor read included. We weren’t able to remove it without reducing video quality, so it’s presented here in its original context.',
  28                 'upload_date': '20180731',
  29                 'timestamp': 1533009600,
  30                 'channel': 'Lindsay Ellis',
  31                 'uploader': 'Lindsay Ellis',
  32             },
  33             'params': {
  34                 'usenetrc': True,
  35             },
  36             'skip': 'All Nebula content requires authentication',
  37         },
  38         {
  39             'url': 'https://nebula.app/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
  40             'md5': '6d4edd14ce65720fa63aba5c583fb328',
  41             'info_dict': {
  42                 'id': '5e7e78171aaf320001fbd6be',
  43                 'ext': 'mp4',
  44                 'title': 'Landing Craft - How The Allies Got Ashore',
  45                 'description': r're:^In this episode we explore the unsung heroes of D-Day, the landing craft.',
  46                 'upload_date': '20200327',
  47                 'timestamp': 1585348140,
  48                 'channel': 'The Logistics of D-Day',
  49                 'uploader': 'The Logistics of D-Day',
  50             },
  51             'params': {
  52                 'usenetrc': True,
  53             },
  54             'skip': 'All Nebula content requires authentication',
  55         },
  56         {
  57             'url': 'https://nebula.app/videos/money-episode-1-the-draw',
  58             'md5': '8c7d272910eea320f6f8e6d3084eecf5',
  59             'info_dict': {
  60                 'id': '5e779ebdd157bc0001d1c75a',
  61                 'ext': 'mp4',
  62                 'title': 'Episode 1: The Draw',
  63                 'description': r'contains:There’s free money on offer… if the players can all work together.',
  64                 'upload_date': '20200323',
  65                 'timestamp': 1584980400,
  66                 'channel': 'Tom Scott Presents: Money',
  67                 'uploader': 'Tom Scott Presents: Money',
  68             },
  69             'params': {
  70                 'usenetrc': True,
  71             },
  72             'skip': 'All Nebula content requires authentication',
  73         },
  74         {
  75             'url': 'https://watchnebula.com/videos/money-episode-1-the-draw',
  76             'only_matching': True,
  77         },
  78     ]
  79     _NETRC_MACHINE = 'watchnebula'
  80
  81     def _retrieve_nebula_auth(self, video_id):
  82         """
  83         Log in to Nebula, and returns a Nebula API token
  84         """
  85
  86         username, password = self._get_login_info()
  87         if not (username and password):
  88             self.raise_login_required()
  89
  90         self.report_login()
  91         data = json.dumps({'email': username, 'password': password}).encode('utf8')
  92         response = self._download_json(
  93             'https://api.watchnebula.com/api/v1/auth/login/',
  94             data=data, fatal=False, video_id=video_id,
  95             headers={
  96                 'content-type': 'application/json',
  97                 # Submitting the 'sessionid' cookie always causes a 403 on auth endpoint
  98                 'cookie': ''
  99             },
 100             note='Authenticating to Nebula with supplied credentials',
 101             errnote='Authentication failed or rejected')
 102         if not response or not response.get('key'):
 103             self.raise_login_required()
 104         return response['key']
 105
 106     def _retrieve_zype_api_key(self, page_url, display_id):
 107         """
 108         Retrieves the Zype API key
 109         """
 110
 111         # Find the js that has the API key from the webpage and download it
 112         webpage = self._download_webpage(page_url, video_id=display_id)
 113         main_script_relpath = self._search_regex(
 114             r'<script[^>]*src="(?P<script_relpath>[^"]*main.[0-9a-f]*.chunk.js)"[^>]*>', webpage,
 115             group='script_relpath', name='script relative path', fatal=True)
 116         main_script_abspath = urljoin(page_url, main_script_relpath)
 117         main_script = self._download_webpage(main_script_abspath, video_id=display_id,
 118                                              note='Retrieving Zype API key')
 119
 120         api_key = self._search_regex(
 121             r'REACT_APP_ZYPE_API_KEY\s*:\s*"(?P<api_key>[\w-]*)"', main_script,
 122             group='api_key', name='API key', fatal=True)
 123
 124         return api_key
 125
 126     def _call_zype_api(self, path, params, video_id, api_key, note):
 127         """
 128         A helper for making calls to the Zype API.
 129         """
 130         query = {'api_key': api_key, 'per_page': 1}
 131         query.update(params)
 132         return self._download_json('https://api.zype.com' + path, video_id, query=query, note=note)
 133
 134     def _call_nebula_api(self, path, video_id, access_token, note):
 135         """
 136         A helper for making calls to the Nebula API.
 137         """
 138         return self._download_json('https://api.watchnebula.com/api/v1' + path, video_id, headers={
 139             'Authorization': 'Token {access_token}'.format(access_token=access_token)
 140         }, note=note)
 141
 142     def _fetch_zype_access_token(self, video_id, nebula_token):
 143         user_object = self._call_nebula_api('/auth/user/', video_id, nebula_token, note='Retrieving Zype access token')
 144         access_token = try_get(user_object, lambda x: x['zype_auth_info']['access_token'], compat_str)
 145         if not access_token:
 146             if try_get(user_object, lambda x: x['is_subscribed'], bool):
 147                 # TODO: Reimplement the same Zype token polling the Nebula frontend implements
 148                 # see https://github.com/ytdl-org/youtube-dl/pull/24805#issuecomment-749231532
 149                 raise ExtractorError(
 150                     'Unable to extract Zype access token from Nebula API authentication endpoint. '
 151                     'Open an arbitrary video in a browser with this account to generate a token',
 152                     expected=True)
 153             raise ExtractorError('Unable to extract Zype access token from Nebula API authentication endpoint')
 154         return access_token
 155
 156     def _extract_channel_title(self, video_meta):
 157         # TODO: Implement the API calls giving us the channel list,
 158         # so that we can do the title lookup and then figure out the channel URL
 159         categories = video_meta.get('categories', []) if video_meta else []
 160         # the channel name is the value of the first category
 161         for category in categories:
 162             if category.get('value'):
 163                 return category['value'][0]
 164
 165     def _real_extract(self, url):
 166         display_id = self._match_id(url)
 167         nebula_token = self._retrieve_nebula_auth(display_id)
 168         api_key = self._retrieve_zype_api_key(url, display_id)
 169
 170         response = self._call_zype_api('/videos', {'friendly_title': display_id},
 171                                        display_id, api_key, note='Retrieving metadata from Zype')
 172         if len(response.get('response') or []) != 1:
 173             raise ExtractorError('Unable to find video on Zype API')
 174         video_meta = response['response'][0]
 175
 176         video_id = video_meta['_id']
 177         zype_access_token = self._fetch_zype_access_token(display_id, nebula_token=nebula_token)
 178
 179         channel_title = self._extract_channel_title(video_meta)
 180
 181         return {
 182             'id': video_id,
 183             'display_id': display_id,
 184             '_type': 'url_transparent',
 185             'ie_key': 'Zype',
 186             'url': 'https://player.zype.com/embed/%s.html?access_token=%s' % (video_id, zype_access_token),
 187             'title': video_meta.get('title'),
 188             'description': video_meta.get('description'),
 189             'timestamp': parse_iso8601(video_meta.get('published_at')),
 190             'thumbnails': [
 191                 {
 192                     'id': tn.get('name'),  # this appears to be null
 193                     'url': tn['url'],
 194                     'width': tn.get('width'),
 195                     'height': tn.get('height'),
 196                 } for tn in video_meta.get('thumbnails', [])],
 197             'duration': video_meta.get('duration'),
 198             'channel': channel_title,
 199             'uploader': channel_title,  # we chose uploader = channel name
 200             # TODO: uploader_url, channel_id, channel_url
 201         }