2 from __future__
import unicode_literals
7 from urllib
.error
import HTTPError
8 from .common
import InfoExtractor
9 from ..compat
import compat_str
, compat_urllib_parse_unquote
, compat_urllib_parse_quote
18 class NebulaIE(InfoExtractor
):
20 _VALID_URL
= r
'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app)/videos/(?P<id>[-\w]+)'
23 'url': 'https://nebula.app/videos/that-time-disney-remade-beauty-and-the-beast',
24 'md5': 'fe79c4df8b3aa2fea98a93d027465c7e',
26 'id': '5c271b40b13fd613090034fd',
28 'title': 'That Time Disney Remade Beauty and the Beast',
29 'description': 'Note: this video was originally posted on YouTube with the sponsor read included. We weren’t able to remove it without reducing video quality, so it’s presented here in its original context.',
30 'upload_date': '20180731',
31 'timestamp': 1533009600,
32 'channel': 'Lindsay Ellis',
33 'uploader': 'Lindsay Ellis',
38 'skip': 'All Nebula content requires authentication',
41 'url': 'https://nebula.app/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
42 'md5': '6d4edd14ce65720fa63aba5c583fb328',
44 'id': '5e7e78171aaf320001fbd6be',
46 'title': 'Landing Craft - How The Allies Got Ashore',
47 'description': r
're:^In this episode we explore the unsung heroes of D-Day, the landing craft.',
48 'upload_date': '20200327',
49 'timestamp': 1585348140,
50 'channel': 'The Logistics of D-Day',
51 'uploader': 'The Logistics of D-Day',
56 'skip': 'All Nebula content requires authentication',
59 'url': 'https://nebula.app/videos/money-episode-1-the-draw',
60 'md5': '8c7d272910eea320f6f8e6d3084eecf5',
62 'id': '5e779ebdd157bc0001d1c75a',
64 'title': 'Episode 1: The Draw',
65 'description': r
'contains:There’s free money on offer… if the players can all work together.',
66 'upload_date': '20200323',
67 'timestamp': 1584980400,
68 'channel': 'Tom Scott Presents: Money',
69 'uploader': 'Tom Scott Presents: Money',
74 'skip': 'All Nebula content requires authentication',
77 'url': 'https://watchnebula.com/videos/money-episode-1-the-draw',
78 'only_matching': True,
81 _NETRC_MACHINE
= 'watchnebula'
85 def _retrieve_nebula_auth(self
):
87 Log in to Nebula, and returns a Nebula API token
90 username
, password
= self
._get
_login
_info
()
91 if not (username
and password
):
92 self
.raise_login_required()
95 data
= json
.dumps({'email': username, 'password': password}
).encode('utf8')
96 response
= self
._download
_json
(
97 'https://api.watchnebula.com/api/v1/auth/login/',
98 data
=data
, fatal
=False, video_id
=None,
100 'content-type': 'application/json',
101 # Submitting the 'sessionid' cookie always causes a 403 on auth endpoint
104 note
='Authenticating to Nebula with supplied credentials',
105 errnote
='Authentication failed or rejected')
106 if not response
or not response
.get('key'):
107 self
.raise_login_required()
109 # save nebula token as cookie
111 'nebula.app', 'nebula-auth',
112 compat_urllib_parse_quote(
114 "apiToken": response
["key"],
115 "isLoggingIn": False,
116 "isLoggingOut": False,
117 }, separators
=(",", ":"))),
118 expire_time
=int(time
.time()) + 86400 * 365,
121 return response
['key']
123 def _retrieve_zype_api_key(self
, page_url
, display_id
):
125 Retrieves the Zype API key
128 # Find the js that has the API key from the webpage and download it
129 webpage
= self
._download
_webpage
(page_url
, video_id
=display_id
)
130 main_script_relpath
= self
._search
_regex
(
131 r
'<script[^>]*src="(?P<script_relpath>[^"]*main.[0-9a-f]*.chunk.js)"[^>]*>', webpage
,
132 group
='script_relpath', name
='script relative path', fatal
=True)
133 main_script_abspath
= urljoin(page_url
, main_script_relpath
)
134 main_script
= self
._download
_webpage
(main_script_abspath
, video_id
=display_id
,
135 note
='Retrieving Zype API key')
137 api_key
= self
._search
_regex
(
138 r
'REACT_APP_ZYPE_API_KEY\s*:\s*"(?P<api_key>[\w-]*)"', main_script
,
139 group
='api_key', name
='API key', fatal
=True)
143 def _call_zype_api(self
, path
, params
, video_id
, api_key
, note
):
145 A helper for making calls to the Zype API.
147 query
= {'api_key': api_key, 'per_page': 1}
149 return self
._download
_json
('https://api.zype.com' + path
, video_id
, query
=query
, note
=note
)
151 def _call_nebula_api(self
, path
, video_id
, access_token
, note
):
153 A helper for making calls to the Nebula API.
155 return self
._download
_json
('https://api.watchnebula.com/api/v1' + path
, video_id
, headers
={
156 'Authorization': 'Token {access_token}'.format(access_token
=access_token
)
159 def _fetch_zype_access_token(self
, video_id
):
161 user_object
= self
._call
_nebula
_api
('/auth/user/', video_id
, self
._nebula
_token
, note
='Retrieving Zype access token')
162 except ExtractorError
as exc
:
163 # if 401, attempt credential auth and retry
164 if exc
.cause
and isinstance(exc
.cause
, HTTPError
) and exc
.cause
.code
== 401:
165 self
._nebula
_token
= self
._retrieve
_nebula
_auth
()
166 user_object
= self
._call
_nebula
_api
('/auth/user/', video_id
, self
._nebula
_token
, note
='Retrieving Zype access token')
170 access_token
= try_get(user_object
, lambda x
: x
['zype_auth_info']['access_token'], compat_str
)
172 if try_get(user_object
, lambda x
: x
['is_subscribed'], bool):
173 # TODO: Reimplement the same Zype token polling the Nebula frontend implements
174 # see https://github.com/ytdl-org/youtube-dl/pull/24805#issuecomment-749231532
175 raise ExtractorError(
176 'Unable to extract Zype access token from Nebula API authentication endpoint. '
177 'Open an arbitrary video in a browser with this account to generate a token',
179 raise ExtractorError('Unable to extract Zype access token from Nebula API authentication endpoint')
182 def _extract_channel_title(self
, video_meta
):
183 # TODO: Implement the API calls giving us the channel list,
184 # so that we can do the title lookup and then figure out the channel URL
185 categories
= video_meta
.get('categories', []) if video_meta
else []
186 # the channel name is the value of the first category
187 for category
in categories
:
188 if category
.get('value'):
189 return category
['value'][0]
191 def _real_initialize(self
):
192 # check cookie jar for valid token
193 nebula_cookies
= self
._get
_cookies
('https://nebula.app')
194 nebula_cookie
= nebula_cookies
.get('nebula-auth')
196 self
.to_screen('Authenticating to Nebula with token from cookie jar')
197 nebula_cookie_value
= compat_urllib_parse_unquote(nebula_cookie
.value
)
198 self
._nebula
_token
= self
._parse
_json
(nebula_cookie_value
, None).get('apiToken')
200 # try to authenticate using credentials if no valid token has been found
201 if not self
._nebula
_token
:
202 self
._nebula
_token
= self
._retrieve
_nebula
_auth
()
204 def _real_extract(self
, url
):
205 display_id
= self
._match
_id
(url
)
206 api_key
= self
._retrieve
_zype
_api
_key
(url
, display_id
)
208 response
= self
._call
_zype
_api
('/videos', {'friendly_title': display_id}
,
209 display_id
, api_key
, note
='Retrieving metadata from Zype')
210 if len(response
.get('response') or []) != 1:
211 raise ExtractorError('Unable to find video on Zype API')
212 video_meta
= response
['response'][0]
214 video_id
= video_meta
['_id']
215 zype_access_token
= self
._fetch
_zype
_access
_token
(display_id
)
217 channel_title
= self
._extract
_channel
_title
(video_meta
)
221 'display_id': display_id
,
222 '_type': 'url_transparent',
224 'url': 'https://player.zype.com/embed/%s.html?access_token=%s' % (video_id
, zype_access_token
),
225 'title': video_meta
.get('title'),
226 'description': video_meta
.get('description'),
227 'timestamp': parse_iso8601(video_meta
.get('published_at')),
229 'id': tn
.get('name'), # this appears to be null
231 'width': tn
.get('width'),
232 'height': tn
.get('height'),
233 } for tn
in video_meta
.get('thumbnails', [])],
234 'duration': video_meta
.get('duration'),
235 'channel': channel_title
,
236 'uploader': channel_title
, # we chose uploader = channel name
237 # TODO: uploader_url, channel_id, channel_url