]>
Commit | Line | Data |
---|---|---|
f1823403 HH |
1 | # coding: utf-8\r |
2 | from __future__ import unicode_literals\r | |
3 | \r | |
4 | import json\r | |
5 | \r | |
6 | from .common import InfoExtractor\r | |
7 | from ..compat import compat_str\r | |
8 | from ..utils import (\r | |
9 | ExtractorError,\r | |
10 | parse_iso8601,\r | |
11 | try_get,\r | |
12 | urljoin,\r | |
13 | )\r | |
14 | \r | |
15 | \r | |
16 | class NebulaIE(InfoExtractor):\r | |
17 | \r | |
1ad047d0 | 18 | _VALID_URL = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app)/videos/(?P<id>[-\w]+)'\r |
f1823403 HH |
19 | _TESTS = [\r |
20 | {\r | |
1ad047d0 | 21 | 'url': 'https://nebula.app/videos/that-time-disney-remade-beauty-and-the-beast',\r |
f1823403 HH |
22 | 'md5': 'fe79c4df8b3aa2fea98a93d027465c7e',\r |
23 | 'info_dict': {\r | |
24 | 'id': '5c271b40b13fd613090034fd',\r | |
25 | 'ext': 'mp4',\r | |
26 | 'title': 'That Time Disney Remade Beauty and the Beast',\r | |
27 | 'description': 'Note: this video was originally posted on YouTube with the sponsor read included. We weren’t able to remove it without reducing video quality, so it’s presented here in its original context.',\r | |
28 | 'upload_date': '20180731',\r | |
29 | 'timestamp': 1533009600,\r | |
30 | 'channel': 'Lindsay Ellis',\r | |
31 | 'uploader': 'Lindsay Ellis',\r | |
32 | },\r | |
33 | 'params': {\r | |
34 | 'usenetrc': True,\r | |
35 | },\r | |
36 | 'skip': 'All Nebula content requires authentication',\r | |
37 | },\r | |
38 | {\r | |
1ad047d0 | 39 | 'url': 'https://nebula.app/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',\r |
f1823403 HH |
40 | 'md5': '6d4edd14ce65720fa63aba5c583fb328',\r |
41 | 'info_dict': {\r | |
42 | 'id': '5e7e78171aaf320001fbd6be',\r | |
43 | 'ext': 'mp4',\r | |
44 | 'title': 'Landing Craft - How The Allies Got Ashore',\r | |
45 | 'description': r're:^In this episode we explore the unsung heroes of D-Day, the landing craft.',\r | |
46 | 'upload_date': '20200327',\r | |
47 | 'timestamp': 1585348140,\r | |
48 | 'channel': 'The Logistics of D-Day',\r | |
49 | 'uploader': 'The Logistics of D-Day',\r | |
50 | },\r | |
51 | 'params': {\r | |
52 | 'usenetrc': True,\r | |
53 | },\r | |
54 | 'skip': 'All Nebula content requires authentication',\r | |
55 | },\r | |
56 | {\r | |
1ad047d0 | 57 | 'url': 'https://nebula.app/videos/money-episode-1-the-draw',\r |
f1823403 HH |
58 | 'md5': '8c7d272910eea320f6f8e6d3084eecf5',\r |
59 | 'info_dict': {\r | |
60 | 'id': '5e779ebdd157bc0001d1c75a',\r | |
61 | 'ext': 'mp4',\r | |
62 | 'title': 'Episode 1: The Draw',\r | |
63 | 'description': r'contains:There’s free money on offer… if the players can all work together.',\r | |
64 | 'upload_date': '20200323',\r | |
65 | 'timestamp': 1584980400,\r | |
66 | 'channel': 'Tom Scott Presents: Money',\r | |
67 | 'uploader': 'Tom Scott Presents: Money',\r | |
68 | },\r | |
69 | 'params': {\r | |
70 | 'usenetrc': True,\r | |
71 | },\r | |
72 | 'skip': 'All Nebula content requires authentication',\r | |
73 | },\r | |
1ad047d0 | 74 | {\r |
75 | 'url': 'https://watchnebula.com/videos/money-episode-1-the-draw',\r | |
76 | 'only_matching': True,\r | |
77 | },\r | |
f1823403 HH |
78 | ]\r |
79 | _NETRC_MACHINE = 'watchnebula'\r | |
80 | \r | |
81 | def _retrieve_nebula_auth(self, video_id):\r | |
82 | """\r | |
83 | Log in to Nebula, and returns a Nebula API token\r | |
84 | """\r | |
85 | \r | |
86 | username, password = self._get_login_info()\r | |
87 | if not (username and password):\r | |
88 | self.raise_login_required()\r | |
89 | \r | |
90 | self.report_login()\r | |
91 | data = json.dumps({'email': username, 'password': password}).encode('utf8')\r | |
92 | response = self._download_json(\r | |
93 | 'https://api.watchnebula.com/api/v1/auth/login/',\r | |
94 | data=data, fatal=False, video_id=video_id,\r | |
95 | headers={\r | |
96 | 'content-type': 'application/json',\r | |
97 | # Submitting the 'sessionid' cookie always causes a 403 on auth endpoint\r | |
98 | 'cookie': ''\r | |
99 | },\r | |
100 | note='Authenticating to Nebula with supplied credentials',\r | |
101 | errnote='Authentication failed or rejected')\r | |
102 | if not response or not response.get('key'):\r | |
103 | self.raise_login_required()\r | |
104 | return response['key']\r | |
105 | \r | |
106 | def _retrieve_zype_api_key(self, page_url, display_id):\r | |
107 | """\r | |
108 | Retrieves the Zype API key\r | |
109 | """\r | |
110 | \r | |
111 | # Find the js that has the API key from the webpage and download it\r | |
112 | webpage = self._download_webpage(page_url, video_id=display_id)\r | |
113 | main_script_relpath = self._search_regex(\r | |
114 | r'<script[^>]*src="(?P<script_relpath>[^"]*main.[0-9a-f]*.chunk.js)"[^>]*>', webpage,\r | |
115 | group='script_relpath', name='script relative path', fatal=True)\r | |
116 | main_script_abspath = urljoin(page_url, main_script_relpath)\r | |
117 | main_script = self._download_webpage(main_script_abspath, video_id=display_id,\r | |
118 | note='Retrieving Zype API key')\r | |
119 | \r | |
120 | api_key = self._search_regex(\r | |
121 | r'REACT_APP_ZYPE_API_KEY\s*:\s*"(?P<api_key>[\w-]*)"', main_script,\r | |
122 | group='api_key', name='API key', fatal=True)\r | |
123 | \r | |
124 | return api_key\r | |
125 | \r | |
126 | def _call_zype_api(self, path, params, video_id, api_key, note):\r | |
127 | """\r | |
128 | A helper for making calls to the Zype API.\r | |
129 | """\r | |
130 | query = {'api_key': api_key, 'per_page': 1}\r | |
131 | query.update(params)\r | |
132 | return self._download_json('https://api.zype.com' + path, video_id, query=query, note=note)\r | |
133 | \r | |
134 | def _call_nebula_api(self, path, video_id, access_token, note):\r | |
135 | """\r | |
136 | A helper for making calls to the Nebula API.\r | |
137 | """\r | |
138 | return self._download_json('https://api.watchnebula.com/api/v1' + path, video_id, headers={\r | |
139 | 'Authorization': 'Token {access_token}'.format(access_token=access_token)\r | |
140 | }, note=note)\r | |
141 | \r | |
142 | def _fetch_zype_access_token(self, video_id, nebula_token):\r | |
143 | user_object = self._call_nebula_api('/auth/user/', video_id, nebula_token, note='Retrieving Zype access token')\r | |
144 | access_token = try_get(user_object, lambda x: x['zype_auth_info']['access_token'], compat_str)\r | |
145 | if not access_token:\r | |
146 | if try_get(user_object, lambda x: x['is_subscribed'], bool):\r | |
147 | # TODO: Reimplement the same Zype token polling the Nebula frontend implements\r | |
148 | # see https://github.com/ytdl-org/youtube-dl/pull/24805#issuecomment-749231532\r | |
149 | raise ExtractorError(\r | |
150 | 'Unable to extract Zype access token from Nebula API authentication endpoint. '\r | |
151 | 'Open an arbitrary video in a browser with this account to generate a token',\r | |
152 | expected=True)\r | |
153 | raise ExtractorError('Unable to extract Zype access token from Nebula API authentication endpoint')\r | |
154 | return access_token\r | |
155 | \r | |
156 | def _extract_channel_title(self, video_meta):\r | |
157 | # TODO: Implement the API calls giving us the channel list,\r | |
158 | # so that we can do the title lookup and then figure out the channel URL\r | |
159 | categories = video_meta.get('categories', []) if video_meta else []\r | |
160 | # the channel name is the value of the first category\r | |
161 | for category in categories:\r | |
162 | if category.get('value'):\r | |
163 | return category['value'][0]\r | |
164 | \r | |
165 | def _real_extract(self, url):\r | |
166 | display_id = self._match_id(url)\r | |
167 | nebula_token = self._retrieve_nebula_auth(display_id)\r | |
168 | api_key = self._retrieve_zype_api_key(url, display_id)\r | |
169 | \r | |
170 | response = self._call_zype_api('/videos', {'friendly_title': display_id},\r | |
171 | display_id, api_key, note='Retrieving metadata from Zype')\r | |
172 | if len(response.get('response') or []) != 1:\r | |
173 | raise ExtractorError('Unable to find video on Zype API')\r | |
174 | video_meta = response['response'][0]\r | |
175 | \r | |
176 | video_id = video_meta['_id']\r | |
177 | zype_access_token = self._fetch_zype_access_token(display_id, nebula_token=nebula_token)\r | |
178 | \r | |
179 | channel_title = self._extract_channel_title(video_meta)\r | |
180 | \r | |
181 | return {\r | |
182 | 'id': video_id,\r | |
183 | 'display_id': display_id,\r | |
184 | '_type': 'url_transparent',\r | |
185 | 'ie_key': 'Zype',\r | |
186 | 'url': 'https://player.zype.com/embed/%s.html?access_token=%s' % (video_id, zype_access_token),\r | |
187 | 'title': video_meta.get('title'),\r | |
188 | 'description': video_meta.get('description'),\r | |
189 | 'timestamp': parse_iso8601(video_meta.get('published_at')),\r | |
190 | 'thumbnails': [\r | |
191 | {\r | |
192 | 'id': tn.get('name'), # this appears to be null\r | |
193 | 'url': tn['url'],\r | |
194 | 'width': tn.get('width'),\r | |
195 | 'height': tn.get('height'),\r | |
196 | } for tn in video_meta.get('thumbnails', [])],\r | |
197 | 'duration': video_meta.get('duration'),\r | |
198 | 'channel': channel_title,\r | |
199 | 'uploader': channel_title, # we chose uploader = channel name\r | |
200 | # TODO: uploader_url, channel_id, channel_url\r | |
201 | }\r |