]>
Commit | Line | Data |
---|---|---|
be2d40a5 TG |
1 | from __future__ import unicode_literals |
2 | ||
bb5ebd44 | 3 | import re |
7913e0fc | 4 | import json |
bb5ebd44 | 5 | |
be2d40a5 | 6 | from .common import InfoExtractor |
7913e0fc | 7 | from .gigya import GigyaBaseIE |
7913e0fc | 8 | from ..compat import compat_HTTPError |
117589df | 9 | from ..utils import ( |
7913e0fc | 10 | ExtractorError, |
117589df | 11 | strip_or_none, |
7913e0fc | 12 | float_or_none, |
13 | int_or_none, | |
fd62b366 | 14 | merge_dicts, |
7913e0fc | 15 | parse_iso8601, |
117589df | 16 | ) |
be2d40a5 TG |
17 | |
18 | ||
19 | class CanvasIE(InfoExtractor): | |
170d6444 | 20 | _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza)/assets/(?P<id>[^/?#&]+)' |
117589df S |
21 | _TESTS = [{ |
22 | 'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', | |
23 | 'md5': '90139b746a0a9bd7bb631283f6e2a64e', | |
24 | 'info_dict': { | |
25 | 'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', | |
26 | 'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', | |
27 | 'ext': 'flv', | |
28 | 'title': 'Nachtwacht: De Greystook', | |
29 | 'description': 'md5:1db3f5dc4c7109c821261e7512975be7', | |
30 | 'thumbnail': r're:^https?://.*\.jpg$', | |
31 | 'duration': 1468.03, | |
32 | }, | |
33 | 'expected_warnings': ['is not a supported codec', 'Unknown MIME type'], | |
34 | }, { | |
35 | 'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e', | |
36 | 'only_matching': True, | |
37 | }] | |
170d6444 RA |
38 | _HLS_ENTRY_PROTOCOLS_MAP = { |
39 | 'HLS': 'm3u8_native', | |
40 | 'HLS_AES': 'm3u8', | |
41 | } | |
117589df S |
42 | |
43 | def _real_extract(self, url): | |
44 | mobj = re.match(self._VALID_URL, url) | |
45 | site_id, video_id = mobj.group('site_id'), mobj.group('id') | |
46 | ||
47 | data = self._download_json( | |
48 | 'https://mediazone.vrt.be/api/v1/%s/assets/%s' | |
49 | % (site_id, video_id), video_id) | |
50 | ||
51 | title = data['title'] | |
52 | description = data.get('description') | |
53 | ||
54 | formats = [] | |
55 | for target in data['targetUrls']: | |
56 | format_url, format_type = target.get('url'), target.get('type') | |
57 | if not format_url or not format_type: | |
58 | continue | |
170d6444 | 59 | if format_type in self._HLS_ENTRY_PROTOCOLS_MAP: |
117589df | 60 | formats.extend(self._extract_m3u8_formats( |
170d6444 | 61 | format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type], |
117589df S |
62 | m3u8_id=format_type, fatal=False)) |
63 | elif format_type == 'HDS': | |
64 | formats.extend(self._extract_f4m_formats( | |
65 | format_url, video_id, f4m_id=format_type, fatal=False)) | |
66 | elif format_type == 'MPEG_DASH': | |
67 | formats.extend(self._extract_mpd_formats( | |
68 | format_url, video_id, mpd_id=format_type, fatal=False)) | |
69 | elif format_type == 'HSS': | |
70 | formats.extend(self._extract_ism_formats( | |
71 | format_url, video_id, ism_id='mss', fatal=False)) | |
72 | else: | |
73 | formats.append({ | |
74 | 'format_id': format_type, | |
75 | 'url': format_url, | |
76 | }) | |
77 | self._sort_formats(formats) | |
78 | ||
79 | subtitles = {} | |
80 | subtitle_urls = data.get('subtitleUrls') | |
81 | if isinstance(subtitle_urls, list): | |
82 | for subtitle in subtitle_urls: | |
83 | subtitle_url = subtitle.get('url') | |
84 | if subtitle_url and subtitle.get('type') == 'CLOSED': | |
85 | subtitles.setdefault('nl', []).append({'url': subtitle_url}) | |
86 | ||
87 | return { | |
88 | 'id': video_id, | |
89 | 'display_id': video_id, | |
90 | 'title': title, | |
91 | 'description': description, | |
92 | 'formats': formats, | |
93 | 'duration': float_or_none(data.get('duration'), 1000), | |
94 | 'thumbnail': data.get('posterImageUrl'), | |
95 | 'subtitles': subtitles, | |
96 | } | |
97 | ||
98 | ||
99 | class CanvasEenIE(InfoExtractor): | |
41b263ac | 100 | IE_DESC = 'canvas.be and een.be' |
bb5ebd44 | 101 | _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)' |
6eff2605 | 102 | _TESTS = [{ |
be2d40a5 | 103 | 'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week', |
117589df | 104 | 'md5': 'ed66976748d12350b118455979cca293', |
be2d40a5 | 105 | 'info_dict': { |
4e2743ab S |
106 | 'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e', |
107 | 'display_id': 'de-afspraak-veilt-voor-de-warmste-week', | |
117589df | 108 | 'ext': 'flv', |
4e2743ab S |
109 | 'title': 'De afspraak veilt voor de Warmste Week', |
110 | 'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6', | |
ec85ded8 | 111 | 'thumbnail': r're:^https?://.*\.jpg$', |
4e2743ab | 112 | 'duration': 49.02, |
117589df S |
113 | }, |
114 | 'expected_warnings': ['is not a supported codec'], | |
6eff2605 S |
115 | }, { |
116 | # with subtitles | |
117 | 'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167', | |
118 | 'info_dict': { | |
119 | 'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625', | |
120 | 'display_id': 'pieter-0167', | |
121 | 'ext': 'mp4', | |
122 | 'title': 'Pieter 0167', | |
123 | 'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e', | |
ec85ded8 | 124 | 'thumbnail': r're:^https?://.*\.jpg$', |
6eff2605 S |
125 | 'duration': 2553.08, |
126 | 'subtitles': { | |
127 | 'nl': [{ | |
128 | 'ext': 'vtt', | |
129 | }], | |
130 | }, | |
131 | }, | |
132 | 'params': { | |
133 | 'skip_download': True, | |
117589df S |
134 | }, |
135 | 'skip': 'Pagina niet gevonden', | |
bb5ebd44 S |
136 | }, { |
137 | 'url': 'https://www.een.be/sorry-voor-alles/herbekijk-sorry-voor-alles', | |
138 | 'info_dict': { | |
139 | 'id': 'mz-ast-11a587f8-b921-4266-82e2-0bce3e80d07f', | |
140 | 'display_id': 'herbekijk-sorry-voor-alles', | |
141 | 'ext': 'mp4', | |
142 | 'title': 'Herbekijk Sorry voor alles', | |
143 | 'description': 'md5:8bb2805df8164e5eb95d6a7a29dc0dd3', | |
ec85ded8 | 144 | 'thumbnail': r're:^https?://.*\.jpg$', |
bb5ebd44 S |
145 | 'duration': 3788.06, |
146 | }, | |
147 | 'params': { | |
148 | 'skip_download': True, | |
117589df S |
149 | }, |
150 | 'skip': 'Episode no longer available', | |
bb5ebd44 S |
151 | }, { |
152 | 'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend', | |
153 | 'only_matching': True, | |
6eff2605 | 154 | }] |
be2d40a5 TG |
155 | |
156 | def _real_extract(self, url): | |
bb5ebd44 S |
157 | mobj = re.match(self._VALID_URL, url) |
158 | site_id, display_id = mobj.group('site_id'), mobj.group('id') | |
be2d40a5 | 159 | |
4e2743ab | 160 | webpage = self._download_webpage(url, display_id) |
be2d40a5 | 161 | |
117589df | 162 | title = strip_or_none(self._search_regex( |
4e2743ab | 163 | r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>', |
bb5ebd44 | 164 | webpage, 'title', default=None) or self._og_search_title( |
117589df | 165 | webpage, default=None)) |
4e2743ab S |
166 | |
167 | video_id = self._html_search_regex( | |
117589df S |
168 | r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id', |
169 | group='id') | |
4e2743ab | 170 | |
be2d40a5 | 171 | return { |
117589df S |
172 | '_type': 'url_transparent', |
173 | 'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id), | |
174 | 'ie_key': CanvasIE.ie_key(), | |
be2d40a5 | 175 | 'id': video_id, |
4e2743ab | 176 | 'display_id': display_id, |
be2d40a5 | 177 | 'title': title, |
4e2743ab | 178 | 'description': self._og_search_description(webpage), |
be2d40a5 | 179 | } |
7913e0fc | 180 | |
181 | ||
182 | class VrtNUIE(GigyaBaseIE): | |
183 | IE_DESC = 'VrtNU.be' | |
184 | _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?P<site_id>vrtnu)/(?:[^/]+/)*(?P<id>[^/?#&]+)' | |
185 | _TESTS = [{ | |
186 | 'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/', | |
187 | 'info_dict': { | |
188 | 'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de', | |
189 | 'ext': 'flv', | |
190 | 'title': 'De zwarte weduwe', | |
191 | 'description': 'md5:d90c21dced7db869a85db89a623998d4', | |
192 | 'duration': 1457.04, | |
193 | 'thumbnail': r're:^https?://.*\.jpg$', | |
194 | 'season': '1', | |
195 | 'season_number': 1, | |
196 | 'episode_number': 1, | |
197 | }, | |
b8c6ffc5 | 198 | 'skip': 'This video is only available for registered users' |
7913e0fc | 199 | }] |
200 | _NETRC_MACHINE = 'vrtnu' | |
201 | _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy' | |
202 | _CONTEXT_ID = 'R3595707040' | |
203 | ||
204 | def _real_initialize(self): | |
205 | self._login() | |
206 | ||
207 | def _login(self): | |
208 | username, password = self._get_login_info() | |
209 | if username is None: | |
b8c6ffc5 | 210 | return |
7913e0fc | 211 | |
212 | auth_data = { | |
213 | 'APIKey': self._APIKEY, | |
214 | 'targetEnv': 'jssdk', | |
215 | 'loginID': username, | |
216 | 'password': password, | |
217 | 'authMode': 'cookie', | |
218 | } | |
219 | ||
220 | auth_info = self._gigya_login(auth_data) | |
221 | ||
222 | # Sometimes authentication fails for no good reason, retry | |
223 | login_attempt = 1 | |
224 | while login_attempt <= 3: | |
225 | try: | |
226 | # When requesting a token, no actual token is returned, but the | |
227 | # necessary cookies are set. | |
228 | self._request_webpage( | |
229 | 'https://token.vrt.be', | |
230 | None, note='Requesting a token', errnote='Could not get a token', | |
231 | headers={ | |
232 | 'Content-Type': 'application/json', | |
233 | 'Referer': 'https://www.vrt.be/vrtnu/', | |
234 | }, | |
235 | data=json.dumps({ | |
236 | 'uid': auth_info['UID'], | |
237 | 'uidsig': auth_info['UIDSignature'], | |
238 | 'ts': auth_info['signatureTimestamp'], | |
239 | 'email': auth_info['profile']['email'], | |
240 | }).encode('utf-8')) | |
241 | except ExtractorError as e: | |
242 | if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401: | |
243 | login_attempt += 1 | |
244 | self.report_warning('Authentication failed') | |
245 | self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again') | |
246 | else: | |
247 | raise e | |
248 | else: | |
249 | break | |
250 | ||
251 | def _real_extract(self, url): | |
252 | display_id = self._match_id(url) | |
253 | ||
64f34528 | 254 | webpage, urlh = self._download_webpage_handle(url, display_id) |
7913e0fc | 255 | |
fd62b366 S |
256 | info = self._search_json_ld(webpage, display_id, default={}) |
257 | ||
258 | # title is optional here since it may be extracted by extractor | |
259 | # that is delegated from here | |
260 | title = strip_or_none(self._html_search_regex( | |
7913e0fc | 261 | r'(?ms)<h1 class="content__heading">(.+?)</h1>', |
fd62b366 | 262 | webpage, 'title', default=None)) |
7913e0fc | 263 | |
264 | description = self._html_search_regex( | |
265 | r'(?ms)<div class="content__description">(.+?)</div>', | |
266 | webpage, 'description', default=None) | |
267 | ||
268 | season = self._html_search_regex( | |
269 | [r'''(?xms)<div\ class="tabs__tab\ tabs__tab--active">\s* | |
270 | <span>seizoen\ (.+?)</span>\s* | |
271 | </div>''', | |
272 | r'<option value="seizoen (\d{1,3})" data-href="[^"]+?" selected>'], | |
273 | webpage, 'season', default=None) | |
274 | ||
275 | season_number = int_or_none(season) | |
276 | ||
277 | episode_number = int_or_none(self._html_search_regex( | |
278 | r'''(?xms)<div\ class="content__episode">\s* | |
279 | <abbr\ title="aflevering">afl</abbr>\s*<span>(\d+)</span> | |
280 | </div>''', | |
281 | webpage, 'episode_number', default=None)) | |
282 | ||
283 | release_date = parse_iso8601(self._html_search_regex( | |
284 | r'(?ms)<div class="content__broadcastdate">\s*<time\ datetime="(.+?)"', | |
285 | webpage, 'release_date', default=None)) | |
286 | ||
287 | # If there's a ? or a # in the URL, remove them and everything after | |
64f34528 | 288 | clean_url = urlh.geturl().split('?')[0].split('#')[0].strip('/') |
7913e0fc | 289 | securevideo_url = clean_url + '.mssecurevideo.json' |
290 | ||
b8c6ffc5 S |
291 | try: |
292 | video = self._download_json(securevideo_url, display_id) | |
293 | except ExtractorError as e: | |
294 | if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401: | |
295 | self.raise_login_required() | |
296 | raise | |
297 | ||
7913e0fc | 298 | # We are dealing with a '../<show>.relevant' URL |
b8c6ffc5 | 299 | redirect_url = video.get('url') |
7913e0fc | 300 | if redirect_url: |
b8c6ffc5 S |
301 | return self.url_result(self._proto_relative_url(redirect_url, 'https:')) |
302 | ||
303 | # There is only one entry, but with an unknown key, so just get | |
304 | # the first one | |
305 | video_id = list(video.values())[0].get('videoid') | |
7913e0fc | 306 | |
fd62b366 | 307 | return merge_dicts(info, { |
7913e0fc | 308 | '_type': 'url_transparent', |
309 | 'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id, | |
310 | 'ie_key': CanvasIE.ie_key(), | |
311 | 'id': video_id, | |
312 | 'display_id': display_id, | |
313 | 'title': title, | |
314 | 'description': description, | |
315 | 'season': season, | |
316 | 'season_number': season_number, | |
317 | 'episode_number': episode_number, | |
318 | 'release_date': release_date, | |
fd62b366 | 319 | }) |