]>
Commit | Line | Data |
---|---|---|
be2d40a5 | 1 | from __future__ import unicode_literals |
aeec0e44 | 2 | import json |
be2d40a5 | 3 | |
bb5ebd44 | 4 | |
be2d40a5 | 5 | from .common import InfoExtractor |
7913e0fc | 6 | from .gigya import GigyaBaseIE |
7913e0fc | 7 | from ..compat import compat_HTTPError |
117589df | 8 | from ..utils import ( |
7913e0fc | 9 | ExtractorError, |
bc2ca1bb | 10 | clean_html, |
11 | extract_attributes, | |
7913e0fc | 12 | float_or_none, |
bc2ca1bb | 13 | get_element_by_class, |
7913e0fc | 14 | int_or_none, |
fd62b366 | 15 | merge_dicts, |
628e5bc0 | 16 | str_or_none, |
bc2ca1bb | 17 | strip_or_none, |
628e5bc0 | 18 | url_or_none, |
888299e6 | 19 | urlencode_postdata |
117589df | 20 | ) |
be2d40a5 TG |
21 | |
22 | ||
23 | class CanvasIE(InfoExtractor): | |
bc2ca1bb | 24 | _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)' |
117589df S |
25 | _TESTS = [{ |
26 | 'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', | |
cdb19aa4 | 27 | 'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9', |
117589df S |
28 | 'info_dict': { |
29 | 'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', | |
30 | 'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', | |
628e5bc0 | 31 | 'ext': 'mp4', |
117589df | 32 | 'title': 'Nachtwacht: De Greystook', |
628e5bc0 | 33 | 'description': 'Nachtwacht: De Greystook', |
117589df | 34 | 'thumbnail': r're:^https?://.*\.jpg$', |
cdb19aa4 | 35 | 'duration': 1468.02, |
117589df | 36 | }, |
cdb19aa4 | 37 | 'expected_warnings': ['is not a supported codec'], |
117589df S |
38 | }, { |
39 | 'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e', | |
40 | 'only_matching': True, | |
41 | }] | |
00dd0cd5 | 42 | _GEO_BYPASS = False |
170d6444 RA |
43 | _HLS_ENTRY_PROTOCOLS_MAP = { |
44 | 'HLS': 'm3u8_native', | |
aeec0e44 | 45 | 'HLS_AES': 'm3u8_native', |
170d6444 | 46 | } |
aeec0e44 | 47 | _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2' |
117589df S |
48 | |
49 | def _real_extract(self, url): | |
5ad28e7f | 50 | mobj = self._match_valid_url(url) |
117589df S |
51 | site_id, video_id = mobj.group('site_id'), mobj.group('id') |
52 | ||
00dd0cd5 | 53 | data = None |
54 | if site_id != 'vrtvideo': | |
55 | # Old API endpoint, serves more formats but may fail for some videos | |
56 | data = self._download_json( | |
57 | 'https://mediazone.vrt.be/api/v1/%s/assets/%s' | |
58 | % (site_id, video_id), video_id, 'Downloading asset JSON', | |
59 | 'Unable to download asset JSON', fatal=False) | |
628e5bc0 S |
60 | |
61 | # New API endpoint | |
62 | if not data: | |
aeec0e44 | 63 | vrtnutoken = self._download_json('https://token.vrt.be/refreshtoken', |
64 | video_id, note='refreshtoken: Retrieve vrtnutoken', | |
65 | errnote='refreshtoken failed')['vrtnutoken'] | |
00dd0cd5 | 66 | headers = self.geo_verification_headers() |
aeec0e44 | 67 | headers.update({'Content-Type': 'application/json; charset=utf-8'}) |
68 | vrtPlayerToken = self._download_json( | |
628e5bc0 | 69 | '%s/tokens' % self._REST_API_BASE, video_id, |
aeec0e44 | 70 | 'Downloading token', headers=headers, data=json.dumps({ |
71 | 'identityToken': vrtnutoken | |
72 | }).encode('utf-8'))['vrtPlayerToken'] | |
628e5bc0 S |
73 | data = self._download_json( |
74 | '%s/videos/%s' % (self._REST_API_BASE, video_id), | |
00dd0cd5 | 75 | video_id, 'Downloading video JSON', query={ |
aeec0e44 | 76 | 'vrtPlayerToken': vrtPlayerToken, |
77 | 'client': 'null', | |
628e5bc0 | 78 | }, expected_status=400) |
3464a272 | 79 | if 'title' not in data: |
00dd0cd5 | 80 | code = data.get('code') |
81 | if code == 'AUTHENTICATION_REQUIRED': | |
82 | self.raise_login_required() | |
83 | elif code == 'INVALID_LOCATION': | |
84 | self.raise_geo_restricted(countries=['BE']) | |
85 | raise ExtractorError(data.get('message') or code, expected=True) | |
117589df | 86 | |
3464a272 | 87 | # Note: The title may be an empty string |
88 | title = data['title'] or f'{site_id} {video_id}' | |
117589df S |
89 | description = data.get('description') |
90 | ||
91 | formats = [] | |
e0e624ca | 92 | subtitles = {} |
117589df | 93 | for target in data['targetUrls']: |
628e5bc0 | 94 | format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type')) |
117589df S |
95 | if not format_url or not format_type: |
96 | continue | |
628e5bc0 | 97 | format_type = format_type.upper() |
170d6444 | 98 | if format_type in self._HLS_ENTRY_PROTOCOLS_MAP: |
e0e624ca | 99 | fmts, subs = self._extract_m3u8_formats_and_subtitles( |
170d6444 | 100 | format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type], |
e0e624ca F |
101 | m3u8_id=format_type, fatal=False) |
102 | formats.extend(fmts) | |
103 | subtitles = self._merge_subtitles(subtitles, subs) | |
117589df S |
104 | elif format_type == 'HDS': |
105 | formats.extend(self._extract_f4m_formats( | |
106 | format_url, video_id, f4m_id=format_type, fatal=False)) | |
107 | elif format_type == 'MPEG_DASH': | |
e0e624ca F |
108 | fmts, subs = self._extract_mpd_formats_and_subtitles( |
109 | format_url, video_id, mpd_id=format_type, fatal=False) | |
110 | formats.extend(fmts) | |
111 | subtitles = self._merge_subtitles(subtitles, subs) | |
117589df | 112 | elif format_type == 'HSS': |
e0e624ca F |
113 | fmts, subs = self._extract_ism_formats_and_subtitles( |
114 | format_url, video_id, ism_id='mss', fatal=False) | |
115 | formats.extend(fmts) | |
116 | subtitles = self._merge_subtitles(subtitles, subs) | |
117589df S |
117 | else: |
118 | formats.append({ | |
119 | 'format_id': format_type, | |
120 | 'url': format_url, | |
121 | }) | |
122 | self._sort_formats(formats) | |
123 | ||
117589df S |
124 | subtitle_urls = data.get('subtitleUrls') |
125 | if isinstance(subtitle_urls, list): | |
126 | for subtitle in subtitle_urls: | |
127 | subtitle_url = subtitle.get('url') | |
128 | if subtitle_url and subtitle.get('type') == 'CLOSED': | |
129 | subtitles.setdefault('nl', []).append({'url': subtitle_url}) | |
130 | ||
131 | return { | |
132 | 'id': video_id, | |
133 | 'display_id': video_id, | |
134 | 'title': title, | |
135 | 'description': description, | |
136 | 'formats': formats, | |
137 | 'duration': float_or_none(data.get('duration'), 1000), | |
138 | 'thumbnail': data.get('posterImageUrl'), | |
139 | 'subtitles': subtitles, | |
140 | } | |
141 | ||
142 | ||
143 | class CanvasEenIE(InfoExtractor): | |
41b263ac | 144 | IE_DESC = 'canvas.be and een.be' |
bb5ebd44 | 145 | _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)' |
6eff2605 | 146 | _TESTS = [{ |
be2d40a5 | 147 | 'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week', |
117589df | 148 | 'md5': 'ed66976748d12350b118455979cca293', |
be2d40a5 | 149 | 'info_dict': { |
4e2743ab S |
150 | 'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e', |
151 | 'display_id': 'de-afspraak-veilt-voor-de-warmste-week', | |
117589df | 152 | 'ext': 'flv', |
4e2743ab S |
153 | 'title': 'De afspraak veilt voor de Warmste Week', |
154 | 'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6', | |
ec85ded8 | 155 | 'thumbnail': r're:^https?://.*\.jpg$', |
4e2743ab | 156 | 'duration': 49.02, |
117589df S |
157 | }, |
158 | 'expected_warnings': ['is not a supported codec'], | |
6eff2605 S |
159 | }, { |
160 | # with subtitles | |
161 | 'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167', | |
162 | 'info_dict': { | |
163 | 'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625', | |
164 | 'display_id': 'pieter-0167', | |
165 | 'ext': 'mp4', | |
166 | 'title': 'Pieter 0167', | |
167 | 'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e', | |
ec85ded8 | 168 | 'thumbnail': r're:^https?://.*\.jpg$', |
6eff2605 S |
169 | 'duration': 2553.08, |
170 | 'subtitles': { | |
171 | 'nl': [{ | |
172 | 'ext': 'vtt', | |
173 | }], | |
174 | }, | |
175 | }, | |
176 | 'params': { | |
177 | 'skip_download': True, | |
117589df S |
178 | }, |
179 | 'skip': 'Pagina niet gevonden', | |
bb5ebd44 | 180 | }, { |
628e5bc0 | 181 | 'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan', |
bb5ebd44 | 182 | 'info_dict': { |
628e5bc0 S |
183 | 'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8', |
184 | 'display_id': 'emma-pakt-thilly-aan', | |
bb5ebd44 | 185 | 'ext': 'mp4', |
628e5bc0 S |
186 | 'title': 'Emma pakt Thilly aan', |
187 | 'description': 'md5:c5c9b572388a99b2690030afa3f3bad7', | |
ec85ded8 | 188 | 'thumbnail': r're:^https?://.*\.jpg$', |
628e5bc0 | 189 | 'duration': 118.24, |
bb5ebd44 S |
190 | }, |
191 | 'params': { | |
192 | 'skip_download': True, | |
117589df | 193 | }, |
628e5bc0 | 194 | 'expected_warnings': ['is not a supported codec'], |
bb5ebd44 S |
195 | }, { |
196 | 'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend', | |
197 | 'only_matching': True, | |
6eff2605 | 198 | }] |
be2d40a5 TG |
199 | |
200 | def _real_extract(self, url): | |
5ad28e7f | 201 | mobj = self._match_valid_url(url) |
bb5ebd44 | 202 | site_id, display_id = mobj.group('site_id'), mobj.group('id') |
be2d40a5 | 203 | |
4e2743ab | 204 | webpage = self._download_webpage(url, display_id) |
be2d40a5 | 205 | |
117589df | 206 | title = strip_or_none(self._search_regex( |
4e2743ab | 207 | r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>', |
bb5ebd44 | 208 | webpage, 'title', default=None) or self._og_search_title( |
117589df | 209 | webpage, default=None)) |
4e2743ab S |
210 | |
211 | video_id = self._html_search_regex( | |
117589df S |
212 | r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id', |
213 | group='id') | |
4e2743ab | 214 | |
be2d40a5 | 215 | return { |
117589df S |
216 | '_type': 'url_transparent', |
217 | 'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id), | |
218 | 'ie_key': CanvasIE.ie_key(), | |
be2d40a5 | 219 | 'id': video_id, |
4e2743ab | 220 | 'display_id': display_id, |
be2d40a5 | 221 | 'title': title, |
4e2743ab | 222 | 'description': self._og_search_description(webpage), |
be2d40a5 | 223 | } |
7913e0fc | 224 | |
225 | ||
226 | class VrtNUIE(GigyaBaseIE): | |
227 | IE_DESC = 'VrtNU.be' | |
00dd0cd5 | 228 | _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)' |
7913e0fc | 229 | _TESTS = [{ |
628e5bc0 | 230 | # Available via old API endpoint |
00dd0cd5 | 231 | 'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/', |
7913e0fc | 232 | 'info_dict': { |
00dd0cd5 | 233 | 'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de', |
628e5bc0 | 234 | 'ext': 'mp4', |
00dd0cd5 | 235 | 'title': 'Postbus X - Aflevering 1 (Seizoen 1989)', |
236 | 'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7', | |
7913e0fc | 237 | 'duration': 1457.04, |
238 | 'thumbnail': r're:^https?://.*\.jpg$', | |
00dd0cd5 | 239 | 'series': 'Postbus X', |
240 | 'season': 'Seizoen 1989', | |
241 | 'season_number': 1989, | |
242 | 'episode': 'De zwarte weduwe', | |
7913e0fc | 243 | 'episode_number': 1, |
00dd0cd5 | 244 | 'timestamp': 1595822400, |
245 | 'upload_date': '20200727', | |
7913e0fc | 246 | }, |
628e5bc0 S |
247 | 'skip': 'This video is only available for registered users', |
248 | 'params': { | |
249 | 'username': '<snip>', | |
250 | 'password': '<snip>', | |
251 | }, | |
252 | 'expected_warnings': ['is not a supported codec'], | |
253 | }, { | |
254 | # Only available via new API endpoint | |
255 | 'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/', | |
256 | 'info_dict': { | |
257 | 'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1', | |
258 | 'ext': 'mp4', | |
259 | 'title': 'Aflevering 5', | |
260 | 'description': 'Wie valt door de mand tijdens een missie?', | |
261 | 'duration': 2967.06, | |
262 | 'season': 'Season 1', | |
263 | 'season_number': 1, | |
264 | 'episode_number': 5, | |
265 | }, | |
266 | 'skip': 'This video is only available for registered users', | |
267 | 'params': { | |
268 | 'username': '<snip>', | |
269 | 'password': '<snip>', | |
270 | }, | |
271 | 'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'], | |
7913e0fc | 272 | }] |
273 | _NETRC_MACHINE = 'vrtnu' | |
aeec0e44 | 274 | _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy' |
7913e0fc | 275 | _CONTEXT_ID = 'R3595707040' |
276 | ||
277 | def _real_initialize(self): | |
278 | self._login() | |
279 | ||
280 | def _login(self): | |
281 | username, password = self._get_login_info() | |
282 | if username is None: | |
b8c6ffc5 | 283 | return |
7913e0fc | 284 | |
aeec0e44 | 285 | auth_info = self._gigya_login({ |
286 | 'APIKey': self._APIKEY, | |
287 | 'targetEnv': 'jssdk', | |
288 | 'loginID': username, | |
289 | 'password': password, | |
290 | 'authMode': 'cookie', | |
291 | }) | |
7913e0fc | 292 | |
cc33cc43 L |
293 | if auth_info.get('errorDetails'): |
294 | raise ExtractorError('Unable to login: VrtNU said: ' + auth_info.get('errorDetails'), expected=True) | |
295 | ||
7913e0fc | 296 | # Sometimes authentication fails for no good reason, retry |
297 | login_attempt = 1 | |
298 | while login_attempt <= 3: | |
299 | try: | |
888299e6 S |
300 | self._request_webpage('https://token.vrt.be/vrtnuinitlogin', |
301 | None, note='Requesting XSRF Token', errnote='Could not get XSRF Token', | |
302 | query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'}) | |
303 | ||
304 | post_data = { | |
305 | 'UID': auth_info['UID'], | |
306 | 'UIDSignature': auth_info['UIDSignature'], | |
307 | 'signatureTimestamp': auth_info['signatureTimestamp'], | |
888299e6 S |
308 | '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value, |
309 | } | |
310 | ||
7913e0fc | 311 | self._request_webpage( |
888299e6 | 312 | 'https://login.vrt.be/perform_login', |
aeec0e44 | 313 | None, note='Performing login', errnote='perform login failed', |
314 | headers={}, query={ | |
315 | 'client_id': 'vrtnu-site' | |
316 | }, data=urlencode_postdata(post_data)) | |
888299e6 | 317 | |
7913e0fc | 318 | except ExtractorError as e: |
319 | if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401: | |
320 | login_attempt += 1 | |
321 | self.report_warning('Authentication failed') | |
322 | self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again') | |
323 | else: | |
324 | raise e | |
325 | else: | |
326 | break | |
327 | ||
328 | def _real_extract(self, url): | |
329 | display_id = self._match_id(url) | |
330 | ||
00dd0cd5 | 331 | webpage = self._download_webpage(url, display_id) |
b8c6ffc5 | 332 | |
00dd0cd5 | 333 | attrs = extract_attributes(self._search_regex( |
334 | r'(<nui-media[^>]+>)', webpage, 'media element')) | |
335 | video_id = attrs['videoid'] | |
336 | publication_id = attrs.get('publicationid') | |
337 | if publication_id: | |
338 | video_id = publication_id + '$' + video_id | |
b8c6ffc5 | 339 | |
00dd0cd5 | 340 | page = (self._parse_json(self._search_regex( |
341 | r'digitalData\s*=\s*({.+?});', webpage, 'digial data', | |
342 | default='{}'), video_id, fatal=False) or {}).get('page') or {} | |
7913e0fc | 343 | |
00dd0cd5 | 344 | info = self._search_json_ld(webpage, display_id, default={}) |
fd62b366 | 345 | return merge_dicts(info, { |
7913e0fc | 346 | '_type': 'url_transparent', |
347 | 'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id, | |
348 | 'ie_key': CanvasIE.ie_key(), | |
349 | 'id': video_id, | |
350 | 'display_id': display_id, | |
00dd0cd5 | 351 | 'season_number': int_or_none(page.get('episode_season')), |
fd62b366 | 352 | }) |
bc2ca1bb | 353 | |
354 | ||
355 | class DagelijkseKostIE(InfoExtractor): | |
356 | IE_DESC = 'dagelijksekost.een.be' | |
357 | _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)' | |
358 | _TEST = { | |
359 | 'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof', | |
360 | 'md5': '30bfffc323009a3e5f689bef6efa2365', | |
361 | 'info_dict': { | |
362 | 'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa', | |
363 | 'display_id': 'hachis-parmentier-met-witloof', | |
364 | 'ext': 'mp4', | |
365 | 'title': 'Hachis parmentier met witloof', | |
366 | 'description': 'md5:9960478392d87f63567b5b117688cdc5', | |
367 | 'thumbnail': r're:^https?://.*\.jpg$', | |
368 | 'duration': 283.02, | |
369 | }, | |
370 | 'expected_warnings': ['is not a supported codec'], | |
371 | } | |
372 | ||
373 | def _real_extract(self, url): | |
374 | display_id = self._match_id(url) | |
375 | webpage = self._download_webpage(url, display_id) | |
376 | ||
377 | title = strip_or_none(get_element_by_class( | |
378 | 'dish-metadata__title', webpage | |
379 | ) or self._html_search_meta( | |
380 | 'twitter:title', webpage)) | |
381 | ||
382 | description = clean_html(get_element_by_class( | |
383 | 'dish-description', webpage) | |
384 | ) or self._html_search_meta( | |
385 | ('description', 'twitter:description', 'og:description'), | |
386 | webpage) | |
387 | ||
388 | video_id = self._html_search_regex( | |
389 | r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id', | |
390 | group='id') | |
391 | ||
392 | return { | |
393 | '_type': 'url_transparent', | |
394 | 'url': 'https://mediazone.vrt.be/api/v1/dako/assets/%s' % video_id, | |
395 | 'ie_key': CanvasIE.ie_key(), | |
396 | 'id': video_id, | |
397 | 'display_id': display_id, | |
398 | 'title': title, | |
399 | 'description': description, | |
400 | } |