1 from __future__
import unicode_literals
5 from .common
import InfoExtractor
6 from .gigya
import GigyaBaseIE
7 from ..compat
import compat_HTTPError
23 class CanvasIE(InfoExtractor
):
24 _VALID_URL
= r
'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)'
26 'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
27 'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
29 'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
30 'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
32 'title': 'Nachtwacht: De Greystook',
33 'description': 'Nachtwacht: De Greystook',
34 'thumbnail': r
're:^https?://.*\.jpg$',
37 'expected_warnings': ['is not a supported codec'],
39 'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
40 'only_matching': True,
43 _HLS_ENTRY_PROTOCOLS_MAP
= {
45 'HLS_AES': 'm3u8_native',
47 _REST_API_BASE
= 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
49 def _real_extract(self
, url
):
50 mobj
= self
._match
_valid
_url
(url
)
51 site_id
, video_id
= mobj
.group('site_id'), mobj
.group('id')
54 if site_id
!= 'vrtvideo':
55 # Old API endpoint, serves more formats but may fail for some videos
56 data
= self
._download
_json
(
57 'https://mediazone.vrt.be/api/v1/%s/assets/%s'
58 % (site_id
, video_id
), video_id
, 'Downloading asset JSON',
59 'Unable to download asset JSON', fatal
=False)
63 vrtnutoken
= self
._download
_json
('https://token.vrt.be/refreshtoken',
64 video_id
, note
='refreshtoken: Retrieve vrtnutoken',
65 errnote
='refreshtoken failed')['vrtnutoken']
66 headers
= self
.geo_verification_headers()
67 headers
.update({'Content-Type': 'application/json; charset=utf-8'}
)
68 vrtPlayerToken
= self
._download
_json
(
69 '%s/tokens' % self
._REST
_API
_BASE
, video_id
,
70 'Downloading token', headers
=headers
, data
=json
.dumps({
71 'identityToken': vrtnutoken
72 }).encode('utf-8'))['vrtPlayerToken']
73 data
= self
._download
_json
(
74 '%s/videos/%s' % (self
._REST
_API
_BASE
, video_id
),
75 video_id
, 'Downloading video JSON', query
={
76 'vrtPlayerToken': vrtPlayerToken
,
78 }, expected_status
=400)
79 if 'title' not in data
:
80 code
= data
.get('code')
81 if code
== 'AUTHENTICATION_REQUIRED':
82 self
.raise_login_required()
83 elif code
== 'INVALID_LOCATION':
84 self
.raise_geo_restricted(countries
=['BE'])
85 raise ExtractorError(data
.get('message') or code
, expected
=True)
87 # Note: The title may be an empty string
88 title
= data
['title'] or f
'{site_id} {video_id}'
89 description
= data
.get('description')
93 for target
in data
['targetUrls']:
94 format_url
, format_type
= url_or_none(target
.get('url')), str_or_none(target
.get('type'))
95 if not format_url
or not format_type
:
97 format_type
= format_type
.upper()
98 if format_type
in self
._HLS
_ENTRY
_PROTOCOLS
_MAP
:
99 fmts
, subs
= self
._extract
_m
3u8_formats
_and
_subtitles
(
100 format_url
, video_id
, 'mp4', self
._HLS
_ENTRY
_PROTOCOLS
_MAP
[format_type
],
101 m3u8_id
=format_type
, fatal
=False)
103 subtitles
= self
._merge
_subtitles
(subtitles
, subs
)
104 elif format_type
== 'HDS':
105 formats
.extend(self
._extract
_f
4m
_formats
(
106 format_url
, video_id
, f4m_id
=format_type
, fatal
=False))
107 elif format_type
== 'MPEG_DASH':
108 fmts
, subs
= self
._extract
_mpd
_formats
_and
_subtitles
(
109 format_url
, video_id
, mpd_id
=format_type
, fatal
=False)
111 subtitles
= self
._merge
_subtitles
(subtitles
, subs
)
112 elif format_type
== 'HSS':
113 fmts
, subs
= self
._extract
_ism
_formats
_and
_subtitles
(
114 format_url
, video_id
, ism_id
='mss', fatal
=False)
116 subtitles
= self
._merge
_subtitles
(subtitles
, subs
)
119 'format_id': format_type
,
122 self
._sort
_formats
(formats
)
124 subtitle_urls
= data
.get('subtitleUrls')
125 if isinstance(subtitle_urls
, list):
126 for subtitle
in subtitle_urls
:
127 subtitle_url
= subtitle
.get('url')
128 if subtitle_url
and subtitle
.get('type') == 'CLOSED':
129 subtitles
.setdefault('nl', []).append({'url': subtitle_url}
)
133 'display_id': video_id
,
135 'description': description
,
137 'duration': float_or_none(data
.get('duration'), 1000),
138 'thumbnail': data
.get('posterImageUrl'),
139 'subtitles': subtitles
,
143 class CanvasEenIE(InfoExtractor
):
144 IE_DESC
= 'canvas.be and een.be'
145 _VALID_URL
= r
'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
147 'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
148 'md5': 'ed66976748d12350b118455979cca293',
150 'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
151 'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
153 'title': 'De afspraak veilt voor de Warmste Week',
154 'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
155 'thumbnail': r
're:^https?://.*\.jpg$',
158 'expected_warnings': ['is not a supported codec'],
161 'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
163 'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
164 'display_id': 'pieter-0167',
166 'title': 'Pieter 0167',
167 'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
168 'thumbnail': r
're:^https?://.*\.jpg$',
177 'skip_download': True,
179 'skip': 'Pagina niet gevonden',
181 'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
183 'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
184 'display_id': 'emma-pakt-thilly-aan',
186 'title': 'Emma pakt Thilly aan',
187 'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
188 'thumbnail': r
're:^https?://.*\.jpg$',
192 'skip_download': True,
194 'expected_warnings': ['is not a supported codec'],
196 'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
197 'only_matching': True,
200 def _real_extract(self
, url
):
201 mobj
= self
._match
_valid
_url
(url
)
202 site_id
, display_id
= mobj
.group('site_id'), mobj
.group('id')
204 webpage
= self
._download
_webpage
(url
, display_id
)
206 title
= strip_or_none(self
._search
_regex
(
207 r
'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
208 webpage
, 'title', default
=None) or self
._og
_search
_title
(
209 webpage
, default
=None))
211 video_id
= self
._html
_search
_regex
(
212 r
'data-video=(["\'])(?P
<id>(?
:(?
!\
1).)+)\
1', webpage, 'video
id',
216 '_type
': 'url_transparent
',
217 'url
': 'https
://mediazone
.vrt
.be
/api
/v1
/%s/assets
/%s' % (site_id, video_id),
218 'ie_key
': CanvasIE.ie_key(),
220 'display_id
': display_id,
222 'description
': self._og_search_description(webpage),
226 class VrtNUIE(GigyaBaseIE):
228 _VALID_URL = r'https?
://(?
:www\
.)?vrt\
.be
/vrtnu
/a
-z
/(?
:[^
/]+/){2}
(?P
<id>[^
/?
#&]+)'
230 # Available via old API endpoint
231 'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
233 'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
235 'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
236 'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
238 'thumbnail': r
're:^https?://.*\.jpg$',
239 'series': 'Postbus X',
240 'season': 'Seizoen 1989',
241 'season_number': 1989,
242 'episode': 'De zwarte weduwe',
244 'timestamp': 1595822400,
245 'upload_date': '20200727',
247 'skip': 'This video is only available for registered users',
249 'username': '<snip>',
250 'password': '<snip>',
252 'expected_warnings': ['is not a supported codec'],
254 # Only available via new API endpoint
255 'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
257 'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
259 'title': 'Aflevering 5',
260 'description': 'Wie valt door de mand tijdens een missie?',
262 'season': 'Season 1',
266 'skip': 'This video is only available for registered users',
268 'username': '<snip>',
269 'password': '<snip>',
271 'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
273 _NETRC_MACHINE
= 'vrtnu'
274 _APIKEY
= '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
275 _CONTEXT_ID
= 'R3595707040'
277 def _perform_login(self
, username
, password
):
278 auth_info
= self
._gigya
_login
({
279 'APIKey': self
._APIKEY
,
280 'targetEnv': 'jssdk',
282 'password': password
,
283 'authMode': 'cookie',
286 if auth_info
.get('errorDetails'):
287 raise ExtractorError('Unable to login: VrtNU said: ' + auth_info
.get('errorDetails'), expected
=True)
289 # Sometimes authentication fails for no good reason, retry
291 while login_attempt
<= 3:
293 self
._request
_webpage
('https://token.vrt.be/vrtnuinitlogin',
294 None, note
='Requesting XSRF Token', errnote
='Could not get XSRF Token',
295 query
={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'}
)
298 'UID': auth_info
['UID'],
299 'UIDSignature': auth_info
['UIDSignature'],
300 'signatureTimestamp': auth_info
['signatureTimestamp'],
301 '_csrf': self
._get
_cookies
('https://login.vrt.be').get('OIDCXSRF').value
,
304 self
._request
_webpage
(
305 'https://login.vrt.be/perform_login',
306 None, note
='Performing login', errnote
='perform login failed',
308 'client_id': 'vrtnu-site'
309 }, data
=urlencode_postdata(post_data
))
311 except ExtractorError
as e
:
312 if isinstance(e
.cause
, compat_HTTPError
) and e
.cause
.code
== 401:
314 self
.report_warning('Authentication failed')
315 self
._sleep
(1, None, msg_template
='Waiting for %(timeout)s seconds before trying again')
321 def _real_extract(self
, url
):
322 display_id
= self
._match
_id
(url
)
324 webpage
= self
._download
_webpage
(url
, display_id
)
326 attrs
= extract_attributes(self
._search
_regex
(
327 r
'(<nui-media[^>]+>)', webpage
, 'media element'))
328 video_id
= attrs
['videoid']
329 publication_id
= attrs
.get('publicationid')
331 video_id
= publication_id
+ '$' + video_id
333 page
= (self
._parse
_json
(self
._search
_regex
(
334 r
'digitalData\s*=\s*({.+?});', webpage
, 'digial data',
335 default
='{}'), video_id, fatal=False) or {}).get('page
') or {}
337 info = self._search_json_ld(webpage, display_id, default={})
338 return merge_dicts(info, {
339 '_type
': 'url_transparent
',
340 'url
': 'https
://mediazone
.vrt
.be
/api
/v1
/vrtvideo
/assets
/%s' % video_id,
341 'ie_key
': CanvasIE.ie_key(),
343 'display_id
': display_id,
344 'season_number
': int_or_none(page.get('episode_season
')),
348 class DagelijkseKostIE(InfoExtractor):
349 IE_DESC = 'dagelijksekost
.een
.be
'
350 _VALID_URL = r'https?
://dagelijksekost\
.een\
.be
/gerechten
/(?P
<id>[^
/?
#&]+)'
352 'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
353 'md5': '30bfffc323009a3e5f689bef6efa2365',
355 'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
356 'display_id': 'hachis-parmentier-met-witloof',
358 'title': 'Hachis parmentier met witloof',
359 'description': 'md5:9960478392d87f63567b5b117688cdc5',
360 'thumbnail': r
're:^https?://.*\.jpg$',
363 'expected_warnings': ['is not a supported codec'],
366 def _real_extract(self
, url
):
367 display_id
= self
._match
_id
(url
)
368 webpage
= self
._download
_webpage
(url
, display_id
)
370 title
= strip_or_none(get_element_by_class(
371 'dish-metadata__title', webpage
372 ) or self
._html
_search
_meta
(
373 'twitter:title', webpage
))
375 description
= clean_html(get_element_by_class(
376 'dish-description', webpage
)
377 ) or self
._html
_search
_meta
(
378 ('description', 'twitter:description', 'og:description'),
381 video_id
= self
._html
_search
_regex
(
382 r
'data-url=(["\'])(?P
<id>(?
:(?
!\
1).)+)\
1', webpage, 'video
id',
386 '_type
': 'url_transparent
',
387 'url
': 'https
://mediazone
.vrt
.be
/api
/v1
/dako
/assets
/%s' % video_id,
388 'ie_key
': CanvasIE.ie_key(),
390 'display_id
': display_id,
392 'description
': description,