4 from .common
import InfoExtractor
5 from .gigya
import GigyaBaseIE
6 from ..compat
import compat_HTTPError
22 class CanvasIE(InfoExtractor
):
23 _VALID_URL
= r
'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)'
25 'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
26 'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
28 'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
29 'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
31 'title': 'Nachtwacht: De Greystook',
32 'description': 'Nachtwacht: De Greystook',
33 'thumbnail': r
're:^https?://.*\.jpg$',
36 'expected_warnings': ['is not a supported codec'],
38 'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
39 'only_matching': True,
42 _HLS_ENTRY_PROTOCOLS_MAP
= {
44 'HLS_AES': 'm3u8_native',
46 _REST_API_BASE
= 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
48 def _real_extract(self
, url
):
49 mobj
= self
._match
_valid
_url
(url
)
50 site_id
, video_id
= mobj
.group('site_id'), mobj
.group('id')
53 if site_id
!= 'vrtvideo':
54 # Old API endpoint, serves more formats but may fail for some videos
55 data
= self
._download
_json
(
56 'https://mediazone.vrt.be/api/v1/%s/assets/%s'
57 % (site_id
, video_id
), video_id
, 'Downloading asset JSON',
58 'Unable to download asset JSON', fatal
=False)
62 vrtnutoken
= self
._download
_json
('https://token.vrt.be/refreshtoken',
63 video_id
, note
='refreshtoken: Retrieve vrtnutoken',
64 errnote
='refreshtoken failed')['vrtnutoken']
65 headers
= self
.geo_verification_headers()
66 headers
.update({'Content-Type': 'application/json; charset=utf-8'}
)
67 vrtPlayerToken
= self
._download
_json
(
68 '%s/tokens' % self
._REST
_API
_BASE
, video_id
,
69 'Downloading token', headers
=headers
, data
=json
.dumps({
70 'identityToken': vrtnutoken
71 }).encode('utf-8'))['vrtPlayerToken']
72 data
= self
._download
_json
(
73 '%s/videos/%s' % (self
._REST
_API
_BASE
, video_id
),
74 video_id
, 'Downloading video JSON', query
={
75 'vrtPlayerToken': vrtPlayerToken
,
77 }, expected_status
=400)
78 if 'title' not in data
:
79 code
= data
.get('code')
80 if code
== 'AUTHENTICATION_REQUIRED':
81 self
.raise_login_required()
82 elif code
== 'INVALID_LOCATION':
83 self
.raise_geo_restricted(countries
=['BE'])
84 raise ExtractorError(data
.get('message') or code
, expected
=True)
86 # Note: The title may be an empty string
87 title
= data
['title'] or f
'{site_id} {video_id}'
88 description
= data
.get('description')
92 for target
in data
['targetUrls']:
93 format_url
, format_type
= url_or_none(target
.get('url')), str_or_none(target
.get('type'))
94 if not format_url
or not format_type
:
96 format_type
= format_type
.upper()
97 if format_type
in self
._HLS
_ENTRY
_PROTOCOLS
_MAP
:
98 fmts
, subs
= self
._extract
_m
3u8_formats
_and
_subtitles
(
99 format_url
, video_id
, 'mp4', self
._HLS
_ENTRY
_PROTOCOLS
_MAP
[format_type
],
100 m3u8_id
=format_type
, fatal
=False)
102 subtitles
= self
._merge
_subtitles
(subtitles
, subs
)
103 elif format_type
== 'HDS':
104 formats
.extend(self
._extract
_f
4m
_formats
(
105 format_url
, video_id
, f4m_id
=format_type
, fatal
=False))
106 elif format_type
== 'MPEG_DASH':
107 fmts
, subs
= self
._extract
_mpd
_formats
_and
_subtitles
(
108 format_url
, video_id
, mpd_id
=format_type
, fatal
=False)
110 subtitles
= self
._merge
_subtitles
(subtitles
, subs
)
111 elif format_type
== 'HSS':
112 fmts
, subs
= self
._extract
_ism
_formats
_and
_subtitles
(
113 format_url
, video_id
, ism_id
='mss', fatal
=False)
115 subtitles
= self
._merge
_subtitles
(subtitles
, subs
)
118 'format_id': format_type
,
121 self
._sort
_formats
(formats
)
123 subtitle_urls
= data
.get('subtitleUrls')
124 if isinstance(subtitle_urls
, list):
125 for subtitle
in subtitle_urls
:
126 subtitle_url
= subtitle
.get('url')
127 if subtitle_url
and subtitle
.get('type') == 'CLOSED':
128 subtitles
.setdefault('nl', []).append({'url': subtitle_url}
)
132 'display_id': video_id
,
134 'description': description
,
136 'duration': float_or_none(data
.get('duration'), 1000),
137 'thumbnail': data
.get('posterImageUrl'),
138 'subtitles': subtitles
,
142 class CanvasEenIE(InfoExtractor
):
143 IE_DESC
= 'canvas.be and een.be'
144 _VALID_URL
= r
'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
146 'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
147 'md5': 'ed66976748d12350b118455979cca293',
149 'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
150 'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
152 'title': 'De afspraak veilt voor de Warmste Week',
153 'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
154 'thumbnail': r
're:^https?://.*\.jpg$',
157 'expected_warnings': ['is not a supported codec'],
160 'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
162 'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
163 'display_id': 'pieter-0167',
165 'title': 'Pieter 0167',
166 'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
167 'thumbnail': r
're:^https?://.*\.jpg$',
176 'skip_download': True,
178 'skip': 'Pagina niet gevonden',
180 'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
182 'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
183 'display_id': 'emma-pakt-thilly-aan',
185 'title': 'Emma pakt Thilly aan',
186 'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
187 'thumbnail': r
're:^https?://.*\.jpg$',
191 'skip_download': True,
193 'expected_warnings': ['is not a supported codec'],
195 'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
196 'only_matching': True,
199 def _real_extract(self
, url
):
200 mobj
= self
._match
_valid
_url
(url
)
201 site_id
, display_id
= mobj
.group('site_id'), mobj
.group('id')
203 webpage
= self
._download
_webpage
(url
, display_id
)
205 title
= strip_or_none(self
._search
_regex
(
206 r
'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
207 webpage
, 'title', default
=None) or self
._og
_search
_title
(
208 webpage
, default
=None))
210 video_id
= self
._html
_search
_regex
(
211 r
'data-video=(["\'])(?P
<id>(?
:(?
!\
1).)+)\
1', webpage, 'video
id',
215 '_type
': 'url_transparent
',
216 'url
': 'https
://mediazone
.vrt
.be
/api
/v1
/%s/assets
/%s' % (site_id, video_id),
217 'ie_key
': CanvasIE.ie_key(),
219 'display_id
': display_id,
221 'description
': self._og_search_description(webpage),
225 class VrtNUIE(GigyaBaseIE):
227 _VALID_URL = r'https?
://(?
:www\
.)?vrt\
.be
/vrtnu
/a
-z
/(?
:[^
/]+/){2}
(?P
<id>[^
/?
#&]+)'
229 # Available via old API endpoint
230 'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
232 'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
234 'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
235 'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
237 'thumbnail': r
're:^https?://.*\.jpg$',
238 'series': 'Postbus X',
239 'season': 'Seizoen 1989',
240 'season_number': 1989,
241 'episode': 'De zwarte weduwe',
243 'timestamp': 1595822400,
244 'upload_date': '20200727',
246 'skip': 'This video is only available for registered users',
247 'expected_warnings': ['is not a supported codec'],
249 # Only available via new API endpoint
250 'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
252 'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
254 'title': 'Aflevering 5',
255 'description': 'Wie valt door de mand tijdens een missie?',
257 'season': 'Season 1',
261 'skip': 'This video is only available for registered users',
262 'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
264 _NETRC_MACHINE
= 'vrtnu'
265 _APIKEY
= '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
266 _CONTEXT_ID
= 'R3595707040'
268 def _perform_login(self
, username
, password
):
269 auth_info
= self
._gigya
_login
({
270 'APIKey': self
._APIKEY
,
271 'targetEnv': 'jssdk',
273 'password': password
,
274 'authMode': 'cookie',
277 if auth_info
.get('errorDetails'):
278 raise ExtractorError('Unable to login: VrtNU said: ' + auth_info
.get('errorDetails'), expected
=True)
280 # Sometimes authentication fails for no good reason, retry
282 while login_attempt
<= 3:
284 self
._request
_webpage
('https://token.vrt.be/vrtnuinitlogin',
285 None, note
='Requesting XSRF Token', errnote
='Could not get XSRF Token',
286 query
={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'}
)
289 'UID': auth_info
['UID'],
290 'UIDSignature': auth_info
['UIDSignature'],
291 'signatureTimestamp': auth_info
['signatureTimestamp'],
292 '_csrf': self
._get
_cookies
('https://login.vrt.be').get('OIDCXSRF').value
,
295 self
._request
_webpage
(
296 'https://login.vrt.be/perform_login',
297 None, note
='Performing login', errnote
='perform login failed',
299 'client_id': 'vrtnu-site'
300 }, data
=urlencode_postdata(post_data
))
302 except ExtractorError
as e
:
303 if isinstance(e
.cause
, compat_HTTPError
) and e
.cause
.code
== 401:
305 self
.report_warning('Authentication failed')
306 self
._sleep
(1, None, msg_template
='Waiting for %(timeout)s seconds before trying again')
312 def _real_extract(self
, url
):
313 display_id
= self
._match
_id
(url
)
315 webpage
= self
._download
_webpage
(url
, display_id
)
317 attrs
= extract_attributes(self
._search
_regex
(
318 r
'(<nui-media[^>]+>)', webpage
, 'media element'))
319 video_id
= attrs
['videoid']
320 publication_id
= attrs
.get('publicationid')
322 video_id
= publication_id
+ '$' + video_id
324 page
= (self
._parse
_json
(self
._search
_regex
(
325 r
'digitalData\s*=\s*({.+?});', webpage
, 'digial data',
326 default
='{}'), video_id, fatal=False) or {}).get('page
') or {}
328 info = self._search_json_ld(webpage, display_id, default={})
329 return merge_dicts(info, {
330 '_type
': 'url_transparent
',
331 'url
': 'https
://mediazone
.vrt
.be
/api
/v1
/vrtvideo
/assets
/%s' % video_id,
332 'ie_key
': CanvasIE.ie_key(),
334 'display_id
': display_id,
335 'season_number
': int_or_none(page.get('episode_season
')),
339 class DagelijkseKostIE(InfoExtractor):
340 IE_DESC = 'dagelijksekost
.een
.be
'
341 _VALID_URL = r'https?
://dagelijksekost\
.een\
.be
/gerechten
/(?P
<id>[^
/?
#&]+)'
343 'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
344 'md5': '30bfffc323009a3e5f689bef6efa2365',
346 'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
347 'display_id': 'hachis-parmentier-met-witloof',
349 'title': 'Hachis parmentier met witloof',
350 'description': 'md5:9960478392d87f63567b5b117688cdc5',
351 'thumbnail': r
're:^https?://.*\.jpg$',
354 'expected_warnings': ['is not a supported codec'],
357 def _real_extract(self
, url
):
358 display_id
= self
._match
_id
(url
)
359 webpage
= self
._download
_webpage
(url
, display_id
)
361 title
= strip_or_none(get_element_by_class(
362 'dish-metadata__title', webpage
363 ) or self
._html
_search
_meta
(
364 'twitter:title', webpage
))
366 description
= clean_html(get_element_by_class(
367 'dish-description', webpage
)
368 ) or self
._html
_search
_meta
(
369 ('description', 'twitter:description', 'og:description'),
372 video_id
= self
._html
_search
_regex
(
373 r
'data-url=(["\'])(?P
<id>(?
:(?
!\
1).)+)\
1', webpage, 'video
id',
377 '_type
': 'url_transparent
',
378 'url
': 'https
://mediazone
.vrt
.be
/api
/v1
/dako
/assets
/%s' % video_id,
379 'ie_key
': CanvasIE.ie_key(),
381 'display_id
': display_id,
383 'description
': description,