yt_dlp/extractor/canvas.py

   1 import json
   2
   3
   4 from .common import InfoExtractor
   5 from .gigya import GigyaBaseIE
   6 from ..compat import compat_HTTPError
   7 from ..utils import (
   8     ExtractorError,
   9     clean_html,
  10     extract_attributes,
  11     float_or_none,
  12     get_element_by_class,
  13     int_or_none,
  14     merge_dicts,
  15     str_or_none,
  16     strip_or_none,
  17     url_or_none,
  18     urlencode_postdata
  19 )
  20
  21
  22 class CanvasIE(InfoExtractor):
  23     _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)'
  24     _TESTS = [{
  25         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  26         'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
  27         'info_dict': {
  28             'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  29             'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  30             'ext': 'mp4',
  31             'title': 'Nachtwacht: De Greystook',
  32             'description': 'Nachtwacht: De Greystook',
  33             'thumbnail': r're:^https?://.*\.jpg$',
  34             'duration': 1468.02,
  35         },
  36         'expected_warnings': ['is not a supported codec'],
  37     }, {
  38         'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
  39         'only_matching': True,
  40     }]
  41     _GEO_BYPASS = False
  42     _HLS_ENTRY_PROTOCOLS_MAP = {
  43         'HLS': 'm3u8_native',
  44         'HLS_AES': 'm3u8_native',
  45     }
  46     _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
  47
  48     def _real_extract(self, url):
  49         mobj = self._match_valid_url(url)
  50         site_id, video_id = mobj.group('site_id'), mobj.group('id')
  51
  52         data = None
  53         if site_id != 'vrtvideo':
  54             # Old API endpoint, serves more formats but may fail for some videos
  55             data = self._download_json(
  56                 'https://mediazone.vrt.be/api/v1/%s/assets/%s'
  57                 % (site_id, video_id), video_id, 'Downloading asset JSON',
  58                 'Unable to download asset JSON', fatal=False)
  59
  60         # New API endpoint
  61         if not data:
  62             vrtnutoken = self._download_json('https://token.vrt.be/refreshtoken',
  63                                              video_id, note='refreshtoken: Retrieve vrtnutoken',
  64                                              errnote='refreshtoken failed')['vrtnutoken']
  65             headers = self.geo_verification_headers()
  66             headers.update({'Content-Type': 'application/json; charset=utf-8'})
  67             vrtPlayerToken = self._download_json(
  68                 '%s/tokens' % self._REST_API_BASE, video_id,
  69                 'Downloading token', headers=headers, data=json.dumps({
  70                     'identityToken': vrtnutoken
  71                 }).encode('utf-8'))['vrtPlayerToken']
  72             data = self._download_json(
  73                 '%s/videos/%s' % (self._REST_API_BASE, video_id),
  74                 video_id, 'Downloading video JSON', query={
  75                     'vrtPlayerToken': vrtPlayerToken,
  76                     'client': 'null',
  77                 }, expected_status=400)
  78             if 'title' not in data:
  79                 code = data.get('code')
  80                 if code == 'AUTHENTICATION_REQUIRED':
  81                     self.raise_login_required()
  82                 elif code == 'INVALID_LOCATION':
  83                     self.raise_geo_restricted(countries=['BE'])
  84                 raise ExtractorError(data.get('message') or code, expected=True)
  85
  86         # Note: The title may be an empty string
  87         title = data['title'] or f'{site_id} {video_id}'
  88         description = data.get('description')
  89
  90         formats = []
  91         subtitles = {}
  92         for target in data['targetUrls']:
  93             format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type'))
  94             if not format_url or not format_type:
  95                 continue
  96             format_type = format_type.upper()
  97             if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
  98                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
  99                     format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
 100                     m3u8_id=format_type, fatal=False)
 101                 formats.extend(fmts)
 102                 subtitles = self._merge_subtitles(subtitles, subs)
 103             elif format_type == 'HDS':
 104                 formats.extend(self._extract_f4m_formats(
 105                     format_url, video_id, f4m_id=format_type, fatal=False))
 106             elif format_type == 'MPEG_DASH':
 107                 fmts, subs = self._extract_mpd_formats_and_subtitles(
 108                     format_url, video_id, mpd_id=format_type, fatal=False)
 109                 formats.extend(fmts)
 110                 subtitles = self._merge_subtitles(subtitles, subs)
 111             elif format_type == 'HSS':
 112                 fmts, subs = self._extract_ism_formats_and_subtitles(
 113                     format_url, video_id, ism_id='mss', fatal=False)
 114                 formats.extend(fmts)
 115                 subtitles = self._merge_subtitles(subtitles, subs)
 116             else:
 117                 formats.append({
 118                     'format_id': format_type,
 119                     'url': format_url,
 120                 })
 121         self._sort_formats(formats)
 122
 123         subtitle_urls = data.get('subtitleUrls')
 124         if isinstance(subtitle_urls, list):
 125             for subtitle in subtitle_urls:
 126                 subtitle_url = subtitle.get('url')
 127                 if subtitle_url and subtitle.get('type') == 'CLOSED':
 128                     subtitles.setdefault('nl', []).append({'url': subtitle_url})
 129
 130         return {
 131             'id': video_id,
 132             'display_id': video_id,
 133             'title': title,
 134             'description': description,
 135             'formats': formats,
 136             'duration': float_or_none(data.get('duration'), 1000),
 137             'thumbnail': data.get('posterImageUrl'),
 138             'subtitles': subtitles,
 139         }
 140
 141
 142 class CanvasEenIE(InfoExtractor):
 143     IE_DESC = 'canvas.be and een.be'
 144     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 145     _TESTS = [{
 146         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
 147         'md5': 'ed66976748d12350b118455979cca293',
 148         'info_dict': {
 149             'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
 150             'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
 151             'ext': 'flv',
 152             'title': 'De afspraak veilt voor de Warmste Week',
 153             'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
 154             'thumbnail': r're:^https?://.*\.jpg$',
 155             'duration': 49.02,
 156         },
 157         'expected_warnings': ['is not a supported codec'],
 158     }, {
 159         # with subtitles
 160         'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
 161         'info_dict': {
 162             'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
 163             'display_id': 'pieter-0167',
 164             'ext': 'mp4',
 165             'title': 'Pieter 0167',
 166             'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
 167             'thumbnail': r're:^https?://.*\.jpg$',
 168             'duration': 2553.08,
 169             'subtitles': {
 170                 'nl': [{
 171                     'ext': 'vtt',
 172                 }],
 173             },
 174         },
 175         'params': {
 176             'skip_download': True,
 177         },
 178         'skip': 'Pagina niet gevonden',
 179     }, {
 180         'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
 181         'info_dict': {
 182             'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
 183             'display_id': 'emma-pakt-thilly-aan',
 184             'ext': 'mp4',
 185             'title': 'Emma pakt Thilly aan',
 186             'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
 187             'thumbnail': r're:^https?://.*\.jpg$',
 188             'duration': 118.24,
 189         },
 190         'params': {
 191             'skip_download': True,
 192         },
 193         'expected_warnings': ['is not a supported codec'],
 194     }, {
 195         'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
 196         'only_matching': True,
 197     }]
 198
 199     def _real_extract(self, url):
 200         mobj = self._match_valid_url(url)
 201         site_id, display_id = mobj.group('site_id'), mobj.group('id')
 202
 203         webpage = self._download_webpage(url, display_id)
 204
 205         title = strip_or_none(self._search_regex(
 206             r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
 207             webpage, 'title', default=None) or self._og_search_title(
 208             webpage, default=None))
 209
 210         video_id = self._html_search_regex(
 211             r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
 212             group='id')
 213
 214         return {
 215             '_type': 'url_transparent',
 216             'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
 217             'ie_key': CanvasIE.ie_key(),
 218             'id': video_id,
 219             'display_id': display_id,
 220             'title': title,
 221             'description': self._og_search_description(webpage),
 222         }
 223
 224
 225 class VrtNUIE(GigyaBaseIE):
 226     IE_DESC = 'VrtNU.be'
 227     _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
 228     _TESTS = [{
 229         # Available via old API endpoint
 230         'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
 231         'info_dict': {
 232             'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
 233             'ext': 'mp4',
 234             'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
 235             'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
 236             'duration': 1457.04,
 237             'thumbnail': r're:^https?://.*\.jpg$',
 238             'series': 'Postbus X',
 239             'season': 'Seizoen 1989',
 240             'season_number': 1989,
 241             'episode': 'De zwarte weduwe',
 242             'episode_number': 1,
 243             'timestamp': 1595822400,
 244             'upload_date': '20200727',
 245         },
 246         'skip': 'This video is only available for registered users',
 247         'expected_warnings': ['is not a supported codec'],
 248     }, {
 249         # Only available via new API endpoint
 250         'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
 251         'info_dict': {
 252             'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
 253             'ext': 'mp4',
 254             'title': 'Aflevering 5',
 255             'description': 'Wie valt door de mand tijdens een missie?',
 256             'duration': 2967.06,
 257             'season': 'Season 1',
 258             'season_number': 1,
 259             'episode_number': 5,
 260         },
 261         'skip': 'This video is only available for registered users',
 262         'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
 263     }]
 264     _NETRC_MACHINE = 'vrtnu'
 265     _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
 266     _CONTEXT_ID = 'R3595707040'
 267
 268     def _perform_login(self, username, password):
 269         auth_info = self._gigya_login({
 270             'APIKey': self._APIKEY,
 271             'targetEnv': 'jssdk',
 272             'loginID': username,
 273             'password': password,
 274             'authMode': 'cookie',
 275         })
 276
 277         if auth_info.get('errorDetails'):
 278             raise ExtractorError('Unable to login: VrtNU said: ' + auth_info.get('errorDetails'), expected=True)
 279
 280         # Sometimes authentication fails for no good reason, retry
 281         login_attempt = 1
 282         while login_attempt <= 3:
 283             try:
 284                 self._request_webpage('https://token.vrt.be/vrtnuinitlogin',
 285                                       None, note='Requesting XSRF Token', errnote='Could not get XSRF Token',
 286                                       query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'})
 287
 288                 post_data = {
 289                     'UID': auth_info['UID'],
 290                     'UIDSignature': auth_info['UIDSignature'],
 291                     'signatureTimestamp': auth_info['signatureTimestamp'],
 292                     '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
 293                 }
 294
 295                 self._request_webpage(
 296                     'https://login.vrt.be/perform_login',
 297                     None, note='Performing login', errnote='perform login failed',
 298                     headers={}, query={
 299                         'client_id': 'vrtnu-site'
 300                     }, data=urlencode_postdata(post_data))
 301
 302             except ExtractorError as e:
 303                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 304                     login_attempt += 1
 305                     self.report_warning('Authentication failed')
 306                     self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
 307                 else:
 308                     raise e
 309             else:
 310                 break
 311
 312     def _real_extract(self, url):
 313         display_id = self._match_id(url)
 314
 315         webpage = self._download_webpage(url, display_id)
 316
 317         attrs = extract_attributes(self._search_regex(
 318             r'(<nui-media[^>]+>)', webpage, 'media element'))
 319         video_id = attrs['videoid']
 320         publication_id = attrs.get('publicationid')
 321         if publication_id:
 322             video_id = publication_id + '$' + video_id
 323
 324         page = (self._parse_json(self._search_regex(
 325             r'digitalData\s*=\s*({.+?});', webpage, 'digial data',
 326             default='{}'), video_id, fatal=False) or {}).get('page') or {}
 327
 328         info = self._search_json_ld(webpage, display_id, default={})
 329         return merge_dicts(info, {
 330             '_type': 'url_transparent',
 331             'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
 332             'ie_key': CanvasIE.ie_key(),
 333             'id': video_id,
 334             'display_id': display_id,
 335             'season_number': int_or_none(page.get('episode_season')),
 336         })
 337
 338
 339 class DagelijkseKostIE(InfoExtractor):
 340     IE_DESC = 'dagelijksekost.een.be'
 341     _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
 342     _TEST = {
 343         'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
 344         'md5': '30bfffc323009a3e5f689bef6efa2365',
 345         'info_dict': {
 346             'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
 347             'display_id': 'hachis-parmentier-met-witloof',
 348             'ext': 'mp4',
 349             'title': 'Hachis parmentier met witloof',
 350             'description': 'md5:9960478392d87f63567b5b117688cdc5',
 351             'thumbnail': r're:^https?://.*\.jpg$',
 352             'duration': 283.02,
 353         },
 354         'expected_warnings': ['is not a supported codec'],
 355     }
 356
 357     def _real_extract(self, url):
 358         display_id = self._match_id(url)
 359         webpage = self._download_webpage(url, display_id)
 360
 361         title = strip_or_none(get_element_by_class(
 362             'dish-metadata__title', webpage
 363         ) or self._html_search_meta(
 364             'twitter:title', webpage))
 365
 366         description = clean_html(get_element_by_class(
 367             'dish-description', webpage)
 368         ) or self._html_search_meta(
 369             ('description', 'twitter:description', 'og:description'),
 370             webpage)
 371
 372         video_id = self._html_search_regex(
 373             r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
 374             group='id')
 375
 376         return {
 377             '_type': 'url_transparent',
 378             'url': 'https://mediazone.vrt.be/api/v1/dako/assets/%s' % video_id,
 379             'ie_key': CanvasIE.ie_key(),
 380             'id': video_id,
 381             'display_id': display_id,
 382             'title': title,
 383             'description': description,
 384         }