yt_dlp/extractor/canvas.py

   1 import json
   2
   3
   4 from .common import InfoExtractor
   5 from .gigya import GigyaBaseIE
   6 from ..compat import compat_HTTPError
   7 from ..utils import (
   8     ExtractorError,
   9     clean_html,
  10     extract_attributes,
  11     float_or_none,
  12     get_element_by_class,
  13     int_or_none,
  14     merge_dicts,
  15     str_or_none,
  16     strip_or_none,
  17     url_or_none,
  18     urlencode_postdata
  19 )
  20
  21
  22 class CanvasIE(InfoExtractor):
  23     _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)'
  24     _TESTS = [{
  25         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  26         'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
  27         'info_dict': {
  28             'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  29             'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  30             'ext': 'mp4',
  31             'title': 'Nachtwacht: De Greystook',
  32             'description': 'Nachtwacht: De Greystook',
  33             'thumbnail': r're:^https?://.*\.jpg$',
  34             'duration': 1468.02,
  35         },
  36         'expected_warnings': ['is not a supported codec'],
  37     }, {
  38         'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
  39         'only_matching': True,
  40     }]
  41     _GEO_BYPASS = False
  42     _HLS_ENTRY_PROTOCOLS_MAP = {
  43         'HLS': 'm3u8_native',
  44         'HLS_AES': 'm3u8_native',
  45     }
  46     _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
  47
  48     def _real_extract(self, url):
  49         mobj = self._match_valid_url(url)
  50         site_id, video_id = mobj.group('site_id'), mobj.group('id')
  51
  52         data = None
  53         if site_id != 'vrtvideo':
  54             # Old API endpoint, serves more formats but may fail for some videos
  55             data = self._download_json(
  56                 'https://mediazone.vrt.be/api/v1/%s/assets/%s'
  57                 % (site_id, video_id), video_id, 'Downloading asset JSON',
  58                 'Unable to download asset JSON', fatal=False)
  59
  60         # New API endpoint
  61         if not data:
  62             vrtnutoken = self._download_json('https://token.vrt.be/refreshtoken',
  63                                              video_id, note='refreshtoken: Retrieve vrtnutoken',
  64                                              errnote='refreshtoken failed')['vrtnutoken']
  65             headers = self.geo_verification_headers()
  66             headers.update({'Content-Type': 'application/json; charset=utf-8'})
  67             vrtPlayerToken = self._download_json(
  68                 '%s/tokens' % self._REST_API_BASE, video_id,
  69                 'Downloading token', headers=headers, data=json.dumps({
  70                     'identityToken': vrtnutoken
  71                 }).encode('utf-8'))['vrtPlayerToken']
  72             data = self._download_json(
  73                 '%s/videos/%s' % (self._REST_API_BASE, video_id),
  74                 video_id, 'Downloading video JSON', query={
  75                     'vrtPlayerToken': vrtPlayerToken,
  76                     'client': 'null',
  77                 }, expected_status=400)
  78             if 'title' not in data:
  79                 code = data.get('code')
  80                 if code == 'AUTHENTICATION_REQUIRED':
  81                     self.raise_login_required()
  82                 elif code == 'INVALID_LOCATION':
  83                     self.raise_geo_restricted(countries=['BE'])
  84                 raise ExtractorError(data.get('message') or code, expected=True)
  85
  86         # Note: The title may be an empty string
  87         title = data['title'] or f'{site_id} {video_id}'
  88         description = data.get('description')
  89
  90         formats = []
  91         subtitles = {}
  92         for target in data['targetUrls']:
  93             format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type'))
  94             if not format_url or not format_type:
  95                 continue
  96             format_type = format_type.upper()
  97             if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
  98                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
  99                     format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
 100                     m3u8_id=format_type, fatal=False)
 101                 formats.extend(fmts)
 102                 subtitles = self._merge_subtitles(subtitles, subs)
 103             elif format_type == 'HDS':
 104                 formats.extend(self._extract_f4m_formats(
 105                     format_url, video_id, f4m_id=format_type, fatal=False))
 106             elif format_type == 'MPEG_DASH':
 107                 fmts, subs = self._extract_mpd_formats_and_subtitles(
 108                     format_url, video_id, mpd_id=format_type, fatal=False)
 109                 formats.extend(fmts)
 110                 subtitles = self._merge_subtitles(subtitles, subs)
 111             elif format_type == 'HSS':
 112                 fmts, subs = self._extract_ism_formats_and_subtitles(
 113                     format_url, video_id, ism_id='mss', fatal=False)
 114                 formats.extend(fmts)
 115                 subtitles = self._merge_subtitles(subtitles, subs)
 116             else:
 117                 formats.append({
 118                     'format_id': format_type,
 119                     'url': format_url,
 120                 })
 121
 122         subtitle_urls = data.get('subtitleUrls')
 123         if isinstance(subtitle_urls, list):
 124             for subtitle in subtitle_urls:
 125                 subtitle_url = subtitle.get('url')
 126                 if subtitle_url and subtitle.get('type') == 'CLOSED':
 127                     subtitles.setdefault('nl', []).append({'url': subtitle_url})
 128
 129         return {
 130             'id': video_id,
 131             'display_id': video_id,
 132             'title': title,
 133             'description': description,
 134             'formats': formats,
 135             'duration': float_or_none(data.get('duration'), 1000),
 136             'thumbnail': data.get('posterImageUrl'),
 137             'subtitles': subtitles,
 138         }
 139
 140
 141 class CanvasEenIE(InfoExtractor):
 142     IE_DESC = 'canvas.be and een.be'
 143     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 144     _TESTS = [{
 145         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
 146         'md5': 'ed66976748d12350b118455979cca293',
 147         'info_dict': {
 148             'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
 149             'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
 150             'ext': 'flv',
 151             'title': 'De afspraak veilt voor de Warmste Week',
 152             'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
 153             'thumbnail': r're:^https?://.*\.jpg$',
 154             'duration': 49.02,
 155         },
 156         'expected_warnings': ['is not a supported codec'],
 157     }, {
 158         # with subtitles
 159         'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
 160         'info_dict': {
 161             'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
 162             'display_id': 'pieter-0167',
 163             'ext': 'mp4',
 164             'title': 'Pieter 0167',
 165             'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
 166             'thumbnail': r're:^https?://.*\.jpg$',
 167             'duration': 2553.08,
 168             'subtitles': {
 169                 'nl': [{
 170                     'ext': 'vtt',
 171                 }],
 172             },
 173         },
 174         'params': {
 175             'skip_download': True,
 176         },
 177         'skip': 'Pagina niet gevonden',
 178     }, {
 179         'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
 180         'info_dict': {
 181             'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
 182             'display_id': 'emma-pakt-thilly-aan',
 183             'ext': 'mp4',
 184             'title': 'Emma pakt Thilly aan',
 185             'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
 186             'thumbnail': r're:^https?://.*\.jpg$',
 187             'duration': 118.24,
 188         },
 189         'params': {
 190             'skip_download': True,
 191         },
 192         'expected_warnings': ['is not a supported codec'],
 193     }, {
 194         'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
 195         'only_matching': True,
 196     }]
 197
 198     def _real_extract(self, url):
 199         mobj = self._match_valid_url(url)
 200         site_id, display_id = mobj.group('site_id'), mobj.group('id')
 201
 202         webpage = self._download_webpage(url, display_id)
 203
 204         title = strip_or_none(self._search_regex(
 205             r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
 206             webpage, 'title', default=None) or self._og_search_title(
 207             webpage, default=None))
 208
 209         video_id = self._html_search_regex(
 210             r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
 211             group='id')
 212
 213         return {
 214             '_type': 'url_transparent',
 215             'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
 216             'ie_key': CanvasIE.ie_key(),
 217             'id': video_id,
 218             'display_id': display_id,
 219             'title': title,
 220             'description': self._og_search_description(webpage),
 221         }
 222
 223
 224 class VrtNUIE(GigyaBaseIE):
 225     IE_DESC = 'VrtNU.be'
 226     _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
 227     _TESTS = [{
 228         # Available via old API endpoint
 229         'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
 230         'info_dict': {
 231             'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
 232             'ext': 'mp4',
 233             'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
 234             'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
 235             'duration': 1457.04,
 236             'thumbnail': r're:^https?://.*\.jpg$',
 237             'series': 'Postbus X',
 238             'season': 'Seizoen 1989',
 239             'season_number': 1989,
 240             'episode': 'De zwarte weduwe',
 241             'episode_number': 1,
 242             'timestamp': 1595822400,
 243             'upload_date': '20200727',
 244         },
 245         'skip': 'This video is only available for registered users',
 246         'expected_warnings': ['is not a supported codec'],
 247     }, {
 248         # Only available via new API endpoint
 249         'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
 250         'info_dict': {
 251             'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
 252             'ext': 'mp4',
 253             'title': 'Aflevering 5',
 254             'description': 'Wie valt door de mand tijdens een missie?',
 255             'duration': 2967.06,
 256             'season': 'Season 1',
 257             'season_number': 1,
 258             'episode_number': 5,
 259         },
 260         'skip': 'This video is only available for registered users',
 261         'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
 262     }]
 263     _NETRC_MACHINE = 'vrtnu'
 264     _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
 265     _CONTEXT_ID = 'R3595707040'
 266
 267     def _perform_login(self, username, password):
 268         auth_info = self._gigya_login({
 269             'APIKey': self._APIKEY,
 270             'targetEnv': 'jssdk',
 271             'loginID': username,
 272             'password': password,
 273             'authMode': 'cookie',
 274         })
 275
 276         if auth_info.get('errorDetails'):
 277             raise ExtractorError('Unable to login: VrtNU said: ' + auth_info.get('errorDetails'), expected=True)
 278
 279         # Sometimes authentication fails for no good reason, retry
 280         login_attempt = 1
 281         while login_attempt <= 3:
 282             try:
 283                 self._request_webpage('https://token.vrt.be/vrtnuinitlogin',
 284                                       None, note='Requesting XSRF Token', errnote='Could not get XSRF Token',
 285                                       query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'})
 286
 287                 post_data = {
 288                     'UID': auth_info['UID'],
 289                     'UIDSignature': auth_info['UIDSignature'],
 290                     'signatureTimestamp': auth_info['signatureTimestamp'],
 291                     '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
 292                 }
 293
 294                 self._request_webpage(
 295                     'https://login.vrt.be/perform_login',
 296                     None, note='Performing login', errnote='perform login failed',
 297                     headers={}, query={
 298                         'client_id': 'vrtnu-site'
 299                     }, data=urlencode_postdata(post_data))
 300
 301             except ExtractorError as e:
 302                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 303                     login_attempt += 1
 304                     self.report_warning('Authentication failed')
 305                     self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
 306                 else:
 307                     raise e
 308             else:
 309                 break
 310
 311     def _real_extract(self, url):
 312         display_id = self._match_id(url)
 313
 314         webpage = self._download_webpage(url, display_id)
 315
 316         attrs = extract_attributes(self._search_regex(
 317             r'(<nui-media[^>]+>)', webpage, 'media element'))
 318         video_id = attrs['videoid']
 319         publication_id = attrs.get('publicationid')
 320         if publication_id:
 321             video_id = publication_id + '$' + video_id
 322
 323         page = (self._parse_json(self._search_regex(
 324             r'digitalData\s*=\s*({.+?});', webpage, 'digial data',
 325             default='{}'), video_id, fatal=False) or {}).get('page') or {}
 326
 327         info = self._search_json_ld(webpage, display_id, default={})
 328         return merge_dicts(info, {
 329             '_type': 'url_transparent',
 330             'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
 331             'ie_key': CanvasIE.ie_key(),
 332             'id': video_id,
 333             'display_id': display_id,
 334             'season_number': int_or_none(page.get('episode_season')),
 335         })
 336
 337
 338 class DagelijkseKostIE(InfoExtractor):
 339     IE_DESC = 'dagelijksekost.een.be'
 340     _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
 341     _TEST = {
 342         'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
 343         'md5': '30bfffc323009a3e5f689bef6efa2365',
 344         'info_dict': {
 345             'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
 346             'display_id': 'hachis-parmentier-met-witloof',
 347             'ext': 'mp4',
 348             'title': 'Hachis parmentier met witloof',
 349             'description': 'md5:9960478392d87f63567b5b117688cdc5',
 350             'thumbnail': r're:^https?://.*\.jpg$',
 351             'duration': 283.02,
 352         },
 353         'expected_warnings': ['is not a supported codec'],
 354     }
 355
 356     def _real_extract(self, url):
 357         display_id = self._match_id(url)
 358         webpage = self._download_webpage(url, display_id)
 359
 360         title = strip_or_none(get_element_by_class(
 361             'dish-metadata__title', webpage
 362         ) or self._html_search_meta(
 363             'twitter:title', webpage))
 364
 365         description = clean_html(get_element_by_class(
 366             'dish-description', webpage)
 367         ) or self._html_search_meta(
 368             ('description', 'twitter:description', 'og:description'),
 369             webpage)
 370
 371         video_id = self._html_search_regex(
 372             r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
 373             group='id')
 374
 375         return {
 376             '_type': 'url_transparent',
 377             'url': 'https://mediazone.vrt.be/api/v1/dako/assets/%s' % video_id,
 378             'ie_key': CanvasIE.ie_key(),
 379             'id': video_id,
 380             'display_id': display_id,
 381             'title': title,
 382             'description': description,
 383         }