yt_dlp/extractor/canvas.py

   1 from __future__ import unicode_literals
   2 import json
   3
   4
   5 from .common import InfoExtractor
   6 from .gigya import GigyaBaseIE
   7 from ..compat import compat_HTTPError
   8 from ..utils import (
   9     ExtractorError,
  10     clean_html,
  11     extract_attributes,
  12     float_or_none,
  13     get_element_by_class,
  14     int_or_none,
  15     merge_dicts,
  16     str_or_none,
  17     strip_or_none,
  18     url_or_none,
  19     urlencode_postdata
  20 )
  21
  22
  23 class CanvasIE(InfoExtractor):
  24     _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)'
  25     _TESTS = [{
  26         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  27         'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
  28         'info_dict': {
  29             'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  30             'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  31             'ext': 'mp4',
  32             'title': 'Nachtwacht: De Greystook',
  33             'description': 'Nachtwacht: De Greystook',
  34             'thumbnail': r're:^https?://.*\.jpg$',
  35             'duration': 1468.02,
  36         },
  37         'expected_warnings': ['is not a supported codec'],
  38     }, {
  39         'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
  40         'only_matching': True,
  41     }]
  42     _GEO_BYPASS = False
  43     _HLS_ENTRY_PROTOCOLS_MAP = {
  44         'HLS': 'm3u8_native',
  45         'HLS_AES': 'm3u8_native',
  46     }
  47     _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2'
  48
  49     def _real_extract(self, url):
  50         mobj = self._match_valid_url(url)
  51         site_id, video_id = mobj.group('site_id'), mobj.group('id')
  52
  53         data = None
  54         if site_id != 'vrtvideo':
  55             # Old API endpoint, serves more formats but may fail for some videos
  56             data = self._download_json(
  57                 'https://mediazone.vrt.be/api/v1/%s/assets/%s'
  58                 % (site_id, video_id), video_id, 'Downloading asset JSON',
  59                 'Unable to download asset JSON', fatal=False)
  60
  61         # New API endpoint
  62         if not data:
  63             vrtnutoken = self._download_json('https://token.vrt.be/refreshtoken',
  64                                              video_id, note='refreshtoken: Retrieve vrtnutoken',
  65                                              errnote='refreshtoken failed')['vrtnutoken']
  66             headers = self.geo_verification_headers()
  67             headers.update({'Content-Type': 'application/json; charset=utf-8'})
  68             vrtPlayerToken = self._download_json(
  69                 '%s/tokens' % self._REST_API_BASE, video_id,
  70                 'Downloading token', headers=headers, data=json.dumps({
  71                     'identityToken': vrtnutoken
  72                 }).encode('utf-8'))['vrtPlayerToken']
  73             data = self._download_json(
  74                 '%s/videos/%s' % (self._REST_API_BASE, video_id),
  75                 video_id, 'Downloading video JSON', query={
  76                     'vrtPlayerToken': vrtPlayerToken,
  77                     'client': 'null',
  78                 }, expected_status=400)
  79             if 'title' not in data:
  80                 code = data.get('code')
  81                 if code == 'AUTHENTICATION_REQUIRED':
  82                     self.raise_login_required()
  83                 elif code == 'INVALID_LOCATION':
  84                     self.raise_geo_restricted(countries=['BE'])
  85                 raise ExtractorError(data.get('message') or code, expected=True)
  86
  87         # Note: The title may be an empty string
  88         title = data['title'] or f'{site_id} {video_id}'
  89         description = data.get('description')
  90
  91         formats = []
  92         subtitles = {}
  93         for target in data['targetUrls']:
  94             format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type'))
  95             if not format_url or not format_type:
  96                 continue
  97             format_type = format_type.upper()
  98             if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
  99                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
 100                     format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
 101                     m3u8_id=format_type, fatal=False)
 102                 formats.extend(fmts)
 103                 subtitles = self._merge_subtitles(subtitles, subs)
 104             elif format_type == 'HDS':
 105                 formats.extend(self._extract_f4m_formats(
 106                     format_url, video_id, f4m_id=format_type, fatal=False))
 107             elif format_type == 'MPEG_DASH':
 108                 fmts, subs = self._extract_mpd_formats_and_subtitles(
 109                     format_url, video_id, mpd_id=format_type, fatal=False)
 110                 formats.extend(fmts)
 111                 subtitles = self._merge_subtitles(subtitles, subs)
 112             elif format_type == 'HSS':
 113                 fmts, subs = self._extract_ism_formats_and_subtitles(
 114                     format_url, video_id, ism_id='mss', fatal=False)
 115                 formats.extend(fmts)
 116                 subtitles = self._merge_subtitles(subtitles, subs)
 117             else:
 118                 formats.append({
 119                     'format_id': format_type,
 120                     'url': format_url,
 121                 })
 122         self._sort_formats(formats)
 123
 124         subtitle_urls = data.get('subtitleUrls')
 125         if isinstance(subtitle_urls, list):
 126             for subtitle in subtitle_urls:
 127                 subtitle_url = subtitle.get('url')
 128                 if subtitle_url and subtitle.get('type') == 'CLOSED':
 129                     subtitles.setdefault('nl', []).append({'url': subtitle_url})
 130
 131         return {
 132             'id': video_id,
 133             'display_id': video_id,
 134             'title': title,
 135             'description': description,
 136             'formats': formats,
 137             'duration': float_or_none(data.get('duration'), 1000),
 138             'thumbnail': data.get('posterImageUrl'),
 139             'subtitles': subtitles,
 140         }
 141
 142
 143 class CanvasEenIE(InfoExtractor):
 144     IE_DESC = 'canvas.be and een.be'
 145     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 146     _TESTS = [{
 147         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
 148         'md5': 'ed66976748d12350b118455979cca293',
 149         'info_dict': {
 150             'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
 151             'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
 152             'ext': 'flv',
 153             'title': 'De afspraak veilt voor de Warmste Week',
 154             'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
 155             'thumbnail': r're:^https?://.*\.jpg$',
 156             'duration': 49.02,
 157         },
 158         'expected_warnings': ['is not a supported codec'],
 159     }, {
 160         # with subtitles
 161         'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
 162         'info_dict': {
 163             'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
 164             'display_id': 'pieter-0167',
 165             'ext': 'mp4',
 166             'title': 'Pieter 0167',
 167             'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
 168             'thumbnail': r're:^https?://.*\.jpg$',
 169             'duration': 2553.08,
 170             'subtitles': {
 171                 'nl': [{
 172                     'ext': 'vtt',
 173                 }],
 174             },
 175         },
 176         'params': {
 177             'skip_download': True,
 178         },
 179         'skip': 'Pagina niet gevonden',
 180     }, {
 181         'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
 182         'info_dict': {
 183             'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
 184             'display_id': 'emma-pakt-thilly-aan',
 185             'ext': 'mp4',
 186             'title': 'Emma pakt Thilly aan',
 187             'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
 188             'thumbnail': r're:^https?://.*\.jpg$',
 189             'duration': 118.24,
 190         },
 191         'params': {
 192             'skip_download': True,
 193         },
 194         'expected_warnings': ['is not a supported codec'],
 195     }, {
 196         'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
 197         'only_matching': True,
 198     }]
 199
 200     def _real_extract(self, url):
 201         mobj = self._match_valid_url(url)
 202         site_id, display_id = mobj.group('site_id'), mobj.group('id')
 203
 204         webpage = self._download_webpage(url, display_id)
 205
 206         title = strip_or_none(self._search_regex(
 207             r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
 208             webpage, 'title', default=None) or self._og_search_title(
 209             webpage, default=None))
 210
 211         video_id = self._html_search_regex(
 212             r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
 213             group='id')
 214
 215         return {
 216             '_type': 'url_transparent',
 217             'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
 218             'ie_key': CanvasIE.ie_key(),
 219             'id': video_id,
 220             'display_id': display_id,
 221             'title': title,
 222             'description': self._og_search_description(webpage),
 223         }
 224
 225
 226 class VrtNUIE(GigyaBaseIE):
 227     IE_DESC = 'VrtNU.be'
 228     _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
 229     _TESTS = [{
 230         # Available via old API endpoint
 231         'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
 232         'info_dict': {
 233             'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
 234             'ext': 'mp4',
 235             'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
 236             'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
 237             'duration': 1457.04,
 238             'thumbnail': r're:^https?://.*\.jpg$',
 239             'series': 'Postbus X',
 240             'season': 'Seizoen 1989',
 241             'season_number': 1989,
 242             'episode': 'De zwarte weduwe',
 243             'episode_number': 1,
 244             'timestamp': 1595822400,
 245             'upload_date': '20200727',
 246         },
 247         'skip': 'This video is only available for registered users',
 248         'params': {
 249             'username': '<snip>',
 250             'password': '<snip>',
 251         },
 252         'expected_warnings': ['is not a supported codec'],
 253     }, {
 254         # Only available via new API endpoint
 255         'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
 256         'info_dict': {
 257             'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
 258             'ext': 'mp4',
 259             'title': 'Aflevering 5',
 260             'description': 'Wie valt door de mand tijdens een missie?',
 261             'duration': 2967.06,
 262             'season': 'Season 1',
 263             'season_number': 1,
 264             'episode_number': 5,
 265         },
 266         'skip': 'This video is only available for registered users',
 267         'params': {
 268             'username': '<snip>',
 269             'password': '<snip>',
 270         },
 271         'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
 272     }]
 273     _NETRC_MACHINE = 'vrtnu'
 274     _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
 275     _CONTEXT_ID = 'R3595707040'
 276
 277     def _perform_login(self, username, password):
 278         auth_info = self._gigya_login({
 279             'APIKey': self._APIKEY,
 280             'targetEnv': 'jssdk',
 281             'loginID': username,
 282             'password': password,
 283             'authMode': 'cookie',
 284         })
 285
 286         if auth_info.get('errorDetails'):
 287             raise ExtractorError('Unable to login: VrtNU said: ' + auth_info.get('errorDetails'), expected=True)
 288
 289         # Sometimes authentication fails for no good reason, retry
 290         login_attempt = 1
 291         while login_attempt <= 3:
 292             try:
 293                 self._request_webpage('https://token.vrt.be/vrtnuinitlogin',
 294                                       None, note='Requesting XSRF Token', errnote='Could not get XSRF Token',
 295                                       query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'})
 296
 297                 post_data = {
 298                     'UID': auth_info['UID'],
 299                     'UIDSignature': auth_info['UIDSignature'],
 300                     'signatureTimestamp': auth_info['signatureTimestamp'],
 301                     '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
 302                 }
 303
 304                 self._request_webpage(
 305                     'https://login.vrt.be/perform_login',
 306                     None, note='Performing login', errnote='perform login failed',
 307                     headers={}, query={
 308                         'client_id': 'vrtnu-site'
 309                     }, data=urlencode_postdata(post_data))
 310
 311             except ExtractorError as e:
 312                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 313                     login_attempt += 1
 314                     self.report_warning('Authentication failed')
 315                     self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
 316                 else:
 317                     raise e
 318             else:
 319                 break
 320
 321     def _real_extract(self, url):
 322         display_id = self._match_id(url)
 323
 324         webpage = self._download_webpage(url, display_id)
 325
 326         attrs = extract_attributes(self._search_regex(
 327             r'(<nui-media[^>]+>)', webpage, 'media element'))
 328         video_id = attrs['videoid']
 329         publication_id = attrs.get('publicationid')
 330         if publication_id:
 331             video_id = publication_id + '$' + video_id
 332
 333         page = (self._parse_json(self._search_regex(
 334             r'digitalData\s*=\s*({.+?});', webpage, 'digial data',
 335             default='{}'), video_id, fatal=False) or {}).get('page') or {}
 336
 337         info = self._search_json_ld(webpage, display_id, default={})
 338         return merge_dicts(info, {
 339             '_type': 'url_transparent',
 340             'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
 341             'ie_key': CanvasIE.ie_key(),
 342             'id': video_id,
 343             'display_id': display_id,
 344             'season_number': int_or_none(page.get('episode_season')),
 345         })
 346
 347
 348 class DagelijkseKostIE(InfoExtractor):
 349     IE_DESC = 'dagelijksekost.een.be'
 350     _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
 351     _TEST = {
 352         'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
 353         'md5': '30bfffc323009a3e5f689bef6efa2365',
 354         'info_dict': {
 355             'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
 356             'display_id': 'hachis-parmentier-met-witloof',
 357             'ext': 'mp4',
 358             'title': 'Hachis parmentier met witloof',
 359             'description': 'md5:9960478392d87f63567b5b117688cdc5',
 360             'thumbnail': r're:^https?://.*\.jpg$',
 361             'duration': 283.02,
 362         },
 363         'expected_warnings': ['is not a supported codec'],
 364     }
 365
 366     def _real_extract(self, url):
 367         display_id = self._match_id(url)
 368         webpage = self._download_webpage(url, display_id)
 369
 370         title = strip_or_none(get_element_by_class(
 371             'dish-metadata__title', webpage
 372         ) or self._html_search_meta(
 373             'twitter:title', webpage))
 374
 375         description = clean_html(get_element_by_class(
 376             'dish-description', webpage)
 377         ) or self._html_search_meta(
 378             ('description', 'twitter:description', 'og:description'),
 379             webpage)
 380
 381         video_id = self._html_search_regex(
 382             r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
 383             group='id')
 384
 385         return {
 386             '_type': 'url_transparent',
 387             'url': 'https://mediazone.vrt.be/api/v1/dako/assets/%s' % video_id,
 388             'ie_key': CanvasIE.ie_key(),
 389             'id': video_id,
 390             'display_id': display_id,
 391             'title': title,
 392             'description': description,
 393         }