youtube_dlc/extractor/canvas.py

   1 from __future__ import unicode_literals
   2
   3 import re
   4 import json
   5
   6 from .common import InfoExtractor
   7 from .gigya import GigyaBaseIE
   8 from ..compat import compat_HTTPError
   9 from ..utils import (
  10     extract_attributes,
  11     ExtractorError,
  12     strip_or_none,
  13     float_or_none,
  14     int_or_none,
  15     merge_dicts,
  16     str_or_none,
  17     url_or_none,
  18 )
  19
  20
  21 class CanvasIE(InfoExtractor):
  22     _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza)/assets/(?P<id>[^/?#&]+)'
  23     _TESTS = [{
  24         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  25         'md5': '68993eda72ef62386a15ea2cf3c93107',
  26         'info_dict': {
  27             'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  28             'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  29             'ext': 'mp4',
  30             'title': 'Nachtwacht: De Greystook',
  31             'description': 'Nachtwacht: De Greystook',
  32             'thumbnail': r're:^https?://.*\.jpg$',
  33             'duration': 1468.04,
  34         },
  35         'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
  36     }, {
  37         'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
  38         'only_matching': True,
  39     }]
  40     _GEO_BYPASS = False
  41     _HLS_ENTRY_PROTOCOLS_MAP = {
  42         'HLS': 'm3u8_native',
  43         'HLS_AES': 'm3u8',
  44     }
  45     _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v1'
  46
  47     def _real_extract(self, url):
  48         mobj = re.match(self._VALID_URL, url)
  49         site_id, video_id = mobj.group('site_id'), mobj.group('id')
  50
  51         data = None
  52         if site_id != 'vrtvideo':
  53             # Old API endpoint, serves more formats but may fail for some videos
  54             data = self._download_json(
  55                 'https://mediazone.vrt.be/api/v1/%s/assets/%s'
  56                 % (site_id, video_id), video_id, 'Downloading asset JSON',
  57                 'Unable to download asset JSON', fatal=False)
  58
  59         # New API endpoint
  60         if not data:
  61             headers = self.geo_verification_headers()
  62             headers.update({'Content-Type': 'application/json'})
  63             token = self._download_json(
  64                 '%s/tokens' % self._REST_API_BASE, video_id,
  65                 'Downloading token', data=b'', headers=headers)['vrtPlayerToken']
  66             data = self._download_json(
  67                 '%s/videos/%s' % (self._REST_API_BASE, video_id),
  68                 video_id, 'Downloading video JSON', query={
  69                     'vrtPlayerToken': token,
  70                     'client': '%s@PROD' % site_id,
  71                 }, expected_status=400)
  72             if not data.get('title'):
  73                 code = data.get('code')
  74                 if code == 'AUTHENTICATION_REQUIRED':
  75                     self.raise_login_required()
  76                 elif code == 'INVALID_LOCATION':
  77                     self.raise_geo_restricted(countries=['BE'])
  78                 raise ExtractorError(data.get('message') or code, expected=True)
  79
  80         title = data['title']
  81         description = data.get('description')
  82
  83         formats = []
  84         for target in data['targetUrls']:
  85             format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type'))
  86             if not format_url or not format_type:
  87                 continue
  88             format_type = format_type.upper()
  89             if format_type in self._HLS_ENTRY_PROTOCOLS_MAP:
  90                 formats.extend(self._extract_m3u8_formats(
  91                     format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type],
  92                     m3u8_id=format_type, fatal=False))
  93             elif format_type == 'HDS':
  94                 formats.extend(self._extract_f4m_formats(
  95                     format_url, video_id, f4m_id=format_type, fatal=False))
  96             elif format_type == 'MPEG_DASH':
  97                 formats.extend(self._extract_mpd_formats(
  98                     format_url, video_id, mpd_id=format_type, fatal=False))
  99             elif format_type == 'HSS':
 100                 formats.extend(self._extract_ism_formats(
 101                     format_url, video_id, ism_id='mss', fatal=False))
 102             else:
 103                 formats.append({
 104                     'format_id': format_type,
 105                     'url': format_url,
 106                 })
 107         self._sort_formats(formats)
 108
 109         subtitles = {}
 110         subtitle_urls = data.get('subtitleUrls')
 111         if isinstance(subtitle_urls, list):
 112             for subtitle in subtitle_urls:
 113                 subtitle_url = subtitle.get('url')
 114                 if subtitle_url and subtitle.get('type') == 'CLOSED':
 115                     subtitles.setdefault('nl', []).append({'url': subtitle_url})
 116
 117         return {
 118             'id': video_id,
 119             'display_id': video_id,
 120             'title': title,
 121             'description': description,
 122             'formats': formats,
 123             'duration': float_or_none(data.get('duration'), 1000),
 124             'thumbnail': data.get('posterImageUrl'),
 125             'subtitles': subtitles,
 126         }
 127
 128
 129 class CanvasEenIE(InfoExtractor):
 130     IE_DESC = 'canvas.be and een.be'
 131     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 132     _TESTS = [{
 133         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
 134         'md5': 'ed66976748d12350b118455979cca293',
 135         'info_dict': {
 136             'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
 137             'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
 138             'ext': 'flv',
 139             'title': 'De afspraak veilt voor de Warmste Week',
 140             'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
 141             'thumbnail': r're:^https?://.*\.jpg$',
 142             'duration': 49.02,
 143         },
 144         'expected_warnings': ['is not a supported codec'],
 145     }, {
 146         # with subtitles
 147         'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
 148         'info_dict': {
 149             'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
 150             'display_id': 'pieter-0167',
 151             'ext': 'mp4',
 152             'title': 'Pieter 0167',
 153             'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
 154             'thumbnail': r're:^https?://.*\.jpg$',
 155             'duration': 2553.08,
 156             'subtitles': {
 157                 'nl': [{
 158                     'ext': 'vtt',
 159                 }],
 160             },
 161         },
 162         'params': {
 163             'skip_download': True,
 164         },
 165         'skip': 'Pagina niet gevonden',
 166     }, {
 167         'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan',
 168         'info_dict': {
 169             'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8',
 170             'display_id': 'emma-pakt-thilly-aan',
 171             'ext': 'mp4',
 172             'title': 'Emma pakt Thilly aan',
 173             'description': 'md5:c5c9b572388a99b2690030afa3f3bad7',
 174             'thumbnail': r're:^https?://.*\.jpg$',
 175             'duration': 118.24,
 176         },
 177         'params': {
 178             'skip_download': True,
 179         },
 180         'expected_warnings': ['is not a supported codec'],
 181     }, {
 182         'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
 183         'only_matching': True,
 184     }]
 185
 186     def _real_extract(self, url):
 187         mobj = re.match(self._VALID_URL, url)
 188         site_id, display_id = mobj.group('site_id'), mobj.group('id')
 189
 190         webpage = self._download_webpage(url, display_id)
 191
 192         title = strip_or_none(self._search_regex(
 193             r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
 194             webpage, 'title', default=None) or self._og_search_title(
 195             webpage, default=None))
 196
 197         video_id = self._html_search_regex(
 198             r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
 199             group='id')
 200
 201         return {
 202             '_type': 'url_transparent',
 203             'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
 204             'ie_key': CanvasIE.ie_key(),
 205             'id': video_id,
 206             'display_id': display_id,
 207             'title': title,
 208             'description': self._og_search_description(webpage),
 209         }
 210
 211
 212 class VrtNUIE(GigyaBaseIE):
 213     IE_DESC = 'VrtNU.be'
 214     _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
 215     _TESTS = [{
 216         # Available via old API endpoint
 217         'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
 218         'info_dict': {
 219             'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
 220             'ext': 'mp4',
 221             'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
 222             'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
 223             'duration': 1457.04,
 224             'thumbnail': r're:^https?://.*\.jpg$',
 225             'series': 'Postbus X',
 226             'season': 'Seizoen 1989',
 227             'season_number': 1989,
 228             'episode': 'De zwarte weduwe',
 229             'episode_number': 1,
 230             'timestamp': 1595822400,
 231             'upload_date': '20200727',
 232         },
 233         'skip': 'This video is only available for registered users',
 234         'params': {
 235             'username': '<snip>',
 236             'password': '<snip>',
 237         },
 238         'expected_warnings': ['is not a supported codec'],
 239     }, {
 240         # Only available via new API endpoint
 241         'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/',
 242         'info_dict': {
 243             'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1',
 244             'ext': 'mp4',
 245             'title': 'Aflevering 5',
 246             'description': 'Wie valt door de mand tijdens een missie?',
 247             'duration': 2967.06,
 248             'season': 'Season 1',
 249             'season_number': 1,
 250             'episode_number': 5,
 251         },
 252         'skip': 'This video is only available for registered users',
 253         'params': {
 254             'username': '<snip>',
 255             'password': '<snip>',
 256         },
 257         'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
 258     }]
 259     _NETRC_MACHINE = 'vrtnu'
 260     _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
 261     _CONTEXT_ID = 'R3595707040'
 262
 263     def _real_initialize(self):
 264         self._login()
 265
 266     def _login(self):
 267         username, password = self._get_login_info()
 268         if username is None:
 269             return
 270
 271         auth_data = {
 272             'APIKey': self._APIKEY,
 273             'targetEnv': 'jssdk',
 274             'loginID': username,
 275             'password': password,
 276             'authMode': 'cookie',
 277         }
 278
 279         auth_info = self._gigya_login(auth_data)
 280
 281         # Sometimes authentication fails for no good reason, retry
 282         login_attempt = 1
 283         while login_attempt <= 3:
 284             try:
 285                 # When requesting a token, no actual token is returned, but the
 286                 # necessary cookies are set.
 287                 self._request_webpage(
 288                     'https://token.vrt.be',
 289                     None, note='Requesting a token', errnote='Could not get a token',
 290                     headers={
 291                         'Content-Type': 'application/json',
 292                         'Referer': 'https://www.vrt.be/vrtnu/',
 293                     },
 294                     data=json.dumps({
 295                         'uid': auth_info['UID'],
 296                         'uidsig': auth_info['UIDSignature'],
 297                         'ts': auth_info['signatureTimestamp'],
 298                         'email': auth_info['profile']['email'],
 299                     }).encode('utf-8'))
 300             except ExtractorError as e:
 301                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 302                     login_attempt += 1
 303                     self.report_warning('Authentication failed')
 304                     self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
 305                 else:
 306                     raise e
 307             else:
 308                 break
 309
 310     def _real_extract(self, url):
 311         display_id = self._match_id(url)
 312
 313         webpage = self._download_webpage(url, display_id)
 314
 315         attrs = extract_attributes(self._search_regex(
 316             r'(<nui-media[^>]+>)', webpage, 'media element'))
 317         video_id = attrs['videoid']
 318         publication_id = attrs.get('publicationid')
 319         if publication_id:
 320             video_id = publication_id + '$' + video_id
 321
 322         page = (self._parse_json(self._search_regex(
 323             r'digitalData\s*=\s*({.+?});', webpage, 'digial data',
 324             default='{}'), video_id, fatal=False) or {}).get('page') or {}
 325
 326         info = self._search_json_ld(webpage, display_id, default={})
 327         return merge_dicts(info, {
 328             '_type': 'url_transparent',
 329             'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
 330             'ie_key': CanvasIE.ie_key(),
 331             'id': video_id,
 332             'display_id': display_id,
 333             'season_number': int_or_none(page.get('episode_season')),
 334         })