yt_dlp/extractor/rtve.py

   1 import base64
   2 import io
   3
   4 from .common import InfoExtractor
   5 from ..compat import (
   6     compat_b64decode,
   7     compat_struct_unpack,
   8 )
   9 from ..utils import (
  10     determine_ext,
  11     ExtractorError,
  12     float_or_none,
  13     qualities,
  14     remove_end,
  15     remove_start,
  16     try_get,
  17 )
  18
  19
  20 class RTVEALaCartaIE(InfoExtractor):
  21     IE_NAME = 'rtve.es:alacarta'
  22     IE_DESC = 'RTVE a la carta'
  23     _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?(alacarta/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
  24
  25     _TESTS = [{
  26         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
  27         'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
  28         'info_dict': {
  29             'id': '2491869',
  30             'ext': 'mp4',
  31             'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
  32             'duration': 5024.566,
  33             'series': 'Balonmano',
  34         },
  35         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
  36     }, {
  37         'note': 'Live stream',
  38         'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',
  39         'info_dict': {
  40             'id': '1694255',
  41             'ext': 'mp4',
  42             'title': 're:^24H LIVE [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
  43             'is_live': True,
  44         },
  45         'params': {
  46             'skip_download': 'live stream',
  47         },
  48     }, {
  49         'url': 'http://www.rtve.es/alacarta/videos/servir-y-proteger/servir-proteger-capitulo-104/4236788/',
  50         'md5': 'd850f3c8731ea53952ebab489cf81cbf',
  51         'info_dict': {
  52             'id': '4236788',
  53             'ext': 'mp4',
  54             'title': 'Servir y proteger - Capítulo 104',
  55             'duration': 3222.0,
  56         },
  57         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
  58     }, {
  59         'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
  60         'only_matching': True,
  61     }, {
  62         'url': 'http://www.rtve.es/filmoteca/no-do/not-1-introduccion-primer-noticiario-espanol/1465256/',
  63         'only_matching': True,
  64     }]
  65
  66     def _real_initialize(self):
  67         user_agent_b64 = base64.b64encode(self.get_param('http_headers')['User-Agent'].encode('utf-8')).decode('utf-8')
  68         self._manager = self._download_json(
  69             'http://www.rtve.es/odin/loki/' + user_agent_b64,
  70             None, 'Fetching manager info')['manager']
  71
  72     @staticmethod
  73     def _decrypt_url(png):
  74         encrypted_data = io.BytesIO(compat_b64decode(png)[8:])
  75         while True:
  76             length = compat_struct_unpack('!I', encrypted_data.read(4))[0]
  77             chunk_type = encrypted_data.read(4)
  78             if chunk_type == b'IEND':
  79                 break
  80             data = encrypted_data.read(length)
  81             if chunk_type == b'tEXt':
  82                 alphabet_data, text = data.split(b'\0')
  83                 quality, url_data = text.split(b'%%')
  84                 alphabet = []
  85                 e = 0
  86                 d = 0
  87                 for l in alphabet_data.decode('iso-8859-1'):
  88                     if d == 0:
  89                         alphabet.append(l)
  90                         d = e = (e + 1) % 4
  91                     else:
  92                         d -= 1
  93                 url = ''
  94                 f = 0
  95                 e = 3
  96                 b = 1
  97                 for letter in url_data.decode('iso-8859-1'):
  98                     if f == 0:
  99                         l = int(letter) * 10
 100                         f = 1
 101                     else:
 102                         if e == 0:
 103                             l += int(letter)
 104                             url += alphabet[l]
 105                             e = (b + 3) % 4
 106                             f = 0
 107                             b += 1
 108                         else:
 109                             e -= 1
 110
 111                 yield quality.decode(), url
 112             encrypted_data.read(4)  # CRC
 113
 114     def _extract_png_formats(self, video_id):
 115         png = self._download_webpage(
 116             'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id),
 117             video_id, 'Downloading url information', query={'q': 'v2'})
 118         q = qualities(['Media', 'Alta', 'HQ', 'HD_READY', 'HD_FULL'])
 119         formats = []
 120         for quality, video_url in self._decrypt_url(png):
 121             ext = determine_ext(video_url)
 122             if ext == 'm3u8':
 123                 formats.extend(self._extract_m3u8_formats(
 124                     video_url, video_id, 'mp4', 'm3u8_native',
 125                     m3u8_id='hls', fatal=False))
 126             elif ext == 'mpd':
 127                 formats.extend(self._extract_mpd_formats(
 128                     video_url, video_id, 'dash', fatal=False))
 129             else:
 130                 formats.append({
 131                     'format_id': quality,
 132                     'quality': q(quality),
 133                     'url': video_url,
 134                 })
 135         self._sort_formats(formats)
 136         return formats
 137
 138     def _real_extract(self, url):
 139         video_id = self._match_id(url)
 140         info = self._download_json(
 141             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
 142             video_id)['page']['items'][0]
 143         if info['state'] == 'DESPU':
 144             raise ExtractorError('The video is no longer available', expected=True)
 145         title = info['title'].strip()
 146         formats = self._extract_png_formats(video_id)
 147
 148         subtitles = None
 149         sbt_file = info.get('sbtFile')
 150         if sbt_file:
 151             subtitles = self.extract_subtitles(video_id, sbt_file)
 152
 153         is_live = info.get('live') is True
 154
 155         return {
 156             'id': video_id,
 157             'title': title,
 158             'formats': formats,
 159             'thumbnail': info.get('image'),
 160             'subtitles': subtitles,
 161             'duration': float_or_none(info.get('duration'), 1000),
 162             'is_live': is_live,
 163             'series': info.get('programTitle'),
 164         }
 165
 166     def _get_subtitles(self, video_id, sub_file):
 167         subs = self._download_json(
 168             sub_file + '.json', video_id,
 169             'Downloading subtitles info')['page']['items']
 170         return dict(
 171             (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
 172             for s in subs)
 173
 174
 175 class RTVEAudioIE(RTVEALaCartaIE):
 176     IE_NAME = 'rtve.es:audio'
 177     IE_DESC = 'RTVE audio'
 178     _VALID_URL = r'https?://(?:www\.)?rtve\.es/(alacarta|play)/audios/[^/]+/[^/]+/(?P<id>[0-9]+)'
 179
 180     _TESTS = [{
 181         'url': 'https://www.rtve.es/alacarta/audios/a-hombros-de-gigantes/palabra-ingeniero-codigos-informaticos-27-04-21/5889192/',
 182         'md5': 'ae06d27bff945c4e87a50f89f6ce48ce',
 183         'info_dict': {
 184             'id': '5889192',
 185             'ext': 'mp3',
 186             'title': 'Códigos informáticos',
 187             'thumbnail': r're:https?://.+/1598856591583.jpg',
 188             'duration': 349.440,
 189             'series': 'A hombros de gigantes',
 190         },
 191     }, {
 192         'url': 'https://www.rtve.es/play/audios/en-radio-3/ignatius-farray/5791165/',
 193         'md5': '072855ab89a9450e0ba314c717fa5ebc',
 194         'info_dict': {
 195             'id': '5791165',
 196             'ext': 'mp3',
 197             'title': 'Ignatius Farray',
 198             'thumbnail': r're:https?://.+/1613243011863.jpg',
 199             'duration': 3559.559,
 200             'series': 'En Radio 3'
 201         },
 202     }, {
 203         'url': 'https://www.rtve.es/play/audios/frankenstein-o-el-moderno-prometeo/capitulo-26-ultimo-muerte-victor-juan-jose-plans-mary-shelley/6082623/',
 204         'md5': '0eadab248cc8dd193fa5765712e84d5c',
 205         'info_dict': {
 206             'id': '6082623',
 207             'ext': 'mp3',
 208             'title': 'Capítulo 26 y último: La muerte de Victor',
 209             'thumbnail': r're:https?://.+/1632147445707.jpg',
 210             'duration': 3174.086,
 211             'series': 'Frankenstein o el moderno Prometeo'
 212         },
 213     }]
 214
 215     def _extract_png_formats(self, audio_id):
 216         """
 217         This function retrieves media related png thumbnail which obfuscate
 218         valuable information about the media. This information is decrypted
 219         via base class _decrypt_url function providing media quality and
 220         media url
 221         """
 222         png = self._download_webpage(
 223             'http://www.rtve.es/ztnr/movil/thumbnail/%s/audios/%s.png' %
 224             (self._manager, audio_id),
 225             audio_id, 'Downloading url information', query={'q': 'v2'})
 226         q = qualities(['Media', 'Alta', 'HQ', 'HD_READY', 'HD_FULL'])
 227         formats = []
 228         for quality, audio_url in self._decrypt_url(png):
 229             ext = determine_ext(audio_url)
 230             if ext == 'm3u8':
 231                 formats.extend(self._extract_m3u8_formats(
 232                     audio_url, audio_id, 'mp4', 'm3u8_native',
 233                     m3u8_id='hls', fatal=False))
 234             elif ext == 'mpd':
 235                 formats.extend(self._extract_mpd_formats(
 236                     audio_url, audio_id, 'dash', fatal=False))
 237             else:
 238                 formats.append({
 239                     'format_id': quality,
 240                     'quality': q(quality),
 241                     'url': audio_url,
 242                 })
 243         self._sort_formats(formats)
 244         return formats
 245
 246     def _real_extract(self, url):
 247         audio_id = self._match_id(url)
 248         info = self._download_json(
 249             'https://www.rtve.es/api/audios/%s.json' % audio_id,
 250             audio_id)['page']['items'][0]
 251
 252         return {
 253             'id': audio_id,
 254             'title': info['title'].strip(),
 255             'thumbnail': info.get('thumbnail'),
 256             'duration': float_or_none(info.get('duration'), 1000),
 257             'series': try_get(info, lambda x: x['programInfo']['title']),
 258             'formats': self._extract_png_formats(audio_id),
 259         }
 260
 261
 262 class RTVEInfantilIE(RTVEALaCartaIE):
 263     IE_NAME = 'rtve.es:infantil'
 264     IE_DESC = 'RTVE infantil'
 265     _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/[^/]+/video/[^/]+/(?P<id>[0-9]+)/'
 266
 267     _TESTS = [{
 268         'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
 269         'md5': '5747454717aedf9f9fdf212d1bcfc48d',
 270         'info_dict': {
 271             'id': '3040283',
 272             'ext': 'mp4',
 273             'title': 'Maneras de vivir',
 274             'thumbnail': r're:https?://.+/1426182947956\.JPG',
 275             'duration': 357.958,
 276         },
 277         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
 278     }]
 279
 280
 281 class RTVELiveIE(RTVEALaCartaIE):
 282     IE_NAME = 'rtve.es:live'
 283     IE_DESC = 'RTVE.es live streams'
 284     _VALID_URL = r'https?://(?:www\.)?rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
 285
 286     _TESTS = [{
 287         'url': 'http://www.rtve.es/directo/la-1/',
 288         'info_dict': {
 289             'id': 'la-1',
 290             'ext': 'mp4',
 291             'title': 're:^La 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 292         },
 293         'params': {
 294             'skip_download': 'live stream',
 295         }
 296     }]
 297
 298     def _real_extract(self, url):
 299         mobj = self._match_valid_url(url)
 300         video_id = mobj.group('id')
 301
 302         webpage = self._download_webpage(url, video_id)
 303         title = remove_end(self._og_search_title(webpage), ' en directo en RTVE.es')
 304         title = remove_start(title, 'Estoy viendo ')
 305
 306         vidplayer_id = self._search_regex(
 307             (r'playerId=player([0-9]+)',
 308              r'class=["\'].*?\blive_mod\b.*?["\'][^>]+data-assetid=["\'](\d+)',
 309              r'data-id=["\'](\d+)'),
 310             webpage, 'internal video ID')
 311
 312         return {
 313             'id': video_id,
 314             'title': title,
 315             'formats': self._extract_png_formats(vidplayer_id),
 316             'is_live': True,
 317         }
 318
 319
 320 class RTVETelevisionIE(InfoExtractor):
 321     IE_NAME = 'rtve.es:television'
 322     _VALID_URL = r'https?://(?:www\.)?rtve\.es/television/[^/]+/[^/]+/(?P<id>\d+).shtml'
 323
 324     _TEST = {
 325         'url': 'http://www.rtve.es/television/20160628/revolucion-del-movil/1364141.shtml',
 326         'info_dict': {
 327             'id': '3069778',
 328             'ext': 'mp4',
 329             'title': 'Documentos TV - La revolución del móvil',
 330             'duration': 3496.948,
 331         },
 332         'params': {
 333             'skip_download': True,
 334         },
 335     }
 336
 337     def _real_extract(self, url):
 338         page_id = self._match_id(url)
 339         webpage = self._download_webpage(url, page_id)
 340
 341         alacarta_url = self._search_regex(
 342             r'data-location="alacarta_videos"[^<]+url&quot;:&quot;(http://www\.rtve\.es/alacarta.+?)&',
 343             webpage, 'alacarta url', default=None)
 344         if alacarta_url is None:
 345             raise ExtractorError(
 346                 'The webpage doesn\'t contain any video', expected=True)
 347
 348         return self.url_result(alacarta_url, ie=RTVEALaCartaIE.ie_key())