yt_dlp/extractor/rtve.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import base64
   5 import io
   6 import sys
   7
   8 from .common import InfoExtractor
   9 from ..compat import (
  10     compat_b64decode,
  11     compat_struct_unpack,
  12 )
  13 from ..utils import (
  14     determine_ext,
  15     ExtractorError,
  16     float_or_none,
  17     qualities,
  18     remove_end,
  19     remove_start,
  20     std_headers,
  21     try_get,
  22 )
  23
  24 _bytes_to_chr = (lambda x: x) if sys.version_info[0] == 2 else (lambda x: map(chr, x))
  25
  26
  27 class RTVEALaCartaIE(InfoExtractor):
  28     IE_NAME = 'rtve.es:alacarta'
  29     IE_DESC = 'RTVE a la carta'
  30     _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?(alacarta/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
  31
  32     _TESTS = [{
  33         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
  34         'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
  35         'info_dict': {
  36             'id': '2491869',
  37             'ext': 'mp4',
  38             'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
  39             'duration': 5024.566,
  40             'series': 'Balonmano',
  41         },
  42         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
  43     }, {
  44         'note': 'Live stream',
  45         'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',
  46         'info_dict': {
  47             'id': '1694255',
  48             'ext': 'mp4',
  49             'title': 're:^24H LIVE [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
  50             'is_live': True,
  51         },
  52         'params': {
  53             'skip_download': 'live stream',
  54         },
  55     }, {
  56         'url': 'http://www.rtve.es/alacarta/videos/servir-y-proteger/servir-proteger-capitulo-104/4236788/',
  57         'md5': 'd850f3c8731ea53952ebab489cf81cbf',
  58         'info_dict': {
  59             'id': '4236788',
  60             'ext': 'mp4',
  61             'title': 'Servir y proteger - Capítulo 104',
  62             'duration': 3222.0,
  63         },
  64         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
  65     }, {
  66         'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
  67         'only_matching': True,
  68     }, {
  69         'url': 'http://www.rtve.es/filmoteca/no-do/not-1-introduccion-primer-noticiario-espanol/1465256/',
  70         'only_matching': True,
  71     }]
  72
  73     def _real_initialize(self):
  74         user_agent_b64 = base64.b64encode(std_headers['User-Agent'].encode('utf-8')).decode('utf-8')
  75         self._manager = self._download_json(
  76             'http://www.rtve.es/odin/loki/' + user_agent_b64,
  77             None, 'Fetching manager info')['manager']
  78
  79     @staticmethod
  80     def _decrypt_url(png):
  81         encrypted_data = io.BytesIO(compat_b64decode(png)[8:])
  82         while True:
  83             length = compat_struct_unpack('!I', encrypted_data.read(4))[0]
  84             chunk_type = encrypted_data.read(4)
  85             if chunk_type == b'IEND':
  86                 break
  87             data = encrypted_data.read(length)
  88             if chunk_type == b'tEXt':
  89                 alphabet_data, text = data.split(b'\0')
  90                 quality, url_data = text.split(b'%%')
  91                 alphabet = []
  92                 e = 0
  93                 d = 0
  94                 for l in _bytes_to_chr(alphabet_data):
  95                     if d == 0:
  96                         alphabet.append(l)
  97                         d = e = (e + 1) % 4
  98                     else:
  99                         d -= 1
 100                 url = ''
 101                 f = 0
 102                 e = 3
 103                 b = 1
 104                 for letter in _bytes_to_chr(url_data):
 105                     if f == 0:
 106                         l = int(letter) * 10
 107                         f = 1
 108                     else:
 109                         if e == 0:
 110                             l += int(letter)
 111                             url += alphabet[l]
 112                             e = (b + 3) % 4
 113                             f = 0
 114                             b += 1
 115                         else:
 116                             e -= 1
 117
 118                 yield quality.decode(), url
 119             encrypted_data.read(4)  # CRC
 120
 121     def _extract_png_formats(self, video_id):
 122         png = self._download_webpage(
 123             'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id),
 124             video_id, 'Downloading url information', query={'q': 'v2'})
 125         q = qualities(['Media', 'Alta', 'HQ', 'HD_READY', 'HD_FULL'])
 126         formats = []
 127         for quality, video_url in self._decrypt_url(png):
 128             ext = determine_ext(video_url)
 129             if ext == 'm3u8':
 130                 formats.extend(self._extract_m3u8_formats(
 131                     video_url, video_id, 'mp4', 'm3u8_native',
 132                     m3u8_id='hls', fatal=False))
 133             elif ext == 'mpd':
 134                 formats.extend(self._extract_mpd_formats(
 135                     video_url, video_id, 'dash', fatal=False))
 136             else:
 137                 formats.append({
 138                     'format_id': quality,
 139                     'quality': q(quality),
 140                     'url': video_url,
 141                 })
 142         self._sort_formats(formats)
 143         return formats
 144
 145     def _real_extract(self, url):
 146         video_id = self._match_id(url)
 147         info = self._download_json(
 148             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
 149             video_id)['page']['items'][0]
 150         if info['state'] == 'DESPU':
 151             raise ExtractorError('The video is no longer available', expected=True)
 152         title = info['title'].strip()
 153         formats = self._extract_png_formats(video_id)
 154
 155         subtitles = None
 156         sbt_file = info.get('sbtFile')
 157         if sbt_file:
 158             subtitles = self.extract_subtitles(video_id, sbt_file)
 159
 160         is_live = info.get('live') is True
 161
 162         return {
 163             'id': video_id,
 164             'title': title,
 165             'formats': formats,
 166             'thumbnail': info.get('image'),
 167             'subtitles': subtitles,
 168             'duration': float_or_none(info.get('duration'), 1000),
 169             'is_live': is_live,
 170             'series': info.get('programTitle'),
 171         }
 172
 173     def _get_subtitles(self, video_id, sub_file):
 174         subs = self._download_json(
 175             sub_file + '.json', video_id,
 176             'Downloading subtitles info')['page']['items']
 177         return dict(
 178             (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
 179             for s in subs)
 180
 181
 182 class RTVEAudioIE(RTVEALaCartaIE):
 183     IE_NAME = 'rtve.es:audio'
 184     IE_DESC = 'RTVE audio'
 185     _VALID_URL = r'https?://(?:www\.)?rtve\.es/(alacarta|play)/audios/[^/]+/[^/]+/(?P<id>[0-9]+)'
 186
 187     _TESTS = [{
 188         'url': 'https://www.rtve.es/alacarta/audios/a-hombros-de-gigantes/palabra-ingeniero-codigos-informaticos-27-04-21/5889192/',
 189         'md5': 'ae06d27bff945c4e87a50f89f6ce48ce',
 190         'info_dict': {
 191             'id': '5889192',
 192             'ext': 'mp3',
 193             'title': 'Códigos informáticos',
 194             'thumbnail': r're:https?://.+/1598856591583.jpg',
 195             'duration': 349.440,
 196             'series': 'A hombros de gigantes',
 197         },
 198     }, {
 199         'url': 'https://www.rtve.es/play/audios/en-radio-3/ignatius-farray/5791165/',
 200         'md5': '072855ab89a9450e0ba314c717fa5ebc',
 201         'info_dict': {
 202             'id': '5791165',
 203             'ext': 'mp3',
 204             'title': 'Ignatius Farray',
 205             'thumbnail': r're:https?://.+/1613243011863.jpg',
 206             'duration': 3559.559,
 207             'series': 'En Radio 3'
 208         },
 209     }, {
 210         'url': 'https://www.rtve.es/play/audios/frankenstein-o-el-moderno-prometeo/capitulo-26-ultimo-muerte-victor-juan-jose-plans-mary-shelley/6082623/',
 211         'md5': '0eadab248cc8dd193fa5765712e84d5c',
 212         'info_dict': {
 213             'id': '6082623',
 214             'ext': 'mp3',
 215             'title': 'Capítulo 26 y último: La muerte de Victor',
 216             'thumbnail': r're:https?://.+/1632147445707.jpg',
 217             'duration': 3174.086,
 218             'series': 'Frankenstein o el moderno Prometeo'
 219         },
 220     }]
 221
 222     def _extract_png_formats(self, audio_id):
 223         """
 224         This function retrieves media related png thumbnail which obfuscate
 225         valuable information about the media. This information is decrypted
 226         via base class _decrypt_url function providing media quality and
 227         media url
 228         """
 229         png = self._download_webpage(
 230             'http://www.rtve.es/ztnr/movil/thumbnail/%s/audios/%s.png' %
 231             (self._manager, audio_id),
 232             audio_id, 'Downloading url information', query={'q': 'v2'})
 233         q = qualities(['Media', 'Alta', 'HQ', 'HD_READY', 'HD_FULL'])
 234         formats = []
 235         for quality, audio_url in self._decrypt_url(png):
 236             ext = determine_ext(audio_url)
 237             if ext == 'm3u8':
 238                 formats.extend(self._extract_m3u8_formats(
 239                     audio_url, audio_id, 'mp4', 'm3u8_native',
 240                     m3u8_id='hls', fatal=False))
 241             elif ext == 'mpd':
 242                 formats.extend(self._extract_mpd_formats(
 243                     audio_url, audio_id, 'dash', fatal=False))
 244             else:
 245                 formats.append({
 246                     'format_id': quality,
 247                     'quality': q(quality),
 248                     'url': audio_url,
 249                 })
 250         self._sort_formats(formats)
 251         return formats
 252
 253     def _real_extract(self, url):
 254         audio_id = self._match_id(url)
 255         info = self._download_json(
 256             'https://www.rtve.es/api/audios/%s.json' % audio_id,
 257             audio_id)['page']['items'][0]
 258
 259         return {
 260             'id': audio_id,
 261             'title': info['title'].strip(),
 262             'thumbnail': info.get('thumbnail'),
 263             'duration': float_or_none(info.get('duration'), 1000),
 264             'series': try_get(info, lambda x: x['programInfo']['title']),
 265             'formats': self._extract_png_formats(audio_id),
 266         }
 267
 268
 269 class RTVEInfantilIE(RTVEALaCartaIE):
 270     IE_NAME = 'rtve.es:infantil'
 271     IE_DESC = 'RTVE infantil'
 272     _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/[^/]+/video/[^/]+/(?P<id>[0-9]+)/'
 273
 274     _TESTS = [{
 275         'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
 276         'md5': '5747454717aedf9f9fdf212d1bcfc48d',
 277         'info_dict': {
 278             'id': '3040283',
 279             'ext': 'mp4',
 280             'title': 'Maneras de vivir',
 281             'thumbnail': r're:https?://.+/1426182947956\.JPG',
 282             'duration': 357.958,
 283         },
 284         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
 285     }]
 286
 287
 288 class RTVELiveIE(RTVEALaCartaIE):
 289     IE_NAME = 'rtve.es:live'
 290     IE_DESC = 'RTVE.es live streams'
 291     _VALID_URL = r'https?://(?:www\.)?rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
 292
 293     _TESTS = [{
 294         'url': 'http://www.rtve.es/directo/la-1/',
 295         'info_dict': {
 296             'id': 'la-1',
 297             'ext': 'mp4',
 298             'title': 're:^La 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 299         },
 300         'params': {
 301             'skip_download': 'live stream',
 302         }
 303     }]
 304
 305     def _real_extract(self, url):
 306         mobj = self._match_valid_url(url)
 307         video_id = mobj.group('id')
 308
 309         webpage = self._download_webpage(url, video_id)
 310         title = remove_end(self._og_search_title(webpage), ' en directo en RTVE.es')
 311         title = remove_start(title, 'Estoy viendo ')
 312
 313         vidplayer_id = self._search_regex(
 314             (r'playerId=player([0-9]+)',
 315              r'class=["\'].*?\blive_mod\b.*?["\'][^>]+data-assetid=["\'](\d+)',
 316              r'data-id=["\'](\d+)'),
 317             webpage, 'internal video ID')
 318
 319         return {
 320             'id': video_id,
 321             'title': title,
 322             'formats': self._extract_png_formats(vidplayer_id),
 323             'is_live': True,
 324         }
 325
 326
 327 class RTVETelevisionIE(InfoExtractor):
 328     IE_NAME = 'rtve.es:television'
 329     _VALID_URL = r'https?://(?:www\.)?rtve\.es/television/[^/]+/[^/]+/(?P<id>\d+).shtml'
 330
 331     _TEST = {
 332         'url': 'http://www.rtve.es/television/20160628/revolucion-del-movil/1364141.shtml',
 333         'info_dict': {
 334             'id': '3069778',
 335             'ext': 'mp4',
 336             'title': 'Documentos TV - La revolución del móvil',
 337             'duration': 3496.948,
 338         },
 339         'params': {
 340             'skip_download': True,
 341         },
 342     }
 343
 344     def _real_extract(self, url):
 345         page_id = self._match_id(url)
 346         webpage = self._download_webpage(url, page_id)
 347
 348         alacarta_url = self._search_regex(
 349             r'data-location="alacarta_videos"[^<]+url&quot;:&quot;(http://www\.rtve\.es/alacarta.+?)&',
 350             webpage, 'alacarta url', default=None)
 351         if alacarta_url is None:
 352             raise ExtractorError(
 353                 'The webpage doesn\'t contain any video', expected=True)
 354
 355         return self.url_result(alacarta_url, ie=RTVEALaCartaIE.ie_key())