yt_dlp/extractor/rtve.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import base64
   5 import io
   6 import re
   7 import sys
   8
   9 from .common import InfoExtractor
  10 from ..compat import (
  11     compat_b64decode,
  12     compat_struct_unpack,
  13 )
  14 from ..utils import (
  15     determine_ext,
  16     ExtractorError,
  17     float_or_none,
  18     qualities,
  19     remove_end,
  20     remove_start,
  21     std_headers,
  22 )
  23
  24 _bytes_to_chr = (lambda x: x) if sys.version_info[0] == 2 else (lambda x: map(chr, x))
  25
  26
  27 class RTVEALaCartaIE(InfoExtractor):
  28     IE_NAME = 'rtve.es:alacarta'
  29     IE_DESC = 'RTVE a la carta'
  30     _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?(alacarta/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
  31
  32     _TESTS = [{
  33         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
  34         'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
  35         'info_dict': {
  36             'id': '2491869',
  37             'ext': 'mp4',
  38             'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
  39             'duration': 5024.566,
  40             'series': 'Balonmano',
  41         },
  42         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
  43     }, {
  44         'note': 'Live stream',
  45         'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',
  46         'info_dict': {
  47             'id': '1694255',
  48             'ext': 'mp4',
  49             'title': 're:^24H LIVE [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
  50             'is_live': True,
  51         },
  52         'params': {
  53             'skip_download': 'live stream',
  54         },
  55     }, {
  56         'url': 'http://www.rtve.es/alacarta/videos/servir-y-proteger/servir-proteger-capitulo-104/4236788/',
  57         'md5': 'd850f3c8731ea53952ebab489cf81cbf',
  58         'info_dict': {
  59             'id': '4236788',
  60             'ext': 'mp4',
  61             'title': 'Servir y proteger - Capítulo 104',
  62             'duration': 3222.0,
  63         },
  64         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
  65     }, {
  66         'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
  67         'only_matching': True,
  68     }, {
  69         'url': 'http://www.rtve.es/filmoteca/no-do/not-1-introduccion-primer-noticiario-espanol/1465256/',
  70         'only_matching': True,
  71     }]
  72
  73     def _real_initialize(self):
  74         user_agent_b64 = base64.b64encode(std_headers['User-Agent'].encode('utf-8')).decode('utf-8')
  75         self._manager = self._download_json(
  76             'http://www.rtve.es/odin/loki/' + user_agent_b64,
  77             None, 'Fetching manager info')['manager']
  78
  79     @staticmethod
  80     def _decrypt_url(png):
  81         encrypted_data = io.BytesIO(compat_b64decode(png)[8:])
  82         while True:
  83             length = compat_struct_unpack('!I', encrypted_data.read(4))[0]
  84             chunk_type = encrypted_data.read(4)
  85             if chunk_type == b'IEND':
  86                 break
  87             data = encrypted_data.read(length)
  88             if chunk_type == b'tEXt':
  89                 alphabet_data, text = data.split(b'\0')
  90                 quality, url_data = text.split(b'%%')
  91                 alphabet = []
  92                 e = 0
  93                 d = 0
  94                 for l in _bytes_to_chr(alphabet_data):
  95                     if d == 0:
  96                         alphabet.append(l)
  97                         d = e = (e + 1) % 4
  98                     else:
  99                         d -= 1
 100                 url = ''
 101                 f = 0
 102                 e = 3
 103                 b = 1
 104                 for letter in _bytes_to_chr(url_data):
 105                     if f == 0:
 106                         l = int(letter) * 10
 107                         f = 1
 108                     else:
 109                         if e == 0:
 110                             l += int(letter)
 111                             url += alphabet[l]
 112                             e = (b + 3) % 4
 113                             f = 0
 114                             b += 1
 115                         else:
 116                             e -= 1
 117
 118                 yield quality.decode(), url
 119             encrypted_data.read(4)  # CRC
 120
 121     def _extract_png_formats(self, video_id):
 122         png = self._download_webpage(
 123             'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id),
 124             video_id, 'Downloading url information', query={'q': 'v2'})
 125         q = qualities(['Media', 'Alta', 'HQ', 'HD_READY', 'HD_FULL'])
 126         formats = []
 127         for quality, video_url in self._decrypt_url(png):
 128             ext = determine_ext(video_url)
 129             if ext == 'm3u8':
 130                 formats.extend(self._extract_m3u8_formats(
 131                     video_url, video_id, 'mp4', 'm3u8_native',
 132                     m3u8_id='hls', fatal=False))
 133             elif ext == 'mpd':
 134                 formats.extend(self._extract_mpd_formats(
 135                     video_url, video_id, 'dash', fatal=False))
 136             else:
 137                 formats.append({
 138                     'format_id': quality,
 139                     'quality': q(quality),
 140                     'url': video_url,
 141                 })
 142         self._sort_formats(formats)
 143         return formats
 144
 145     def _real_extract(self, url):
 146         video_id = self._match_id(url)
 147         info = self._download_json(
 148             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
 149             video_id)['page']['items'][0]
 150         if info['state'] == 'DESPU':
 151             raise ExtractorError('The video is no longer available', expected=True)
 152         title = info['title'].strip()
 153         formats = self._extract_png_formats(video_id)
 154
 155         subtitles = None
 156         sbt_file = info.get('sbtFile')
 157         if sbt_file:
 158             subtitles = self.extract_subtitles(video_id, sbt_file)
 159
 160         is_live = info.get('live') is True
 161
 162         return {
 163             'id': video_id,
 164             'title': self._live_title(title) if is_live else title,
 165             'formats': formats,
 166             'thumbnail': info.get('image'),
 167             'subtitles': subtitles,
 168             'duration': float_or_none(info.get('duration'), 1000),
 169             'is_live': is_live,
 170             'series': info.get('programTitle'),
 171         }
 172
 173     def _get_subtitles(self, video_id, sub_file):
 174         subs = self._download_json(
 175             sub_file + '.json', video_id,
 176             'Downloading subtitles info')['page']['items']
 177         return dict(
 178             (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
 179             for s in subs)
 180
 181
 182 class RTVEInfantilIE(RTVEALaCartaIE):
 183     IE_NAME = 'rtve.es:infantil'
 184     IE_DESC = 'RTVE infantil'
 185     _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/[^/]+/video/[^/]+/(?P<id>[0-9]+)/'
 186
 187     _TESTS = [{
 188         'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
 189         'md5': '5747454717aedf9f9fdf212d1bcfc48d',
 190         'info_dict': {
 191             'id': '3040283',
 192             'ext': 'mp4',
 193             'title': 'Maneras de vivir',
 194             'thumbnail': r're:https?://.+/1426182947956\.JPG',
 195             'duration': 357.958,
 196         },
 197         'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
 198     }]
 199
 200
 201 class RTVELiveIE(RTVEALaCartaIE):
 202     IE_NAME = 'rtve.es:live'
 203     IE_DESC = 'RTVE.es live streams'
 204     _VALID_URL = r'https?://(?:www\.)?rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
 205
 206     _TESTS = [{
 207         'url': 'http://www.rtve.es/directo/la-1/',
 208         'info_dict': {
 209             'id': 'la-1',
 210             'ext': 'mp4',
 211             'title': 're:^La 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 212         },
 213         'params': {
 214             'skip_download': 'live stream',
 215         }
 216     }]
 217
 218     def _real_extract(self, url):
 219         mobj = re.match(self._VALID_URL, url)
 220         video_id = mobj.group('id')
 221
 222         webpage = self._download_webpage(url, video_id)
 223         title = remove_end(self._og_search_title(webpage), ' en directo en RTVE.es')
 224         title = remove_start(title, 'Estoy viendo ')
 225
 226         vidplayer_id = self._search_regex(
 227             (r'playerId=player([0-9]+)',
 228              r'class=["\'].*?\blive_mod\b.*?["\'][^>]+data-assetid=["\'](\d+)',
 229              r'data-id=["\'](\d+)'),
 230             webpage, 'internal video ID')
 231
 232         return {
 233             'id': video_id,
 234             'title': self._live_title(title),
 235             'formats': self._extract_png_formats(vidplayer_id),
 236             'is_live': True,
 237         }
 238
 239
 240 class RTVETelevisionIE(InfoExtractor):
 241     IE_NAME = 'rtve.es:television'
 242     _VALID_URL = r'https?://(?:www\.)?rtve\.es/television/[^/]+/[^/]+/(?P<id>\d+).shtml'
 243
 244     _TEST = {
 245         'url': 'http://www.rtve.es/television/20160628/revolucion-del-movil/1364141.shtml',
 246         'info_dict': {
 247             'id': '3069778',
 248             'ext': 'mp4',
 249             'title': 'Documentos TV - La revolución del móvil',
 250             'duration': 3496.948,
 251         },
 252         'params': {
 253             'skip_download': True,
 254         },
 255     }
 256
 257     def _real_extract(self, url):
 258         page_id = self._match_id(url)
 259         webpage = self._download_webpage(url, page_id)
 260
 261         alacarta_url = self._search_regex(
 262             r'data-location="alacarta_videos"[^<]+url&quot;:&quot;(http://www\.rtve\.es/alacarta.+?)&',
 263             webpage, 'alacarta url', default=None)
 264         if alacarta_url is None:
 265             raise ExtractorError(
 266                 'The webpage doesn\'t contain any video', expected=True)
 267
 268         return self.url_result(alacarta_url, ie=RTVEALaCartaIE.ie_key())