[yt-dlp.git] / youtube_dl / extractor / rtve.py

# encoding: utf-8
from __future__ import unicode_literals

import base64
import re
import time

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    ExtractorError,
    float_or_none,
    remove_end,
    std_headers,
    struct_unpack,
)


def _decrypt_url(png):
    encrypted_data = base64.b64decode(png.encode('utf-8'))
    text_index = encrypted_data.find(b'tEXt')
    text_chunk = encrypted_data[text_index - 4:]
    length = struct_unpack('!I', text_chunk[:4])[0]
    # Use bytearray to get integers when iterating in both python 2.x and 3.x
    data = bytearray(text_chunk[8:8 + length])
    data = [chr(b) for b in data if b != 0]
    hash_index = data.index('#')
    alphabet_data = data[:hash_index]
    url_data = data[hash_index + 1:]

    alphabet = []
    e = 0
    d = 0
    for l in alphabet_data:
        if d == 0:
            alphabet.append(l)
            d = e = (e + 1) % 4
        else:
            d -= 1
    url = ''
    f = 0
    e = 3
    b = 1
    for letter in url_data:
        if f == 0:
            l = int(letter) * 10
            f = 1
        else:
            if e == 0:
                l += int(letter)
                url += alphabet[l]
                e = (b + 3) % 4
                f = 0
                b += 1
            else:
                e -= 1

    return url


class RTVEALaCartaIE(InfoExtractor):
    IE_NAME = 'rtve.es:alacarta'
    IE_DESC = 'RTVE a la carta'
    _VALID_URL = r'http://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'

    _TESTS = [{
        'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
        'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
        'info_dict': {
            'id': '2491869',
            'ext': 'mp4',
            'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
            'duration': 5024.566,
        },
    }, {
        'note': 'Live stream',
        'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',
        'info_dict': {
            'id': '1694255',
            'ext': 'flv',
            'title': 'TODO',
        },
        'skip': 'The f4m manifest can\'t be used yet',
    }, {
        'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
        'only_matching': True,
    }]

    def _real_initialize(self):
        user_agent_b64 = base64.b64encode(std_headers['User-Agent'].encode('utf-8')).decode('utf-8')
        manager_info = self._download_json(
            'http://www.rtve.es/odin/loki/' + user_agent_b64,
            None, 'Fetching manager info')
        self._manager = manager_info['manager']

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        info = self._download_json(
            'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
            video_id)['page']['items'][0]
        if info['state'] == 'DESPU':
            raise ExtractorError('The video is no longer available', expected=True)
        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
        png = self._download_webpage(png_url, video_id, 'Downloading url information')
        video_url = _decrypt_url(png)
        if not video_url.endswith('.f4m'):
            auth_url = video_url.replace(
                'resources/', 'auth/resources/'
            ).replace('.net.rtve', '.multimedia.cdn.rtve')
            video_path = self._download_webpage(
                auth_url, video_id, 'Getting video url')
            # Use mvod1.akcdn instead of flash.akamaihd.multimedia.cdn to get
            # the right Content-Length header and the mp4 format
            video_url = compat_urlparse.urljoin(
                'http://mvod1.akcdn.rtve.es/', video_path)

        subtitles = None
        if info.get('sbtFile') is not None:
            subtitles = self.extract_subtitles(video_id, info['sbtFile'])

        return {
            'id': video_id,
            'title': info['title'],
            'url': video_url,
            'thumbnail': info.get('image'),
            'page_url': url,
            'subtitles': subtitles,
            'duration': float_or_none(info.get('duration'), scale=1000),
        }

    def _get_subtitles(self, video_id, sub_file):
        subs = self._download_json(
            sub_file + '.json', video_id,
            'Downloading subtitles info')['page']['items']
        return dict(
            (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
            for s in subs)


class RTVEInfantilIE(InfoExtractor):
    IE_NAME = 'rtve.es:infantil'
    IE_DESC = 'RTVE infantil'
    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_title>[^/]*)/(?P<id>[0-9]+)/'

    _TESTS = [{
        'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
        'md5': '915319587b33720b8e0357caaa6617e6',
        'info_dict': {
            'id': '3040283',
            'ext': 'mp4',
            'title': 'Maneras de vivir',
            'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG',
            'duration': 357.958,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        info = self._download_json(
            'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
            video_id)['page']['items'][0]

        webpage = self._download_webpage(url, video_id)
        vidplayer_id = self._search_regex(
            r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')

        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
        png = self._download_webpage(png_url, video_id, 'Downloading url information')
        video_url = _decrypt_url(png)

        return {
            'id': video_id,
            'ext': 'mp4',
            'title': info['title'],
            'url': video_url,
            'thumbnail': info.get('image'),
            'duration': float_or_none(info.get('duration'), scale=1000),
        }


class RTVELiveIE(InfoExtractor):
    IE_NAME = 'rtve.es:live'
    IE_DESC = 'RTVE.es live streams'
    _VALID_URL = r'http://www\.rtve\.es/(?:deportes/directo|noticias|television)/(?P<id>[a-zA-Z0-9-]+)'

    _TESTS = [{
        'url': 'http://www.rtve.es/noticias/directo-la-1/',
        'info_dict': {
            'id': 'directo-la-1',
            'ext': 'flv',
            'title': 're:^La 1 de TVE [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
        },
        'params': {
            'skip_download': 'live stream',
        }
    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        start_time = time.gmtime()
        video_id = mobj.group('id')

        webpage = self._download_webpage(url, video_id)
        player_url = self._search_regex(
            r'<param name="movie" value="([^"]+)"/>', webpage, 'player URL')
        title = remove_end(self._og_search_title(webpage), ' en directo')
        title += ' ' + time.strftime('%Y-%m-%dZ%H%M%S', start_time)

        vidplayer_id = self._search_regex(
            r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
        png = self._download_webpage(png_url, video_id, 'Downloading url information')
        video_url = _decrypt_url(png)

        return {
            'id': video_id,
            'ext': 'flv',
            'title': title,
            'url': video_url,
            'app': 'rtve-live-live?ovpfv=2.1.2',
            'player_url': player_url,
            'rtmp_live': True,
        }
Commit	Line	Data
91a6adde JMF	1	# encoding: utf-8
	2	from __future__ import unicode_literals
	3
91a6adde	4	import base64
2b9faf55 PH	5	import re
2b9faf55 PH	6	import time
91a6adde JMF	7
91a6adde JMF	8	from .common import InfoExtractor
93540ee1	9	from ..compat import compat_urlparse
91a6adde	10	from ..utils import (
ce73839f	11	ExtractorError,
f3bff94c	12	float_or_none,
2b9faf55	13	remove_end,
e9f65f87	14	std_headers,
f3bff94c	15	struct_unpack,
91a6adde JMF	16	)
	17
	18
2b9faf55	19	def _decrypt_url(png):
afe8b594	20	encrypted_data = base64.b64decode(png.encode('utf-8'))
2b9faf55 PH	21	text_index = encrypted_data.find(b'tEXt')
	22	text_chunk = encrypted_data[text_index - 4:]
	23	length = struct_unpack('!I', text_chunk[:4])[0]
	24	# Use bytearray to get integers when iterating in both python 2.x and 3.x
	25	data = bytearray(text_chunk[8:8 + length])
	26	data = [chr(b) for b in data if b != 0]
	27	hash_index = data.index('#')
	28	alphabet_data = data[:hash_index]
	29	url_data = data[hash_index + 1:]
	30
	31	alphabet = []
	32	e = 0
	33	d = 0
	34	for l in alphabet_data:
	35	if d == 0:
	36	alphabet.append(l)
	37	d = e = (e + 1) % 4
	38	else:
	39	d -= 1
	40	url = ''
	41	f = 0
	42	e = 3
	43	b = 1
	44	for letter in url_data:
	45	if f == 0:
	46	l = int(letter) * 10
	47	f = 1
	48	else:
	49	if e == 0:
	50	l += int(letter)
	51	url += alphabet[l]
	52	e = (b + 3) % 4
	53	f = 0
	54	b += 1
	55	else:
	56	e -= 1
	57
	58	return url
	59
	60
91a6adde JMF	61	class RTVEALaCartaIE(InfoExtractor):
	62	IE_NAME = 'rtve.es:alacarta'
	63	IE_DESC = 'RTVE a la carta'
cd596028	64	_VALID_URL = r'http://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
91a6adde	65
2b9faf55	66	_TESTS = [{
91a6adde	67	'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
9aeaf730	68	'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
91a6adde JMF	69	'info_dict': {
	70	'id': '2491869',
	71	'ext': 'mp4',
	72	'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
f3bff94c	73	'duration': 5024.566,
91a6adde	74	},
2b9faf55 PH	75	}, {
	76	'note': 'Live stream',
	77	'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',
	78	'info_dict': {
	79	'id': '1694255',
	80	'ext': 'flv',
	81	'title': 'TODO',
cc57bd33 JMF	82	},
cc57bd33 JMF	83	'skip': 'The f4m manifest can\'t be used yet',
cd596028 JMF	84	}, {
	85	'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
	86	'only_matching': True,
2b9faf55	87	}]
91a6adde	88
e9f65f87 JMF	89	def _real_initialize(self):
	90	user_agent_b64 = base64.b64encode(std_headers['User-Agent'].encode('utf-8')).decode('utf-8')
	91	manager_info = self._download_json(
	92	'http://www.rtve.es/odin/loki/' + user_agent_b64,
	93	None, 'Fetching manager info')
	94	self._manager = manager_info['manager']
	95
91a6adde JMF	96	def _real_extract(self, url):
	97	mobj = re.match(self._VALID_URL, url)
	98	video_id = mobj.group('id')
	99	info = self._download_json(
	100	'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
	101	video_id)['page']['items'][0]
ce73839f JMF	102	if info['state'] == 'DESPU':
ce73839f JMF	103	raise ExtractorError('The video is no longer available', expected=True)
e9f65f87	104	png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
91a6adde	105	png = self._download_webpage(png_url, video_id, 'Downloading url information')
2b9faf55	106	video_url = _decrypt_url(png)
cc57bd33 JMF	107	if not video_url.endswith('.f4m'):
	108	auth_url = video_url.replace(
	109	'resources/', 'auth/resources/'
	110	).replace('.net.rtve', '.multimedia.cdn.rtve')
	111	video_path = self._download_webpage(
	112	auth_url, video_id, 'Getting video url')
93540ee1	113	# Use mvod1.akcdn instead of flash.akamaihd.multimedia.cdn to get
cc57bd33	114	# the right Content-Length header and the mp4 format
93540ee1 JMF	115	video_url = compat_urlparse.urljoin(
93540ee1 JMF	116	'http://mvod1.akcdn.rtve.es/', video_path)
91a6adde	117
25ac63ed JMF	118	subtitles = None
	119	if info.get('sbtFile') is not None:
	120	subtitles = self.extract_subtitles(video_id, info['sbtFile'])
	121
91a6adde JMF	122	return {
	123	'id': video_id,
	124	'title': info['title'],
	125	'url': video_url,
2b9faf55 PH	126	'thumbnail': info.get('image'),
2b9faf55 PH	127	'page_url': url,
25ac63ed	128	'subtitles': subtitles,
f3bff94c	129	'duration': float_or_none(info.get('duration'), scale=1000),
2b9faf55 PH	130	}
2b9faf55 PH	131
25ac63ed JMF	132	def _get_subtitles(self, video_id, sub_file):
	133	subs = self._download_json(
	134	sub_file + '.json', video_id,
	135	'Downloading subtitles info')['page']['items']
9c665ab7 PH	136	return dict(
9c665ab7 PH	137	(s['lang'], [{'ext': 'vtt', 'url': s['src']}])
25ac63ed JMF	138	for s in subs)
25ac63ed JMF	139
b68eedba	140
d5b55939	141	class RTVEInfantilIE(InfoExtractor):
b68eedba JMF	142	IE_NAME = 'rtve.es:infantil'
	143	IE_DESC = 'RTVE infantil'
	144	_VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/])/video/(?P<short_title>[^/])/(?P<id>[0-9]+)/'
d5b55939 EF	145
	146	_TESTS = [{
	147	'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
	148	'md5': '915319587b33720b8e0357caaa6617e6',
	149	'info_dict': {
	150	'id': '3040283',
	151	'ext': 'mp4',
	152	'title': 'Maneras de vivir',
	153	'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG',
	154	'duration': 357.958,
	155	},
b68eedba	156	}]
d5b55939 EF	157
d5b55939 EF	158	def _real_extract(self, url):
b68eedba	159	video_id = self._match_id(url)
d5b55939 EF	160	info = self._download_json(
	161	'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
	162	video_id)['page']['items'][0]
	163
	164	webpage = self._download_webpage(url, video_id)
	165	vidplayer_id = self._search_regex(
	166	r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
	167
	168	png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
	169	png = self._download_webpage(png_url, video_id, 'Downloading url information')
	170	video_url = _decrypt_url(png)
	171
	172	return {
	173	'id': video_id,
	174	'ext': 'mp4',
	175	'title': info['title'],
	176	'url': video_url,
	177	'thumbnail': info.get('image'),
	178	'duration': float_or_none(info.get('duration'), scale=1000),
	179	}
	180
	181
2b9faf55 PH	182	class RTVELiveIE(InfoExtractor):
	183	IE_NAME = 'rtve.es:live'
	184	IE_DESC = 'RTVE.es live streams'
9d048a17	185	_VALID_URL = r'http://www\.rtve\.es/(?:deportes/directo\|noticias\|television)/(?P<id>[a-zA-Z0-9-]+)'
2b9faf55 PH	186
	187	_TESTS = [{
	188	'url': 'http://www.rtve.es/noticias/directo-la-1/',
	189	'info_dict': {
	190	'id': 'directo-la-1',
	191	'ext': 'flv',
	192	'title': 're:^La 1 de TVE [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
	193	},
	194	'params': {
	195	'skip_download': 'live stream',
	196	}
	197	}]
	198
	199	def _real_extract(self, url):
	200	mobj = re.match(self._VALID_URL, url)
	201	start_time = time.gmtime()
	202	video_id = mobj.group('id')
	203
	204	webpage = self._download_webpage(url, video_id)
	205	player_url = self._search_regex(
	206	r'<param name="movie" value="([^"]+)"/>', webpage, 'player URL')
	207	title = remove_end(self._og_search_title(webpage), ' en directo')
	208	title += ' ' + time.strftime('%Y-%m-%dZ%H%M%S', start_time)
	209
	210	vidplayer_id = self._search_regex(
	211	r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
	212	png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
	213	png = self._download_webpage(png_url, video_id, 'Downloading url information')
	214	video_url = _decrypt_url(png)
	215
2b9faf55 PH	216	return {
	217	'id': video_id,
	218	'ext': 'flv',
	219	'title': title,
	220	'url': video_url,
	221	'app': 'rtve-live-live?ovpfv=2.1.2',
	222	'player_url': player_url,
	223	'rtmp_live': True,
91a6adde	224	}