[yt-dlp.git] / youtube_dl / extractor / itv.py

# coding: utf-8
from __future__ import unicode_literals

import uuid
import xml.etree.ElementTree as etree
import json

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_etree_register_namespace,
)
from ..utils import (
    extract_attributes,
    xpath_with_ns,
    xpath_element,
    xpath_text,
    int_or_none,
    parse_duration,
    ExtractorError,
    determine_ext,
)


class ITVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
    _GEO_COUNTRIES = ['GB']
    _TEST = {
        'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
        'info_dict': {
            'id': '2a2936a0053',
            'ext': 'flv',
            'title': 'Home Movie',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        params = extract_attributes(self._search_regex(
            r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))

        ns_map = {
            'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/',
            'tem': 'http://tempuri.org/',
            'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types',
            'com': 'http://schemas.itv.com/2009/05/Common',
        }
        for ns, full_ns in ns_map.items():
            compat_etree_register_namespace(ns, full_ns)

        def _add_ns(name):
            return xpath_with_ns(name, ns_map)

        def _add_sub_element(element, name):
            return etree.SubElement(element, _add_ns(name))

        req_env = etree.Element(_add_ns('soapenv:Envelope'))
        _add_sub_element(req_env, 'soapenv:Header')
        body = _add_sub_element(req_env, 'soapenv:Body')
        get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
        request = _add_sub_element(get_playlist, 'tem:request')
        _add_sub_element(request, 'itv:ProductionId').text = params['data-video-id']
        _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
        vodcrid = _add_sub_element(request, 'itv:Vodcrid')
        _add_sub_element(vodcrid, 'com:Id')
        _add_sub_element(request, 'itv:Partition')
        user_info = _add_sub_element(get_playlist, 'tem:userInfo')
        _add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv'
        _add_sub_element(user_info, 'itv:DM')
        _add_sub_element(user_info, 'itv:RevenueScienceValue')
        _add_sub_element(user_info, 'itv:SessionId')
        _add_sub_element(user_info, 'itv:SsoToken')
        _add_sub_element(user_info, 'itv:UserToken')
        site_info = _add_sub_element(get_playlist, 'tem:siteInfo')
        _add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None'
        _add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV'
        _add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any'
        _add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO'
        _add_sub_element(site_info, 'itv:Category')
        _add_sub_element(site_info, 'itv:Platform').text = 'DotCom'
        _add_sub_element(site_info, 'itv:Site').text = 'ItvCom'
        device_info = _add_sub_element(get_playlist, 'tem:deviceInfo')
        _add_sub_element(device_info, 'itv:ScreenSize').text = 'Big'
        player_info = _add_sub_element(get_playlist, 'tem:playerInfo')
        _add_sub_element(player_info, 'itv:Version').text = '2'

        headers = self.geo_verification_headers()
        headers.update({
            'Content-Type': 'text/xml; charset=utf-8',
            'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
        })
        resp_env = self._download_xml(
            params['data-playlist-url'], video_id,
            headers=headers, data=etree.tostring(req_env))
        playlist = xpath_element(resp_env, './/Playlist')
        if playlist is None:
            fault_code = xpath_text(resp_env, './/faultcode')
            fault_string = xpath_text(resp_env, './/faultstring')
            if fault_code == 'InvalidGeoRegion':
                self.raise_geo_restricted(
                    msg=fault_string, countries=self._GEO_COUNTRIES)
            raise ExtractorError('%s said: %s' % (self.IE_NAME, fault_string))
        title = xpath_text(playlist, 'EpisodeTitle', fatal=True)
        video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
        media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
        rtmp_url = media_files.attrib['base']

        formats = []
        for media_file in media_files.findall('MediaFile'):
            play_path = xpath_text(media_file, 'URL')
            if not play_path:
                continue
            tbr = int_or_none(media_file.get('bitrate'), 1000)
            formats.append({
                'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
                'url': rtmp_url,
                'play_path': play_path,
                'tbr': tbr,
                'ext': 'flv',
                # rtmp formats are now stop downloading at ~72MiB
                'preference': -10,
            })

        ios_playlist_url = params.get('data-video-playlist')
        hmac = params.get('data-video-hmac')
        if ios_playlist_url and hmac:
            headers = self.geo_verification_headers()
            headers.update({
                'Accept': 'application/vnd.itv.vod.playlist.v2+json',
                'Content-Type': 'application/json',
                'hmac': hmac.upper(),
            })
            ios_playlist = self._download_json(
                ios_playlist_url, video_id, data=json.dumps({
                    'user': {
                        'itvUserId': '',
                        'entitlements': [],
                        'token': ''
                    },
                    'device': {
                        'manufacturer': 'Apple',
                        'model': 'iPad',
                        'os': {
                            'name': 'iPhone OS',
                            'version': '9.3',
                            'type': 'ios'
                        }
                    },
                    'client': {
                        'version': '4.1',
                        'id': 'browser'
                    },
                    'variantAvailability': {
                        'featureset': {
                            'min': ['hls', 'aes'],
                            'max': ['hls', 'aes']
                        },
                        'platformTag': 'mobile'
                    }
                }).encode(), headers=headers, fatal=False)
            if ios_playlist:
                video_data = ios_playlist.get('Playlist', {}).get('Video', {})
                ios_base_url = video_data.get('Base')
                for media_file in video_data.get('MediaFiles', []):
                    href = media_file.get('Href')
                    if not href:
                        continue
                    if ios_base_url:
                        href = ios_base_url + href
                    ext = determine_ext(href)
                    if ext == 'm3u8':
                        formats.extend(self._extract_m3u8_formats(
                            href, video_id, 'mp4', entry_protocol='m3u8_native',
                            m3u8_id='hls', fatal=False))
                    else:
                        formats.append({
                            'url': href,
                        })
        self._sort_formats(formats)

        subtitles = {}
        for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
            if not caption_url.text:
                continue
            ext = determine_ext(caption_url.text, 'ttml')
            subtitles.setdefault('en', []).append({
                'url': caption_url.text,
                'ext': 'ttml' if ext == 'xml' else ext,
            })

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'subtitles': subtitles,
            'episode_title': title,
            'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
            'series': xpath_text(playlist, 'ProgrammeTitle'),
            'duartion': parse_duration(xpath_text(playlist, 'Duration')),
        }
Commit	Line	Data
a71b8d3b RA	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
	4	import uuid
	5	import xml.etree.ElementTree as etree
	6	import json
	7
	8	from .common import InfoExtractor
47194199 RA	9	from ..compat import (
	10	compat_str,
	11	compat_etree_register_namespace,
	12	)
a71b8d3b RA	13	from ..utils import (
	14	extract_attributes,
	15	xpath_with_ns,
	16	xpath_element,
	17	xpath_text,
	18	int_or_none,
	19	parse_duration,
	20	ExtractorError,
	21	determine_ext,
	22	)
	23
	24
	25	class ITVIE(InfoExtractor):
f592ff98	26	_VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
4248dad9	27	_GEO_COUNTRIES = ['GB']
a71b8d3b RA	28	_TEST = {
	29	'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
	30	'info_dict': {
	31	'id': '2a2936a0053',
	32	'ext': 'flv',
	33	'title': 'Home Movie',
	34	},
	35	'params': {
	36	# rtmp download
	37	'skip_download': True,
	38	},
	39	}
	40
	41	def _real_extract(self, url):
	42	video_id = self._match_id(url)
	43	webpage = self._download_webpage(url, video_id)
	44	params = extract_attributes(self._search_regex(
	45	r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
	46
	47	ns_map = {
	48	'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/',
	49	'tem': 'http://tempuri.org/',
	50	'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types',
	51	'com': 'http://schemas.itv.com/2009/05/Common',
	52	}
	53	for ns, full_ns in ns_map.items():
47194199	54	compat_etree_register_namespace(ns, full_ns)
a71b8d3b RA	55
	56	def _add_ns(name):
	57	return xpath_with_ns(name, ns_map)
	58
	59	def _add_sub_element(element, name):
	60	return etree.SubElement(element, _add_ns(name))
	61
	62	req_env = etree.Element(_add_ns('soapenv:Envelope'))
	63	_add_sub_element(req_env, 'soapenv:Header')
	64	body = _add_sub_element(req_env, 'soapenv:Body')
	65	get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
	66	request = _add_sub_element(get_playlist, 'tem:request')
	67	_add_sub_element(request, 'itv:ProductionId').text = params['data-video-id']
	68	_add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
	69	vodcrid = _add_sub_element(request, 'itv:Vodcrid')
	70	_add_sub_element(vodcrid, 'com:Id')
	71	_add_sub_element(request, 'itv:Partition')
	72	user_info = _add_sub_element(get_playlist, 'tem:userInfo')
	73	_add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv'
	74	_add_sub_element(user_info, 'itv:DM')
	75	_add_sub_element(user_info, 'itv:RevenueScienceValue')
	76	_add_sub_element(user_info, 'itv:SessionId')
	77	_add_sub_element(user_info, 'itv:SsoToken')
	78	_add_sub_element(user_info, 'itv:UserToken')
	79	site_info = _add_sub_element(get_playlist, 'tem:siteInfo')
	80	_add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None'
	81	_add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV'
	82	_add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any'
	83	_add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO'
	84	_add_sub_element(site_info, 'itv:Category')
	85	_add_sub_element(site_info, 'itv:Platform').text = 'DotCom'
	86	_add_sub_element(site_info, 'itv:Site').text = 'ItvCom'
	87	device_info = _add_sub_element(get_playlist, 'tem:deviceInfo')
	88	_add_sub_element(device_info, 'itv:ScreenSize').text = 'Big'
	89	player_info = _add_sub_element(get_playlist, 'tem:playerInfo')
	90	_add_sub_element(player_info, 'itv:Version').text = '2'
	91
	92	headers = self.geo_verification_headers()
	93	headers.update({
	94	'Content-Type': 'text/xml; charset=utf-8',
	95	'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
	96	})
	97	resp_env = self._download_xml(
	98	params['data-playlist-url'], video_id,
	99	headers=headers, data=etree.tostring(req_env))
	100	playlist = xpath_element(resp_env, './/Playlist')
	101	if playlist is None:
28200e65	102	fault_code = xpath_text(resp_env, './/faultcode')
a71b8d3b	103	fault_string = xpath_text(resp_env, './/faultstring')
28200e65	104	if fault_code == 'InvalidGeoRegion':
4248dad9 S	105	self.raise_geo_restricted(
4248dad9 S	106	msg=fault_string, countries=self._GEO_COUNTRIES)
a71b8d3b RA	107	raise ExtractorError('%s said: %s' % (self.IE_NAME, fault_string))
a71b8d3b RA	108	title = xpath_text(playlist, 'EpisodeTitle', fatal=True)
f592ff98 RA	109	video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
f592ff98 RA	110	media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
a71b8d3b RA	111	rtmp_url = media_files.attrib['base']
	112
	113	formats = []
	114	for media_file in media_files.findall('MediaFile'):
	115	play_path = xpath_text(media_file, 'URL')
	116	if not play_path:
	117	continue
	118	tbr = int_or_none(media_file.get('bitrate'), 1000)
	119	formats.append({
	120	'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
	121	'url': rtmp_url,
	122	'play_path': play_path,
	123	'tbr': tbr,
	124	'ext': 'flv',
5935ef3c S	125	# rtmp formats are now stop downloading at ~72MiB
5935ef3c S	126	'preference': -10,
a71b8d3b RA	127	})
	128
	129	ios_playlist_url = params.get('data-video-playlist')
	130	hmac = params.get('data-video-hmac')
	131	if ios_playlist_url and hmac:
	132	headers = self.geo_verification_headers()
	133	headers.update({
	134	'Accept': 'application/vnd.itv.vod.playlist.v2+json',
	135	'Content-Type': 'application/json',
	136	'hmac': hmac.upper(),
	137	})
	138	ios_playlist = self._download_json(
	139	ios_playlist_url, video_id, data=json.dumps({
	140	'user': {
	141	'itvUserId': '',
	142	'entitlements': [],
	143	'token': ''
	144	},
	145	'device': {
	146	'manufacturer': 'Apple',
	147	'model': 'iPad',
	148	'os': {
	149	'name': 'iPhone OS',
	150	'version': '9.3',
	151	'type': 'ios'
	152	}
	153	},
	154	'client': {
	155	'version': '4.1',
	156	'id': 'browser'
	157	},
	158	'variantAvailability': {
	159	'featureset': {
	160	'min': ['hls', 'aes'],
	161	'max': ['hls', 'aes']
	162	},
	163	'platformTag': 'mobile'
	164	}
	165	}).encode(), headers=headers, fatal=False)
	166	if ios_playlist:
	167	video_data = ios_playlist.get('Playlist', {}).get('Video', {})
	168	ios_base_url = video_data.get('Base')
	169	for media_file in video_data.get('MediaFiles', []):
	170	href = media_file.get('Href')
	171	if not href:
	172	continue
	173	if ios_base_url:
	174	href = ios_base_url + href
	175	ext = determine_ext(href)
	176	if ext == 'm3u8':
f67177ca S	177	formats.extend(self._extract_m3u8_formats(
	178	href, video_id, 'mp4', entry_protocol='m3u8_native',
	179	m3u8_id='hls', fatal=False))
a71b8d3b RA	180	else:
	181	formats.append({
	182	'url': href,
	183	})
	184	self._sort_formats(formats)
	185
f592ff98 RA	186	subtitles = {}
	187	for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
	188	if not caption_url.text:
	189	continue
	190	ext = determine_ext(caption_url.text, 'ttml')
	191	subtitles.setdefault('en', []).append({
4edeac5b	192	'url': caption_url.text,
f592ff98 RA	193	'ext': 'ttml' if ext == 'xml' else ext,
	194	})
	195
a71b8d3b RA	196	return {
	197	'id': video_id,
	198	'title': title,
	199	'formats': formats,
f592ff98	200	'subtitles': subtitles,
a71b8d3b RA	201	'episode_title': title,
	202	'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
	203	'series': xpath_text(playlist, 'ProgrammeTitle'),
	204	'duartion': parse_duration(xpath_text(playlist, 'Duration')),
	205	}