[yt-dlp.git] / youtube_dl / extractor / itv.py

# coding: utf-8
from __future__ import unicode_literals

import uuid
import xml.etree.ElementTree as etree
import json
import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_etree_register_namespace,
)
from ..utils import (
    extract_attributes,
    xpath_with_ns,
    xpath_element,
    xpath_text,
    int_or_none,
    parse_duration,
    ExtractorError,
    determine_ext,
)


class ITVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
    _GEO_COUNTRIES = ['GB']
    _TESTS = [{
        'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
        'info_dict': {
            'id': '2a2936a0053',
            'ext': 'flv',
            'title': 'Home Movie',
        },
        'params': {
            # rtmp download
            'skip_download': True,
        },
    }, {
        # unavailable via data-playlist-url
        'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033',
        'only_matching': True,
    }, {
        # InvalidVodcrid
        'url': 'https://www.itv.com/hub/james-martins-saturday-morning/2a5159a0034',
        'only_matching': True,
    }, {
        # ContentUnavailable
        'url': 'https://www.itv.com/hub/whos-doing-the-dishes/2a2898a0024',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        params = extract_attributes(self._search_regex(
            r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))

        ns_map = {
            'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/',
            'tem': 'http://tempuri.org/',
            'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types',
            'com': 'http://schemas.itv.com/2009/05/Common',
        }
        for ns, full_ns in ns_map.items():
            compat_etree_register_namespace(ns, full_ns)

        def _add_ns(name):
            return xpath_with_ns(name, ns_map)

        def _add_sub_element(element, name):
            return etree.SubElement(element, _add_ns(name))

        production_id = (
            params.get('data-video-autoplay-id') or
            '%s#001' % (
                params.get('data-video-episode-id') or
                video_id.replace('a', '/')))

        req_env = etree.Element(_add_ns('soapenv:Envelope'))
        _add_sub_element(req_env, 'soapenv:Header')
        body = _add_sub_element(req_env, 'soapenv:Body')
        get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
        request = _add_sub_element(get_playlist, 'tem:request')
        _add_sub_element(request, 'itv:ProductionId').text = production_id
        _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
        vodcrid = _add_sub_element(request, 'itv:Vodcrid')
        _add_sub_element(vodcrid, 'com:Id')
        _add_sub_element(request, 'itv:Partition')
        user_info = _add_sub_element(get_playlist, 'tem:userInfo')
        _add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv'
        _add_sub_element(user_info, 'itv:DM')
        _add_sub_element(user_info, 'itv:RevenueScienceValue')
        _add_sub_element(user_info, 'itv:SessionId')
        _add_sub_element(user_info, 'itv:SsoToken')
        _add_sub_element(user_info, 'itv:UserToken')
        site_info = _add_sub_element(get_playlist, 'tem:siteInfo')
        _add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None'
        _add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV'
        _add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any'
        _add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO'
        _add_sub_element(site_info, 'itv:Category')
        _add_sub_element(site_info, 'itv:Platform').text = 'DotCom'
        _add_sub_element(site_info, 'itv:Site').text = 'ItvCom'
        device_info = _add_sub_element(get_playlist, 'tem:deviceInfo')
        _add_sub_element(device_info, 'itv:ScreenSize').text = 'Big'
        player_info = _add_sub_element(get_playlist, 'tem:playerInfo')
        _add_sub_element(player_info, 'itv:Version').text = '2'

        headers = self.geo_verification_headers()
        headers.update({
            'Content-Type': 'text/xml; charset=utf-8',
            'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
        })

        info = self._search_json_ld(webpage, video_id, default={})
        formats = []
        subtitles = {}

        def extract_subtitle(sub_url):
            ext = determine_ext(sub_url, 'ttml')
            subtitles.setdefault('en', []).append({
                'url': sub_url,
                'ext': 'ttml' if ext == 'xml' else ext,
            })

        resp_env = self._download_xml(
            params['data-playlist-url'], video_id,
            headers=headers, data=etree.tostring(req_env))
        playlist = xpath_element(resp_env, './/Playlist')
        if playlist is None:
            fault_code = xpath_text(resp_env, './/faultcode')
            fault_string = xpath_text(resp_env, './/faultstring')
            if fault_code == 'InvalidGeoRegion':
                self.raise_geo_restricted(
                    msg=fault_string, countries=self._GEO_COUNTRIES)
            elif fault_code not in (
                    'InvalidEntity', 'InvalidVodcrid', 'ContentUnavailable'):
                raise ExtractorError(
                    '%s said: %s' % (self.IE_NAME, fault_string), expected=True)
            info.update({
                'title': self._og_search_title(webpage),
                'episode_title': params.get('data-video-episode'),
                'series': params.get('data-video-title'),
            })
        else:
            title = xpath_text(playlist, 'EpisodeTitle', default=None)
            info.update({
                'title': title,
                'episode_title': title,
                'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
                'series': xpath_text(playlist, 'ProgrammeTitle'),
                'duration': parse_duration(xpath_text(playlist, 'Duration')),
            })
            video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
            media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
            rtmp_url = media_files.attrib['base']

            for media_file in media_files.findall('MediaFile'):
                play_path = xpath_text(media_file, 'URL')
                if not play_path:
                    continue
                tbr = int_or_none(media_file.get('bitrate'), 1000)
                f = {
                    'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
                    'play_path': play_path,
                    # Providing this swfVfy allows to avoid truncated downloads
                    'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
                    'page_url': url,
                    'tbr': tbr,
                    'ext': 'flv',
                }
                app = self._search_regex(
                    'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
                if app:
                    f.update({
                        'url': rtmp_url.split('?', 1)[0],
                        'app': app,
                    })
                else:
                    f['url'] = rtmp_url
                formats.append(f)

            for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
                if caption_url.text:
                    extract_subtitle(caption_url.text)

        ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')
        hmac = params.get('data-video-hmac')
        if ios_playlist_url and hmac and re.match(r'https?://', ios_playlist_url):
            headers = self.geo_verification_headers()
            headers.update({
                'Accept': 'application/vnd.itv.vod.playlist.v2+json',
                'Content-Type': 'application/json',
                'hmac': hmac.upper(),
            })
            ios_playlist = self._download_json(
                ios_playlist_url, video_id, data=json.dumps({
                    'user': {
                        'itvUserId': '',
                        'entitlements': [],
                        'token': ''
                    },
                    'device': {
                        'manufacturer': 'Safari',
                        'model': '5',
                        'os': {
                            'name': 'Windows NT',
                            'version': '6.1',
                            'type': 'desktop'
                        }
                    },
                    'client': {
                        'version': '4.1',
                        'id': 'browser'
                    },
                    'variantAvailability': {
                        'featureset': {
                            'min': ['hls', 'aes', 'outband-webvtt'],
                            'max': ['hls', 'aes', 'outband-webvtt']
                        },
                        'platformTag': 'dotcom'
                    }
                }).encode(), headers=headers, fatal=False)
            if ios_playlist:
                video_data = ios_playlist.get('Playlist', {}).get('Video', {})
                ios_base_url = video_data.get('Base')
                for media_file in video_data.get('MediaFiles', []):
                    href = media_file.get('Href')
                    if not href:
                        continue
                    if ios_base_url:
                        href = ios_base_url + href
                    ext = determine_ext(href)
                    if ext == 'm3u8':
                        formats.extend(self._extract_m3u8_formats(
                            href, video_id, 'mp4', entry_protocol='m3u8_native',
                            m3u8_id='hls', fatal=False))
                    else:
                        formats.append({
                            'url': href,
                        })
                subs = video_data.get('Subtitles')
                if isinstance(subs, list):
                    for sub in subs:
                        if not isinstance(sub, dict):
                            continue
                        href = sub.get('Href')
                        if isinstance(href, compat_str):
                            extract_subtitle(href)
                if not info.get('duration'):
                    info['duration'] = parse_duration(video_data.get('Duration'))

        self._sort_formats(formats)

        info.update({
            'id': video_id,
            'formats': formats,
            'subtitles': subtitles,
        })
        return info
Commit	Line	Data
a71b8d3b RA	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
	4	import uuid
	5	import xml.etree.ElementTree as etree
	6	import json
115afb77	7	import re
a71b8d3b RA	8
a71b8d3b RA	9	from .common import InfoExtractor
47194199 RA	10	from ..compat import (
	11	compat_str,
	12	compat_etree_register_namespace,
	13	)
a71b8d3b RA	14	from ..utils import (
	15	extract_attributes,
	16	xpath_with_ns,
	17	xpath_element,
	18	xpath_text,
	19	int_or_none,
	20	parse_duration,
	21	ExtractorError,
	22	determine_ext,
	23	)
	24
	25
	26	class ITVIE(InfoExtractor):
f592ff98	27	_VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
4248dad9	28	_GEO_COUNTRIES = ['GB']
3fae11ac	29	_TESTS = [{
a71b8d3b RA	30	'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
	31	'info_dict': {
	32	'id': '2a2936a0053',
	33	'ext': 'flv',
	34	'title': 'Home Movie',
	35	},
	36	'params': {
	37	# rtmp download
	38	'skip_download': True,
	39	},
3fae11ac S	40	}, {
	41	# unavailable via data-playlist-url
	42	'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033',
	43	'only_matching': True,
c18142da S	44	}, {
	45	# InvalidVodcrid
	46	'url': 'https://www.itv.com/hub/james-martins-saturday-morning/2a5159a0034',
	47	'only_matching': True,
	48	}, {
	49	# ContentUnavailable
	50	'url': 'https://www.itv.com/hub/whos-doing-the-dishes/2a2898a0024',
	51	'only_matching': True,
3fae11ac	52	}]
a71b8d3b RA	53
	54	def _real_extract(self, url):
	55	video_id = self._match_id(url)
	56	webpage = self._download_webpage(url, video_id)
	57	params = extract_attributes(self._search_regex(
	58	r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
	59
	60	ns_map = {
	61	'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/',
	62	'tem': 'http://tempuri.org/',
	63	'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types',
	64	'com': 'http://schemas.itv.com/2009/05/Common',
	65	}
	66	for ns, full_ns in ns_map.items():
47194199	67	compat_etree_register_namespace(ns, full_ns)
a71b8d3b RA	68
	69	def _add_ns(name):
	70	return xpath_with_ns(name, ns_map)
	71
	72	def _add_sub_element(element, name):
	73	return etree.SubElement(element, _add_ns(name))
	74
71dde5ee S	75	production_id = (
	76	params.get('data-video-autoplay-id') or
	77	'%s#001' % (
	78	params.get('data-video-episode-id') or
	79	video_id.replace('a', '/')))
	80
a71b8d3b RA	81	req_env = etree.Element(_add_ns('soapenv:Envelope'))
	82	_add_sub_element(req_env, 'soapenv:Header')
	83	body = _add_sub_element(req_env, 'soapenv:Body')
	84	get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
	85	request = _add_sub_element(get_playlist, 'tem:request')
71dde5ee	86	_add_sub_element(request, 'itv:ProductionId').text = production_id
a71b8d3b RA	87	_add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
	88	vodcrid = _add_sub_element(request, 'itv:Vodcrid')
	89	_add_sub_element(vodcrid, 'com:Id')
	90	_add_sub_element(request, 'itv:Partition')
	91	user_info = _add_sub_element(get_playlist, 'tem:userInfo')
	92	_add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv'
	93	_add_sub_element(user_info, 'itv:DM')
	94	_add_sub_element(user_info, 'itv:RevenueScienceValue')
	95	_add_sub_element(user_info, 'itv:SessionId')
	96	_add_sub_element(user_info, 'itv:SsoToken')
	97	_add_sub_element(user_info, 'itv:UserToken')
	98	site_info = _add_sub_element(get_playlist, 'tem:siteInfo')
	99	_add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None'
	100	_add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV'
	101	_add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any'
	102	_add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO'
	103	_add_sub_element(site_info, 'itv:Category')
	104	_add_sub_element(site_info, 'itv:Platform').text = 'DotCom'
	105	_add_sub_element(site_info, 'itv:Site').text = 'ItvCom'
	106	device_info = _add_sub_element(get_playlist, 'tem:deviceInfo')
	107	_add_sub_element(device_info, 'itv:ScreenSize').text = 'Big'
	108	player_info = _add_sub_element(get_playlist, 'tem:playerInfo')
	109	_add_sub_element(player_info, 'itv:Version').text = '2'
	110
	111	headers = self.geo_verification_headers()
	112	headers.update({
	113	'Content-Type': 'text/xml; charset=utf-8',
	114	'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
	115	})
3fae11ac S	116
	117	info = self._search_json_ld(webpage, video_id, default={})
	118	formats = []
	119	subtitles = {}
	120
	121	def extract_subtitle(sub_url):
	122	ext = determine_ext(sub_url, 'ttml')
	123	subtitles.setdefault('en', []).append({
	124	'url': sub_url,
	125	'ext': 'ttml' if ext == 'xml' else ext,
	126	})
	127
a71b8d3b RA	128	resp_env = self._download_xml(
	129	params['data-playlist-url'], video_id,
	130	headers=headers, data=etree.tostring(req_env))
	131	playlist = xpath_element(resp_env, './/Playlist')
	132	if playlist is None:
28200e65	133	fault_code = xpath_text(resp_env, './/faultcode')
a71b8d3b	134	fault_string = xpath_text(resp_env, './/faultstring')
28200e65	135	if fault_code == 'InvalidGeoRegion':
4248dad9 S	136	self.raise_geo_restricted(
4248dad9 S	137	msg=fault_string, countries=self._GEO_COUNTRIES)
c18142da S	138	elif fault_code not in (
c18142da S	139	'InvalidEntity', 'InvalidVodcrid', 'ContentUnavailable'):
3fae11ac S	140	raise ExtractorError(
	141	'%s said: %s' % (self.IE_NAME, fault_string), expected=True)
	142	info.update({
	143	'title': self._og_search_title(webpage),
	144	'episode_title': params.get('data-video-episode'),
	145	'series': params.get('data-video-title'),
	146	})
	147	else:
	148	title = xpath_text(playlist, 'EpisodeTitle', default=None)
	149	info.update({
	150	'title': title,
	151	'episode_title': title,
	152	'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
	153	'series': xpath_text(playlist, 'ProgrammeTitle'),
	154	'duration': parse_duration(xpath_text(playlist, 'Duration')),
	155	})
	156	video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
	157	media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
	158	rtmp_url = media_files.attrib['base']
a71b8d3b	159
3fae11ac S	160	for media_file in media_files.findall('MediaFile'):
	161	play_path = xpath_text(media_file, 'URL')
	162	if not play_path:
	163	continue
	164	tbr = int_or_none(media_file.get('bitrate'), 1000)
	165	f = {
	166	'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
	167	'play_path': play_path,
	168	# Providing this swfVfy allows to avoid truncated downloads
	169	'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
	170	'page_url': url,
	171	'tbr': tbr,
	172	'ext': 'flv',
	173	}
	174	app = self._search_regex(
	175	'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
	176	if app:
	177	f.update({
	178	'url': rtmp_url.split('?', 1)[0],
	179	'app': app,
	180	})
	181	else:
	182	f['url'] = rtmp_url
	183	formats.append(f)
	184
	185	for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
	186	if caption_url.text:
	187	extract_subtitle(caption_url.text)
a71b8d3b	188
115afb77	189	ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')
a71b8d3b	190	hmac = params.get('data-video-hmac')
115afb77	191	if ios_playlist_url and hmac and re.match(r'https?://', ios_playlist_url):
a71b8d3b RA	192	headers = self.geo_verification_headers()
	193	headers.update({
	194	'Accept': 'application/vnd.itv.vod.playlist.v2+json',
	195	'Content-Type': 'application/json',
	196	'hmac': hmac.upper(),
	197	})
	198	ios_playlist = self._download_json(
	199	ios_playlist_url, video_id, data=json.dumps({
	200	'user': {
	201	'itvUserId': '',
	202	'entitlements': [],
	203	'token': ''
	204	},
	205	'device': {
115afb77 RA	206	'manufacturer': 'Safari',
115afb77 RA	207	'model': '5',
a71b8d3b	208	'os': {
115afb77 RA	209	'name': 'Windows NT',
	210	'version': '6.1',
	211	'type': 'desktop'
a71b8d3b RA	212	}
	213	},
	214	'client': {
	215	'version': '4.1',
	216	'id': 'browser'
	217	},
	218	'variantAvailability': {
	219	'featureset': {
115afb77 RA	220	'min': ['hls', 'aes', 'outband-webvtt'],
115afb77 RA	221	'max': ['hls', 'aes', 'outband-webvtt']
a71b8d3b	222	},
115afb77	223	'platformTag': 'dotcom'
a71b8d3b RA	224	}
	225	}).encode(), headers=headers, fatal=False)
	226	if ios_playlist:
	227	video_data = ios_playlist.get('Playlist', {}).get('Video', {})
	228	ios_base_url = video_data.get('Base')
	229	for media_file in video_data.get('MediaFiles', []):
	230	href = media_file.get('Href')
	231	if not href:
	232	continue
	233	if ios_base_url:
	234	href = ios_base_url + href
	235	ext = determine_ext(href)
	236	if ext == 'm3u8':
f67177ca S	237	formats.extend(self._extract_m3u8_formats(
	238	href, video_id, 'mp4', entry_protocol='m3u8_native',
	239	m3u8_id='hls', fatal=False))
a71b8d3b RA	240	else:
	241	formats.append({
	242	'url': href,
	243	})
3fae11ac S	244	subs = video_data.get('Subtitles')
	245	if isinstance(subs, list):
	246	for sub in subs:
	247	if not isinstance(sub, dict):
	248	continue
	249	href = sub.get('Href')
	250	if isinstance(href, compat_str):
	251	extract_subtitle(href)
	252	if not info.get('duration'):
	253	info['duration'] = parse_duration(video_data.get('Duration'))
a71b8d3b	254
3fae11ac	255	self._sort_formats(formats)
f592ff98	256
751c89a2	257	info.update({
a71b8d3b	258	'id': video_id,
a71b8d3b	259	'formats': formats,
f592ff98	260	'subtitles': subtitles,
751c89a2 S	261	})
751c89a2 S	262	return info