]>
Commit | Line | Data |
---|---|---|
a71b8d3b RA |
1 | # coding: utf-8 |
2 | from __future__ import unicode_literals | |
3 | ||
4 | import uuid | |
5 | import xml.etree.ElementTree as etree | |
6 | import json | |
115afb77 | 7 | import re |
a71b8d3b RA |
8 | |
9 | from .common import InfoExtractor | |
ea1f5e5d | 10 | from .brightcove import BrightcoveNewIE |
47194199 RA |
11 | from ..compat import ( |
12 | compat_str, | |
13 | compat_etree_register_namespace, | |
14 | ) | |
a71b8d3b | 15 | from ..utils import ( |
a4ec4517 S |
16 | determine_ext, |
17 | ExtractorError, | |
a71b8d3b | 18 | extract_attributes, |
a71b8d3b | 19 | int_or_none, |
30374f4d | 20 | merge_dicts, |
a71b8d3b | 21 | parse_duration, |
ea1f5e5d | 22 | smuggle_url, |
3052a30d | 23 | url_or_none, |
a4ec4517 S |
24 | xpath_with_ns, |
25 | xpath_element, | |
26 | xpath_text, | |
a71b8d3b RA |
27 | ) |
28 | ||
29 | ||
30 | class ITVIE(InfoExtractor): | |
f592ff98 | 31 | _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)' |
4248dad9 | 32 | _GEO_COUNTRIES = ['GB'] |
3fae11ac | 33 | _TESTS = [{ |
a71b8d3b RA |
34 | 'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053', |
35 | 'info_dict': { | |
36 | 'id': '2a2936a0053', | |
37 | 'ext': 'flv', | |
38 | 'title': 'Home Movie', | |
39 | }, | |
40 | 'params': { | |
41 | # rtmp download | |
42 | 'skip_download': True, | |
43 | }, | |
3fae11ac S |
44 | }, { |
45 | # unavailable via data-playlist-url | |
46 | 'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033', | |
47 | 'only_matching': True, | |
c18142da S |
48 | }, { |
49 | # InvalidVodcrid | |
50 | 'url': 'https://www.itv.com/hub/james-martins-saturday-morning/2a5159a0034', | |
51 | 'only_matching': True, | |
52 | }, { | |
53 | # ContentUnavailable | |
54 | 'url': 'https://www.itv.com/hub/whos-doing-the-dishes/2a2898a0024', | |
55 | 'only_matching': True, | |
3fae11ac | 56 | }] |
a71b8d3b RA |
57 | |
58 | def _real_extract(self, url): | |
59 | video_id = self._match_id(url) | |
60 | webpage = self._download_webpage(url, video_id) | |
61 | params = extract_attributes(self._search_regex( | |
62 | r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params')) | |
63 | ||
64 | ns_map = { | |
65 | 'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/', | |
66 | 'tem': 'http://tempuri.org/', | |
67 | 'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types', | |
68 | 'com': 'http://schemas.itv.com/2009/05/Common', | |
69 | } | |
70 | for ns, full_ns in ns_map.items(): | |
47194199 | 71 | compat_etree_register_namespace(ns, full_ns) |
a71b8d3b RA |
72 | |
73 | def _add_ns(name): | |
74 | return xpath_with_ns(name, ns_map) | |
75 | ||
76 | def _add_sub_element(element, name): | |
77 | return etree.SubElement(element, _add_ns(name)) | |
78 | ||
71dde5ee | 79 | production_id = ( |
3089bc74 S |
80 | params.get('data-video-autoplay-id') |
81 | or '%s#001' % ( | |
82 | params.get('data-video-episode-id') | |
83 | or video_id.replace('a', '/'))) | |
71dde5ee | 84 | |
a71b8d3b RA |
85 | req_env = etree.Element(_add_ns('soapenv:Envelope')) |
86 | _add_sub_element(req_env, 'soapenv:Header') | |
87 | body = _add_sub_element(req_env, 'soapenv:Body') | |
88 | get_playlist = _add_sub_element(body, ('tem:GetPlaylist')) | |
89 | request = _add_sub_element(get_playlist, 'tem:request') | |
71dde5ee | 90 | _add_sub_element(request, 'itv:ProductionId').text = production_id |
a71b8d3b RA |
91 | _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper() |
92 | vodcrid = _add_sub_element(request, 'itv:Vodcrid') | |
93 | _add_sub_element(vodcrid, 'com:Id') | |
94 | _add_sub_element(request, 'itv:Partition') | |
95 | user_info = _add_sub_element(get_playlist, 'tem:userInfo') | |
96 | _add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv' | |
97 | _add_sub_element(user_info, 'itv:DM') | |
98 | _add_sub_element(user_info, 'itv:RevenueScienceValue') | |
99 | _add_sub_element(user_info, 'itv:SessionId') | |
100 | _add_sub_element(user_info, 'itv:SsoToken') | |
101 | _add_sub_element(user_info, 'itv:UserToken') | |
102 | site_info = _add_sub_element(get_playlist, 'tem:siteInfo') | |
103 | _add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None' | |
104 | _add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV' | |
105 | _add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any' | |
106 | _add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO' | |
107 | _add_sub_element(site_info, 'itv:Category') | |
108 | _add_sub_element(site_info, 'itv:Platform').text = 'DotCom' | |
109 | _add_sub_element(site_info, 'itv:Site').text = 'ItvCom' | |
110 | device_info = _add_sub_element(get_playlist, 'tem:deviceInfo') | |
111 | _add_sub_element(device_info, 'itv:ScreenSize').text = 'Big' | |
112 | player_info = _add_sub_element(get_playlist, 'tem:playerInfo') | |
113 | _add_sub_element(player_info, 'itv:Version').text = '2' | |
114 | ||
115 | headers = self.geo_verification_headers() | |
116 | headers.update({ | |
117 | 'Content-Type': 'text/xml; charset=utf-8', | |
118 | 'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist', | |
119 | }) | |
3fae11ac S |
120 | |
121 | info = self._search_json_ld(webpage, video_id, default={}) | |
122 | formats = [] | |
123 | subtitles = {} | |
124 | ||
125 | def extract_subtitle(sub_url): | |
126 | ext = determine_ext(sub_url, 'ttml') | |
127 | subtitles.setdefault('en', []).append({ | |
128 | 'url': sub_url, | |
129 | 'ext': 'ttml' if ext == 'xml' else ext, | |
130 | }) | |
131 | ||
a71b8d3b RA |
132 | resp_env = self._download_xml( |
133 | params['data-playlist-url'], video_id, | |
30374f4d S |
134 | headers=headers, data=etree.tostring(req_env), fatal=False) |
135 | if resp_env: | |
136 | playlist = xpath_element(resp_env, './/Playlist') | |
137 | if playlist is None: | |
138 | fault_code = xpath_text(resp_env, './/faultcode') | |
139 | fault_string = xpath_text(resp_env, './/faultstring') | |
140 | if fault_code == 'InvalidGeoRegion': | |
141 | self.raise_geo_restricted( | |
142 | msg=fault_string, countries=self._GEO_COUNTRIES) | |
143 | elif fault_code not in ( | |
144 | 'InvalidEntity', 'InvalidVodcrid', 'ContentUnavailable'): | |
145 | raise ExtractorError( | |
146 | '%s said: %s' % (self.IE_NAME, fault_string), expected=True) | |
147 | info.update({ | |
148 | 'title': self._og_search_title(webpage), | |
149 | 'episode_title': params.get('data-video-episode'), | |
150 | 'series': params.get('data-video-title'), | |
151 | }) | |
152 | else: | |
153 | title = xpath_text(playlist, 'EpisodeTitle', default=None) | |
154 | info.update({ | |
155 | 'title': title, | |
156 | 'episode_title': title, | |
157 | 'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')), | |
158 | 'series': xpath_text(playlist, 'ProgrammeTitle'), | |
159 | 'duration': parse_duration(xpath_text(playlist, 'Duration')), | |
160 | }) | |
161 | video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True) | |
162 | media_files = xpath_element(video_element, 'MediaFiles', fatal=True) | |
163 | rtmp_url = media_files.attrib['base'] | |
a71b8d3b | 164 | |
30374f4d S |
165 | for media_file in media_files.findall('MediaFile'): |
166 | play_path = xpath_text(media_file, 'URL') | |
167 | if not play_path: | |
168 | continue | |
169 | tbr = int_or_none(media_file.get('bitrate'), 1000) | |
170 | f = { | |
171 | 'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''), | |
172 | 'play_path': play_path, | |
173 | # Providing this swfVfy allows to avoid truncated downloads | |
174 | 'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf', | |
175 | 'page_url': url, | |
176 | 'tbr': tbr, | |
177 | 'ext': 'flv', | |
178 | } | |
179 | app = self._search_regex( | |
180 | 'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None) | |
181 | if app: | |
182 | f.update({ | |
183 | 'url': rtmp_url.split('?', 1)[0], | |
184 | 'app': app, | |
185 | }) | |
186 | else: | |
187 | f['url'] = rtmp_url | |
188 | formats.append(f) | |
3fae11ac | 189 | |
30374f4d S |
190 | for caption_url in video_element.findall('ClosedCaptioningURIs/URL'): |
191 | if caption_url.text: | |
192 | extract_subtitle(caption_url.text) | |
a71b8d3b | 193 | |
115afb77 | 194 | ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id') |
a71b8d3b | 195 | hmac = params.get('data-video-hmac') |
115afb77 | 196 | if ios_playlist_url and hmac and re.match(r'https?://', ios_playlist_url): |
a71b8d3b RA |
197 | headers = self.geo_verification_headers() |
198 | headers.update({ | |
199 | 'Accept': 'application/vnd.itv.vod.playlist.v2+json', | |
200 | 'Content-Type': 'application/json', | |
201 | 'hmac': hmac.upper(), | |
202 | }) | |
203 | ios_playlist = self._download_json( | |
204 | ios_playlist_url, video_id, data=json.dumps({ | |
205 | 'user': { | |
206 | 'itvUserId': '', | |
207 | 'entitlements': [], | |
208 | 'token': '' | |
209 | }, | |
210 | 'device': { | |
115afb77 RA |
211 | 'manufacturer': 'Safari', |
212 | 'model': '5', | |
a71b8d3b | 213 | 'os': { |
115afb77 RA |
214 | 'name': 'Windows NT', |
215 | 'version': '6.1', | |
216 | 'type': 'desktop' | |
a71b8d3b RA |
217 | } |
218 | }, | |
219 | 'client': { | |
220 | 'version': '4.1', | |
221 | 'id': 'browser' | |
222 | }, | |
223 | 'variantAvailability': { | |
224 | 'featureset': { | |
115afb77 RA |
225 | 'min': ['hls', 'aes', 'outband-webvtt'], |
226 | 'max': ['hls', 'aes', 'outband-webvtt'] | |
a71b8d3b | 227 | }, |
115afb77 | 228 | 'platformTag': 'dotcom' |
a71b8d3b RA |
229 | } |
230 | }).encode(), headers=headers, fatal=False) | |
231 | if ios_playlist: | |
232 | video_data = ios_playlist.get('Playlist', {}).get('Video', {}) | |
233 | ios_base_url = video_data.get('Base') | |
234 | for media_file in video_data.get('MediaFiles', []): | |
235 | href = media_file.get('Href') | |
236 | if not href: | |
237 | continue | |
238 | if ios_base_url: | |
239 | href = ios_base_url + href | |
240 | ext = determine_ext(href) | |
241 | if ext == 'm3u8': | |
f67177ca S |
242 | formats.extend(self._extract_m3u8_formats( |
243 | href, video_id, 'mp4', entry_protocol='m3u8_native', | |
244 | m3u8_id='hls', fatal=False)) | |
a71b8d3b RA |
245 | else: |
246 | formats.append({ | |
247 | 'url': href, | |
248 | }) | |
3fae11ac S |
249 | subs = video_data.get('Subtitles') |
250 | if isinstance(subs, list): | |
251 | for sub in subs: | |
252 | if not isinstance(sub, dict): | |
253 | continue | |
3052a30d S |
254 | href = url_or_none(sub.get('Href')) |
255 | if href: | |
3fae11ac S |
256 | extract_subtitle(href) |
257 | if not info.get('duration'): | |
258 | info['duration'] = parse_duration(video_data.get('Duration')) | |
a71b8d3b | 259 | |
3fae11ac | 260 | self._sort_formats(formats) |
f592ff98 | 261 | |
751c89a2 | 262 | info.update({ |
a71b8d3b | 263 | 'id': video_id, |
a71b8d3b | 264 | 'formats': formats, |
f592ff98 | 265 | 'subtitles': subtitles, |
751c89a2 | 266 | }) |
30374f4d S |
267 | |
268 | webpage_info = self._search_json_ld(webpage, video_id, default={}) | |
269 | if not webpage_info.get('title'): | |
270 | webpage_info['title'] = self._html_search_regex( | |
271 | r'(?s)<h\d+[^>]+\bclass=["\'][^>]*episode-title["\'][^>]*>([^<]+)<', | |
272 | webpage, 'title', default=None) or self._og_search_title( | |
273 | webpage, default=None) or self._html_search_meta( | |
274 | 'twitter:title', webpage, 'title', | |
275 | default=None) or webpage_info['episode'] | |
276 | ||
277 | return merge_dicts(info, webpage_info) | |
ea1f5e5d S |
278 | |
279 | ||
280 | class ITVBTCCIE(InfoExtractor): | |
281 | _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)' | |
282 | _TEST = { | |
283 | 'url': 'http://www.itv.com/btcc/races/btcc-2018-all-the-action-from-brands-hatch', | |
284 | 'info_dict': { | |
285 | 'id': 'btcc-2018-all-the-action-from-brands-hatch', | |
286 | 'title': 'BTCC 2018: All the action from Brands Hatch', | |
287 | }, | |
288 | 'playlist_mincount': 9, | |
289 | } | |
290 | BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s' | |
291 | ||
292 | def _real_extract(self, url): | |
293 | playlist_id = self._match_id(url) | |
294 | ||
295 | webpage = self._download_webpage(url, playlist_id) | |
296 | ||
297 | entries = [ | |
298 | self.url_result( | |
299 | smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, { | |
300 | # ITV does not like some GB IP ranges, so here are some | |
301 | # IP blocks it accepts | |
302 | 'geo_ip_blocks': [ | |
303 | '193.113.0.0/16', '54.36.162.0/23', '159.65.16.0/21' | |
304 | ], | |
305 | 'referrer': url, | |
306 | }), | |
307 | ie=BrightcoveNewIE.ie_key(), video_id=video_id) | |
308 | for video_id in re.findall(r'data-video-id=["\'](\d+)', webpage)] | |
309 | ||
310 | title = self._og_search_title(webpage, fatal=False) | |
311 | ||
312 | return self.playlist_result(entries, playlist_id, title) |