]>
Commit | Line | Data |
---|---|---|
b0adbe98 S |
1 | from __future__ import unicode_literals |
2 | ||
b8d8cced S |
3 | import re |
4 | ||
afbdd3ac | 5 | from .common import InfoExtractor |
b8d8cced S |
6 | from ..compat import ( |
7 | compat_urlparse, | |
8 | compat_str, | |
9 | ) | |
b0adbe98 | 10 | from ..utils import ( |
f1388739 | 11 | ExtractorError, |
51342717 | 12 | determine_ext, |
f1388739 YCH |
13 | find_xpath_attr, |
14 | fix_xml_ampersands, | |
b8d8cced | 15 | GeoRestrictedError, |
f1388739 | 16 | int_or_none, |
b0adbe98 | 17 | parse_duration, |
b8d8cced S |
18 | strip_or_none, |
19 | try_get, | |
b0adbe98 | 20 | unified_strdate, |
b8d8cced | 21 | unified_timestamp, |
f1388739 | 22 | update_url_query, |
b8d8cced | 23 | urljoin, |
06d5556d | 24 | xpath_text, |
b0adbe98 S |
25 | ) |
26 | ||
27 | ||
034a8849 | 28 | class RaiBaseIE(InfoExtractor): |
b8d8cced S |
29 | _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}' |
30 | _GEO_COUNTRIES = ['IT'] | |
31 | _GEO_BYPASS = False | |
32 | ||
33 | def _extract_relinker_info(self, relinker_url, video_id): | |
034a8849 | 34 | formats = [] |
b8d8cced S |
35 | geoprotection = None |
36 | is_live = None | |
37 | duration = None | |
034a8849 YCH |
38 | |
39 | for platform in ('mon', 'flash', 'native'): | |
034a8849 YCH |
40 | relinker = self._download_xml( |
41 | relinker_url, video_id, | |
42 | note='Downloading XML metadata for platform %s' % platform, | |
43 | transform_source=fix_xml_ampersands, | |
38cce791 YCH |
44 | query={'output': 45, 'pl': platform}, |
45 | headers=self.geo_verification_headers()) | |
034a8849 | 46 | |
b8d8cced S |
47 | if not geoprotection: |
48 | geoprotection = xpath_text( | |
49 | relinker, './geoprotection', default=None) == 'Y' | |
50 | ||
51 | if not is_live: | |
52 | is_live = xpath_text( | |
53 | relinker, './is_live', default=None) == 'Y' | |
54 | if not duration: | |
55 | duration = parse_duration(xpath_text( | |
56 | relinker, './duration', default=None)) | |
57 | ||
58 | url_elem = find_xpath_attr(relinker, './url', 'type', 'content') | |
59 | if url_elem is None: | |
60 | continue | |
61 | ||
62 | media_url = url_elem.text | |
63 | ||
64 | # This does not imply geo restriction (e.g. | |
65 | # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html) | |
034a8849 | 66 | if media_url == 'http://download.rai.it/video_no_available.mp4': |
b8d8cced | 67 | continue |
034a8849 YCH |
68 | |
69 | ext = determine_ext(media_url) | |
70 | if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'): | |
71 | continue | |
72 | ||
73 | if ext == 'm3u8': | |
74 | formats.extend(self._extract_m3u8_formats( | |
75 | media_url, video_id, 'mp4', 'm3u8_native', | |
76 | m3u8_id='hls', fatal=False)) | |
77 | elif ext == 'f4m': | |
78 | manifest_url = update_url_query( | |
79 | media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'), | |
80 | {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'}) | |
81 | formats.extend(self._extract_f4m_formats( | |
82 | manifest_url, video_id, f4m_id='hds', fatal=False)) | |
83 | else: | |
84 | bitrate = int_or_none(xpath_text(relinker, 'bitrate')) | |
85 | formats.append({ | |
86 | 'url': media_url, | |
87 | 'tbr': bitrate if bitrate > 0 else None, | |
88 | 'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http', | |
89 | }) | |
90 | ||
b8d8cced S |
91 | if not formats and geoprotection is True: |
92 | self.raise_geo_restricted(countries=self._GEO_COUNTRIES) | |
93 | ||
94 | return dict((k, v) for k, v in { | |
95 | 'is_live': is_live, | |
96 | 'duration': duration, | |
97 | 'formats': formats, | |
98 | }.items() if v is not None) | |
034a8849 | 99 | |
1b3feca0 S |
100 | @staticmethod |
101 | def _extract_subtitles(url, subtitle_url): | |
102 | subtitles = {} | |
103 | if subtitle_url and isinstance(subtitle_url, compat_str): | |
104 | subtitle_url = urljoin(url, subtitle_url) | |
105 | STL_EXT = '.stl' | |
106 | SRT_EXT = '.srt' | |
107 | subtitles['it'] = [{ | |
108 | 'ext': 'stl', | |
109 | 'url': subtitle_url, | |
110 | }] | |
111 | if subtitle_url.endswith(STL_EXT): | |
112 | srt_url = subtitle_url[:-len(STL_EXT)] + SRT_EXT | |
113 | subtitles['it'].append({ | |
114 | 'ext': 'srt', | |
115 | 'url': srt_url, | |
116 | }) | |
117 | return subtitles | |
118 | ||
2b28b892 | 119 | |
51342717 | 120 | class RaiPlayIE(RaiBaseIE): |
b8d8cced | 121 | _VALID_URL = r'(?P<url>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s)\.html)' % RaiBaseIE._UUID_RE |
51342717 T |
122 | _TESTS = [{ |
123 | 'url': 'http://www.raiplay.it/video/2016/10/La-Casa-Bianca-e06118bb-59a9-4636-b914-498e4cfd2c66.html?source=twitter', | |
124 | 'md5': '340aa3b7afb54bfd14a8c11786450d76', | |
125 | 'info_dict': { | |
126 | 'id': 'e06118bb-59a9-4636-b914-498e4cfd2c66', | |
127 | 'ext': 'mp4', | |
128 | 'title': 'La Casa Bianca', | |
b8d8cced S |
129 | 'alt_title': 'S2016 - Puntata del 23/10/2016', |
130 | 'description': 'md5:a09d45890850458077d1f68bb036e0a5', | |
51342717 | 131 | 'thumbnail': r're:^https?://.*\.jpg$', |
b8d8cced S |
132 | 'uploader': 'Rai 3', |
133 | 'creator': 'Rai 3', | |
134 | 'duration': 3278, | |
135 | 'timestamp': 1477764300, | |
136 | 'upload_date': '20161029', | |
137 | 'series': 'La Casa Bianca', | |
138 | 'season': '2016', | |
139 | }, | |
51342717 T |
140 | }, { |
141 | 'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html', | |
142 | 'md5': '8970abf8caf8aef4696e7b1f2adfc696', | |
143 | 'info_dict': { | |
144 | 'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391', | |
145 | 'ext': 'mp4', | |
b8d8cced S |
146 | 'title': 'Report del 07/04/2014', |
147 | 'alt_title': 'S2013/14 - Puntata del 07/04/2014', | |
148 | 'description': 'md5:f27c544694cacb46a078db84ec35d2d9', | |
51342717 | 149 | 'thumbnail': r're:^https?://.*\.jpg$', |
b8d8cced S |
150 | 'uploader': 'Rai 5', |
151 | 'creator': 'Rai 5', | |
152 | 'duration': 6160, | |
153 | 'series': 'Report', | |
154 | 'season_number': 5, | |
155 | 'season': '2013/14', | |
156 | }, | |
157 | 'params': { | |
158 | 'skip_download': True, | |
159 | }, | |
160 | }, { | |
161 | 'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?', | |
162 | 'only_matching': True, | |
51342717 | 163 | }] |
2b28b892 | 164 | |
51342717 | 165 | def _real_extract(self, url): |
b8d8cced S |
166 | mobj = re.match(self._VALID_URL, url) |
167 | url, video_id = mobj.group('url', 'id') | |
2b28b892 | 168 | |
b8d8cced S |
169 | media = self._download_json( |
170 | '%s?json' % url, video_id, 'Downloading video JSON') | |
2b28b892 | 171 | |
b8d8cced S |
172 | title = media['name'] |
173 | ||
174 | video = media['video'] | |
175 | ||
176 | relinker_info = self._extract_relinker_info(video['contentUrl'], video_id) | |
177 | self._sort_formats(relinker_info['formats']) | |
2b28b892 | 178 | |
51342717 T |
179 | thumbnails = [] |
180 | if 'images' in media: | |
181 | for _, value in media.get('images').items(): | |
182 | if value: | |
183 | thumbnails.append({ | |
b8d8cced | 184 | 'url': value.replace('[RESOLUTION]', '600x400') |
51342717 | 185 | }) |
034a8849 | 186 | |
b8d8cced S |
187 | timestamp = unified_timestamp(try_get( |
188 | media, lambda x: x['availabilities'][0]['start'], compat_str)) | |
b0adbe98 | 189 | |
1b3feca0 S |
190 | subtitles = self._extract_subtitles(url, video.get('subtitles')) |
191 | ||
b8d8cced | 192 | info = { |
51342717 | 193 | 'id': video_id, |
b8d8cced S |
194 | 'title': title, |
195 | 'alt_title': media.get('subtitle'), | |
196 | 'description': media.get('description'), | |
51342717 | 197 | 'uploader': media.get('channel'), |
b8d8cced S |
198 | 'creator': media.get('editor'), |
199 | 'duration': parse_duration(video.get('duration')), | |
200 | 'timestamp': timestamp, | |
51342717 | 201 | 'thumbnails': thumbnails, |
b8d8cced S |
202 | 'series': try_get( |
203 | media, lambda x: x['isPartOf']['name'], compat_str), | |
204 | 'season_number': int_or_none(try_get( | |
205 | media, lambda x: x['isPartOf']['numeroStagioni'])), | |
206 | 'season': media.get('stagione') or None, | |
1b3feca0 | 207 | 'subtitles': subtitles, |
51342717 | 208 | } |
b0adbe98 | 209 | |
b8d8cced S |
210 | info.update(relinker_info) |
211 | ||
212 | return info | |
213 | ||
06d5556d | 214 | |
034a8849 | 215 | class RaiIE(RaiBaseIE): |
b8d8cced | 216 | _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/dl/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE |
51342717 | 217 | _TESTS = [{ |
b8d8cced S |
218 | # var uniquename = "ContentItem-..." |
219 | # data-id="ContentItem-..." | |
51342717 T |
220 | 'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html', |
221 | 'info_dict': { | |
222 | 'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9', | |
223 | 'ext': 'mp4', | |
224 | 'title': 'TG PRIMO TEMPO', | |
b8d8cced | 225 | 'thumbnail': r're:^https?://.*\.jpg$', |
51342717 | 226 | 'duration': 1758, |
b8d8cced | 227 | 'upload_date': '20140612', |
51342717 T |
228 | } |
229 | }, { | |
b8d8cced | 230 | # with ContentItem in many metas |
51342717 T |
231 | 'url': 'http://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html', |
232 | 'info_dict': { | |
233 | 'id': '1632c009-c843-4836-bb65-80c33084a64b', | |
234 | 'ext': 'mp4', | |
b8d8cced S |
235 | 'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor "La ragazza del treno"', |
236 | 'description': 'I film in uscita questa settimana.', | |
51342717 | 237 | 'thumbnail': r're:^https?://.*\.png$', |
b8d8cced S |
238 | 'duration': 833, |
239 | 'upload_date': '20161103', | |
51342717 T |
240 | } |
241 | }, { | |
b8d8cced | 242 | # with ContentItem in og:url |
51342717 T |
243 | 'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html', |
244 | 'md5': '11959b4e44fa74de47011b5799490adf', | |
245 | 'info_dict': { | |
246 | 'id': 'efb17665-691c-45d5-a60c-5301333cbb0c', | |
247 | 'ext': 'mp4', | |
248 | 'title': 'TG1 ore 20:00 del 03/11/2016', | |
b8d8cced | 249 | 'description': 'TG1 edizione integrale ore 20:00 del giorno 03/11/2016', |
51342717 | 250 | 'thumbnail': r're:^https?://.*\.jpg$', |
b8d8cced | 251 | 'duration': 2214, |
51342717 | 252 | 'upload_date': '20161103', |
51342717 T |
253 | } |
254 | }, { | |
b8d8cced | 255 | # drawMediaRaiTV(...) |
51342717 T |
256 | 'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html', |
257 | 'md5': '2dd727e61114e1ee9c47f0da6914e178', | |
258 | 'info_dict': { | |
259 | 'id': '59d69d28-6bb6-409d-a4b5-ed44096560af', | |
260 | 'ext': 'mp4', | |
261 | 'title': 'Il pacco', | |
262 | 'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a', | |
b8d8cced | 263 | 'thumbnail': r're:^https?://.*\.jpg$', |
51342717 | 264 | 'upload_date': '20141221', |
034a8849 | 265 | }, |
51342717 | 266 | }, { |
b8d8cced S |
267 | # initEdizione('ContentItem-...' |
268 | 'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined', | |
269 | 'info_dict': { | |
270 | 'id': 'c2187016-8484-4e3a-8ac8-35e475b07303', | |
271 | 'ext': 'mp4', | |
272 | 'title': r're:TG1 ore \d{2}:\d{2} del \d{2}/\d{2}/\d{4}', | |
273 | 'duration': 2274, | |
274 | 'upload_date': '20170401', | |
275 | }, | |
276 | 'skip': 'Changes daily', | |
277 | }, { | |
278 | # HDS live stream with only relinker URL | |
51342717 | 279 | 'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews', |
51342717 T |
280 | 'info_dict': { |
281 | 'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc', | |
282 | 'ext': 'flv', | |
283 | 'title': 'EuroNews', | |
2b28b892 | 284 | }, |
b8d8cced S |
285 | 'params': { |
286 | 'skip_download': True, | |
2b28b892 | 287 | }, |
51342717 | 288 | }, { |
b8d8cced | 289 | # HLS live stream with ContentItem in og:url |
51342717 | 290 | 'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html', |
51342717 T |
291 | 'info_dict': { |
292 | 'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9', | |
293 | 'ext': 'mp4', | |
294 | 'title': 'La diretta di Rainews24', | |
15e4b6b7 | 295 | }, |
b8d8cced S |
296 | 'params': { |
297 | 'skip_download': True, | |
298 | }, | |
51342717 | 299 | }] |
06d5556d | 300 | |
51342717 T |
301 | def _extract_from_content_id(self, content_id, url): |
302 | media = self._download_json( | |
303 | 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id, | |
304 | content_id, 'Downloading video JSON') | |
305 | ||
b8d8cced S |
306 | title = media['name'].strip() |
307 | ||
308 | media_type = media['type'] | |
309 | if 'Audio' in media_type: | |
310 | relinker_info = { | |
311 | 'formats': { | |
312 | 'format_id': media.get('formatoAudio'), | |
313 | 'url': media['audioUrl'], | |
314 | 'ext': media.get('formatoAudio'), | |
315 | } | |
316 | } | |
317 | elif 'Video' in media_type: | |
318 | relinker_info = self._extract_relinker_info(media['mediaUri'], content_id) | |
319 | else: | |
320 | raise ExtractorError('not a media file') | |
321 | ||
322 | self._sort_formats(relinker_info['formats']) | |
323 | ||
51342717 T |
324 | thumbnails = [] |
325 | for image_type in ('image', 'image_medium', 'image_300'): | |
326 | thumbnail_url = media.get(image_type) | |
327 | if thumbnail_url: | |
328 | thumbnails.append({ | |
329 | 'url': compat_urlparse.urljoin(url, thumbnail_url), | |
330 | }) | |
331 | ||
1b3feca0 | 332 | subtitles = self._extract_subtitles(url, media.get('subtitlesUrl')) |
51342717 | 333 | |
b8d8cced | 334 | info = { |
51342717 | 335 | 'id': content_id, |
b8d8cced S |
336 | 'title': title, |
337 | 'description': strip_or_none(media.get('desc')), | |
51342717 T |
338 | 'thumbnails': thumbnails, |
339 | 'uploader': media.get('author'), | |
340 | 'upload_date': unified_strdate(media.get('date')), | |
341 | 'duration': parse_duration(media.get('length')), | |
51342717 T |
342 | 'subtitles': subtitles, |
343 | } | |
b8d8cced S |
344 | |
345 | info.update(relinker_info) | |
346 | ||
347 | return info | |
348 | ||
349 | def _real_extract(self, url): | |
350 | video_id = self._match_id(url) | |
351 | ||
352 | webpage = self._download_webpage(url, video_id) | |
353 | ||
354 | content_item_id = None | |
355 | ||
356 | content_item_url = self._html_search_meta( | |
357 | ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url', | |
358 | 'twitter:player', 'jsonlink'), webpage, default=None) | |
359 | if content_item_url: | |
360 | content_item_id = self._search_regex( | |
361 | r'ContentItem-(%s)' % self._UUID_RE, content_item_url, | |
362 | 'content item id', default=None) | |
363 | ||
364 | if not content_item_id: | |
365 | content_item_id = self._search_regex( | |
366 | r'''(?x) | |
367 | (?: | |
368 | (?:initEdizione|drawMediaRaiTV)\(| | |
369 | <(?:[^>]+\bdata-id|var\s+uniquename)= | |
370 | ) | |
371 | (["\']) | |
372 | (?:(?!\1).)*\bContentItem-(?P<id>%s) | |
373 | ''' % self._UUID_RE, | |
374 | webpage, 'content item id', default=None, group='id') | |
375 | ||
376 | content_item_ids = set() | |
361f293a S |
377 | if content_item_id: |
378 | content_item_ids.add(content_item_id) | |
b8d8cced S |
379 | if video_id not in content_item_ids: |
380 | content_item_ids.add(video_id) | |
381 | ||
382 | for content_item_id in content_item_ids: | |
383 | try: | |
384 | return self._extract_from_content_id(content_item_id, url) | |
385 | except GeoRestrictedError: | |
386 | raise | |
387 | except ExtractorError: | |
388 | pass | |
389 | ||
390 | relinker_url = self._search_regex( | |
391 | r'''(?x) | |
392 | (?: | |
393 | var\s+videoURL| | |
394 | mediaInfo\.mediaUri | |
395 | )\s*=\s* | |
396 | ([\'"]) | |
397 | (?P<url> | |
398 | (?:https?:)? | |
399 | //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\? | |
400 | (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1 | |
401 | ''', | |
402 | webpage, 'relinker URL', group='url') | |
403 | ||
404 | relinker_info = self._extract_relinker_info( | |
405 | urljoin(url, relinker_url), video_id) | |
406 | self._sort_formats(relinker_info['formats']) | |
407 | ||
408 | title = self._search_regex( | |
409 | r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1', | |
410 | webpage, 'title', group='title', | |
411 | default=None) or self._og_search_title(webpage) | |
412 | ||
413 | info = { | |
414 | 'id': video_id, | |
415 | 'title': title, | |
416 | } | |
417 | ||
418 | info.update(relinker_info) | |
419 | ||
420 | return info |