]>
Commit | Line | Data |
---|---|---|
b0adbe98 S |
1 | from __future__ import unicode_literals |
2 | ||
b8d8cced S |
3 | import re |
4 | ||
afbdd3ac | 5 | from .common import InfoExtractor |
b8d8cced S |
6 | from ..compat import ( |
7 | compat_urlparse, | |
8 | compat_str, | |
9 | ) | |
b0adbe98 | 10 | from ..utils import ( |
f1388739 | 11 | ExtractorError, |
51342717 | 12 | determine_ext, |
f1388739 YCH |
13 | find_xpath_attr, |
14 | fix_xml_ampersands, | |
b8d8cced | 15 | GeoRestrictedError, |
f1388739 | 16 | int_or_none, |
b0adbe98 | 17 | parse_duration, |
b8d8cced S |
18 | strip_or_none, |
19 | try_get, | |
b0adbe98 | 20 | unified_strdate, |
b8d8cced | 21 | unified_timestamp, |
f1388739 | 22 | update_url_query, |
b8d8cced | 23 | urljoin, |
06d5556d | 24 | xpath_text, |
b0adbe98 S |
25 | ) |
26 | ||
27 | ||
034a8849 | 28 | class RaiBaseIE(InfoExtractor): |
b8d8cced S |
29 | _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}' |
30 | _GEO_COUNTRIES = ['IT'] | |
31 | _GEO_BYPASS = False | |
32 | ||
33 | def _extract_relinker_info(self, relinker_url, video_id): | |
034a8849 | 34 | formats = [] |
b8d8cced S |
35 | geoprotection = None |
36 | is_live = None | |
37 | duration = None | |
034a8849 YCH |
38 | |
39 | for platform in ('mon', 'flash', 'native'): | |
034a8849 YCH |
40 | relinker = self._download_xml( |
41 | relinker_url, video_id, | |
42 | note='Downloading XML metadata for platform %s' % platform, | |
43 | transform_source=fix_xml_ampersands, | |
38cce791 YCH |
44 | query={'output': 45, 'pl': platform}, |
45 | headers=self.geo_verification_headers()) | |
034a8849 | 46 | |
b8d8cced S |
47 | if not geoprotection: |
48 | geoprotection = xpath_text( | |
49 | relinker, './geoprotection', default=None) == 'Y' | |
50 | ||
51 | if not is_live: | |
52 | is_live = xpath_text( | |
53 | relinker, './is_live', default=None) == 'Y' | |
54 | if not duration: | |
55 | duration = parse_duration(xpath_text( | |
56 | relinker, './duration', default=None)) | |
57 | ||
58 | url_elem = find_xpath_attr(relinker, './url', 'type', 'content') | |
59 | if url_elem is None: | |
60 | continue | |
61 | ||
62 | media_url = url_elem.text | |
63 | ||
64 | # This does not imply geo restriction (e.g. | |
65 | # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html) | |
034a8849 | 66 | if media_url == 'http://download.rai.it/video_no_available.mp4': |
b8d8cced | 67 | continue |
034a8849 YCH |
68 | |
69 | ext = determine_ext(media_url) | |
70 | if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'): | |
71 | continue | |
72 | ||
73 | if ext == 'm3u8': | |
74 | formats.extend(self._extract_m3u8_formats( | |
75 | media_url, video_id, 'mp4', 'm3u8_native', | |
76 | m3u8_id='hls', fatal=False)) | |
77 | elif ext == 'f4m': | |
78 | manifest_url = update_url_query( | |
79 | media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'), | |
80 | {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'}) | |
81 | formats.extend(self._extract_f4m_formats( | |
82 | manifest_url, video_id, f4m_id='hds', fatal=False)) | |
83 | else: | |
84 | bitrate = int_or_none(xpath_text(relinker, 'bitrate')) | |
85 | formats.append({ | |
86 | 'url': media_url, | |
87 | 'tbr': bitrate if bitrate > 0 else None, | |
88 | 'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http', | |
89 | }) | |
90 | ||
b8d8cced S |
91 | if not formats and geoprotection is True: |
92 | self.raise_geo_restricted(countries=self._GEO_COUNTRIES) | |
93 | ||
94 | return dict((k, v) for k, v in { | |
95 | 'is_live': is_live, | |
96 | 'duration': duration, | |
97 | 'formats': formats, | |
98 | }.items() if v is not None) | |
034a8849 | 99 | |
2b28b892 | 100 | |
51342717 | 101 | class RaiPlayIE(RaiBaseIE): |
b8d8cced | 102 | _VALID_URL = r'(?P<url>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s)\.html)' % RaiBaseIE._UUID_RE |
51342717 T |
103 | _TESTS = [{ |
104 | 'url': 'http://www.raiplay.it/video/2016/10/La-Casa-Bianca-e06118bb-59a9-4636-b914-498e4cfd2c66.html?source=twitter', | |
105 | 'md5': '340aa3b7afb54bfd14a8c11786450d76', | |
106 | 'info_dict': { | |
107 | 'id': 'e06118bb-59a9-4636-b914-498e4cfd2c66', | |
108 | 'ext': 'mp4', | |
109 | 'title': 'La Casa Bianca', | |
b8d8cced S |
110 | 'alt_title': 'S2016 - Puntata del 23/10/2016', |
111 | 'description': 'md5:a09d45890850458077d1f68bb036e0a5', | |
51342717 | 112 | 'thumbnail': r're:^https?://.*\.jpg$', |
b8d8cced S |
113 | 'uploader': 'Rai 3', |
114 | 'creator': 'Rai 3', | |
115 | 'duration': 3278, | |
116 | 'timestamp': 1477764300, | |
117 | 'upload_date': '20161029', | |
118 | 'series': 'La Casa Bianca', | |
119 | 'season': '2016', | |
120 | }, | |
51342717 T |
121 | }, { |
122 | 'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html', | |
123 | 'md5': '8970abf8caf8aef4696e7b1f2adfc696', | |
124 | 'info_dict': { | |
125 | 'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391', | |
126 | 'ext': 'mp4', | |
b8d8cced S |
127 | 'title': 'Report del 07/04/2014', |
128 | 'alt_title': 'S2013/14 - Puntata del 07/04/2014', | |
129 | 'description': 'md5:f27c544694cacb46a078db84ec35d2d9', | |
51342717 | 130 | 'thumbnail': r're:^https?://.*\.jpg$', |
b8d8cced S |
131 | 'uploader': 'Rai 5', |
132 | 'creator': 'Rai 5', | |
133 | 'duration': 6160, | |
134 | 'series': 'Report', | |
135 | 'season_number': 5, | |
136 | 'season': '2013/14', | |
137 | }, | |
138 | 'params': { | |
139 | 'skip_download': True, | |
140 | }, | |
141 | }, { | |
142 | 'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?', | |
143 | 'only_matching': True, | |
51342717 | 144 | }] |
2b28b892 | 145 | |
51342717 | 146 | def _real_extract(self, url): |
b8d8cced S |
147 | mobj = re.match(self._VALID_URL, url) |
148 | url, video_id = mobj.group('url', 'id') | |
2b28b892 | 149 | |
b8d8cced S |
150 | media = self._download_json( |
151 | '%s?json' % url, video_id, 'Downloading video JSON') | |
2b28b892 | 152 | |
b8d8cced S |
153 | title = media['name'] |
154 | ||
155 | video = media['video'] | |
156 | ||
157 | relinker_info = self._extract_relinker_info(video['contentUrl'], video_id) | |
158 | self._sort_formats(relinker_info['formats']) | |
2b28b892 | 159 | |
51342717 T |
160 | thumbnails = [] |
161 | if 'images' in media: | |
162 | for _, value in media.get('images').items(): | |
163 | if value: | |
164 | thumbnails.append({ | |
b8d8cced | 165 | 'url': value.replace('[RESOLUTION]', '600x400') |
51342717 | 166 | }) |
034a8849 | 167 | |
b8d8cced S |
168 | timestamp = unified_timestamp(try_get( |
169 | media, lambda x: x['availabilities'][0]['start'], compat_str)) | |
b0adbe98 | 170 | |
b8d8cced | 171 | info = { |
51342717 | 172 | 'id': video_id, |
b8d8cced S |
173 | 'title': title, |
174 | 'alt_title': media.get('subtitle'), | |
175 | 'description': media.get('description'), | |
51342717 | 176 | 'uploader': media.get('channel'), |
b8d8cced S |
177 | 'creator': media.get('editor'), |
178 | 'duration': parse_duration(video.get('duration')), | |
179 | 'timestamp': timestamp, | |
51342717 | 180 | 'thumbnails': thumbnails, |
b8d8cced S |
181 | 'series': try_get( |
182 | media, lambda x: x['isPartOf']['name'], compat_str), | |
183 | 'season_number': int_or_none(try_get( | |
184 | media, lambda x: x['isPartOf']['numeroStagioni'])), | |
185 | 'season': media.get('stagione') or None, | |
51342717 | 186 | } |
b0adbe98 | 187 | |
b8d8cced S |
188 | info.update(relinker_info) |
189 | ||
190 | return info | |
191 | ||
06d5556d | 192 | |
034a8849 | 193 | class RaiIE(RaiBaseIE): |
b8d8cced | 194 | _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/dl/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE |
51342717 | 195 | _TESTS = [{ |
b8d8cced S |
196 | # var uniquename = "ContentItem-..." |
197 | # data-id="ContentItem-..." | |
51342717 T |
198 | 'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html', |
199 | 'info_dict': { | |
200 | 'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9', | |
201 | 'ext': 'mp4', | |
202 | 'title': 'TG PRIMO TEMPO', | |
b8d8cced | 203 | 'thumbnail': r're:^https?://.*\.jpg$', |
51342717 | 204 | 'duration': 1758, |
b8d8cced | 205 | 'upload_date': '20140612', |
51342717 T |
206 | } |
207 | }, { | |
b8d8cced | 208 | # with ContentItem in many metas |
51342717 T |
209 | 'url': 'http://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html', |
210 | 'info_dict': { | |
211 | 'id': '1632c009-c843-4836-bb65-80c33084a64b', | |
212 | 'ext': 'mp4', | |
b8d8cced S |
213 | 'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor "La ragazza del treno"', |
214 | 'description': 'I film in uscita questa settimana.', | |
51342717 | 215 | 'thumbnail': r're:^https?://.*\.png$', |
b8d8cced S |
216 | 'duration': 833, |
217 | 'upload_date': '20161103', | |
51342717 T |
218 | } |
219 | }, { | |
b8d8cced | 220 | # with ContentItem in og:url |
51342717 T |
221 | 'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html', |
222 | 'md5': '11959b4e44fa74de47011b5799490adf', | |
223 | 'info_dict': { | |
224 | 'id': 'efb17665-691c-45d5-a60c-5301333cbb0c', | |
225 | 'ext': 'mp4', | |
226 | 'title': 'TG1 ore 20:00 del 03/11/2016', | |
b8d8cced | 227 | 'description': 'TG1 edizione integrale ore 20:00 del giorno 03/11/2016', |
51342717 | 228 | 'thumbnail': r're:^https?://.*\.jpg$', |
b8d8cced | 229 | 'duration': 2214, |
51342717 | 230 | 'upload_date': '20161103', |
51342717 T |
231 | } |
232 | }, { | |
b8d8cced | 233 | # drawMediaRaiTV(...) |
51342717 T |
234 | 'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html', |
235 | 'md5': '2dd727e61114e1ee9c47f0da6914e178', | |
236 | 'info_dict': { | |
237 | 'id': '59d69d28-6bb6-409d-a4b5-ed44096560af', | |
238 | 'ext': 'mp4', | |
239 | 'title': 'Il pacco', | |
240 | 'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a', | |
b8d8cced | 241 | 'thumbnail': r're:^https?://.*\.jpg$', |
51342717 | 242 | 'upload_date': '20141221', |
034a8849 | 243 | }, |
51342717 | 244 | }, { |
b8d8cced S |
245 | # initEdizione('ContentItem-...' |
246 | 'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined', | |
247 | 'info_dict': { | |
248 | 'id': 'c2187016-8484-4e3a-8ac8-35e475b07303', | |
249 | 'ext': 'mp4', | |
250 | 'title': r're:TG1 ore \d{2}:\d{2} del \d{2}/\d{2}/\d{4}', | |
251 | 'duration': 2274, | |
252 | 'upload_date': '20170401', | |
253 | }, | |
254 | 'skip': 'Changes daily', | |
255 | }, { | |
256 | # HDS live stream with only relinker URL | |
51342717 | 257 | 'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews', |
51342717 T |
258 | 'info_dict': { |
259 | 'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc', | |
260 | 'ext': 'flv', | |
261 | 'title': 'EuroNews', | |
2b28b892 | 262 | }, |
b8d8cced S |
263 | 'params': { |
264 | 'skip_download': True, | |
2b28b892 | 265 | }, |
51342717 | 266 | }, { |
b8d8cced | 267 | # HLS live stream with ContentItem in og:url |
51342717 | 268 | 'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html', |
51342717 T |
269 | 'info_dict': { |
270 | 'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9', | |
271 | 'ext': 'mp4', | |
272 | 'title': 'La diretta di Rainews24', | |
15e4b6b7 | 273 | }, |
b8d8cced S |
274 | 'params': { |
275 | 'skip_download': True, | |
276 | }, | |
51342717 | 277 | }] |
06d5556d | 278 | |
51342717 T |
279 | def _extract_from_content_id(self, content_id, url): |
280 | media = self._download_json( | |
281 | 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id, | |
282 | content_id, 'Downloading video JSON') | |
283 | ||
b8d8cced S |
284 | title = media['name'].strip() |
285 | ||
286 | media_type = media['type'] | |
287 | if 'Audio' in media_type: | |
288 | relinker_info = { | |
289 | 'formats': { | |
290 | 'format_id': media.get('formatoAudio'), | |
291 | 'url': media['audioUrl'], | |
292 | 'ext': media.get('formatoAudio'), | |
293 | } | |
294 | } | |
295 | elif 'Video' in media_type: | |
296 | relinker_info = self._extract_relinker_info(media['mediaUri'], content_id) | |
297 | else: | |
298 | raise ExtractorError('not a media file') | |
299 | ||
300 | self._sort_formats(relinker_info['formats']) | |
301 | ||
51342717 T |
302 | thumbnails = [] |
303 | for image_type in ('image', 'image_medium', 'image_300'): | |
304 | thumbnail_url = media.get(image_type) | |
305 | if thumbnail_url: | |
306 | thumbnails.append({ | |
307 | 'url': compat_urlparse.urljoin(url, thumbnail_url), | |
308 | }) | |
309 | ||
51342717 T |
310 | subtitles = {} |
311 | captions = media.get('subtitlesUrl') | |
312 | if captions: | |
313 | STL_EXT = '.stl' | |
314 | SRT_EXT = '.srt' | |
315 | if captions.endswith(STL_EXT): | |
316 | captions = captions[:-len(STL_EXT)] + SRT_EXT | |
317 | subtitles['it'] = [{ | |
318 | 'ext': 'srt', | |
319 | 'url': captions, | |
320 | }] | |
321 | ||
b8d8cced | 322 | info = { |
51342717 | 323 | 'id': content_id, |
b8d8cced S |
324 | 'title': title, |
325 | 'description': strip_or_none(media.get('desc')), | |
51342717 T |
326 | 'thumbnails': thumbnails, |
327 | 'uploader': media.get('author'), | |
328 | 'upload_date': unified_strdate(media.get('date')), | |
329 | 'duration': parse_duration(media.get('length')), | |
51342717 T |
330 | 'subtitles': subtitles, |
331 | } | |
b8d8cced S |
332 | |
333 | info.update(relinker_info) | |
334 | ||
335 | return info | |
336 | ||
337 | def _real_extract(self, url): | |
338 | video_id = self._match_id(url) | |
339 | ||
340 | webpage = self._download_webpage(url, video_id) | |
341 | ||
342 | content_item_id = None | |
343 | ||
344 | content_item_url = self._html_search_meta( | |
345 | ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url', | |
346 | 'twitter:player', 'jsonlink'), webpage, default=None) | |
347 | if content_item_url: | |
348 | content_item_id = self._search_regex( | |
349 | r'ContentItem-(%s)' % self._UUID_RE, content_item_url, | |
350 | 'content item id', default=None) | |
351 | ||
352 | if not content_item_id: | |
353 | content_item_id = self._search_regex( | |
354 | r'''(?x) | |
355 | (?: | |
356 | (?:initEdizione|drawMediaRaiTV)\(| | |
357 | <(?:[^>]+\bdata-id|var\s+uniquename)= | |
358 | ) | |
359 | (["\']) | |
360 | (?:(?!\1).)*\bContentItem-(?P<id>%s) | |
361 | ''' % self._UUID_RE, | |
362 | webpage, 'content item id', default=None, group='id') | |
363 | ||
364 | content_item_ids = set() | |
361f293a S |
365 | if content_item_id: |
366 | content_item_ids.add(content_item_id) | |
b8d8cced S |
367 | if video_id not in content_item_ids: |
368 | content_item_ids.add(video_id) | |
369 | ||
370 | for content_item_id in content_item_ids: | |
371 | try: | |
372 | return self._extract_from_content_id(content_item_id, url) | |
373 | except GeoRestrictedError: | |
374 | raise | |
375 | except ExtractorError: | |
376 | pass | |
377 | ||
378 | relinker_url = self._search_regex( | |
379 | r'''(?x) | |
380 | (?: | |
381 | var\s+videoURL| | |
382 | mediaInfo\.mediaUri | |
383 | )\s*=\s* | |
384 | ([\'"]) | |
385 | (?P<url> | |
386 | (?:https?:)? | |
387 | //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\? | |
388 | (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1 | |
389 | ''', | |
390 | webpage, 'relinker URL', group='url') | |
391 | ||
392 | relinker_info = self._extract_relinker_info( | |
393 | urljoin(url, relinker_url), video_id) | |
394 | self._sort_formats(relinker_info['formats']) | |
395 | ||
396 | title = self._search_regex( | |
397 | r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1', | |
398 | webpage, 'title', group='title', | |
399 | default=None) or self._og_search_title(webpage) | |
400 | ||
401 | info = { | |
402 | 'id': video_id, | |
403 | 'title': title, | |
404 | } | |
405 | ||
406 | info.update(relinker_info) | |
407 | ||
408 | return info |