]>
Commit | Line | Data |
---|---|---|
abea9145 | 1 | # coding: utf-8 |
b0adbe98 S |
2 | from __future__ import unicode_literals |
3 | ||
b8d8cced S |
4 | import re |
5 | ||
afbdd3ac | 6 | from .common import InfoExtractor |
b8d8cced S |
7 | from ..compat import ( |
8 | compat_urlparse, | |
9 | compat_str, | |
10 | ) | |
b0adbe98 | 11 | from ..utils import ( |
f1388739 | 12 | ExtractorError, |
51342717 | 13 | determine_ext, |
f1388739 YCH |
14 | find_xpath_attr, |
15 | fix_xml_ampersands, | |
b8d8cced | 16 | GeoRestrictedError, |
f1388739 | 17 | int_or_none, |
b0adbe98 | 18 | parse_duration, |
a0566bbf | 19 | remove_start, |
b8d8cced | 20 | strip_or_none, |
a0566bbf | 21 | try_get, |
b0adbe98 | 22 | unified_strdate, |
b8d8cced | 23 | unified_timestamp, |
f1388739 | 24 | update_url_query, |
b8d8cced | 25 | urljoin, |
06d5556d | 26 | xpath_text, |
b0adbe98 S |
27 | ) |
28 | ||
29 | ||
034a8849 | 30 | class RaiBaseIE(InfoExtractor): |
b8d8cced S |
31 | _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}' |
32 | _GEO_COUNTRIES = ['IT'] | |
33 | _GEO_BYPASS = False | |
34 | ||
35 | def _extract_relinker_info(self, relinker_url, video_id): | |
0c7b4f49 RA |
36 | if not re.match(r'https?://', relinker_url): |
37 | return {'formats': [{'url': relinker_url}]} | |
38 | ||
034a8849 | 39 | formats = [] |
b8d8cced S |
40 | geoprotection = None |
41 | is_live = None | |
42 | duration = None | |
034a8849 YCH |
43 | |
44 | for platform in ('mon', 'flash', 'native'): | |
034a8849 YCH |
45 | relinker = self._download_xml( |
46 | relinker_url, video_id, | |
47 | note='Downloading XML metadata for platform %s' % platform, | |
48 | transform_source=fix_xml_ampersands, | |
38cce791 YCH |
49 | query={'output': 45, 'pl': platform}, |
50 | headers=self.geo_verification_headers()) | |
034a8849 | 51 | |
b8d8cced S |
52 | if not geoprotection: |
53 | geoprotection = xpath_text( | |
54 | relinker, './geoprotection', default=None) == 'Y' | |
55 | ||
56 | if not is_live: | |
57 | is_live = xpath_text( | |
58 | relinker, './is_live', default=None) == 'Y' | |
59 | if not duration: | |
60 | duration = parse_duration(xpath_text( | |
61 | relinker, './duration', default=None)) | |
62 | ||
63 | url_elem = find_xpath_attr(relinker, './url', 'type', 'content') | |
64 | if url_elem is None: | |
65 | continue | |
66 | ||
67 | media_url = url_elem.text | |
68 | ||
69 | # This does not imply geo restriction (e.g. | |
70 | # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html) | |
a0566bbf | 71 | if '/video_no_available.mp4' in media_url: |
b8d8cced | 72 | continue |
034a8849 YCH |
73 | |
74 | ext = determine_ext(media_url) | |
75 | if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'): | |
76 | continue | |
77 | ||
c17eb5b4 | 78 | if ext == 'm3u8' or 'format=m3u8' in media_url or platform == 'mon': |
034a8849 YCH |
79 | formats.extend(self._extract_m3u8_formats( |
80 | media_url, video_id, 'mp4', 'm3u8_native', | |
81 | m3u8_id='hls', fatal=False)) | |
c17eb5b4 | 82 | elif ext == 'f4m' or platform == 'flash': |
034a8849 YCH |
83 | manifest_url = update_url_query( |
84 | media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'), | |
85 | {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'}) | |
86 | formats.extend(self._extract_f4m_formats( | |
87 | manifest_url, video_id, f4m_id='hds', fatal=False)) | |
88 | else: | |
89 | bitrate = int_or_none(xpath_text(relinker, 'bitrate')) | |
90 | formats.append({ | |
91 | 'url': media_url, | |
92 | 'tbr': bitrate if bitrate > 0 else None, | |
93 | 'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http', | |
94 | }) | |
95 | ||
b8d8cced S |
96 | if not formats and geoprotection is True: |
97 | self.raise_geo_restricted(countries=self._GEO_COUNTRIES) | |
98 | ||
99 | return dict((k, v) for k, v in { | |
100 | 'is_live': is_live, | |
101 | 'duration': duration, | |
102 | 'formats': formats, | |
103 | }.items() if v is not None) | |
034a8849 | 104 | |
1b3feca0 S |
105 | @staticmethod |
106 | def _extract_subtitles(url, subtitle_url): | |
107 | subtitles = {} | |
108 | if subtitle_url and isinstance(subtitle_url, compat_str): | |
109 | subtitle_url = urljoin(url, subtitle_url) | |
110 | STL_EXT = '.stl' | |
111 | SRT_EXT = '.srt' | |
112 | subtitles['it'] = [{ | |
113 | 'ext': 'stl', | |
114 | 'url': subtitle_url, | |
115 | }] | |
116 | if subtitle_url.endswith(STL_EXT): | |
117 | srt_url = subtitle_url[:-len(STL_EXT)] + SRT_EXT | |
118 | subtitles['it'].append({ | |
119 | 'ext': 'srt', | |
120 | 'url': srt_url, | |
121 | }) | |
122 | return subtitles | |
123 | ||
2b28b892 | 124 | |
51342717 | 125 | class RaiPlayIE(RaiBaseIE): |
a0566bbf | 126 | _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s))\.(?:html|json)' % RaiBaseIE._UUID_RE |
51342717 | 127 | _TESTS = [{ |
51342717 T |
128 | 'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html', |
129 | 'md5': '8970abf8caf8aef4696e7b1f2adfc696', | |
130 | 'info_dict': { | |
131 | 'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391', | |
132 | 'ext': 'mp4', | |
b8d8cced | 133 | 'title': 'Report del 07/04/2014', |
8bdd16b4 | 134 | 'alt_title': 'St 2013/14 - Espresso nel caffè - 07/04/2014', |
abea9145 | 135 | 'description': 'md5:d730c168a58f4bb35600fc2f881ec04e', |
51342717 | 136 | 'thumbnail': r're:^https?://.*\.jpg$', |
abea9145 | 137 | 'uploader': 'Rai Gulp', |
b8d8cced | 138 | 'duration': 6160, |
8bdd16b4 | 139 | 'series': 'Report', |
140 | 'season': '2013/14', | |
b8d8cced S |
141 | }, |
142 | 'params': { | |
143 | 'skip_download': True, | |
144 | }, | |
145 | }, { | |
146 | 'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?', | |
147 | 'only_matching': True, | |
51342717 | 148 | }] |
2b28b892 | 149 | |
51342717 | 150 | def _real_extract(self, url): |
a0566bbf | 151 | base, video_id = re.match(self._VALID_URL, url).groups() |
2b28b892 | 152 | |
b8d8cced | 153 | media = self._download_json( |
a0566bbf | 154 | base + '.json', video_id, 'Downloading video JSON') |
2b28b892 | 155 | |
b8d8cced | 156 | title = media['name'] |
b8d8cced S |
157 | video = media['video'] |
158 | ||
abea9145 | 159 | relinker_info = self._extract_relinker_info(video['content_url'], video_id) |
b8d8cced | 160 | self._sort_formats(relinker_info['formats']) |
2b28b892 | 161 | |
51342717 | 162 | thumbnails = [] |
8bdd16b4 | 163 | for _, value in media.get('images', {}).items(): |
164 | if value: | |
165 | thumbnails.append({ | |
166 | 'url': urljoin(url, value), | |
167 | }) | |
034a8849 | 168 | |
8bdd16b4 | 169 | date_published = media.get('date_published') |
170 | time_published = media.get('time_published') | |
171 | if date_published and time_published: | |
172 | date_published += ' ' + time_published | |
b0adbe98 | 173 | |
1b3feca0 S |
174 | subtitles = self._extract_subtitles(url, video.get('subtitles')) |
175 | ||
8bdd16b4 | 176 | program_info = media.get('program_info') or {} |
177 | season = media.get('season') | |
178 | ||
b8d8cced | 179 | info = { |
a0566bbf | 180 | 'id': remove_start(media.get('id'), 'ContentItem-') or video_id, |
181 | 'display_id': video_id, | |
9c48b5a1 S |
182 | 'title': self._live_title(title) if relinker_info.get( |
183 | 'is_live') else title, | |
8bdd16b4 | 184 | 'alt_title': strip_or_none(media.get('subtitle')), |
b8d8cced | 185 | 'description': media.get('description'), |
9c48b5a1 | 186 | 'uploader': strip_or_none(media.get('channel')), |
8bdd16b4 | 187 | 'creator': strip_or_none(media.get('editor') or None), |
b8d8cced | 188 | 'duration': parse_duration(video.get('duration')), |
8bdd16b4 | 189 | 'timestamp': unified_timestamp(date_published), |
51342717 | 190 | 'thumbnails': thumbnails, |
8bdd16b4 | 191 | 'series': program_info.get('name'), |
192 | 'season_number': int_or_none(season), | |
193 | 'season': season if (season and not season.isdigit()) else None, | |
194 | 'episode': media.get('episode_title'), | |
195 | 'episode_number': int_or_none(media.get('episode')), | |
1b3feca0 | 196 | 'subtitles': subtitles, |
51342717 | 197 | } |
b0adbe98 | 198 | |
b8d8cced | 199 | info.update(relinker_info) |
b8d8cced S |
200 | return info |
201 | ||
06d5556d | 202 | |
a0566bbf | 203 | class RaiPlayLiveIE(RaiPlayIE): |
204 | _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/dirette/(?P<id>[^/?#&]+))' | |
205 | _TESTS = [{ | |
9c48b5a1 S |
206 | 'url': 'http://www.raiplay.it/dirette/rainews24', |
207 | 'info_dict': { | |
208 | 'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c', | |
209 | 'display_id': 'rainews24', | |
210 | 'ext': 'mp4', | |
211 | 'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', | |
a0566bbf | 212 | 'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497', |
9c48b5a1 S |
213 | 'uploader': 'Rai News 24', |
214 | 'creator': 'Rai News 24', | |
215 | 'is_live': True, | |
216 | }, | |
217 | 'params': { | |
218 | 'skip_download': True, | |
219 | }, | |
a0566bbf | 220 | }] |
449c6657 | 221 | |
222 | ||
1115271a | 223 | class RaiPlayPlaylistIE(InfoExtractor): |
a0566bbf | 224 | _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+))' |
1115271a S |
225 | _TESTS = [{ |
226 | 'url': 'http://www.raiplay.it/programmi/nondirloalmiocapo/', | |
227 | 'info_dict': { | |
228 | 'id': 'nondirloalmiocapo', | |
229 | 'title': 'Non dirlo al mio capo', | |
a0566bbf | 230 | 'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b', |
1115271a S |
231 | }, |
232 | 'playlist_mincount': 12, | |
233 | }] | |
234 | ||
235 | def _real_extract(self, url): | |
a0566bbf | 236 | base, playlist_id = re.match(self._VALID_URL, url).groups() |
1115271a | 237 | |
a0566bbf | 238 | program = self._download_json( |
239 | base + '.json', playlist_id, 'Downloading program JSON') | |
1115271a S |
240 | |
241 | entries = [] | |
a0566bbf | 242 | for b in (program.get('blocks') or []): |
243 | for s in (b.get('sets') or []): | |
244 | s_id = s.get('id') | |
245 | if not s_id: | |
246 | continue | |
247 | medias = self._download_json( | |
248 | '%s/%s.json' % (base, s_id), s_id, | |
249 | 'Downloading content set JSON', fatal=False) | |
250 | if not medias: | |
251 | continue | |
252 | for m in (medias.get('items') or []): | |
253 | path_id = m.get('path_id') | |
254 | if not path_id: | |
255 | continue | |
256 | video_url = urljoin(url, path_id) | |
257 | entries.append(self.url_result( | |
258 | video_url, ie=RaiPlayIE.ie_key(), | |
259 | video_id=RaiPlayIE._match_id(video_url))) | |
260 | ||
261 | return self.playlist_result( | |
262 | entries, playlist_id, program.get('name'), | |
263 | try_get(program, lambda x: x['program_info']['description'])) | |
1115271a S |
264 | |
265 | ||
034a8849 | 266 | class RaiIE(RaiBaseIE): |
2b2da3ba | 267 | _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE |
51342717 | 268 | _TESTS = [{ |
b8d8cced S |
269 | # var uniquename = "ContentItem-..." |
270 | # data-id="ContentItem-..." | |
51342717 T |
271 | 'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html', |
272 | 'info_dict': { | |
273 | 'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9', | |
274 | 'ext': 'mp4', | |
275 | 'title': 'TG PRIMO TEMPO', | |
b8d8cced | 276 | 'thumbnail': r're:^https?://.*\.jpg$', |
51342717 | 277 | 'duration': 1758, |
b8d8cced | 278 | 'upload_date': '20140612', |
8bdd16b4 | 279 | }, |
280 | 'skip': 'This content is available only in Italy', | |
51342717 | 281 | }, { |
b8d8cced | 282 | # with ContentItem in many metas |
51342717 T |
283 | 'url': 'http://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html', |
284 | 'info_dict': { | |
285 | 'id': '1632c009-c843-4836-bb65-80c33084a64b', | |
286 | 'ext': 'mp4', | |
b8d8cced S |
287 | 'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor "La ragazza del treno"', |
288 | 'description': 'I film in uscita questa settimana.', | |
51342717 | 289 | 'thumbnail': r're:^https?://.*\.png$', |
b8d8cced S |
290 | 'duration': 833, |
291 | 'upload_date': '20161103', | |
51342717 T |
292 | } |
293 | }, { | |
b8d8cced | 294 | # with ContentItem in og:url |
51342717 | 295 | 'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html', |
abea9145 | 296 | 'md5': '6865dd00cf0bbf5772fdd89d59bd768a', |
51342717 T |
297 | 'info_dict': { |
298 | 'id': 'efb17665-691c-45d5-a60c-5301333cbb0c', | |
299 | 'ext': 'mp4', | |
300 | 'title': 'TG1 ore 20:00 del 03/11/2016', | |
b8d8cced | 301 | 'description': 'TG1 edizione integrale ore 20:00 del giorno 03/11/2016', |
51342717 | 302 | 'thumbnail': r're:^https?://.*\.jpg$', |
b8d8cced | 303 | 'duration': 2214, |
51342717 | 304 | 'upload_date': '20161103', |
51342717 | 305 | } |
51342717 | 306 | }, { |
b8d8cced S |
307 | # initEdizione('ContentItem-...' |
308 | 'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined', | |
309 | 'info_dict': { | |
310 | 'id': 'c2187016-8484-4e3a-8ac8-35e475b07303', | |
311 | 'ext': 'mp4', | |
312 | 'title': r're:TG1 ore \d{2}:\d{2} del \d{2}/\d{2}/\d{4}', | |
313 | 'duration': 2274, | |
314 | 'upload_date': '20170401', | |
315 | }, | |
316 | 'skip': 'Changes daily', | |
51342717 | 317 | }, { |
b8d8cced | 318 | # HLS live stream with ContentItem in og:url |
51342717 | 319 | 'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html', |
51342717 T |
320 | 'info_dict': { |
321 | 'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9', | |
322 | 'ext': 'mp4', | |
323 | 'title': 'La diretta di Rainews24', | |
15e4b6b7 | 324 | }, |
b8d8cced S |
325 | 'params': { |
326 | 'skip_download': True, | |
327 | }, | |
0c7b4f49 RA |
328 | }, { |
329 | # Direct MMS URL | |
330 | 'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html', | |
331 | 'only_matching': True, | |
2b2da3ba S |
332 | }, { |
333 | 'url': 'https://www.rainews.it/tgr/marche/notiziari/video/2019/02/ContentItem-6ba945a2-889c-4a80-bdeb-8489c70a8db9.html', | |
334 | 'only_matching': True, | |
51342717 | 335 | }] |
06d5556d | 336 | |
51342717 T |
337 | def _extract_from_content_id(self, content_id, url): |
338 | media = self._download_json( | |
339 | 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id, | |
340 | content_id, 'Downloading video JSON') | |
341 | ||
b8d8cced S |
342 | title = media['name'].strip() |
343 | ||
344 | media_type = media['type'] | |
345 | if 'Audio' in media_type: | |
346 | relinker_info = { | |
085d9dd9 | 347 | 'formats': [{ |
b8d8cced S |
348 | 'format_id': media.get('formatoAudio'), |
349 | 'url': media['audioUrl'], | |
350 | 'ext': media.get('formatoAudio'), | |
085d9dd9 | 351 | }] |
b8d8cced S |
352 | } |
353 | elif 'Video' in media_type: | |
354 | relinker_info = self._extract_relinker_info(media['mediaUri'], content_id) | |
355 | else: | |
356 | raise ExtractorError('not a media file') | |
357 | ||
358 | self._sort_formats(relinker_info['formats']) | |
359 | ||
51342717 T |
360 | thumbnails = [] |
361 | for image_type in ('image', 'image_medium', 'image_300'): | |
362 | thumbnail_url = media.get(image_type) | |
363 | if thumbnail_url: | |
364 | thumbnails.append({ | |
365 | 'url': compat_urlparse.urljoin(url, thumbnail_url), | |
366 | }) | |
367 | ||
1b3feca0 | 368 | subtitles = self._extract_subtitles(url, media.get('subtitlesUrl')) |
51342717 | 369 | |
b8d8cced | 370 | info = { |
51342717 | 371 | 'id': content_id, |
b8d8cced S |
372 | 'title': title, |
373 | 'description': strip_or_none(media.get('desc')), | |
51342717 T |
374 | 'thumbnails': thumbnails, |
375 | 'uploader': media.get('author'), | |
376 | 'upload_date': unified_strdate(media.get('date')), | |
377 | 'duration': parse_duration(media.get('length')), | |
51342717 T |
378 | 'subtitles': subtitles, |
379 | } | |
b8d8cced S |
380 | |
381 | info.update(relinker_info) | |
382 | ||
383 | return info | |
384 | ||
385 | def _real_extract(self, url): | |
386 | video_id = self._match_id(url) | |
387 | ||
388 | webpage = self._download_webpage(url, video_id) | |
389 | ||
390 | content_item_id = None | |
391 | ||
392 | content_item_url = self._html_search_meta( | |
393 | ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url', | |
394 | 'twitter:player', 'jsonlink'), webpage, default=None) | |
395 | if content_item_url: | |
396 | content_item_id = self._search_regex( | |
397 | r'ContentItem-(%s)' % self._UUID_RE, content_item_url, | |
398 | 'content item id', default=None) | |
399 | ||
400 | if not content_item_id: | |
401 | content_item_id = self._search_regex( | |
402 | r'''(?x) | |
403 | (?: | |
404 | (?:initEdizione|drawMediaRaiTV)\(| | |
405 | <(?:[^>]+\bdata-id|var\s+uniquename)= | |
406 | ) | |
407 | (["\']) | |
408 | (?:(?!\1).)*\bContentItem-(?P<id>%s) | |
409 | ''' % self._UUID_RE, | |
410 | webpage, 'content item id', default=None, group='id') | |
411 | ||
412 | content_item_ids = set() | |
361f293a S |
413 | if content_item_id: |
414 | content_item_ids.add(content_item_id) | |
b8d8cced S |
415 | if video_id not in content_item_ids: |
416 | content_item_ids.add(video_id) | |
417 | ||
418 | for content_item_id in content_item_ids: | |
419 | try: | |
420 | return self._extract_from_content_id(content_item_id, url) | |
421 | except GeoRestrictedError: | |
422 | raise | |
423 | except ExtractorError: | |
424 | pass | |
425 | ||
a0566bbf | 426 | relinker_url = self._proto_relative_url(self._search_regex( |
b8d8cced S |
427 | r'''(?x) |
428 | (?: | |
429 | var\s+videoURL| | |
430 | mediaInfo\.mediaUri | |
431 | )\s*=\s* | |
432 | ([\'"]) | |
433 | (?P<url> | |
434 | (?:https?:)? | |
435 | //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\? | |
436 | (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1 | |
437 | ''', | |
a0566bbf | 438 | webpage, 'relinker URL', group='url')) |
b8d8cced S |
439 | |
440 | relinker_info = self._extract_relinker_info( | |
441 | urljoin(url, relinker_url), video_id) | |
442 | self._sort_formats(relinker_info['formats']) | |
443 | ||
444 | title = self._search_regex( | |
445 | r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1', | |
446 | webpage, 'title', group='title', | |
447 | default=None) or self._og_search_title(webpage) | |
448 | ||
449 | info = { | |
450 | 'id': video_id, | |
451 | 'title': title, | |
452 | } | |
453 | ||
454 | info.update(relinker_info) | |
455 | ||
456 | return info |