]>
Commit | Line | Data |
---|---|---|
a85e131b | 1 | # coding: utf-8 |
2 | from __future__ import unicode_literals | |
3 | ||
4 | import re | |
5 | ||
6 | from .common import InfoExtractor | |
7 | from ..utils import ( | |
8 | clean_html, | |
9 | ExtractorError, | |
10 | js_to_json, | |
11 | base_url, | |
12 | url_basename, | |
13 | urljoin, | |
14 | ) | |
15 | ||
16 | ||
5867a167 | 17 | class RCSBaseIE(InfoExtractor): |
18c1f043 | 18 | # based on VideoPlayerLoader.prototype.getVideoSrc |
19 | # and VideoPlayerLoader.prototype.transformSrc from | |
20 | # https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs | |
a85e131b | 21 | _ALL_REPLACE = { |
22 | 'media2vam.corriere.it.edgesuite.net': | |
23 | 'media2vam-corriere-it.akamaized.net', | |
24 | 'media.youreporter.it.edgesuite.net': | |
25 | 'media-youreporter-it.akamaized.net', | |
26 | 'corrierepmd.corriere.it.edgesuite.net': | |
27 | 'corrierepmd-corriere-it.akamaized.net', | |
28 | 'media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/': | |
29 | 'video.corriere.it/vr360/videos/', | |
30 | '.net//': '.net/', | |
31 | } | |
32 | _MP4_REPLACE = { | |
33 | 'media2vam.corbologna.corriere.it.edgesuite.net': | |
34 | 'media2vam-bologna-corriere-it.akamaized.net', | |
35 | 'media2vam.corfiorentino.corriere.it.edgesuite.net': | |
36 | 'media2vam-fiorentino-corriere-it.akamaized.net', | |
37 | 'media2vam.cormezzogiorno.corriere.it.edgesuite.net': | |
38 | 'media2vam-mezzogiorno-corriere-it.akamaized.net', | |
39 | 'media2vam.corveneto.corriere.it.edgesuite.net': | |
40 | 'media2vam-veneto-corriere-it.akamaized.net', | |
41 | 'media2.oggi.it.edgesuite.net': | |
42 | 'media2-oggi-it.akamaized.net', | |
43 | 'media2.quimamme.it.edgesuite.net': | |
44 | 'media2-quimamme-it.akamaized.net', | |
45 | 'media2.amica.it.edgesuite.net': | |
46 | 'media2-amica-it.akamaized.net', | |
47 | 'media2.living.corriere.it.edgesuite.net': | |
48 | 'media2-living-corriere-it.akamaized.net', | |
49 | 'media2.style.corriere.it.edgesuite.net': | |
50 | 'media2-style-corriere-it.akamaized.net', | |
51 | 'media2.iodonna.it.edgesuite.net': | |
52 | 'media2-iodonna-it.akamaized.net', | |
53 | 'media2.leitv.it.edgesuite.net': | |
54 | 'media2-leitv-it.akamaized.net', | |
55 | } | |
56 | _MIGRATION_MAP = { | |
57 | 'videoamica-vh.akamaihd': 'amica', | |
58 | 'media2-amica-it.akamaized': 'amica', | |
59 | 'corrierevam-vh.akamaihd': 'corriere', | |
60 | 'media2vam-corriere-it.akamaized': 'corriere', | |
61 | 'cormezzogiorno-vh.akamaihd': 'corrieredelmezzogiorno', | |
62 | 'media2vam-mezzogiorno-corriere-it.akamaized': 'corrieredelmezzogiorno', | |
63 | 'corveneto-vh.akamaihd': 'corrieredelveneto', | |
64 | 'media2vam-veneto-corriere-it.akamaized': 'corrieredelveneto', | |
65 | 'corbologna-vh.akamaihd': 'corrieredibologna', | |
66 | 'media2vam-bologna-corriere-it.akamaized': 'corrieredibologna', | |
67 | 'corfiorentino-vh.akamaihd': 'corrierefiorentino', | |
68 | 'media2vam-fiorentino-corriere-it.akamaized': 'corrierefiorentino', | |
69 | 'corinnovazione-vh.akamaihd': 'corriereinnovazione', | |
70 | 'media2-gazzanet-gazzetta-it.akamaized': 'gazzanet', | |
71 | 'videogazzanet-vh.akamaihd': 'gazzanet', | |
72 | 'videogazzaworld-vh.akamaihd': 'gazzaworld', | |
73 | 'gazzettavam-vh.akamaihd': 'gazzetta', | |
74 | 'media2vam-gazzetta-it.akamaized': 'gazzetta', | |
75 | 'videoiodonna-vh.akamaihd': 'iodonna', | |
76 | 'media2-leitv-it.akamaized': 'leitv', | |
77 | 'videoleitv-vh.akamaihd': 'leitv', | |
78 | 'videoliving-vh.akamaihd': 'living', | |
79 | 'media2-living-corriere-it.akamaized': 'living', | |
80 | 'media2-oggi-it.akamaized': 'oggi', | |
81 | 'videooggi-vh.akamaihd': 'oggi', | |
82 | 'media2-quimamme-it.akamaized': 'quimamme', | |
83 | 'quimamme-vh.akamaihd': 'quimamme', | |
84 | 'videorunning-vh.akamaihd': 'running', | |
85 | 'media2-style-corriere-it.akamaized': 'style', | |
86 | 'style-vh.akamaihd': 'style', | |
87 | 'videostyle-vh.akamaihd': 'style', | |
88 | 'media2-stylepiccoli-it.akamaized': 'stylepiccoli', | |
89 | 'stylepiccoli-vh.akamaihd': 'stylepiccoli', | |
90 | 'doveviaggi-vh.akamaihd': 'viaggi', | |
91 | 'media2-doveviaggi-it.akamaized': 'viaggi', | |
92 | 'media2-vivimilano-corriere-it.akamaized': 'vivimilano', | |
93 | 'vivimilano-vh.akamaihd': 'vivimilano', | |
94 | 'media2-youreporter-it.akamaized': 'youreporter' | |
95 | } | |
96 | _MIGRATION_MEDIA = { | |
97 | 'advrcs-vh.akamaihd': '', | |
98 | 'corriere-f.akamaihd': '', | |
99 | 'corrierepmd-corriere-it.akamaized': '', | |
100 | 'corrprotetto-vh.akamaihd': '', | |
101 | 'gazzetta-f.akamaihd': '', | |
102 | 'gazzettapmd-gazzetta-it.akamaized': '', | |
103 | 'gazzprotetto-vh.akamaihd': '', | |
104 | 'periodici-f.akamaihd': '', | |
105 | 'periodicisecure-vh.akamaihd': '', | |
106 | 'videocoracademy-vh.akamaihd': '' | |
107 | } | |
108 | ||
109 | def _get_video_src(self, video): | |
508649e6 | 110 | mediaFiles = video.get('mediaProfile').get('mediaFile') |
a85e131b | 111 | src = {} |
112 | # audio | |
508649e6 | 113 | if video.get('mediaType') == 'AUDIO': |
a85e131b | 114 | for aud in mediaFiles: |
115 | # todo: check | |
508649e6 | 116 | src['mp3'] = aud.get('value') |
a85e131b | 117 | # video |
118 | else: | |
119 | for vid in mediaFiles: | |
508649e6 | 120 | if vid.get('mimeType') == 'application/vnd.apple.mpegurl': |
121 | src['m3u8'] = vid.get('value') | |
122 | if vid.get('mimeType') == 'video/mp4': | |
123 | src['mp4'] = vid.get('value') | |
a85e131b | 124 | |
125 | # replace host | |
126 | for t in src: | |
127 | for s, r in self._ALL_REPLACE.items(): | |
128 | src[t] = src[t].replace(s, r) | |
129 | for s, r in self._MP4_REPLACE.items(): | |
130 | src[t] = src[t].replace(s, r) | |
131 | ||
132 | # switch cdn | |
133 | if 'mp4' in src and 'm3u8' in src: | |
508649e6 | 134 | if ('-lh.akamaihd' not in src.get('m3u8') |
135 | and 'akamai' in src.get('mp4')): | |
a85e131b | 136 | if 'm3u8' in src: |
508649e6 | 137 | matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('m3u8')) |
a85e131b | 138 | src['m3u8'] = 'https://vod.rcsobjects.it/hls/%s%s' % ( |
139 | self._MIGRATION_MAP[matches.group('host')], | |
140 | matches.group('path').replace( | |
141 | '///', '/').replace( | |
142 | '//', '/').replace( | |
143 | '.csmil', '.urlset' | |
144 | ) | |
145 | ) | |
146 | if 'mp4' in src: | |
508649e6 | 147 | matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('mp4')) |
a85e131b | 148 | if matches: |
149 | if matches.group('host') in self._MIGRATION_MEDIA: | |
150 | vh_stream = 'https://media2.corriereobjects.it' | |
508649e6 | 151 | if src.get('mp4').find('fcs.quotidiani_!'): |
a85e131b | 152 | vh_stream = 'https://media2-it.corriereobjects.it' |
153 | src['mp4'] = '%s%s' % ( | |
154 | vh_stream, | |
155 | matches.group('path').replace( | |
156 | '///', '/').replace( | |
157 | '//', '/').replace( | |
158 | '/fcs.quotidiani/mediacenter', '').replace( | |
159 | '/fcs.quotidiani_!/mediacenter', '').replace( | |
160 | 'corriere/content/mediacenter/', '').replace( | |
161 | 'gazzetta/content/mediacenter/', '') | |
162 | ) | |
163 | else: | |
164 | src['mp4'] = 'https://vod.rcsobjects.it/%s%s' % ( | |
165 | self._MIGRATION_MAP[matches.group('host')], | |
166 | matches.group('path').replace('///', '/').replace('//', '/') | |
167 | ) | |
168 | ||
169 | if 'mp3' in src: | |
508649e6 | 170 | src['mp3'] = src.get('mp3').replace( |
a85e131b | 171 | 'media2vam-corriere-it.akamaized.net', |
172 | 'vod.rcsobjects.it/corriere') | |
173 | if 'mp4' in src: | |
508649e6 | 174 | if src.get('mp4').find('fcs.quotidiani_!'): |
175 | src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects') | |
a85e131b | 176 | if 'm3u8' in src: |
508649e6 | 177 | if src.get('m3u8').find('fcs.quotidiani_!'): |
178 | src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects') | |
a85e131b | 179 | |
508649e6 | 180 | if 'geoblocking' in video.get('mediaProfile'): |
a85e131b | 181 | if 'm3u8' in src: |
508649e6 | 182 | src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects') |
a85e131b | 183 | if 'mp4' in src: |
508649e6 | 184 | src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects') |
a85e131b | 185 | if 'm3u8' in src: |
508649e6 | 186 | if src.get('m3u8').find('csmil') and src.get('m3u8').find('vod'): |
187 | src['m3u8'] = src.get('m3u8').replace('.csmil', '.urlset') | |
a85e131b | 188 | |
189 | return src | |
190 | ||
191 | def _create_formats(self, urls, video_id): | |
192 | formats = [] | |
193 | formats = self._extract_m3u8_formats( | |
508649e6 | 194 | urls.get('m3u8'), video_id, 'mp4', entry_protocol='m3u8_native', |
a85e131b | 195 | m3u8_id='hls', fatal=False) |
196 | ||
18c1f043 | 197 | if urls.get('mp4'): |
a85e131b | 198 | formats.append({ |
199 | 'format_id': 'http-mp4', | |
18c1f043 | 200 | 'url': urls['mp4'] |
a85e131b | 201 | }) |
202 | self._sort_formats(formats) | |
203 | return formats | |
204 | ||
205 | def _real_extract(self, url): | |
206 | video_id = self._match_id(url) | |
508649e6 | 207 | mobj = re.search(self._VALID_URL, url) |
a85e131b | 208 | |
508649e6 | 209 | if 'cdn' not in mobj.groupdict(): |
a85e131b | 210 | raise ExtractorError('CDN not found in url: %s' % url) |
211 | ||
212 | # for leitv/youreporter/viaggi don't use the embed page | |
508649e6 | 213 | if ((mobj.group('cdn') not in ['leitv.it', 'youreporter.it']) |
214 | and (mobj.group('vid') == 'video')): | |
215 | url = 'https://video.%s/video-embed/%s' % (mobj.group('cdn'), video_id) | |
a85e131b | 216 | |
217 | page = self._download_webpage(url, video_id) | |
218 | ||
219 | video_data = None | |
220 | # look for json video data url | |
221 | json = self._search_regex( | |
18c1f043 | 222 | r'''(?x)url\s*=\s*(["']) |
223 | (?P<url> | |
224 | (?:https?:)?//video\.rcs\.it | |
225 | /fragment-includes/video-includes/.+?\.json | |
226 | )\1;''', | |
227 | page, video_id, group='url', default=None) | |
a85e131b | 228 | if json: |
229 | if json.startswith('//'): | |
230 | json = 'https:%s' % json | |
231 | video_data = self._download_json(json, video_id) | |
232 | ||
508649e6 | 233 | # if json url not found, look for json video data directly in the page |
a85e131b | 234 | else: |
18c1f043 | 235 | # RCS normal pages and most of the embeds |
a85e131b | 236 | json = self._search_regex( |
237 | r'[\s;]video\s*=\s*({[\s\S]+?})(?:;|,playlist=)', | |
238 | page, video_id, default=None) | |
18c1f043 | 239 | if not json and 'video-embed' in url: |
240 | page = self._download_webpage(url.replace('video-embed', 'video-json'), video_id) | |
241 | json = self._search_regex( | |
242 | r'##start-video##({[\s\S]+?})##end-video##', | |
243 | page, video_id, default=None) | |
244 | if not json: | |
a85e131b | 245 | # if no video data found try search for iframes |
246 | emb = RCSEmbedsIE._extract_url(page) | |
247 | if emb: | |
5867a167 | 248 | return { |
249 | '_type': 'url_transparent', | |
250 | 'url': emb, | |
251 | 'ie_key': RCSEmbedsIE.ie_key() | |
252 | } | |
18c1f043 | 253 | if json: |
254 | video_data = self._parse_json( | |
255 | json, video_id, transform_source=js_to_json) | |
a85e131b | 256 | |
257 | if not video_data: | |
258 | raise ExtractorError('Video data not found in the page') | |
259 | ||
260 | formats = self._create_formats( | |
261 | self._get_video_src(video_data), video_id) | |
262 | ||
508649e6 | 263 | description = (video_data.get('description') |
18c1f043 | 264 | or clean_html(video_data.get('htmlDescription')) |
265 | or self._html_search_meta('description', page)) | |
5867a167 | 266 | uploader = video_data.get('provider') or mobj.group('cdn') |
508649e6 | 267 | |
a85e131b | 268 | return { |
269 | 'id': video_id, | |
508649e6 | 270 | 'title': video_data.get('title'), |
271 | 'description': description, | |
272 | 'uploader': uploader, | |
a85e131b | 273 | 'formats': formats |
274 | } | |
275 | ||
276 | ||
5867a167 | 277 | class RCSEmbedsIE(RCSBaseIE): |
a85e131b | 278 | _VALID_URL = r'''(?x) |
279 | https?://(?P<vid>video)\. | |
280 | (?P<cdn> | |
281 | (?: | |
282 | rcs| | |
283 | (?:corriere\w+\.)?corriere| | |
284 | (?:gazzanet\.)?gazzetta | |
285 | )\.it) | |
286 | /video-embed/(?P<id>[^/=&\?]+?)(?:$|\?)''' | |
287 | _TESTS = [{ | |
288 | 'url': 'https://video.rcs.it/video-embed/iodonna-0001585037', | |
289 | 'md5': '623ecc8ffe7299b2d0c1046d8331a9df', | |
290 | 'info_dict': { | |
291 | 'id': 'iodonna-0001585037', | |
292 | 'ext': 'mp4', | |
293 | 'title': 'Sky Arte racconta Madonna nella serie "Artist to icon"', | |
294 | 'description': 'md5:65b09633df9ffee57f48b39e34c9e067', | |
295 | 'uploader': 'rcs.it', | |
296 | } | |
5867a167 | 297 | }, { |
18c1f043 | 298 | # redownload the page changing 'video-embed' in 'video-json' |
5867a167 | 299 | 'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789', |
300 | 'md5': 'a043e3fecbe4d9ed7fc5d888652a5440', | |
301 | 'info_dict': { | |
302 | 'id': 'gazzanet-mo05-0000260789', | |
303 | 'ext': 'mp4', | |
304 | 'title': 'Valentino Rossi e papà Graziano si divertono col drifting', | |
305 | 'description': 'md5:a8bf90d6adafd9815f70fc74c0fc370a', | |
306 | 'uploader': 'rcd', | |
307 | } | |
a85e131b | 308 | }, { |
309 | 'url': 'https://video.corriere.it/video-embed/b727632a-f9d0-11ea-91b0-38d50a849abb?player', | |
310 | 'match_only': True | |
311 | }, { | |
312 | 'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140', | |
313 | 'match_only': True | |
314 | }] | |
315 | ||
316 | @staticmethod | |
317 | def _sanitize_urls(urls): | |
318 | # add protocol if missing | |
319 | for i, e in enumerate(urls): | |
320 | if e.startswith('//'): | |
321 | urls[i] = 'https:%s' % e | |
322 | # clean iframes urls | |
323 | for i, e in enumerate(urls): | |
324 | urls[i] = urljoin(base_url(e), url_basename(e)) | |
325 | return urls | |
326 | ||
327 | @staticmethod | |
328 | def _extract_urls(webpage): | |
329 | entries = [ | |
330 | mobj.group('url') | |
331 | for mobj in re.finditer(r'''(?x) | |
332 | (?: | |
333 | data-frame-src=| | |
334 | <iframe[^\n]+src= | |
335 | ) | |
336 | (["']) | |
337 | (?P<url>(?:https?:)?//video\. | |
338 | (?: | |
339 | rcs| | |
340 | (?:corriere\w+\.)?corriere| | |
341 | (?:gazzanet\.)?gazzetta | |
342 | ) | |
343 | \.it/video-embed/.+?) | |
344 | \1''', webpage)] | |
345 | return RCSEmbedsIE._sanitize_urls(entries) | |
346 | ||
347 | @staticmethod | |
348 | def _extract_url(webpage): | |
349 | urls = RCSEmbedsIE._extract_urls(webpage) | |
350 | return urls[0] if urls else None | |
351 | ||
352 | ||
5867a167 | 353 | class RCSIE(RCSBaseIE): |
a85e131b | 354 | _VALID_URL = r'''(?x)https?://(?P<vid>video|viaggi)\. |
355 | (?P<cdn> | |
356 | (?: | |
5867a167 | 357 | corrieredelmezzogiorno\. |
358 | |corrieredelveneto\. | |
359 | |corrieredibologna\. | |
360 | |corrierefiorentino\. | |
361 | )?corriere\.it | |
362 | |(?:gazzanet\.)?gazzetta\.it) | |
363 | /(?!video-embed/).+?/(?P<id>[^/\?]+)(?=\?|/$|$)''' | |
a85e131b | 364 | _TESTS = [{ |
365 | 'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb', | |
366 | 'md5': '0f4ededc202b0f00b6e509d831e2dcda', | |
367 | 'info_dict': { | |
368 | 'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb', | |
369 | 'ext': 'mp4', | |
370 | 'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante', | |
371 | 'description': 'md5:93b51c9161ac8a64fb2f997b054d0152', | |
372 | 'uploader': 'Corriere Tv', | |
373 | } | |
374 | }, { | |
18c1f043 | 375 | # video data inside iframe |
a85e131b | 376 | 'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/', |
377 | 'md5': 'da378e4918d2afbf7d61c35abb948d4c', | |
378 | 'info_dict': { | |
379 | 'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2', | |
380 | 'ext': 'mp4', | |
381 | 'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen', | |
382 | 'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8', | |
383 | 'uploader': 'DOVE Viaggi', | |
384 | } | |
385 | }, { | |
a85e131b | 386 | 'url': 'https://video.gazzetta.it/video-motogp-catalogna-cadute-dovizioso-vale-rossi/49612410-00ca-11eb-bcd8-30d4253e0140?vclk=Videobar', |
387 | 'md5': 'eedc1b5defd18e67383afef51ff7bdf9', | |
388 | 'info_dict': { | |
389 | 'id': '49612410-00ca-11eb-bcd8-30d4253e0140', | |
390 | 'ext': 'mp4', | |
391 | 'title': 'Dovizioso, il contatto con Zarco e la caduta. E anche Vale finisce a terra', | |
392 | 'description': 'md5:8c6e905dc3b9413218beca11ebd69778', | |
393 | 'uploader': 'AMorici', | |
394 | } | |
395 | }, { | |
5867a167 | 396 | 'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945', |
a85e131b | 397 | 'match_only': True |
398 | }] | |
399 | ||
400 | ||
5867a167 | 401 | class RCSVariousIE(RCSBaseIE): |
a85e131b | 402 | _VALID_URL = r'''(?x)https?://www\. |
403 | (?P<cdn> | |
404 | leitv\.it| | |
405 | youreporter\.it | |
18c1f043 | 406 | )/(?:[^/]+/)?(?P<id>[^/]+?)(?:$|\?|/)''' |
a85e131b | 407 | _TESTS = [{ |
18c1f043 | 408 | 'url': 'https://www.leitv.it/benessere/mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa/', |
409 | 'md5': '92b4e63667b8f95acb0a04da25ae28a1', | |
a85e131b | 410 | 'info_dict': { |
18c1f043 | 411 | 'id': 'mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa', |
a85e131b | 412 | 'ext': 'mp4', |
18c1f043 | 413 | 'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto', |
414 | 'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5', | |
a85e131b | 415 | 'uploader': 'leitv.it', |
416 | } | |
417 | }, { | |
418 | 'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/', | |
419 | 'md5': '8dccd436b47a830bab5b4a88232f391a', | |
420 | 'info_dict': { | |
421 | 'id': 'fiume-sesia-3-ottobre-2020', | |
422 | 'ext': 'mp4', | |
423 | 'title': 'Fiume Sesia 3 ottobre 2020', | |
424 | 'description': 'md5:0070eef1cc884d13c970a4125063de55', | |
425 | 'uploader': 'youreporter.it', | |
426 | } | |
427 | }] |