]>
Commit | Line | Data |
---|---|---|
a85e131b | 1 | import re |
2 | ||
3 | from .common import InfoExtractor | |
3d2623a8 | 4 | from ..networking import HEADRequest |
a85e131b | 5 | from ..utils import ( |
a85e131b | 6 | ExtractorError, |
a85e131b | 7 | base_url, |
304ad45a | 8 | clean_html, |
c6b65786 | 9 | extract_attributes, |
10 | get_element_html_by_class, | |
11 | get_element_html_by_id, | |
12 | int_or_none, | |
304ad45a | 13 | js_to_json, |
c6b65786 | 14 | mimetype2ext, |
15 | sanitize_url, | |
16 | traverse_obj, | |
17 | try_call, | |
a85e131b | 18 | url_basename, |
19 | urljoin, | |
20 | ) | |
21 | ||
22 | ||
5867a167 | 23 | class RCSBaseIE(InfoExtractor): |
18c1f043 | 24 | # based on VideoPlayerLoader.prototype.getVideoSrc |
25 | # and VideoPlayerLoader.prototype.transformSrc from | |
26 | # https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs | |
c6b65786 | 27 | _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}' |
28 | _RCS_ID_RE = r'[\w-]+-\d{10}' | |
a85e131b | 29 | _MIGRATION_MAP = { |
30 | 'videoamica-vh.akamaihd': 'amica', | |
31 | 'media2-amica-it.akamaized': 'amica', | |
32 | 'corrierevam-vh.akamaihd': 'corriere', | |
33 | 'media2vam-corriere-it.akamaized': 'corriere', | |
34 | 'cormezzogiorno-vh.akamaihd': 'corrieredelmezzogiorno', | |
35 | 'media2vam-mezzogiorno-corriere-it.akamaized': 'corrieredelmezzogiorno', | |
36 | 'corveneto-vh.akamaihd': 'corrieredelveneto', | |
37 | 'media2vam-veneto-corriere-it.akamaized': 'corrieredelveneto', | |
38 | 'corbologna-vh.akamaihd': 'corrieredibologna', | |
39 | 'media2vam-bologna-corriere-it.akamaized': 'corrieredibologna', | |
40 | 'corfiorentino-vh.akamaihd': 'corrierefiorentino', | |
41 | 'media2vam-fiorentino-corriere-it.akamaized': 'corrierefiorentino', | |
42 | 'corinnovazione-vh.akamaihd': 'corriereinnovazione', | |
43 | 'media2-gazzanet-gazzetta-it.akamaized': 'gazzanet', | |
44 | 'videogazzanet-vh.akamaihd': 'gazzanet', | |
45 | 'videogazzaworld-vh.akamaihd': 'gazzaworld', | |
46 | 'gazzettavam-vh.akamaihd': 'gazzetta', | |
47 | 'media2vam-gazzetta-it.akamaized': 'gazzetta', | |
48 | 'videoiodonna-vh.akamaihd': 'iodonna', | |
49 | 'media2-leitv-it.akamaized': 'leitv', | |
50 | 'videoleitv-vh.akamaihd': 'leitv', | |
51 | 'videoliving-vh.akamaihd': 'living', | |
52 | 'media2-living-corriere-it.akamaized': 'living', | |
53 | 'media2-oggi-it.akamaized': 'oggi', | |
54 | 'videooggi-vh.akamaihd': 'oggi', | |
55 | 'media2-quimamme-it.akamaized': 'quimamme', | |
56 | 'quimamme-vh.akamaihd': 'quimamme', | |
57 | 'videorunning-vh.akamaihd': 'running', | |
58 | 'media2-style-corriere-it.akamaized': 'style', | |
59 | 'style-vh.akamaihd': 'style', | |
60 | 'videostyle-vh.akamaihd': 'style', | |
61 | 'media2-stylepiccoli-it.akamaized': 'stylepiccoli', | |
62 | 'stylepiccoli-vh.akamaihd': 'stylepiccoli', | |
63 | 'doveviaggi-vh.akamaihd': 'viaggi', | |
64 | 'media2-doveviaggi-it.akamaized': 'viaggi', | |
65 | 'media2-vivimilano-corriere-it.akamaized': 'vivimilano', | |
66 | 'vivimilano-vh.akamaihd': 'vivimilano', | |
67 | 'media2-youreporter-it.akamaized': 'youreporter' | |
68 | } | |
a85e131b | 69 | |
70 | def _get_video_src(self, video): | |
c6b65786 | 71 | for source in traverse_obj(video, ( |
72 | 'mediaProfile', 'mediaFile', lambda _, v: v.get('mimeType'))): | |
73 | url = source['value'] | |
74 | for s, r in ( | |
75 | ('media2vam.corriere.it.edgesuite.net', 'media2vam-corriere-it.akamaized.net'), | |
76 | ('media.youreporter.it.edgesuite.net', 'media-youreporter-it.akamaized.net'), | |
77 | ('corrierepmd.corriere.it.edgesuite.net', 'corrierepmd-corriere-it.akamaized.net'), | |
78 | ('media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/', 'video.corriere.it/vr360/videos/'), | |
79 | ('http://', 'https://'), | |
80 | ): | |
81 | url = url.replace(s, r) | |
a85e131b | 82 | |
c6b65786 | 83 | type_ = mimetype2ext(source['mimeType']) |
84 | if type_ == 'm3u8' and '-vh.akamaihd' in url: | |
85 | # still needed for some old content: see _TESTS #3 | |
86 | matches = re.search(r'(?:https?:)?//(?P<host>[\w\.\-]+)\.net/i(?P<path>.+)$', url) | |
87 | if matches: | |
88 | url = f'https://vod.rcsobjects.it/hls/{self._MIGRATION_MAP[matches.group("host")]}{matches.group("path")}' | |
89 | if traverse_obj(video, ('mediaProfile', 'geoblocking')) or ( | |
90 | type_ == 'm3u8' and 'fcs.quotidiani_!' in url): | |
91 | url = url.replace('vod.rcsobjects', 'vod-it.rcsobjects') | |
92 | if type_ == 'm3u8' and 'vod' in url: | |
93 | url = url.replace('.csmil', '.urlset') | |
94 | if type_ == 'mp3': | |
95 | url = url.replace('media2vam-corriere-it.akamaized.net', 'vod.rcsobjects.it/corriere') | |
a85e131b | 96 | |
c6b65786 | 97 | yield { |
98 | 'type': type_, | |
99 | 'url': url, | |
100 | 'bitrate': source.get('bitrate') | |
101 | } | |
a85e131b | 102 | |
c6b65786 | 103 | def _create_http_formats(self, m3u8_formats, video_id): |
104 | for f in m3u8_formats: | |
105 | if f['vcodec'] == 'none': | |
106 | continue | |
107 | http_url = re.sub(r'(https?://[^/]+)/hls/([^?#]+?\.mp4).+', r'\g<1>/\g<2>', f['url']) | |
108 | if http_url == f['url']: | |
109 | continue | |
a85e131b | 110 | |
c6b65786 | 111 | http_f = f.copy() |
112 | del http_f['manifest_url'] | |
113 | format_id = try_call(lambda: http_f['format_id'].replace('hls-', 'https-')) | |
114 | urlh = self._request_webpage(HEADRequest(http_url), video_id, fatal=False, | |
115 | note=f'Check filesize for {format_id}') | |
116 | if not urlh: | |
117 | continue | |
a85e131b | 118 | |
c6b65786 | 119 | http_f.update({ |
120 | 'format_id': format_id, | |
121 | 'url': http_url, | |
122 | 'protocol': 'https', | |
123 | 'filesize_approx': int_or_none(urlh.headers.get('Content-Length', None)), | |
a85e131b | 124 | }) |
c6b65786 | 125 | yield http_f |
126 | ||
127 | def _create_formats(self, sources, video_id): | |
128 | for source in sources: | |
129 | if source['type'] == 'm3u8': | |
130 | m3u8_formats = self._extract_m3u8_formats( | |
131 | source['url'], video_id, 'mp4', m3u8_id='hls', fatal=False) | |
132 | yield from m3u8_formats | |
133 | yield from self._create_http_formats(m3u8_formats, video_id) | |
134 | elif source['type'] == 'mp3': | |
135 | yield { | |
136 | 'format_id': 'https-mp3', | |
137 | 'ext': 'mp3', | |
138 | 'acodec': 'mp3', | |
139 | 'vcodec': 'none', | |
140 | 'abr': source.get('bitrate'), | |
141 | 'url': source['url'], | |
142 | } | |
a85e131b | 143 | |
144 | def _real_extract(self, url): | |
c6b65786 | 145 | cdn, video_id = self._match_valid_url(url).group('cdn', 'id') |
146 | display_id, video_data = None, None | |
a85e131b | 147 | |
c6b65786 | 148 | if re.match(self._UUID_RE, video_id) or re.match(self._RCS_ID_RE, video_id): |
149 | url = f'https://video.{cdn}/video-json/{video_id}' | |
150 | else: | |
151 | webpage = self._download_webpage(url, video_id) | |
152 | data_config = get_element_html_by_id('divVideoPlayer', webpage) or get_element_html_by_class('divVideoPlayer', webpage) | |
a85e131b | 153 | |
c6b65786 | 154 | if data_config: |
155 | data_config = self._parse_json( | |
156 | extract_attributes(data_config).get('data-config'), | |
157 | video_id, fatal=False) or {} | |
158 | if data_config.get('newspaper'): | |
159 | cdn = f'{data_config["newspaper"]}.it' | |
160 | display_id, video_id = video_id, data_config.get('uuid') or video_id | |
161 | url = f'https://video.{cdn}/video-json/{video_id}' | |
162 | else: | |
163 | json_url = self._search_regex( | |
164 | r'''(?x)url\s*=\s*(["']) | |
165 | (?P<url> | |
166 | (?:https?:)?//video\.rcs\.it | |
167 | /fragment-includes/video-includes/[^"']+?\.json | |
168 | )\1;''', | |
169 | webpage, video_id, group='url', default=None) | |
170 | if json_url: | |
171 | video_data = self._download_json(sanitize_url(json_url, scheme='https'), video_id) | |
172 | display_id, video_id = video_id, video_data.get('id') or video_id | |
a85e131b | 173 | |
c6b65786 | 174 | if not video_data: |
175 | webpage = self._download_webpage(url, video_id) | |
a85e131b | 176 | |
c6b65786 | 177 | video_data = self._search_json( |
178 | '##start-video##', webpage, 'video data', video_id, default=None, | |
179 | end_pattern='##end-video##', transform_source=js_to_json) | |
a85e131b | 180 | |
c6b65786 | 181 | if not video_data: |
182 | # try search for iframes | |
183 | emb = RCSEmbedsIE._extract_url(webpage) | |
a85e131b | 184 | if emb: |
5867a167 | 185 | return { |
186 | '_type': 'url_transparent', | |
187 | 'url': emb, | |
188 | 'ie_key': RCSEmbedsIE.ie_key() | |
189 | } | |
a85e131b | 190 | |
191 | if not video_data: | |
192 | raise ExtractorError('Video data not found in the page') | |
193 | ||
a85e131b | 194 | return { |
195 | 'id': video_id, | |
c6b65786 | 196 | 'display_id': display_id, |
508649e6 | 197 | 'title': video_data.get('title'), |
c6b65786 | 198 | 'description': (clean_html(video_data.get('description')) |
199 | or clean_html(video_data.get('htmlDescription')) | |
200 | or self._html_search_meta('description', webpage)), | |
201 | 'uploader': video_data.get('provider') or cdn, | |
202 | 'formats': list(self._create_formats(self._get_video_src(video_data), video_id)), | |
a85e131b | 203 | } |
204 | ||
205 | ||
5867a167 | 206 | class RCSEmbedsIE(RCSBaseIE): |
a85e131b | 207 | _VALID_URL = r'''(?x) |
208 | https?://(?P<vid>video)\. | |
209 | (?P<cdn> | |
210 | (?: | |
211 | rcs| | |
212 | (?:corriere\w+\.)?corriere| | |
213 | (?:gazzanet\.)?gazzetta | |
214 | )\.it) | |
215 | /video-embed/(?P<id>[^/=&\?]+?)(?:$|\?)''' | |
bfd973ec | 216 | _EMBED_REGEX = [r'''(?x) |
217 | (?: | |
218 | data-frame-src=| | |
219 | <iframe[^\n]+src= | |
220 | ) | |
221 | (["']) | |
222 | (?P<url>(?:https?:)?//video\. | |
223 | (?: | |
224 | rcs| | |
225 | (?:corriere\w+\.)?corriere| | |
226 | (?:gazzanet\.)?gazzetta | |
227 | ) | |
228 | \.it/video-embed/.+?) | |
229 | \1'''] | |
a85e131b | 230 | _TESTS = [{ |
231 | 'url': 'https://video.rcs.it/video-embed/iodonna-0001585037', | |
c6b65786 | 232 | 'md5': '0faca97df525032bb9847f690bc3720c', |
a85e131b | 233 | 'info_dict': { |
234 | 'id': 'iodonna-0001585037', | |
235 | 'ext': 'mp4', | |
236 | 'title': 'Sky Arte racconta Madonna nella serie "Artist to icon"', | |
237 | 'description': 'md5:65b09633df9ffee57f48b39e34c9e067', | |
238 | 'uploader': 'rcs.it', | |
239 | } | |
5867a167 | 240 | }, { |
241 | 'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789', | |
a85e131b | 242 | 'match_only': True |
243 | }, { | |
244 | 'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140', | |
245 | 'match_only': True | |
246 | }] | |
c6b65786 | 247 | _WEBPAGE_TESTS = [{ |
248 | 'url': 'https://www.iodonna.it/video-iodonna/personaggi-video/monica-bellucci-piu-del-lavoro-oggi-per-me-sono-importanti-lamicizia-e-la-famiglia/', | |
249 | 'info_dict': { | |
250 | 'id': 'iodonna-0002033648', | |
251 | 'ext': 'mp4', | |
252 | 'title': 'Monica Bellucci: «Più del lavoro, oggi per me sono importanti l\'amicizia e la famiglia»', | |
253 | 'description': 'md5:daea6d9837351e56b1ab615c06bebac1', | |
254 | 'uploader': 'rcs.it', | |
255 | } | |
256 | }] | |
a85e131b | 257 | |
258 | @staticmethod | |
c6b65786 | 259 | def _sanitize_url(url): |
260 | url = sanitize_url(url, scheme='https') | |
261 | return urljoin(base_url(url), url_basename(url)) | |
a85e131b | 262 | |
bfd973ec | 263 | @classmethod |
264 | def _extract_embed_urls(cls, url, webpage): | |
c6b65786 | 265 | return map(cls._sanitize_url, super()._extract_embed_urls(url, webpage)) |
a85e131b | 266 | |
267 | ||
5867a167 | 268 | class RCSIE(RCSBaseIE): |
a85e131b | 269 | _VALID_URL = r'''(?x)https?://(?P<vid>video|viaggi)\. |
270 | (?P<cdn> | |
271 | (?: | |
5867a167 | 272 | corrieredelmezzogiorno\. |
273 | |corrieredelveneto\. | |
274 | |corrieredibologna\. | |
275 | |corrierefiorentino\. | |
276 | )?corriere\.it | |
277 | |(?:gazzanet\.)?gazzetta\.it) | |
c6b65786 | 278 | /(?!video-embed/)[^?#]+?/(?P<id>[^/\?]+)(?=\?|/$|$)''' |
a85e131b | 279 | _TESTS = [{ |
c6b65786 | 280 | # json iframe directly from id |
a85e131b | 281 | 'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb', |
c6b65786 | 282 | 'md5': '14946840dec46ecfddf66ba4eea7d2b2', |
a85e131b | 283 | 'info_dict': { |
284 | 'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb', | |
285 | 'ext': 'mp4', | |
286 | 'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante', | |
c6b65786 | 287 | 'description': 'md5:3915ce5ebb3d2571deb69a5eb85ac9b5', |
a85e131b | 288 | 'uploader': 'Corriere Tv', |
289 | } | |
290 | }, { | |
c6b65786 | 291 | # search for video id inside the page |
a85e131b | 292 | 'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/', |
c6b65786 | 293 | 'md5': 'f22a92d9e666e80f2fffbf2825359c81', |
a85e131b | 294 | 'info_dict': { |
295 | 'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2', | |
c6b65786 | 296 | 'display_id': 'norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen', |
a85e131b | 297 | 'ext': 'mp4', |
298 | 'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen', | |
299 | 'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8', | |
300 | 'uploader': 'DOVE Viaggi', | |
301 | } | |
302 | }, { | |
c6b65786 | 303 | # only audio format https://github.com/yt-dlp/yt-dlp/issues/5683 |
304 | 'url': 'https://video.corriere.it/cronaca/audio-telefonata-il-papa-becciu-santita-lettera-che-mi-ha-inviato-condanna/b94c0d20-70c2-11ed-9572-e4b947a0ebd2', | |
305 | 'md5': 'aaffb08d02f2ce4292a4654694c78150', | |
306 | 'info_dict': { | |
307 | 'id': 'b94c0d20-70c2-11ed-9572-e4b947a0ebd2', | |
308 | 'ext': 'mp3', | |
309 | 'title': 'L\'audio della telefonata tra il Papa e Becciu: «Santità, la lettera che mi ha inviato è una condanna»', | |
310 | 'description': 'md5:c0ddb61bd94a8d4e0d4bb9cda50a689b', | |
311 | 'uploader': 'Corriere Tv', | |
312 | 'formats': [{'format_id': 'https-mp3', 'ext': 'mp3'}], | |
313 | } | |
314 | }, { | |
315 | # old content still needs cdn migration | |
316 | 'url': 'https://viaggi.corriere.it/video/milano-varallo-sesia-sul-treno-a-vapore/', | |
317 | 'md5': '2dfdce7af249654ad27eeba03fe1e08d', | |
a85e131b | 318 | 'info_dict': { |
c6b65786 | 319 | 'id': 'd8f6c8d0-f7d7-11e8-bfca-f74cf4634191', |
320 | 'display_id': 'milano-varallo-sesia-sul-treno-a-vapore', | |
a85e131b | 321 | 'ext': 'mp4', |
c6b65786 | 322 | 'title': 'Milano-Varallo Sesia sul treno a vapore', |
323 | 'description': 'md5:6348f47aac230397fe341a74f7678d53', | |
324 | 'uploader': 'DOVE Viaggi', | |
a85e131b | 325 | } |
326 | }, { | |
5867a167 | 327 | 'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945', |
a85e131b | 328 | 'match_only': True |
329 | }] | |
330 | ||
331 | ||
5867a167 | 332 | class RCSVariousIE(RCSBaseIE): |
a85e131b | 333 | _VALID_URL = r'''(?x)https?://www\. |
334 | (?P<cdn> | |
335 | leitv\.it| | |
c6b65786 | 336 | youreporter\.it| |
337 | amica\.it | |
18c1f043 | 338 | )/(?:[^/]+/)?(?P<id>[^/]+?)(?:$|\?|/)''' |
a85e131b | 339 | _TESTS = [{ |
c6b65786 | 340 | 'url': 'https://www.leitv.it/benessere/mal-di-testa/', |
341 | 'md5': '3b7a683d105a7313ec7513b014443631', | |
a85e131b | 342 | 'info_dict': { |
c6b65786 | 343 | 'id': 'leitv-0000125151', |
344 | 'display_id': 'mal-di-testa', | |
a85e131b | 345 | 'ext': 'mp4', |
18c1f043 | 346 | 'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto', |
347 | 'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5', | |
a85e131b | 348 | 'uploader': 'leitv.it', |
349 | } | |
350 | }, { | |
351 | 'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/', | |
c6b65786 | 352 | 'md5': '3989b6d603482611a2abd2f32b79f739', |
a85e131b | 353 | 'info_dict': { |
c6b65786 | 354 | 'id': 'youreporter-0000332574', |
355 | 'display_id': 'fiume-sesia-3-ottobre-2020', | |
a85e131b | 356 | 'ext': 'mp4', |
357 | 'title': 'Fiume Sesia 3 ottobre 2020', | |
358 | 'description': 'md5:0070eef1cc884d13c970a4125063de55', | |
359 | 'uploader': 'youreporter.it', | |
360 | } | |
c6b65786 | 361 | }, { |
362 | 'url': 'https://www.amica.it/video-post/saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi/', | |
363 | 'md5': '187cce524dfd0343c95646c047375fc4', | |
364 | 'info_dict': { | |
365 | 'id': 'amica-0001225365', | |
366 | 'display_id': 'saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi', | |
367 | 'ext': 'mp4', | |
368 | 'title': '"Saint Omer": al cinema il film Leone d\'argento che ribalta gli stereotipi', | |
369 | 'description': 'md5:b1c8869c2dcfd6073a2a311ba0008aa8', | |
370 | 'uploader': 'rcs.it', | |
371 | } | |
a85e131b | 372 | }] |