]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/rcs.py
[extractors] Use new framework for existing embeds (#4307)
[yt-dlp.git] / yt_dlp / extractor / rcs.py
CommitLineData
a85e131b 1import re
2
3from .common import InfoExtractor
4from ..utils import (
5 clean_html,
6 ExtractorError,
7 js_to_json,
8 base_url,
9 url_basename,
10 urljoin,
11)
12
13
5867a167 14class RCSBaseIE(InfoExtractor):
18c1f043 15 # based on VideoPlayerLoader.prototype.getVideoSrc
16 # and VideoPlayerLoader.prototype.transformSrc from
17 # https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs
a85e131b 18 _ALL_REPLACE = {
19 'media2vam.corriere.it.edgesuite.net':
20 'media2vam-corriere-it.akamaized.net',
21 'media.youreporter.it.edgesuite.net':
22 'media-youreporter-it.akamaized.net',
23 'corrierepmd.corriere.it.edgesuite.net':
24 'corrierepmd-corriere-it.akamaized.net',
25 'media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/':
26 'video.corriere.it/vr360/videos/',
27 '.net//': '.net/',
28 }
29 _MP4_REPLACE = {
30 'media2vam.corbologna.corriere.it.edgesuite.net':
31 'media2vam-bologna-corriere-it.akamaized.net',
32 'media2vam.corfiorentino.corriere.it.edgesuite.net':
33 'media2vam-fiorentino-corriere-it.akamaized.net',
34 'media2vam.cormezzogiorno.corriere.it.edgesuite.net':
35 'media2vam-mezzogiorno-corriere-it.akamaized.net',
36 'media2vam.corveneto.corriere.it.edgesuite.net':
37 'media2vam-veneto-corriere-it.akamaized.net',
38 'media2.oggi.it.edgesuite.net':
39 'media2-oggi-it.akamaized.net',
40 'media2.quimamme.it.edgesuite.net':
41 'media2-quimamme-it.akamaized.net',
42 'media2.amica.it.edgesuite.net':
43 'media2-amica-it.akamaized.net',
44 'media2.living.corriere.it.edgesuite.net':
45 'media2-living-corriere-it.akamaized.net',
46 'media2.style.corriere.it.edgesuite.net':
47 'media2-style-corriere-it.akamaized.net',
48 'media2.iodonna.it.edgesuite.net':
49 'media2-iodonna-it.akamaized.net',
50 'media2.leitv.it.edgesuite.net':
51 'media2-leitv-it.akamaized.net',
52 }
53 _MIGRATION_MAP = {
54 'videoamica-vh.akamaihd': 'amica',
55 'media2-amica-it.akamaized': 'amica',
56 'corrierevam-vh.akamaihd': 'corriere',
57 'media2vam-corriere-it.akamaized': 'corriere',
58 'cormezzogiorno-vh.akamaihd': 'corrieredelmezzogiorno',
59 'media2vam-mezzogiorno-corriere-it.akamaized': 'corrieredelmezzogiorno',
60 'corveneto-vh.akamaihd': 'corrieredelveneto',
61 'media2vam-veneto-corriere-it.akamaized': 'corrieredelveneto',
62 'corbologna-vh.akamaihd': 'corrieredibologna',
63 'media2vam-bologna-corriere-it.akamaized': 'corrieredibologna',
64 'corfiorentino-vh.akamaihd': 'corrierefiorentino',
65 'media2vam-fiorentino-corriere-it.akamaized': 'corrierefiorentino',
66 'corinnovazione-vh.akamaihd': 'corriereinnovazione',
67 'media2-gazzanet-gazzetta-it.akamaized': 'gazzanet',
68 'videogazzanet-vh.akamaihd': 'gazzanet',
69 'videogazzaworld-vh.akamaihd': 'gazzaworld',
70 'gazzettavam-vh.akamaihd': 'gazzetta',
71 'media2vam-gazzetta-it.akamaized': 'gazzetta',
72 'videoiodonna-vh.akamaihd': 'iodonna',
73 'media2-leitv-it.akamaized': 'leitv',
74 'videoleitv-vh.akamaihd': 'leitv',
75 'videoliving-vh.akamaihd': 'living',
76 'media2-living-corriere-it.akamaized': 'living',
77 'media2-oggi-it.akamaized': 'oggi',
78 'videooggi-vh.akamaihd': 'oggi',
79 'media2-quimamme-it.akamaized': 'quimamme',
80 'quimamme-vh.akamaihd': 'quimamme',
81 'videorunning-vh.akamaihd': 'running',
82 'media2-style-corriere-it.akamaized': 'style',
83 'style-vh.akamaihd': 'style',
84 'videostyle-vh.akamaihd': 'style',
85 'media2-stylepiccoli-it.akamaized': 'stylepiccoli',
86 'stylepiccoli-vh.akamaihd': 'stylepiccoli',
87 'doveviaggi-vh.akamaihd': 'viaggi',
88 'media2-doveviaggi-it.akamaized': 'viaggi',
89 'media2-vivimilano-corriere-it.akamaized': 'vivimilano',
90 'vivimilano-vh.akamaihd': 'vivimilano',
91 'media2-youreporter-it.akamaized': 'youreporter'
92 }
93 _MIGRATION_MEDIA = {
94 'advrcs-vh.akamaihd': '',
95 'corriere-f.akamaihd': '',
96 'corrierepmd-corriere-it.akamaized': '',
97 'corrprotetto-vh.akamaihd': '',
98 'gazzetta-f.akamaihd': '',
99 'gazzettapmd-gazzetta-it.akamaized': '',
100 'gazzprotetto-vh.akamaihd': '',
101 'periodici-f.akamaihd': '',
102 'periodicisecure-vh.akamaihd': '',
103 'videocoracademy-vh.akamaihd': ''
104 }
105
106 def _get_video_src(self, video):
508649e6 107 mediaFiles = video.get('mediaProfile').get('mediaFile')
a85e131b 108 src = {}
109 # audio
508649e6 110 if video.get('mediaType') == 'AUDIO':
a85e131b 111 for aud in mediaFiles:
112 # todo: check
508649e6 113 src['mp3'] = aud.get('value')
a85e131b 114 # video
115 else:
116 for vid in mediaFiles:
508649e6 117 if vid.get('mimeType') == 'application/vnd.apple.mpegurl':
118 src['m3u8'] = vid.get('value')
119 if vid.get('mimeType') == 'video/mp4':
120 src['mp4'] = vid.get('value')
a85e131b 121
122 # replace host
123 for t in src:
124 for s, r in self._ALL_REPLACE.items():
125 src[t] = src[t].replace(s, r)
126 for s, r in self._MP4_REPLACE.items():
127 src[t] = src[t].replace(s, r)
128
129 # switch cdn
130 if 'mp4' in src and 'm3u8' in src:
508649e6 131 if ('-lh.akamaihd' not in src.get('m3u8')
132 and 'akamai' in src.get('mp4')):
a85e131b 133 if 'm3u8' in src:
508649e6 134 matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('m3u8'))
a85e131b 135 src['m3u8'] = 'https://vod.rcsobjects.it/hls/%s%s' % (
136 self._MIGRATION_MAP[matches.group('host')],
137 matches.group('path').replace(
138 '///', '/').replace(
139 '//', '/').replace(
140 '.csmil', '.urlset'
141 )
142 )
143 if 'mp4' in src:
508649e6 144 matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('mp4'))
a85e131b 145 if matches:
146 if matches.group('host') in self._MIGRATION_MEDIA:
147 vh_stream = 'https://media2.corriereobjects.it'
508649e6 148 if src.get('mp4').find('fcs.quotidiani_!'):
a85e131b 149 vh_stream = 'https://media2-it.corriereobjects.it'
150 src['mp4'] = '%s%s' % (
151 vh_stream,
152 matches.group('path').replace(
153 '///', '/').replace(
154 '//', '/').replace(
155 '/fcs.quotidiani/mediacenter', '').replace(
156 '/fcs.quotidiani_!/mediacenter', '').replace(
157 'corriere/content/mediacenter/', '').replace(
158 'gazzetta/content/mediacenter/', '')
159 )
160 else:
161 src['mp4'] = 'https://vod.rcsobjects.it/%s%s' % (
162 self._MIGRATION_MAP[matches.group('host')],
163 matches.group('path').replace('///', '/').replace('//', '/')
164 )
165
166 if 'mp3' in src:
508649e6 167 src['mp3'] = src.get('mp3').replace(
a85e131b 168 'media2vam-corriere-it.akamaized.net',
169 'vod.rcsobjects.it/corriere')
170 if 'mp4' in src:
508649e6 171 if src.get('mp4').find('fcs.quotidiani_!'):
172 src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
a85e131b 173 if 'm3u8' in src:
508649e6 174 if src.get('m3u8').find('fcs.quotidiani_!'):
175 src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')
a85e131b 176
508649e6 177 if 'geoblocking' in video.get('mediaProfile'):
a85e131b 178 if 'm3u8' in src:
508649e6 179 src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')
a85e131b 180 if 'mp4' in src:
508649e6 181 src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
a85e131b 182 if 'm3u8' in src:
508649e6 183 if src.get('m3u8').find('csmil') and src.get('m3u8').find('vod'):
184 src['m3u8'] = src.get('m3u8').replace('.csmil', '.urlset')
a85e131b 185
186 return src
187
188 def _create_formats(self, urls, video_id):
189 formats = []
190 formats = self._extract_m3u8_formats(
508649e6 191 urls.get('m3u8'), video_id, 'mp4', entry_protocol='m3u8_native',
a85e131b 192 m3u8_id='hls', fatal=False)
193
18c1f043 194 if urls.get('mp4'):
a85e131b 195 formats.append({
196 'format_id': 'http-mp4',
18c1f043 197 'url': urls['mp4']
a85e131b 198 })
199 self._sort_formats(formats)
200 return formats
201
202 def _real_extract(self, url):
5ad28e7f 203 mobj = self._match_valid_url(url)
204 video_id = mobj.group('id')
a85e131b 205
508649e6 206 if 'cdn' not in mobj.groupdict():
a85e131b 207 raise ExtractorError('CDN not found in url: %s' % url)
208
209 # for leitv/youreporter/viaggi don't use the embed page
508649e6 210 if ((mobj.group('cdn') not in ['leitv.it', 'youreporter.it'])
211 and (mobj.group('vid') == 'video')):
212 url = 'https://video.%s/video-embed/%s' % (mobj.group('cdn'), video_id)
a85e131b 213
214 page = self._download_webpage(url, video_id)
215
216 video_data = None
217 # look for json video data url
218 json = self._search_regex(
18c1f043 219 r'''(?x)url\s*=\s*(["'])
220 (?P<url>
221 (?:https?:)?//video\.rcs\.it
222 /fragment-includes/video-includes/.+?\.json
223 )\1;''',
224 page, video_id, group='url', default=None)
a85e131b 225 if json:
226 if json.startswith('//'):
227 json = 'https:%s' % json
228 video_data = self._download_json(json, video_id)
229
508649e6 230 # if json url not found, look for json video data directly in the page
a85e131b 231 else:
18c1f043 232 # RCS normal pages and most of the embeds
a85e131b 233 json = self._search_regex(
234 r'[\s;]video\s*=\s*({[\s\S]+?})(?:;|,playlist=)',
235 page, video_id, default=None)
18c1f043 236 if not json and 'video-embed' in url:
237 page = self._download_webpage(url.replace('video-embed', 'video-json'), video_id)
238 json = self._search_regex(
239 r'##start-video##({[\s\S]+?})##end-video##',
240 page, video_id, default=None)
241 if not json:
a85e131b 242 # if no video data found try search for iframes
243 emb = RCSEmbedsIE._extract_url(page)
244 if emb:
5867a167 245 return {
246 '_type': 'url_transparent',
247 'url': emb,
248 'ie_key': RCSEmbedsIE.ie_key()
249 }
18c1f043 250 if json:
251 video_data = self._parse_json(
252 json, video_id, transform_source=js_to_json)
a85e131b 253
254 if not video_data:
255 raise ExtractorError('Video data not found in the page')
256
257 formats = self._create_formats(
258 self._get_video_src(video_data), video_id)
259
508649e6 260 description = (video_data.get('description')
18c1f043 261 or clean_html(video_data.get('htmlDescription'))
262 or self._html_search_meta('description', page))
5867a167 263 uploader = video_data.get('provider') or mobj.group('cdn')
508649e6 264
a85e131b 265 return {
266 'id': video_id,
508649e6 267 'title': video_data.get('title'),
268 'description': description,
269 'uploader': uploader,
a85e131b 270 'formats': formats
271 }
272
273
5867a167 274class RCSEmbedsIE(RCSBaseIE):
a85e131b 275 _VALID_URL = r'''(?x)
276 https?://(?P<vid>video)\.
277 (?P<cdn>
278 (?:
279 rcs|
280 (?:corriere\w+\.)?corriere|
281 (?:gazzanet\.)?gazzetta
282 )\.it)
283 /video-embed/(?P<id>[^/=&\?]+?)(?:$|\?)'''
bfd973ec 284 _EMBED_REGEX = [r'''(?x)
285 (?:
286 data-frame-src=|
287 <iframe[^\n]+src=
288 )
289 (["'])
290 (?P<url>(?:https?:)?//video\.
291 (?:
292 rcs|
293 (?:corriere\w+\.)?corriere|
294 (?:gazzanet\.)?gazzetta
295 )
296 \.it/video-embed/.+?)
297 \1''']
a85e131b 298 _TESTS = [{
299 'url': 'https://video.rcs.it/video-embed/iodonna-0001585037',
300 'md5': '623ecc8ffe7299b2d0c1046d8331a9df',
301 'info_dict': {
302 'id': 'iodonna-0001585037',
303 'ext': 'mp4',
304 'title': 'Sky Arte racconta Madonna nella serie "Artist to icon"',
305 'description': 'md5:65b09633df9ffee57f48b39e34c9e067',
306 'uploader': 'rcs.it',
307 }
5867a167 308 }, {
18c1f043 309 # redownload the page changing 'video-embed' in 'video-json'
5867a167 310 'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789',
311 'md5': 'a043e3fecbe4d9ed7fc5d888652a5440',
312 'info_dict': {
313 'id': 'gazzanet-mo05-0000260789',
314 'ext': 'mp4',
315 'title': 'Valentino Rossi e papà Graziano si divertono col drifting',
316 'description': 'md5:a8bf90d6adafd9815f70fc74c0fc370a',
317 'uploader': 'rcd',
318 }
a85e131b 319 }, {
320 'url': 'https://video.corriere.it/video-embed/b727632a-f9d0-11ea-91b0-38d50a849abb?player',
321 'match_only': True
322 }, {
323 'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140',
324 'match_only': True
325 }]
326
327 @staticmethod
328 def _sanitize_urls(urls):
329 # add protocol if missing
330 for i, e in enumerate(urls):
331 if e.startswith('//'):
332 urls[i] = 'https:%s' % e
333 # clean iframes urls
334 for i, e in enumerate(urls):
335 urls[i] = urljoin(base_url(e), url_basename(e))
336 return urls
337
bfd973ec 338 @classmethod
339 def _extract_embed_urls(cls, url, webpage):
340 return cls._sanitize_urls(tuple(super()._extract_embed_urls(url, webpage)))
a85e131b 341
342
5867a167 343class RCSIE(RCSBaseIE):
a85e131b 344 _VALID_URL = r'''(?x)https?://(?P<vid>video|viaggi)\.
345 (?P<cdn>
346 (?:
5867a167 347 corrieredelmezzogiorno\.
348 |corrieredelveneto\.
349 |corrieredibologna\.
350 |corrierefiorentino\.
351 )?corriere\.it
352 |(?:gazzanet\.)?gazzetta\.it)
353 /(?!video-embed/).+?/(?P<id>[^/\?]+)(?=\?|/$|$)'''
a85e131b 354 _TESTS = [{
355 'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb',
356 'md5': '0f4ededc202b0f00b6e509d831e2dcda',
357 'info_dict': {
358 'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb',
359 'ext': 'mp4',
360 'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante',
361 'description': 'md5:93b51c9161ac8a64fb2f997b054d0152',
362 'uploader': 'Corriere Tv',
363 }
364 }, {
18c1f043 365 # video data inside iframe
a85e131b 366 'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/',
367 'md5': 'da378e4918d2afbf7d61c35abb948d4c',
368 'info_dict': {
369 'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2',
370 'ext': 'mp4',
371 'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen',
372 'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8',
373 'uploader': 'DOVE Viaggi',
374 }
375 }, {
a85e131b 376 'url': 'https://video.gazzetta.it/video-motogp-catalogna-cadute-dovizioso-vale-rossi/49612410-00ca-11eb-bcd8-30d4253e0140?vclk=Videobar',
377 'md5': 'eedc1b5defd18e67383afef51ff7bdf9',
378 'info_dict': {
379 'id': '49612410-00ca-11eb-bcd8-30d4253e0140',
380 'ext': 'mp4',
381 'title': 'Dovizioso, il contatto con Zarco e la caduta. E anche Vale finisce a terra',
382 'description': 'md5:8c6e905dc3b9413218beca11ebd69778',
383 'uploader': 'AMorici',
384 }
385 }, {
5867a167 386 'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945',
a85e131b 387 'match_only': True
388 }]
389
390
5867a167 391class RCSVariousIE(RCSBaseIE):
a85e131b 392 _VALID_URL = r'''(?x)https?://www\.
393 (?P<cdn>
394 leitv\.it|
395 youreporter\.it
18c1f043 396 )/(?:[^/]+/)?(?P<id>[^/]+?)(?:$|\?|/)'''
a85e131b 397 _TESTS = [{
18c1f043 398 'url': 'https://www.leitv.it/benessere/mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa/',
399 'md5': '92b4e63667b8f95acb0a04da25ae28a1',
a85e131b 400 'info_dict': {
18c1f043 401 'id': 'mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa',
a85e131b 402 'ext': 'mp4',
18c1f043 403 'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto',
404 'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5',
a85e131b 405 'uploader': 'leitv.it',
406 }
407 }, {
408 'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/',
409 'md5': '8dccd436b47a830bab5b4a88232f391a',
410 'info_dict': {
411 'id': 'fiume-sesia-3-ottobre-2020',
412 'ext': 'mp4',
413 'title': 'Fiume Sesia 3 ottobre 2020',
414 'description': 'md5:0070eef1cc884d13c970a4125063de55',
415 'uploader': 'youreporter.it',
416 }
417 }]