[yt-dlp.git] / yt_dlp / extractor / rcs.py

import re

from .common import InfoExtractor
from ..networking import HEADRequest
from ..utils import (
    ExtractorError,
    base_url,
    clean_html,
    extract_attributes,
    get_element_html_by_class,
    get_element_html_by_id,
    int_or_none,
    js_to_json,
    mimetype2ext,
    sanitize_url,
    traverse_obj,
    try_call,
    url_basename,
    urljoin,
)


class RCSBaseIE(InfoExtractor):
    # based on VideoPlayerLoader.prototype.getVideoSrc
    # and VideoPlayerLoader.prototype.transformSrc from
    # https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs
    _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
    _RCS_ID_RE = r'[\w-]+-\d{10}'
    _MIGRATION_MAP = {
        'videoamica-vh.akamaihd': 'amica',
        'media2-amica-it.akamaized': 'amica',
        'corrierevam-vh.akamaihd': 'corriere',
        'media2vam-corriere-it.akamaized': 'corriere',
        'cormezzogiorno-vh.akamaihd': 'corrieredelmezzogiorno',
        'media2vam-mezzogiorno-corriere-it.akamaized': 'corrieredelmezzogiorno',
        'corveneto-vh.akamaihd': 'corrieredelveneto',
        'media2vam-veneto-corriere-it.akamaized': 'corrieredelveneto',
        'corbologna-vh.akamaihd': 'corrieredibologna',
        'media2vam-bologna-corriere-it.akamaized': 'corrieredibologna',
        'corfiorentino-vh.akamaihd': 'corrierefiorentino',
        'media2vam-fiorentino-corriere-it.akamaized': 'corrierefiorentino',
        'corinnovazione-vh.akamaihd': 'corriereinnovazione',
        'media2-gazzanet-gazzetta-it.akamaized': 'gazzanet',
        'videogazzanet-vh.akamaihd': 'gazzanet',
        'videogazzaworld-vh.akamaihd': 'gazzaworld',
        'gazzettavam-vh.akamaihd': 'gazzetta',
        'media2vam-gazzetta-it.akamaized': 'gazzetta',
        'videoiodonna-vh.akamaihd': 'iodonna',
        'media2-leitv-it.akamaized': 'leitv',
        'videoleitv-vh.akamaihd': 'leitv',
        'videoliving-vh.akamaihd': 'living',
        'media2-living-corriere-it.akamaized': 'living',
        'media2-oggi-it.akamaized': 'oggi',
        'videooggi-vh.akamaihd': 'oggi',
        'media2-quimamme-it.akamaized': 'quimamme',
        'quimamme-vh.akamaihd': 'quimamme',
        'videorunning-vh.akamaihd': 'running',
        'media2-style-corriere-it.akamaized': 'style',
        'style-vh.akamaihd': 'style',
        'videostyle-vh.akamaihd': 'style',
        'media2-stylepiccoli-it.akamaized': 'stylepiccoli',
        'stylepiccoli-vh.akamaihd': 'stylepiccoli',
        'doveviaggi-vh.akamaihd': 'viaggi',
        'media2-doveviaggi-it.akamaized': 'viaggi',
        'media2-vivimilano-corriere-it.akamaized': 'vivimilano',
        'vivimilano-vh.akamaihd': 'vivimilano',
        'media2-youreporter-it.akamaized': 'youreporter'
    }

    def _get_video_src(self, video):
        for source in traverse_obj(video, (
                'mediaProfile', 'mediaFile', lambda _, v: v.get('mimeType'))):
            url = source['value']
            for s, r in (
                ('media2vam.corriere.it.edgesuite.net', 'media2vam-corriere-it.akamaized.net'),
                ('media.youreporter.it.edgesuite.net', 'media-youreporter-it.akamaized.net'),
                ('corrierepmd.corriere.it.edgesuite.net', 'corrierepmd-corriere-it.akamaized.net'),
                ('media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/', 'video.corriere.it/vr360/videos/'),
                ('http://', 'https://'),
            ):
                url = url.replace(s, r)

            type_ = mimetype2ext(source['mimeType'])
            if type_ == 'm3u8' and '-vh.akamaihd' in url:
                # still needed for some old content: see _TESTS #3
                matches = re.search(r'(?:https?:)?//(?P<host>[\w\.\-]+)\.net/i(?P<path>.+)$', url)
                if matches:
                    url = f'https://vod.rcsobjects.it/hls/{self._MIGRATION_MAP[matches.group("host")]}{matches.group("path")}'
            if traverse_obj(video, ('mediaProfile', 'geoblocking')) or (
                    type_ == 'm3u8' and 'fcs.quotidiani_!' in url):
                url = url.replace('vod.rcsobjects', 'vod-it.rcsobjects')
            if type_ == 'm3u8' and 'vod' in url:
                url = url.replace('.csmil', '.urlset')
            if type_ == 'mp3':
                url = url.replace('media2vam-corriere-it.akamaized.net', 'vod.rcsobjects.it/corriere')

            yield {
                'type': type_,
                'url': url,
                'bitrate': source.get('bitrate')
            }

    def _create_http_formats(self, m3u8_formats, video_id):
        for f in m3u8_formats:
            if f['vcodec'] == 'none':
                continue
            http_url = re.sub(r'(https?://[^/]+)/hls/([^?#]+?\.mp4).+', r'\g<1>/\g<2>', f['url'])
            if http_url == f['url']:
                continue

            http_f = f.copy()
            del http_f['manifest_url']
            format_id = try_call(lambda: http_f['format_id'].replace('hls-', 'https-'))
            urlh = self._request_webpage(HEADRequest(http_url), video_id, fatal=False,
                                         note=f'Check filesize for {format_id}')
            if not urlh:
                continue

            http_f.update({
                'format_id': format_id,
                'url': http_url,
                'protocol': 'https',
                'filesize_approx': int_or_none(urlh.headers.get('Content-Length', None)),
            })
            yield http_f

    def _create_formats(self, sources, video_id):
        for source in sources:
            if source['type'] == 'm3u8':
                m3u8_formats = self._extract_m3u8_formats(
                    source['url'], video_id, 'mp4', m3u8_id='hls', fatal=False)
                yield from m3u8_formats
                yield from self._create_http_formats(m3u8_formats, video_id)
            elif source['type'] == 'mp3':
                yield {
                    'format_id': 'https-mp3',
                    'ext': 'mp3',
                    'acodec': 'mp3',
                    'vcodec': 'none',
                    'abr': source.get('bitrate'),
                    'url': source['url'],
                }

    def _real_extract(self, url):
        cdn, video_id = self._match_valid_url(url).group('cdn', 'id')
        display_id, video_data = None, None

        if re.match(self._UUID_RE, video_id) or re.match(self._RCS_ID_RE, video_id):
            url = f'https://video.{cdn}/video-json/{video_id}'
        else:
            webpage = self._download_webpage(url, video_id)
            data_config = get_element_html_by_id('divVideoPlayer', webpage) or get_element_html_by_class('divVideoPlayer', webpage)

            if data_config:
                data_config = self._parse_json(
                    extract_attributes(data_config).get('data-config'),
                    video_id, fatal=False) or {}
                if data_config.get('newspaper'):
                    cdn = f'{data_config["newspaper"]}.it'
                display_id, video_id = video_id, data_config.get('uuid') or video_id
                url = f'https://video.{cdn}/video-json/{video_id}'
            else:
                json_url = self._search_regex(
                    r'''(?x)url\s*=\s*(["'])
                    (?P<url>
                        (?:https?:)?//video\.rcs\.it
                        /fragment-includes/video-includes/[^"']+?\.json
                    )\1;''',
                    webpage, video_id, group='url', default=None)
                if json_url:
                    video_data = self._download_json(sanitize_url(json_url, scheme='https'), video_id)
                    display_id, video_id = video_id, video_data.get('id') or video_id

        if not video_data:
            webpage = self._download_webpage(url, video_id)

            video_data = self._search_json(
                '##start-video##', webpage, 'video data', video_id, default=None,
                end_pattern='##end-video##', transform_source=js_to_json)

            if not video_data:
                # try search for iframes
                emb = RCSEmbedsIE._extract_url(webpage)
                if emb:
                    return {
                        '_type': 'url_transparent',
                        'url': emb,
                        'ie_key': RCSEmbedsIE.ie_key()
                    }

        if not video_data:
            raise ExtractorError('Video data not found in the page')

        return {
            'id': video_id,
            'display_id': display_id,
            'title': video_data.get('title'),
            'description': (clean_html(video_data.get('description'))
                            or clean_html(video_data.get('htmlDescription'))
                            or self._html_search_meta('description', webpage)),
            'uploader': video_data.get('provider') or cdn,
            'formats': list(self._create_formats(self._get_video_src(video_data), video_id)),
        }


class RCSEmbedsIE(RCSBaseIE):
    _VALID_URL = r'''(?x)
                    https?://(?P<vid>video)\.
                    (?P<cdn>
                    (?:
                        rcs|
                        (?:corriere\w+\.)?corriere|
                        (?:gazzanet\.)?gazzetta
                    )\.it)
                    /video-embed/(?P<id>[^/=&\?]+?)(?:$|\?)'''
    _EMBED_REGEX = [r'''(?x)
            (?:
                data-frame-src=|
                <iframe[^\n]+src=
            )
            (["'])
                (?P<url>(?:https?:)?//video\.
                    (?:
                        rcs|
                        (?:corriere\w+\.)?corriere|
                        (?:gazzanet\.)?gazzetta
                    )
                \.it/video-embed/.+?)
            \1''']
    _TESTS = [{
        'url': 'https://video.rcs.it/video-embed/iodonna-0001585037',
        'md5': '0faca97df525032bb9847f690bc3720c',
        'info_dict': {
            'id': 'iodonna-0001585037',
            'ext': 'mp4',
            'title': 'Sky Arte racconta Madonna nella serie "Artist to icon"',
            'description': 'md5:65b09633df9ffee57f48b39e34c9e067',
            'uploader': 'rcs.it',
        }
    }, {
        'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789',
        'only_matching': True
    }, {
        'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140',
        'only_matching': True
    }]
    _WEBPAGE_TESTS = [{
        'url': 'https://www.iodonna.it/video-iodonna/personaggi-video/monica-bellucci-piu-del-lavoro-oggi-per-me-sono-importanti-lamicizia-e-la-famiglia/',
        'info_dict': {
            'id': 'iodonna-0002033648',
            'ext': 'mp4',
            'title': 'Monica Bellucci: «Più del lavoro, oggi per me sono importanti l\'amicizia e la famiglia»',
            'description': 'md5:daea6d9837351e56b1ab615c06bebac1',
            'uploader': 'rcs.it',
        }
    }]

    @staticmethod
    def _sanitize_url(url):
        url = sanitize_url(url, scheme='https')
        return urljoin(base_url(url), url_basename(url))

    @classmethod
    def _extract_embed_urls(cls, url, webpage):
        return map(cls._sanitize_url, super()._extract_embed_urls(url, webpage))


class RCSIE(RCSBaseIE):
    _VALID_URL = r'''(?x)https?://(?P<vid>video|viaggi)\.
                    (?P<cdn>
                    (?:
                        corrieredelmezzogiorno\.
                        |corrieredelveneto\.
                        |corrieredibologna\.
                        |corrierefiorentino\.
                    )?corriere\.it
                    |(?:gazzanet\.)?gazzetta\.it)
                    /(?!video-embed/)[^?#]+?/(?P<id>[^/\?]+)(?=\?|/$|$)'''
    _TESTS = [{
        # json iframe directly from id
        'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb',
        'md5': '14946840dec46ecfddf66ba4eea7d2b2',
        'info_dict': {
            'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb',
            'ext': 'mp4',
            'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante',
            'description': 'md5:3915ce5ebb3d2571deb69a5eb85ac9b5',
            'uploader': 'Corriere Tv',
        }
    }, {
        # search for video id inside the page
        'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/',
        'md5': 'f22a92d9e666e80f2fffbf2825359c81',
        'info_dict': {
            'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2',
            'display_id': 'norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen',
            'ext': 'mp4',
            'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen',
            'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8',
            'uploader': 'DOVE Viaggi',
        }
    }, {
        # only audio format https://github.com/yt-dlp/yt-dlp/issues/5683
        'url': 'https://video.corriere.it/cronaca/audio-telefonata-il-papa-becciu-santita-lettera-che-mi-ha-inviato-condanna/b94c0d20-70c2-11ed-9572-e4b947a0ebd2',
        'md5': 'aaffb08d02f2ce4292a4654694c78150',
        'info_dict': {
            'id': 'b94c0d20-70c2-11ed-9572-e4b947a0ebd2',
            'ext': 'mp3',
            'title': 'L\'audio della telefonata tra il Papa e Becciu: «Santità, la lettera che mi ha inviato è una condanna»',
            'description': 'md5:c0ddb61bd94a8d4e0d4bb9cda50a689b',
            'uploader': 'Corriere Tv',
            'formats': [{'format_id': 'https-mp3', 'ext': 'mp3'}],
        }
    }, {
        # old content still needs cdn migration
        'url': 'https://viaggi.corriere.it/video/milano-varallo-sesia-sul-treno-a-vapore/',
        'md5': '2dfdce7af249654ad27eeba03fe1e08d',
        'info_dict': {
            'id': 'd8f6c8d0-f7d7-11e8-bfca-f74cf4634191',
            'display_id': 'milano-varallo-sesia-sul-treno-a-vapore',
            'ext': 'mp4',
            'title': 'Milano-Varallo Sesia sul treno a vapore',
            'description': 'md5:6348f47aac230397fe341a74f7678d53',
            'uploader': 'DOVE Viaggi',
        }
    }, {
        'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945',
        'only_matching': True
    }]


class RCSVariousIE(RCSBaseIE):
    _VALID_URL = r'''(?x)https?://www\.
                    (?P<cdn>
                        leitv\.it|
                        youreporter\.it|
                        amica\.it
                    )/(?:[^/]+/)?(?P<id>[^/]+?)(?:$|\?|/)'''
    _TESTS = [{
        'url': 'https://www.leitv.it/benessere/mal-di-testa/',
        'md5': '3b7a683d105a7313ec7513b014443631',
        'info_dict': {
            'id': 'leitv-0000125151',
            'display_id': 'mal-di-testa',
            'ext': 'mp4',
            'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto',
            'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5',
            'uploader': 'leitv.it',
        }
    }, {
        'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/',
        'md5': '3989b6d603482611a2abd2f32b79f739',
        'info_dict': {
            'id': 'youreporter-0000332574',
            'display_id': 'fiume-sesia-3-ottobre-2020',
            'ext': 'mp4',
            'title': 'Fiume Sesia 3 ottobre 2020',
            'description': 'md5:0070eef1cc884d13c970a4125063de55',
            'uploader': 'youreporter.it',
        }
    }, {
        'url': 'https://www.amica.it/video-post/saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi/',
        'md5': '187cce524dfd0343c95646c047375fc4',
        'info_dict': {
            'id': 'amica-0001225365',
            'display_id': 'saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi',
            'ext': 'mp4',
            'title': '"Saint Omer": al cinema il film Leone d\'argento che ribalta gli stereotipi',
            'description': 'md5:b1c8869c2dcfd6073a2a311ba0008aa8',
            'uploader': 'rcs.it',
        }
    }]
Commit	Line	Data
a85e131b	1	import re
	2
	3	from .common import InfoExtractor
3d2623a8	4	from ..networking import HEADRequest
a85e131b	5	from ..utils import (
a85e131b	6	ExtractorError,
a85e131b	7	base_url,
304ad45a	8	clean_html,
c6b65786	9	extract_attributes,
	10	get_element_html_by_class,
	11	get_element_html_by_id,
	12	int_or_none,
304ad45a	13	js_to_json,
c6b65786	14	mimetype2ext,
	15	sanitize_url,
	16	traverse_obj,
	17	try_call,
a85e131b	18	url_basename,
	19	urljoin,
	20	)
	21
	22
5867a167	23	class RCSBaseIE(InfoExtractor):
18c1f043	24	# based on VideoPlayerLoader.prototype.getVideoSrc
	25	# and VideoPlayerLoader.prototype.transformSrc from
	26	# https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs
c6b65786	27	_UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
c6b65786	28	_RCS_ID_RE = r'[\w-]+-\d{10}'
a85e131b	29	_MIGRATION_MAP = {
	30	'videoamica-vh.akamaihd': 'amica',
	31	'media2-amica-it.akamaized': 'amica',
	32	'corrierevam-vh.akamaihd': 'corriere',
	33	'media2vam-corriere-it.akamaized': 'corriere',
	34	'cormezzogiorno-vh.akamaihd': 'corrieredelmezzogiorno',
	35	'media2vam-mezzogiorno-corriere-it.akamaized': 'corrieredelmezzogiorno',
	36	'corveneto-vh.akamaihd': 'corrieredelveneto',
	37	'media2vam-veneto-corriere-it.akamaized': 'corrieredelveneto',
	38	'corbologna-vh.akamaihd': 'corrieredibologna',
	39	'media2vam-bologna-corriere-it.akamaized': 'corrieredibologna',
	40	'corfiorentino-vh.akamaihd': 'corrierefiorentino',
	41	'media2vam-fiorentino-corriere-it.akamaized': 'corrierefiorentino',
	42	'corinnovazione-vh.akamaihd': 'corriereinnovazione',
	43	'media2-gazzanet-gazzetta-it.akamaized': 'gazzanet',
	44	'videogazzanet-vh.akamaihd': 'gazzanet',
	45	'videogazzaworld-vh.akamaihd': 'gazzaworld',
	46	'gazzettavam-vh.akamaihd': 'gazzetta',
	47	'media2vam-gazzetta-it.akamaized': 'gazzetta',
	48	'videoiodonna-vh.akamaihd': 'iodonna',
	49	'media2-leitv-it.akamaized': 'leitv',
	50	'videoleitv-vh.akamaihd': 'leitv',
	51	'videoliving-vh.akamaihd': 'living',
	52	'media2-living-corriere-it.akamaized': 'living',
	53	'media2-oggi-it.akamaized': 'oggi',
	54	'videooggi-vh.akamaihd': 'oggi',
	55	'media2-quimamme-it.akamaized': 'quimamme',
	56	'quimamme-vh.akamaihd': 'quimamme',
	57	'videorunning-vh.akamaihd': 'running',
	58	'media2-style-corriere-it.akamaized': 'style',
	59	'style-vh.akamaihd': 'style',
	60	'videostyle-vh.akamaihd': 'style',
	61	'media2-stylepiccoli-it.akamaized': 'stylepiccoli',
	62	'stylepiccoli-vh.akamaihd': 'stylepiccoli',
	63	'doveviaggi-vh.akamaihd': 'viaggi',
	64	'media2-doveviaggi-it.akamaized': 'viaggi',
	65	'media2-vivimilano-corriere-it.akamaized': 'vivimilano',
	66	'vivimilano-vh.akamaihd': 'vivimilano',
	67	'media2-youreporter-it.akamaized': 'youreporter'
	68	}
a85e131b	69
a85e131b	70	def _get_video_src(self, video):
c6b65786	71	for source in traverse_obj(video, (
	72	'mediaProfile', 'mediaFile', lambda _, v: v.get('mimeType'))):
	73	url = source['value']
	74	for s, r in (
	75	('media2vam.corriere.it.edgesuite.net', 'media2vam-corriere-it.akamaized.net'),
	76	('media.youreporter.it.edgesuite.net', 'media-youreporter-it.akamaized.net'),
	77	('corrierepmd.corriere.it.edgesuite.net', 'corrierepmd-corriere-it.akamaized.net'),
	78	('media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/', 'video.corriere.it/vr360/videos/'),
	79	('http://', 'https://'),
	80	):
	81	url = url.replace(s, r)
a85e131b	82
c6b65786	83	type_ = mimetype2ext(source['mimeType'])
	84	if type_ == 'm3u8' and '-vh.akamaihd' in url:
	85	# still needed for some old content: see _TESTS #3
	86	matches = re.search(r'(?:https?:)?//(?P<host>[\w\.\-]+)\.net/i(?P<path>.+)$', url)
	87	if matches:
	88	url = f'https://vod.rcsobjects.it/hls/{self._MIGRATION_MAP[matches.group("host")]}{matches.group("path")}'
	89	if traverse_obj(video, ('mediaProfile', 'geoblocking')) or (
	90	type_ == 'm3u8' and 'fcs.quotidiani_!' in url):
	91	url = url.replace('vod.rcsobjects', 'vod-it.rcsobjects')
	92	if type_ == 'm3u8' and 'vod' in url:
	93	url = url.replace('.csmil', '.urlset')
	94	if type_ == 'mp3':
	95	url = url.replace('media2vam-corriere-it.akamaized.net', 'vod.rcsobjects.it/corriere')
a85e131b	96
c6b65786	97	yield {
	98	'type': type_,
	99	'url': url,
	100	'bitrate': source.get('bitrate')
	101	}
a85e131b	102
c6b65786	103	def _create_http_formats(self, m3u8_formats, video_id):
	104	for f in m3u8_formats:
	105	if f['vcodec'] == 'none':
	106	continue
	107	http_url = re.sub(r'(https?://[^/]+)/hls/([^?#]+?\.mp4).+', r'\g<1>/\g<2>', f['url'])
	108	if http_url == f['url']:
	109	continue
a85e131b	110
c6b65786	111	http_f = f.copy()
	112	del http_f['manifest_url']
	113	format_id = try_call(lambda: http_f['format_id'].replace('hls-', 'https-'))
	114	urlh = self._request_webpage(HEADRequest(http_url), video_id, fatal=False,
	115	note=f'Check filesize for {format_id}')
	116	if not urlh:
	117	continue
a85e131b	118
c6b65786	119	http_f.update({
	120	'format_id': format_id,
	121	'url': http_url,
	122	'protocol': 'https',
	123	'filesize_approx': int_or_none(urlh.headers.get('Content-Length', None)),
a85e131b	124	})
c6b65786	125	yield http_f
	126
	127	def _create_formats(self, sources, video_id):
	128	for source in sources:
	129	if source['type'] == 'm3u8':
	130	m3u8_formats = self._extract_m3u8_formats(
	131	source['url'], video_id, 'mp4', m3u8_id='hls', fatal=False)
	132	yield from m3u8_formats
	133	yield from self._create_http_formats(m3u8_formats, video_id)
	134	elif source['type'] == 'mp3':
	135	yield {
	136	'format_id': 'https-mp3',
	137	'ext': 'mp3',
	138	'acodec': 'mp3',
	139	'vcodec': 'none',
	140	'abr': source.get('bitrate'),
	141	'url': source['url'],
	142	}
a85e131b	143
a85e131b	144	def _real_extract(self, url):
c6b65786	145	cdn, video_id = self._match_valid_url(url).group('cdn', 'id')
c6b65786	146	display_id, video_data = None, None
a85e131b	147
c6b65786	148	if re.match(self._UUID_RE, video_id) or re.match(self._RCS_ID_RE, video_id):
	149	url = f'https://video.{cdn}/video-json/{video_id}'
	150	else:
	151	webpage = self._download_webpage(url, video_id)
	152	data_config = get_element_html_by_id('divVideoPlayer', webpage) or get_element_html_by_class('divVideoPlayer', webpage)
a85e131b	153
c6b65786	154	if data_config:
	155	data_config = self._parse_json(
	156	extract_attributes(data_config).get('data-config'),
	157	video_id, fatal=False) or {}
	158	if data_config.get('newspaper'):
	159	cdn = f'{data_config["newspaper"]}.it'
	160	display_id, video_id = video_id, data_config.get('uuid') or video_id
	161	url = f'https://video.{cdn}/video-json/{video_id}'
	162	else:
	163	json_url = self._search_regex(
	164	r'''(?x)url\s=\s(["'])
	165	(?P<url>
	166	(?:https?:)?//video\.rcs\.it
	167	/fragment-includes/video-includes/[^"']+?\.json
	168	)\1;''',
	169	webpage, video_id, group='url', default=None)
	170	if json_url:
	171	video_data = self._download_json(sanitize_url(json_url, scheme='https'), video_id)
	172	display_id, video_id = video_id, video_data.get('id') or video_id
a85e131b	173
c6b65786	174	if not video_data:
c6b65786	175	webpage = self._download_webpage(url, video_id)
a85e131b	176
c6b65786	177	video_data = self._search_json(
	178	'##start-video##', webpage, 'video data', video_id, default=None,
	179	end_pattern='##end-video##', transform_source=js_to_json)
a85e131b	180
c6b65786	181	if not video_data:
	182	# try search for iframes
	183	emb = RCSEmbedsIE._extract_url(webpage)
a85e131b	184	if emb:
5867a167	185	return {
	186	'_type': 'url_transparent',
	187	'url': emb,
	188	'ie_key': RCSEmbedsIE.ie_key()
	189	}
a85e131b	190
	191	if not video_data:
	192	raise ExtractorError('Video data not found in the page')
	193
a85e131b	194	return {
a85e131b	195	'id': video_id,
c6b65786	196	'display_id': display_id,
508649e6	197	'title': video_data.get('title'),
c6b65786	198	'description': (clean_html(video_data.get('description'))
	199	or clean_html(video_data.get('htmlDescription'))
	200	or self._html_search_meta('description', webpage)),
	201	'uploader': video_data.get('provider') or cdn,
	202	'formats': list(self._create_formats(self._get_video_src(video_data), video_id)),
a85e131b	203	}
	204
	205
5867a167	206	class RCSEmbedsIE(RCSBaseIE):
a85e131b	207	_VALID_URL = r'''(?x)
	208	https?://(?P<vid>video)\.
	209	(?P<cdn>
	210	(?:
	211	rcs\|
	212	(?:corriere\w+\.)?corriere\|
	213	(?:gazzanet\.)?gazzetta
	214	)\.it)
	215	/video-embed/(?P<id>[^/=&\?]+?)(?:$\|\?)'''
bfd973ec	216	_EMBED_REGEX = [r'''(?x)
	217	(?:
	218	data-frame-src=\|
	219	<iframe[^\n]+src=
	220	)
	221	(["'])
	222	(?P<url>(?:https?:)?//video\.
	223	(?:
	224	rcs\|
	225	(?:corriere\w+\.)?corriere\|
	226	(?:gazzanet\.)?gazzetta
	227	)
	228	\.it/video-embed/.+?)
	229	\1''']
a85e131b	230	_TESTS = [{
a85e131b	231	'url': 'https://video.rcs.it/video-embed/iodonna-0001585037',
c6b65786	232	'md5': '0faca97df525032bb9847f690bc3720c',
a85e131b	233	'info_dict': {
	234	'id': 'iodonna-0001585037',
	235	'ext': 'mp4',
	236	'title': 'Sky Arte racconta Madonna nella serie "Artist to icon"',
	237	'description': 'md5:65b09633df9ffee57f48b39e34c9e067',
	238	'uploader': 'rcs.it',
	239	}
5867a167	240	}, {
5867a167	241	'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789',
5ca095cb	242	'only_matching': True
a85e131b	243	}, {
a85e131b	244	'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140',
5ca095cb	245	'only_matching': True
a85e131b	246	}]
c6b65786	247	_WEBPAGE_TESTS = [{
	248	'url': 'https://www.iodonna.it/video-iodonna/personaggi-video/monica-bellucci-piu-del-lavoro-oggi-per-me-sono-importanti-lamicizia-e-la-famiglia/',
	249	'info_dict': {
	250	'id': 'iodonna-0002033648',
	251	'ext': 'mp4',
	252	'title': 'Monica Bellucci: «Più del lavoro, oggi per me sono importanti l\'amicizia e la famiglia»',
	253	'description': 'md5:daea6d9837351e56b1ab615c06bebac1',
	254	'uploader': 'rcs.it',
	255	}
	256	}]
a85e131b	257
a85e131b	258	@staticmethod
c6b65786	259	def _sanitize_url(url):
	260	url = sanitize_url(url, scheme='https')
	261	return urljoin(base_url(url), url_basename(url))
a85e131b	262
bfd973ec	263	@classmethod
bfd973ec	264	def _extract_embed_urls(cls, url, webpage):
c6b65786	265	return map(cls._sanitize_url, super()._extract_embed_urls(url, webpage))
a85e131b	266
a85e131b	267
5867a167	268	class RCSIE(RCSBaseIE):
a85e131b	269	_VALID_URL = r'''(?x)https?://(?P<vid>video\|viaggi)\.
	270	(?P<cdn>
	271	(?:
5867a167	272	corrieredelmezzogiorno\.
	273	\|corrieredelveneto\.
	274	\|corrieredibologna\.
	275	\|corrierefiorentino\.
	276	)?corriere\.it
	277	\|(?:gazzanet\.)?gazzetta\.it)
c6b65786	278	/(?!video-embed/)[^?#]+?/(?P<id>[^/\?]+)(?=\?\|/$\|$)'''
a85e131b	279	_TESTS = [{
c6b65786	280	# json iframe directly from id
a85e131b	281	'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb',
c6b65786	282	'md5': '14946840dec46ecfddf66ba4eea7d2b2',
a85e131b	283	'info_dict': {
	284	'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb',
	285	'ext': 'mp4',
	286	'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante',
c6b65786	287	'description': 'md5:3915ce5ebb3d2571deb69a5eb85ac9b5',
a85e131b	288	'uploader': 'Corriere Tv',
	289	}
	290	}, {
c6b65786	291	# search for video id inside the page
a85e131b	292	'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/',
c6b65786	293	'md5': 'f22a92d9e666e80f2fffbf2825359c81',
a85e131b	294	'info_dict': {
a85e131b	295	'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2',
c6b65786	296	'display_id': 'norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen',
a85e131b	297	'ext': 'mp4',
	298	'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen',
	299	'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8',
	300	'uploader': 'DOVE Viaggi',
	301	}
	302	}, {
c6b65786	303	# only audio format https://github.com/yt-dlp/yt-dlp/issues/5683
	304	'url': 'https://video.corriere.it/cronaca/audio-telefonata-il-papa-becciu-santita-lettera-che-mi-ha-inviato-condanna/b94c0d20-70c2-11ed-9572-e4b947a0ebd2',
	305	'md5': 'aaffb08d02f2ce4292a4654694c78150',
	306	'info_dict': {
	307	'id': 'b94c0d20-70c2-11ed-9572-e4b947a0ebd2',
	308	'ext': 'mp3',
	309	'title': 'L\'audio della telefonata tra il Papa e Becciu: «Santità, la lettera che mi ha inviato è una condanna»',
	310	'description': 'md5:c0ddb61bd94a8d4e0d4bb9cda50a689b',
	311	'uploader': 'Corriere Tv',
	312	'formats': [{'format_id': 'https-mp3', 'ext': 'mp3'}],
	313	}
	314	}, {
	315	# old content still needs cdn migration
	316	'url': 'https://viaggi.corriere.it/video/milano-varallo-sesia-sul-treno-a-vapore/',
	317	'md5': '2dfdce7af249654ad27eeba03fe1e08d',
a85e131b	318	'info_dict': {
c6b65786	319	'id': 'd8f6c8d0-f7d7-11e8-bfca-f74cf4634191',
c6b65786	320	'display_id': 'milano-varallo-sesia-sul-treno-a-vapore',
a85e131b	321	'ext': 'mp4',
c6b65786	322	'title': 'Milano-Varallo Sesia sul treno a vapore',
	323	'description': 'md5:6348f47aac230397fe341a74f7678d53',
	324	'uploader': 'DOVE Viaggi',
a85e131b	325	}
a85e131b	326	}, {
5867a167	327	'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945',
5ca095cb	328	'only_matching': True
a85e131b	329	}]
	330
	331
5867a167	332	class RCSVariousIE(RCSBaseIE):
a85e131b	333	_VALID_URL = r'''(?x)https?://www\.
	334	(?P<cdn>
	335	leitv\.it\|
c6b65786	336	youreporter\.it\|
c6b65786	337	amica\.it
18c1f043	338	)/(?:[^/]+/)?(?P<id>[^/]+?)(?:$\|\?\|/)'''
a85e131b	339	_TESTS = [{
c6b65786	340	'url': 'https://www.leitv.it/benessere/mal-di-testa/',
c6b65786	341	'md5': '3b7a683d105a7313ec7513b014443631',
a85e131b	342	'info_dict': {
c6b65786	343	'id': 'leitv-0000125151',
c6b65786	344	'display_id': 'mal-di-testa',
a85e131b	345	'ext': 'mp4',
18c1f043	346	'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto',
18c1f043	347	'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5',
a85e131b	348	'uploader': 'leitv.it',
	349	}
	350	}, {
	351	'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/',
c6b65786	352	'md5': '3989b6d603482611a2abd2f32b79f739',
a85e131b	353	'info_dict': {
c6b65786	354	'id': 'youreporter-0000332574',
c6b65786	355	'display_id': 'fiume-sesia-3-ottobre-2020',
a85e131b	356	'ext': 'mp4',
	357	'title': 'Fiume Sesia 3 ottobre 2020',
	358	'description': 'md5:0070eef1cc884d13c970a4125063de55',
	359	'uploader': 'youreporter.it',
	360	}
c6b65786	361	}, {
	362	'url': 'https://www.amica.it/video-post/saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi/',
	363	'md5': '187cce524dfd0343c95646c047375fc4',
	364	'info_dict': {
	365	'id': 'amica-0001225365',
	366	'display_id': 'saint-omer-al-cinema-il-film-leone-dargento-che-ribalta-gli-stereotipi',
	367	'ext': 'mp4',
	368	'title': '"Saint Omer": al cinema il film Leone d\'argento che ribalta gli stereotipi',
	369	'description': 'md5:b1c8869c2dcfd6073a2a311ba0008aa8',
	370	'uploader': 'rcs.it',
	371	}
a85e131b	372	}]