[yt-dlp.git] / yt_dlp / extractor / rcs.py

# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    ExtractorError,
    js_to_json,
    base_url,
    url_basename,
    urljoin,
)


class RCSBaseIE(InfoExtractor):
    # based on VideoPlayerLoader.prototype.getVideoSrc
    # and VideoPlayerLoader.prototype.transformSrc from
    # https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs
    _ALL_REPLACE = {
        'media2vam.corriere.it.edgesuite.net':
            'media2vam-corriere-it.akamaized.net',
        'media.youreporter.it.edgesuite.net':
            'media-youreporter-it.akamaized.net',
        'corrierepmd.corriere.it.edgesuite.net':
            'corrierepmd-corriere-it.akamaized.net',
        'media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/':
            'video.corriere.it/vr360/videos/',
        '.net//': '.net/',
    }
    _MP4_REPLACE = {
        'media2vam.corbologna.corriere.it.edgesuite.net':
            'media2vam-bologna-corriere-it.akamaized.net',
        'media2vam.corfiorentino.corriere.it.edgesuite.net':
            'media2vam-fiorentino-corriere-it.akamaized.net',
        'media2vam.cormezzogiorno.corriere.it.edgesuite.net':
            'media2vam-mezzogiorno-corriere-it.akamaized.net',
        'media2vam.corveneto.corriere.it.edgesuite.net':
            'media2vam-veneto-corriere-it.akamaized.net',
        'media2.oggi.it.edgesuite.net':
            'media2-oggi-it.akamaized.net',
        'media2.quimamme.it.edgesuite.net':
            'media2-quimamme-it.akamaized.net',
        'media2.amica.it.edgesuite.net':
            'media2-amica-it.akamaized.net',
        'media2.living.corriere.it.edgesuite.net':
            'media2-living-corriere-it.akamaized.net',
        'media2.style.corriere.it.edgesuite.net':
            'media2-style-corriere-it.akamaized.net',
        'media2.iodonna.it.edgesuite.net':
            'media2-iodonna-it.akamaized.net',
        'media2.leitv.it.edgesuite.net':
            'media2-leitv-it.akamaized.net',
    }
    _MIGRATION_MAP = {
        'videoamica-vh.akamaihd': 'amica',
        'media2-amica-it.akamaized': 'amica',
        'corrierevam-vh.akamaihd': 'corriere',
        'media2vam-corriere-it.akamaized': 'corriere',
        'cormezzogiorno-vh.akamaihd': 'corrieredelmezzogiorno',
        'media2vam-mezzogiorno-corriere-it.akamaized': 'corrieredelmezzogiorno',
        'corveneto-vh.akamaihd': 'corrieredelveneto',
        'media2vam-veneto-corriere-it.akamaized': 'corrieredelveneto',
        'corbologna-vh.akamaihd': 'corrieredibologna',
        'media2vam-bologna-corriere-it.akamaized': 'corrieredibologna',
        'corfiorentino-vh.akamaihd': 'corrierefiorentino',
        'media2vam-fiorentino-corriere-it.akamaized': 'corrierefiorentino',
        'corinnovazione-vh.akamaihd': 'corriereinnovazione',
        'media2-gazzanet-gazzetta-it.akamaized': 'gazzanet',
        'videogazzanet-vh.akamaihd': 'gazzanet',
        'videogazzaworld-vh.akamaihd': 'gazzaworld',
        'gazzettavam-vh.akamaihd': 'gazzetta',
        'media2vam-gazzetta-it.akamaized': 'gazzetta',
        'videoiodonna-vh.akamaihd': 'iodonna',
        'media2-leitv-it.akamaized': 'leitv',
        'videoleitv-vh.akamaihd': 'leitv',
        'videoliving-vh.akamaihd': 'living',
        'media2-living-corriere-it.akamaized': 'living',
        'media2-oggi-it.akamaized': 'oggi',
        'videooggi-vh.akamaihd': 'oggi',
        'media2-quimamme-it.akamaized': 'quimamme',
        'quimamme-vh.akamaihd': 'quimamme',
        'videorunning-vh.akamaihd': 'running',
        'media2-style-corriere-it.akamaized': 'style',
        'style-vh.akamaihd': 'style',
        'videostyle-vh.akamaihd': 'style',
        'media2-stylepiccoli-it.akamaized': 'stylepiccoli',
        'stylepiccoli-vh.akamaihd': 'stylepiccoli',
        'doveviaggi-vh.akamaihd': 'viaggi',
        'media2-doveviaggi-it.akamaized': 'viaggi',
        'media2-vivimilano-corriere-it.akamaized': 'vivimilano',
        'vivimilano-vh.akamaihd': 'vivimilano',
        'media2-youreporter-it.akamaized': 'youreporter'
    }
    _MIGRATION_MEDIA = {
        'advrcs-vh.akamaihd': '',
        'corriere-f.akamaihd': '',
        'corrierepmd-corriere-it.akamaized': '',
        'corrprotetto-vh.akamaihd': '',
        'gazzetta-f.akamaihd': '',
        'gazzettapmd-gazzetta-it.akamaized': '',
        'gazzprotetto-vh.akamaihd': '',
        'periodici-f.akamaihd': '',
        'periodicisecure-vh.akamaihd': '',
        'videocoracademy-vh.akamaihd': ''
    }

    def _get_video_src(self, video):
        mediaFiles = video.get('mediaProfile').get('mediaFile')
        src = {}
        # audio
        if video.get('mediaType') == 'AUDIO':
            for aud in mediaFiles:
                # todo: check
                src['mp3'] = aud.get('value')
        # video
        else:
            for vid in mediaFiles:
                if vid.get('mimeType') == 'application/vnd.apple.mpegurl':
                    src['m3u8'] = vid.get('value')
                if vid.get('mimeType') == 'video/mp4':
                    src['mp4'] = vid.get('value')

        # replace host
        for t in src:
            for s, r in self._ALL_REPLACE.items():
                src[t] = src[t].replace(s, r)
            for s, r in self._MP4_REPLACE.items():
                src[t] = src[t].replace(s, r)

        # switch cdn
        if 'mp4' in src and 'm3u8' in src:
            if ('-lh.akamaihd' not in src.get('m3u8')
                    and 'akamai' in src.get('mp4')):
                if 'm3u8' in src:
                    matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('m3u8'))
                    src['m3u8'] = 'https://vod.rcsobjects.it/hls/%s%s' % (
                        self._MIGRATION_MAP[matches.group('host')],
                        matches.group('path').replace(
                            '///', '/').replace(
                            '//', '/').replace(
                            '.csmil', '.urlset'
                        )
                    )
                if 'mp4' in src:
                    matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('mp4'))
                    if matches:
                        if matches.group('host') in self._MIGRATION_MEDIA:
                            vh_stream = 'https://media2.corriereobjects.it'
                            if src.get('mp4').find('fcs.quotidiani_!'):
                                vh_stream = 'https://media2-it.corriereobjects.it'
                            src['mp4'] = '%s%s' % (
                                vh_stream,
                                matches.group('path').replace(
                                    '///', '/').replace(
                                    '//', '/').replace(
                                    '/fcs.quotidiani/mediacenter', '').replace(
                                    '/fcs.quotidiani_!/mediacenter', '').replace(
                                    'corriere/content/mediacenter/', '').replace(
                                    'gazzetta/content/mediacenter/', '')
                            )
                        else:
                            src['mp4'] = 'https://vod.rcsobjects.it/%s%s' % (
                                self._MIGRATION_MAP[matches.group('host')],
                                matches.group('path').replace('///', '/').replace('//', '/')
                            )

        if 'mp3' in src:
            src['mp3'] = src.get('mp3').replace(
                'media2vam-corriere-it.akamaized.net',
                'vod.rcsobjects.it/corriere')
        if 'mp4' in src:
            if src.get('mp4').find('fcs.quotidiani_!'):
                src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
        if 'm3u8' in src:
            if src.get('m3u8').find('fcs.quotidiani_!'):
                src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')

        if 'geoblocking' in video.get('mediaProfile'):
            if 'm3u8' in src:
                src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')
            if 'mp4' in src:
                src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
        if 'm3u8' in src:
            if src.get('m3u8').find('csmil') and src.get('m3u8').find('vod'):
                src['m3u8'] = src.get('m3u8').replace('.csmil', '.urlset')

        return src

    def _create_formats(self, urls, video_id):
        formats = []
        formats = self._extract_m3u8_formats(
            urls.get('m3u8'), video_id, 'mp4', entry_protocol='m3u8_native',
            m3u8_id='hls', fatal=False)

        if urls.get('mp4'):
            formats.append({
                'format_id': 'http-mp4',
                'url': urls['mp4']
            })
        self._sort_formats(formats)
        return formats

    def _real_extract(self, url):
        video_id = self._match_id(url)
        mobj = re.search(self._VALID_URL, url)

        if 'cdn' not in mobj.groupdict():
            raise ExtractorError('CDN not found in url: %s' % url)

        # for leitv/youreporter/viaggi don't use the embed page
        if ((mobj.group('cdn') not in ['leitv.it', 'youreporter.it'])
                and (mobj.group('vid') == 'video')):
            url = 'https://video.%s/video-embed/%s' % (mobj.group('cdn'), video_id)

        page = self._download_webpage(url, video_id)

        video_data = None
        # look for json video data url
        json = self._search_regex(
            r'''(?x)url\s*=\s*(["'])
            (?P<url>
                (?:https?:)?//video\.rcs\.it
                /fragment-includes/video-includes/.+?\.json
            )\1;''',
            page, video_id, group='url', default=None)
        if json:
            if json.startswith('//'):
                json = 'https:%s' % json
            video_data = self._download_json(json, video_id)

        # if json url not found, look for json video data directly in the page
        else:
            # RCS normal pages and most of the embeds
            json = self._search_regex(
                r'[\s;]video\s*=\s*({[\s\S]+?})(?:;|,playlist=)',
                page, video_id, default=None)
            if not json and 'video-embed' in url:
                page = self._download_webpage(url.replace('video-embed', 'video-json'), video_id)
                json = self._search_regex(
                    r'##start-video##({[\s\S]+?})##end-video##',
                    page, video_id, default=None)
            if not json:
                # if no video data found try search for iframes
                emb = RCSEmbedsIE._extract_url(page)
                if emb:
                    return {
                        '_type': 'url_transparent',
                        'url': emb,
                        'ie_key': RCSEmbedsIE.ie_key()
                    }
            if json:
                video_data = self._parse_json(
                    json, video_id, transform_source=js_to_json)

        if not video_data:
            raise ExtractorError('Video data not found in the page')

        formats = self._create_formats(
            self._get_video_src(video_data), video_id)

        description = (video_data.get('description')
                       or clean_html(video_data.get('htmlDescription'))
                       or self._html_search_meta('description', page))
        uploader = video_data.get('provider') or mobj.group('cdn')

        return {
            'id': video_id,
            'title': video_data.get('title'),
            'description': description,
            'uploader': uploader,
            'formats': formats
        }


class RCSEmbedsIE(RCSBaseIE):
    _VALID_URL = r'''(?x)
                    https?://(?P<vid>video)\.
                    (?P<cdn>
                    (?:
                        rcs|
                        (?:corriere\w+\.)?corriere|
                        (?:gazzanet\.)?gazzetta
                    )\.it)
                    /video-embed/(?P<id>[^/=&\?]+?)(?:$|\?)'''
    _TESTS = [{
        'url': 'https://video.rcs.it/video-embed/iodonna-0001585037',
        'md5': '623ecc8ffe7299b2d0c1046d8331a9df',
        'info_dict': {
            'id': 'iodonna-0001585037',
            'ext': 'mp4',
            'title': 'Sky Arte racconta Madonna nella serie "Artist to icon"',
            'description': 'md5:65b09633df9ffee57f48b39e34c9e067',
            'uploader': 'rcs.it',
        }
    }, {
        # redownload the page changing 'video-embed' in 'video-json'
        'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789',
        'md5': 'a043e3fecbe4d9ed7fc5d888652a5440',
        'info_dict': {
            'id': 'gazzanet-mo05-0000260789',
            'ext': 'mp4',
            'title': 'Valentino Rossi e papà Graziano si divertono col drifting',
            'description': 'md5:a8bf90d6adafd9815f70fc74c0fc370a',
            'uploader': 'rcd',
        }
    }, {
        'url': 'https://video.corriere.it/video-embed/b727632a-f9d0-11ea-91b0-38d50a849abb?player',
        'match_only': True
    }, {
        'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140',
        'match_only': True
    }]

    @staticmethod
    def _sanitize_urls(urls):
        # add protocol if missing
        for i, e in enumerate(urls):
            if e.startswith('//'):
                urls[i] = 'https:%s' % e
        # clean iframes urls
        for i, e in enumerate(urls):
            urls[i] = urljoin(base_url(e), url_basename(e))
        return urls

    @staticmethod
    def _extract_urls(webpage):
        entries = [
            mobj.group('url')
            for mobj in re.finditer(r'''(?x)
            (?:
                data-frame-src=|
                <iframe[^\n]+src=
            )
            (["'])
                (?P<url>(?:https?:)?//video\.
                    (?:
                        rcs|
                        (?:corriere\w+\.)?corriere|
                        (?:gazzanet\.)?gazzetta
                    )
                \.it/video-embed/.+?)
            \1''', webpage)]
        return RCSEmbedsIE._sanitize_urls(entries)

    @staticmethod
    def _extract_url(webpage):
        urls = RCSEmbedsIE._extract_urls(webpage)
        return urls[0] if urls else None


class RCSIE(RCSBaseIE):
    _VALID_URL = r'''(?x)https?://(?P<vid>video|viaggi)\.
                    (?P<cdn>
                    (?:
                        corrieredelmezzogiorno\.
                        |corrieredelveneto\.
                        |corrieredibologna\.
                        |corrierefiorentino\.
                    )?corriere\.it
                    |(?:gazzanet\.)?gazzetta\.it)
                    /(?!video-embed/).+?/(?P<id>[^/\?]+)(?=\?|/$|$)'''
    _TESTS = [{
        'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb',
        'md5': '0f4ededc202b0f00b6e509d831e2dcda',
        'info_dict': {
            'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb',
            'ext': 'mp4',
            'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante',
            'description': 'md5:93b51c9161ac8a64fb2f997b054d0152',
            'uploader': 'Corriere Tv',
        }
    }, {
        # video data inside iframe
        'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/',
        'md5': 'da378e4918d2afbf7d61c35abb948d4c',
        'info_dict': {
            'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2',
            'ext': 'mp4',
            'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen',
            'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8',
            'uploader': 'DOVE Viaggi',
        }
    }, {
        'url': 'https://video.gazzetta.it/video-motogp-catalogna-cadute-dovizioso-vale-rossi/49612410-00ca-11eb-bcd8-30d4253e0140?vclk=Videobar',
        'md5': 'eedc1b5defd18e67383afef51ff7bdf9',
        'info_dict': {
            'id': '49612410-00ca-11eb-bcd8-30d4253e0140',
            'ext': 'mp4',
            'title': 'Dovizioso, il contatto con Zarco e la caduta. E anche Vale finisce a terra',
            'description': 'md5:8c6e905dc3b9413218beca11ebd69778',
            'uploader': 'AMorici',
        }
    }, {
        'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945',
        'match_only': True
    }]


class RCSVariousIE(RCSBaseIE):
    _VALID_URL = r'''(?x)https?://www\.
                    (?P<cdn>
                        leitv\.it|
                        youreporter\.it
                    )/(?:[^/]+/)?(?P<id>[^/]+?)(?:$|\?|/)'''
    _TESTS = [{
        'url': 'https://www.leitv.it/benessere/mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa/',
        'md5': '92b4e63667b8f95acb0a04da25ae28a1',
        'info_dict': {
            'id': 'mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa',
            'ext': 'mp4',
            'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto',
            'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5',
            'uploader': 'leitv.it',
        }
    }, {
        'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/',
        'md5': '8dccd436b47a830bab5b4a88232f391a',
        'info_dict': {
            'id': 'fiume-sesia-3-ottobre-2020',
            'ext': 'mp4',
            'title': 'Fiume Sesia 3 ottobre 2020',
            'description': 'md5:0070eef1cc884d13c970a4125063de55',
            'uploader': 'youreporter.it',
        }
    }]
Commit	Line	Data
a85e131b	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
	4	import re
	5
	6	from .common import InfoExtractor
	7	from ..utils import (
	8	clean_html,
	9	ExtractorError,
	10	js_to_json,
	11	base_url,
	12	url_basename,
	13	urljoin,
	14	)
	15
	16
5867a167	17	class RCSBaseIE(InfoExtractor):
18c1f043	18	# based on VideoPlayerLoader.prototype.getVideoSrc
	19	# and VideoPlayerLoader.prototype.transformSrc from
	20	# https://js2.corriereobjects.it/includes2013/LIBS/js/corriere_video.sjs
a85e131b	21	_ALL_REPLACE = {
	22	'media2vam.corriere.it.edgesuite.net':
	23	'media2vam-corriere-it.akamaized.net',
	24	'media.youreporter.it.edgesuite.net':
	25	'media-youreporter-it.akamaized.net',
	26	'corrierepmd.corriere.it.edgesuite.net':
	27	'corrierepmd-corriere-it.akamaized.net',
	28	'media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/':
	29	'video.corriere.it/vr360/videos/',
	30	'.net//': '.net/',
	31	}
	32	_MP4_REPLACE = {
	33	'media2vam.corbologna.corriere.it.edgesuite.net':
	34	'media2vam-bologna-corriere-it.akamaized.net',
	35	'media2vam.corfiorentino.corriere.it.edgesuite.net':
	36	'media2vam-fiorentino-corriere-it.akamaized.net',
	37	'media2vam.cormezzogiorno.corriere.it.edgesuite.net':
	38	'media2vam-mezzogiorno-corriere-it.akamaized.net',
	39	'media2vam.corveneto.corriere.it.edgesuite.net':
	40	'media2vam-veneto-corriere-it.akamaized.net',
	41	'media2.oggi.it.edgesuite.net':
	42	'media2-oggi-it.akamaized.net',
	43	'media2.quimamme.it.edgesuite.net':
	44	'media2-quimamme-it.akamaized.net',
	45	'media2.amica.it.edgesuite.net':
	46	'media2-amica-it.akamaized.net',
	47	'media2.living.corriere.it.edgesuite.net':
	48	'media2-living-corriere-it.akamaized.net',
	49	'media2.style.corriere.it.edgesuite.net':
	50	'media2-style-corriere-it.akamaized.net',
	51	'media2.iodonna.it.edgesuite.net':
	52	'media2-iodonna-it.akamaized.net',
	53	'media2.leitv.it.edgesuite.net':
	54	'media2-leitv-it.akamaized.net',
	55	}
	56	_MIGRATION_MAP = {
	57	'videoamica-vh.akamaihd': 'amica',
	58	'media2-amica-it.akamaized': 'amica',
	59	'corrierevam-vh.akamaihd': 'corriere',
	60	'media2vam-corriere-it.akamaized': 'corriere',
	61	'cormezzogiorno-vh.akamaihd': 'corrieredelmezzogiorno',
	62	'media2vam-mezzogiorno-corriere-it.akamaized': 'corrieredelmezzogiorno',
	63	'corveneto-vh.akamaihd': 'corrieredelveneto',
	64	'media2vam-veneto-corriere-it.akamaized': 'corrieredelveneto',
	65	'corbologna-vh.akamaihd': 'corrieredibologna',
	66	'media2vam-bologna-corriere-it.akamaized': 'corrieredibologna',
	67	'corfiorentino-vh.akamaihd': 'corrierefiorentino',
	68	'media2vam-fiorentino-corriere-it.akamaized': 'corrierefiorentino',
	69	'corinnovazione-vh.akamaihd': 'corriereinnovazione',
	70	'media2-gazzanet-gazzetta-it.akamaized': 'gazzanet',
	71	'videogazzanet-vh.akamaihd': 'gazzanet',
	72	'videogazzaworld-vh.akamaihd': 'gazzaworld',
	73	'gazzettavam-vh.akamaihd': 'gazzetta',
	74	'media2vam-gazzetta-it.akamaized': 'gazzetta',
	75	'videoiodonna-vh.akamaihd': 'iodonna',
	76	'media2-leitv-it.akamaized': 'leitv',
	77	'videoleitv-vh.akamaihd': 'leitv',
	78	'videoliving-vh.akamaihd': 'living',
	79	'media2-living-corriere-it.akamaized': 'living',
	80	'media2-oggi-it.akamaized': 'oggi',
	81	'videooggi-vh.akamaihd': 'oggi',
	82	'media2-quimamme-it.akamaized': 'quimamme',
	83	'quimamme-vh.akamaihd': 'quimamme',
	84	'videorunning-vh.akamaihd': 'running',
85	'media2-style-corriere-it.akamaized': 'style',
86	'style-vh.akamaihd': 'style',
87	'videostyle-vh.akamaihd': 'style',
88	'media2-stylepiccoli-it.akamaized': 'stylepiccoli',
89	'stylepiccoli-vh.akamaihd': 'stylepiccoli',
90	'doveviaggi-vh.akamaihd': 'viaggi',
91	'media2-doveviaggi-it.akamaized': 'viaggi',
92	'media2-vivimilano-corriere-it.akamaized': 'vivimilano',
93	'vivimilano-vh.akamaihd': 'vivimilano',
94	'media2-youreporter-it.akamaized': 'youreporter'
95	}
96	_MIGRATION_MEDIA = {
97	'advrcs-vh.akamaihd': '',
98	'corriere-f.akamaihd': '',
99	'corrierepmd-corriere-it.akamaized': '',
100	'corrprotetto-vh.akamaihd': '',
101	'gazzetta-f.akamaihd': '',
102	'gazzettapmd-gazzetta-it.akamaized': '',
103	'gazzprotetto-vh.akamaihd': '',
104	'periodici-f.akamaihd': '',
105	'periodicisecure-vh.akamaihd': '',
106	'videocoracademy-vh.akamaihd': ''
107	}
108
109	def _get_video_src(self, video):
508649e6	110	mediaFiles = video.get('mediaProfile').get('mediaFile')
a85e131b	111	src = {}
a85e131b	112	# audio
508649e6	113	if video.get('mediaType') == 'AUDIO':
a85e131b	114	for aud in mediaFiles:
a85e131b	115	# todo: check
508649e6	116	src['mp3'] = aud.get('value')
a85e131b	117	# video
	118	else:
	119	for vid in mediaFiles:
508649e6	120	if vid.get('mimeType') == 'application/vnd.apple.mpegurl':
	121	src['m3u8'] = vid.get('value')
	122	if vid.get('mimeType') == 'video/mp4':
	123	src['mp4'] = vid.get('value')
a85e131b	124
	125	# replace host
	126	for t in src:
	127	for s, r in self._ALL_REPLACE.items():
	128	src[t] = src[t].replace(s, r)
	129	for s, r in self._MP4_REPLACE.items():
	130	src[t] = src[t].replace(s, r)
	131
	132	# switch cdn
	133	if 'mp4' in src and 'm3u8' in src:
508649e6	134	if ('-lh.akamaihd' not in src.get('m3u8')
508649e6	135	and 'akamai' in src.get('mp4')):
a85e131b	136	if 'm3u8' in src:
508649e6	137	matches = re.search(r'(?:https:)?\/\/(?P<host>.)\.net\/i(?P<path>.*)$', src.get('m3u8'))
a85e131b	138	src['m3u8'] = 'https://vod.rcsobjects.it/hls/%s%s' % (
	139	self._MIGRATION_MAP[matches.group('host')],
	140	matches.group('path').replace(
	141	'///', '/').replace(
	142	'//', '/').replace(
	143	'.csmil', '.urlset'
	144	)
	145	)
	146	if 'mp4' in src:
508649e6	147	matches = re.search(r'(?:https:)?\/\/(?P<host>.)\.net\/i(?P<path>.*)$', src.get('mp4'))
a85e131b	148	if matches:
	149	if matches.group('host') in self._MIGRATION_MEDIA:
	150	vh_stream = 'https://media2.corriereobjects.it'
508649e6	151	if src.get('mp4').find('fcs.quotidiani_!'):
a85e131b	152	vh_stream = 'https://media2-it.corriereobjects.it'
	153	src['mp4'] = '%s%s' % (
	154	vh_stream,
	155	matches.group('path').replace(
	156	'///', '/').replace(
	157	'//', '/').replace(
	158	'/fcs.quotidiani/mediacenter', '').replace(
	159	'/fcs.quotidiani_!/mediacenter', '').replace(
	160	'corriere/content/mediacenter/', '').replace(
	161	'gazzetta/content/mediacenter/', '')
	162	)
	163	else:
	164	src['mp4'] = 'https://vod.rcsobjects.it/%s%s' % (
	165	self._MIGRATION_MAP[matches.group('host')],
	166	matches.group('path').replace('///', '/').replace('//', '/')
	167	)
	168
	169	if 'mp3' in src:
508649e6	170	src['mp3'] = src.get('mp3').replace(
a85e131b	171	'media2vam-corriere-it.akamaized.net',
	172	'vod.rcsobjects.it/corriere')
	173	if 'mp4' in src:
508649e6	174	if src.get('mp4').find('fcs.quotidiani_!'):
508649e6	175	src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
a85e131b	176	if 'm3u8' in src:
508649e6	177	if src.get('m3u8').find('fcs.quotidiani_!'):
508649e6	178	src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')
a85e131b	179
508649e6	180	if 'geoblocking' in video.get('mediaProfile'):
a85e131b	181	if 'm3u8' in src:
508649e6	182	src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')
a85e131b	183	if 'mp4' in src:
508649e6	184	src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
a85e131b	185	if 'm3u8' in src:
508649e6	186	if src.get('m3u8').find('csmil') and src.get('m3u8').find('vod'):
508649e6	187	src['m3u8'] = src.get('m3u8').replace('.csmil', '.urlset')
a85e131b	188
	189	return src
	190
	191	def _create_formats(self, urls, video_id):
	192	formats = []
	193	formats = self._extract_m3u8_formats(
508649e6	194	urls.get('m3u8'), video_id, 'mp4', entry_protocol='m3u8_native',
a85e131b	195	m3u8_id='hls', fatal=False)
a85e131b	196
18c1f043	197	if urls.get('mp4'):
a85e131b	198	formats.append({
a85e131b	199	'format_id': 'http-mp4',
18c1f043	200	'url': urls['mp4']
a85e131b	201	})
	202	self._sort_formats(formats)
	203	return formats
	204
	205	def _real_extract(self, url):
	206	video_id = self._match_id(url)
508649e6	207	mobj = re.search(self._VALID_URL, url)
a85e131b	208
508649e6	209	if 'cdn' not in mobj.groupdict():
a85e131b	210	raise ExtractorError('CDN not found in url: %s' % url)
	211
	212	# for leitv/youreporter/viaggi don't use the embed page
508649e6	213	if ((mobj.group('cdn') not in ['leitv.it', 'youreporter.it'])
	214	and (mobj.group('vid') == 'video')):
	215	url = 'https://video.%s/video-embed/%s' % (mobj.group('cdn'), video_id)
a85e131b	216
	217	page = self._download_webpage(url, video_id)
	218
	219	video_data = None
	220	# look for json video data url
	221	json = self._search_regex(
18c1f043	222	r'''(?x)url\s=\s(["'])
	223	(?P<url>
	224	(?:https?:)?//video\.rcs\.it
	225	/fragment-includes/video-includes/.+?\.json
	226	)\1;''',
	227	page, video_id, group='url', default=None)
a85e131b	228	if json:
	229	if json.startswith('//'):
	230	json = 'https:%s' % json
	231	video_data = self._download_json(json, video_id)
	232
508649e6	233	# if json url not found, look for json video data directly in the page
a85e131b	234	else:
18c1f043	235	# RCS normal pages and most of the embeds
a85e131b	236	json = self._search_regex(
	237	r'[\s;]video\s=\s({[\s\S]+?})(?:;\|,playlist=)',
	238	page, video_id, default=None)
18c1f043	239	if not json and 'video-embed' in url:
	240	page = self._download_webpage(url.replace('video-embed', 'video-json'), video_id)
	241	json = self._search_regex(
	242	r'##start-video##({[\s\S]+?})##end-video##',
	243	page, video_id, default=None)
	244	if not json:
a85e131b	245	# if no video data found try search for iframes
	246	emb = RCSEmbedsIE._extract_url(page)
	247	if emb:
5867a167	248	return {
	249	'_type': 'url_transparent',
	250	'url': emb,
	251	'ie_key': RCSEmbedsIE.ie_key()
	252	}
18c1f043	253	if json:
	254	video_data = self._parse_json(
	255	json, video_id, transform_source=js_to_json)
a85e131b	256
	257	if not video_data:
	258	raise ExtractorError('Video data not found in the page')
	259
	260	formats = self._create_formats(
	261	self._get_video_src(video_data), video_id)
	262
508649e6	263	description = (video_data.get('description')
18c1f043	264	or clean_html(video_data.get('htmlDescription'))
18c1f043	265	or self._html_search_meta('description', page))
5867a167	266	uploader = video_data.get('provider') or mobj.group('cdn')
508649e6	267
a85e131b	268	return {
a85e131b	269	'id': video_id,
508649e6	270	'title': video_data.get('title'),
	271	'description': description,
	272	'uploader': uploader,
a85e131b	273	'formats': formats
	274	}
	275
	276
5867a167	277	class RCSEmbedsIE(RCSBaseIE):
a85e131b	278	_VALID_URL = r'''(?x)
	279	https?://(?P<vid>video)\.
	280	(?P<cdn>
	281	(?:
	282	rcs\|
	283	(?:corriere\w+\.)?corriere\|
	284	(?:gazzanet\.)?gazzetta
	285	)\.it)
	286	/video-embed/(?P<id>[^/=&\?]+?)(?:$\|\?)'''
	287	_TESTS = [{
	288	'url': 'https://video.rcs.it/video-embed/iodonna-0001585037',
	289	'md5': '623ecc8ffe7299b2d0c1046d8331a9df',
	290	'info_dict': {
	291	'id': 'iodonna-0001585037',
	292	'ext': 'mp4',
	293	'title': 'Sky Arte racconta Madonna nella serie "Artist to icon"',
	294	'description': 'md5:65b09633df9ffee57f48b39e34c9e067',
	295	'uploader': 'rcs.it',
	296	}
5867a167	297	}, {
18c1f043	298	# redownload the page changing 'video-embed' in 'video-json'
5867a167	299	'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789',
	300	'md5': 'a043e3fecbe4d9ed7fc5d888652a5440',
	301	'info_dict': {
	302	'id': 'gazzanet-mo05-0000260789',
	303	'ext': 'mp4',
	304	'title': 'Valentino Rossi e papà Graziano si divertono col drifting',
	305	'description': 'md5:a8bf90d6adafd9815f70fc74c0fc370a',
	306	'uploader': 'rcd',
	307	}
a85e131b	308	}, {
	309	'url': 'https://video.corriere.it/video-embed/b727632a-f9d0-11ea-91b0-38d50a849abb?player',
	310	'match_only': True
	311	}, {
	312	'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140',
	313	'match_only': True
	314	}]
	315
	316	@staticmethod
	317	def _sanitize_urls(urls):
	318	# add protocol if missing
	319	for i, e in enumerate(urls):
	320	if e.startswith('//'):
	321	urls[i] = 'https:%s' % e
	322	# clean iframes urls
	323	for i, e in enumerate(urls):
	324	urls[i] = urljoin(base_url(e), url_basename(e))
	325	return urls
	326
	327	@staticmethod
	328	def _extract_urls(webpage):
	329	entries = [
	330	mobj.group('url')
	331	for mobj in re.finditer(r'''(?x)
	332	(?:
	333	data-frame-src=\|
	334	<iframe[^\n]+src=
	335	)
	336	(["'])
	337	(?P<url>(?:https?:)?//video\.
	338	(?:
	339	rcs\|
	340	(?:corriere\w+\.)?corriere\|
	341	(?:gazzanet\.)?gazzetta
	342	)
	343	\.it/video-embed/.+?)
	344	\1''', webpage)]
	345	return RCSEmbedsIE._sanitize_urls(entries)
	346
	347	@staticmethod
	348	def _extract_url(webpage):
	349	urls = RCSEmbedsIE._extract_urls(webpage)
	350	return urls[0] if urls else None
	351
	352
5867a167	353	class RCSIE(RCSBaseIE):
a85e131b	354	_VALID_URL = r'''(?x)https?://(?P<vid>video\|viaggi)\.
	355	(?P<cdn>
	356	(?:
5867a167	357	corrieredelmezzogiorno\.
	358	\|corrieredelveneto\.
	359	\|corrieredibologna\.
	360	\|corrierefiorentino\.
	361	)?corriere\.it
	362	\|(?:gazzanet\.)?gazzetta\.it)
	363	/(?!video-embed/).+?/(?P<id>[^/\?]+)(?=\?\|/$\|$)'''
a85e131b	364	_TESTS = [{
	365	'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb',
	366	'md5': '0f4ededc202b0f00b6e509d831e2dcda',
	367	'info_dict': {
	368	'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb',
	369	'ext': 'mp4',
	370	'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante',
	371	'description': 'md5:93b51c9161ac8a64fb2f997b054d0152',
	372	'uploader': 'Corriere Tv',
	373	}
	374	}, {
18c1f043	375	# video data inside iframe
a85e131b	376	'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/',
	377	'md5': 'da378e4918d2afbf7d61c35abb948d4c',
	378	'info_dict': {
	379	'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2',
	380	'ext': 'mp4',
	381	'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen',
	382	'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8',
	383	'uploader': 'DOVE Viaggi',
	384	}
	385	}, {
a85e131b	386	'url': 'https://video.gazzetta.it/video-motogp-catalogna-cadute-dovizioso-vale-rossi/49612410-00ca-11eb-bcd8-30d4253e0140?vclk=Videobar',
	387	'md5': 'eedc1b5defd18e67383afef51ff7bdf9',
	388	'info_dict': {
	389	'id': '49612410-00ca-11eb-bcd8-30d4253e0140',
	390	'ext': 'mp4',
	391	'title': 'Dovizioso, il contatto con Zarco e la caduta. E anche Vale finisce a terra',
	392	'description': 'md5:8c6e905dc3b9413218beca11ebd69778',
	393	'uploader': 'AMorici',
	394	}
	395	}, {
5867a167	396	'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945',
a85e131b	397	'match_only': True
	398	}]
	399
	400
5867a167	401	class RCSVariousIE(RCSBaseIE):
a85e131b	402	_VALID_URL = r'''(?x)https?://www\.
	403	(?P<cdn>
	404	leitv\.it\|
	405	youreporter\.it
18c1f043	406	)/(?:[^/]+/)?(?P<id>[^/]+?)(?:$\|\?\|/)'''
a85e131b	407	_TESTS = [{
18c1f043	408	'url': 'https://www.leitv.it/benessere/mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa/',
18c1f043	409	'md5': '92b4e63667b8f95acb0a04da25ae28a1',
a85e131b	410	'info_dict': {
18c1f043	411	'id': 'mal-di-testa-come-combatterlo-ed-evitarne-la-comparsa',
a85e131b	412	'ext': 'mp4',
18c1f043	413	'title': 'Cervicalgia e mal di testa, il video con i suggerimenti dell\'esperto',
18c1f043	414	'description': 'md5:ae21418f34cee0b8d02a487f55bcabb5',
a85e131b	415	'uploader': 'leitv.it',
	416	}
	417	}, {
	418	'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/',
	419	'md5': '8dccd436b47a830bab5b4a88232f391a',
	420	'info_dict': {
	421	'id': 'fiume-sesia-3-ottobre-2020',
	422	'ext': 'mp4',
	423	'title': 'Fiume Sesia 3 ottobre 2020',
	424	'description': 'md5:0070eef1cc884d13c970a4125063de55',
	425	'uploader': 'youreporter.it',
	426	}
	427	}]