yt_dlp/extractor/rcs.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     clean_html,
   9     ExtractorError,
  10     js_to_json,
  11     base_url,
  12     url_basename,
  13     urljoin,
  14 )
  15
  16
  17 class RCSBaseIE(InfoExtractor):
  18     _ALL_REPLACE = {
  19         'media2vam.corriere.it.edgesuite.net':
  20             'media2vam-corriere-it.akamaized.net',
  21         'media.youreporter.it.edgesuite.net':
  22             'media-youreporter-it.akamaized.net',
  23         'corrierepmd.corriere.it.edgesuite.net':
  24             'corrierepmd-corriere-it.akamaized.net',
  25         'media2vam-corriere-it.akamaized.net/fcs.quotidiani/vr/videos/':
  26             'video.corriere.it/vr360/videos/',
  27         '.net//': '.net/',
  28     }
  29     _MP4_REPLACE = {
  30         'media2vam.corbologna.corriere.it.edgesuite.net':
  31             'media2vam-bologna-corriere-it.akamaized.net',
  32         'media2vam.corfiorentino.corriere.it.edgesuite.net':
  33             'media2vam-fiorentino-corriere-it.akamaized.net',
  34         'media2vam.cormezzogiorno.corriere.it.edgesuite.net':
  35             'media2vam-mezzogiorno-corriere-it.akamaized.net',
  36         'media2vam.corveneto.corriere.it.edgesuite.net':
  37             'media2vam-veneto-corriere-it.akamaized.net',
  38         'media2.oggi.it.edgesuite.net':
  39             'media2-oggi-it.akamaized.net',
  40         'media2.quimamme.it.edgesuite.net':
  41             'media2-quimamme-it.akamaized.net',
  42         'media2.amica.it.edgesuite.net':
  43             'media2-amica-it.akamaized.net',
  44         'media2.living.corriere.it.edgesuite.net':
  45             'media2-living-corriere-it.akamaized.net',
  46         'media2.style.corriere.it.edgesuite.net':
  47             'media2-style-corriere-it.akamaized.net',
  48         'media2.iodonna.it.edgesuite.net':
  49             'media2-iodonna-it.akamaized.net',
  50         'media2.leitv.it.edgesuite.net':
  51             'media2-leitv-it.akamaized.net',
  52     }
  53     _MIGRATION_MAP = {
  54         'videoamica-vh.akamaihd': 'amica',
  55         'media2-amica-it.akamaized': 'amica',
  56         'corrierevam-vh.akamaihd': 'corriere',
  57         'media2vam-corriere-it.akamaized': 'corriere',
  58         'cormezzogiorno-vh.akamaihd': 'corrieredelmezzogiorno',
  59         'media2vam-mezzogiorno-corriere-it.akamaized': 'corrieredelmezzogiorno',
  60         'corveneto-vh.akamaihd': 'corrieredelveneto',
  61         'media2vam-veneto-corriere-it.akamaized': 'corrieredelveneto',
  62         'corbologna-vh.akamaihd': 'corrieredibologna',
  63         'media2vam-bologna-corriere-it.akamaized': 'corrieredibologna',
  64         'corfiorentino-vh.akamaihd': 'corrierefiorentino',
  65         'media2vam-fiorentino-corriere-it.akamaized': 'corrierefiorentino',
  66         'corinnovazione-vh.akamaihd': 'corriereinnovazione',
  67         'media2-gazzanet-gazzetta-it.akamaized': 'gazzanet',
  68         'videogazzanet-vh.akamaihd': 'gazzanet',
  69         'videogazzaworld-vh.akamaihd': 'gazzaworld',
  70         'gazzettavam-vh.akamaihd': 'gazzetta',
  71         'media2vam-gazzetta-it.akamaized': 'gazzetta',
  72         'videoiodonna-vh.akamaihd': 'iodonna',
  73         'media2-leitv-it.akamaized': 'leitv',
  74         'videoleitv-vh.akamaihd': 'leitv',
  75         'videoliving-vh.akamaihd': 'living',
  76         'media2-living-corriere-it.akamaized': 'living',
  77         'media2-oggi-it.akamaized': 'oggi',
  78         'videooggi-vh.akamaihd': 'oggi',
  79         'media2-quimamme-it.akamaized': 'quimamme',
  80         'quimamme-vh.akamaihd': 'quimamme',
  81         'videorunning-vh.akamaihd': 'running',
  82         'media2-style-corriere-it.akamaized': 'style',
  83         'style-vh.akamaihd': 'style',
  84         'videostyle-vh.akamaihd': 'style',
  85         'media2-stylepiccoli-it.akamaized': 'stylepiccoli',
  86         'stylepiccoli-vh.akamaihd': 'stylepiccoli',
  87         'doveviaggi-vh.akamaihd': 'viaggi',
  88         'media2-doveviaggi-it.akamaized': 'viaggi',
  89         'media2-vivimilano-corriere-it.akamaized': 'vivimilano',
  90         'vivimilano-vh.akamaihd': 'vivimilano',
  91         'media2-youreporter-it.akamaized': 'youreporter'
  92     }
  93     _MIGRATION_MEDIA = {
  94         'advrcs-vh.akamaihd': '',
  95         'corriere-f.akamaihd': '',
  96         'corrierepmd-corriere-it.akamaized': '',
  97         'corrprotetto-vh.akamaihd': '',
  98         'gazzetta-f.akamaihd': '',
  99         'gazzettapmd-gazzetta-it.akamaized': '',
 100         'gazzprotetto-vh.akamaihd': '',
 101         'periodici-f.akamaihd': '',
 102         'periodicisecure-vh.akamaihd': '',
 103         'videocoracademy-vh.akamaihd': ''
 104     }
 105
 106     def _get_video_src(self, video):
 107         mediaFiles = video.get('mediaProfile').get('mediaFile')
 108         src = {}
 109         # audio
 110         if video.get('mediaType') == 'AUDIO':
 111             for aud in mediaFiles:
 112                 # todo: check
 113                 src['mp3'] = aud.get('value')
 114         # video
 115         else:
 116             for vid in mediaFiles:
 117                 if vid.get('mimeType') == 'application/vnd.apple.mpegurl':
 118                     src['m3u8'] = vid.get('value')
 119                 if vid.get('mimeType') == 'video/mp4':
 120                     src['mp4'] = vid.get('value')
 121
 122         # replace host
 123         for t in src:
 124             for s, r in self._ALL_REPLACE.items():
 125                 src[t] = src[t].replace(s, r)
 126             for s, r in self._MP4_REPLACE.items():
 127                 src[t] = src[t].replace(s, r)
 128
 129         # switch cdn
 130         if 'mp4' in src and 'm3u8' in src:
 131             if ('-lh.akamaihd' not in src.get('m3u8')
 132                     and 'akamai' in src.get('mp4')):
 133                 if 'm3u8' in src:
 134                     matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('m3u8'))
 135                     src['m3u8'] = 'https://vod.rcsobjects.it/hls/%s%s' % (
 136                         self._MIGRATION_MAP[matches.group('host')],
 137                         matches.group('path').replace(
 138                             '///', '/').replace(
 139                             '//', '/').replace(
 140                             '.csmil', '.urlset'
 141                         )
 142                     )
 143                 if 'mp4' in src:
 144                     matches = re.search(r'(?:https*:)?\/\/(?P<host>.*)\.net\/i(?P<path>.*)$', src.get('mp4'))
 145                     if matches:
 146                         if matches.group('host') in self._MIGRATION_MEDIA:
 147                             vh_stream = 'https://media2.corriereobjects.it'
 148                             if src.get('mp4').find('fcs.quotidiani_!'):
 149                                 vh_stream = 'https://media2-it.corriereobjects.it'
 150                             src['mp4'] = '%s%s' % (
 151                                 vh_stream,
 152                                 matches.group('path').replace(
 153                                     '///', '/').replace(
 154                                     '//', '/').replace(
 155                                     '/fcs.quotidiani/mediacenter', '').replace(
 156                                     '/fcs.quotidiani_!/mediacenter', '').replace(
 157                                     'corriere/content/mediacenter/', '').replace(
 158                                     'gazzetta/content/mediacenter/', '')
 159                             )
 160                         else:
 161                             src['mp4'] = 'https://vod.rcsobjects.it/%s%s' % (
 162                                 self._MIGRATION_MAP[matches.group('host')],
 163                                 matches.group('path').replace('///', '/').replace('//', '/')
 164                             )
 165
 166         if 'mp3' in src:
 167             src['mp3'] = src.get('mp3').replace(
 168                 'media2vam-corriere-it.akamaized.net',
 169                 'vod.rcsobjects.it/corriere')
 170         if 'mp4' in src:
 171             if src.get('mp4').find('fcs.quotidiani_!'):
 172                 src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
 173         if 'm3u8' in src:
 174             if src.get('m3u8').find('fcs.quotidiani_!'):
 175                 src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')
 176
 177         if 'geoblocking' in video.get('mediaProfile'):
 178             if 'm3u8' in src:
 179                 src['m3u8'] = src.get('m3u8').replace('vod.rcsobjects', 'vod-it.rcsobjects')
 180             if 'mp4' in src:
 181                 src['mp4'] = src.get('mp4').replace('vod.rcsobjects', 'vod-it.rcsobjects')
 182         if 'm3u8' in src:
 183             if src.get('m3u8').find('csmil') and src.get('m3u8').find('vod'):
 184                 src['m3u8'] = src.get('m3u8').replace('.csmil', '.urlset')
 185
 186         return src
 187
 188     def _create_formats(self, urls, video_id):
 189         formats = []
 190         formats = self._extract_m3u8_formats(
 191             urls.get('m3u8'), video_id, 'mp4', entry_protocol='m3u8_native',
 192             m3u8_id='hls', fatal=False)
 193
 194         if not formats:
 195             formats.append({
 196                 'format_id': 'http-mp4',
 197                 'url': urls.get('mp4')
 198             })
 199         self._sort_formats(formats)
 200         return formats
 201
 202     def _real_extract(self, url):
 203         video_id = self._match_id(url)
 204         mobj = re.search(self._VALID_URL, url)
 205
 206         if 'cdn' not in mobj.groupdict():
 207             raise ExtractorError('CDN not found in url: %s' % url)
 208
 209         # for leitv/youreporter/viaggi don't use the embed page
 210         if ((mobj.group('cdn') not in ['leitv.it', 'youreporter.it'])
 211                 and (mobj.group('vid') == 'video')):
 212             url = 'https://video.%s/video-embed/%s' % (mobj.group('cdn'), video_id)
 213
 214         page = self._download_webpage(url, video_id)
 215
 216         video_data = None
 217         # look for json video data url
 218         json = self._search_regex(
 219             r'''(?x)var url\s*=\s*["']((?:https?:)?
 220                 //video\.rcs\.it
 221                 /fragment-includes/video-includes/.+?\.json)["'];''',
 222             page, video_id, default=None)
 223         if json:
 224             if json.startswith('//'):
 225                 json = 'https:%s' % json
 226             video_data = self._download_json(json, video_id)
 227
 228         # if json url not found, look for json video data directly in the page
 229         else:
 230             json = self._search_regex(
 231                 r'[\s;]video\s*=\s*({[\s\S]+?})(?:;|,playlist=)',
 232                 page, video_id, default=None)
 233             if json:
 234                 video_data = self._parse_json(
 235                     json, video_id, transform_source=js_to_json)
 236             else:
 237                 # if no video data found try search for iframes
 238                 emb = RCSEmbedsIE._extract_url(page)
 239                 if emb:
 240                     return {
 241                         '_type': 'url_transparent',
 242                         'url': emb,
 243                         'ie_key': RCSEmbedsIE.ie_key()
 244                     }
 245
 246         if not video_data:
 247             raise ExtractorError('Video data not found in the page')
 248
 249         formats = self._create_formats(
 250             self._get_video_src(video_data), video_id)
 251
 252         description = (video_data.get('description')
 253                        or clean_html(video_data.get('htmlDescription')))
 254         uploader = video_data.get('provider') or mobj.group('cdn')
 255
 256         return {
 257             'id': video_id,
 258             'title': video_data.get('title'),
 259             'description': description,
 260             'uploader': uploader,
 261             'formats': formats
 262         }
 263
 264
 265 class RCSEmbedsIE(RCSBaseIE):
 266     _VALID_URL = r'''(?x)
 267                     https?://(?P<vid>video)\.
 268                     (?P<cdn>
 269                     (?:
 270                         rcs|
 271                         (?:corriere\w+\.)?corriere|
 272                         (?:gazzanet\.)?gazzetta
 273                     )\.it)
 274                     /video-embed/(?P<id>[^/=&\?]+?)(?:$|\?)'''
 275     _TESTS = [{
 276         'url': 'https://video.rcs.it/video-embed/iodonna-0001585037',
 277         'md5': '623ecc8ffe7299b2d0c1046d8331a9df',
 278         'info_dict': {
 279             'id': 'iodonna-0001585037',
 280             'ext': 'mp4',
 281             'title': 'Sky Arte racconta Madonna nella serie "Artist to icon"',
 282             'description': 'md5:65b09633df9ffee57f48b39e34c9e067',
 283             'uploader': 'rcs.it',
 284         }
 285     }, {
 286         'url': 'https://video.gazzanet.gazzetta.it/video-embed/gazzanet-mo05-0000260789',
 287         'md5': 'a043e3fecbe4d9ed7fc5d888652a5440',
 288         'info_dict': {
 289             'id': 'gazzanet-mo05-0000260789',
 290             'ext': 'mp4',
 291             'title': 'Valentino Rossi e papà Graziano si divertono col drifting',
 292             'description': 'md5:a8bf90d6adafd9815f70fc74c0fc370a',
 293             'uploader': 'rcd',
 294         }
 295     }, {
 296         'url': 'https://video.corriere.it/video-embed/b727632a-f9d0-11ea-91b0-38d50a849abb?player',
 297         'match_only': True
 298     }, {
 299         'url': 'https://video.gazzetta.it/video-embed/49612410-00ca-11eb-bcd8-30d4253e0140',
 300         'match_only': True
 301     }]
 302
 303     @staticmethod
 304     def _sanitize_urls(urls):
 305         # add protocol if missing
 306         for i, e in enumerate(urls):
 307             if e.startswith('//'):
 308                 urls[i] = 'https:%s' % e
 309         # clean iframes urls
 310         for i, e in enumerate(urls):
 311             urls[i] = urljoin(base_url(e), url_basename(e))
 312         return urls
 313
 314     @staticmethod
 315     def _extract_urls(webpage):
 316         entries = [
 317             mobj.group('url')
 318             for mobj in re.finditer(r'''(?x)
 319             (?:
 320                 data-frame-src=|
 321                 <iframe[^\n]+src=
 322             )
 323             (["'])
 324                 (?P<url>(?:https?:)?//video\.
 325                     (?:
 326                         rcs|
 327                         (?:corriere\w+\.)?corriere|
 328                         (?:gazzanet\.)?gazzetta
 329                     )
 330                 \.it/video-embed/.+?)
 331             \1''', webpage)]
 332         return RCSEmbedsIE._sanitize_urls(entries)
 333
 334     @staticmethod
 335     def _extract_url(webpage):
 336         urls = RCSEmbedsIE._extract_urls(webpage)
 337         return urls[0] if urls else None
 338
 339
 340 class RCSIE(RCSBaseIE):
 341     _VALID_URL = r'''(?x)https?://(?P<vid>video|viaggi)\.
 342                     (?P<cdn>
 343                     (?:
 344                         corrieredelmezzogiorno\.
 345                         |corrieredelveneto\.
 346                         |corrieredibologna\.
 347                         |corrierefiorentino\.
 348                     )?corriere\.it
 349                     |(?:gazzanet\.)?gazzetta\.it)
 350                     /(?!video-embed/).+?/(?P<id>[^/\?]+)(?=\?|/$|$)'''
 351     _TESTS = [{
 352         'url': 'https://video.corriere.it/sport/formula-1/vettel-guida-ferrari-sf90-mugello-suo-fianco-c-elecrerc-bendato-video-esilarante/b727632a-f9d0-11ea-91b0-38d50a849abb',
 353         'md5': '0f4ededc202b0f00b6e509d831e2dcda',
 354         'info_dict': {
 355             'id': 'b727632a-f9d0-11ea-91b0-38d50a849abb',
 356             'ext': 'mp4',
 357             'title': 'Vettel guida la Ferrari SF90 al Mugello e al suo fianco c\'è Leclerc (bendato): il video è esilarante',
 358             'description': 'md5:93b51c9161ac8a64fb2f997b054d0152',
 359             'uploader': 'Corriere Tv',
 360         }
 361     }, {
 362         'url': 'https://viaggi.corriere.it/video/norvegia-il-nuovo-ponte-spettacolare-sopra-la-cascata-di-voringsfossen/',
 363         'md5': 'da378e4918d2afbf7d61c35abb948d4c',
 364         'info_dict': {
 365             'id': '5b7cd134-e2c1-11ea-89b3-b56dd0df2aa2',
 366             'ext': 'mp4',
 367             'title': 'La nuova spettacolare attrazione in Norvegia: il ponte sopra Vøringsfossen',
 368             'description': 'md5:18b35a291f6746c0c8dacd16e5f5f4f8',
 369             'uploader': 'DOVE Viaggi',
 370         }
 371     }, {
 372         'url': 'https://video.gazzetta.it/video-motogp-catalogna-cadute-dovizioso-vale-rossi/49612410-00ca-11eb-bcd8-30d4253e0140?vclk=Videobar',
 373         'md5': 'eedc1b5defd18e67383afef51ff7bdf9',
 374         'info_dict': {
 375             'id': '49612410-00ca-11eb-bcd8-30d4253e0140',
 376             'ext': 'mp4',
 377             'title': 'Dovizioso, il contatto con Zarco e la caduta. E anche Vale finisce a terra',
 378             'description': 'md5:8c6e905dc3b9413218beca11ebd69778',
 379             'uploader': 'AMorici',
 380         }
 381     }, {
 382         'url': 'https://video.corriere.it/video-360/metro-copenaghen-tutta-italiana/a248a7f0-e2db-11e9-9830-af2de6b1f945',
 383         'match_only': True
 384     }]
 385
 386
 387 class RCSVariousIE(RCSBaseIE):
 388     _VALID_URL = r'''(?x)https?://www\.
 389                     (?P<cdn>
 390                         leitv\.it|
 391                         youreporter\.it
 392                     )/(?:video/)?(?P<id>[^/]+?)(?:$|\?|/)'''
 393     _TESTS = [{
 394         'url': 'https://www.leitv.it/video/marmellata-di-ciliegie-fatta-in-casa/',
 395         'md5': '618aaabac32152199c1af86784d4d554',
 396         'info_dict': {
 397             'id': 'marmellata-di-ciliegie-fatta-in-casa',
 398             'ext': 'mp4',
 399             'title': 'Marmellata di ciliegie fatta in casa',
 400             'description': 'md5:89133864d6aad456dbcf6e7a29f86263',
 401             'uploader': 'leitv.it',
 402         }
 403     }, {
 404         'url': 'https://www.youreporter.it/fiume-sesia-3-ottobre-2020/',
 405         'md5': '8dccd436b47a830bab5b4a88232f391a',
 406         'info_dict': {
 407             'id': 'fiume-sesia-3-ottobre-2020',
 408             'ext': 'mp4',
 409             'title': 'Fiume Sesia 3 ottobre 2020',
 410             'description': 'md5:0070eef1cc884d13c970a4125063de55',
 411             'uploader': 'youreporter.it',
 412         }
 413     }]