yt_dlp/extractor/gedidigital.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     base_url,
   9     determine_ext,
  10     int_or_none,
  11     url_basename,
  12     urljoin,
  13 )
  14
  15
  16 class GediDigitalIE(InfoExtractor):
  17     _VALID_URL = r'''(?x)(?P<url>(?:https?:)//video\.
  18         (?:
  19             (?:
  20                 (?:espresso\.)?repubblica
  21                 |lastampa
  22                 |ilsecoloxix
  23                 |huffingtonpost
  24             )|
  25             (?:
  26                 iltirreno
  27                 |messaggeroveneto
  28                 |ilpiccolo
  29                 |gazzettadimantova
  30                 |mattinopadova
  31                 |laprovinciapavese
  32                 |tribunatreviso
  33                 |nuovavenezia
  34                 |gazzettadimodena
  35                 |lanuovaferrara
  36                 |corrierealpi
  37                 |lasentinella
  38             )\.gelocal
  39         )\.it(?:/[^/]+){2,4}/(?P<id>\d+))(?:$|[?&].*)'''
  40     _TESTS = [{
  41         'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683',
  42         'md5': '84658d7fb9e55a6e57ecc77b73137494',
  43         'info_dict': {
  44             'id': '121683',
  45             'ext': 'mp4',
  46             'title': 'Il paradosso delle Regionali: ecco perché la Lega vince ma sembra aver perso',
  47             'description': 'md5:de7f4d6eaaaf36c153b599b10f8ce7ca',
  48             'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-full-.+?\.jpg$',
  49             'duration': 125,
  50         },
  51     }, {
  52         'url': 'https://video.huffingtonpost.it/embed/politica/cotticelli-non-so-cosa-mi-sia-successo-sto-cercando-di-capire-se-ho-avuto-un-malore/29312/29276?responsive=true&el=video971040871621586700',
  53         'only_matching': True,
  54     }, {
  55         'url': 'https://video.espresso.repubblica.it/embed/tutti-i-video/01-ted-villa/14772/14870&width=640&height=360',
  56         'only_matching': True,
  57     }, {
  58         'url': 'https://video.repubblica.it/motori/record-della-pista-a-spa-francorchamps-la-pagani-huayra-roadster-bc-stupisce/367415/367963',
  59         'only_matching': True,
  60     }, {
  61         'url': 'https://video.ilsecoloxix.it/sport/cassani-e-i-brividi-azzurri-ai-mondiali-di-imola-qui-mi-sono-innamorato-del-ciclismo-da-ragazzino-incredibile-tornarci-da-ct/66184/66267',
  62         'only_matching': True,
  63     }, {
  64         'url': 'https://video.iltirreno.gelocal.it/sport/dentro-la-notizia-ferrari-cosa-succede-a-maranello/141059/142723',
  65         'only_matching': True,
  66     }, {
  67         'url': 'https://video.messaggeroveneto.gelocal.it/locale/maria-giovanna-elmi-covid-vaccino/138155/139268',
  68         'only_matching': True,
  69     }, {
  70         'url': 'https://video.ilpiccolo.gelocal.it/dossier/big-john/dinosauro-big-john-al-via-le-visite-guidate-a-trieste/135226/135751',
  71         'only_matching': True,
  72     }, {
  73         'url': 'https://video.gazzettadimantova.gelocal.it/locale/dal-ponte-visconteo-di-valeggio-l-and-8217sos-dei-ristoratori-aprire-anche-a-cena/137310/137818',
  74         'only_matching': True,
  75     }, {
  76         'url': 'https://video.mattinopadova.gelocal.it/dossier/coronavirus-in-veneto/covid-a-vo-un-anno-dopo-un-cuore-tricolore-per-non-dimenticare/138402/138964',
  77         'only_matching': True,
  78     }, {
  79         'url': 'https://video.laprovinciapavese.gelocal.it/locale/mede-zona-rossa-via-alle-vaccinazioni-per-gli-over-80/137545/138120',
  80         'only_matching': True,
  81     }, {
  82         'url': 'https://video.tribunatreviso.gelocal.it/dossier/coronavirus-in-veneto/ecco-le-prima-vaccinazioni-di-massa-nella-marca/134485/135024',
  83         'only_matching': True,
  84     }, {
  85         'url': 'https://video.nuovavenezia.gelocal.it/locale/camion-troppo-alto-per-il-ponte-ferroviario-perde-il-carico/135734/136266',
  86         'only_matching': True,
  87     }, {
  88         'url': 'https://video.gazzettadimodena.gelocal.it/locale/modena-scoperta-la-proteina-che-predice-il-livello-di-gravita-del-covid/139109/139796',
  89         'only_matching': True,
  90     }, {
  91         'url': 'https://video.lanuovaferrara.gelocal.it/locale/due-bombole-di-gpl-aperte-e-abbandonate-i-vigili-bruciano-il-gas/134391/134957',
  92         'only_matching': True,
  93     }, {
  94         'url': 'https://video.corrierealpi.gelocal.it/dossier/cortina-2021-i-mondiali-di-sci-alpino/mondiali-di-sci-il-timelapse-sulla-splendida-olympia/133760/134331',
  95         'only_matching': True,
  96     }, {
  97         'url': 'https://video.lasentinella.gelocal.it/locale/vestigne-centra-un-auto-e-si-ribalta/138931/139466',
  98         'only_matching': True,
  99     }, {
 100         'url': 'https://video.espresso.repubblica.it/tutti-i-video/01-ted-villa/14772',
 101         'only_matching': True,
 102     }]
 103
 104     @staticmethod
 105     def _sanitize_urls(urls):
 106         # add protocol if missing
 107         for i, e in enumerate(urls):
 108             if e.startswith('//'):
 109                 urls[i] = 'https:%s' % e
 110         # clean iframes urls
 111         for i, e in enumerate(urls):
 112             urls[i] = urljoin(base_url(e), url_basename(e))
 113         return urls
 114
 115     @staticmethod
 116     def _extract_urls(webpage):
 117         entries = [
 118             mobj.group('eurl')
 119             for mobj in re.finditer(r'''(?x)
 120             (?:
 121                 data-frame-src=|
 122                 <iframe[^\n]+src=
 123             )
 124             (["'])(?P<eurl>%s)\1''' % GediDigitalIE._VALID_URL, webpage)]
 125         return GediDigitalIE._sanitize_urls(entries)
 126
 127     @staticmethod
 128     def _extract_url(webpage):
 129         urls = GediDigitalIE._extract_urls(webpage)
 130         return urls[0] if urls else None
 131
 132     @staticmethod
 133     def _clean_formats(formats):
 134         format_urls = set()
 135         clean_formats = []
 136         for f in formats:
 137             if f['url'] not in format_urls:
 138                 if f.get('audio_ext') != 'none' and not f.get('acodec'):
 139                     continue
 140                 format_urls.add(f['url'])
 141                 clean_formats.append(f)
 142         formats[:] = clean_formats
 143
 144     def _real_extract(self, url):
 145         video_id = self._match_id(url)
 146         url = re.match(self._VALID_URL, url).group('url')
 147         webpage = self._download_webpage(url, video_id)
 148         title = self._html_search_meta(
 149             ['twitter:title', 'og:title'], webpage, fatal=True)
 150         player_data = re.findall(
 151             r"PlayerFactory\.setParam\('(?P<type>format|param)',\s*'(?P<name>[^']+)',\s*'(?P<val>[^']+)'\);",
 152             webpage)
 153
 154         formats = []
 155         duration = thumb = None
 156         for t, n, v in player_data:
 157             if t == 'format':
 158                 if n in ('video-hds-vod-ec', 'video-hls-vod-ec', 'video-viralize', 'video-youtube-pfp'):
 159                     continue
 160                 elif n.endswith('-vod-ak'):
 161                     formats.extend(self._extract_akamai_formats(
 162                         v, video_id, {'http': 'media.gedidigital.it'}))
 163                 else:
 164                     ext = determine_ext(v)
 165                     if ext == 'm3u8':
 166                         formats.extend(self._extract_m3u8_formats(
 167                             v, video_id, 'mp4', 'm3u8_native', m3u8_id=n, fatal=False))
 168                         continue
 169                     f = {
 170                         'format_id': n,
 171                         'url': v,
 172                     }
 173                     if ext == 'mp3':
 174                         abr = int_or_none(self._search_regex(
 175                             r'-mp3-audio-(\d+)', v, 'abr', default=None))
 176                         f.update({
 177                             'abr': abr,
 178                             'tbr': abr,
 179                             'acodec': ext,
 180                             'vcodec': 'none'
 181                         })
 182                     else:
 183                         mobj = re.match(r'^video-rrtv-(\d+)(?:-(\d+))?$', n)
 184                         if mobj:
 185                             f.update({
 186                                 'height': int(mobj.group(1)),
 187                                 'vbr': int_or_none(mobj.group(2)),
 188                             })
 189                         if not f.get('vbr'):
 190                             f['vbr'] = int_or_none(self._search_regex(
 191                                 r'-video-rrtv-(\d+)', v, 'abr', default=None))
 192                     formats.append(f)
 193             elif t == 'param':
 194                 if n in ['image_full', 'image']:
 195                     thumb = v
 196                 elif n == 'videoDuration':
 197                     duration = int_or_none(v)
 198
 199         self._clean_formats(formats)
 200         self._sort_formats(formats)
 201
 202         return {
 203             'id': video_id,
 204             'title': title,
 205             'description': self._html_search_meta(
 206                 ['twitter:description', 'og:description', 'description'], webpage),
 207             'thumbnail': thumb or self._og_search_thumbnail(webpage),
 208             'formats': formats,
 209             'duration': duration,
 210         }