yt_dlp/extractor/gedi.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import compat_str
   8 from ..utils import (
   9     base_url,
  10     url_basename,
  11     urljoin,
  12 )
  13
  14
  15 class GediBaseIE(InfoExtractor):
  16     @staticmethod
  17     def _clean_audio_fmts(formats):
  18         unique_formats = []
  19         for f in formats:
  20             if 'acodec' in f:
  21                 unique_formats.append(f)
  22         formats[:] = unique_formats
  23
  24     def _real_extract(self, url):
  25         video_id = self._match_id(url)
  26
  27         webpage = self._download_webpage(url, video_id)
  28         player_data = re.findall(
  29             r'PlayerFactory\.setParam\(\'(?P<type>.+?)\',\s*\'(?P<name>.+?)\',\s*\'(?P<val>.+?)\'\);',
  30             webpage)
  31
  32         formats = []
  33         audio_fmts = []
  34         hls_fmts = []
  35         http_fmts = []
  36         title = ''
  37         thumb = ''
  38
  39         fmt_reg = r'(?P<t>video|audio)-(?P<p>rrtv|hls)-(?P<h>[\w\d]+)(?:-(?P<br>[\w\d]+))?$'
  40         br_reg = r'video-rrtv-(?P<br>\d+)-'
  41
  42         for t, n, v in player_data:
  43             if t == 'format':
  44                 m = re.match(fmt_reg, n)
  45                 if m:
  46                     # audio formats
  47                     if m.group('t') == 'audio':
  48                         if m.group('p') == 'hls':
  49                             audio_fmts.extend(self._extract_m3u8_formats(
  50                                 v, video_id, 'm4a', m3u8_id='hls', fatal=False))
  51                         elif m.group('p') == 'rrtv':
  52                             audio_fmts.append({
  53                                 'format_id': 'mp3',
  54                                 'url': v,
  55                                 'tbr': 128,
  56                                 'ext': 'mp3',
  57                                 'vcodec': 'none',
  58                                 'acodec': 'mp3',
  59                             })
  60
  61                     # video formats
  62                     elif m.group('t') == 'video':
  63                         # hls manifest video
  64                         if m.group('p') == 'hls':
  65                             hls_fmts.extend(self._extract_m3u8_formats(
  66                                 v, video_id, 'mp4', m3u8_id='hls', fatal=False))
  67                         # direct mp4 video
  68                         elif m.group('p') == 'rrtv':
  69                             if not m.group('br'):
  70                                 mm = re.search(br_reg, v)
  71                             http_fmts.append({
  72                                 'format_id': 'https-' + m.group('h'),
  73                                 'protocol': 'https',
  74                                 'url': v,
  75                                 'tbr': int(m.group('br')) if m.group('br') else
  76                                 (int(mm.group('br')) if mm.group('br') else 0),
  77                                 'height': int(m.group('h'))
  78                             })
  79
  80             elif t == 'param':
  81                 if n == 'videotitle':
  82                     title = v
  83                 if n == 'image_full_play':
  84                     thumb = v
  85
  86         title = self._og_search_title(webpage) if title == '' else title
  87
  88         # clean weird char
  89         title = compat_str(title).encode('utf8', 'replace').replace(b'\xc3\x82', b'').decode('utf8', 'replace')
  90
  91         if audio_fmts:
  92             self._clean_audio_fmts(audio_fmts)
  93             self._sort_formats(audio_fmts)
  94         if hls_fmts:
  95             self._sort_formats(hls_fmts)
  96         if http_fmts:
  97             self._sort_formats(http_fmts)
  98
  99         formats.extend(audio_fmts)
 100         formats.extend(hls_fmts)
 101         formats.extend(http_fmts)
 102
 103         return {
 104             'id': video_id,
 105             'title': title,
 106             'description': self._html_search_meta('twitter:description', webpage),
 107             'thumbnail': thumb,
 108             'formats': formats,
 109         }
 110
 111
 112 class GediIE(GediBaseIE):
 113     _VALID_URL = r'''(?x)https?://video\.
 114                     (?:
 115                         (?:espresso\.)?repubblica
 116                         |lastampa
 117                         |huffingtonpost
 118                         |ilsecoloxix
 119                         |iltirreno
 120                         |messaggeroveneto
 121                         |ilpiccolo
 122                         |gazzettadimantova
 123                         |mattinopadova
 124                         |laprovinciapavese
 125                         |tribunatreviso
 126                         |nuovavenezia
 127                         |gazzettadimodena
 128                         |lanuovaferrara
 129                         |corrierealpi
 130                         |lasentinella
 131                     )
 132                     (?:\.gelocal)?\.it/(?!embed/).+?/(?P<id>[\d/]+)(?:\?|\&|$)'''
 133     _TESTS = [{
 134         'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683',
 135         'md5': '84658d7fb9e55a6e57ecc77b73137494',
 136         'info_dict': {
 137             'id': '121559/121683',
 138             'ext': 'mp4',
 139             'title': 'Il paradosso delle Regionali: ecco perché la Lega vince ma sembra aver perso',
 140             'description': 'md5:de7f4d6eaaaf36c153b599b10f8ce7ca',
 141             'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$',
 142         },
 143     }, {
 144         'url': 'https://video.repubblica.it/motori/record-della-pista-a-spa-francorchamps-la-pagani-huayra-roadster-bc-stupisce/367415/367963',
 145         'md5': 'e763b94b7920799a0e0e23ffefa2d157',
 146         'info_dict': {
 147             'id': '367415/367963',
 148             'ext': 'mp4',
 149             'title': 'Record della pista a Spa Francorchamps, la Pagani Huayra Roadster BC stupisce',
 150             'description': 'md5:5deb503cefe734a3eb3f07ed74303920',
 151             'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$',
 152         },
 153     }, {
 154         'url': 'https://video.ilsecoloxix.it/sport/cassani-e-i-brividi-azzurri-ai-mondiali-di-imola-qui-mi-sono-innamorato-del-ciclismo-da-ragazzino-incredibile-tornarci-da-ct/66184/66267',
 155         'md5': 'e48108e97b1af137d22a8469f2019057',
 156         'info_dict': {
 157             'id': '66184/66267',
 158             'ext': 'mp4',
 159             'title': 'Cassani e i brividi azzurri ai Mondiali di Imola: \\"Qui mi sono innamorato del ciclismo da ragazzino, incredibile tornarci da ct\\"',
 160             'description': 'md5:fc9c50894f70a2469bb9b54d3d0a3d3b',
 161             'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$',
 162         },
 163     }, {
 164         'url': 'https://video.iltirreno.gelocal.it/sport/dentro-la-notizia-ferrari-cosa-succede-a-maranello/141059/142723',
 165         'md5': 'a6e39f3bdc1842bbd92abbbbef230817',
 166         'info_dict': {
 167             'id': '141059/142723',
 168             'ext': 'mp4',
 169             'title': 'Dentro la notizia - Ferrari, cosa succede a Maranello',
 170             'description': 'md5:9907d65b53765681fa3a0b3122617c1f',
 171             'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$',
 172         },
 173     }]
 174
 175
 176 class GediEmbedsIE(GediBaseIE):
 177     _VALID_URL = r'''(?x)https?://video\.
 178                     (?:
 179                         (?:espresso\.)?repubblica
 180                         |lastampa
 181                         |huffingtonpost
 182                         |ilsecoloxix
 183                         |iltirreno
 184                         |messaggeroveneto
 185                         |ilpiccolo
 186                         |gazzettadimantova
 187                         |mattinopadova
 188                         |laprovinciapavese
 189                         |tribunatreviso
 190                         |nuovavenezia
 191                         |gazzettadimodena
 192                         |lanuovaferrara
 193                         |corrierealpi
 194                         |lasentinella
 195                     )
 196                     (?:\.gelocal)?\.it/embed/.+?/(?P<id>[\d/]+)(?:\?|\&|$)'''
 197     _TESTS = [{
 198         'url': 'https://video.huffingtonpost.it/embed/politica/cotticelli-non-so-cosa-mi-sia-successo-sto-cercando-di-capire-se-ho-avuto-un-malore/29312/29276?responsive=true&el=video971040871621586700',
 199         'md5': 'f4ac23cadfea7fef89bea536583fa7ed',
 200         'info_dict': {
 201             'id': '29312/29276',
 202             'ext': 'mp4',
 203             'title': 'Cotticelli: \\"Non so cosa mi sia successo. Sto cercando di capire se ho avuto un malore\\"',
 204             'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
 205             'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$',
 206         },
 207     }, {
 208         'url': 'https://video.espresso.repubblica.it/embed/tutti-i-video/01-ted-villa/14772/14870&width=640&height=360',
 209         'md5': '0391c2c83c6506581003aaf0255889c0',
 210         'info_dict': {
 211             'id': '14772/14870',
 212             'ext': 'mp4',
 213             'title': 'Festival EMERGENCY, Villa: «La buona informazione aiuta la salute» (14772-14870)',
 214             'description': 'md5:2bce954d278248f3c950be355b7c2226',
 215             'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$',
 216         },
 217     }]
 218
 219     @staticmethod
 220     def _sanitize_urls(urls):
 221         # add protocol if missing
 222         for i, e in enumerate(urls):
 223             if e.startswith('//'):
 224                 urls[i] = 'https:%s' % e
 225         # clean iframes urls
 226         for i, e in enumerate(urls):
 227             urls[i] = urljoin(base_url(e), url_basename(e))
 228         return urls
 229
 230     @staticmethod
 231     def _extract_urls(webpage):
 232         entries = [
 233             mobj.group('url')
 234             for mobj in re.finditer(r'''(?x)
 235             (?:
 236                 data-frame-src=|
 237                 <iframe[^\n]+src=
 238             )
 239             (["'])
 240                 (?P<url>https?://video\.
 241                     (?:
 242                         (?:espresso\.)?repubblica
 243                         |lastampa
 244                         |huffingtonpost
 245                         |ilsecoloxix
 246                         |iltirreno
 247                         |messaggeroveneto
 248                         |ilpiccolo
 249                         |gazzettadimantova
 250                         |mattinopadova
 251                         |laprovinciapavese
 252                         |tribunatreviso
 253                         |nuovavenezia
 254                         |gazzettadimodena
 255                         |lanuovaferrara
 256                         |corrierealpi
 257                         |lasentinella
 258                     )
 259                     (?:\.gelocal)?\.it/embed/.+?)
 260             \1''', webpage)]
 261         return GediEmbedsIE._sanitize_urls(entries)
 262
 263     @staticmethod
 264     def _extract_url(webpage):
 265         urls = GediEmbedsIE._extract_urls(webpage)
 266         return urls[0] if urls else None