yt_dlp/extractor/rai.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..compat import (
   5     compat_str,
   6     compat_urlparse,
   7 )
   8 from ..utils import (
   9     clean_html,
  10     determine_ext,
  11     ExtractorError,
  12     filter_dict,
  13     find_xpath_attr,
  14     fix_xml_ampersands,
  15     GeoRestrictedError,
  16     HEADRequest,
  17     int_or_none,
  18     join_nonempty,
  19     parse_duration,
  20     remove_start,
  21     strip_or_none,
  22     traverse_obj,
  23     try_get,
  24     unified_strdate,
  25     unified_timestamp,
  26     update_url_query,
  27     urljoin,
  28     xpath_text,
  29 )
  30
  31
  32 class RaiBaseIE(InfoExtractor):
  33     _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
  34     _GEO_COUNTRIES = ['IT']
  35     _GEO_BYPASS = False
  36
  37     def _extract_relinker_info(self, relinker_url, video_id, audio_only=False):
  38         if not re.match(r'https?://', relinker_url):
  39             return {'formats': [{'url': relinker_url}]}
  40
  41         formats = []
  42         geoprotection = None
  43         is_live = None
  44         duration = None
  45
  46         for platform in ('mon', 'flash', 'native'):
  47             relinker = self._download_xml(
  48                 relinker_url, video_id,
  49                 note=f'Downloading XML metadata for platform {platform}',
  50                 transform_source=fix_xml_ampersands,
  51                 query={'output': 45, 'pl': platform},
  52                 headers=self.geo_verification_headers())
  53
  54             if not geoprotection:
  55                 geoprotection = xpath_text(
  56                     relinker, './geoprotection', default=None) == 'Y'
  57
  58             if not is_live:
  59                 is_live = xpath_text(
  60                     relinker, './is_live', default=None) == 'Y'
  61             if not duration:
  62                 duration = parse_duration(xpath_text(
  63                     relinker, './duration', default=None))
  64
  65             url_elem = find_xpath_attr(relinker, './url', 'type', 'content')
  66             if url_elem is None:
  67                 continue
  68
  69             media_url = url_elem.text
  70
  71             # This does not imply geo restriction (e.g.
  72             # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html)
  73             if '/video_no_available.mp4' in media_url:
  74                 continue
  75
  76             ext = determine_ext(media_url)
  77             if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
  78                 continue
  79
  80             if ext == 'mp3':
  81                 formats.append({
  82                     'url': media_url,
  83                     'vcodec': 'none',
  84                     'acodec': 'mp3',
  85                     'format_id': 'http-mp3',
  86                 })
  87                 break
  88             elif ext == 'm3u8' or 'format=m3u8' in media_url or platform == 'mon':
  89                 formats.extend(self._extract_m3u8_formats(
  90                     media_url, video_id, 'mp4', 'm3u8_native',
  91                     m3u8_id='hls', fatal=False))
  92             elif ext == 'f4m' or platform == 'flash':
  93                 manifest_url = update_url_query(
  94                     media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
  95                     {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
  96                 formats.extend(self._extract_f4m_formats(
  97                     manifest_url, video_id, f4m_id='hds', fatal=False))
  98             else:
  99                 bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
 100                 formats.append({
 101                     'url': media_url,
 102                     'tbr': bitrate if bitrate > 0 else None,
 103                     'format_id': f'http-{bitrate if bitrate > 0 else "http"}',
 104                 })
 105
 106         if not formats and geoprotection is True:
 107             self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
 108
 109         if not audio_only:
 110             formats.extend(self._create_http_urls(relinker_url, formats))
 111
 112         return filter_dict({
 113             'is_live': is_live,
 114             'duration': duration,
 115             'formats': formats,
 116         })
 117
 118     def _create_http_urls(self, relinker_url, fmts):
 119         _RELINKER_REG = r'https?://(?P<host>[^/]+?)/(?:i/)?(?P<extra>[^/]+?)/(?P<path>.+?)/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4|/playlist\.m3u8).+?'
 120         _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
 121         _QUALITY = {
 122             # tbr: w, h
 123             '250': [352, 198],
 124             '400': [512, 288],
 125             '700': [512, 288],
 126             '800': [700, 394],
 127             '1200': [736, 414],
 128             '1800': [1024, 576],
 129             '2400': [1280, 720],
 130             '3200': [1440, 810],
 131             '3600': [1440, 810],
 132             '5000': [1920, 1080],
 133             '10000': [1920, 1080],
 134         }
 135
 136         def test_url(url):
 137             resp = self._request_webpage(
 138                 HEADRequest(url), None, headers={'User-Agent': 'Rai'},
 139                 fatal=False, errnote=False, note=False)
 140
 141             if resp is False:
 142                 return False
 143
 144             if resp.code == 200:
 145                 return False if resp.url == url else resp.url
 146             return None
 147
 148         # filter out audio-only formats
 149         fmts = [f for f in fmts if not f.get('vcodec') == 'none']
 150
 151         def get_format_info(tbr):
 152             import math
 153             br = int_or_none(tbr)
 154             if len(fmts) == 1 and not br:
 155                 br = fmts[0].get('tbr')
 156             if br or 0 > 300:
 157                 tbr = compat_str(math.floor(br / 100) * 100)
 158             else:
 159                 tbr = '250'
 160
 161             # try extracting info from available m3u8 formats
 162             format_copy = None
 163             for f in fmts:
 164                 if f.get('tbr'):
 165                     br_limit = math.floor(br / 100)
 166                     if br_limit - 1 <= math.floor(f['tbr'] / 100) <= br_limit + 1:
 167                         format_copy = f.copy()
 168             return {
 169                 'width': format_copy.get('width'),
 170                 'height': format_copy.get('height'),
 171                 'tbr': format_copy.get('tbr'),
 172                 'vcodec': format_copy.get('vcodec'),
 173                 'acodec': format_copy.get('acodec'),
 174                 'fps': format_copy.get('fps'),
 175                 'format_id': f'https-{tbr}',
 176             } if format_copy else {
 177                 'width': _QUALITY[tbr][0],
 178                 'height': _QUALITY[tbr][1],
 179                 'format_id': f'https-{tbr}',
 180                 'tbr': int(tbr),
 181             }
 182
 183         loc = test_url(_MP4_TMPL % (relinker_url, '*'))
 184         if not isinstance(loc, compat_str):
 185             return []
 186
 187         mobj = re.match(
 188             _RELINKER_REG,
 189             test_url(relinker_url) or '')
 190         if not mobj:
 191             return []
 192
 193         available_qualities = mobj.group('quality').split(',') if mobj.group('quality') else ['*']
 194         available_qualities = [i for i in available_qualities if i]
 195
 196         formats = []
 197         for q in available_qualities:
 198             fmt = {
 199                 'url': _MP4_TMPL % (relinker_url, q),
 200                 'protocol': 'https',
 201                 'ext': 'mp4',
 202                 **get_format_info(q)
 203             }
 204             formats.append(fmt)
 205         return formats
 206
 207     @staticmethod
 208     def _extract_subtitles(url, video_data):
 209         STL_EXT = 'stl'
 210         SRT_EXT = 'srt'
 211         subtitles = {}
 212         subtitles_array = video_data.get('subtitlesArray') or []
 213         for k in ('subtitles', 'subtitlesUrl'):
 214             subtitles_array.append({'url': video_data.get(k)})
 215         for subtitle in subtitles_array:
 216             sub_url = subtitle.get('url')
 217             if sub_url and isinstance(sub_url, compat_str):
 218                 sub_lang = subtitle.get('language') or 'it'
 219                 sub_url = urljoin(url, sub_url)
 220                 sub_ext = determine_ext(sub_url, SRT_EXT)
 221                 subtitles.setdefault(sub_lang, []).append({
 222                     'ext': sub_ext,
 223                     'url': sub_url,
 224                 })
 225                 if STL_EXT == sub_ext:
 226                     subtitles[sub_lang].append({
 227                         'ext': SRT_EXT,
 228                         'url': sub_url[:-len(STL_EXT)] + SRT_EXT,
 229                     })
 230         return subtitles
 231
 232
 233 class RaiPlayIE(RaiBaseIE):
 234     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
 235     _TESTS = [{
 236         'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
 237         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
 238         'info_dict': {
 239             'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
 240             'ext': 'mp4',
 241             'title': 'Report del 07/04/2014',
 242             'alt_title': 'St 2013/14 - Report - Espresso nel caffè - 07/04/2014',
 243             'description': 'md5:d730c168a58f4bb35600fc2f881ec04e',
 244             'thumbnail': r're:^https?://.*\.jpg$',
 245             'uploader': 'Rai Gulp',
 246             'duration': 6160,
 247             'series': 'Report',
 248             'season': '2013/14',
 249             'subtitles': {
 250                 'it': 'count:4',
 251             },
 252             'release_year': 2022,
 253             'episode': 'Espresso nel caffè - 07/04/2014',
 254         },
 255         'params': {
 256             'skip_download': True,
 257         },
 258     }, {
 259         # 1080p direct mp4 url
 260         'url': 'https://www.raiplay.it/video/2021/11/Blanca-S1E1-Senza-occhi-b1255a4a-8e72-4a2f-b9f3-fc1308e00736.html',
 261         'md5': 'aeda7243115380b2dd5e881fd42d949a',
 262         'info_dict': {
 263             'id': 'b1255a4a-8e72-4a2f-b9f3-fc1308e00736',
 264             'ext': 'mp4',
 265             'title': 'Blanca - S1E1 - Senza occhi',
 266             'alt_title': 'St 1 Ep 1 - Blanca - Senza occhi',
 267             'description': 'md5:75f95d5c030ec8bac263b1212322e28c',
 268             'thumbnail': r're:^https?://.*\.jpg$',
 269             'uploader': 'Rai 1',
 270             'duration': 6493,
 271             'series': 'Blanca',
 272             'season': 'Season 1',
 273             'episode_number': 1,
 274             'release_year': 2021,
 275             'season_number': 1,
 276             'episode': 'Senza occhi',
 277         },
 278     }, {
 279         'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
 280         'only_matching': True,
 281     }, {
 282         # subtitles at 'subtitlesArray' key (see #27698)
 283         'url': 'https://www.raiplay.it/video/2020/12/Report---04-01-2021-2e90f1de-8eee-4de4-ac0e-78d21db5b600.html',
 284         'only_matching': True,
 285     }, {
 286         # DRM protected
 287         'url': 'https://www.raiplay.it/video/2020/09/Lo-straordinario-mondo-di-Zoey-S1E1-Lo-straordinario-potere-di-Zoey-ed493918-1d32-44b7-8454-862e473d00ff.html',
 288         'only_matching': True,
 289     }]
 290
 291     def _real_extract(self, url):
 292         base, video_id = self._match_valid_url(url).groups()
 293
 294         media = self._download_json(
 295             base + '.json', video_id, 'Downloading video JSON')
 296
 297         if not self.get_param('allow_unplayable_formats'):
 298             if try_get(
 299                     media,
 300                     (lambda x: x['rights_management']['rights']['drm'],
 301                      lambda x: x['program_info']['rights_management']['rights']['drm']),
 302                     dict):
 303                 self.report_drm(video_id)
 304
 305         title = media['name']
 306         video = media['video']
 307
 308         relinker_info = self._extract_relinker_info(video['content_url'], video_id)
 309         self._sort_formats(relinker_info['formats'])
 310
 311         thumbnails = []
 312         for _, value in media.get('images', {}).items():
 313             if value:
 314                 thumbnails.append({
 315                     'url': urljoin(url, value),
 316                 })
 317
 318         date_published = media.get('date_published')
 319         time_published = media.get('time_published')
 320         if date_published and time_published:
 321             date_published += ' ' + time_published
 322
 323         subtitles = self._extract_subtitles(url, video)
 324
 325         program_info = media.get('program_info') or {}
 326         season = media.get('season')
 327
 328         alt_title = join_nonempty(media.get('subtitle'), media.get('toptitle'), delim=' - ')
 329
 330         return {
 331             'id': remove_start(media.get('id'), 'ContentItem-') or video_id,
 332             'display_id': video_id,
 333             'title': title,
 334             'alt_title': strip_or_none(alt_title or None),
 335             'description': media.get('description'),
 336             'uploader': strip_or_none(media.get('channel') or None),
 337             'creator': strip_or_none(media.get('editor') or None),
 338             'duration': parse_duration(video.get('duration')),
 339             'timestamp': unified_timestamp(date_published),
 340             'thumbnails': thumbnails,
 341             'series': program_info.get('name'),
 342             'season_number': int_or_none(season),
 343             'season': season if (season and not season.isdigit()) else None,
 344             'episode': media.get('episode_title'),
 345             'episode_number': int_or_none(media.get('episode')),
 346             'subtitles': subtitles,
 347             'release_year': int_or_none(traverse_obj(media, ('track_info', 'edit_year'))),
 348             **relinker_info
 349         }
 350
 351
 352 class RaiPlayLiveIE(RaiPlayIE):
 353     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/dirette/(?P<id>[^/?#&]+))'
 354     _TESTS = [{
 355         'url': 'http://www.raiplay.it/dirette/rainews24',
 356         'info_dict': {
 357             'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c',
 358             'display_id': 'rainews24',
 359             'ext': 'mp4',
 360             'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 361             'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497',
 362             'uploader': 'Rai News 24',
 363             'creator': 'Rai News 24',
 364             'is_live': True,
 365             'live_status': 'is_live',
 366         },
 367         'params': {
 368             'skip_download': True,
 369         },
 370     }]
 371
 372
 373 class RaiPlayPlaylistIE(InfoExtractor):
 374     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
 375     _TESTS = [{
 376         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/',
 377         'info_dict': {
 378             'id': 'nondirloalmiocapo',
 379             'title': 'Non dirlo al mio capo',
 380             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
 381         },
 382         'playlist_mincount': 12,
 383     }, {
 384         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/episodi/stagione-2/',
 385         'info_dict': {
 386             'id': 'nondirloalmiocapo',
 387             'title': 'Non dirlo al mio capo - Stagione 2',
 388             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
 389         },
 390         'playlist_mincount': 12,
 391     }]
 392
 393     def _real_extract(self, url):
 394         base, playlist_id, extra_id = self._match_valid_url(url).groups()
 395
 396         program = self._download_json(
 397             base + '.json', playlist_id, 'Downloading program JSON')
 398
 399         if extra_id:
 400             extra_id = extra_id.upper().rstrip('/')
 401
 402         playlist_title = program.get('name')
 403         entries = []
 404         for b in (program.get('blocks') or []):
 405             for s in (b.get('sets') or []):
 406                 if extra_id:
 407                     if extra_id != join_nonempty(
 408                             b.get('name'), s.get('name'), delim='/').replace(' ', '-').upper():
 409                         continue
 410                     playlist_title = join_nonempty(playlist_title, s.get('name'), delim=' - ')
 411
 412                 s_id = s.get('id')
 413                 if not s_id:
 414                     continue
 415                 medias = self._download_json(
 416                     f'{base}/{s_id}.json', s_id,
 417                     'Downloading content set JSON', fatal=False)
 418                 if not medias:
 419                     continue
 420                 for m in (medias.get('items') or []):
 421                     path_id = m.get('path_id')
 422                     if not path_id:
 423                         continue
 424                     video_url = urljoin(url, path_id)
 425                     entries.append(self.url_result(
 426                         video_url, ie=RaiPlayIE.ie_key(),
 427                         video_id=RaiPlayIE._match_id(video_url)))
 428
 429         return self.playlist_result(
 430             entries, playlist_id, playlist_title,
 431             try_get(program, lambda x: x['program_info']['description']))
 432
 433
 434 class RaiPlaySoundIE(RaiBaseIE):
 435     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplaysound\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
 436     _TESTS = [{
 437         'url': 'https://www.raiplaysound.it/audio/2021/12/IL-RUGGITO-DEL-CONIGLIO-1ebae2a7-7cdb-42bb-842e-fe0d193e9707.html',
 438         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
 439         'info_dict': {
 440             'id': '1ebae2a7-7cdb-42bb-842e-fe0d193e9707',
 441             'ext': 'mp3',
 442             'title': 'Il Ruggito del Coniglio del 10/12/2021',
 443             'alt_title': 'md5:0e6476cd57858bb0f3fcc835d305b455',
 444             'description': 'md5:2a17d2107e59a4a8faa0e18334139ee2',
 445             'thumbnail': r're:^https?://.*\.jpg$',
 446             'uploader': 'rai radio 2',
 447             'duration': 5685,
 448             'series': 'Il Ruggito del Coniglio',
 449             'episode': 'Il Ruggito del Coniglio del 10/12/2021',
 450             'creator': 'rai radio 2',
 451         },
 452         'params': {
 453             'skip_download': True,
 454         },
 455     }]
 456
 457     def _real_extract(self, url):
 458         base, audio_id = self._match_valid_url(url).group('base', 'id')
 459         media = self._download_json(f'{base}.json', audio_id, 'Downloading audio JSON')
 460         uid = try_get(media, lambda x: remove_start(remove_start(x['uniquename'], 'ContentItem-'), 'Page-'))
 461
 462         info = {}
 463         formats = []
 464         relinkers = set(traverse_obj(media, (('downloadable_audio', 'audio', ('live', 'cards', 0, 'audio')), 'url')))
 465         for r in relinkers:
 466             info = self._extract_relinker_info(r, audio_id, True)
 467             formats.extend(info.get('formats'))
 468
 469         date_published = try_get(media, (lambda x: f'{x["create_date"]} {x.get("create_time") or ""}',
 470                                          lambda x: x['live']['create_date']))
 471
 472         podcast_info = traverse_obj(media, 'podcast_info', ('live', 'cards', 0)) or {}
 473         thumbnails = [{
 474             'url': urljoin(url, thumb_url),
 475         } for thumb_url in (podcast_info.get('images') or {}).values() if thumb_url]
 476
 477         return {
 478             **info,
 479             'id': uid or audio_id,
 480             'display_id': audio_id,
 481             'title': traverse_obj(media, 'title', 'episode_title'),
 482             'alt_title': traverse_obj(media, ('track_info', 'media_name'), expected_type=strip_or_none),
 483             'description': media.get('description'),
 484             'uploader': traverse_obj(media, ('track_info', 'channel'), expected_type=strip_or_none),
 485             'creator': traverse_obj(media, ('track_info', 'editor'), expected_type=strip_or_none),
 486             'timestamp': unified_timestamp(date_published),
 487             'thumbnails': thumbnails,
 488             'series': podcast_info.get('title'),
 489             'season_number': int_or_none(media.get('season')),
 490             'episode': media.get('episode_title'),
 491             'episode_number': int_or_none(media.get('episode')),
 492             'formats': formats,
 493         }
 494
 495
 496 class RaiPlaySoundLiveIE(RaiPlaySoundIE):
 497     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?P<id>[^/?#&]+)$)'
 498     _TESTS = [{
 499         'url': 'https://www.raiplaysound.it/radio2',
 500         'info_dict': {
 501             'id': 'b00a50e6-f404-4af6-8f8c-ff3b9af73a44',
 502             'display_id': 'radio2',
 503             'ext': 'mp4',
 504             'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+',
 505             'thumbnail': r're:https://www.raiplaysound.it/dl/img/.+?png',
 506             'uploader': 'rai radio 2',
 507             'series': 'Rai Radio 2',
 508             'creator': 'raiplaysound',
 509             'is_live': True,
 510             'live_status': 'is_live',
 511         },
 512         'params': {
 513             'skip_download': 'live',
 514         },
 515     }]
 516
 517
 518 class RaiPlaySoundPlaylistIE(InfoExtractor):
 519     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?:programmi|playlist|audiolibri)/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
 520     _TESTS = [{
 521         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio',
 522         'info_dict': {
 523             'id': 'ilruggitodelconiglio',
 524             'title': 'Il Ruggito del Coniglio',
 525             'description': 'md5:1bbaf631245a7ab1ec4d9fbb3c7aa8f3',
 526         },
 527         'playlist_mincount': 65,
 528     }, {
 529         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio/puntate/prima-stagione-1995',
 530         'info_dict': {
 531             'id': 'ilruggitodelconiglio_puntate_prima-stagione-1995',
 532             'title': 'Prima Stagione 1995',
 533         },
 534         'playlist_count': 1,
 535     }]
 536
 537     def _real_extract(self, url):
 538         base, playlist_id, extra_id = self._match_valid_url(url).group('base', 'id', 'extra_id')
 539         url = f'{base}.json'
 540         program = self._download_json(url, playlist_id, 'Downloading program JSON')
 541
 542         if extra_id:
 543             extra_id = extra_id.rstrip('/')
 544             playlist_id += '_' + extra_id.replace('/', '_')
 545             path = next(c['path_id'] for c in program.get('filters') or [] if extra_id in c.get('weblink'))
 546             program = self._download_json(
 547                 urljoin('https://www.raiplaysound.it', path), playlist_id, 'Downloading program secondary JSON')
 548
 549         entries = [
 550             self.url_result(urljoin(base, c['path_id']), ie=RaiPlaySoundIE.ie_key())
 551             for c in traverse_obj(program, 'cards', ('block', 'cards')) or []
 552             if c.get('path_id')]
 553
 554         return self.playlist_result(entries, playlist_id, program.get('title'),
 555                                     traverse_obj(program, ('podcast_info', 'description')))
 556
 557
 558 class RaiIE(RaiBaseIE):
 559     _VALID_URL = rf'https?://[^/]+\.(?:rai\.(?:it|tv))/.+?-(?P<id>{RaiBaseIE._UUID_RE})(?:-.+?)?\.html'
 560     _TESTS = [{
 561         # var uniquename = "ContentItem-..."
 562         # data-id="ContentItem-..."
 563         'url': 'https://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
 564         'info_dict': {
 565             'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
 566             'ext': 'mp4',
 567             'title': 'TG PRIMO TEMPO',
 568             'thumbnail': r're:^https?://.*\.jpg$',
 569             'duration': 1758,
 570             'upload_date': '20140612',
 571         },
 572         'skip': 'This content is available only in Italy',
 573     }, {
 574         # with ContentItem in og:url
 575         'url': 'https://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
 576         'md5': '06345bd97c932f19ffb129973d07a020',
 577         'info_dict': {
 578             'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
 579             'ext': 'mp4',
 580             'title': 'TG1 ore 20:00 del 03/11/2016',
 581             'description': 'TG1 edizione integrale ore 20:00 del giorno 03/11/2016',
 582             'thumbnail': r're:^https?://.*\.jpg$',
 583             'duration': 2214,
 584             'upload_date': '20161103'
 585         }
 586     }, {
 587         # Direct MMS URL
 588         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html',
 589         'only_matching': True,
 590     }]
 591
 592     def _extract_from_content_id(self, content_id, url):
 593         media = self._download_json(
 594             f'https://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-{content_id}.html?json',
 595             content_id, 'Downloading video JSON')
 596
 597         title = media['name'].strip()
 598
 599         media_type = media['type']
 600         if 'Audio' in media_type:
 601             relinker_info = {
 602                 'formats': [{
 603                     'format_id': media.get('formatoAudio'),
 604                     'url': media['audioUrl'],
 605                     'ext': media.get('formatoAudio'),
 606                 }]
 607             }
 608         elif 'Video' in media_type:
 609             relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)
 610         else:
 611             raise ExtractorError('not a media file')
 612
 613         self._sort_formats(relinker_info['formats'])
 614
 615         thumbnails = []
 616         for image_type in ('image', 'image_medium', 'image_300'):
 617             thumbnail_url = media.get(image_type)
 618             if thumbnail_url:
 619                 thumbnails.append({
 620                     'url': compat_urlparse.urljoin(url, thumbnail_url),
 621                 })
 622
 623         subtitles = self._extract_subtitles(url, media)
 624
 625         return {
 626             'id': content_id,
 627             'title': title,
 628             'description': strip_or_none(media.get('desc') or None),
 629             'thumbnails': thumbnails,
 630             'uploader': strip_or_none(media.get('author') or None),
 631             'upload_date': unified_strdate(media.get('date')),
 632             'duration': parse_duration(media.get('length')),
 633             'subtitles': subtitles,
 634             **relinker_info
 635         }
 636
 637     def _real_extract(self, url):
 638         video_id = self._match_id(url)
 639
 640         webpage = self._download_webpage(url, video_id)
 641
 642         content_item_id = None
 643
 644         content_item_url = self._html_search_meta(
 645             ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url',
 646              'twitter:player', 'jsonlink'), webpage, default=None)
 647         if content_item_url:
 648             content_item_id = self._search_regex(
 649                 rf'ContentItem-({self._UUID_RE})', content_item_url,
 650                 'content item id', default=None)
 651
 652         if not content_item_id:
 653             content_item_id = self._search_regex(
 654                 rf'''(?x)
 655                     (?:
 656                         (?:initEdizione|drawMediaRaiTV)\(|
 657                         <(?:[^>]+\bdata-id|var\s+uniquename)=|
 658                         <iframe[^>]+\bsrc=
 659                     )
 660                     (["\'])
 661                     (?:(?!\1).)*\bContentItem-(?P<id>{self._UUID_RE})
 662                 ''',
 663                 webpage, 'content item id', default=None, group='id')
 664
 665         content_item_ids = set()
 666         if content_item_id:
 667             content_item_ids.add(content_item_id)
 668         if video_id not in content_item_ids:
 669             content_item_ids.add(video_id)
 670
 671         for content_item_id in content_item_ids:
 672             try:
 673                 return self._extract_from_content_id(content_item_id, url)
 674             except GeoRestrictedError:
 675                 raise
 676             except ExtractorError:
 677                 pass
 678
 679         relinker_url = self._proto_relative_url(self._search_regex(
 680             r'''(?x)
 681                 (?:
 682                     var\s+videoURL|
 683                     mediaInfo\.mediaUri
 684                 )\s*=\s*
 685                 ([\'"])
 686                 (?P<url>
 687                     (?:https?:)?
 688                     //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\?
 689                     (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1
 690             ''',
 691             webpage, 'relinker URL', group='url'))
 692
 693         relinker_info = self._extract_relinker_info(
 694             urljoin(url, relinker_url), video_id)
 695         self._sort_formats(relinker_info['formats'])
 696
 697         title = self._search_regex(
 698             r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
 699             webpage, 'title', group='title',
 700             default=None) or self._og_search_title(webpage)
 701
 702         return {
 703             'id': video_id,
 704             'title': title,
 705             **relinker_info
 706         }
 707
 708
 709 class RaiNewsIE(RaiIE):
 710     _VALID_URL = rf'https?://(www\.)?rainews\.it/[^?#]+-(?P<id>{RaiBaseIE._UUID_RE})(?:-[^/?#]+)?\.html'
 711     _TESTS = [{
 712         # new rainews player (#3911)
 713         'url': 'https://www.rainews.it/rubriche/24mm/video/2022/05/24mm-del-29052022-12cf645d-1ffd-4220-b27c-07c226dbdecf.html',
 714         'info_dict': {
 715             'id': '12cf645d-1ffd-4220-b27c-07c226dbdecf',
 716             'ext': 'mp4',
 717             'title': 'Puntata del 29/05/2022',
 718             'duration': 1589,
 719             'upload_date': '20220529',
 720             'uploader': 'rainews',
 721         }
 722     }, {
 723         # old content with fallback method to extract media urls
 724         'url': 'https://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
 725         'info_dict': {
 726             'id': '1632c009-c843-4836-bb65-80c33084a64b',
 727             'ext': 'mp4',
 728             'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor "La ragazza del treno"',
 729             'description': 'I film in uscita questa settimana.',
 730             'thumbnail': r're:^https?://.*\.png$',
 731             'duration': 833,
 732             'upload_date': '20161103'
 733         },
 734         'expected_warnings': ['unable to extract player_data'],
 735     }]
 736
 737     def _real_extract(self, url):
 738         video_id = self._match_id(url)
 739
 740         webpage = self._download_webpage(url, video_id)
 741
 742         player_data = self._search_json(
 743             r'<rainews-player\s*data=\'', webpage, 'player_data', video_id,
 744             transform_source=clean_html, fatal=False)
 745         track_info = player_data.get('track_info')
 746         relinker_url = traverse_obj(player_data, 'mediapolis', 'content_url')
 747
 748         if not relinker_url:
 749             # fallback on old implementation for some old content
 750             try:
 751                 return self._extract_from_content_id(video_id, url)
 752             except GeoRestrictedError:
 753                 raise
 754             except ExtractorError as e:
 755                 raise ExtractorError('Relinker URL not found', cause=e)
 756
 757         relinker_info = self._extract_relinker_info(urljoin(url, relinker_url), video_id)
 758         self._sort_formats(relinker_info['formats'])
 759
 760         return {
 761             'id': video_id,
 762             'title': track_info.get('title') or self._og_search_title(webpage),
 763             'upload_date': unified_strdate(track_info.get('date')),
 764             'uploader': strip_or_none(track_info.get('editor') or None),
 765             **relinker_info
 766         }