yt_dlp/extractor/rai.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..compat import (
   5     compat_str,
   6     compat_urlparse,
   7 )
   8 from ..utils import (
   9     clean_html,
  10     determine_ext,
  11     ExtractorError,
  12     filter_dict,
  13     find_xpath_attr,
  14     fix_xml_ampersands,
  15     GeoRestrictedError,
  16     HEADRequest,
  17     int_or_none,
  18     join_nonempty,
  19     parse_duration,
  20     remove_start,
  21     strip_or_none,
  22     traverse_obj,
  23     try_get,
  24     unified_strdate,
  25     unified_timestamp,
  26     update_url_query,
  27     urljoin,
  28     xpath_text,
  29 )
  30
  31
  32 class RaiBaseIE(InfoExtractor):
  33     _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
  34     _GEO_COUNTRIES = ['IT']
  35     _GEO_BYPASS = False
  36
  37     def _extract_relinker_info(self, relinker_url, video_id, audio_only=False):
  38         if not re.match(r'https?://', relinker_url):
  39             return {'formats': [{'url': relinker_url}]}
  40
  41         formats = []
  42         geoprotection = None
  43         is_live = None
  44         duration = None
  45
  46         for platform in ('mon', 'flash', 'native'):
  47             relinker = self._download_xml(
  48                 relinker_url, video_id,
  49                 note=f'Downloading XML metadata for platform {platform}',
  50                 transform_source=fix_xml_ampersands,
  51                 query={'output': 45, 'pl': platform},
  52                 headers=self.geo_verification_headers())
  53
  54             if xpath_text(relinker, './license_url', default='{}') != '{}':
  55                 self.report_drm(video_id)
  56
  57             if not geoprotection:
  58                 geoprotection = xpath_text(
  59                     relinker, './geoprotection', default=None) == 'Y'
  60
  61             if not is_live:
  62                 is_live = xpath_text(
  63                     relinker, './is_live', default=None) == 'Y'
  64             if not duration:
  65                 duration = parse_duration(xpath_text(
  66                     relinker, './duration', default=None))
  67
  68             url_elem = find_xpath_attr(relinker, './url', 'type', 'content')
  69             if url_elem is None:
  70                 continue
  71
  72             media_url = url_elem.text
  73
  74             # This does not imply geo restriction (e.g.
  75             # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html)
  76             if '/video_no_available.mp4' in media_url:
  77                 continue
  78
  79             ext = determine_ext(media_url)
  80             if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
  81                 continue
  82
  83             if ext == 'mp3':
  84                 formats.append({
  85                     'url': media_url,
  86                     'vcodec': 'none',
  87                     'acodec': 'mp3',
  88                     'format_id': 'http-mp3',
  89                 })
  90                 break
  91             elif ext == 'm3u8' or 'format=m3u8' in media_url or platform == 'mon':
  92                 formats.extend(self._extract_m3u8_formats(
  93                     media_url, video_id, 'mp4', 'm3u8_native',
  94                     m3u8_id='hls', fatal=False))
  95             elif ext == 'f4m' or platform == 'flash':
  96                 manifest_url = update_url_query(
  97                     media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
  98                     {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
  99                 formats.extend(self._extract_f4m_formats(
 100                     manifest_url, video_id, f4m_id='hds', fatal=False))
 101             else:
 102                 bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
 103                 formats.append({
 104                     'url': media_url,
 105                     'tbr': bitrate if bitrate > 0 else None,
 106                     'format_id': f'http-{bitrate if bitrate > 0 else "http"}',
 107                 })
 108
 109         if not formats and geoprotection is True:
 110             self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
 111
 112         if not audio_only:
 113             formats.extend(self._create_http_urls(relinker_url, formats))
 114
 115         return filter_dict({
 116             'is_live': is_live,
 117             'duration': duration,
 118             'formats': formats,
 119         })
 120
 121     def _create_http_urls(self, relinker_url, fmts):
 122         _RELINKER_REG = r'https?://(?P<host>[^/]+?)/(?:i/)?(?P<extra>[^/]+?)/(?P<path>.+?)/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4|/playlist\.m3u8).+?'
 123         _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
 124         _QUALITY = {
 125             # tbr: w, h
 126             '250': [352, 198],
 127             '400': [512, 288],
 128             '700': [512, 288],
 129             '800': [700, 394],
 130             '1200': [736, 414],
 131             '1800': [1024, 576],
 132             '2400': [1280, 720],
 133             '3200': [1440, 810],
 134             '3600': [1440, 810],
 135             '5000': [1920, 1080],
 136             '10000': [1920, 1080],
 137         }
 138
 139         def test_url(url):
 140             resp = self._request_webpage(
 141                 HEADRequest(url), None, headers={'User-Agent': 'Rai'},
 142                 fatal=False, errnote=False, note=False)
 143
 144             if resp is False:
 145                 return False
 146
 147             if resp.code == 200:
 148                 return False if resp.url == url else resp.url
 149             return None
 150
 151         # filter out audio-only formats
 152         fmts = [f for f in fmts if not f.get('vcodec') == 'none']
 153
 154         def get_format_info(tbr):
 155             import math
 156             br = int_or_none(tbr)
 157             if len(fmts) == 1 and not br:
 158                 br = fmts[0].get('tbr')
 159             if br and br > 300:
 160                 tbr = compat_str(math.floor(br / 100) * 100)
 161             else:
 162                 tbr = '250'
 163
 164             # try extracting info from available m3u8 formats
 165             format_copy = None
 166             for f in fmts:
 167                 if f.get('tbr'):
 168                     br_limit = math.floor(br / 100)
 169                     if br_limit - 1 <= math.floor(f['tbr'] / 100) <= br_limit + 1:
 170                         format_copy = f.copy()
 171             return {
 172                 'width': format_copy.get('width'),
 173                 'height': format_copy.get('height'),
 174                 'tbr': format_copy.get('tbr'),
 175                 'vcodec': format_copy.get('vcodec'),
 176                 'acodec': format_copy.get('acodec'),
 177                 'fps': format_copy.get('fps'),
 178                 'format_id': f'https-{tbr}',
 179             } if format_copy else {
 180                 'width': _QUALITY[tbr][0],
 181                 'height': _QUALITY[tbr][1],
 182                 'format_id': f'https-{tbr}',
 183                 'tbr': int(tbr),
 184             }
 185
 186         loc = test_url(_MP4_TMPL % (relinker_url, '*'))
 187         if not isinstance(loc, compat_str):
 188             return []
 189
 190         mobj = re.match(
 191             _RELINKER_REG,
 192             test_url(relinker_url) or '')
 193         if not mobj:
 194             return []
 195
 196         available_qualities = mobj.group('quality').split(',') if mobj.group('quality') else ['*']
 197         available_qualities = [i for i in available_qualities if i]
 198
 199         formats = []
 200         for q in available_qualities:
 201             fmt = {
 202                 'url': _MP4_TMPL % (relinker_url, q),
 203                 'protocol': 'https',
 204                 'ext': 'mp4',
 205                 **get_format_info(q)
 206             }
 207             formats.append(fmt)
 208         return formats
 209
 210     @staticmethod
 211     def _extract_subtitles(url, video_data):
 212         STL_EXT = 'stl'
 213         SRT_EXT = 'srt'
 214         subtitles = {}
 215         subtitles_array = video_data.get('subtitlesArray') or []
 216         for k in ('subtitles', 'subtitlesUrl'):
 217             subtitles_array.append({'url': video_data.get(k)})
 218         for subtitle in subtitles_array:
 219             sub_url = subtitle.get('url')
 220             if sub_url and isinstance(sub_url, compat_str):
 221                 sub_lang = subtitle.get('language') or 'it'
 222                 sub_url = urljoin(url, sub_url)
 223                 sub_ext = determine_ext(sub_url, SRT_EXT)
 224                 subtitles.setdefault(sub_lang, []).append({
 225                     'ext': sub_ext,
 226                     'url': sub_url,
 227                 })
 228                 if STL_EXT == sub_ext:
 229                     subtitles[sub_lang].append({
 230                         'ext': SRT_EXT,
 231                         'url': sub_url[:-len(STL_EXT)] + SRT_EXT,
 232                     })
 233         return subtitles
 234
 235
 236 class RaiPlayIE(RaiBaseIE):
 237     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
 238     _TESTS = [{
 239         'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
 240         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
 241         'info_dict': {
 242             'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
 243             'ext': 'mp4',
 244             'title': 'Report del 07/04/2014',
 245             'alt_title': 'St 2013/14 - Report - Espresso nel caffè - 07/04/2014',
 246             'description': 'md5:d730c168a58f4bb35600fc2f881ec04e',
 247             'thumbnail': r're:^https?://.*\.jpg$',
 248             'uploader': 'Rai Gulp',
 249             'duration': 6160,
 250             'series': 'Report',
 251             'season': '2013/14',
 252             'subtitles': {
 253                 'it': 'count:4',
 254             },
 255             'release_year': 2022,
 256             'episode': 'Espresso nel caffè - 07/04/2014',
 257             'timestamp': 1396919880,
 258             'upload_date': '20140408',
 259         },
 260         'params': {
 261             'skip_download': True,
 262         },
 263     }, {
 264         # 1080p direct mp4 url
 265         'url': 'https://www.raiplay.it/video/2021/11/Blanca-S1E1-Senza-occhi-b1255a4a-8e72-4a2f-b9f3-fc1308e00736.html',
 266         'md5': 'aeda7243115380b2dd5e881fd42d949a',
 267         'info_dict': {
 268             'id': 'b1255a4a-8e72-4a2f-b9f3-fc1308e00736',
 269             'ext': 'mp4',
 270             'title': 'Blanca - S1E1 - Senza occhi',
 271             'alt_title': 'St 1 Ep 1 - Blanca - Senza occhi',
 272             'description': 'md5:75f95d5c030ec8bac263b1212322e28c',
 273             'thumbnail': r're:^https?://.*\.jpg$',
 274             'uploader': 'Rai 1',
 275             'duration': 6493,
 276             'series': 'Blanca',
 277             'season': 'Season 1',
 278             'episode_number': 1,
 279             'release_year': 2021,
 280             'season_number': 1,
 281             'episode': 'Senza occhi',
 282             'timestamp': 1637318940,
 283             'upload_date': '20211119',
 284         },
 285     }, {
 286         'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
 287         'only_matching': True,
 288     }, {
 289         # subtitles at 'subtitlesArray' key (see #27698)
 290         'url': 'https://www.raiplay.it/video/2020/12/Report---04-01-2021-2e90f1de-8eee-4de4-ac0e-78d21db5b600.html',
 291         'only_matching': True,
 292     }, {
 293         # DRM protected
 294         'url': 'https://www.raiplay.it/video/2021/06/Lo-straordinario-mondo-di-Zoey-S2E1-Lo-straordinario-ritorno-di-Zoey-3ba992de-2332-41ad-9214-73e32ab209f4.html',
 295         'only_matching': True,
 296     }]
 297
 298     def _real_extract(self, url):
 299         base, video_id = self._match_valid_url(url).groups()
 300
 301         media = self._download_json(
 302             base + '.json', video_id, 'Downloading video JSON')
 303
 304         if not self.get_param('allow_unplayable_formats'):
 305             if try_get(
 306                     media,
 307                     (lambda x: x['rights_management']['rights']['drm'],
 308                      lambda x: x['program_info']['rights_management']['rights']['drm']),
 309                     dict):
 310                 self.report_drm(video_id)
 311
 312         title = media['name']
 313         video = media['video']
 314
 315         relinker_info = self._extract_relinker_info(video['content_url'], video_id)
 316
 317         thumbnails = []
 318         for _, value in media.get('images', {}).items():
 319             if value:
 320                 thumbnails.append({
 321                     'url': urljoin(url, value),
 322                 })
 323
 324         date_published = media.get('date_published')
 325         time_published = media.get('time_published')
 326         if date_published and time_published:
 327             date_published += ' ' + time_published
 328
 329         subtitles = self._extract_subtitles(url, video)
 330
 331         program_info = media.get('program_info') or {}
 332         season = media.get('season')
 333
 334         alt_title = join_nonempty(media.get('subtitle'), media.get('toptitle'), delim=' - ')
 335
 336         return {
 337             'id': remove_start(media.get('id'), 'ContentItem-') or video_id,
 338             'display_id': video_id,
 339             'title': title,
 340             'alt_title': strip_or_none(alt_title or None),
 341             'description': media.get('description'),
 342             'uploader': strip_or_none(media.get('channel') or None),
 343             'creator': strip_or_none(media.get('editor') or None),
 344             'duration': parse_duration(video.get('duration')),
 345             'timestamp': unified_timestamp(date_published),
 346             'thumbnails': thumbnails,
 347             'series': program_info.get('name'),
 348             'season_number': int_or_none(season),
 349             'season': season if (season and not season.isdigit()) else None,
 350             'episode': media.get('episode_title'),
 351             'episode_number': int_or_none(media.get('episode')),
 352             'subtitles': subtitles,
 353             'release_year': int_or_none(traverse_obj(media, ('track_info', 'edit_year'))),
 354             **relinker_info
 355         }
 356
 357
 358 class RaiPlayLiveIE(RaiPlayIE):  # XXX: Do not subclass from concrete IE
 359     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/dirette/(?P<id>[^/?#&]+))'
 360     _TESTS = [{
 361         'url': 'http://www.raiplay.it/dirette/rainews24',
 362         'info_dict': {
 363             'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c',
 364             'display_id': 'rainews24',
 365             'ext': 'mp4',
 366             'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 367             'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497',
 368             'uploader': 'Rai News 24',
 369             'creator': 'Rai News 24',
 370             'is_live': True,
 371             'live_status': 'is_live',
 372             'upload_date': '20090502',
 373             'timestamp': 1241276220,
 374         },
 375         'params': {
 376             'skip_download': True,
 377         },
 378     }]
 379
 380
 381 class RaiPlayPlaylistIE(InfoExtractor):
 382     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
 383     _TESTS = [{
 384         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/',
 385         'info_dict': {
 386             'id': 'nondirloalmiocapo',
 387             'title': 'Non dirlo al mio capo',
 388             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
 389         },
 390         'playlist_mincount': 12,
 391     }, {
 392         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/episodi/stagione-2/',
 393         'info_dict': {
 394             'id': 'nondirloalmiocapo',
 395             'title': 'Non dirlo al mio capo - Stagione 2',
 396             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
 397         },
 398         'playlist_mincount': 12,
 399     }]
 400
 401     def _real_extract(self, url):
 402         base, playlist_id, extra_id = self._match_valid_url(url).groups()
 403
 404         program = self._download_json(
 405             base + '.json', playlist_id, 'Downloading program JSON')
 406
 407         if extra_id:
 408             extra_id = extra_id.upper().rstrip('/')
 409
 410         playlist_title = program.get('name')
 411         entries = []
 412         for b in (program.get('blocks') or []):
 413             for s in (b.get('sets') or []):
 414                 if extra_id:
 415                     if extra_id != join_nonempty(
 416                             b.get('name'), s.get('name'), delim='/').replace(' ', '-').upper():
 417                         continue
 418                     playlist_title = join_nonempty(playlist_title, s.get('name'), delim=' - ')
 419
 420                 s_id = s.get('id')
 421                 if not s_id:
 422                     continue
 423                 medias = self._download_json(
 424                     f'{base}/{s_id}.json', s_id,
 425                     'Downloading content set JSON', fatal=False)
 426                 if not medias:
 427                     continue
 428                 for m in (medias.get('items') or []):
 429                     path_id = m.get('path_id')
 430                     if not path_id:
 431                         continue
 432                     video_url = urljoin(url, path_id)
 433                     entries.append(self.url_result(
 434                         video_url, ie=RaiPlayIE.ie_key(),
 435                         video_id=RaiPlayIE._match_id(video_url)))
 436
 437         return self.playlist_result(
 438             entries, playlist_id, playlist_title,
 439             try_get(program, lambda x: x['program_info']['description']))
 440
 441
 442 class RaiPlaySoundIE(RaiBaseIE):
 443     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplaysound\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
 444     _TESTS = [{
 445         'url': 'https://www.raiplaysound.it/audio/2021/12/IL-RUGGITO-DEL-CONIGLIO-1ebae2a7-7cdb-42bb-842e-fe0d193e9707.html',
 446         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
 447         'info_dict': {
 448             'id': '1ebae2a7-7cdb-42bb-842e-fe0d193e9707',
 449             'ext': 'mp3',
 450             'title': 'Il Ruggito del Coniglio del 10/12/2021',
 451             'alt_title': 'md5:0e6476cd57858bb0f3fcc835d305b455',
 452             'description': 'md5:2a17d2107e59a4a8faa0e18334139ee2',
 453             'thumbnail': r're:^https?://.*\.jpg$',
 454             'uploader': 'rai radio 2',
 455             'duration': 5685,
 456             'series': 'Il Ruggito del Coniglio',
 457             'episode': 'Il Ruggito del Coniglio del 10/12/2021',
 458             'creator': 'rai radio 2',
 459             'timestamp': 1638346620,
 460             'upload_date': '20211201',
 461         },
 462         'params': {
 463             'skip_download': True,
 464         },
 465     }]
 466
 467     def _real_extract(self, url):
 468         base, audio_id = self._match_valid_url(url).group('base', 'id')
 469         media = self._download_json(f'{base}.json', audio_id, 'Downloading audio JSON')
 470         uid = try_get(media, lambda x: remove_start(remove_start(x['uniquename'], 'ContentItem-'), 'Page-'))
 471
 472         info = {}
 473         formats = []
 474         relinkers = set(traverse_obj(media, (('downloadable_audio', 'audio', ('live', 'cards', 0, 'audio')), 'url')))
 475         for r in relinkers:
 476             info = self._extract_relinker_info(r, audio_id, True)
 477             formats.extend(info.get('formats'))
 478
 479         date_published = try_get(media, (lambda x: f'{x["create_date"]} {x.get("create_time") or ""}',
 480                                          lambda x: x['live']['create_date']))
 481
 482         podcast_info = traverse_obj(media, 'podcast_info', ('live', 'cards', 0)) or {}
 483         thumbnails = [{
 484             'url': urljoin(url, thumb_url),
 485         } for thumb_url in (podcast_info.get('images') or {}).values() if thumb_url]
 486
 487         return {
 488             **info,
 489             'id': uid or audio_id,
 490             'display_id': audio_id,
 491             'title': traverse_obj(media, 'title', 'episode_title'),
 492             'alt_title': traverse_obj(media, ('track_info', 'media_name'), expected_type=strip_or_none),
 493             'description': media.get('description'),
 494             'uploader': traverse_obj(media, ('track_info', 'channel'), expected_type=strip_or_none),
 495             'creator': traverse_obj(media, ('track_info', 'editor'), expected_type=strip_or_none),
 496             'timestamp': unified_timestamp(date_published),
 497             'thumbnails': thumbnails,
 498             'series': podcast_info.get('title'),
 499             'season_number': int_or_none(media.get('season')),
 500             'episode': media.get('episode_title'),
 501             'episode_number': int_or_none(media.get('episode')),
 502             'formats': formats,
 503         }
 504
 505
 506 class RaiPlaySoundLiveIE(RaiPlaySoundIE):  # XXX: Do not subclass from concrete IE
 507     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?P<id>[^/?#&]+)$)'
 508     _TESTS = [{
 509         'url': 'https://www.raiplaysound.it/radio2',
 510         'info_dict': {
 511             'id': 'b00a50e6-f404-4af6-8f8c-ff3b9af73a44',
 512             'display_id': 'radio2',
 513             'ext': 'mp4',
 514             'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+',
 515             'thumbnail': r're:https://www.raiplaysound.it/dl/img/.+?png',
 516             'uploader': 'rai radio 2',
 517             'series': 'Rai Radio 2',
 518             'creator': 'raiplaysound',
 519             'is_live': True,
 520             'live_status': 'is_live',
 521         },
 522         'params': {
 523             'skip_download': 'live',
 524         },
 525     }]
 526
 527
 528 class RaiPlaySoundPlaylistIE(InfoExtractor):
 529     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?:programmi|playlist|audiolibri)/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
 530     _TESTS = [{
 531         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio',
 532         'info_dict': {
 533             'id': 'ilruggitodelconiglio',
 534             'title': 'Il Ruggito del Coniglio',
 535             'description': 'md5:1bbaf631245a7ab1ec4d9fbb3c7aa8f3',
 536         },
 537         'playlist_mincount': 65,
 538     }, {
 539         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio/puntate/prima-stagione-1995',
 540         'info_dict': {
 541             'id': 'ilruggitodelconiglio_puntate_prima-stagione-1995',
 542             'title': 'Prima Stagione 1995',
 543         },
 544         'playlist_count': 1,
 545     }]
 546
 547     def _real_extract(self, url):
 548         base, playlist_id, extra_id = self._match_valid_url(url).group('base', 'id', 'extra_id')
 549         url = f'{base}.json'
 550         program = self._download_json(url, playlist_id, 'Downloading program JSON')
 551
 552         if extra_id:
 553             extra_id = extra_id.rstrip('/')
 554             playlist_id += '_' + extra_id.replace('/', '_')
 555             path = next(c['path_id'] for c in program.get('filters') or [] if extra_id in c.get('weblink'))
 556             program = self._download_json(
 557                 urljoin('https://www.raiplaysound.it', path), playlist_id, 'Downloading program secondary JSON')
 558
 559         entries = [
 560             self.url_result(urljoin(base, c['path_id']), ie=RaiPlaySoundIE.ie_key())
 561             for c in traverse_obj(program, 'cards', ('block', 'cards')) or []
 562             if c.get('path_id')]
 563
 564         return self.playlist_result(entries, playlist_id, program.get('title'),
 565                                     traverse_obj(program, ('podcast_info', 'description')))
 566
 567
 568 class RaiIE(RaiBaseIE):
 569     _VALID_URL = rf'https?://[^/]+\.(?:rai\.(?:it|tv))/.+?-(?P<id>{RaiBaseIE._UUID_RE})(?:-.+?)?\.html'
 570     _TESTS = [{
 571         # var uniquename = "ContentItem-..."
 572         # data-id="ContentItem-..."
 573         'url': 'https://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
 574         'info_dict': {
 575             'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
 576             'ext': 'mp4',
 577             'title': 'TG PRIMO TEMPO',
 578             'thumbnail': r're:^https?://.*\.jpg$',
 579             'duration': 1758,
 580             'upload_date': '20140612',
 581         },
 582         'skip': 'This content is available only in Italy',
 583     }, {
 584         # with ContentItem in og:url
 585         'url': 'https://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
 586         'md5': '06345bd97c932f19ffb129973d07a020',
 587         'info_dict': {
 588             'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
 589             'ext': 'mp4',
 590             'title': 'TG1 ore 20:00 del 03/11/2016',
 591             'description': 'TG1 edizione integrale ore 20:00 del giorno 03/11/2016',
 592             'thumbnail': r're:^https?://.*\.jpg$',
 593             'duration': 2214,
 594             'upload_date': '20161103'
 595         }
 596     }, {
 597         # Direct MMS URL
 598         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html',
 599         'only_matching': True,
 600     }]
 601
 602     def _extract_from_content_id(self, content_id, url):
 603         media = self._download_json(
 604             f'https://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-{content_id}.html?json',
 605             content_id, 'Downloading video JSON')
 606
 607         title = media['name'].strip()
 608
 609         media_type = media['type']
 610         if 'Audio' in media_type:
 611             relinker_info = {
 612                 'formats': [{
 613                     'format_id': media.get('formatoAudio'),
 614                     'url': media['audioUrl'],
 615                     'ext': media.get('formatoAudio'),
 616                 }]
 617             }
 618         elif 'Video' in media_type:
 619             relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)
 620         else:
 621             raise ExtractorError('not a media file')
 622
 623         thumbnails = []
 624         for image_type in ('image', 'image_medium', 'image_300'):
 625             thumbnail_url = media.get(image_type)
 626             if thumbnail_url:
 627                 thumbnails.append({
 628                     'url': compat_urlparse.urljoin(url, thumbnail_url),
 629                 })
 630
 631         subtitles = self._extract_subtitles(url, media)
 632
 633         return {
 634             'id': content_id,
 635             'title': title,
 636             'description': strip_or_none(media.get('desc') or None),
 637             'thumbnails': thumbnails,
 638             'uploader': strip_or_none(media.get('author') or None),
 639             'upload_date': unified_strdate(media.get('date')),
 640             'duration': parse_duration(media.get('length')),
 641             'subtitles': subtitles,
 642             **relinker_info
 643         }
 644
 645     def _real_extract(self, url):
 646         video_id = self._match_id(url)
 647
 648         webpage = self._download_webpage(url, video_id)
 649
 650         content_item_id = None
 651
 652         content_item_url = self._html_search_meta(
 653             ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url',
 654              'twitter:player', 'jsonlink'), webpage, default=None)
 655         if content_item_url:
 656             content_item_id = self._search_regex(
 657                 rf'ContentItem-({self._UUID_RE})', content_item_url,
 658                 'content item id', default=None)
 659
 660         if not content_item_id:
 661             content_item_id = self._search_regex(
 662                 rf'''(?x)
 663                     (?:
 664                         (?:initEdizione|drawMediaRaiTV)\(|
 665                         <(?:[^>]+\bdata-id|var\s+uniquename)=|
 666                         <iframe[^>]+\bsrc=
 667                     )
 668                     (["\'])
 669                     (?:(?!\1).)*\bContentItem-(?P<id>{self._UUID_RE})
 670                 ''',
 671                 webpage, 'content item id', default=None, group='id')
 672
 673         content_item_ids = set()
 674         if content_item_id:
 675             content_item_ids.add(content_item_id)
 676         if video_id not in content_item_ids:
 677             content_item_ids.add(video_id)
 678
 679         for content_item_id in content_item_ids:
 680             try:
 681                 return self._extract_from_content_id(content_item_id, url)
 682             except GeoRestrictedError:
 683                 raise
 684             except ExtractorError:
 685                 pass
 686
 687         relinker_url = self._proto_relative_url(self._search_regex(
 688             r'''(?x)
 689                 (?:
 690                     var\s+videoURL|
 691                     mediaInfo\.mediaUri
 692                 )\s*=\s*
 693                 ([\'"])
 694                 (?P<url>
 695                     (?:https?:)?
 696                     //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\?
 697                     (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1
 698             ''',
 699             webpage, 'relinker URL', group='url'))
 700
 701         relinker_info = self._extract_relinker_info(
 702             urljoin(url, relinker_url), video_id)
 703
 704         title = self._search_regex(
 705             r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
 706             webpage, 'title', group='title',
 707             default=None) or self._og_search_title(webpage)
 708
 709         return {
 710             'id': video_id,
 711             'title': title,
 712             **relinker_info
 713         }
 714
 715
 716 class RaiNewsIE(RaiIE):  # XXX: Do not subclass from concrete IE
 717     _VALID_URL = rf'https?://(www\.)?rainews\.it/(?!articoli)[^?#]+-(?P<id>{RaiBaseIE._UUID_RE})(?:-[^/?#]+)?\.html'
 718     _EMBED_REGEX = [rf'<iframe[^>]+data-src="(?P<url>/iframe/[^?#]+?{RaiBaseIE._UUID_RE}\.html)']
 719     _TESTS = [{
 720         # new rainews player (#3911)
 721         'url': 'https://www.rainews.it/rubriche/24mm/video/2022/05/24mm-del-29052022-12cf645d-1ffd-4220-b27c-07c226dbdecf.html',
 722         'info_dict': {
 723             'id': '12cf645d-1ffd-4220-b27c-07c226dbdecf',
 724             'ext': 'mp4',
 725             'title': 'Puntata del 29/05/2022',
 726             'duration': 1589,
 727             'upload_date': '20220529',
 728             'uploader': 'rainews',
 729         }
 730     }, {
 731         # old content with fallback method to extract media urls
 732         'url': 'https://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
 733         'info_dict': {
 734             'id': '1632c009-c843-4836-bb65-80c33084a64b',
 735             'ext': 'mp4',
 736             'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor "La ragazza del treno"',
 737             'description': 'I film in uscita questa settimana.',
 738             'thumbnail': r're:^https?://.*\.png$',
 739             'duration': 833,
 740             'upload_date': '20161103'
 741         },
 742         'expected_warnings': ['unable to extract player_data'],
 743     }, {
 744         # iframe + drm
 745         'url': 'https://www.rainews.it/iframe/video/2022/07/euro2022-europei-calcio-femminile-italia-belgio-gol-0-1-video-4de06a69-de75-4e32-a657-02f0885f8118.html',
 746         'only_matching': True,
 747     }]
 748
 749     def _real_extract(self, url):
 750         video_id = self._match_id(url)
 751
 752         webpage = self._download_webpage(url, video_id)
 753
 754         player_data = self._search_json(
 755             r'<rainews-player\s*data=\'', webpage, 'player_data', video_id,
 756             transform_source=clean_html, fatal=False)
 757         track_info = player_data.get('track_info')
 758         relinker_url = traverse_obj(player_data, 'mediapolis', 'content_url')
 759
 760         if not relinker_url:
 761             # fallback on old implementation for some old content
 762             try:
 763                 return self._extract_from_content_id(video_id, url)
 764             except GeoRestrictedError:
 765                 raise
 766             except ExtractorError as e:
 767                 raise ExtractorError('Relinker URL not found', cause=e)
 768
 769         relinker_info = self._extract_relinker_info(urljoin(url, relinker_url), video_id)
 770
 771         return {
 772             'id': video_id,
 773             'title': track_info.get('title') or self._og_search_title(webpage),
 774             'upload_date': unified_strdate(track_info.get('date')),
 775             'uploader': strip_or_none(track_info.get('editor') or None),
 776             **relinker_info
 777         }
 778
 779
 780 class RaiSudtirolIE(RaiBaseIE):
 781     _VALID_URL = r'https?://raisudtirol\.rai\.it/.+?media=(?P<id>[TP]tv\d+)'
 782     _TESTS = [{
 783         'url': 'https://raisudtirol.rai.it/la/index.php?media=Ptv1619729460',
 784         'info_dict': {
 785             'id': 'Ptv1619729460',
 786             'ext': 'mp4',
 787             'title': 'Euro: trasmisciun d\'economia - 29-04-2021 20:51',
 788             'series': 'Euro: trasmisciun d\'economia',
 789             'upload_date': '20210429',
 790             'thumbnail': r're:https://raisudtirol\.rai\.it/img/.+?\.jpg',
 791             'uploader': 'raisudtirol',
 792         }
 793     }]
 794
 795     def _real_extract(self, url):
 796         video_id = self._match_id(url)
 797         webpage = self._download_webpage(url, video_id)
 798
 799         video_date = self._html_search_regex(r'<span class="med_data">(.+?)</span>', webpage, 'video_date', fatal=False)
 800         video_title = self._html_search_regex(r'<span class="med_title">(.+?)</span>', webpage, 'video_title', fatal=False)
 801         video_url = self._html_search_regex(r'sources:\s*\[\{file:\s*"(.+?)"\}\]', webpage, 'video_url')
 802         video_thumb = self._html_search_regex(r'image: \'(.+?)\'', webpage, 'video_thumb', fatal=False)
 803
 804         return {
 805             'id': video_id,
 806             'title': join_nonempty(video_title, video_date, delim=' - '),
 807             'series': video_title,
 808             'upload_date': unified_strdate(video_date),
 809             'thumbnail': urljoin('https://raisudtirol.rai.it/', video_thumb),
 810             'uploader': 'raisudtirol',
 811             'formats': [{
 812                 'format_id': 'https-mp4',
 813                 'url': self._proto_relative_url(video_url),
 814                 'width': 1024,
 815                 'height': 576,
 816                 'fps': 25,
 817                 'vcodec': 'h264',
 818                 'acodec': 'aac',
 819             }],
 820         }