yt_dlp/extractor/rai.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..compat import (
   5     compat_str,
   6     compat_urlparse,
   7 )
   8 from ..utils import (
   9     clean_html,
  10     determine_ext,
  11     ExtractorError,
  12     filter_dict,
  13     find_xpath_attr,
  14     fix_xml_ampersands,
  15     GeoRestrictedError,
  16     HEADRequest,
  17     int_or_none,
  18     join_nonempty,
  19     parse_duration,
  20     remove_start,
  21     strip_or_none,
  22     traverse_obj,
  23     try_get,
  24     unified_strdate,
  25     unified_timestamp,
  26     update_url_query,
  27     urljoin,
  28     xpath_text,
  29 )
  30
  31
  32 class RaiBaseIE(InfoExtractor):
  33     _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
  34     _GEO_COUNTRIES = ['IT']
  35     _GEO_BYPASS = False
  36
  37     def _extract_relinker_info(self, relinker_url, video_id, audio_only=False):
  38         if not re.match(r'https?://', relinker_url):
  39             return {'formats': [{'url': relinker_url}]}
  40
  41         formats = []
  42         geoprotection = None
  43         is_live = None
  44         duration = None
  45
  46         for platform in ('mon', 'flash', 'native'):
  47             relinker = self._download_xml(
  48                 relinker_url, video_id,
  49                 note=f'Downloading XML metadata for platform {platform}',
  50                 transform_source=fix_xml_ampersands,
  51                 query={'output': 45, 'pl': platform},
  52                 headers=self.geo_verification_headers())
  53
  54             if xpath_text(relinker, './license_url', default='{}') != '{}':
  55                 self.report_drm(video_id)
  56
  57             if not geoprotection:
  58                 geoprotection = xpath_text(
  59                     relinker, './geoprotection', default=None) == 'Y'
  60
  61             if not is_live:
  62                 is_live = xpath_text(
  63                     relinker, './is_live', default=None) == 'Y'
  64             if not duration:
  65                 duration = parse_duration(xpath_text(
  66                     relinker, './duration', default=None))
  67
  68             url_elem = find_xpath_attr(relinker, './url', 'type', 'content')
  69             if url_elem is None:
  70                 continue
  71
  72             media_url = url_elem.text
  73
  74             # This does not imply geo restriction (e.g.
  75             # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html)
  76             if '/video_no_available.mp4' in media_url:
  77                 continue
  78
  79             ext = determine_ext(media_url)
  80             if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
  81                 continue
  82
  83             if ext == 'mp3':
  84                 formats.append({
  85                     'url': media_url,
  86                     'vcodec': 'none',
  87                     'acodec': 'mp3',
  88                     'format_id': 'http-mp3',
  89                 })
  90                 break
  91             elif ext == 'm3u8' or 'format=m3u8' in media_url or platform == 'mon':
  92                 formats.extend(self._extract_m3u8_formats(
  93                     media_url, video_id, 'mp4', 'm3u8_native',
  94                     m3u8_id='hls', fatal=False))
  95             elif ext == 'f4m' or platform == 'flash':
  96                 manifest_url = update_url_query(
  97                     media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
  98                     {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
  99                 formats.extend(self._extract_f4m_formats(
 100                     manifest_url, video_id, f4m_id='hds', fatal=False))
 101             else:
 102                 bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
 103                 formats.append({
 104                     'url': media_url,
 105                     'tbr': bitrate if bitrate > 0 else None,
 106                     'format_id': f'http-{bitrate if bitrate > 0 else "http"}',
 107                 })
 108
 109         if not formats and geoprotection is True:
 110             self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
 111
 112         if not audio_only:
 113             formats.extend(self._create_http_urls(relinker_url, formats))
 114
 115         return filter_dict({
 116             'is_live': is_live,
 117             'duration': duration,
 118             'formats': formats,
 119         })
 120
 121     def _create_http_urls(self, relinker_url, fmts):
 122         _RELINKER_REG = r'https?://(?P<host>[^/]+?)/(?:i/)?(?P<extra>[^/]+?)/(?P<path>.+?)/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4|/playlist\.m3u8).+?'
 123         _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
 124         _QUALITY = {
 125             # tbr: w, h
 126             '250': [352, 198],
 127             '400': [512, 288],
 128             '700': [512, 288],
 129             '800': [700, 394],
 130             '1200': [736, 414],
 131             '1800': [1024, 576],
 132             '2400': [1280, 720],
 133             '3200': [1440, 810],
 134             '3600': [1440, 810],
 135             '5000': [1920, 1080],
 136             '10000': [1920, 1080],
 137         }
 138
 139         def test_url(url):
 140             resp = self._request_webpage(
 141                 HEADRequest(url), None, headers={'User-Agent': 'Rai'},
 142                 fatal=False, errnote=False, note=False)
 143
 144             if resp is False:
 145                 return False
 146
 147             if resp.code == 200:
 148                 return False if resp.url == url else resp.url
 149             return None
 150
 151         # filter out audio-only formats
 152         fmts = [f for f in fmts if not f.get('vcodec') == 'none']
 153
 154         def get_format_info(tbr):
 155             import math
 156             br = int_or_none(tbr)
 157             if len(fmts) == 1 and not br:
 158                 br = fmts[0].get('tbr')
 159             if br and br > 300:
 160                 tbr = compat_str(math.floor(br / 100) * 100)
 161             else:
 162                 tbr = '250'
 163
 164             # try extracting info from available m3u8 formats
 165             format_copy = None
 166             for f in fmts:
 167                 if f.get('tbr'):
 168                     br_limit = math.floor(br / 100)
 169                     if br_limit - 1 <= math.floor(f['tbr'] / 100) <= br_limit + 1:
 170                         format_copy = f.copy()
 171             return {
 172                 'width': format_copy.get('width'),
 173                 'height': format_copy.get('height'),
 174                 'tbr': format_copy.get('tbr'),
 175                 'vcodec': format_copy.get('vcodec'),
 176                 'acodec': format_copy.get('acodec'),
 177                 'fps': format_copy.get('fps'),
 178                 'format_id': f'https-{tbr}',
 179             } if format_copy else {
 180                 'width': _QUALITY[tbr][0],
 181                 'height': _QUALITY[tbr][1],
 182                 'format_id': f'https-{tbr}',
 183                 'tbr': int(tbr),
 184             }
 185
 186         loc = test_url(_MP4_TMPL % (relinker_url, '*'))
 187         if not isinstance(loc, compat_str):
 188             return []
 189
 190         mobj = re.match(
 191             _RELINKER_REG,
 192             test_url(relinker_url) or '')
 193         if not mobj:
 194             return []
 195
 196         available_qualities = mobj.group('quality').split(',') if mobj.group('quality') else ['*']
 197         available_qualities = [i for i in available_qualities if i]
 198
 199         formats = []
 200         for q in available_qualities:
 201             fmt = {
 202                 'url': _MP4_TMPL % (relinker_url, q),
 203                 'protocol': 'https',
 204                 'ext': 'mp4',
 205                 **get_format_info(q)
 206             }
 207             formats.append(fmt)
 208         return formats
 209
 210     @staticmethod
 211     def _extract_subtitles(url, video_data):
 212         STL_EXT = 'stl'
 213         SRT_EXT = 'srt'
 214         subtitles = {}
 215         subtitles_array = video_data.get('subtitlesArray') or []
 216         for k in ('subtitles', 'subtitlesUrl'):
 217             subtitles_array.append({'url': video_data.get(k)})
 218         for subtitle in subtitles_array:
 219             sub_url = subtitle.get('url')
 220             if sub_url and isinstance(sub_url, compat_str):
 221                 sub_lang = subtitle.get('language') or 'it'
 222                 sub_url = urljoin(url, sub_url)
 223                 sub_ext = determine_ext(sub_url, SRT_EXT)
 224                 subtitles.setdefault(sub_lang, []).append({
 225                     'ext': sub_ext,
 226                     'url': sub_url,
 227                 })
 228                 if STL_EXT == sub_ext:
 229                     subtitles[sub_lang].append({
 230                         'ext': SRT_EXT,
 231                         'url': sub_url[:-len(STL_EXT)] + SRT_EXT,
 232                     })
 233         return subtitles
 234
 235
 236 class RaiPlayIE(RaiBaseIE):
 237     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
 238     _TESTS = [{
 239         'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
 240         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
 241         'info_dict': {
 242             'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
 243             'ext': 'mp4',
 244             'title': 'Report del 07/04/2014',
 245             'alt_title': 'St 2013/14 - Report - Espresso nel caffè - 07/04/2014',
 246             'description': 'md5:d730c168a58f4bb35600fc2f881ec04e',
 247             'thumbnail': r're:^https?://.*\.jpg$',
 248             'uploader': 'Rai Gulp',
 249             'duration': 6160,
 250             'series': 'Report',
 251             'season': '2013/14',
 252             'subtitles': {
 253                 'it': 'count:4',
 254             },
 255             'release_year': 2022,
 256             'episode': 'Espresso nel caffè - 07/04/2014',
 257             'timestamp': 1396919880,
 258             'upload_date': '20140408',
 259         },
 260         'params': {
 261             'skip_download': True,
 262         },
 263     }, {
 264         # 1080p direct mp4 url
 265         'url': 'https://www.raiplay.it/video/2021/11/Blanca-S1E1-Senza-occhi-b1255a4a-8e72-4a2f-b9f3-fc1308e00736.html',
 266         'md5': 'aeda7243115380b2dd5e881fd42d949a',
 267         'info_dict': {
 268             'id': 'b1255a4a-8e72-4a2f-b9f3-fc1308e00736',
 269             'ext': 'mp4',
 270             'title': 'Blanca - S1E1 - Senza occhi',
 271             'alt_title': 'St 1 Ep 1 - Blanca - Senza occhi',
 272             'description': 'md5:75f95d5c030ec8bac263b1212322e28c',
 273             'thumbnail': r're:^https?://.*\.jpg$',
 274             'uploader': 'Rai 1',
 275             'duration': 6493,
 276             'series': 'Blanca',
 277             'season': 'Season 1',
 278             'episode_number': 1,
 279             'release_year': 2021,
 280             'season_number': 1,
 281             'episode': 'Senza occhi',
 282             'timestamp': 1637318940,
 283             'upload_date': '20211119',
 284         },
 285     }, {
 286         'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
 287         'only_matching': True,
 288     }, {
 289         # subtitles at 'subtitlesArray' key (see #27698)
 290         'url': 'https://www.raiplay.it/video/2020/12/Report---04-01-2021-2e90f1de-8eee-4de4-ac0e-78d21db5b600.html',
 291         'only_matching': True,
 292     }, {
 293         # DRM protected
 294         'url': 'https://www.raiplay.it/video/2021/06/Lo-straordinario-mondo-di-Zoey-S2E1-Lo-straordinario-ritorno-di-Zoey-3ba992de-2332-41ad-9214-73e32ab209f4.html',
 295         'only_matching': True,
 296     }]
 297
 298     def _real_extract(self, url):
 299         base, video_id = self._match_valid_url(url).groups()
 300
 301         media = self._download_json(
 302             base + '.json', video_id, 'Downloading video JSON')
 303
 304         if not self.get_param('allow_unplayable_formats'):
 305             if try_get(
 306                     media,
 307                     (lambda x: x['rights_management']['rights']['drm'],
 308                      lambda x: x['program_info']['rights_management']['rights']['drm']),
 309                     dict):
 310                 self.report_drm(video_id)
 311
 312         title = media['name']
 313         video = media['video']
 314
 315         relinker_info = self._extract_relinker_info(video['content_url'], video_id)
 316         self._sort_formats(relinker_info['formats'])
 317
 318         thumbnails = []
 319         for _, value in media.get('images', {}).items():
 320             if value:
 321                 thumbnails.append({
 322                     'url': urljoin(url, value),
 323                 })
 324
 325         date_published = media.get('date_published')
 326         time_published = media.get('time_published')
 327         if date_published and time_published:
 328             date_published += ' ' + time_published
 329
 330         subtitles = self._extract_subtitles(url, video)
 331
 332         program_info = media.get('program_info') or {}
 333         season = media.get('season')
 334
 335         alt_title = join_nonempty(media.get('subtitle'), media.get('toptitle'), delim=' - ')
 336
 337         return {
 338             'id': remove_start(media.get('id'), 'ContentItem-') or video_id,
 339             'display_id': video_id,
 340             'title': title,
 341             'alt_title': strip_or_none(alt_title or None),
 342             'description': media.get('description'),
 343             'uploader': strip_or_none(media.get('channel') or None),
 344             'creator': strip_or_none(media.get('editor') or None),
 345             'duration': parse_duration(video.get('duration')),
 346             'timestamp': unified_timestamp(date_published),
 347             'thumbnails': thumbnails,
 348             'series': program_info.get('name'),
 349             'season_number': int_or_none(season),
 350             'season': season if (season and not season.isdigit()) else None,
 351             'episode': media.get('episode_title'),
 352             'episode_number': int_or_none(media.get('episode')),
 353             'subtitles': subtitles,
 354             'release_year': int_or_none(traverse_obj(media, ('track_info', 'edit_year'))),
 355             **relinker_info
 356         }
 357
 358
 359 class RaiPlayLiveIE(RaiPlayIE):
 360     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/dirette/(?P<id>[^/?#&]+))'
 361     _TESTS = [{
 362         'url': 'http://www.raiplay.it/dirette/rainews24',
 363         'info_dict': {
 364             'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c',
 365             'display_id': 'rainews24',
 366             'ext': 'mp4',
 367             'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 368             'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497',
 369             'uploader': 'Rai News 24',
 370             'creator': 'Rai News 24',
 371             'is_live': True,
 372             'live_status': 'is_live',
 373             'upload_date': '20090502',
 374             'timestamp': 1241276220,
 375         },
 376         'params': {
 377             'skip_download': True,
 378         },
 379     }]
 380
 381
 382 class RaiPlayPlaylistIE(InfoExtractor):
 383     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
 384     _TESTS = [{
 385         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/',
 386         'info_dict': {
 387             'id': 'nondirloalmiocapo',
 388             'title': 'Non dirlo al mio capo',
 389             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
 390         },
 391         'playlist_mincount': 12,
 392     }, {
 393         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/episodi/stagione-2/',
 394         'info_dict': {
 395             'id': 'nondirloalmiocapo',
 396             'title': 'Non dirlo al mio capo - Stagione 2',
 397             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
 398         },
 399         'playlist_mincount': 12,
 400     }]
 401
 402     def _real_extract(self, url):
 403         base, playlist_id, extra_id = self._match_valid_url(url).groups()
 404
 405         program = self._download_json(
 406             base + '.json', playlist_id, 'Downloading program JSON')
 407
 408         if extra_id:
 409             extra_id = extra_id.upper().rstrip('/')
 410
 411         playlist_title = program.get('name')
 412         entries = []
 413         for b in (program.get('blocks') or []):
 414             for s in (b.get('sets') or []):
 415                 if extra_id:
 416                     if extra_id != join_nonempty(
 417                             b.get('name'), s.get('name'), delim='/').replace(' ', '-').upper():
 418                         continue
 419                     playlist_title = join_nonempty(playlist_title, s.get('name'), delim=' - ')
 420
 421                 s_id = s.get('id')
 422                 if not s_id:
 423                     continue
 424                 medias = self._download_json(
 425                     f'{base}/{s_id}.json', s_id,
 426                     'Downloading content set JSON', fatal=False)
 427                 if not medias:
 428                     continue
 429                 for m in (medias.get('items') or []):
 430                     path_id = m.get('path_id')
 431                     if not path_id:
 432                         continue
 433                     video_url = urljoin(url, path_id)
 434                     entries.append(self.url_result(
 435                         video_url, ie=RaiPlayIE.ie_key(),
 436                         video_id=RaiPlayIE._match_id(video_url)))
 437
 438         return self.playlist_result(
 439             entries, playlist_id, playlist_title,
 440             try_get(program, lambda x: x['program_info']['description']))
 441
 442
 443 class RaiPlaySoundIE(RaiBaseIE):
 444     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplaysound\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
 445     _TESTS = [{
 446         'url': 'https://www.raiplaysound.it/audio/2021/12/IL-RUGGITO-DEL-CONIGLIO-1ebae2a7-7cdb-42bb-842e-fe0d193e9707.html',
 447         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
 448         'info_dict': {
 449             'id': '1ebae2a7-7cdb-42bb-842e-fe0d193e9707',
 450             'ext': 'mp3',
 451             'title': 'Il Ruggito del Coniglio del 10/12/2021',
 452             'alt_title': 'md5:0e6476cd57858bb0f3fcc835d305b455',
 453             'description': 'md5:2a17d2107e59a4a8faa0e18334139ee2',
 454             'thumbnail': r're:^https?://.*\.jpg$',
 455             'uploader': 'rai radio 2',
 456             'duration': 5685,
 457             'series': 'Il Ruggito del Coniglio',
 458             'episode': 'Il Ruggito del Coniglio del 10/12/2021',
 459             'creator': 'rai radio 2',
 460             'timestamp': 1638346620,
 461             'upload_date': '20211201',
 462         },
 463         'params': {
 464             'skip_download': True,
 465         },
 466     }]
 467
 468     def _real_extract(self, url):
 469         base, audio_id = self._match_valid_url(url).group('base', 'id')
 470         media = self._download_json(f'{base}.json', audio_id, 'Downloading audio JSON')
 471         uid = try_get(media, lambda x: remove_start(remove_start(x['uniquename'], 'ContentItem-'), 'Page-'))
 472
 473         info = {}
 474         formats = []
 475         relinkers = set(traverse_obj(media, (('downloadable_audio', 'audio', ('live', 'cards', 0, 'audio')), 'url')))
 476         for r in relinkers:
 477             info = self._extract_relinker_info(r, audio_id, True)
 478             formats.extend(info.get('formats'))
 479
 480         date_published = try_get(media, (lambda x: f'{x["create_date"]} {x.get("create_time") or ""}',
 481                                          lambda x: x['live']['create_date']))
 482
 483         podcast_info = traverse_obj(media, 'podcast_info', ('live', 'cards', 0)) or {}
 484         thumbnails = [{
 485             'url': urljoin(url, thumb_url),
 486         } for thumb_url in (podcast_info.get('images') or {}).values() if thumb_url]
 487
 488         return {
 489             **info,
 490             'id': uid or audio_id,
 491             'display_id': audio_id,
 492             'title': traverse_obj(media, 'title', 'episode_title'),
 493             'alt_title': traverse_obj(media, ('track_info', 'media_name'), expected_type=strip_or_none),
 494             'description': media.get('description'),
 495             'uploader': traverse_obj(media, ('track_info', 'channel'), expected_type=strip_or_none),
 496             'creator': traverse_obj(media, ('track_info', 'editor'), expected_type=strip_or_none),
 497             'timestamp': unified_timestamp(date_published),
 498             'thumbnails': thumbnails,
 499             'series': podcast_info.get('title'),
 500             'season_number': int_or_none(media.get('season')),
 501             'episode': media.get('episode_title'),
 502             'episode_number': int_or_none(media.get('episode')),
 503             'formats': formats,
 504         }
 505
 506
 507 class RaiPlaySoundLiveIE(RaiPlaySoundIE):
 508     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?P<id>[^/?#&]+)$)'
 509     _TESTS = [{
 510         'url': 'https://www.raiplaysound.it/radio2',
 511         'info_dict': {
 512             'id': 'b00a50e6-f404-4af6-8f8c-ff3b9af73a44',
 513             'display_id': 'radio2',
 514             'ext': 'mp4',
 515             'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+',
 516             'thumbnail': r're:https://www.raiplaysound.it/dl/img/.+?png',
 517             'uploader': 'rai radio 2',
 518             'series': 'Rai Radio 2',
 519             'creator': 'raiplaysound',
 520             'is_live': True,
 521             'live_status': 'is_live',
 522         },
 523         'params': {
 524             'skip_download': 'live',
 525         },
 526     }]
 527
 528
 529 class RaiPlaySoundPlaylistIE(InfoExtractor):
 530     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?:programmi|playlist|audiolibri)/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
 531     _TESTS = [{
 532         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio',
 533         'info_dict': {
 534             'id': 'ilruggitodelconiglio',
 535             'title': 'Il Ruggito del Coniglio',
 536             'description': 'md5:1bbaf631245a7ab1ec4d9fbb3c7aa8f3',
 537         },
 538         'playlist_mincount': 65,
 539     }, {
 540         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio/puntate/prima-stagione-1995',
 541         'info_dict': {
 542             'id': 'ilruggitodelconiglio_puntate_prima-stagione-1995',
 543             'title': 'Prima Stagione 1995',
 544         },
 545         'playlist_count': 1,
 546     }]
 547
 548     def _real_extract(self, url):
 549         base, playlist_id, extra_id = self._match_valid_url(url).group('base', 'id', 'extra_id')
 550         url = f'{base}.json'
 551         program = self._download_json(url, playlist_id, 'Downloading program JSON')
 552
 553         if extra_id:
 554             extra_id = extra_id.rstrip('/')
 555             playlist_id += '_' + extra_id.replace('/', '_')
 556             path = next(c['path_id'] for c in program.get('filters') or [] if extra_id in c.get('weblink'))
 557             program = self._download_json(
 558                 urljoin('https://www.raiplaysound.it', path), playlist_id, 'Downloading program secondary JSON')
 559
 560         entries = [
 561             self.url_result(urljoin(base, c['path_id']), ie=RaiPlaySoundIE.ie_key())
 562             for c in traverse_obj(program, 'cards', ('block', 'cards')) or []
 563             if c.get('path_id')]
 564
 565         return self.playlist_result(entries, playlist_id, program.get('title'),
 566                                     traverse_obj(program, ('podcast_info', 'description')))
 567
 568
 569 class RaiIE(RaiBaseIE):
 570     _VALID_URL = rf'https?://[^/]+\.(?:rai\.(?:it|tv))/.+?-(?P<id>{RaiBaseIE._UUID_RE})(?:-.+?)?\.html'
 571     _TESTS = [{
 572         # var uniquename = "ContentItem-..."
 573         # data-id="ContentItem-..."
 574         'url': 'https://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
 575         'info_dict': {
 576             'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
 577             'ext': 'mp4',
 578             'title': 'TG PRIMO TEMPO',
 579             'thumbnail': r're:^https?://.*\.jpg$',
 580             'duration': 1758,
 581             'upload_date': '20140612',
 582         },
 583         'skip': 'This content is available only in Italy',
 584     }, {
 585         # with ContentItem in og:url
 586         'url': 'https://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
 587         'md5': '06345bd97c932f19ffb129973d07a020',
 588         'info_dict': {
 589             'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
 590             'ext': 'mp4',
 591             'title': 'TG1 ore 20:00 del 03/11/2016',
 592             'description': 'TG1 edizione integrale ore 20:00 del giorno 03/11/2016',
 593             'thumbnail': r're:^https?://.*\.jpg$',
 594             'duration': 2214,
 595             'upload_date': '20161103'
 596         }
 597     }, {
 598         # Direct MMS URL
 599         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html',
 600         'only_matching': True,
 601     }]
 602
 603     def _extract_from_content_id(self, content_id, url):
 604         media = self._download_json(
 605             f'https://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-{content_id}.html?json',
 606             content_id, 'Downloading video JSON')
 607
 608         title = media['name'].strip()
 609
 610         media_type = media['type']
 611         if 'Audio' in media_type:
 612             relinker_info = {
 613                 'formats': [{
 614                     'format_id': media.get('formatoAudio'),
 615                     'url': media['audioUrl'],
 616                     'ext': media.get('formatoAudio'),
 617                 }]
 618             }
 619         elif 'Video' in media_type:
 620             relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)
 621         else:
 622             raise ExtractorError('not a media file')
 623
 624         self._sort_formats(relinker_info['formats'])
 625
 626         thumbnails = []
 627         for image_type in ('image', 'image_medium', 'image_300'):
 628             thumbnail_url = media.get(image_type)
 629             if thumbnail_url:
 630                 thumbnails.append({
 631                     'url': compat_urlparse.urljoin(url, thumbnail_url),
 632                 })
 633
 634         subtitles = self._extract_subtitles(url, media)
 635
 636         return {
 637             'id': content_id,
 638             'title': title,
 639             'description': strip_or_none(media.get('desc') or None),
 640             'thumbnails': thumbnails,
 641             'uploader': strip_or_none(media.get('author') or None),
 642             'upload_date': unified_strdate(media.get('date')),
 643             'duration': parse_duration(media.get('length')),
 644             'subtitles': subtitles,
 645             **relinker_info
 646         }
 647
 648     def _real_extract(self, url):
 649         video_id = self._match_id(url)
 650
 651         webpage = self._download_webpage(url, video_id)
 652
 653         content_item_id = None
 654
 655         content_item_url = self._html_search_meta(
 656             ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url',
 657              'twitter:player', 'jsonlink'), webpage, default=None)
 658         if content_item_url:
 659             content_item_id = self._search_regex(
 660                 rf'ContentItem-({self._UUID_RE})', content_item_url,
 661                 'content item id', default=None)
 662
 663         if not content_item_id:
 664             content_item_id = self._search_regex(
 665                 rf'''(?x)
 666                     (?:
 667                         (?:initEdizione|drawMediaRaiTV)\(|
 668                         <(?:[^>]+\bdata-id|var\s+uniquename)=|
 669                         <iframe[^>]+\bsrc=
 670                     )
 671                     (["\'])
 672                     (?:(?!\1).)*\bContentItem-(?P<id>{self._UUID_RE})
 673                 ''',
 674                 webpage, 'content item id', default=None, group='id')
 675
 676         content_item_ids = set()
 677         if content_item_id:
 678             content_item_ids.add(content_item_id)
 679         if video_id not in content_item_ids:
 680             content_item_ids.add(video_id)
 681
 682         for content_item_id in content_item_ids:
 683             try:
 684                 return self._extract_from_content_id(content_item_id, url)
 685             except GeoRestrictedError:
 686                 raise
 687             except ExtractorError:
 688                 pass
 689
 690         relinker_url = self._proto_relative_url(self._search_regex(
 691             r'''(?x)
 692                 (?:
 693                     var\s+videoURL|
 694                     mediaInfo\.mediaUri
 695                 )\s*=\s*
 696                 ([\'"])
 697                 (?P<url>
 698                     (?:https?:)?
 699                     //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\?
 700                     (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1
 701             ''',
 702             webpage, 'relinker URL', group='url'))
 703
 704         relinker_info = self._extract_relinker_info(
 705             urljoin(url, relinker_url), video_id)
 706         self._sort_formats(relinker_info['formats'])
 707
 708         title = self._search_regex(
 709             r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
 710             webpage, 'title', group='title',
 711             default=None) or self._og_search_title(webpage)
 712
 713         return {
 714             'id': video_id,
 715             'title': title,
 716             **relinker_info
 717         }
 718
 719
 720 class RaiNewsIE(RaiIE):
 721     _VALID_URL = rf'https?://(www\.)?rainews\.it/(?!articoli)[^?#]+-(?P<id>{RaiBaseIE._UUID_RE})(?:-[^/?#]+)?\.html'
 722     _EMBED_REGEX = [rf'<iframe[^>]+data-src="(?P<url>/iframe/[^?#]+?{RaiBaseIE._UUID_RE}\.html)']
 723     _TESTS = [{
 724         # new rainews player (#3911)
 725         'url': 'https://www.rainews.it/rubriche/24mm/video/2022/05/24mm-del-29052022-12cf645d-1ffd-4220-b27c-07c226dbdecf.html',
 726         'info_dict': {
 727             'id': '12cf645d-1ffd-4220-b27c-07c226dbdecf',
 728             'ext': 'mp4',
 729             'title': 'Puntata del 29/05/2022',
 730             'duration': 1589,
 731             'upload_date': '20220529',
 732             'uploader': 'rainews',
 733         }
 734     }, {
 735         # old content with fallback method to extract media urls
 736         'url': 'https://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
 737         'info_dict': {
 738             'id': '1632c009-c843-4836-bb65-80c33084a64b',
 739             'ext': 'mp4',
 740             'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor "La ragazza del treno"',
 741             'description': 'I film in uscita questa settimana.',
 742             'thumbnail': r're:^https?://.*\.png$',
 743             'duration': 833,
 744             'upload_date': '20161103'
 745         },
 746         'expected_warnings': ['unable to extract player_data'],
 747     }, {
 748         # iframe + drm
 749         'url': 'https://www.rainews.it/iframe/video/2022/07/euro2022-europei-calcio-femminile-italia-belgio-gol-0-1-video-4de06a69-de75-4e32-a657-02f0885f8118.html',
 750         'only_matching': True,
 751     }]
 752
 753     def _real_extract(self, url):
 754         video_id = self._match_id(url)
 755
 756         webpage = self._download_webpage(url, video_id)
 757
 758         player_data = self._search_json(
 759             r'<rainews-player\s*data=\'', webpage, 'player_data', video_id,
 760             transform_source=clean_html, fatal=False)
 761         track_info = player_data.get('track_info')
 762         relinker_url = traverse_obj(player_data, 'mediapolis', 'content_url')
 763
 764         if not relinker_url:
 765             # fallback on old implementation for some old content
 766             try:
 767                 return self._extract_from_content_id(video_id, url)
 768             except GeoRestrictedError:
 769                 raise
 770             except ExtractorError as e:
 771                 raise ExtractorError('Relinker URL not found', cause=e)
 772
 773         relinker_info = self._extract_relinker_info(urljoin(url, relinker_url), video_id)
 774
 775         self._sort_formats(relinker_info['formats'])
 776
 777         return {
 778             'id': video_id,
 779             'title': track_info.get('title') or self._og_search_title(webpage),
 780             'upload_date': unified_strdate(track_info.get('date')),
 781             'uploader': strip_or_none(track_info.get('editor') or None),
 782             **relinker_info
 783         }
 784
 785
 786 class RaiSudtirolIE(RaiBaseIE):
 787     _VALID_URL = r'https?://raisudtirol\.rai\.it/.+?media=(?P<id>[TP]tv\d+)'
 788     _TESTS = [{
 789         'url': 'https://raisudtirol.rai.it/la/index.php?media=Ptv1619729460',
 790         'info_dict': {
 791             'id': 'Ptv1619729460',
 792             'ext': 'mp4',
 793             'title': 'Euro: trasmisciun d\'economia - 29-04-2021 20:51',
 794             'series': 'Euro: trasmisciun d\'economia',
 795             'upload_date': '20210429',
 796             'thumbnail': r're:https://raisudtirol\.rai\.it/img/.+?\.jpg',
 797             'uploader': 'raisudtirol',
 798         }
 799     }]
 800
 801     def _real_extract(self, url):
 802         video_id = self._match_id(url)
 803         webpage = self._download_webpage(url, video_id)
 804
 805         video_date = self._html_search_regex(r'<span class="med_data">(.+?)</span>', webpage, 'video_date', fatal=False)
 806         video_title = self._html_search_regex(r'<span class="med_title">(.+?)</span>', webpage, 'video_title', fatal=False)
 807         video_url = self._html_search_regex(r'sources:\s*\[\{file:\s*"(.+?)"\}\]', webpage, 'video_url')
 808         video_thumb = self._html_search_regex(r'image: \'(.+?)\'', webpage, 'video_thumb', fatal=False)
 809
 810         return {
 811             'id': video_id,
 812             'title': join_nonempty(video_title, video_date, delim=' - '),
 813             'series': video_title,
 814             'upload_date': unified_strdate(video_date),
 815             'thumbnail': urljoin('https://raisudtirol.rai.it/', video_thumb),
 816             'uploader': 'raisudtirol',
 817             'formats': [{
 818                 'format_id': 'https-mp4',
 819                 'url': self._proto_relative_url(video_url),
 820                 'width': 1024,
 821                 'height': 576,
 822                 'fps': 25,
 823                 'vcodec': 'h264',
 824                 'acodec': 'aac',
 825             }],
 826         }