yt_dlp/extractor/rai.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     clean_html,
   6     determine_ext,
   7     ExtractorError,
   8     filter_dict,
   9     GeoRestrictedError,
  10     int_or_none,
  11     join_nonempty,
  12     parse_duration,
  13     remove_start,
  14     strip_or_none,
  15     traverse_obj,
  16     try_get,
  17     unified_strdate,
  18     unified_timestamp,
  19     update_url_query,
  20     urljoin,
  21     xpath_text,
  22 )
  23
  24
  25 class RaiBaseIE(InfoExtractor):
  26     _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
  27     _GEO_COUNTRIES = ['IT']
  28     _GEO_BYPASS = False
  29
  30     def _extract_relinker_info(self, relinker_url, video_id, audio_only=False):
  31         def fix_cdata(s):
  32             # remove \r\n\t before and after <![CDATA[ ]]> to avoid
  33             # polluted text with xpath_text
  34             s = re.sub(r'(\]\]>)[\r\n\t]+(</)', '\\1\\2', s)
  35             return re.sub(r'(>)[\r\n\t]+(<!\[CDATA\[)', '\\1\\2', s)
  36
  37         if not re.match(r'https?://', relinker_url):
  38             return {'formats': [{'url': relinker_url}]}
  39
  40         # set User-Agent to generic 'Rai' to avoid quality filtering from
  41         # the media server and get the maximum qualities available
  42         relinker = self._download_xml(
  43             relinker_url, video_id, note='Downloading XML metadata',
  44             transform_source=fix_cdata, query={'output': 64},
  45             headers={**self.geo_verification_headers(), 'User-Agent': 'Rai'})
  46
  47         if xpath_text(relinker, './license_url', default='{}') != '{}':
  48             self.report_drm(video_id)
  49
  50         is_live = xpath_text(relinker, './is_live', default='N') == 'Y'
  51         duration = parse_duration(xpath_text(relinker, './duration', default=None))
  52         media_url = xpath_text(relinker, './url[@type="content"]', default=None)
  53
  54         if not media_url:
  55             self.raise_no_formats('The relinker returned no media url')
  56
  57         # geo flag is a bit unreliable and not properly set all the time
  58         geoprotection = xpath_text(relinker, './geoprotection', default='N') == 'Y'
  59
  60         ext = determine_ext(media_url)
  61         formats = []
  62
  63         if ext == 'mp3':
  64             formats.append({
  65                 'url': media_url,
  66                 'vcodec': 'none',
  67                 'acodec': 'mp3',
  68                 'format_id': 'https-mp3',
  69             })
  70         elif ext == 'm3u8' or 'format=m3u8' in media_url:
  71             formats.extend(self._extract_m3u8_formats(
  72                 media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
  73         elif ext == 'f4m':
  74             # very likely no longer needed. Cannot find any url that uses it.
  75             manifest_url = update_url_query(
  76                 media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
  77                 {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
  78             formats.extend(self._extract_f4m_formats(
  79                 manifest_url, video_id, f4m_id='hds', fatal=False))
  80         elif ext == 'mp4':
  81             bitrate = int_or_none(xpath_text(relinker, './bitrate'))
  82             formats.append({
  83                 'url': media_url,
  84                 'tbr': bitrate if bitrate > 0 else None,
  85                 'format_id': join_nonempty('https', bitrate, delim='-'),
  86             })
  87         else:
  88             raise ExtractorError('Unrecognized media file found')
  89
  90         if (not formats and geoprotection is True) or '/video_no_available.mp4' in media_url:
  91             self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
  92
  93         if not audio_only and not is_live:
  94             formats.extend(self._create_http_urls(media_url, relinker_url, formats))
  95
  96         return filter_dict({
  97             'is_live': is_live,
  98             'duration': duration,
  99             'formats': formats,
 100         })
 101
 102     def _create_http_urls(self, manifest_url, relinker_url, fmts):
 103         _MANIFEST_REG = r'/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4)?(?:\.csmil)?/playlist\.m3u8'
 104         _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
 105         _QUALITY = {
 106             # tbr: w, h
 107             250: [352, 198],
 108             400: [512, 288],
 109             600: [512, 288],
 110             700: [512, 288],
 111             800: [700, 394],
 112             1200: [736, 414],
 113             1500: [920, 518],
 114             1800: [1024, 576],
 115             2400: [1280, 720],
 116             3200: [1440, 810],
 117             3600: [1440, 810],
 118             5000: [1920, 1080],
 119             10000: [1920, 1080],
 120         }
 121
 122         def percentage(number, target, pc=20, roof=125):
 123             '''check if the target is in the range of number +/- percent'''
 124             if not number or number < 0:
 125                 return False
 126             return abs(target - number) < min(float(number) * float(pc) / 100.0, roof)
 127
 128         def get_format_info(tbr):
 129             import math
 130             br = int_or_none(tbr)
 131             if len(fmts) == 1 and not br:
 132                 br = fmts[0].get('tbr')
 133             if br and br > 300:
 134                 tbr = math.floor(br / 100) * 100
 135             else:
 136                 tbr = 250
 137
 138             # try extracting info from available m3u8 formats
 139             format_copy = [None, None]
 140             for f in fmts:
 141                 if f.get('tbr'):
 142                     if percentage(tbr, f['tbr']):
 143                         format_copy[0] = f.copy()
 144                 if [f.get('width'), f.get('height')] == _QUALITY.get(tbr):
 145                     format_copy[1] = f.copy()
 146                     format_copy[1]['tbr'] = tbr
 147
 148             # prefer format with similar bitrate because there might be
 149             # multiple video with the same resolution but different bitrate
 150             format_copy = format_copy[0] or format_copy[1] or {}
 151             return {
 152                 'format_id': f'https-{tbr}',
 153                 'width': format_copy.get('width'),
 154                 'height': format_copy.get('height'),
 155                 'tbr': format_copy.get('tbr'),
 156                 'vcodec': format_copy.get('vcodec'),
 157                 'acodec': format_copy.get('acodec'),
 158                 'fps': format_copy.get('fps'),
 159             } if format_copy else {
 160                 'format_id': f'https-{tbr}',
 161                 'width': _QUALITY[tbr][0],
 162                 'height': _QUALITY[tbr][1],
 163                 'tbr': tbr,
 164                 'vcodec': 'avc1',
 165                 'acodec': 'mp4a',
 166                 'fps': 25,
 167             }
 168
 169         # filter out single-stream formats
 170         fmts = [f for f in fmts
 171                 if not f.get('vcodec') == 'none' and not f.get('acodec') == 'none']
 172
 173         mobj = re.search(_MANIFEST_REG, manifest_url)
 174         if not mobj:
 175             return []
 176         available_qualities = mobj.group('quality').split(',') if mobj.group('quality') else ['*']
 177
 178         formats = []
 179         for q in filter(None, available_qualities):
 180             self.write_debug(f'Creating https format for quality {q}')
 181             formats.append({
 182                 'url': _MP4_TMPL % (relinker_url, q),
 183                 'protocol': 'https',
 184                 'ext': 'mp4',
 185                 **get_format_info(q)
 186             })
 187         return formats
 188
 189     @staticmethod
 190     def _get_thumbnails_list(thumbs, url):
 191         return [{
 192             'url': urljoin(url, thumb_url),
 193         } for thumb_url in (thumbs or {}).values() if thumb_url]
 194
 195     @staticmethod
 196     def _extract_subtitles(url, video_data):
 197         STL_EXT = 'stl'
 198         SRT_EXT = 'srt'
 199         subtitles = {}
 200         subtitles_array = video_data.get('subtitlesArray') or video_data.get('subtitleList') or []
 201         for k in ('subtitles', 'subtitlesUrl'):
 202             subtitles_array.append({'url': video_data.get(k)})
 203         for subtitle in subtitles_array:
 204             sub_url = subtitle.get('url')
 205             if sub_url and isinstance(sub_url, str):
 206                 sub_lang = subtitle.get('language') or 'it'
 207                 sub_url = urljoin(url, sub_url)
 208                 sub_ext = determine_ext(sub_url, SRT_EXT)
 209                 subtitles.setdefault(sub_lang, []).append({
 210                     'ext': sub_ext,
 211                     'url': sub_url,
 212                 })
 213                 if STL_EXT == sub_ext:
 214                     subtitles[sub_lang].append({
 215                         'ext': SRT_EXT,
 216                         'url': sub_url[:-len(STL_EXT)] + SRT_EXT,
 217                     })
 218         return subtitles
 219
 220
 221 class RaiPlayIE(RaiBaseIE):
 222     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
 223     _TESTS = [{
 224         'url': 'https://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
 225         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
 226         'info_dict': {
 227             'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
 228             'ext': 'mp4',
 229             'title': 'Report del 07/04/2014',
 230             'alt_title': 'St 2013/14 - Report - Espresso nel caffè - 07/04/2014',
 231             'description': 'md5:d730c168a58f4bb35600fc2f881ec04e',
 232             'thumbnail': r're:^https?://www\.raiplay\.it/.+\.jpg',
 233             'uploader': 'Rai 3',
 234             'creator': 'Rai 3',
 235             'duration': 6160,
 236             'series': 'Report',
 237             'season': '2013/14',
 238             'subtitles': {'it': 'count:4'},
 239             'release_year': 2022,
 240             'episode': 'Espresso nel caffè - 07/04/2014',
 241             'timestamp': 1396919880,
 242             'upload_date': '20140408',
 243             'formats': 'count:4',
 244         },
 245         'params': {'skip_download': True},
 246     }, {
 247         # 1080p direct mp4 url
 248         'url': 'https://www.raiplay.it/video/2021/11/Blanca-S1E1-Senza-occhi-b1255a4a-8e72-4a2f-b9f3-fc1308e00736.html',
 249         'md5': 'aeda7243115380b2dd5e881fd42d949a',
 250         'info_dict': {
 251             'id': 'b1255a4a-8e72-4a2f-b9f3-fc1308e00736',
 252             'ext': 'mp4',
 253             'title': 'Blanca - S1E1 - Senza occhi',
 254             'alt_title': 'St 1 Ep 1 - Blanca - Senza occhi',
 255             'description': 'md5:75f95d5c030ec8bac263b1212322e28c',
 256             'thumbnail': r're:^https://www\.raiplay\.it/dl/img/.+\.jpg',
 257             'uploader': 'Rai Premium',
 258             'creator': 'Rai Fiction',
 259             'duration': 6493,
 260             'series': 'Blanca',
 261             'season': 'Season 1',
 262             'episode_number': 1,
 263             'release_year': 2021,
 264             'season_number': 1,
 265             'episode': 'Senza occhi',
 266             'timestamp': 1637318940,
 267             'upload_date': '20211119',
 268             'formats': 'count:12',
 269         },
 270         'params': {'skip_download': True},
 271         'expected_warnings': ['Video not available. Likely due to geo-restriction.']
 272     }, {
 273         # 1500 quality
 274         'url': 'https://www.raiplay.it/video/2012/09/S1E11---Tutto-cio-che-luccica-0cab3323-732e-45d6-8e86-7704acab6598.html',
 275         'md5': 'a634d20e8ab2d43724c273563f6bf87a',
 276         'info_dict': {
 277             'id': '0cab3323-732e-45d6-8e86-7704acab6598',
 278             'ext': 'mp4',
 279             'title': 'Mia and Me - S1E11 - Tutto ciò che luccica',
 280             'alt_title': 'St 1 Ep 11 - Mia and Me - Tutto ciò che luccica',
 281             'description': 'md5:4969e594184b1920c4c1f2b704da9dea',
 282             'thumbnail': r're:^https?://.*\.jpg$',
 283             'uploader': 'Rai Gulp',
 284             'series': 'Mia and Me',
 285             'season': 'Season 1',
 286             'episode_number': 11,
 287             'release_year': 2015,
 288             'season_number': 1,
 289             'episode': 'Tutto ciò che luccica',
 290             'timestamp': 1348495020,
 291             'upload_date': '20120924',
 292         },
 293     }, {
 294         'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
 295         'only_matching': True,
 296     }, {
 297         # subtitles at 'subtitlesArray' key (see #27698)
 298         'url': 'https://www.raiplay.it/video/2020/12/Report---04-01-2021-2e90f1de-8eee-4de4-ac0e-78d21db5b600.html',
 299         'only_matching': True,
 300     }, {
 301         # DRM protected
 302         'url': 'https://www.raiplay.it/video/2021/06/Lo-straordinario-mondo-di-Zoey-S2E1-Lo-straordinario-ritorno-di-Zoey-3ba992de-2332-41ad-9214-73e32ab209f4.html',
 303         'only_matching': True,
 304     }]
 305
 306     def _real_extract(self, url):
 307         base, video_id = self._match_valid_url(url).groups()
 308
 309         media = self._download_json(
 310             f'{base}.json', video_id, 'Downloading video JSON')
 311
 312         if not self.get_param('allow_unplayable_formats'):
 313             if traverse_obj(media, (('program_info', None), 'rights_management', 'rights', 'drm')):
 314                 self.report_drm(video_id)
 315
 316         video = media['video']
 317         relinker_info = self._extract_relinker_info(video['content_url'], video_id)
 318         date_published = join_nonempty(
 319             media.get('date_published'), media.get('time_published'), delim=' ')
 320         season = media.get('season')
 321         alt_title = join_nonempty(media.get('subtitle'), media.get('toptitle'), delim=' - ')
 322
 323         return {
 324             'id': remove_start(media.get('id'), 'ContentItem-') or video_id,
 325             'display_id': video_id,
 326             'title': media.get('name'),
 327             'alt_title': strip_or_none(alt_title or None),
 328             'description': media.get('description'),
 329             'uploader': strip_or_none(
 330                 traverse_obj(media, ('program_info', 'channel'))
 331                 or media.get('channel') or None),
 332             'creator': strip_or_none(
 333                 traverse_obj(media, ('program_info', 'editor'))
 334                 or media.get('editor') or None),
 335             'duration': parse_duration(video.get('duration')),
 336             'timestamp': unified_timestamp(date_published),
 337             'thumbnails': self._get_thumbnails_list(media.get('images'), url),
 338             'series': traverse_obj(media, ('program_info', 'name')),
 339             'season_number': int_or_none(season),
 340             'season': season if (season and not season.isdigit()) else None,
 341             'episode': media.get('episode_title'),
 342             'episode_number': int_or_none(media.get('episode')),
 343             'subtitles': self._extract_subtitles(url, video),
 344             'release_year': int_or_none(traverse_obj(media, ('track_info', 'edit_year'))),
 345             **relinker_info
 346         }
 347
 348
 349 class RaiPlayLiveIE(RaiPlayIE):  # XXX: Do not subclass from concrete IE
 350     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/dirette/(?P<id>[^/?#&]+))'
 351     _TESTS = [{
 352         'url': 'http://www.raiplay.it/dirette/rainews24',
 353         'info_dict': {
 354             'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c',
 355             'display_id': 'rainews24',
 356             'ext': 'mp4',
 357             'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 358             'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497',
 359             'uploader': 'Rai News 24',
 360             'creator': 'Rai News 24',
 361             'is_live': True,
 362             'live_status': 'is_live',
 363             'upload_date': '20090502',
 364             'timestamp': 1241276220,
 365             'formats': 'count:3',
 366         },
 367         'params': {'skip_download': True},
 368     }]
 369
 370
 371 class RaiPlayPlaylistIE(InfoExtractor):
 372     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
 373     _TESTS = [{
 374         # entire series episodes + extras...
 375         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/',
 376         'info_dict': {
 377             'id': 'nondirloalmiocapo',
 378             'title': 'Non dirlo al mio capo',
 379             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
 380         },
 381         'playlist_mincount': 30,
 382     }, {
 383         # single season
 384         'url': 'https://www.raiplay.it/programmi/nondirloalmiocapo/episodi/stagione-2/',
 385         'info_dict': {
 386             'id': 'nondirloalmiocapo',
 387             'title': 'Non dirlo al mio capo - Stagione 2',
 388             'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
 389         },
 390         'playlist_count': 12,
 391     }]
 392
 393     def _real_extract(self, url):
 394         base, playlist_id, extra_id = self._match_valid_url(url).groups()
 395
 396         program = self._download_json(
 397             f'{base}.json', playlist_id, 'Downloading program JSON')
 398
 399         if extra_id:
 400             extra_id = extra_id.upper().rstrip('/')
 401
 402         playlist_title = program.get('name')
 403         entries = []
 404         for b in (program.get('blocks') or []):
 405             for s in (b.get('sets') or []):
 406                 if extra_id:
 407                     if extra_id != join_nonempty(
 408                             b.get('name'), s.get('name'), delim='/').replace(' ', '-').upper():
 409                         continue
 410                     playlist_title = join_nonempty(playlist_title, s.get('name'), delim=' - ')
 411
 412                 s_id = s.get('id')
 413                 if not s_id:
 414                     continue
 415                 medias = self._download_json(
 416                     f'{base}/{s_id}.json', s_id,
 417                     'Downloading content set JSON', fatal=False)
 418                 if not medias:
 419                     continue
 420                 for m in (medias.get('items') or []):
 421                     path_id = m.get('path_id')
 422                     if not path_id:
 423                         continue
 424                     video_url = urljoin(url, path_id)
 425                     entries.append(self.url_result(
 426                         video_url, ie=RaiPlayIE.ie_key(),
 427                         video_id=RaiPlayIE._match_id(video_url)))
 428
 429         return self.playlist_result(
 430             entries, playlist_id, playlist_title,
 431             try_get(program, lambda x: x['program_info']['description']))
 432
 433
 434 class RaiPlaySoundIE(RaiBaseIE):
 435     _VALID_URL = rf'(?P<base>https?://(?:www\.)?raiplaysound\.it/.+?-(?P<id>{RaiBaseIE._UUID_RE}))\.(?:html|json)'
 436     _TESTS = [{
 437         'url': 'https://www.raiplaysound.it/audio/2021/12/IL-RUGGITO-DEL-CONIGLIO-1ebae2a7-7cdb-42bb-842e-fe0d193e9707.html',
 438         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
 439         'info_dict': {
 440             'id': '1ebae2a7-7cdb-42bb-842e-fe0d193e9707',
 441             'ext': 'mp3',
 442             'title': 'Il Ruggito del Coniglio del 10/12/2021',
 443             'alt_title': 'md5:0e6476cd57858bb0f3fcc835d305b455',
 444             'description': 'md5:2a17d2107e59a4a8faa0e18334139ee2',
 445             'thumbnail': r're:^https?://.+\.jpg$',
 446             'uploader': 'rai radio 2',
 447             'duration': 5685,
 448             'series': 'Il Ruggito del Coniglio',
 449             'episode': 'Il Ruggito del Coniglio del 10/12/2021',
 450             'creator': 'rai radio 2',
 451             'timestamp': 1638346620,
 452             'upload_date': '20211201',
 453         },
 454         'params': {'skip_download': True},
 455     }]
 456
 457     def _real_extract(self, url):
 458         base, audio_id = self._match_valid_url(url).group('base', 'id')
 459         media = self._download_json(f'{base}.json', audio_id, 'Downloading audio JSON')
 460         uid = try_get(media, lambda x: remove_start(remove_start(x['uniquename'], 'ContentItem-'), 'Page-'))
 461
 462         info = {}
 463         formats = []
 464         relinkers = set(traverse_obj(media, (('downloadable_audio', 'audio', ('live', 'cards', 0, 'audio')), 'url')))
 465         for r in relinkers:
 466             info = self._extract_relinker_info(r, audio_id, True)
 467             formats.extend(info.get('formats'))
 468
 469         date_published = try_get(media, (lambda x: f'{x["create_date"]} {x.get("create_time") or ""}',
 470                                          lambda x: x['live']['create_date']))
 471
 472         podcast_info = traverse_obj(media, 'podcast_info', ('live', 'cards', 0)) or {}
 473
 474         return {
 475             **info,
 476             'id': uid or audio_id,
 477             'display_id': audio_id,
 478             'title': traverse_obj(media, 'title', 'episode_title'),
 479             'alt_title': traverse_obj(media, ('track_info', 'media_name'), expected_type=strip_or_none),
 480             'description': media.get('description'),
 481             'uploader': traverse_obj(media, ('track_info', 'channel'), expected_type=strip_or_none),
 482             'creator': traverse_obj(media, ('track_info', 'editor'), expected_type=strip_or_none),
 483             'timestamp': unified_timestamp(date_published),
 484             'thumbnails': self._get_thumbnails_list(podcast_info.get('images'), url),
 485             'series': podcast_info.get('title'),
 486             'season_number': int_or_none(media.get('season')),
 487             'episode': media.get('episode_title'),
 488             'episode_number': int_or_none(media.get('episode')),
 489             'formats': formats,
 490         }
 491
 492
 493 class RaiPlaySoundLiveIE(RaiPlaySoundIE):  # XXX: Do not subclass from concrete IE
 494     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?P<id>[^/?#&]+)$)'
 495     _TESTS = [{
 496         'url': 'https://www.raiplaysound.it/radio2',
 497         'info_dict': {
 498             'id': 'b00a50e6-f404-4af6-8f8c-ff3b9af73a44',
 499             'display_id': 'radio2',
 500             'ext': 'mp4',
 501             'title': r're:Rai Radio 2 \d+-\d+-\d+ \d+:\d+',
 502             'thumbnail': r're:^https://www\.raiplaysound\.it/dl/img/.+\.png',
 503             'uploader': 'rai radio 2',
 504             'series': 'Rai Radio 2',
 505             'creator': 'raiplaysound',
 506             'is_live': True,
 507             'live_status': 'is_live',
 508         },
 509         'params': {'skip_download': True},
 510     }]
 511
 512
 513 class RaiPlaySoundPlaylistIE(InfoExtractor):
 514     _VALID_URL = r'(?P<base>https?://(?:www\.)?raiplaysound\.it/(?:programmi|playlist|audiolibri)/(?P<id>[^/?#&]+))(?:/(?P<extra_id>[^?#&]+))?'
 515     _TESTS = [{
 516         # entire show
 517         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio',
 518         'info_dict': {
 519             'id': 'ilruggitodelconiglio',
 520             'title': 'Il Ruggito del Coniglio',
 521             'description': 'md5:48cff6972435964284614d70474132e6',
 522         },
 523         'playlist_mincount': 65,
 524     }, {
 525         # single season
 526         'url': 'https://www.raiplaysound.it/programmi/ilruggitodelconiglio/puntate/prima-stagione-1995',
 527         'info_dict': {
 528             'id': 'ilruggitodelconiglio_puntate_prima-stagione-1995',
 529             'title': 'Prima Stagione 1995',
 530         },
 531         'playlist_count': 1,
 532     }]
 533
 534     def _real_extract(self, url):
 535         base, playlist_id, extra_id = self._match_valid_url(url).group('base', 'id', 'extra_id')
 536         url = f'{base}.json'
 537         program = self._download_json(url, playlist_id, 'Downloading program JSON')
 538
 539         if extra_id:
 540             extra_id = extra_id.rstrip('/')
 541             playlist_id += '_' + extra_id.replace('/', '_')
 542             path = next(c['path_id'] for c in program.get('filters') or [] if extra_id in c.get('weblink'))
 543             program = self._download_json(
 544                 urljoin('https://www.raiplaysound.it', path), playlist_id, 'Downloading program secondary JSON')
 545
 546         entries = [
 547             self.url_result(urljoin(base, c['path_id']), ie=RaiPlaySoundIE.ie_key())
 548             for c in traverse_obj(program, 'cards', ('block', 'cards')) or []
 549             if c.get('path_id')]
 550
 551         return self.playlist_result(entries, playlist_id, program.get('title'),
 552                                     traverse_obj(program, ('podcast_info', 'description')))
 553
 554
 555 class RaiIE(RaiBaseIE):
 556     _VALID_URL = rf'https?://[^/]+\.(?:rai\.(?:it|tv))/.+?-(?P<id>{RaiBaseIE._UUID_RE})(?:-.+?)?\.html'
 557     _TESTS = [{
 558         'url': 'https://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
 559         'info_dict': {
 560             'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
 561             'ext': 'mp4',
 562             'title': 'TG PRIMO TEMPO',
 563             'thumbnail': r're:^https?://.*\.jpg',
 564             'duration': 1758,
 565             'upload_date': '20140612',
 566         },
 567         'params': {'skip_download': True},
 568         'expected_warnings': ['Video not available. Likely due to geo-restriction.']
 569     }, {
 570         'url': 'https://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
 571         'info_dict': {
 572             'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
 573             'ext': 'mp4',
 574             'title': 'TG1 ore 20:00 del 03/11/2016',
 575             'description': 'TG1 edizione integrale ore 20:00 del giorno 03/11/2016',
 576             'thumbnail': r're:^https?://.*\.jpg$',
 577             'duration': 2214,
 578             'upload_date': '20161103'
 579         },
 580         'params': {'skip_download': True},
 581     }, {
 582         # Direct MMS: Media URL no longer works.
 583         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html',
 584         'only_matching': True,
 585     }]
 586
 587     def _real_extract(self, url):
 588         content_id = self._match_id(url)
 589         media = self._download_json(
 590             f'https://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-{content_id}.html?json',
 591             content_id, 'Downloading video JSON', fatal=False, expected_status=404)
 592
 593         if media is None:
 594             return None
 595
 596         if 'Audio' in media['type']:
 597             relinker_info = {
 598                 'formats': [{
 599                     'format_id': join_nonempty('https', media.get('formatoAudio'), delim='-'),
 600                     'url': media['audioUrl'],
 601                     'ext': media.get('formatoAudio'),
 602                     'vcodec': 'none',
 603                     'acodec': media.get('formatoAudio'),
 604                 }]
 605             }
 606         elif 'Video' in media['type']:
 607             relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)
 608         else:
 609             raise ExtractorError('not a media file')
 610
 611         thumbnails = self._get_thumbnails_list(
 612             {image_type: media.get(image_type) for image_type in (
 613                 'image', 'image_medium', 'image_300')}, url)
 614
 615         return {
 616             'id': content_id,
 617             'title': strip_or_none(media.get('name') or media.get('title')),
 618             'description': strip_or_none(media.get('desc')) or None,
 619             'thumbnails': thumbnails,
 620             'uploader': strip_or_none(media.get('author')) or None,
 621             'upload_date': unified_strdate(media.get('date')),
 622             'duration': parse_duration(media.get('length')),
 623             'subtitles': self._extract_subtitles(url, media),
 624             **relinker_info
 625         }
 626
 627
 628 class RaiNewsIE(RaiIE):  # XXX: Do not subclass from concrete IE
 629     _VALID_URL = rf'https?://(www\.)?rainews\.it/(?!articoli)[^?#]+-(?P<id>{RaiBaseIE._UUID_RE})(?:-[^/?#]+)?\.html'
 630     _EMBED_REGEX = [rf'<iframe[^>]+data-src="(?P<url>/iframe/[^?#]+?{RaiBaseIE._UUID_RE}\.html)']
 631     _TESTS = [{
 632         # new rainews player (#3911)
 633         'url': 'https://www.rainews.it/rubriche/24mm/video/2022/05/24mm-del-29052022-12cf645d-1ffd-4220-b27c-07c226dbdecf.html',
 634         'info_dict': {
 635             'id': '12cf645d-1ffd-4220-b27c-07c226dbdecf',
 636             'ext': 'mp4',
 637             'title': 'Puntata del 29/05/2022',
 638             'duration': 1589,
 639             'upload_date': '20220529',
 640             'uploader': 'rainews',
 641         },
 642         'params': {'skip_download': True},
 643     }, {
 644         # old content with fallback method to extract media urls
 645         'url': 'https://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
 646         'info_dict': {
 647             'id': '1632c009-c843-4836-bb65-80c33084a64b',
 648             'ext': 'mp4',
 649             'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor "La ragazza del treno"',
 650             'description': 'I film in uscita questa settimana.',
 651             'thumbnail': r're:^https?://.*\.png$',
 652             'duration': 833,
 653             'upload_date': '20161103'
 654         },
 655         'params': {'skip_download': True},
 656         'expected_warnings': ['unable to extract player_data'],
 657     }, {
 658         # iframe + drm
 659         'url': 'https://www.rainews.it/iframe/video/2022/07/euro2022-europei-calcio-femminile-italia-belgio-gol-0-1-video-4de06a69-de75-4e32-a657-02f0885f8118.html',
 660         'only_matching': True,
 661     }]
 662     _PLAYER_TAG = 'news'
 663
 664     def _real_extract(self, url):
 665         video_id = self._match_id(url)
 666
 667         webpage = self._download_webpage(url, video_id)
 668
 669         player_data = self._search_json(
 670             rf'<rai{self._PLAYER_TAG}-player\s*data=\'', webpage, 'player_data', video_id,
 671             transform_source=clean_html, default={})
 672         track_info = player_data.get('track_info')
 673         relinker_url = traverse_obj(player_data, 'mediapolis', 'content_url')
 674
 675         if not relinker_url:
 676             # fallback on old implementation for some old content
 677             try:
 678                 return self._extract_from_content_id(video_id, url)
 679             except GeoRestrictedError:
 680                 raise
 681             except ExtractorError as e:
 682                 raise ExtractorError('Relinker URL not found', cause=e)
 683
 684         relinker_info = self._extract_relinker_info(urljoin(url, relinker_url), video_id)
 685
 686         return {
 687             'id': video_id,
 688             'title': player_data.get('title') or track_info.get('title') or self._og_search_title(webpage),
 689             'upload_date': unified_strdate(track_info.get('date')),
 690             'uploader': strip_or_none(track_info.get('editor') or None),
 691             **relinker_info
 692         }
 693
 694
 695 class RaiCulturaIE(RaiNewsIE):  # XXX: Do not subclass from concrete IE
 696     _VALID_URL = rf'https?://(www\.)?raicultura\.it/(?!articoli)[^?#]+-(?P<id>{RaiBaseIE._UUID_RE})(?:-[^/?#]+)?\.html'
 697     _EMBED_REGEX = [rf'<iframe[^>]+data-src="(?P<url>/iframe/[^?#]+?{RaiBaseIE._UUID_RE}\.html)']
 698     _TESTS = [{
 699         'url': 'https://www.raicultura.it/letteratura/articoli/2018/12/Alberto-Asor-Rosa-Letteratura-e-potere-05ba8775-82b5-45c5-a89d-dd955fbde1fb.html',
 700         'info_dict': {
 701             'id': '05ba8775-82b5-45c5-a89d-dd955fbde1fb',
 702             'ext': 'mp4',
 703             'title': 'Alberto Asor Rosa: Letteratura e potere',
 704             'duration': 1756,
 705             'upload_date': '20181206',
 706             'uploader': 'raicultura',
 707             'formats': 'count:2',
 708         },
 709         'params': {'skip_download': True},
 710     }]
 711     _PLAYER_TAG = 'cultura'
 712
 713
 714 class RaiSudtirolIE(RaiBaseIE):
 715     _VALID_URL = r'https?://raisudtirol\.rai\.it/.+media=(?P<id>\w+)'
 716     _TESTS = [{
 717         # mp4 file
 718         'url': 'https://raisudtirol.rai.it/la/index.php?media=Ptv1619729460',
 719         'info_dict': {
 720             'id': 'Ptv1619729460',
 721             'ext': 'mp4',
 722             'title': 'Euro: trasmisciun d\'economia - 29-04-2021 20:51',
 723             'series': 'Euro: trasmisciun d\'economia',
 724             'upload_date': '20210429',
 725             'thumbnail': r're:https://raisudtirol\.rai\.it/img/.+\.jpg',
 726             'uploader': 'raisudtirol',
 727             'formats': 'count:1',
 728         },
 729         'params': {'skip_download': True},
 730     }, {
 731         # m3u manifest
 732         'url': 'https://raisudtirol.rai.it/it/kidsplayer.php?lang=it&media=GUGGUG_P1.smil',
 733         'info_dict': {
 734             'id': 'GUGGUG_P1',
 735             'ext': 'mp4',
 736             'title': 'GUGGUG! La Prospettiva - Die Perspektive',
 737             'uploader': 'raisudtirol',
 738             'formats': 'count:6',
 739         },
 740         'params': {'skip_download': True},
 741     }]
 742
 743     def _real_extract(self, url):
 744         video_id = self._match_id(url)
 745         webpage = self._download_webpage(url, video_id)
 746
 747         video_date = self._html_search_regex(
 748             r'<span class="med_data">(.+?)</span>', webpage, 'video_date', default=None)
 749         video_title = self._html_search_regex([
 750             r'<span class="med_title">(.+?)</span>', r'title: \'(.+?)\','],
 751             webpage, 'video_title', default=None)
 752         video_url = self._html_search_regex([
 753             r'sources:\s*\[\{file:\s*"(.+?)"\}\]',
 754             r'<source\s+src="(.+?)"\s+type="application/x-mpegURL"'],
 755             webpage, 'video_url', default=None)
 756
 757         ext = determine_ext(video_url)
 758         if ext == 'm3u8':
 759             formats = self._extract_m3u8_formats(video_url, video_id)
 760         elif ext == 'mp4':
 761             formats = [{
 762                 'format_id': 'https-mp4',
 763                 'url': self._proto_relative_url(video_url),
 764                 'width': 1024,
 765                 'height': 576,
 766                 'fps': 25,
 767                 'vcodec': 'avc1',
 768                 'acodec': 'mp4a',
 769             }]
 770         else:
 771             formats = []
 772             self.raise_no_formats(f'Unrecognized media file: {video_url}')
 773
 774         return {
 775             'id': video_id,
 776             'title': join_nonempty(video_title, video_date, delim=' - '),
 777             'series': video_title if video_date else None,
 778             'upload_date': unified_strdate(video_date),
 779             'thumbnail': urljoin('https://raisudtirol.rai.it/', self._html_search_regex(
 780                 r'image: \'(.+?)\'', webpage, 'video_thumb', default=None)),
 781             'uploader': 'raisudtirol',
 782             'formats': formats,
 783         }