]> jfr.im git - yt-dlp.git/blame_incremental - yt_dlp/extractor/mediaset.py
projects / yt-dlp.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Allow extractors to specify section_start/end for clips
[yt-dlp.git] / yt_dlp / extractor / mediaset.py
... / ...
CommitLineData
1import functools
2import re
3
4from .theplatform import ThePlatformBaseIE
5from ..utils import (
6 ExtractorError,
7 GeoRestrictedError,
8 int_or_none,
9 OnDemandPagedList,
10 parse_qs,
11 try_get,
12 urljoin,
13 update_url_query,
14)
15
16
17class MediasetIE(ThePlatformBaseIE):
18 _TP_TLD = 'eu'
19 _VALID_URL = r'''(?x)
20 (?:
21 mediaset:|
22 https?://
23 (?:\w+\.)+mediaset\.it/
24 (?:
25 (?:video|on-demand|movie)/(?:[^/]+/)+[^/]+_|
26 player/index\.html\?.*?\bprogramGuid=
27 )
28 )(?P<id>[0-9A-Z]{16,})
29 '''
30 _TESTS = [{
31 # full episode
32 'url': 'https://www.mediasetplay.mediaset.it/video/mrwronglezionidamore/episodio-1_F310575103000102',
33 'md5': 'a7e75c6384871f322adb781d3bd72c26',
34 'info_dict': {
35 'id': 'F310575103000102',
36 'ext': 'mp4',
37 'title': 'Episodio 1',
38 'description': 'md5:e8017b7d7194e9bfb75299c2b8d81e02',
39 'thumbnail': r're:^https?://.*\.jpg$',
40 'duration': 2682.0,
41 'upload_date': '20210530',
42 'series': 'Mr Wrong - Lezioni d\'amore',
43 'timestamp': 1622413946,
44 'uploader': 'Canale 5',
45 'uploader_id': 'C5',
46 'season': 'Season 1',
47 'episode': 'Episode 1',
48 'season_number': 1,
49 'episode_number': 1,
50 'chapters': [{'start_time': 0.0, 'end_time': 439.88}, {'start_time': 439.88, 'end_time': 1685.84}, {'start_time': 1685.84, 'end_time': 2682.0}],
51 },
52 }, {
53 'url': 'https://www.mediasetplay.mediaset.it/video/matrix/puntata-del-25-maggio_F309013801000501',
54 'md5': '1276f966ac423d16ba255ce867de073e',
55 'info_dict': {
56 'id': 'F309013801000501',
57 'ext': 'mp4',
58 'title': 'Puntata del 25 maggio',
59 'description': 'md5:ee2e456e3eb1dba5e814596655bb5296',
60 'thumbnail': r're:^https?://.*\.jpg$',
61 'duration': 6565.008,
62 'upload_date': '20200903',
63 'series': 'Matrix',
64 'timestamp': 1599172492,
65 'uploader': 'Canale 5',
66 'uploader_id': 'C5',
67 'season': 'Season 5',
68 'episode': 'Episode 5',
69 'season_number': 5,
70 'episode_number': 5,
71 'chapters': [{'start_time': 0.0, 'end_time': 3409.08}, {'start_time': 3409.08, 'end_time': 6565.008}],
72 },
73 }, {
74 'url': 'https://www.mediasetplay.mediaset.it/video/cameracafe5/episodio-69-pezzo-di-luna_F303843101017801',
75 'md5': 'd1650ac9ff944f185556126a736df148',
76 'info_dict': {
77 'id': 'F303843101017801',
78 'ext': 'mp4',
79 'title': 'Episodio 69 - Pezzo di luna',
80 'description': 'md5:7c32c8ec4118b72588b9412f11353f73',
81 'thumbnail': r're:^https?://.*\.jpg$',
82 'duration': 263.008,
83 'upload_date': '20200902',
84 'series': 'Camera Café 5',
85 'timestamp': 1599064700,
86 'uploader': 'Italia 1',
87 'uploader_id': 'I1',
88 'season': 'Season 5',
89 'episode': 'Episode 178',
90 'season_number': 5,
91 'episode_number': 178,
92 'chapters': [{'start_time': 0.0, 'end_time': 261.88}, {'start_time': 261.88, 'end_time': 263.008}],
93 },
94 }, {
95 'url': 'https://www.mediasetplay.mediaset.it/video/cameracafe5/episodio-51-tu-chi-sei_F303843107000601',
96 'md5': '567e9ad375b7a27a0e370650f572a1e3',
97 'info_dict': {
98 'id': 'F303843107000601',
99 'ext': 'mp4',
100 'title': 'Episodio 51 - Tu chi sei?',
101 'description': 'md5:42ef006e56824cc31787a547590923f4',
102 'thumbnail': r're:^https?://.*\.jpg$',
103 'duration': 367.021,
104 'upload_date': '20200902',
105 'series': 'Camera Café 5',
106 'timestamp': 1599069817,
107 'uploader': 'Italia 1',
108 'uploader_id': 'I1',
109 'season': 'Season 5',
110 'episode': 'Episode 6',
111 'season_number': 5,
112 'episode_number': 6,
113 'chapters': [{'start_time': 0.0, 'end_time': 358.68}, {'start_time': 358.68, 'end_time': 367.021}],
114 },
115 }, {
116 # movie
117 'url': 'https://www.mediasetplay.mediaset.it/movie/selvaggi/selvaggi_F006474501000101',
118 'md5': '720440187a2ae26af8148eb9e6b901ed',
119 'info_dict': {
120 'id': 'F006474501000101',
121 'ext': 'mp4',
122 'title': 'Selvaggi',
123 'description': 'md5:cfdedbbfdd12d4d0e5dcf1fa1b75284f',
124 'thumbnail': r're:^https?://.*\.jpg$',
125 'duration': 5233.01,
126 'upload_date': '20210729',
127 'timestamp': 1627594716,
128 'uploader': 'Cine34',
129 'uploader_id': 'B6',
130 'chapters': [{'start_time': 0.0, 'end_time': 1938.56}, {'start_time': 1938.56, 'end_time': 5233.01}],
131 },
132 }, {
133 # clip
134 'url': 'https://www.mediasetplay.mediaset.it/video/gogglebox/un-grande-classico-della-commedia-sexy_FAFU000000661680',
135 'only_matching': True,
136 }, {
137 # iframe simple
138 'url': 'https://static3.mediasetplay.mediaset.it/player/index.html?appKey=5ad3966b1de1c4000d5cec48&programGuid=FAFU000000665924&id=665924',
139 'only_matching': True,
140 }, {
141 # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
142 'url': 'https://static3.mediasetplay.mediaset.it/player/index.html?appKey=5ad3966b1de1c4000d5cec48&programGuid=FAFU000000665104&id=665104',
143 'only_matching': True,
144 }, {
145 'url': 'mediaset:FAFU000000665924',
146 'only_matching': True,
147 }, {
148 'url': 'https://www.mediasetplay.mediaset.it/video/mediasethaacuoreilfuturo/palmieri-alicudi-lisola-dei-tre-bambini-felici--un-decreto-per-alicudi-e-tutte-le-microscuole_FD00000000102295',
149 'only_matching': True,
150 }, {
151 'url': 'https://www.mediasetplay.mediaset.it/video/cherryseason/anticipazioni-degli-episodi-del-23-ottobre_F306837101005C02',
152 'only_matching': True,
153 }, {
154 'url': 'https://www.mediasetplay.mediaset.it/video/tg5/ambiente-onda-umana-per-salvare-il-pianeta_F309453601079D01',
155 'only_matching': True,
156 }, {
157 'url': 'https://www.mediasetplay.mediaset.it/video/grandefratellovip/benedetta-una-doccia-gelata_F309344401044C135',
158 'only_matching': True,
159 }, {
160 'url': 'https://www.mediasetplay.mediaset.it/movie/herculeslaleggendahainizio/hercules-la-leggenda-ha-inizio_F305927501000102',
161 'only_matching': True,
162 }, {
163 'url': 'https://mediasetinfinity.mediaset.it/video/braveandbeautiful/episodio-113_F310948005000402',
164 'only_matching': True,
165 }]
166
167 @staticmethod
168 def _extract_urls(ie, webpage):
169 def _qs(url):
170 return parse_qs(url)
171
172 def _program_guid(qs):
173 return qs.get('programGuid', [None])[0]
174
175 entries = []
176 for mobj in re.finditer(
177 r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?video\.mediaset\.it/player/playerIFrame(?:Twitter)?\.shtml.*?)\1',
178 webpage):
179 embed_url = mobj.group('url')
180 embed_qs = _qs(embed_url)
181 program_guid = _program_guid(embed_qs)
182 if program_guid:
183 entries.append(embed_url)
184 continue
185 video_id = embed_qs.get('id', [None])[0]
186 if not video_id:
187 continue
188 urlh = ie._request_webpage(
189 embed_url, video_id, note='Following embed URL redirect')
190 embed_url = urlh.geturl()
191 program_guid = _program_guid(_qs(embed_url))
192 if program_guid:
193 entries.append(embed_url)
194 return entries
195
196 def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
197 for video in smil.findall(self._xpath_ns('.//video', namespace)):
198 video.attrib['src'] = re.sub(r'(https?://vod05)t(-mediaset-it\.akamaized\.net/.+?.mpd)\?.+', r'\1\2', video.attrib['src'])
199 return super(MediasetIE, self)._parse_smil_formats(smil, smil_url, video_id, namespace, f4m_params, transform_rtmp_url)
200
201 def _check_drm_formats(self, tp_formats, video_id):
202 has_nondrm, drm_manifest = False, ''
203 for f in tp_formats:
204 if '_sampleaes/' in (f.get('manifest_url') or ''):
205 drm_manifest = drm_manifest or f['manifest_url']
206 f['has_drm'] = True
207 if not f.get('has_drm') and f.get('manifest_url'):
208 has_nondrm = True
209
210 nodrm_manifest = re.sub(r'_sampleaes/(\w+)_fp_', r'/\1_no_', drm_manifest)
211 if has_nondrm or nodrm_manifest == drm_manifest:
212 return
213
214 tp_formats.extend(self._extract_m3u8_formats(
215 nodrm_manifest, video_id, m3u8_id='hls', fatal=False) or [])
216
217 def _real_extract(self, url):
218 guid = self._match_id(url)
219 tp_path = 'PR1GhC/media/guid/2702976343/' + guid
220 info = self._extract_theplatform_metadata(tp_path, guid)
221
222 formats = []
223 subtitles = {}
224 first_e = geo_e = None
225 asset_type = 'geoNo:HD,browser,geoIT|geoNo:HD,geoIT|geoNo:SD,browser,geoIT|geoNo:SD,geoIT|geoNo|HD|SD'
226 # TODO: fixup ISM+none manifest URLs
227 for f in ('MPEG4', 'M3U'):
228 try:
229 tp_formats, tp_subtitles = self._extract_theplatform_smil(
230 update_url_query('http://link.theplatform.%s/s/%s' % (self._TP_TLD, tp_path), {
231 'mbr': 'true',
232 'formats': f,
233 'assetTypes': asset_type,
234 }), guid, 'Downloading %s SMIL data' % (f.split('+')[0]))
235 except ExtractorError as e:
236 if not geo_e and isinstance(e, GeoRestrictedError):
237 geo_e = e
238 if not first_e:
239 first_e = e
240 continue
241 self._check_drm_formats(tp_formats, guid)
242 formats.extend(tp_formats)
243 subtitles = self._merge_subtitles(subtitles, tp_subtitles)
244
245 # check for errors and report them
246 if (first_e or geo_e) and not formats:
247 raise geo_e or first_e
248
249 self._sort_formats(formats)
250
251 feed_data = self._download_json(
252 'https://feed.entertainment.tv.theplatform.eu/f/PR1GhC/mediaset-prod-all-programs-v2/guid/-/' + guid,
253 guid, fatal=False)
254 if feed_data:
255 publish_info = feed_data.get('mediasetprogram$publishInfo') or {}
256 thumbnails = feed_data.get('thumbnails') or {}
257 thumbnail = None
258 for key, value in thumbnails.items():
259 if key.startswith('image_keyframe_poster-'):
260 thumbnail = value.get('url')
261 break
262
263 info.update({
264 'description': info.get('description') or feed_data.get('description') or feed_data.get('longDescription'),
265 'uploader': publish_info.get('description'),
266 'uploader_id': publish_info.get('channel'),
267 'view_count': int_or_none(feed_data.get('mediasetprogram$numberOfViews')),
268 'thumbnail': thumbnail,
269 })
270
271 if feed_data.get('programType') == 'episode':
272 info.update({
273 'episode_number': int_or_none(
274 feed_data.get('tvSeasonEpisodeNumber')),
275 'season_number': int_or_none(
276 feed_data.get('tvSeasonNumber')),
277 'series': feed_data.get('mediasetprogram$brandTitle'),
278 })
279
280 info.update({
281 'id': guid,
282 'formats': formats,
283 'subtitles': subtitles,
284 })
285 return info
286
287
288class MediasetShowIE(MediasetIE):
289 _VALID_URL = r'''(?x)
290 (?:
291 https?://
292 (\w+\.)+mediaset\.it/
293 (?:
294 (?:fiction|programmi-tv|serie-tv|kids)/(?:.+?/)?
295 (?:[a-z-]+)_SE(?P<id>\d{12})
296 (?:,ST(?P<st>\d{12}))?
297 (?:,sb(?P<sb>\d{9}))?$
298 )
299 )
300 '''
301 _TESTS = [{
302 # TV Show webpage (general webpage)
303 'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/leiene_SE000000000061',
304 'info_dict': {
305 'id': '000000000061',
306 'title': 'Le Iene',
307 },
308 'playlist_mincount': 7,
309 }, {
310 # TV Show webpage (specific season)
311 'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/leiene_SE000000000061,ST000000002763',
312 'info_dict': {
313 'id': '000000002763',
314 'title': 'Le Iene',
315 },
316 'playlist_mincount': 7,
317 }, {
318 # TV Show specific playlist (with multiple pages)
319 'url': 'https://www.mediasetplay.mediaset.it/programmi-tv/leiene/iservizi_SE000000000061,ST000000002763,sb100013375',
320 'info_dict': {
321 'id': '100013375',
322 'title': 'I servizi',
323 },
324 'playlist_mincount': 50,
325 }]
326
327 _BY_SUBBRAND = 'https://feed.entertainment.tv.theplatform.eu/f/PR1GhC/mediaset-prod-all-programs-v2?byCustomValue={subBrandId}{%s}&sort=:publishInfo_lastPublished|desc,tvSeasonEpisodeNumber|desc&range=%d-%d'
328 _PAGE_SIZE = 25
329
330 def _fetch_page(self, sb, page):
331 lower_limit = page * self._PAGE_SIZE + 1
332 upper_limit = lower_limit + self._PAGE_SIZE - 1
333 content = self._download_json(
334 self._BY_SUBBRAND % (sb, lower_limit, upper_limit), sb)
335 for entry in content.get('entries') or []:
336 yield self.url_result(
337 'mediaset:' + entry['guid'],
338 playlist_title=entry['mediasetprogram$subBrandDescription'])
339
340 def _real_extract(self, url):
341 playlist_id, st, sb = self._match_valid_url(url).group('id', 'st', 'sb')
342 if not sb:
343 page = self._download_webpage(url, st or playlist_id)
344 entries = [self.url_result(urljoin('https://www.mediasetplay.mediaset.it', url))
345 for url in re.findall(r'href="([^<>=]+SE\d{12},ST\d{12},sb\d{9})">[^<]+<', page)]
346 title = (self._html_search_regex(r'(?s)<h1[^>]*>(.+?)</h1>', page, 'title', default=None)
347 or self._og_search_title(page))
348 return self.playlist_result(entries, st or playlist_id, title)
349
350 entries = OnDemandPagedList(
351 functools.partial(self._fetch_page, sb),
352 self._PAGE_SIZE)
353 title = try_get(entries, lambda x: x[0]['playlist_title'])
354
355 return self.playlist_result(entries, sb, title)
fork of https://github.com/yt-dlp/yt-dlp/