]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/orf.py
[ie/youtube] Extract upload timestamp if available (#9856)
[yt-dlp.git] / yt_dlp / extractor / orf.py
CommitLineData
a0d50aab 1import base64
b695e3f9 2import functools
5d73273f 3import re
54543467
JMF
4
5from .common import InfoExtractor
6from ..utils import (
aca2fd22 7 clean_html,
0732a905
PH
8 determine_ext,
9 float_or_none,
14137b57 10 int_or_none,
6ba30856
E
11 make_archive_id,
12 mimetype2ext,
0732a905 13 orderedSet,
0dd53fae 14 parse_age_limit,
14137b57 15 remove_end,
0732a905 16 strip_jsonp,
5770293d 17 try_call,
0732a905 18 unified_strdate,
4b6aca17 19 url_or_none,
54543467 20)
6ba30856 21from ..utils.traversal import traverse_obj
54543467 22
5d73273f 23
efe93167 24class ORFRadioIE(InfoExtractor):
5770293d 25 IE_NAME = 'orf:radio'
26
27 STATION_INFO = {
28 'fm4': ('fm4', 'fm4', 'orffm4'),
29 'noe': ('noe', 'oe2n', 'orfnoe'),
30 'wien': ('wie', 'oe2w', 'orfwie'),
31 'burgenland': ('bgl', 'oe2b', 'orfbgl'),
32 'ooe': ('ooe', 'oe2o', 'orfooe'),
33 'steiermark': ('stm', 'oe2st', 'orfstm'),
34 'kaernten': ('ktn', 'oe2k', 'orfktn'),
35 'salzburg': ('sbg', 'oe2s', 'orfsbg'),
36 'tirol': ('tir', 'oe2t', 'orftir'),
37 'vorarlberg': ('vbg', 'oe2v', 'orfvbg'),
38 'oe3': ('oe3', 'oe3', 'orfoe3'),
39 'oe1': ('oe1', 'oe1', 'orfoe1'),
40 }
41 _STATION_RE = '|'.join(map(re.escape, STATION_INFO.keys()))
eb368012 42
5770293d 43 _VALID_URL = rf'''(?x)
44 https?://(?:
45 (?P<station>{_STATION_RE})\.orf\.at/player|
46 radiothek\.orf\.at/(?P<station2>{_STATION_RE})
47 )/(?P<date>[0-9]+)/(?P<show>\w+)'''
aca2fd22 48
5770293d 49 _TESTS = [{
50 'url': 'https://radiothek.orf.at/ooe/20220801/OGMO',
51 'info_dict': {
52 'id': 'OGMO',
53 'title': 'Guten Morgen OÖ',
54 'description': 'md5:a3f6083399ef92b8cbe2d421b180835a',
55 },
56 'playlist': [{
57 'md5': 'f33147d954a326e338ea52572c2810e8',
58 'info_dict': {
59 'id': '2022-08-01_0459_tl_66_7DaysMon1_319062',
0146c6cd 60 'ext': 'mp3',
5770293d 61 'title': 'Guten Morgen OÖ',
62 'upload_date': '20220801',
63 'duration': 18000,
64 'timestamp': 1659322789,
65 'description': 'md5:a3f6083399ef92b8cbe2d421b180835a',
66 }
67 }]
68 }, {
69 'url': 'https://ooe.orf.at/player/20220801/OGMO',
1339ecb2 70 'info_dict': {
5770293d 71 'id': 'OGMO',
72 'title': 'Guten Morgen OÖ',
73 'description': 'md5:a3f6083399ef92b8cbe2d421b180835a',
1339ecb2 74 },
5770293d 75 'playlist': [{
76 'md5': 'f33147d954a326e338ea52572c2810e8',
77 'info_dict': {
78 'id': '2022-08-01_0459_tl_66_7DaysMon1_319062',
79 'ext': 'mp3',
80 'title': 'Guten Morgen OÖ',
81 'upload_date': '20220801',
82 'duration': 18000,
83 'timestamp': 1659322789,
84 'description': 'md5:a3f6083399ef92b8cbe2d421b180835a',
85 }
86 }]
87 }, {
88 'url': 'http://fm4.orf.at/player/20170107/4CC',
aca2fd22 89 'only_matching': True,
5770293d 90 }, {
13283058 91 'url': 'https://noe.orf.at/player/20200423/NGM',
92 'only_matching': True,
5770293d 93 }, {
13283058 94 'url': 'https://wien.orf.at/player/20200423/WGUM',
95 'only_matching': True,
5770293d 96 }, {
13283058 97 'url': 'https://burgenland.orf.at/player/20200423/BGM',
98 'only_matching': True,
5770293d 99 }, {
13283058 100 'url': 'https://steiermark.orf.at/player/20200423/STGMS',
101 'only_matching': True,
5770293d 102 }, {
13283058 103 'url': 'https://kaernten.orf.at/player/20200423/KGUMO',
104 'only_matching': True,
5770293d 105 }, {
13283058 106 'url': 'https://salzburg.orf.at/player/20200423/SGUM',
107 'only_matching': True,
5770293d 108 }, {
13283058 109 'url': 'https://tirol.orf.at/player/20200423/TGUMO',
110 'only_matching': True,
5770293d 111 }, {
13283058 112 'url': 'https://vorarlberg.orf.at/player/20200423/VGUM',
113 'only_matching': True,
5770293d 114 }, {
13283058 115 'url': 'https://oe3.orf.at/player/20200424/3WEK',
116 'only_matching': True,
5770293d 117 }, {
1339ecb2
S
118 'url': 'http://oe1.orf.at/player/20170108/456544',
119 'md5': '34d8a6e67ea888293741c86a099b745b',
120 'info_dict': {
121 'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
122 'ext': 'mp3',
123 'title': 'Morgenjournal',
124 'duration': 609,
125 'timestamp': 1483858796,
126 'upload_date': '20170108',
127 },
128 'skip': 'Shows from ORF radios are only available for 7 days.'
5770293d 129 }]
130
131 def _entries(self, data, station):
132 _, loop_station, old_ie = self.STATION_INFO[station]
133 for info in data['streams']:
134 item_id = info.get('loopStreamId')
135 if not item_id:
136 continue
137 video_id = item_id.replace('.mp3', '')
138 yield {
139 'id': video_id,
140 'ext': 'mp3',
141 'url': f'https://loopstream01.apa.at/?channel={loop_station}&id={item_id}',
142 '_old_archive_ids': [make_archive_id(old_ie, video_id)],
143 'title': data.get('title'),
144 'description': clean_html(data.get('subtitle')),
145 'duration': try_call(lambda: (info['end'] - info['start']) / 1000),
146 'timestamp': int_or_none(info.get('start'), scale=1000),
147 'series': data.get('programTitle'),
148 }
149
150 def _real_extract(self, url):
151 station, station2, show_date, show_id = self._match_valid_url(url).group('station', 'station2', 'date', 'show')
152 api_station, _, _ = self.STATION_INFO[station or station2]
153 data = self._download_json(
154 f'http://audioapi.orf.at/{api_station}/api/json/current/broadcast/{show_id}/{show_date}', show_id)
155
156 return self.playlist_result(
157 self._entries(data, station or station2), show_id, data.get('title'), clean_html(data.get('subtitle')))
efe93167 158
159
6ba30856
E
160class ORFPodcastIE(InfoExtractor):
161 IE_NAME = 'orf:podcast'
162 _STATION_RE = '|'.join(map(re.escape, (
163 'bgl', 'fm4', 'ktn', 'noe', 'oe1', 'oe3',
164 'ooe', 'sbg', 'stm', 'tir', 'tv', 'vbg', 'wie')))
165 _VALID_URL = rf'https?://sound\.orf\.at/podcast/(?P<station>{_STATION_RE})/(?P<show>[\w-]+)/(?P<id>[\w-]+)'
166 _TESTS = [{
167 'url': 'https://sound.orf.at/podcast/oe3/fruehstueck-bei-mir/nicolas-stockhammer-15102023',
168 'md5': '526a5700e03d271a1505386a8721ab9b',
169 'info_dict': {
170 'id': 'nicolas-stockhammer-15102023',
171 'ext': 'mp3',
172 'title': 'Nicolas Stockhammer (15.10.2023)',
173 'duration': 3396.0,
174 'series': 'Frühstück bei mir',
175 },
176 'skip': 'ORF podcasts are only available for a limited time'
177 }]
178
179 def _real_extract(self, url):
180 station, show, show_id = self._match_valid_url(url).group('station', 'show', 'id')
181 data = self._download_json(
182 f'https://audioapi.orf.at/radiothek/api/2.0/podcast/{station}/{show}/{show_id}', show_id)
183
184 return {
185 'id': show_id,
186 'ext': 'mp3',
187 'vcodec': 'none',
188 **traverse_obj(data, ('payload', {
189 'url': ('enclosures', 0, 'url'),
190 'ext': ('enclosures', 0, 'type', {mimetype2ext}),
191 'title': 'title',
192 'description': ('description', {clean_html}),
193 'duration': ('duration', {functools.partial(float_or_none, scale=1000)}),
194 'series': ('podcast', 'title'),
195 })),
196 }
197
198
14137b57
S
199class ORFIPTVIE(InfoExtractor):
200 IE_NAME = 'orf:iptv'
201 IE_DESC = 'iptv.ORF.at'
5886b38d 202 _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
14137b57
S
203
204 _TEST = {
529d26c3
S
205 'url': 'http://iptv.orf.at/stories/2275236/',
206 'md5': 'c8b22af4718a4b4af58342529453e3e5',
14137b57 207 'info_dict': {
529d26c3 208 'id': '350612',
14137b57 209 'ext': 'flv',
529d26c3
S
210 'title': 'Weitere Evakuierungen um Vulkan Calbuco',
211 'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
212 'duration': 68.197,
ec85ded8 213 'thumbnail': r're:^https?://.*\.jpg$',
529d26c3 214 'upload_date': '20150425',
14137b57
S
215 },
216 }
217
218 def _real_extract(self, url):
219 story_id = self._match_id(url)
220
221 webpage = self._download_webpage(
222 'http://iptv.orf.at/stories/%s' % story_id, story_id)
223
224 video_id = self._search_regex(
225 r'data-video(?:id)?="(\d+)"', webpage, 'video id')
226
227 data = self._download_json(
228 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
229 video_id)[0]
230
231 duration = float_or_none(data['duration'], 1000)
232
233 video = data['sources']['default']
234 load_balancer_url = video['loadBalancerUrl']
235 abr = int_or_none(video.get('audioBitrate'))
236 vbr = int_or_none(video.get('bitrate'))
237 fps = int_or_none(video.get('videoFps'))
238 width = int_or_none(video.get('videoWidth'))
239 height = int_or_none(video.get('videoHeight'))
240 thumbnail = video.get('preview')
241
242 rendition = self._download_json(
243 load_balancer_url, video_id, transform_source=strip_jsonp)
244
245 f = {
246 'abr': abr,
247 'vbr': vbr,
248 'fps': fps,
249 'width': width,
250 'height': height,
251 }
252
253 formats = []
254 for format_id, format_url in rendition['redirect'].items():
255 if format_id == 'rtmp':
256 ff = f.copy()
257 ff.update({
258 'url': format_url,
259 'format_id': format_id,
260 })
261 formats.append(ff)
262 elif determine_ext(format_url) == 'f4m':
263 formats.extend(self._extract_f4m_formats(
264 format_url, video_id, f4m_id=format_id))
265 elif determine_ext(format_url) == 'm3u8':
266 formats.extend(self._extract_m3u8_formats(
267 format_url, video_id, 'mp4', m3u8_id=format_id))
268 else:
269 continue
14137b57
S
270
271 title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
272 description = self._og_search_description(webpage)
273 upload_date = unified_strdate(self._html_search_meta(
274 'dc.date', webpage, 'upload date'))
275
276 return {
277 'id': video_id,
278 'title': title,
279 'description': description,
280 'duration': duration,
281 'thumbnail': thumbnail,
282 'upload_date': upload_date,
283 'formats': formats,
284 }
0732a905
PH
285
286
287class ORFFM4StoryIE(InfoExtractor):
288 IE_NAME = 'orf:fm4:story'
289 IE_DESC = 'fm4.orf.at stories'
290 _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
291
292 _TEST = {
293 'url': 'http://fm4.orf.at/stories/2865738/',
294 'playlist': [{
295 'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
296 'info_dict': {
297 'id': '547792',
298 'ext': 'flv',
299 'title': 'Manu Delago und Inner Tongue live',
300 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
301 'duration': 1748.52,
302 'thumbnail': r're:^https?://.*\.jpg$',
303 'upload_date': '20170913',
304 },
305 }, {
306 'md5': 'c6dd2179731f86f4f55a7b49899d515f',
307 'info_dict': {
308 'id': '547798',
309 'ext': 'flv',
310 'title': 'Manu Delago und Inner Tongue live (2)',
311 'duration': 1504.08,
312 'thumbnail': r're:^https?://.*\.jpg$',
313 'upload_date': '20170913',
314 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
315 },
316 }],
317 }
318
319 def _real_extract(self, url):
320 story_id = self._match_id(url)
321 webpage = self._download_webpage(url, story_id)
322
323 entries = []
324 all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
325 for idx, video_id in enumerate(all_ids):
326 data = self._download_json(
327 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
328 video_id)[0]
329
330 duration = float_or_none(data['duration'], 1000)
331
332 video = data['sources']['q8c']
333 load_balancer_url = video['loadBalancerUrl']
334 abr = int_or_none(video.get('audioBitrate'))
335 vbr = int_or_none(video.get('bitrate'))
336 fps = int_or_none(video.get('videoFps'))
337 width = int_or_none(video.get('videoWidth'))
338 height = int_or_none(video.get('videoHeight'))
339 thumbnail = video.get('preview')
340
341 rendition = self._download_json(
342 load_balancer_url, video_id, transform_source=strip_jsonp)
343
344 f = {
345 'abr': abr,
346 'vbr': vbr,
347 'fps': fps,
348 'width': width,
349 'height': height,
350 }
351
352 formats = []
353 for format_id, format_url in rendition['redirect'].items():
354 if format_id == 'rtmp':
355 ff = f.copy()
356 ff.update({
357 'url': format_url,
358 'format_id': format_id,
359 })
360 formats.append(ff)
361 elif determine_ext(format_url) == 'f4m':
362 formats.extend(self._extract_f4m_formats(
363 format_url, video_id, f4m_id=format_id))
364 elif determine_ext(format_url) == 'm3u8':
365 formats.extend(self._extract_m3u8_formats(
366 format_url, video_id, 'mp4', m3u8_id=format_id))
367 else:
368 continue
0732a905
PH
369
370 title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
371 if idx >= 1:
372 # Titles are duplicates, make them unique
373 title += ' (' + str(idx + 1) + ')'
374 description = self._og_search_description(webpage)
375 upload_date = unified_strdate(self._html_search_meta(
376 'dc.date', webpage, 'upload date'))
377
378 entries.append({
379 'id': video_id,
380 'title': title,
381 'description': description,
382 'duration': duration,
383 'thumbnail': thumbnail,
384 'upload_date': upload_date,
385 'formats': formats,
386 })
387
388 return self.playlist_result(entries)
a0d50aab
H
389
390
391class ORFONIE(InfoExtractor):
392 IE_NAME = 'orf:on'
0dd53fae 393 _VALID_URL = r'https?://on\.orf\.at/video/(?P<id>\d+)'
a0d50aab
H
394 _TESTS = [{
395 'url': 'https://on.orf.at/video/14210000/school-of-champions-48',
396 'info_dict': {
397 'id': '14210000',
398 'ext': 'mp4',
399 'duration': 2651.08,
400 'thumbnail': 'https://api-tvthek.orf.at/assets/segments/0167/98/thumb_16697671_segments_highlight_teaser.jpeg',
401 'title': 'School of Champions (4/8)',
402 'description': 'md5:d09ad279fc2e8502611e7648484b6afd',
403 'media_type': 'episode',
404 'timestamp': 1706472362,
405 'upload_date': '20240128',
3779f2a3 406 '_old_archive_ids': ['orftvthek 14210000'],
a0d50aab 407 }
0dd53fae
T
408 }, {
409 'url': 'https://on.orf.at/video/3220355',
410 'md5': 'f94d98e667cf9a3851317efb4e136662',
411 'info_dict': {
412 'id': '3220355',
413 'ext': 'mp4',
414 'duration': 445.04,
415 'thumbnail': 'https://api-tvthek.orf.at/assets/segments/0002/60/thumb_159573_segments_highlight_teaser.png',
416 'title': '50 Jahre Burgenland: Der Festumzug',
417 'description': 'md5:1560bf855119544ee8c4fa5376a2a6b0',
418 'media_type': 'episode',
419 'timestamp': 52916400,
420 'upload_date': '19710905',
3779f2a3 421 '_old_archive_ids': ['orftvthek 3220355'],
0dd53fae 422 }
a0d50aab
H
423 }]
424
0dd53fae 425 def _extract_video(self, video_id):
a0d50aab
H
426 encrypted_id = base64.b64encode(f'3dSlfek03nsLKdj4Jsd{video_id}'.encode()).decode()
427 api_json = self._download_json(
0dd53fae
T
428 f'https://api-tvthek.orf.at/api/v4.3/public/episode/encrypted/{encrypted_id}', video_id)
429
430 if traverse_obj(api_json, 'is_drm_protected'):
431 self.report_drm(video_id)
a0d50aab
H
432
433 formats, subtitles = [], {}
434 for manifest_type in traverse_obj(api_json, ('sources', {dict.keys}, ...)):
435 for manifest_url in traverse_obj(api_json, ('sources', manifest_type, ..., 'src', {url_or_none})):
436 if manifest_type == 'hls':
437 fmts, subs = self._extract_m3u8_formats_and_subtitles(
0dd53fae 438 manifest_url, video_id, fatal=False, m3u8_id='hls')
a0d50aab
H
439 elif manifest_type == 'dash':
440 fmts, subs = self._extract_mpd_formats_and_subtitles(
0dd53fae 441 manifest_url, video_id, fatal=False, mpd_id='dash')
a0d50aab
H
442 else:
443 continue
444 formats.extend(fmts)
445 self._merge_subtitles(subs, target=subtitles)
446
0dd53fae
T
447 for sub_url in traverse_obj(api_json, (
448 '_embedded', 'subtitle',
449 ('xml_url', 'sami_url', 'stl_url', 'ttml_url', 'srt_url', 'vtt_url'), {url_or_none})):
450 self._merge_subtitles({'de': [{'url': sub_url}]}, target=subtitles)
451
a0d50aab
H
452 return {
453 'id': video_id,
454 'formats': formats,
455 'subtitles': subtitles,
3779f2a3 456 '_old_archive_ids': [make_archive_id('ORFTVthek', video_id)],
a0d50aab 457 **traverse_obj(api_json, {
0dd53fae 458 'age_limit': ('age_classification', {parse_age_limit}),
a0d50aab
H
459 'duration': ('duration_second', {float_or_none}),
460 'title': (('title', 'headline'), {str}),
461 'description': (('description', 'teaser_text'), {str}),
462 'media_type': ('video_type', {str}),
463 }, get_all=False),
464 }
465
466 def _real_extract(self, url):
0dd53fae
T
467 video_id = self._match_id(url)
468 webpage = self._download_webpage(url, video_id)
a0d50aab
H
469
470 return {
471 'id': video_id,
472 'title': self._html_search_meta(['og:title', 'twitter:title'], webpage, default=None),
473 'description': self._html_search_meta(
474 ['description', 'og:description', 'twitter:description'], webpage, default=None),
0dd53fae
T
475 **self._search_json_ld(webpage, video_id, fatal=False),
476 **self._extract_video(video_id),
a0d50aab 477 }