]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/ard.py
[extractor/youtube] Handle `consent.youtube`
[yt-dlp.git] / yt_dlp / extractor / ard.py
CommitLineData
c968f738 1import json
d5822b96
PH
2import re
3
4from .common import InfoExtractor
3741302a 5from .generic import GenericIE
d5822b96 6from ..utils import (
f9b85496 7 determine_ext,
d5822b96 8 ExtractorError,
6d3d3fc0
PH
9 int_or_none,
10 parse_duration,
75258218
S
11 qualities,
12 str_or_none,
13 try_get,
6d3d3fc0 14 unified_strdate,
75258218 15 unified_timestamp,
31eeab9f 16 update_url_query,
3052a30d 17 url_or_none,
75258218 18 xpath_text,
d5822b96 19)
f7854627 20from ..compat import compat_etree_fromstring
d5822b96 21
f9b85496 22
c968f738
RA
23class ARDMediathekBaseIE(InfoExtractor):
24 _GEO_COUNTRIES = ['DE']
1c821227 25
e37c92ec
S
26 def _extract_media_info(self, media_info_url, webpage, video_id):
27 media_info = self._download_json(
28 media_info_url, video_id, 'Downloading media JSON')
c968f738 29 return self._parse_media_info(media_info, video_id, '"fsk"' in webpage)
e37c92ec 30
c968f738 31 def _parse_media_info(self, media_info, video_id, fsk):
e37c92ec
S
32 formats = self._extract_formats(media_info, video_id)
33
34 if not formats:
c968f738 35 if fsk:
b7da73eb 36 self.raise_no_formats(
e37c92ec
S
37 'This video is only available after 20:00', expected=True)
38 elif media_info.get('_geoblocked'):
c968f738
RA
39 self.raise_geo_restricted(
40 'This video is not available due to geoblocking',
b7da73eb 41 countries=self._GEO_COUNTRIES, metadata_available=True)
e37c92ec 42
e37c92ec
S
43 subtitles = {}
44 subtitle_url = media_info.get('_subtitleUrl')
45 if subtitle_url:
46 subtitles['de'] = [{
ffa2cecf 47 'ext': 'ttml',
e37c92ec 48 'url': subtitle_url,
d61ef7f3
GM
49 }, {
50 'ext': 'vtt',
51 'url': subtitle_url.replace('/ebutt/', '/webvtt/') + '.vtt',
e37c92ec
S
52 }]
53
54 return {
55 'id': video_id,
c968f738
RA
56 'duration': int_or_none(media_info.get('_duration')),
57 'thumbnail': media_info.get('_previewImage'),
58 'is_live': media_info.get('_isLive') is True,
e37c92ec
S
59 'formats': formats,
60 'subtitles': subtitles,
61 }
62
e6e5d98c 63 def _ARD_extract_episode_info(self, title):
64 """Try to extract season/episode data from the title."""
65 res = {}
66 if not title:
67 return res
68
69 for pattern in [
70 # Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
71 # from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
72 r'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
73 # E.g.: title="Fritjof aus Norwegen (2) (AD)"
74 # from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
75 r'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
76 r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
77 # E.g.: title="Folge 25/42: Symmetrie"
78 # from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
79 # E.g.: title="Folge 1063 - Vertrauen"
80 # from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
81 r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
82 ]:
83 m = re.match(pattern, title)
84 if m:
85 groupdict = m.groupdict()
86 res['season_number'] = int_or_none(groupdict.get('season_number'))
87 res['episode_number'] = int_or_none(groupdict.get('episode_number'))
88 res['episode'] = str_or_none(groupdict.get('episode'))
89 # Build the episode title by removing numeric episode information:
90 if groupdict.get('ep_info') and not res['episode']:
91 res['episode'] = str_or_none(
92 title.replace(groupdict.get('ep_info'), ''))
93 if res['episode']:
94 res['episode'] = res['episode'].strip()
95 break
96
97 # As a fallback use the whole title as the episode name:
98 if not res.get('episode'):
99 res['episode'] = title.strip()
100 return res
101
e37c92ec
S
102 def _extract_formats(self, media_info, video_id):
103 type_ = media_info.get('_type')
104 media_array = media_info.get('_mediaArray', [])
105 formats = []
106 for num, media in enumerate(media_array):
107 for stream in media.get('_mediaStreamArray', []):
108 stream_urls = stream.get('_stream')
109 if not stream_urls:
110 continue
111 if not isinstance(stream_urls, list):
112 stream_urls = [stream_urls]
113 quality = stream.get('_quality')
114 server = stream.get('_server')
115 for stream_url in stream_urls:
3052a30d 116 if not url_or_none(stream_url):
91328f26 117 continue
e37c92ec 118 ext = determine_ext(stream_url)
1fc0b47f 119 if quality != 'auto' and ext in ('f4m', 'm3u8'):
120 continue
e37c92ec 121 if ext == 'f4m':
7e5edcfd 122 formats.extend(self._extract_f4m_formats(
31eeab9f
RA
123 update_url_query(stream_url, {
124 'hdcore': '3.1.1',
125 'plugin': 'aasp-3.1.1.69.124'
c968f738 126 }), video_id, f4m_id='hds', fatal=False))
e37c92ec 127 elif ext == 'm3u8':
7e5edcfd 128 formats.extend(self._extract_m3u8_formats(
c968f738
RA
129 stream_url, video_id, 'mp4', 'm3u8_native',
130 m3u8_id='hls', fatal=False))
e37c92ec
S
131 else:
132 if server and server.startswith('rtmp'):
133 f = {
134 'url': server,
135 'play_path': stream_url,
136 'format_id': 'a%s-rtmp-%s' % (num, quality),
137 }
91328f26 138 else:
e37c92ec
S
139 f = {
140 'url': stream_url,
141 'format_id': 'a%s-%s-%s' % (num, ext, quality)
142 }
c968f738
RA
143 m = re.search(
144 r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$',
145 stream_url)
e37c92ec
S
146 if m:
147 f.update({
148 'width': int(m.group('width')),
149 'height': int(m.group('height')),
150 })
151 if type_ == 'audio':
152 f['vcodec'] = 'none'
153 formats.append(f)
154 return formats
155
c968f738
RA
156
157class ARDMediathekIE(ARDMediathekBaseIE):
158 IE_NAME = 'ARD:mediathek'
159 _VALID_URL = r'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
160
161 _TESTS = [{
162 # available till 26.07.2022
163 'url': 'http://www.ardmediathek.de/tv/S%C3%9CDLICHT/Was-ist-die-Kunst-der-Zukunft-liebe-Ann/BR-Fernsehen/Video?bcastId=34633636&documentId=44726822',
164 'info_dict': {
165 'id': '44726822',
166 'ext': 'mp4',
167 'title': 'Was ist die Kunst der Zukunft, liebe Anna McCarthy?',
168 'description': 'md5:4ada28b3e3b5df01647310e41f3a62f5',
169 'duration': 1740,
170 },
171 'params': {
172 # m3u8 download
173 'skip_download': True,
174 }
175 }, {
176 'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872',
177 'only_matching': True,
178 }, {
179 # audio
180 'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
181 'only_matching': True,
182 }, {
183 'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
184 'only_matching': True,
185 }, {
186 # audio
187 'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
188 'only_matching': True,
189 }, {
190 'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698',
191 'only_matching': True,
192 }]
193
194 @classmethod
195 def suitable(cls, url):
196 return False if ARDBetaMediathekIE.suitable(url) else super(ARDMediathekIE, cls).suitable(url)
197
d5822b96
PH
198 def _real_extract(self, url):
199 # determine video id from url
5ad28e7f 200 m = self._match_valid_url(url)
d5822b96 201
ca127ab2
S
202 document_id = None
203
d5822b96
PH
204 numid = re.search(r'documentId=([0-9]+)', url)
205 if numid:
ca127ab2 206 document_id = video_id = numid.group(1)
d5822b96
PH
207 else:
208 video_id = m.group('video_id')
209
5622f29a 210 webpage = self._download_webpage(url, video_id)
f9b85496 211
3791d84a
S
212 ERRORS = (
213 ('>Leider liegt eine Störung vor.', 'Video %s is unavailable'),
214 ('>Der gewünschte Beitrag ist nicht mehr verfügbar.<',
215 'Video %s is no longer available'),
3791d84a
S
216 )
217
218 for pattern, message in ERRORS:
219 if pattern in webpage:
220 raise ExtractorError(message % video_id, expected=True)
39aa42ff 221
bfd91588 222 if re.search(r'[\?&]rss($|[=&])', url):
f7854627 223 doc = compat_etree_fromstring(webpage.encode('utf-8'))
3741302a
OE
224 if doc.tag == 'rss':
225 return GenericIE()._extract_rss(url, video_id, doc)
226
a820dc72 227 title = self._og_search_title(webpage, default=None) or self._html_search_regex(
0f97c9a0 228 [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
197224b7 229 r'<meta name="dcterms\.title" content="(.*?)"/>',
8c587971
AS
230 r'<h4 class="headline">(.*?)</h4>',
231 r'<title[^>]*>(.*?)</title>'],
0f97c9a0 232 webpage, 'title')
a820dc72 233 description = self._og_search_description(webpage, default=None) or self._html_search_meta(
29546b34
PH
234 'dcterms.abstract', webpage, 'description', default=None)
235 if description is None:
236 description = self._html_search_meta(
8c587971
AS
237 'description', webpage, 'meta description', default=None)
238 if description is None:
239 description = self._html_search_regex(
240 r'<p\s+class="teasertext">(.+?)</p>',
241 webpage, 'teaser text', default=None)
29546b34
PH
242
243 # Thumbnail is sometimes not present.
244 # It is in the mobile version, but that seems to use a different URL
245 # structure altogether.
246 thumbnail = self._og_search_thumbnail(webpage, default=None)
247
248 media_streams = re.findall(r'''(?x)
249 mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
250 "([^"]+)"''', webpage)
251
252 if media_streams:
253 QUALITIES = qualities(['lo', 'hi', 'hq'])
254 formats = []
255 for furl in set(media_streams):
256 if furl.endswith('.f4m'):
257 fid = 'f4m'
258 else:
259 fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
260 fid = fid_m.group(1) if fid_m else None
261 formats.append({
262 'quality': QUALITIES(fid),
263 'format_id': fid,
264 'url': furl,
265 })
e37c92ec
S
266 info = {
267 'formats': formats,
268 }
29546b34 269 else: # request JSON file
ca127ab2
S
270 if not document_id:
271 video_id = self._search_regex(
3f6a90eb 272 (r'/play/(?:config|media|sola)/(\d+)', r'contentId["\']\s*:\s*(\d+)'),
273 webpage, 'media id', default=None)
e37c92ec 274 info = self._extract_media_info(
ca127ab2
S
275 'http://www.ardmediathek.de/play/media/%s' % video_id,
276 webpage, video_id)
f9b85496 277
e37c92ec 278 info.update({
f9b85496 279 'id': video_id,
39ca3b5c 280 'title': title,
f9b85496 281 'description': description,
f9b85496 282 'thumbnail': thumbnail,
e37c92ec 283 })
e6e5d98c 284 info.update(self._ARD_extract_episode_info(info['title']))
e37c92ec
S
285
286 return info
6d3d3fc0
PH
287
288
289class ARDIE(InfoExtractor):
14eb1ee1 290 _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html'
ad29ef04 291 _TESTS = [{
d61ef7f3
GM
292 # available till 7.12.2023
293 'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-video-424.html',
294 'md5': 'a438f671e87a7eba04000336a119ccc4',
ad29ef04 295 'info_dict': {
d61ef7f3
GM
296 'id': 'maischberger-video-424',
297 'display_id': 'maischberger-video-424',
ad29ef04 298 'ext': 'mp4',
d61ef7f3
GM
299 'duration': 4452.0,
300 'title': 'maischberger am 07.12.2022',
301 'upload_date': '20221207',
ad29ef04
W
302 'thumbnail': r're:^https?://.*\.jpg$',
303 },
a54c5f83 304 }, {
14eb1ee1 305 'url': 'https://www.daserste.de/information/politik-weltgeschehen/morgenmagazin/videosextern/dominik-kahun-aus-der-nhl-direkt-zur-weltmeisterschaft-100.html',
306 'only_matching': True,
307 }, {
308 'url': 'https://www.daserste.de/information/nachrichten-wetter/tagesthemen/videosextern/tagesthemen-17736.html',
a54c5f83 309 'only_matching': True,
f17c7022
OF
310 }, {
311 'url': 'https://www.daserste.de/unterhaltung/serie/in-aller-freundschaft-die-jungen-aerzte/videos/diversity-tag-sanam-afrashteh100.html',
312 'only_matching': True,
d6a03502 313 }, {
6d3d3fc0 314 'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
d6a03502 315 'only_matching': True,
14eb1ee1 316 }, {
317 'url': 'https://www.daserste.de/unterhaltung/serie/in-aller-freundschaft-die-jungen-aerzte/Drehpause-100.html',
318 'only_matching': True,
319 }, {
320 'url': 'https://www.daserste.de/unterhaltung/film/filmmittwoch-im-ersten/videos/making-ofwendezeit-video-100.html',
321 'only_matching': True,
ad29ef04 322 }]
6d3d3fc0
PH
323
324 def _real_extract(self, url):
5ad28e7f 325 mobj = self._match_valid_url(url)
14eb1ee1 326 display_id = mobj.group('id')
6d3d3fc0
PH
327
328 player_url = mobj.group('mainurl') + '~playerXml.xml'
329 doc = self._download_xml(player_url, display_id)
330 video_node = doc.find('./video')
bf0ff932
PH
331 upload_date = unified_strdate(xpath_text(
332 video_node, './broadcastDate'))
333 thumbnail = xpath_text(video_node, './/teaserImage//variant/url')
6d3d3fc0
PH
334
335 formats = []
336 for a in video_node.findall('.//asset'):
bc2ca1bb 337 file_name = xpath_text(a, './fileName', default=None)
338 if not file_name:
339 continue
340 format_type = a.attrib.get('type')
341 format_url = url_or_none(file_name)
342 if format_url:
343 ext = determine_ext(file_name)
344 if ext == 'm3u8':
345 formats.extend(self._extract_m3u8_formats(
346 format_url, display_id, 'mp4', entry_protocol='m3u8_native',
347 m3u8_id=format_type or 'hls', fatal=False))
348 continue
349 elif ext == 'f4m':
350 formats.extend(self._extract_f4m_formats(
351 update_url_query(format_url, {'hdcore': '3.7.0'}),
352 display_id, f4m_id=format_type or 'hds', fatal=False))
353 continue
6d3d3fc0 354 f = {
bc2ca1bb 355 'format_id': format_type,
356 'width': int_or_none(xpath_text(a, './frameWidth')),
357 'height': int_or_none(xpath_text(a, './frameHeight')),
358 'vbr': int_or_none(xpath_text(a, './bitrateVideo')),
359 'abr': int_or_none(xpath_text(a, './bitrateAudio')),
360 'vcodec': xpath_text(a, './codecVideo'),
361 'tbr': int_or_none(xpath_text(a, './totalBitrate')),
6d3d3fc0 362 }
bc2ca1bb 363 server_prefix = xpath_text(a, './serverPrefix', default=None)
364 if server_prefix:
365 f.update({
366 'url': server_prefix,
367 'playpath': file_name,
368 })
6d3d3fc0 369 else:
bc2ca1bb 370 if not format_url:
371 continue
372 f['url'] = format_url
6d3d3fc0 373 formats.append(f)
6d3d3fc0 374
ad9158d5
F
375 _SUB_FORMATS = (
376 ('./dataTimedText', 'ttml'),
377 ('./dataTimedTextNoOffset', 'ttml'),
378 ('./dataTimedTextVtt', 'vtt'),
379 )
380
381 subtitles = {}
382 for subsel, subext in _SUB_FORMATS:
383 for node in video_node.findall(subsel):
384 subtitles.setdefault('de', []).append({
385 'url': node.attrib['url'],
386 'ext': subext,
387 })
388
6d3d3fc0 389 return {
14eb1ee1 390 'id': xpath_text(video_node, './videoId', default=display_id),
6d3d3fc0 391 'formats': formats,
ad9158d5 392 'subtitles': subtitles,
6d3d3fc0
PH
393 'display_id': display_id,
394 'title': video_node.find('./title').text,
395 'duration': parse_duration(video_node.find('./duration').text),
396 'upload_date': upload_date,
397 'thumbnail': thumbnail,
398 }
c1a37eb2
PH
399
400
c968f738 401class ARDBetaMediathekIE(ARDMediathekBaseIE):
14a08605 402 _VALID_URL = r'''(?x)https://
403 (?:(?:beta|www)\.)?ardmediathek\.de/
404 (?:(?P<client>[^/]+)/)?
405 (?:player|live|video|(?P<playlist>sendung|sammlung))/
409cdd1e 406 (?:(?P<display_id>(?(playlist)[^?#]+?|[^?#]+))/)?
407 (?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)
408 (?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
14a08605 409
c1a37eb2 410 _TESTS = [{
a820dc72
RA
411 'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
412 'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
c1a37eb2
PH
413 'info_dict': {
414 'display_id': 'die-robuste-roswita',
a820dc72 415 'id': '78566716',
c968f738 416 'title': 'Die robuste Roswita',
a820dc72 417 'description': r're:^Der Mord.*totgeglaubte Ehefrau Roswita',
c1a37eb2 418 'duration': 5316,
a820dc72
RA
419 'thumbnail': 'https://img.ardmediathek.de/standard/00/78/56/67/84/575672121/16x9/960?mandant=ard',
420 'timestamp': 1596658200,
421 'upload_date': '20200805',
c1a37eb2
PH
422 'ext': 'mp4',
423 },
14a08605 424 'skip': 'Error',
425 }, {
426 'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
427 'md5': 'f1837e563323b8a642a8ddeff0131f51',
428 'info_dict': {
429 'id': '10049223',
430 'ext': 'mp4',
431 'title': 'tagesschau, 20:00 Uhr',
432 'timestamp': 1636398000,
433 'description': 'md5:39578c7b96c9fe50afdf5674ad985e6b',
434 'upload_date': '20211108',
435 },
409cdd1e 436 }, {
437 'url': 'https://www.ardmediathek.de/sendung/beforeigners/beforeigners/staffel-1/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw/1',
438 'playlist_count': 6,
439 'info_dict': {
440 'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw',
441 'title': 'beforeigners/beforeigners/staffel-1',
442 },
fe515e5c
S
443 }, {
444 'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
445 'only_matching': True,
446 }, {
447 'url': 'https://ardmediathek.de/ard/video/saartalk/saartalk-gesellschaftsgift-haltung-gegen-hass/sr-fernsehen/Y3JpZDovL3NyLW9ubGluZS5kZS9TVF84MTY4MA/',
448 'only_matching': True,
449 }, {
450 'url': 'https://www.ardmediathek.de/ard/video/trailer/private-eyes-s01-e01/one/Y3JpZDovL3dkci5kZS9CZWl0cmFnLTE1MTgwYzczLWNiMTEtNGNkMS1iMjUyLTg5MGYzOWQxZmQ1YQ/',
451 'only_matching': True,
1c821227
S
452 }, {
453 'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3N3ci5kZS9hZXgvbzEwNzE5MTU/',
454 'only_matching': True,
455 }, {
456 'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
457 'only_matching': True,
e6e5d98c 458 }, {
459 # playlist of type 'sendung'
460 'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/',
461 'only_matching': True,
462 }, {
463 # playlist of type 'sammlung'
464 'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
465 'only_matching': True,
14a08605 466 }, {
467 'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
468 'only_matching': True,
469 }, {
470 'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3dkci5kZS9CZWl0cmFnLWQ2NDJjYWEzLTMwZWYtNGI4NS1iMTI2LTU1N2UxYTcxOGIzOQ/tatort-duo-koeln-leipzig-ihr-kinderlein-kommet',
471 'only_matching': True,
c1a37eb2
PH
472 }]
473
e6e5d98c 474 def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, pageNumber):
475 """ Query the ARD server for playlist information
476 and returns the data in "raw" format """
477 if mode == 'sendung':
478 graphQL = json.dumps({
479 'query': '''{
480 showPage(
481 client: "%s"
482 showId: "%s"
483 pageNumber: %d
484 ) {
485 pagination {
486 pageSize
487 totalElements
488 }
489 teasers { # Array
490 mediumTitle
491 links { target { id href title } }
492 type
493 }
494 }}''' % (client, playlist_id, pageNumber),
495 }).encode()
496 else: # mode == 'sammlung'
497 graphQL = json.dumps({
498 'query': '''{
499 morePage(
500 client: "%s"
501 compilationId: "%s"
502 pageNumber: %d
503 ) {
504 widget {
505 pagination {
506 pageSize
507 totalElements
508 }
509 teasers { # Array
510 mediumTitle
511 links { target { id href title } }
512 type
513 }
514 }
515 }}''' % (client, playlist_id, pageNumber),
516 }).encode()
517 # Ressources for ARD graphQL debugging:
518 # https://api-test.ardmediathek.de/public-gateway
519 show_page = self._download_json(
520 'https://api.ardmediathek.de/public-gateway',
521 '[Playlist] %s' % display_id,
522 data=graphQL,
523 headers={'Content-Type': 'application/json'})['data']
524 # align the structure of the returned data:
525 if mode == 'sendung':
526 show_page = show_page['showPage']
527 else: # mode == 'sammlung'
528 show_page = show_page['morePage']['widget']
529 return show_page
530
531 def _ARD_extract_playlist(self, url, playlist_id, display_id, client, mode):
532 """ Collects all playlist entries and returns them as info dict.
533 Supports playlists of mode 'sendung' and 'sammlung', and also nested
534 playlists. """
535 entries = []
536 pageNumber = 0
537 while True: # iterate by pageNumber
538 show_page = self._ARD_load_playlist_snipped(
539 playlist_id, display_id, client, mode, pageNumber)
540 for teaser in show_page['teasers']: # process playlist items
541 if '/compilation/' in teaser['links']['target']['href']:
542 # alternativ cond.: teaser['type'] == "compilation"
543 # => This is an nested compilation, e.g. like:
544 # https://www.ardmediathek.de/ard/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2/
545 link_mode = 'sammlung'
546 else:
547 link_mode = 'video'
548
549 item_url = 'https://www.ardmediathek.de/%s/%s/%s/%s/%s' % (
550 client, link_mode, display_id,
551 # perform HTLM quoting of episode title similar to ARD:
552 re.sub('^-|-$', '', # remove '-' from begin/end
553 re.sub('[^a-zA-Z0-9]+', '-', # replace special chars by -
554 teaser['links']['target']['title'].lower()
555 .replace('ä', 'ae').replace('ö', 'oe')
556 .replace('ü', 'ue').replace('ß', 'ss'))),
557 teaser['links']['target']['id'])
558 entries.append(self.url_result(
559 item_url,
560 ie=ARDBetaMediathekIE.ie_key()))
561
562 if (show_page['pagination']['pageSize'] * (pageNumber + 1)
563 >= show_page['pagination']['totalElements']):
564 # we've processed enough pages to get all playlist entries
565 break
566 pageNumber = pageNumber + 1
567
409cdd1e 568 return self.playlist_result(entries, playlist_id, playlist_title=display_id)
e6e5d98c 569
c1a37eb2 570 def _real_extract(self, url):
409cdd1e 571 video_id, display_id, playlist_type, client, season_number = self._match_valid_url(url).group(
572 'id', 'display_id', 'playlist', 'client', 'season')
14a08605 573 display_id, client = display_id or video_id, client or 'ard'
574
575 if playlist_type:
409cdd1e 576 # TODO: Extract only specified season
14a08605 577 return self._ARD_extract_playlist(url, video_id, display_id, client, playlist_type)
e6e5d98c 578
c968f738
RA
579 player_page = self._download_json(
580 'https://api.ardmediathek.de/public-gateway',
b704fc1a 581 display_id, data=json.dumps({
c968f738 582 'query': '''{
b704fc1a 583 playerPage(client:"%s", clipId: "%s") {
c968f738
RA
584 blockedByFsk
585 broadcastedOn
586 maturityContentRating
587 mediaCollection {
588 _duration
589 _geoblocked
590 _isLive
591 _mediaArray {
592 _mediaStreamArray {
593 _quality
594 _server
595 _stream
c1a37eb2 596 }
c968f738
RA
597 }
598 _previewImage
599 _subtitleUrl
600 _type
601 }
602 show {
603 title
604 }
605 synopsis
606 title
607 tracking {
608 atiCustomVars {
609 contentId
610 }
611 }
612 }
14a08605 613}''' % (client, video_id),
c968f738
RA
614 }).encode(), headers={
615 'Content-Type': 'application/json'
616 })['data']['playerPage']
617 title = player_page['title']
618 content_id = str_or_none(try_get(
619 player_page, lambda x: x['tracking']['atiCustomVars']['contentId']))
620 media_collection = player_page.get('mediaCollection') or {}
621 if not media_collection and content_id:
622 media_collection = self._download_json(
623 'https://www.ardmediathek.de/play/media/' + content_id,
624 content_id, fatal=False) or {}
625 info = self._parse_media_info(
626 media_collection, content_id or video_id,
627 player_page.get('blockedByFsk'))
628 age_limit = None
629 description = player_page.get('synopsis')
630 maturity_content_rating = player_page.get('maturityContentRating')
631 if maturity_content_rating:
632 age_limit = int_or_none(maturity_content_rating.lstrip('FSK'))
f4a18db7
RA
633 if not age_limit and description:
634 age_limit = int_or_none(self._search_regex(
635 r'\(FSK\s*(\d+)\)\s*$', description, 'age limit', default=None))
c968f738
RA
636 info.update({
637 'age_limit': age_limit,
b704fc1a 638 'display_id': display_id,
c968f738
RA
639 'title': title,
640 'description': description,
641 'timestamp': unified_timestamp(player_page.get('broadcastedOn')),
642 'series': try_get(player_page, lambda x: x['show']['title']),
75258218 643 })
e6e5d98c 644 info.update(self._ARD_extract_episode_info(info['title']))
c968f738 645 return info