4 from .common
import InfoExtractor
5 from .generic
import GenericIE
20 from ..compat
import compat_etree_fromstring
23 class ARDMediathekBaseIE(InfoExtractor
):
24 _GEO_COUNTRIES
= ['DE']
26 def _extract_media_info(self
, media_info_url
, webpage
, video_id
):
27 media_info
= self
._download
_json
(
28 media_info_url
, video_id
, 'Downloading media JSON')
29 return self
._parse
_media
_info
(media_info
, video_id
, '"fsk"' in webpage
)
31 def _parse_media_info(self
, media_info
, video_id
, fsk
):
32 formats
= self
._extract
_formats
(media_info
, video_id
)
36 self
.raise_no_formats(
37 'This video is only available after 20:00', expected
=True)
38 elif media_info
.get('_geoblocked'):
39 self
.raise_geo_restricted(
40 'This video is not available due to geoblocking',
41 countries
=self
._GEO
_COUNTRIES
, metadata_available
=True)
44 subtitle_url
= media_info
.get('_subtitleUrl')
51 'url': subtitle_url
.replace('/ebutt/', '/webvtt/') + '.vtt',
56 'duration': int_or_none(media_info
.get('_duration')),
57 'thumbnail': media_info
.get('_previewImage'),
58 'is_live': media_info
.get('_isLive') is True,
60 'subtitles': subtitles
,
63 def _ARD_extract_episode_info(self
, title
):
64 """Try to extract season/episode data from the title."""
70 # Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
71 # from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
72 r
'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
73 # E.g.: title="Fritjof aus Norwegen (2) (AD)"
74 # from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
75 r
'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
76 r
'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
77 # E.g.: title="Folge 25/42: Symmetrie"
78 # from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
79 # E.g.: title="Folge 1063 - Vertrauen"
80 # from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
81 r
'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
83 m
= re
.match(pattern
, title
)
85 groupdict
= m
.groupdict()
86 res
['season_number'] = int_or_none(groupdict
.get('season_number'))
87 res
['episode_number'] = int_or_none(groupdict
.get('episode_number'))
88 res
['episode'] = str_or_none(groupdict
.get('episode'))
89 # Build the episode title by removing numeric episode information:
90 if groupdict
.get('ep_info') and not res
['episode']:
91 res
['episode'] = str_or_none(
92 title
.replace(groupdict
.get('ep_info'), ''))
94 res
['episode'] = res
['episode'].strip()
97 # As a fallback use the whole title as the episode name:
98 if not res
.get('episode'):
99 res
['episode'] = title
.strip()
102 def _extract_formats(self
, media_info
, video_id
):
103 type_
= media_info
.get('_type')
104 media_array
= media_info
.get('_mediaArray', [])
106 for num
, media
in enumerate(media_array
):
107 for stream
in media
.get('_mediaStreamArray', []):
108 stream_urls
= stream
.get('_stream')
111 if not isinstance(stream_urls
, list):
112 stream_urls
= [stream_urls
]
113 quality
= stream
.get('_quality')
114 server
= stream
.get('_server')
115 for stream_url
in stream_urls
:
116 if not url_or_none(stream_url
):
118 ext
= determine_ext(stream_url
)
119 if quality
!= 'auto' and ext
in ('f4m', 'm3u8'):
122 formats
.extend(self
._extract
_f
4m
_formats
(
123 update_url_query(stream_url
, {
125 'plugin': 'aasp-3.1.1.69.124'
126 }), video_id
, f4m_id
='hds', fatal
=False))
128 formats
.extend(self
._extract
_m
3u8_formats
(
129 stream_url
, video_id
, 'mp4', 'm3u8_native',
130 m3u8_id
='hls', fatal
=False))
132 if server
and server
.startswith('rtmp'):
135 'play_path': stream_url
,
136 'format_id': 'a%s-rtmp-%s' % (num
, quality
),
141 'format_id': 'a%s-%s-%s' % (num
, ext
, quality
)
144 r
'_(?P<width>\d+)x(?P<height>\d+)\.mp4$',
148 'width': int(m
.group('width')),
149 'height': int(m
.group('height')),
157 class ARDMediathekIE(ARDMediathekBaseIE
):
158 IE_NAME
= 'ARD:mediathek'
159 _VALID_URL
= r
'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
162 # available till 26.07.2022
163 'url': 'http://www.ardmediathek.de/tv/S%C3%9CDLICHT/Was-ist-die-Kunst-der-Zukunft-liebe-Ann/BR-Fernsehen/Video?bcastId=34633636&documentId=44726822',
167 'title': 'Was ist die Kunst der Zukunft, liebe Anna McCarthy?',
168 'description': 'md5:4ada28b3e3b5df01647310e41f3a62f5',
173 'skip_download': True,
176 'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872',
177 'only_matching': True,
180 'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
181 'only_matching': True,
183 'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
184 'only_matching': True,
187 'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
188 'only_matching': True,
190 'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698',
191 'only_matching': True,
195 def suitable(cls
, url
):
196 return False if ARDBetaMediathekIE
.suitable(url
) else super(ARDMediathekIE
, cls
).suitable(url
)
198 def _real_extract(self
, url
):
199 # determine video id from url
200 m
= self
._match
_valid
_url
(url
)
204 numid
= re
.search(r
'documentId=([0-9]+)', url
)
206 document_id
= video_id
= numid
.group(1)
208 video_id
= m
.group('video_id')
210 webpage
= self
._download
_webpage
(url
, video_id
)
213 ('>Leider liegt eine Störung vor.', 'Video %s is unavailable'),
214 ('>Der gewünschte Beitrag ist nicht mehr verfügbar.<',
215 'Video %s is no longer available'),
218 for pattern
, message
in ERRORS
:
219 if pattern
in webpage
:
220 raise ExtractorError(message
% video_id
, expected
=True)
222 if re
.search(r
'[\?&]rss($|[=&])', url
):
223 doc
= compat_etree_fromstring(webpage
.encode('utf-8'))
225 return GenericIE()._extract
_rss
(url
, video_id
, doc
)
227 title
= self
._og
_search
_title
(webpage
, default
=None) or self
._html
_search
_regex
(
228 [r
'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
229 r
'<meta name="dcterms\.title" content="(.*?)"/>',
230 r
'<h4 class="headline">(.*?)</h4>',
231 r
'<title[^>]*>(.*?)</title>'],
233 description
= self
._og
_search
_description
(webpage
, default
=None) or self
._html
_search
_meta
(
234 'dcterms.abstract', webpage
, 'description', default
=None)
235 if description
is None:
236 description
= self
._html
_search
_meta
(
237 'description', webpage
, 'meta description', default
=None)
238 if description
is None:
239 description
= self
._html
_search
_regex
(
240 r
'<p\s+class="teasertext">(.+?)</p>',
241 webpage
, 'teaser text', default
=None)
243 # Thumbnail is sometimes not present.
244 # It is in the mobile version, but that seems to use a different URL
245 # structure altogether.
246 thumbnail
= self
._og
_search
_thumbnail
(webpage
, default
=None)
248 media_streams
= re
.findall(r
'''(?x)
249 mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
250 "([^"]+)"''', webpage
)
253 QUALITIES
= qualities(['lo', 'hi', 'hq'])
255 for furl
in set(media_streams
):
256 if furl
.endswith('.f4m'):
259 fid_m
= re
.match(r
'.*\.([^.]+)\.[^.]+$', furl
)
260 fid
= fid_m
.group(1) if fid_m
else None
262 'quality': QUALITIES(fid
),
269 else: # request JSON file
271 video_id
= self
._search
_regex
(
272 (r
'/play/(?:config|media|sola)/(\d+)', r
'contentId["\']\s
*:\s
*(\d
+)'),
273 webpage, 'media
id', default=None)
274 info = self._extract_media_info(
275 'http
://www
.ardmediathek
.de
/play
/media
/%s' % video_id,
281 'description
': description,
282 'thumbnail
': thumbnail,
284 info.update(self._ARD_extract_episode_info(info['title
']))
289 class ARDIE(InfoExtractor):
290 _VALID_URL = r'(?P
<mainurl
>https?
://(?
:www\
.)?daserste\
.de
/(?
:[^
/?
#&]+/)+(?P<id>[^/?#&]+))\.html'
292 # available till 7.12.2023
293 'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-video-424.html',
294 'md5': 'a438f671e87a7eba04000336a119ccc4',
296 'id': 'maischberger-video-424',
297 'display_id': 'maischberger-video-424',
300 'title': 'maischberger am 07.12.2022',
301 'upload_date': '20221207',
302 'thumbnail': r
're:^https?://.*\.jpg$',
305 'url': 'https://www.daserste.de/information/politik-weltgeschehen/morgenmagazin/videosextern/dominik-kahun-aus-der-nhl-direkt-zur-weltmeisterschaft-100.html',
306 'only_matching': True,
308 'url': 'https://www.daserste.de/information/nachrichten-wetter/tagesthemen/videosextern/tagesthemen-17736.html',
309 'only_matching': True,
311 'url': 'https://www.daserste.de/unterhaltung/serie/in-aller-freundschaft-die-jungen-aerzte/videos/diversity-tag-sanam-afrashteh100.html',
312 'only_matching': True,
314 'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
315 'only_matching': True,
317 'url': 'https://www.daserste.de/unterhaltung/serie/in-aller-freundschaft-die-jungen-aerzte/Drehpause-100.html',
318 'only_matching': True,
320 'url': 'https://www.daserste.de/unterhaltung/film/filmmittwoch-im-ersten/videos/making-ofwendezeit-video-100.html',
321 'only_matching': True,
324 def _real_extract(self
, url
):
325 mobj
= self
._match
_valid
_url
(url
)
326 display_id
= mobj
.group('id')
328 player_url
= mobj
.group('mainurl') + '~playerXml.xml'
329 doc
= self
._download
_xml
(player_url
, display_id
)
330 video_node
= doc
.find('./video')
331 upload_date
= unified_strdate(xpath_text(
332 video_node
, './broadcastDate'))
333 thumbnail
= xpath_text(video_node
, './/teaserImage//variant/url')
336 for a
in video_node
.findall('.//asset'):
337 file_name
= xpath_text(a
, './fileName', default
=None)
340 format_type
= a
.attrib
.get('type')
341 format_url
= url_or_none(file_name
)
343 ext
= determine_ext(file_name
)
345 formats
.extend(self
._extract
_m
3u8_formats
(
346 format_url
, display_id
, 'mp4', entry_protocol
='m3u8_native',
347 m3u8_id
=format_type
or 'hls', fatal
=False))
350 formats
.extend(self
._extract
_f
4m
_formats
(
351 update_url_query(format_url
, {'hdcore': '3.7.0'}
),
352 display_id
, f4m_id
=format_type
or 'hds', fatal
=False))
355 'format_id': format_type
,
356 'width': int_or_none(xpath_text(a
, './frameWidth')),
357 'height': int_or_none(xpath_text(a
, './frameHeight')),
358 'vbr': int_or_none(xpath_text(a
, './bitrateVideo')),
359 'abr': int_or_none(xpath_text(a
, './bitrateAudio')),
360 'vcodec': xpath_text(a
, './codecVideo'),
361 'tbr': int_or_none(xpath_text(a
, './totalBitrate')),
363 server_prefix
= xpath_text(a
, './serverPrefix', default
=None)
366 'url': server_prefix
,
367 'playpath': file_name
,
372 f
['url'] = format_url
376 ('./dataTimedText', 'ttml'),
377 ('./dataTimedTextNoOffset', 'ttml'),
378 ('./dataTimedTextVtt', 'vtt'),
382 for subsel
, subext
in _SUB_FORMATS
:
383 for node
in video_node
.findall(subsel
):
384 subtitles
.setdefault('de', []).append({
385 'url': node
.attrib
['url'],
390 'id': xpath_text(video_node
, './videoId', default
=display_id
),
392 'subtitles': subtitles
,
393 'display_id': display_id
,
394 'title': video_node
.find('./title').text
,
395 'duration': parse_duration(video_node
.find('./duration').text
),
396 'upload_date': upload_date
,
397 'thumbnail': thumbnail
,
401 class ARDBetaMediathekIE(ARDMediathekBaseIE
):
402 _VALID_URL
= r
'''(?x)https://
403 (?:(?:beta|www)\.)?ardmediathek\.de/
404 (?:(?P<client>[^/]+)/)?
405 (?:player|live|video|(?P<playlist>sendung|sammlung))/
406 (?:(?P<display_id>(?(playlist)[^?#]+?|[^?#]+))/)?
407 (?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)
408 (?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
411 'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
412 'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
414 'display_id': 'die-robuste-roswita',
416 'title': 'Die robuste Roswita',
417 'description': r
're:^Der Mord.*totgeglaubte Ehefrau Roswita',
419 'thumbnail': 'https://img.ardmediathek.de/standard/00/78/56/67/84/575672121/16x9/960?mandant=ard',
420 'timestamp': 1596658200,
421 'upload_date': '20200805',
426 'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
427 'md5': 'f1837e563323b8a642a8ddeff0131f51',
431 'title': 'tagesschau, 20:00 Uhr',
432 'timestamp': 1636398000,
433 'description': 'md5:39578c7b96c9fe50afdf5674ad985e6b',
434 'upload_date': '20211108',
437 'url': 'https://www.ardmediathek.de/sendung/beforeigners/beforeigners/staffel-1/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw/1',
440 'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JlZm9yZWlnbmVycw',
441 'title': 'beforeigners/beforeigners/staffel-1',
444 'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
445 'only_matching': True,
447 'url': 'https://ardmediathek.de/ard/video/saartalk/saartalk-gesellschaftsgift-haltung-gegen-hass/sr-fernsehen/Y3JpZDovL3NyLW9ubGluZS5kZS9TVF84MTY4MA/',
448 'only_matching': True,
450 'url': 'https://www.ardmediathek.de/ard/video/trailer/private-eyes-s01-e01/one/Y3JpZDovL3dkci5kZS9CZWl0cmFnLTE1MTgwYzczLWNiMTEtNGNkMS1iMjUyLTg5MGYzOWQxZmQ1YQ/',
451 'only_matching': True,
453 'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3N3ci5kZS9hZXgvbzEwNzE5MTU/',
454 'only_matching': True,
456 'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
457 'only_matching': True,
459 # playlist of type 'sendung'
460 'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/',
461 'only_matching': True,
463 # playlist of type 'sammlung'
464 'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
465 'only_matching': True,
467 'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
468 'only_matching': True,
470 'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3dkci5kZS9CZWl0cmFnLWQ2NDJjYWEzLTMwZWYtNGI4NS1iMTI2LTU1N2UxYTcxOGIzOQ/tatort-duo-koeln-leipzig-ihr-kinderlein-kommet',
471 'only_matching': True,
474 def _ARD_load_playlist_snipped(self
, playlist_id
, display_id
, client
, mode
, pageNumber
):
475 """ Query the ARD server for playlist information
476 and returns the data in "raw" format """
477 if mode
== 'sendung':
478 graphQL
= json
.dumps({
491 links { target { id href title } }
494 }}''' % (client
, playlist_id
, pageNumber
),
496 else: # mode == 'sammlung'
497 graphQL
= json
.dumps({
511 links { target { id href title } }
515 }}''' % (client
, playlist_id
, pageNumber
),
517 # Ressources for ARD graphQL debugging:
518 # https://api-test.ardmediathek.de/public-gateway
519 show_page
= self
._download
_json
(
520 'https://api.ardmediathek.de/public-gateway',
521 '[Playlist] %s' % display_id
,
523 headers
={'Content-Type': 'application/json'}
)['data']
524 # align the structure of the returned data:
525 if mode
== 'sendung':
526 show_page
= show_page
['showPage']
527 else: # mode == 'sammlung'
528 show_page
= show_page
['morePage']['widget']
531 def _ARD_extract_playlist(self
, url
, playlist_id
, display_id
, client
, mode
):
532 """ Collects all playlist entries and returns them as info dict.
533 Supports playlists of mode 'sendung' and 'sammlung', and also nested
537 while True: # iterate by pageNumber
538 show_page
= self
._ARD
_load
_playlist
_snipped
(
539 playlist_id
, display_id
, client
, mode
, pageNumber
)
540 for teaser
in show_page
['teasers']: # process playlist items
541 if '/compilation/' in teaser
['links']['target']['href']:
542 # alternativ cond.: teaser['type'] == "compilation"
543 # => This is an nested compilation, e.g. like:
544 # https://www.ardmediathek.de/ard/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2/
545 link_mode
= 'sammlung'
549 item_url
= 'https://www.ardmediathek.de/%s/%s/%s/%s/%s' % (
550 client
, link_mode
, display_id
,
551 # perform HTLM quoting of episode title similar to ARD:
552 re
.sub('^-|-$', '', # remove '-' from begin/end
553 re
.sub('[^a-zA-Z0-9]+', '-', # replace special chars by -
554 teaser
['links']['target']['title'].lower()
555 .replace('ä', 'ae').replace('ö', 'oe')
556 .replace('ü', 'ue').replace('ß', 'ss'))),
557 teaser
['links']['target']['id'])
558 entries
.append(self
.url_result(
560 ie
=ARDBetaMediathekIE
.ie_key()))
562 if (show_page
['pagination']['pageSize'] * (pageNumber
+ 1)
563 >= show_page
['pagination']['totalElements']):
564 # we've processed enough pages to get all playlist entries
566 pageNumber
= pageNumber
+ 1
568 return self
.playlist_result(entries
, playlist_id
, playlist_title
=display_id
)
570 def _real_extract(self
, url
):
571 video_id
, display_id
, playlist_type
, client
, season_number
= self
._match
_valid
_url
(url
).group(
572 'id', 'display_id', 'playlist', 'client', 'season')
573 display_id
, client
= display_id
or video_id
, client
or 'ard'
576 # TODO: Extract only specified season
577 return self
._ARD
_extract
_playlist
(url
, video_id
, display_id
, client
, playlist_type
)
579 player_page
= self
._download
_json
(
580 'https://api.ardmediathek.de/public-gateway',
581 display_id
, data
=json
.dumps({
583 playerPage(client:"%s", clipId: "%s") {
586 maturityContentRating
613 }''' % (client
, video_id
),
614 }).encode(), headers
={
615 'Content-Type': 'application/json'
616 })['data']['playerPage']
617 title
= player_page
['title']
618 content_id
= str_or_none(try_get(
619 player_page
, lambda x
: x
['tracking']['atiCustomVars']['contentId']))
620 media_collection
= player_page
.get('mediaCollection') or {}
621 if not media_collection
and content_id
:
622 media_collection
= self
._download
_json
(
623 'https://www.ardmediathek.de/play/media/' + content_id
,
624 content_id
, fatal
=False) or {}
625 info
= self
._parse
_media
_info
(
626 media_collection
, content_id
or video_id
,
627 player_page
.get('blockedByFsk'))
629 description
= player_page
.get('synopsis')
630 maturity_content_rating
= player_page
.get('maturityContentRating')
631 if maturity_content_rating
:
632 age_limit
= int_or_none(maturity_content_rating
.lstrip('FSK'))
633 if not age_limit
and description
:
634 age_limit
= int_or_none(self
._search
_regex
(
635 r
'\(FSK\s*(\d+)\)\s*$', description
, 'age limit', default
=None))
637 'age_limit': age_limit
,
638 'display_id': display_id
,
640 'description': description
,
641 'timestamp': unified_timestamp(player_page
.get('broadcastedOn')),
642 'series': try_get(player_page
, lambda x
: x
['show']['title']),
644 info
.update(self
._ARD
_extract
_episode
_info
(info
['title']))