2 from __future__
import unicode_literals
7 from .common
import InfoExtractor
8 from .generic
import GenericIE
23 from ..compat
import compat_etree_fromstring
26 class ARDMediathekBaseIE(InfoExtractor
):
27 _GEO_COUNTRIES
= ['DE']
29 def _extract_media_info(self
, media_info_url
, webpage
, video_id
):
30 media_info
= self
._download
_json
(
31 media_info_url
, video_id
, 'Downloading media JSON')
32 return self
._parse
_media
_info
(media_info
, video_id
, '"fsk"' in webpage
)
34 def _parse_media_info(self
, media_info
, video_id
, fsk
):
35 formats
= self
._extract
_formats
(media_info
, video_id
)
40 'This video is only available after 20:00', expected
=True)
41 elif media_info
.get('_geoblocked'):
42 self
.raise_geo_restricted(
43 'This video is not available due to geoblocking',
44 countries
=self
._GEO
_COUNTRIES
)
46 self
._sort
_formats
(formats
)
49 subtitle_url
= media_info
.get('_subtitleUrl')
58 'duration': int_or_none(media_info
.get('_duration')),
59 'thumbnail': media_info
.get('_previewImage'),
60 'is_live': media_info
.get('_isLive') is True,
62 'subtitles': subtitles
,
65 def _ARD_extract_episode_info(self
, title
):
66 """Try to extract season/episode data from the title."""
72 # Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
73 # from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
74 r
'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
75 # E.g.: title="Fritjof aus Norwegen (2) (AD)"
76 # from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
77 r
'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
78 r
'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
79 # E.g.: title="Folge 25/42: Symmetrie"
80 # from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
81 # E.g.: title="Folge 1063 - Vertrauen"
82 # from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
83 r
'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
85 m
= re
.match(pattern
, title
)
87 groupdict
= m
.groupdict()
88 res
['season_number'] = int_or_none(groupdict
.get('season_number'))
89 res
['episode_number'] = int_or_none(groupdict
.get('episode_number'))
90 res
['episode'] = str_or_none(groupdict
.get('episode'))
91 # Build the episode title by removing numeric episode information:
92 if groupdict
.get('ep_info') and not res
['episode']:
93 res
['episode'] = str_or_none(
94 title
.replace(groupdict
.get('ep_info'), ''))
96 res
['episode'] = res
['episode'].strip()
99 # As a fallback use the whole title as the episode name:
100 if not res
.get('episode'):
101 res
['episode'] = title
.strip()
104 def _extract_formats(self
, media_info
, video_id
):
105 type_
= media_info
.get('_type')
106 media_array
= media_info
.get('_mediaArray', [])
108 for num
, media
in enumerate(media_array
):
109 for stream
in media
.get('_mediaStreamArray', []):
110 stream_urls
= stream
.get('_stream')
113 if not isinstance(stream_urls
, list):
114 stream_urls
= [stream_urls
]
115 quality
= stream
.get('_quality')
116 server
= stream
.get('_server')
117 for stream_url
in stream_urls
:
118 if not url_or_none(stream_url
):
120 ext
= determine_ext(stream_url
)
121 if quality
!= 'auto' and ext
in ('f4m', 'm3u8'):
124 formats
.extend(self
._extract
_f
4m
_formats
(
125 update_url_query(stream_url
, {
127 'plugin': 'aasp-3.1.1.69.124'
128 }), video_id
, f4m_id
='hds', fatal
=False))
130 formats
.extend(self
._extract
_m
3u8_formats
(
131 stream_url
, video_id
, 'mp4', 'm3u8_native',
132 m3u8_id
='hls', fatal
=False))
134 if server
and server
.startswith('rtmp'):
137 'play_path': stream_url
,
138 'format_id': 'a%s-rtmp-%s' % (num
, quality
),
143 'format_id': 'a%s-%s-%s' % (num
, ext
, quality
)
146 r
'_(?P<width>\d+)x(?P<height>\d+)\.mp4$',
150 'width': int(m
.group('width')),
151 'height': int(m
.group('height')),
159 class ARDMediathekIE(ARDMediathekBaseIE
):
160 IE_NAME
= 'ARD:mediathek'
161 _VALID_URL
= r
'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
164 # available till 26.07.2022
165 'url': 'http://www.ardmediathek.de/tv/S%C3%9CDLICHT/Was-ist-die-Kunst-der-Zukunft-liebe-Ann/BR-Fernsehen/Video?bcastId=34633636&documentId=44726822',
169 'title': 'Was ist die Kunst der Zukunft, liebe Anna McCarthy?',
170 'description': 'md5:4ada28b3e3b5df01647310e41f3a62f5',
175 'skip_download': True,
178 'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872',
179 'only_matching': True,
182 'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
183 'only_matching': True,
185 'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
186 'only_matching': True,
189 'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
190 'only_matching': True,
192 'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698',
193 'only_matching': True,
197 def suitable(cls
, url
):
198 return False if ARDBetaMediathekIE
.suitable(url
) else super(ARDMediathekIE
, cls
).suitable(url
)
200 def _real_extract(self
, url
):
201 # determine video id from url
202 m
= re
.match(self
._VALID
_URL
, url
)
206 numid
= re
.search(r
'documentId=([0-9]+)', url
)
208 document_id
= video_id
= numid
.group(1)
210 video_id
= m
.group('video_id')
212 webpage
= self
._download
_webpage
(url
, video_id
)
215 ('>Leider liegt eine Störung vor.', 'Video %s is unavailable'),
216 ('>Der gewünschte Beitrag ist nicht mehr verfügbar.<',
217 'Video %s is no longer available'),
220 for pattern
, message
in ERRORS
:
221 if pattern
in webpage
:
222 raise ExtractorError(message
% video_id
, expected
=True)
224 if re
.search(r
'[\?&]rss($|[=&])', url
):
225 doc
= compat_etree_fromstring(webpage
.encode('utf-8'))
227 return GenericIE()._extract
_rss
(url
, video_id
, doc
)
229 title
= self
._og
_search
_title
(webpage
, default
=None) or self
._html
_search
_regex
(
230 [r
'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
231 r
'<meta name="dcterms\.title" content="(.*?)"/>',
232 r
'<h4 class="headline">(.*?)</h4>',
233 r
'<title[^>]*>(.*?)</title>'],
235 description
= self
._og
_search
_description
(webpage
, default
=None) or self
._html
_search
_meta
(
236 'dcterms.abstract', webpage
, 'description', default
=None)
237 if description
is None:
238 description
= self
._html
_search
_meta
(
239 'description', webpage
, 'meta description', default
=None)
240 if description
is None:
241 description
= self
._html
_search
_regex
(
242 r
'<p\s+class="teasertext">(.+?)</p>',
243 webpage
, 'teaser text', default
=None)
245 # Thumbnail is sometimes not present.
246 # It is in the mobile version, but that seems to use a different URL
247 # structure altogether.
248 thumbnail
= self
._og
_search
_thumbnail
(webpage
, default
=None)
250 media_streams
= re
.findall(r
'''(?x)
251 mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
252 "([^"]+)"''', webpage
)
255 QUALITIES
= qualities(['lo', 'hi', 'hq'])
257 for furl
in set(media_streams
):
258 if furl
.endswith('.f4m'):
261 fid_m
= re
.match(r
'.*\.([^.]+)\.[^.]+$', furl
)
262 fid
= fid_m
.group(1) if fid_m
else None
264 'quality': QUALITIES(fid
),
268 self
._sort
_formats
(formats
)
272 else: # request JSON file
274 video_id
= self
._search
_regex
(
275 r
'/play/(?:config|media)/(\d+)', webpage
, 'media id')
276 info
= self
._extract
_media
_info
(
277 'http://www.ardmediathek.de/play/media/%s' % video_id
,
282 'title': self
._live
_title
(title
) if info
.get('is_live') else title
,
283 'description': description
,
284 'thumbnail': thumbnail
,
286 info
.update(self
._ARD
_extract
_episode
_info
(info
['title']))
291 class ARDIE(InfoExtractor
):
292 _VALID_URL
= r
'(?P<mainurl>https?://(?:www\.)?daserste\.de/[^?#]+/videos(?:extern)?/(?P<display_id>[^/?#]+)-(?:video-?)?(?P<id>[0-9]+))\.html'
294 # available till 7.01.2022
295 'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-die-woche-video100.html',
296 'md5': '867d8aa39eeaf6d76407c5ad1bb0d4c1',
298 'display_id': 'maischberger-die-woche',
302 'title': 'maischberger. die woche vom 7. Januar 2021',
303 'upload_date': '20210107',
304 'thumbnail': r
're:^https?://.*\.jpg$',
307 'url': 'https://www.daserste.de/information/reportage-dokumentation/erlebnis-erde/videosextern/woelfe-und-herdenschutzhunde-ungleiche-brueder-102.html',
308 'only_matching': True,
310 'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
311 'only_matching': True,
314 def _real_extract(self
, url
):
315 mobj
= re
.match(self
._VALID
_URL
, url
)
316 display_id
= mobj
.group('display_id')
318 player_url
= mobj
.group('mainurl') + '~playerXml.xml'
319 doc
= self
._download
_xml
(player_url
, display_id
)
320 video_node
= doc
.find('./video')
321 upload_date
= unified_strdate(xpath_text(
322 video_node
, './broadcastDate'))
323 thumbnail
= xpath_text(video_node
, './/teaserImage//variant/url')
326 for a
in video_node
.findall('.//asset'):
328 'format_id': a
.attrib
['type'],
329 'width': int_or_none(a
.find('./frameWidth').text
),
330 'height': int_or_none(a
.find('./frameHeight').text
),
331 'vbr': int_or_none(a
.find('./bitrateVideo').text
),
332 'abr': int_or_none(a
.find('./bitrateAudio').text
),
333 'vcodec': a
.find('./codecVideo').text
,
334 'tbr': int_or_none(a
.find('./totalBitrate').text
),
336 if a
.find('./serverPrefix').text
:
337 f
['url'] = a
.find('./serverPrefix').text
338 f
['playpath'] = a
.find('./fileName').text
340 f
['url'] = a
.find('./fileName').text
342 self
._sort
_formats
(formats
)
345 'id': mobj
.group('id'),
347 'display_id': display_id
,
348 'title': video_node
.find('./title').text
,
349 'duration': parse_duration(video_node
.find('./duration').text
),
350 'upload_date': upload_date
,
351 'thumbnail': thumbnail
,
355 class ARDBetaMediathekIE(ARDMediathekBaseIE
):
356 _VALID_URL
= r
'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?P<client>[^/]+)/(?P<mode>player|live|video|sendung|sammlung)/(?P<display_id>(?:[^/]+/)*)(?P<video_id>[a-zA-Z0-9]+)'
358 'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
359 'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
361 'display_id': 'die-robuste-roswita',
363 'title': 'Die robuste Roswita',
364 'description': r
're:^Der Mord.*totgeglaubte Ehefrau Roswita',
366 'thumbnail': 'https://img.ardmediathek.de/standard/00/78/56/67/84/575672121/16x9/960?mandant=ard',
367 'timestamp': 1596658200,
368 'upload_date': '20200805',
372 'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
373 'only_matching': True,
375 'url': 'https://ardmediathek.de/ard/video/saartalk/saartalk-gesellschaftsgift-haltung-gegen-hass/sr-fernsehen/Y3JpZDovL3NyLW9ubGluZS5kZS9TVF84MTY4MA/',
376 'only_matching': True,
378 'url': 'https://www.ardmediathek.de/ard/video/trailer/private-eyes-s01-e01/one/Y3JpZDovL3dkci5kZS9CZWl0cmFnLTE1MTgwYzczLWNiMTEtNGNkMS1iMjUyLTg5MGYzOWQxZmQ1YQ/',
379 'only_matching': True,
381 'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3N3ci5kZS9hZXgvbzEwNzE5MTU/',
382 'only_matching': True,
384 'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
385 'only_matching': True,
387 # playlist of type 'sendung'
388 'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/',
389 'only_matching': True,
391 # playlist of type 'sammlung'
392 'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
393 'only_matching': True,
396 def _ARD_load_playlist_snipped(self
, playlist_id
, display_id
, client
, mode
, pageNumber
):
397 """ Query the ARD server for playlist information
398 and returns the data in "raw" format """
399 if mode
== 'sendung':
400 graphQL
= json
.dumps({
413 links { target { id href title } }
416 }}''' % (client
, playlist_id
, pageNumber
),
418 else: # mode == 'sammlung'
419 graphQL
= json
.dumps({
433 links { target { id href title } }
437 }}''' % (client
, playlist_id
, pageNumber
),
439 # Ressources for ARD graphQL debugging:
440 # https://api-test.ardmediathek.de/public-gateway
441 show_page
= self
._download
_json
(
442 'https://api.ardmediathek.de/public-gateway',
443 '[Playlist] %s' % display_id
,
445 headers
={'Content-Type': 'application/json'}
)['data']
446 # align the structure of the returned data:
447 if mode
== 'sendung':
448 show_page
= show_page
['showPage']
449 else: # mode == 'sammlung'
450 show_page
= show_page
['morePage']['widget']
453 def _ARD_extract_playlist(self
, url
, playlist_id
, display_id
, client
, mode
):
454 """ Collects all playlist entries and returns them as info dict.
455 Supports playlists of mode 'sendung' and 'sammlung', and also nested
459 while True: # iterate by pageNumber
460 show_page
= self
._ARD
_load
_playlist
_snipped
(
461 playlist_id
, display_id
, client
, mode
, pageNumber
)
462 for teaser
in show_page
['teasers']: # process playlist items
463 if '/compilation/' in teaser
['links']['target']['href']:
464 # alternativ cond.: teaser['type'] == "compilation"
465 # => This is an nested compilation, e.g. like:
466 # https://www.ardmediathek.de/ard/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2/
467 link_mode
= 'sammlung'
471 item_url
= 'https://www.ardmediathek.de/%s/%s/%s/%s/%s' % (
472 client
, link_mode
, display_id
,
473 # perform HTLM quoting of episode title similar to ARD:
474 re
.sub('^-|-$', '', # remove '-' from begin/end
475 re
.sub('[^a-zA-Z0-9]+', '-', # replace special chars by -
476 teaser
['links']['target']['title'].lower()
477 .replace('ä', 'ae').replace('ö', 'oe')
478 .replace('ü', 'ue').replace('ß', 'ss'))),
479 teaser
['links']['target']['id'])
480 entries
.append(self
.url_result(
482 ie
=ARDBetaMediathekIE
.ie_key()))
484 if (show_page
['pagination']['pageSize'] * (pageNumber
+ 1)
485 >= show_page
['pagination']['totalElements']):
486 # we've processed enough pages to get all playlist entries
488 pageNumber
= pageNumber
+ 1
490 return self
.playlist_result(entries
, playlist_title
=display_id
)
492 def _real_extract(self
, url
):
493 mobj
= re
.match(self
._VALID
_URL
, url
)
494 video_id
= mobj
.group('video_id')
495 display_id
= mobj
.group('display_id')
497 display_id
= display_id
.rstrip('/')
499 display_id
= video_id
501 if mobj
.group('mode') in ('sendung', 'sammlung'):
502 # this is a playlist-URL
503 return self
._ARD
_extract
_playlist
(
504 url
, video_id
, display_id
,
505 mobj
.group('client'),
508 player_page
= self
._download
_json
(
509 'https://api.ardmediathek.de/public-gateway',
510 display_id
, data
=json
.dumps({
512 playerPage(client:"%s", clipId: "%s") {
515 maturityContentRating
542 }''' % (mobj
.group('client'), video_id
),
543 }).encode(), headers
={
544 'Content-Type': 'application/json'
545 })['data']['playerPage']
546 title
= player_page
['title']
547 content_id
= str_or_none(try_get(
548 player_page
, lambda x
: x
['tracking']['atiCustomVars']['contentId']))
549 media_collection
= player_page
.get('mediaCollection') or {}
550 if not media_collection
and content_id
:
551 media_collection
= self
._download
_json
(
552 'https://www.ardmediathek.de/play/media/' + content_id
,
553 content_id
, fatal
=False) or {}
554 info
= self
._parse
_media
_info
(
555 media_collection
, content_id
or video_id
,
556 player_page
.get('blockedByFsk'))
558 description
= player_page
.get('synopsis')
559 maturity_content_rating
= player_page
.get('maturityContentRating')
560 if maturity_content_rating
:
561 age_limit
= int_or_none(maturity_content_rating
.lstrip('FSK'))
562 if not age_limit
and description
:
563 age_limit
= int_or_none(self
._search
_regex
(
564 r
'\(FSK\s*(\d+)\)\s*$', description
, 'age limit', default
=None))
566 'age_limit': age_limit
,
567 'display_id': display_id
,
569 'description': description
,
570 'timestamp': unified_timestamp(player_page
.get('broadcastedOn')),
571 'series': try_get(player_page
, lambda x
: x
['show']['title']),
573 info
.update(self
._ARD
_extract
_episode
_info
(info
['title']))