]> jfr.im git - yt-dlp.git/blob - youtube_dlc/extractor/ard.py
Merge branch 'ard.py_add_playlist_support' of https://github.com/martin54/youtube...
[yt-dlp.git] / youtube_dlc / extractor / ard.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import json
5 import re
6
7 from .common import InfoExtractor
8 from .generic import GenericIE
9 from ..utils import (
10 determine_ext,
11 ExtractorError,
12 int_or_none,
13 parse_duration,
14 qualities,
15 str_or_none,
16 try_get,
17 unified_strdate,
18 unified_timestamp,
19 update_url_query,
20 url_or_none,
21 xpath_text,
22 )
23 from ..compat import compat_etree_fromstring
24
25
26 class ARDMediathekBaseIE(InfoExtractor):
27 _GEO_COUNTRIES = ['DE']
28
29 def _extract_media_info(self, media_info_url, webpage, video_id):
30 media_info = self._download_json(
31 media_info_url, video_id, 'Downloading media JSON')
32 return self._parse_media_info(media_info, video_id, '"fsk"' in webpage)
33
34 def _parse_media_info(self, media_info, video_id, fsk):
35 formats = self._extract_formats(media_info, video_id)
36
37 if not formats:
38 if fsk:
39 raise ExtractorError(
40 'This video is only available after 20:00', expected=True)
41 elif media_info.get('_geoblocked'):
42 self.raise_geo_restricted(
43 'This video is not available due to geoblocking',
44 countries=self._GEO_COUNTRIES)
45
46 self._sort_formats(formats)
47
48 subtitles = {}
49 subtitle_url = media_info.get('_subtitleUrl')
50 if subtitle_url:
51 subtitles['de'] = [{
52 'ext': 'ttml',
53 'url': subtitle_url,
54 }]
55
56 return {
57 'id': video_id,
58 'duration': int_or_none(media_info.get('_duration')),
59 'thumbnail': media_info.get('_previewImage'),
60 'is_live': media_info.get('_isLive') is True,
61 'formats': formats,
62 'subtitles': subtitles,
63 }
64
65 def _ARD_extract_episode_info(self, title):
66 """Try to extract season/episode data from the title."""
67 res = {}
68 if not title:
69 return res
70
71 for pattern in [
72 # Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
73 # from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
74 r'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
75 # E.g.: title="Fritjof aus Norwegen (2) (AD)"
76 # from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
77 r'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
78 r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
79 # E.g.: title="Folge 25/42: Symmetrie"
80 # from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
81 # E.g.: title="Folge 1063 - Vertrauen"
82 # from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
83 r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
84 ]:
85 m = re.match(pattern, title)
86 if m:
87 groupdict = m.groupdict()
88 res['season_number'] = int_or_none(groupdict.get('season_number'))
89 res['episode_number'] = int_or_none(groupdict.get('episode_number'))
90 res['episode'] = str_or_none(groupdict.get('episode'))
91 # Build the episode title by removing numeric episode information:
92 if groupdict.get('ep_info') and not res['episode']:
93 res['episode'] = str_or_none(
94 title.replace(groupdict.get('ep_info'), ''))
95 if res['episode']:
96 res['episode'] = res['episode'].strip()
97 break
98
99 # As a fallback use the whole title as the episode name:
100 if not res.get('episode'):
101 res['episode'] = title.strip()
102 return res
103
104 def _extract_formats(self, media_info, video_id):
105 type_ = media_info.get('_type')
106 media_array = media_info.get('_mediaArray', [])
107 formats = []
108 for num, media in enumerate(media_array):
109 for stream in media.get('_mediaStreamArray', []):
110 stream_urls = stream.get('_stream')
111 if not stream_urls:
112 continue
113 if not isinstance(stream_urls, list):
114 stream_urls = [stream_urls]
115 quality = stream.get('_quality')
116 server = stream.get('_server')
117 for stream_url in stream_urls:
118 if not url_or_none(stream_url):
119 continue
120 ext = determine_ext(stream_url)
121 if quality != 'auto' and ext in ('f4m', 'm3u8'):
122 continue
123 if ext == 'f4m':
124 formats.extend(self._extract_f4m_formats(
125 update_url_query(stream_url, {
126 'hdcore': '3.1.1',
127 'plugin': 'aasp-3.1.1.69.124'
128 }), video_id, f4m_id='hds', fatal=False))
129 elif ext == 'm3u8':
130 formats.extend(self._extract_m3u8_formats(
131 stream_url, video_id, 'mp4', 'm3u8_native',
132 m3u8_id='hls', fatal=False))
133 else:
134 if server and server.startswith('rtmp'):
135 f = {
136 'url': server,
137 'play_path': stream_url,
138 'format_id': 'a%s-rtmp-%s' % (num, quality),
139 }
140 else:
141 f = {
142 'url': stream_url,
143 'format_id': 'a%s-%s-%s' % (num, ext, quality)
144 }
145 m = re.search(
146 r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$',
147 stream_url)
148 if m:
149 f.update({
150 'width': int(m.group('width')),
151 'height': int(m.group('height')),
152 })
153 if type_ == 'audio':
154 f['vcodec'] = 'none'
155 formats.append(f)
156 return formats
157
158
159 class ARDMediathekIE(ARDMediathekBaseIE):
160 IE_NAME = 'ARD:mediathek'
161 _VALID_URL = r'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
162
163 _TESTS = [{
164 # available till 26.07.2022
165 'url': 'http://www.ardmediathek.de/tv/S%C3%9CDLICHT/Was-ist-die-Kunst-der-Zukunft-liebe-Ann/BR-Fernsehen/Video?bcastId=34633636&documentId=44726822',
166 'info_dict': {
167 'id': '44726822',
168 'ext': 'mp4',
169 'title': 'Was ist die Kunst der Zukunft, liebe Anna McCarthy?',
170 'description': 'md5:4ada28b3e3b5df01647310e41f3a62f5',
171 'duration': 1740,
172 },
173 'params': {
174 # m3u8 download
175 'skip_download': True,
176 }
177 }, {
178 'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872',
179 'only_matching': True,
180 }, {
181 # audio
182 'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
183 'only_matching': True,
184 }, {
185 'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
186 'only_matching': True,
187 }, {
188 # audio
189 'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
190 'only_matching': True,
191 }, {
192 'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698',
193 'only_matching': True,
194 }]
195
196 @classmethod
197 def suitable(cls, url):
198 return False if ARDBetaMediathekIE.suitable(url) else super(ARDMediathekIE, cls).suitable(url)
199
200 def _real_extract(self, url):
201 # determine video id from url
202 m = re.match(self._VALID_URL, url)
203
204 document_id = None
205
206 numid = re.search(r'documentId=([0-9]+)', url)
207 if numid:
208 document_id = video_id = numid.group(1)
209 else:
210 video_id = m.group('video_id')
211
212 webpage = self._download_webpage(url, video_id)
213
214 ERRORS = (
215 ('>Leider liegt eine Störung vor.', 'Video %s is unavailable'),
216 ('>Der gewünschte Beitrag ist nicht mehr verfügbar.<',
217 'Video %s is no longer available'),
218 )
219
220 for pattern, message in ERRORS:
221 if pattern in webpage:
222 raise ExtractorError(message % video_id, expected=True)
223
224 if re.search(r'[\?&]rss($|[=&])', url):
225 doc = compat_etree_fromstring(webpage.encode('utf-8'))
226 if doc.tag == 'rss':
227 return GenericIE()._extract_rss(url, video_id, doc)
228
229 title = self._html_search_regex(
230 [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
231 r'<meta name="dcterms\.title" content="(.*?)"/>',
232 r'<h4 class="headline">(.*?)</h4>',
233 r'<title[^>]*>(.*?)</title>'],
234 webpage, 'title')
235 description = self._html_search_meta(
236 'dcterms.abstract', webpage, 'description', default=None)
237 if description is None:
238 description = self._html_search_meta(
239 'description', webpage, 'meta description', default=None)
240 if description is None:
241 description = self._html_search_regex(
242 r'<p\s+class="teasertext">(.+?)</p>',
243 webpage, 'teaser text', default=None)
244
245 # Thumbnail is sometimes not present.
246 # It is in the mobile version, but that seems to use a different URL
247 # structure altogether.
248 thumbnail = self._og_search_thumbnail(webpage, default=None)
249
250 media_streams = re.findall(r'''(?x)
251 mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
252 "([^"]+)"''', webpage)
253
254 if media_streams:
255 QUALITIES = qualities(['lo', 'hi', 'hq'])
256 formats = []
257 for furl in set(media_streams):
258 if furl.endswith('.f4m'):
259 fid = 'f4m'
260 else:
261 fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
262 fid = fid_m.group(1) if fid_m else None
263 formats.append({
264 'quality': QUALITIES(fid),
265 'format_id': fid,
266 'url': furl,
267 })
268 self._sort_formats(formats)
269 info = {
270 'formats': formats,
271 }
272 else: # request JSON file
273 if not document_id:
274 video_id = self._search_regex(
275 r'/play/(?:config|media)/(\d+)', webpage, 'media id')
276 info = self._extract_media_info(
277 'http://www.ardmediathek.de/play/media/%s' % video_id,
278 webpage, video_id)
279
280 info.update({
281 'id': video_id,
282 'title': self._live_title(title) if info.get('is_live') else title,
283 'description': description,
284 'thumbnail': thumbnail,
285 })
286 info.update(self._ARD_extract_episode_info(info['title']))
287
288 return info
289
290
291 class ARDIE(InfoExtractor):
292 _VALID_URL = r'(?P<mainurl>https?://(www\.)?daserste\.de/[^?#]+/videos(?:extern)?/(?P<display_id>[^/?#]+)-(?P<id>[0-9]+))\.html'
293 _TESTS = [{
294 # available till 14.02.2019
295 'url': 'http://www.daserste.de/information/talk/maischberger/videos/das-groko-drama-zerlegen-sich-die-volksparteien-video-102.html',
296 'md5': '8e4ec85f31be7c7fc08a26cdbc5a1f49',
297 'info_dict': {
298 'display_id': 'das-groko-drama-zerlegen-sich-die-volksparteien-video',
299 'id': '102',
300 'ext': 'mp4',
301 'duration': 4435.0,
302 'title': 'Das GroKo-Drama: Zerlegen sich die Volksparteien?',
303 'upload_date': '20180214',
304 'thumbnail': r're:^https?://.*\.jpg$',
305 },
306 }, {
307 'url': 'https://www.daserste.de/information/reportage-dokumentation/erlebnis-erde/videosextern/woelfe-und-herdenschutzhunde-ungleiche-brueder-102.html',
308 'only_matching': True,
309 }, {
310 'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
311 'only_matching': True,
312 }]
313
314 def _real_extract(self, url):
315 mobj = re.match(self._VALID_URL, url)
316 display_id = mobj.group('display_id')
317
318 player_url = mobj.group('mainurl') + '~playerXml.xml'
319 doc = self._download_xml(player_url, display_id)
320 video_node = doc.find('./video')
321 upload_date = unified_strdate(xpath_text(
322 video_node, './broadcastDate'))
323 thumbnail = xpath_text(video_node, './/teaserImage//variant/url')
324
325 formats = []
326 for a in video_node.findall('.//asset'):
327 f = {
328 'format_id': a.attrib['type'],
329 'width': int_or_none(a.find('./frameWidth').text),
330 'height': int_or_none(a.find('./frameHeight').text),
331 'vbr': int_or_none(a.find('./bitrateVideo').text),
332 'abr': int_or_none(a.find('./bitrateAudio').text),
333 'vcodec': a.find('./codecVideo').text,
334 'tbr': int_or_none(a.find('./totalBitrate').text),
335 }
336 if a.find('./serverPrefix').text:
337 f['url'] = a.find('./serverPrefix').text
338 f['playpath'] = a.find('./fileName').text
339 else:
340 f['url'] = a.find('./fileName').text
341 formats.append(f)
342 self._sort_formats(formats)
343
344 return {
345 'id': mobj.group('id'),
346 'formats': formats,
347 'display_id': display_id,
348 'title': video_node.find('./title').text,
349 'duration': parse_duration(video_node.find('./duration').text),
350 'upload_date': upload_date,
351 'thumbnail': thumbnail,
352 }
353
354
355 class ARDBetaMediathekIE(ARDMediathekBaseIE):
356 _VALID_URL = r'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?P<client>[^/]+)/(?P<mode>player|live|video|sendung|sammlung)/(?P<display_id>(?:[^/]+/)*)(?P<video_id>[a-zA-Z0-9]+)'
357 _TESTS = [{
358 'url': 'https://ardmediathek.de/ard/video/die-robuste-roswita/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
359 'md5': 'dfdc87d2e7e09d073d5a80770a9ce88f',
360 'info_dict': {
361 'display_id': 'die-robuste-roswita',
362 'id': '70153354',
363 'title': 'Die robuste Roswita',
364 'description': r're:^Der Mord.*trüber ist als die Ilm.',
365 'duration': 5316,
366 'thumbnail': 'https://img.ardmediathek.de/standard/00/70/15/33/90/-1852531467/16x9/960?mandant=ard',
367 'timestamp': 1577047500,
368 'upload_date': '20191222',
369 'ext': 'mp4',
370 },
371 }, {
372 'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
373 'only_matching': True,
374 }, {
375 'url': 'https://ardmediathek.de/ard/video/saartalk/saartalk-gesellschaftsgift-haltung-gegen-hass/sr-fernsehen/Y3JpZDovL3NyLW9ubGluZS5kZS9TVF84MTY4MA/',
376 'only_matching': True,
377 }, {
378 'url': 'https://www.ardmediathek.de/ard/video/trailer/private-eyes-s01-e01/one/Y3JpZDovL3dkci5kZS9CZWl0cmFnLTE1MTgwYzczLWNiMTEtNGNkMS1iMjUyLTg5MGYzOWQxZmQ1YQ/',
379 'only_matching': True,
380 }, {
381 'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3N3ci5kZS9hZXgvbzEwNzE5MTU/',
382 'only_matching': True,
383 }, {
384 'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
385 'only_matching': True,
386 }, {
387 # playlist of type 'sendung'
388 'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/',
389 'only_matching': True,
390 }, {
391 # playlist of type 'sammlung'
392 'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
393 'only_matching': True,
394 }]
395
396 def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, pageNumber):
397 """ Query the ARD server for playlist information
398 and returns the data in "raw" format """
399 if mode == 'sendung':
400 graphQL = json.dumps({
401 'query': '''{
402 showPage(
403 client: "%s"
404 showId: "%s"
405 pageNumber: %d
406 ) {
407 pagination {
408 pageSize
409 totalElements
410 }
411 teasers { # Array
412 mediumTitle
413 links { target { id href title } }
414 type
415 }
416 }}''' % (client, playlist_id, pageNumber),
417 }).encode()
418 else: # mode == 'sammlung'
419 graphQL = json.dumps({
420 'query': '''{
421 morePage(
422 client: "%s"
423 compilationId: "%s"
424 pageNumber: %d
425 ) {
426 widget {
427 pagination {
428 pageSize
429 totalElements
430 }
431 teasers { # Array
432 mediumTitle
433 links { target { id href title } }
434 type
435 }
436 }
437 }}''' % (client, playlist_id, pageNumber),
438 }).encode()
439 # Ressources for ARD graphQL debugging:
440 # https://api-test.ardmediathek.de/public-gateway
441 show_page = self._download_json(
442 'https://api.ardmediathek.de/public-gateway',
443 '[Playlist] %s' % display_id,
444 data=graphQL,
445 headers={'Content-Type': 'application/json'})['data']
446 # align the structure of the returned data:
447 if mode == 'sendung':
448 show_page = show_page['showPage']
449 else: # mode == 'sammlung'
450 show_page = show_page['morePage']['widget']
451 return show_page
452
453 def _ARD_extract_playlist(self, url, playlist_id, display_id, client, mode):
454 """ Collects all playlist entries and returns them as info dict.
455 Supports playlists of mode 'sendung' and 'sammlung', and also nested
456 playlists. """
457 entries = []
458 pageNumber = 0
459 while True: # iterate by pageNumber
460 show_page = self._ARD_load_playlist_snipped(
461 playlist_id, display_id, client, mode, pageNumber)
462 for teaser in show_page['teasers']: # process playlist items
463 if '/compilation/' in teaser['links']['target']['href']:
464 # alternativ cond.: teaser['type'] == "compilation"
465 # => This is an nested compilation, e.g. like:
466 # https://www.ardmediathek.de/ard/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2/
467 link_mode = 'sammlung'
468 else:
469 link_mode = 'video'
470
471 item_url = 'https://www.ardmediathek.de/%s/%s/%s/%s/%s' % (
472 client, link_mode, display_id,
473 # perform HTLM quoting of episode title similar to ARD:
474 re.sub('^-|-$', '', # remove '-' from begin/end
475 re.sub('[^a-zA-Z0-9]+', '-', # replace special chars by -
476 teaser['links']['target']['title'].lower()
477 .replace('ä', 'ae').replace('ö', 'oe')
478 .replace('ü', 'ue').replace('ß', 'ss'))),
479 teaser['links']['target']['id'])
480 entries.append(self.url_result(
481 item_url,
482 ie=ARDBetaMediathekIE.ie_key()))
483
484 if (show_page['pagination']['pageSize'] * (pageNumber + 1)
485 >= show_page['pagination']['totalElements']):
486 # we've processed enough pages to get all playlist entries
487 break
488 pageNumber = pageNumber + 1
489
490 return self.playlist_result(entries, playlist_title=display_id)
491
492 def _real_extract(self, url):
493 mobj = re.match(self._VALID_URL, url)
494 video_id = mobj.group('video_id')
495 display_id = mobj.group('display_id')
496 if display_id:
497 display_id = display_id.rstrip('/')
498 if not display_id:
499 display_id = video_id
500
501 if mobj.group('mode') in ('sendung', 'sammlung'):
502 # this is a playlist-URL
503 return self._ARD_extract_playlist(
504 url, video_id, display_id,
505 mobj.group('client'),
506 mobj.group('mode'))
507
508 player_page = self._download_json(
509 'https://api.ardmediathek.de/public-gateway',
510 display_id, data=json.dumps({
511 'query': '''{
512 playerPage(client:"%s", clipId: "%s") {
513 blockedByFsk
514 broadcastedOn
515 maturityContentRating
516 mediaCollection {
517 _duration
518 _geoblocked
519 _isLive
520 _mediaArray {
521 _mediaStreamArray {
522 _quality
523 _server
524 _stream
525 }
526 }
527 _previewImage
528 _subtitleUrl
529 _type
530 }
531 show {
532 title
533 }
534 synopsis
535 title
536 tracking {
537 atiCustomVars {
538 contentId
539 }
540 }
541 }
542 }''' % (mobj.group('client'), video_id),
543 }).encode(), headers={
544 'Content-Type': 'application/json'
545 })['data']['playerPage']
546 title = player_page['title']
547 content_id = str_or_none(try_get(
548 player_page, lambda x: x['tracking']['atiCustomVars']['contentId']))
549 media_collection = player_page.get('mediaCollection') or {}
550 if not media_collection and content_id:
551 media_collection = self._download_json(
552 'https://www.ardmediathek.de/play/media/' + content_id,
553 content_id, fatal=False) or {}
554 info = self._parse_media_info(
555 media_collection, content_id or video_id,
556 player_page.get('blockedByFsk'))
557 age_limit = None
558 description = player_page.get('synopsis')
559 maturity_content_rating = player_page.get('maturityContentRating')
560 if maturity_content_rating:
561 age_limit = int_or_none(maturity_content_rating.lstrip('FSK'))
562 if not age_limit and description:
563 age_limit = int_or_none(self._search_regex(
564 r'\(FSK\s*(\d+)\)\s*$', description, 'age limit', default=None))
565 info.update({
566 'age_limit': age_limit,
567 'display_id': display_id,
568 'title': title,
569 'description': description,
570 'timestamp': unified_timestamp(player_page.get('broadcastedOn')),
571 'series': try_get(player_page, lambda x: x['show']['title']),
572 })
573 info.update(self._ARD_extract_episode_info(info['title']))
574 return info