youtube_dl/extractor/ard.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     determine_ext,
   9     ExtractorError,
  10     qualities,
  11     compat_urllib_parse_urlparse,
  12     compat_urllib_parse,
  13     int_or_none,
  14     parse_duration,
  15     unified_strdate,
  16 )
  17
  18
  19 class ARDMediathekIE(InfoExtractor):
  20     IE_NAME = 'ARD:mediathek'
  21     _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
  22
  23     _TESTS = [{
  24         'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
  25         'file': '22429276.mp4',
  26         'md5': '469751912f1de0816a9fc9df8336476c',
  27         'info_dict': {
  28             'title': 'Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?',
  29             'description': 'Das Erste Mediathek [ARD]: Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?, Anne Will, Über die Spionage-Affäre diskutieren Clemens Binninger, Katrin Göring-Eckardt, Georg Mascolo, Andrew B. Denison und Constanze Kurz.. Das Video zur Sendung Anne Will am Mittwoch, 16.07.2014',
  30         },
  31         'skip': 'Blocked outside of Germany',
  32     }, {
  33         'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
  34         'info_dict': {
  35             'id': '22490580',
  36             'ext': 'mp4',
  37             'title': 'Das Wunder von Wolbeck (Video tgl. ab 20 Uhr)',
  38             'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
  39         },
  40         'skip': 'Blocked outside of Germany',
  41     }]
  42
  43     def _real_extract(self, url):
  44         # determine video id from url
  45         m = re.match(self._VALID_URL, url)
  46
  47         numid = re.search(r'documentId=([0-9]+)', url)
  48         if numid:
  49             video_id = numid.group(1)
  50         else:
  51             video_id = m.group('video_id')
  52
  53         urlp = compat_urllib_parse_urlparse(url)
  54         url = urlp._replace(path=compat_urllib_parse.quote(urlp.path.encode('utf-8'))).geturl()
  55
  56         webpage = self._download_webpage(url, video_id)
  57
  58         if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
  59             raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
  60
  61         title = self._html_search_regex(
  62             [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
  63              r'<meta name="dcterms.title" content="(.*?)"/>',
  64              r'<h4 class="headline">(.*?)</h4>'],
  65             webpage, 'title')
  66         description = self._html_search_meta(
  67             'dcterms.abstract', webpage, 'description', default=None)
  68         if description is None:
  69             description = self._html_search_meta(
  70                 'description', webpage, 'meta description')
  71
  72         # Thumbnail is sometimes not present.
  73         # It is in the mobile version, but that seems to use a different URL
  74         # structure altogether.
  75         thumbnail = self._og_search_thumbnail(webpage, default=None)
  76
  77         media_streams = re.findall(r'''(?x)
  78             mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
  79             "([^"]+)"''', webpage)
  80
  81         if media_streams:
  82             QUALITIES = qualities(['lo', 'hi', 'hq'])
  83             formats = []
  84             for furl in set(media_streams):
  85                 if furl.endswith('.f4m'):
  86                     fid = 'f4m'
  87                 else:
  88                     fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
  89                     fid = fid_m.group(1) if fid_m else None
  90                 formats.append({
  91                     'quality': QUALITIES(fid),
  92                     'format_id': fid,
  93                     'url': furl,
  94                 })
  95         else:  # request JSON file
  96             media_info = self._download_json(
  97                 'http://www.ardmediathek.de/play/media/%s' % video_id, video_id)
  98             # The second element of the _mediaArray contains the standard http urls
  99             streams = media_info['_mediaArray'][1]['_mediaStreamArray']
 100             if not streams:
 101                 if '"fsk"' in webpage:
 102                     raise ExtractorError('This video is only available after 20:00')
 103
 104             formats = []
 105             for s in streams:
 106                 if type(s['_stream']) == list:
 107                     for index, url in enumerate(s['_stream'][::-1]):
 108                         quality = s['_quality'] + index
 109                         formats.append({
 110                             'quality': quality,
 111                             'url': url,
 112                             'format_id': '%s-%s' % (determine_ext(url), quality)
 113                         })
 114                     continue
 115
 116                 format = {
 117                     'quality': s['_quality'],
 118                     'url': s['_stream'],
 119                 }
 120
 121                 format['format_id'] = '%s-%s' % (
 122                     determine_ext(format['url']), format['quality'])
 123
 124                 formats.append(format)
 125
 126         self._sort_formats(formats)
 127
 128         return {
 129             'id': video_id,
 130             'title': title,
 131             'description': description,
 132             'formats': formats,
 133             'thumbnail': thumbnail,
 134         }
 135
 136
 137 class ARDIE(InfoExtractor):
 138     _VALID_URL = '(?P<mainurl>https?://(www\.)?daserste\.de/[^?#]+/videos/(?P<display_id>[^/?#]+)-(?P<id>[0-9]+))\.html'
 139     _TEST = {
 140         'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
 141         'md5': 'd216c3a86493f9322545e045ddc3eb35',
 142         'info_dict': {
 143             'display_id': 'die-story-im-ersten-mission-unter-falscher-flagge',
 144             'id': '100',
 145             'ext': 'mp4',
 146             'duration': 2600,
 147             'title': 'Die Story im Ersten: Mission unter falscher Flagge',
 148             'upload_date': '20140804',
 149             'thumbnail': 're:^https?://.*\.jpg$',
 150         }
 151     }
 152
 153     def _real_extract(self, url):
 154         mobj = re.match(self._VALID_URL, url)
 155         display_id = mobj.group('display_id')
 156
 157         player_url = mobj.group('mainurl') + '~playerXml.xml'
 158         doc = self._download_xml(player_url, display_id)
 159         video_node = doc.find('./video')
 160         upload_date = unified_strdate(video_node.find('./broadcastDate').text)
 161         thumbnail = video_node.find('.//teaserImage//variant/url').text
 162
 163         formats = []
 164         for a in video_node.findall('.//asset'):
 165             f = {
 166                 'format_id': a.attrib['type'],
 167                 'width': int_or_none(a.find('./frameWidth').text),
 168                 'height': int_or_none(a.find('./frameHeight').text),
 169                 'vbr': int_or_none(a.find('./bitrateVideo').text),
 170                 'abr': int_or_none(a.find('./bitrateAudio').text),
 171                 'vcodec': a.find('./codecVideo').text,
 172                 'tbr': int_or_none(a.find('./totalBitrate').text),
 173             }
 174             if a.find('./serverPrefix').text:
 175                 f['url'] = a.find('./serverPrefix').text
 176                 f['playpath'] = a.find('./fileName').text
 177             else:
 178                 f['url'] = a.find('./fileName').text
 179             formats.append(f)
 180         self._sort_formats(formats)
 181
 182         return {
 183             'id': mobj.group('id'),
 184             'formats': formats,
 185             'display_id': display_id,
 186             'title': video_node.find('./title').text,
 187             'duration': parse_duration(video_node.find('./duration').text),
 188             'upload_date': upload_date,
 189             'thumbnail': thumbnail,
 190         }
 191