youtube_dl/extractor/ard.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     determine_ext,
   9     ExtractorError,
  10     qualities,
  11     compat_urllib_parse_urlparse,
  12     compat_urllib_parse,
  13     int_or_none,
  14     parse_duration,
  15     unified_strdate,
  16     xpath_text,
  17 )
  18
  19
  20 class ARDMediathekIE(InfoExtractor):
  21     IE_NAME = 'ARD:mediathek'
  22     _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
  23
  24     _TESTS = [{
  25         'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
  26         'file': '22429276.mp4',
  27         'md5': '469751912f1de0816a9fc9df8336476c',
  28         'info_dict': {
  29             'title': 'Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?',
  30             'description': 'Das Erste Mediathek [ARD]: Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?, Anne Will, Über die Spionage-Affäre diskutieren Clemens Binninger, Katrin Göring-Eckardt, Georg Mascolo, Andrew B. Denison und Constanze Kurz.. Das Video zur Sendung Anne Will am Mittwoch, 16.07.2014',
  31         },
  32         'skip': 'Blocked outside of Germany',
  33     }, {
  34         'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
  35         'info_dict': {
  36             'id': '22490580',
  37             'ext': 'mp4',
  38             'title': 'Das Wunder von Wolbeck (Video tgl. ab 20 Uhr)',
  39             'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
  40         },
  41         'skip': 'Blocked outside of Germany',
  42     }]
  43
  44     def _real_extract(self, url):
  45         # determine video id from url
  46         m = re.match(self._VALID_URL, url)
  47
  48         numid = re.search(r'documentId=([0-9]+)', url)
  49         if numid:
  50             video_id = numid.group(1)
  51         else:
  52             video_id = m.group('video_id')
  53
  54         urlp = compat_urllib_parse_urlparse(url)
  55         url = urlp._replace(path=compat_urllib_parse.quote(urlp.path.encode('utf-8'))).geturl()
  56
  57         webpage = self._download_webpage(url, video_id)
  58
  59         if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
  60             raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
  61
  62         title = self._html_search_regex(
  63             [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
  64              r'<meta name="dcterms.title" content="(.*?)"/>',
  65              r'<h4 class="headline">(.*?)</h4>'],
  66             webpage, 'title')
  67         description = self._html_search_meta(
  68             'dcterms.abstract', webpage, 'description', default=None)
  69         if description is None:
  70             description = self._html_search_meta(
  71                 'description', webpage, 'meta description')
  72
  73         # Thumbnail is sometimes not present.
  74         # It is in the mobile version, but that seems to use a different URL
  75         # structure altogether.
  76         thumbnail = self._og_search_thumbnail(webpage, default=None)
  77
  78         media_streams = re.findall(r'''(?x)
  79             mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
  80             "([^"]+)"''', webpage)
  81
  82         if media_streams:
  83             QUALITIES = qualities(['lo', 'hi', 'hq'])
  84             formats = []
  85             for furl in set(media_streams):
  86                 if furl.endswith('.f4m'):
  87                     fid = 'f4m'
  88                 else:
  89                     fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
  90                     fid = fid_m.group(1) if fid_m else None
  91                 formats.append({
  92                     'quality': QUALITIES(fid),
  93                     'format_id': fid,
  94                     'url': furl,
  95                 })
  96         else:  # request JSON file
  97             media_info = self._download_json(
  98                 'http://www.ardmediathek.de/play/media/%s' % video_id, video_id)
  99             # The second element of the _mediaArray contains the standard http urls
 100             streams = media_info['_mediaArray'][1]['_mediaStreamArray']
 101             if not streams:
 102                 if '"fsk"' in webpage:
 103                     raise ExtractorError('This video is only available after 20:00')
 104
 105             formats = []
 106             for s in streams:
 107                 if type(s['_stream']) == list:
 108                     for index, url in enumerate(s['_stream'][::-1]):
 109                         quality = s['_quality'] + index
 110                         formats.append({
 111                             'quality': quality,
 112                             'url': url,
 113                             'format_id': '%s-%s' % (determine_ext(url), quality)
 114                         })
 115                     continue
 116
 117                 format = {
 118                     'quality': s['_quality'],
 119                     'url': s['_stream'],
 120                 }
 121
 122                 format['format_id'] = '%s-%s' % (
 123                     determine_ext(format['url']), format['quality'])
 124
 125                 formats.append(format)
 126
 127         self._sort_formats(formats)
 128
 129         return {
 130             'id': video_id,
 131             'title': title,
 132             'description': description,
 133             'formats': formats,
 134             'thumbnail': thumbnail,
 135         }
 136
 137
 138 class ARDIE(InfoExtractor):
 139     _VALID_URL = '(?P<mainurl>https?://(www\.)?daserste\.de/[^?#]+/videos/(?P<display_id>[^/?#]+)-(?P<id>[0-9]+))\.html'
 140     _TEST = {
 141         'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
 142         'md5': 'd216c3a86493f9322545e045ddc3eb35',
 143         'info_dict': {
 144             'display_id': 'die-story-im-ersten-mission-unter-falscher-flagge',
 145             'id': '100',
 146             'ext': 'mp4',
 147             'duration': 2600,
 148             'title': 'Die Story im Ersten: Mission unter falscher Flagge',
 149             'upload_date': '20140804',
 150             'thumbnail': 're:^https?://.*\.jpg$',
 151         }
 152     }
 153
 154     def _real_extract(self, url):
 155         mobj = re.match(self._VALID_URL, url)
 156         display_id = mobj.group('display_id')
 157
 158         player_url = mobj.group('mainurl') + '~playerXml.xml'
 159         doc = self._download_xml(player_url, display_id)
 160         video_node = doc.find('./video')
 161         upload_date = unified_strdate(xpath_text(
 162             video_node, './broadcastDate'))
 163         thumbnail = xpath_text(video_node, './/teaserImage//variant/url')
 164
 165         formats = []
 166         for a in video_node.findall('.//asset'):
 167             f = {
 168                 'format_id': a.attrib['type'],
 169                 'width': int_or_none(a.find('./frameWidth').text),
 170                 'height': int_or_none(a.find('./frameHeight').text),
 171                 'vbr': int_or_none(a.find('./bitrateVideo').text),
 172                 'abr': int_or_none(a.find('./bitrateAudio').text),
 173                 'vcodec': a.find('./codecVideo').text,
 174                 'tbr': int_or_none(a.find('./totalBitrate').text),
 175             }
 176             if a.find('./serverPrefix').text:
 177                 f['url'] = a.find('./serverPrefix').text
 178                 f['playpath'] = a.find('./fileName').text
 179             else:
 180                 f['url'] = a.find('./fileName').text
 181             formats.append(f)
 182         self._sort_formats(formats)
 183
 184         return {
 185             'id': mobj.group('id'),
 186             'formats': formats,
 187             'display_id': display_id,
 188             'title': video_node.find('./title').text,
 189             'duration': parse_duration(video_node.find('./duration').text),
 190             'upload_date': upload_date,
 191             'thumbnail': thumbnail,
 192         }
 193