youtube_dl/extractor/ted.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import re
   5
   6 from .subtitles import SubtitlesInfoExtractor
   7
   8 from ..utils import (
   9     compat_str,
  10 )
  11
  12
  13 class TEDIE(SubtitlesInfoExtractor):
  14     _VALID_URL = r'''(?x)
  15         (?P<proto>https?://)
  16         (?P<type>www|embed)(?P<urlmain>\.ted\.com/
  17         (
  18             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
  19             |
  20             ((?P<type_talk>talks)) # We have a simple talk
  21             |
  22             (?P<type_watch>watch)/[^/]+/[^/]+
  23         )
  24         (/lang/(.*?))? # The url may contain the language
  25         /(?P<name>[\w-]+) # Here goes the name and then ".html"
  26         .*)$
  27         '''
  28     _TESTS = [{
  29         'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
  30         'md5': 'fc94ac279feebbce69f21c0c6ee82810',
  31         'info_dict': {
  32             'id': '102',
  33             'ext': 'mp4',
  34             'title': 'The illusion of consciousness',
  35             'description': ('Philosopher Dan Dennett makes a compelling '
  36                 'argument that not only don\'t we understand our own '
  37                 'consciousness, but that half the time our brains are '
  38                 'actively fooling us.'),
  39             'uploader': 'Dan Dennett',
  40             'width': 854,
  41         }
  42     }, {
  43         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
  44         'md5': '226f4fb9c62380d11b7995efa4c87994',
  45         'info_dict': {
  46             'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
  47             'ext': 'mp4',
  48             'title': 'Vishal Sikka: The beauty and power of algorithms',
  49             'thumbnail': 're:^https?://.+\.jpg',
  50             'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
  51         }
  52     }, {
  53         'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
  54         'info_dict': {
  55             'id': '1972',
  56             'ext': 'mp4',
  57             'title': 'Be passionate. Be courageous. Be your best.',
  58             'uploader': 'Gabby Giffords and Mark Kelly',
  59             'description': 'md5:5174aed4d0f16021b704120360f72b92',
  60         },
  61     }, {
  62         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
  63         'info_dict': {
  64             'id': '10',
  65             'title': 'Who are the hackers?',
  66         },
  67         'playlist_mincount': 6,
  68     }]
  69
  70     _NATIVE_FORMATS = {
  71         'low': {'preference': 1, 'width': 320, 'height': 180},
  72         'medium': {'preference': 2, 'width': 512, 'height': 288},
  73         'high': {'preference': 3, 'width': 854, 'height': 480},
  74     }
  75
  76     def _extract_info(self, webpage):
  77         info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
  78             webpage, 'info json')
  79         return json.loads(info_json)
  80
  81     def _real_extract(self, url):
  82         m = re.match(self._VALID_URL, url, re.VERBOSE)
  83         if m.group('type') == 'embed':
  84             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
  85             return self.url_result(desktop_url, 'TED')
  86         name = m.group('name')
  87         if m.group('type_talk'):
  88             return self._talk_info(url, name)
  89         elif m.group('type_watch'):
  90             return self._watch_info(url, name)
  91         else:
  92             return self._playlist_videos_info(url, name)
  93
  94     def _playlist_videos_info(self, url, name):
  95         '''Returns the videos of the playlist'''
  96
  97         webpage = self._download_webpage(url, name,
  98             'Downloading playlist webpage')
  99         info = self._extract_info(webpage)
 100         playlist_info = info['playlist']
 101
 102         playlist_entries = [
 103             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
 104             for talk in info['talks']
 105         ]
 106         return self.playlist_result(
 107             playlist_entries,
 108             playlist_id=compat_str(playlist_info['id']),
 109             playlist_title=playlist_info['title'])
 110
 111     def _talk_info(self, url, video_name):
 112         webpage = self._download_webpage(url, video_name)
 113         self.report_extraction(video_name)
 114
 115         talk_info = self._extract_info(webpage)['talks'][0]
 116
 117         formats = [{
 118             'url': format_url,
 119             'format_id': format_id,
 120             'format': format_id,
 121         } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
 122         if formats:
 123             for f in formats:
 124                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
 125                 if finfo:
 126                     f.update(finfo)
 127         else:
 128             # Use rtmp downloads
 129             formats = [{
 130                 'format_id': f['name'],
 131                 'url': talk_info['streamer'],
 132                 'play_path': f['file'],
 133                 'ext': 'flv',
 134                 'width': f['width'],
 135                 'height': f['height'],
 136                 'tbr': f['bitrate'],
 137             } for f in talk_info['resources']['rtmp']]
 138         self._sort_formats(formats)
 139
 140         video_id = compat_str(talk_info['id'])
 141         # subtitles
 142         video_subtitles = self.extract_subtitles(video_id, talk_info)
 143         if self._downloader.params.get('listsubtitles', False):
 144             self._list_available_subtitles(video_id, talk_info)
 145             return
 146
 147         thumbnail = talk_info['thumb']
 148         if not thumbnail.startswith('http'):
 149             thumbnail = 'http://' + thumbnail
 150         return {
 151             'id': video_id,
 152             'title': talk_info['title'],
 153             'uploader': talk_info['speaker'],
 154             'thumbnail': thumbnail,
 155             'description': self._og_search_description(webpage),
 156             'subtitles': video_subtitles,
 157             'formats': formats,
 158         }
 159
 160     def _get_available_subtitles(self, video_id, talk_info):
 161         languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
 162         if languages:
 163             sub_lang_list = {}
 164             for l in languages:
 165                 url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
 166                 sub_lang_list[l] = url
 167             return sub_lang_list
 168         else:
 169             self._downloader.report_warning('video doesn\'t have subtitles')
 170             return {}
 171
 172     def _watch_info(self, url, name):
 173         webpage = self._download_webpage(url, name)
 174
 175         config_json = self._html_search_regex(
 176             r"data-config='([^']+)", webpage, 'config')
 177         config = json.loads(config_json)
 178         video_url = config['video']['url']
 179         thumbnail = config.get('image', {}).get('url')
 180
 181         title = self._html_search_regex(
 182             r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
 183         description = self._html_search_regex(
 184             [
 185                 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
 186                 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
 187             ],
 188             webpage, 'description', fatal=False)
 189
 190         return {
 191             'id': name,
 192             'url': video_url,
 193             'title': title,
 194             'thumbnail': thumbnail,
 195             'description': description,
 196         }