yt_dlp/extractor/lecturio.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     clean_html,
   6     determine_ext,
   7     ExtractorError,
   8     float_or_none,
   9     int_or_none,
  10     str_or_none,
  11     url_or_none,
  12     urlencode_postdata,
  13     urljoin,
  14 )
  15
  16
  17 class LecturioBaseIE(InfoExtractor):
  18     _API_BASE_URL = 'https://app.lecturio.com/api/en/latest/html5/'
  19     _LOGIN_URL = 'https://app.lecturio.com/en/login'
  20     _NETRC_MACHINE = 'lecturio'
  21
  22     def _perform_login(self, username, password):
  23         # Sets some cookies
  24         _, urlh = self._download_webpage_handle(
  25             self._LOGIN_URL, None, 'Downloading login popup')
  26
  27         def is_logged(url_handle):
  28             return self._LOGIN_URL not in url_handle.geturl()
  29
  30         # Already logged in
  31         if is_logged(urlh):
  32             return
  33
  34         login_form = {
  35             'signin[email]': username,
  36             'signin[password]': password,
  37             'signin[remember]': 'on',
  38         }
  39
  40         response, urlh = self._download_webpage_handle(
  41             self._LOGIN_URL, None, 'Logging in',
  42             data=urlencode_postdata(login_form))
  43
  44         # Logged in successfully
  45         if is_logged(urlh):
  46             return
  47
  48         errors = self._html_search_regex(
  49             r'(?s)<ul[^>]+class=["\']error_list[^>]+>(.+?)</ul>', response,
  50             'errors', default=None)
  51         if errors:
  52             raise ExtractorError('Unable to login: %s' % errors, expected=True)
  53         raise ExtractorError('Unable to log in')
  54
  55
  56 class LecturioIE(LecturioBaseIE):
  57     _VALID_URL = r'''(?x)
  58                     https://
  59                         (?:
  60                             app\.lecturio\.com/([^/]+/(?P<nt>[^/?#&]+)\.lecture|(?:\#/)?lecture/c/\d+/(?P<id>\d+))|
  61                             (?:www\.)?lecturio\.de/[^/]+/(?P<nt_de>[^/?#&]+)\.vortrag
  62                         )
  63                     '''
  64     _TESTS = [{
  65         'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
  66         'md5': '9a42cf1d8282a6311bf7211bbde26fde',
  67         'info_dict': {
  68             'id': '39634',
  69             'ext': 'mp4',
  70             'title': 'Important Concepts and Terms — Introduction to Microbiology',
  71         },
  72         'skip': 'Requires lecturio account credentials',
  73     }, {
  74         'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
  75         'only_matching': True,
  76     }, {
  77         'url': 'https://app.lecturio.com/#/lecture/c/6434/39634',
  78         'only_matching': True,
  79     }]
  80
  81     _CC_LANGS = {
  82         'Arabic': 'ar',
  83         'Bulgarian': 'bg',
  84         'German': 'de',
  85         'English': 'en',
  86         'Spanish': 'es',
  87         'Persian': 'fa',
  88         'French': 'fr',
  89         'Japanese': 'ja',
  90         'Polish': 'pl',
  91         'Pashto': 'ps',
  92         'Russian': 'ru',
  93     }
  94
  95     def _real_extract(self, url):
  96         mobj = self._match_valid_url(url)
  97         nt = mobj.group('nt') or mobj.group('nt_de')
  98         lecture_id = mobj.group('id')
  99         display_id = nt or lecture_id
 100         api_path = 'lectures/' + lecture_id if lecture_id else 'lecture/' + nt + '.json'
 101         video = self._download_json(
 102             self._API_BASE_URL + api_path, display_id)
 103         title = video['title'].strip()
 104         if not lecture_id:
 105             pid = video.get('productId') or video.get('uid')
 106             if pid:
 107                 spid = pid.split('_')
 108                 if spid and len(spid) == 2:
 109                     lecture_id = spid[1]
 110
 111         formats = []
 112         for format_ in video['content']['media']:
 113             if not isinstance(format_, dict):
 114                 continue
 115             file_ = format_.get('file')
 116             if not file_:
 117                 continue
 118             ext = determine_ext(file_)
 119             if ext == 'smil':
 120                 # smil contains only broken RTMP formats anyway
 121                 continue
 122             file_url = url_or_none(file_)
 123             if not file_url:
 124                 continue
 125             label = str_or_none(format_.get('label'))
 126             filesize = int_or_none(format_.get('fileSize'))
 127             f = {
 128                 'url': file_url,
 129                 'format_id': label,
 130                 'filesize': float_or_none(filesize, invscale=1000)
 131             }
 132             if label:
 133                 mobj = re.match(r'(\d+)p\s*\(([^)]+)\)', label)
 134                 if mobj:
 135                     f.update({
 136                         'format_id': mobj.group(2),
 137                         'height': int(mobj.group(1)),
 138                     })
 139             formats.append(f)
 140         self._sort_formats(formats)
 141
 142         subtitles = {}
 143         automatic_captions = {}
 144         captions = video.get('captions') or []
 145         for cc in captions:
 146             cc_url = cc.get('url')
 147             if not cc_url:
 148                 continue
 149             cc_label = cc.get('translatedCode')
 150             lang = cc.get('languageCode') or self._search_regex(
 151                 r'/([a-z]{2})_', cc_url, 'lang',
 152                 default=cc_label.split()[0] if cc_label else 'en')
 153             original_lang = self._search_regex(
 154                 r'/[a-z]{2}_([a-z]{2})_', cc_url, 'original lang',
 155                 default=None)
 156             sub_dict = (automatic_captions
 157                         if 'auto-translated' in cc_label or original_lang
 158                         else subtitles)
 159             sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
 160                 'url': cc_url,
 161             })
 162
 163         return {
 164             'id': lecture_id or nt,
 165             'title': title,
 166             'formats': formats,
 167             'subtitles': subtitles,
 168             'automatic_captions': automatic_captions,
 169         }
 170
 171
 172 class LecturioCourseIE(LecturioBaseIE):
 173     _VALID_URL = r'https://app\.lecturio\.com/(?:[^/]+/(?P<nt>[^/?#&]+)\.course|(?:#/)?course/c/(?P<id>\d+))'
 174     _TESTS = [{
 175         'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
 176         'info_dict': {
 177             'id': 'microbiology-introduction',
 178             'title': 'Microbiology: Introduction',
 179             'description': 'md5:13da8500c25880c6016ae1e6d78c386a',
 180         },
 181         'playlist_count': 45,
 182         'skip': 'Requires lecturio account credentials',
 183     }, {
 184         'url': 'https://app.lecturio.com/#/course/c/6434',
 185         'only_matching': True,
 186     }]
 187
 188     def _real_extract(self, url):
 189         nt, course_id = self._match_valid_url(url).groups()
 190         display_id = nt or course_id
 191         api_path = 'courses/' + course_id if course_id else 'course/content/' + nt + '.json'
 192         course = self._download_json(
 193             self._API_BASE_URL + api_path, display_id)
 194         entries = []
 195         for lecture in course.get('lectures', []):
 196             lecture_id = str_or_none(lecture.get('id'))
 197             lecture_url = lecture.get('url')
 198             if lecture_url:
 199                 lecture_url = urljoin(url, lecture_url)
 200             else:
 201                 lecture_url = 'https://app.lecturio.com/#/lecture/c/%s/%s' % (course_id, lecture_id)
 202             entries.append(self.url_result(
 203                 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
 204         return self.playlist_result(
 205             entries, display_id, course.get('title'),
 206             clean_html(course.get('description')))
 207
 208
 209 class LecturioDeCourseIE(LecturioBaseIE):
 210     _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
 211     _TEST = {
 212         'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
 213         'only_matching': True,
 214     }
 215
 216     def _real_extract(self, url):
 217         display_id = self._match_id(url)
 218
 219         webpage = self._download_webpage(url, display_id)
 220
 221         entries = []
 222         for mobj in re.finditer(
 223                 r'(?s)<td[^>]+\bdata-lecture-id=["\'](?P<id>\d+).+?\bhref=(["\'])(?P<url>(?:(?!\2).)+\.vortrag)\b[^>]+>',
 224                 webpage):
 225             lecture_url = urljoin(url, mobj.group('url'))
 226             lecture_id = mobj.group('id')
 227             entries.append(self.url_result(
 228                 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
 229
 230         title = self._search_regex(
 231             r'<h1[^>]*>([^<]+)', webpage, 'title', default=None)
 232
 233         return self.playlist_result(entries, display_id, title)