yt_dlp/extractor/frontendmasters.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import (
   8     compat_str,
   9     compat_urlparse,
  10 )
  11 from ..utils import (
  12     ExtractorError,
  13     parse_duration,
  14     url_or_none,
  15     urlencode_postdata,
  16 )
  17
  18
  19 class FrontendMastersBaseIE(InfoExtractor):
  20     _API_BASE = 'https://api.frontendmasters.com/v1/kabuki'
  21     _LOGIN_URL = 'https://frontendmasters.com/login/'
  22
  23     _NETRC_MACHINE = 'frontendmasters'
  24
  25     _QUALITIES = {
  26         'low': {'width': 480, 'height': 360},
  27         'mid': {'width': 1280, 'height': 720},
  28         'high': {'width': 1920, 'height': 1080}
  29     }
  30
  31     def _perform_login(self, username, password):
  32         login_page = self._download_webpage(
  33             self._LOGIN_URL, None, 'Downloading login page')
  34
  35         login_form = self._hidden_inputs(login_page)
  36
  37         login_form.update({
  38             'username': username,
  39             'password': password
  40         })
  41
  42         post_url = self._search_regex(
  43             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
  44             'post_url', default=self._LOGIN_URL, group='url')
  45
  46         if not post_url.startswith('http'):
  47             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
  48
  49         response = self._download_webpage(
  50             post_url, None, 'Logging in', data=urlencode_postdata(login_form),
  51             headers={'Content-Type': 'application/x-www-form-urlencoded'})
  52
  53         # Successful login
  54         if any(p in response for p in (
  55                 'wp-login.php?action=logout', '>Logout')):
  56             return
  57
  58         error = self._html_search_regex(
  59             r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P<error>[^<]+)<',
  60             response, 'error message', default=None, group='error')
  61         if error:
  62             raise ExtractorError('Unable to login: %s' % error, expected=True)
  63         raise ExtractorError('Unable to log in')
  64
  65
  66 class FrontendMastersPageBaseIE(FrontendMastersBaseIE):
  67     def _download_course(self, course_name, url):
  68         return self._download_json(
  69             '%s/courses/%s' % (self._API_BASE, course_name), course_name,
  70             'Downloading course JSON', headers={'Referer': url})
  71
  72     @staticmethod
  73     def _extract_chapters(course):
  74         chapters = []
  75         lesson_elements = course.get('lessonElements')
  76         if isinstance(lesson_elements, list):
  77             chapters = [url_or_none(e) for e in lesson_elements if url_or_none(e)]
  78         return chapters
  79
  80     @staticmethod
  81     def _extract_lesson(chapters, lesson_id, lesson):
  82         title = lesson.get('title') or lesson_id
  83         display_id = lesson.get('slug')
  84         description = lesson.get('description')
  85         thumbnail = lesson.get('thumbnail')
  86
  87         chapter_number = None
  88         index = lesson.get('index')
  89         element_index = lesson.get('elementIndex')
  90         if (isinstance(index, int) and isinstance(element_index, int)
  91                 and index < element_index):
  92             chapter_number = element_index - index
  93         chapter = (chapters[chapter_number - 1]
  94                    if chapter_number - 1 < len(chapters) else None)
  95
  96         duration = None
  97         timestamp = lesson.get('timestamp')
  98         if isinstance(timestamp, compat_str):
  99             mobj = re.search(
 100                 r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P<end>\s*\d{1,2}:\d{1,2}:\d{1,2})',
 101                 timestamp)
 102             if mobj:
 103                 duration = parse_duration(mobj.group('end')) - parse_duration(
 104                     mobj.group('start'))
 105
 106         return {
 107             '_type': 'url_transparent',
 108             'url': 'frontendmasters:%s' % lesson_id,
 109             'ie_key': FrontendMastersIE.ie_key(),
 110             'id': lesson_id,
 111             'display_id': display_id,
 112             'title': title,
 113             'description': description,
 114             'thumbnail': thumbnail,
 115             'duration': duration,
 116             'chapter': chapter,
 117             'chapter_number': chapter_number,
 118         }
 119
 120
 121 class FrontendMastersIE(FrontendMastersBaseIE):
 122     _VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)'
 123     _TESTS = [{
 124         'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba',
 125         'md5': '7f161159710d6b7016a4f4af6fcb05e2',
 126         'info_dict': {
 127             'id': 'a2qogef6ba',
 128             'ext': 'mp4',
 129             'title': 'a2qogef6ba',
 130         },
 131         'skip': 'Requires FrontendMasters account credentials',
 132     }, {
 133         'url': 'frontendmasters:a2qogef6ba',
 134         'only_matching': True,
 135     }]
 136
 137     def _real_extract(self, url):
 138         lesson_id = self._match_id(url)
 139
 140         source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id)
 141
 142         formats = []
 143         for ext in ('webm', 'mp4'):
 144             for quality in ('low', 'mid', 'high'):
 145                 resolution = self._QUALITIES[quality].copy()
 146                 format_id = '%s-%s' % (ext, quality)
 147                 format_url = self._download_json(
 148                     source_url, lesson_id,
 149                     'Downloading %s source JSON' % format_id, query={
 150                         'f': ext,
 151                         'r': resolution['height'],
 152                     }, headers={
 153                         'Referer': url,
 154                     }, fatal=False)['url']
 155
 156                 if not format_url:
 157                     continue
 158
 159                 f = resolution.copy()
 160                 f.update({
 161                     'url': format_url,
 162                     'ext': ext,
 163                     'format_id': format_id,
 164                 })
 165                 formats.append(f)
 166         self._sort_formats(formats)
 167
 168         subtitles = {
 169             'en': [{
 170                 'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id),
 171             }]
 172         }
 173
 174         return {
 175             'id': lesson_id,
 176             'title': lesson_id,
 177             'formats': formats,
 178             'subtitles': subtitles
 179         }
 180
 181
 182 class FrontendMastersLessonIE(FrontendMastersPageBaseIE):
 183     _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)'
 184     _TEST = {
 185         'url': 'https://frontendmasters.com/courses/web-development/tools',
 186         'info_dict': {
 187             'id': 'a2qogef6ba',
 188             'display_id': 'tools',
 189             'ext': 'mp4',
 190             'title': 'Tools',
 191             'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7',
 192             'thumbnail': r're:^https?://.*\.jpg$',
 193             'chapter': 'Introduction',
 194             'chapter_number': 1,
 195         },
 196         'params': {
 197             'skip_download': True,
 198         },
 199         'skip': 'Requires FrontendMasters account credentials',
 200     }
 201
 202     def _real_extract(self, url):
 203         mobj = self._match_valid_url(url)
 204         course_name, lesson_name = mobj.group('course_name', 'lesson_name')
 205
 206         course = self._download_course(course_name, url)
 207
 208         lesson_id, lesson = next(
 209             (video_id, data)
 210             for video_id, data in course['lessonData'].items()
 211             if data.get('slug') == lesson_name)
 212
 213         chapters = self._extract_chapters(course)
 214         return self._extract_lesson(chapters, lesson_id, lesson)
 215
 216
 217 class FrontendMastersCourseIE(FrontendMastersPageBaseIE):
 218     _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)'
 219     _TEST = {
 220         'url': 'https://frontendmasters.com/courses/web-development/',
 221         'info_dict': {
 222             'id': 'web-development',
 223             'title': 'Introduction to Web Development',
 224             'description': 'md5:9317e6e842098bf725d62360e52d49a6',
 225         },
 226         'playlist_count': 81,
 227         'skip': 'Requires FrontendMasters account credentials',
 228     }
 229
 230     @classmethod
 231     def suitable(cls, url):
 232         return False if FrontendMastersLessonIE.suitable(url) else super(
 233             FrontendMastersBaseIE, cls).suitable(url)
 234
 235     def _real_extract(self, url):
 236         course_name = self._match_id(url)
 237
 238         course = self._download_course(course_name, url)
 239
 240         chapters = self._extract_chapters(course)
 241
 242         lessons = sorted(
 243             course['lessonData'].values(), key=lambda data: data['index'])
 244
 245         entries = []
 246         for lesson in lessons:
 247             lesson_name = lesson.get('slug')
 248             lesson_id = lesson.get('hash') or lesson.get('statsId')
 249             if not lesson_id or not lesson_name:
 250                 continue
 251             entries.append(self._extract_lesson(chapters, lesson_id, lesson))
 252
 253         title = course.get('title')
 254         description = course.get('description')
 255
 256         return self.playlist_result(entries, course_name, title, description)