[yt-dlp.git] / yt_dlp / extractor / frontendmasters.py

import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    parse_duration,
    url_or_none,
    urlencode_postdata,
)


class FrontendMastersBaseIE(InfoExtractor):
    _API_BASE = 'https://api.frontendmasters.com/v1/kabuki'
    _LOGIN_URL = 'https://frontendmasters.com/login/'

    _NETRC_MACHINE = 'frontendmasters'

    _QUALITIES = {
        'low': {'width': 480, 'height': 360},
        'mid': {'width': 1280, 'height': 720},
        'high': {'width': 1920, 'height': 1080}
    }

    def _perform_login(self, username, password):
        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'username': username,
            'password': password
        })

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post_url', default=self._LOGIN_URL, group='url')

        if not post_url.startswith('http'):
            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)

        response = self._download_webpage(
            post_url, None, 'Logging in', data=urlencode_postdata(login_form),
            headers={'Content-Type': 'application/x-www-form-urlencoded'})

        # Successful login
        if any(p in response for p in (
                'wp-login.php?action=logout', '>Logout')):
            return

        error = self._html_search_regex(
            r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P<error>[^<]+)<',
            response, 'error message', default=None, group='error')
        if error:
            raise ExtractorError('Unable to login: %s' % error, expected=True)
        raise ExtractorError('Unable to log in')


class FrontendMastersPageBaseIE(FrontendMastersBaseIE):
    def _download_course(self, course_name, url):
        return self._download_json(
            '%s/courses/%s' % (self._API_BASE, course_name), course_name,
            'Downloading course JSON', headers={'Referer': url})

    @staticmethod
    def _extract_chapters(course):
        chapters = []
        lesson_elements = course.get('lessonElements')
        if isinstance(lesson_elements, list):
            chapters = [url_or_none(e) for e in lesson_elements if url_or_none(e)]
        return chapters

    @staticmethod
    def _extract_lesson(chapters, lesson_id, lesson):
        title = lesson.get('title') or lesson_id
        display_id = lesson.get('slug')
        description = lesson.get('description')
        thumbnail = lesson.get('thumbnail')

        chapter_number = None
        index = lesson.get('index')
        element_index = lesson.get('elementIndex')
        if (isinstance(index, int) and isinstance(element_index, int)
                and index < element_index):
            chapter_number = element_index - index
        chapter = (chapters[chapter_number - 1]
                   if chapter_number - 1 < len(chapters) else None)

        duration = None
        timestamp = lesson.get('timestamp')
        if isinstance(timestamp, compat_str):
            mobj = re.search(
                r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P<end>\s*\d{1,2}:\d{1,2}:\d{1,2})',
                timestamp)
            if mobj:
                duration = parse_duration(mobj.group('end')) - parse_duration(
                    mobj.group('start'))

        return {
            '_type': 'url_transparent',
            'url': 'frontendmasters:%s' % lesson_id,
            'ie_key': FrontendMastersIE.ie_key(),
            'id': lesson_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'chapter': chapter,
            'chapter_number': chapter_number,
        }


class FrontendMastersIE(FrontendMastersBaseIE):
    _VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)'
    _TESTS = [{
        'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba',
        'md5': '7f161159710d6b7016a4f4af6fcb05e2',
        'info_dict': {
            'id': 'a2qogef6ba',
            'ext': 'mp4',
            'title': 'a2qogef6ba',
        },
        'skip': 'Requires FrontendMasters account credentials',
    }, {
        'url': 'frontendmasters:a2qogef6ba',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        lesson_id = self._match_id(url)

        source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id)

        formats = []
        for ext in ('webm', 'mp4'):
            for quality in ('low', 'mid', 'high'):
                resolution = self._QUALITIES[quality].copy()
                format_id = '%s-%s' % (ext, quality)
                format_url = self._download_json(
                    source_url, lesson_id,
                    'Downloading %s source JSON' % format_id, query={
                        'f': ext,
                        'r': resolution['height'],
                    }, headers={
                        'Referer': url,
                    }, fatal=False)['url']

                if not format_url:
                    continue

                f = resolution.copy()
                f.update({
                    'url': format_url,
                    'ext': ext,
                    'format_id': format_id,
                })
                formats.append(f)

        subtitles = {
            'en': [{
                'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id),
            }]
        }

        return {
            'id': lesson_id,
            'title': lesson_id,
            'formats': formats,
            'subtitles': subtitles
        }


class FrontendMastersLessonIE(FrontendMastersPageBaseIE):
    _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)'
    _TEST = {
        'url': 'https://frontendmasters.com/courses/web-development/tools',
        'info_dict': {
            'id': 'a2qogef6ba',
            'display_id': 'tools',
            'ext': 'mp4',
            'title': 'Tools',
            'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7',
            'thumbnail': r're:^https?://.*\.jpg$',
            'chapter': 'Introduction',
            'chapter_number': 1,
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'Requires FrontendMasters account credentials',
    }

    def _real_extract(self, url):
        mobj = self._match_valid_url(url)
        course_name, lesson_name = mobj.group('course_name', 'lesson_name')

        course = self._download_course(course_name, url)

        lesson_id, lesson = next(
            (video_id, data)
            for video_id, data in course['lessonData'].items()
            if data.get('slug') == lesson_name)

        chapters = self._extract_chapters(course)
        return self._extract_lesson(chapters, lesson_id, lesson)


class FrontendMastersCourseIE(FrontendMastersPageBaseIE):
    _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)'
    _TEST = {
        'url': 'https://frontendmasters.com/courses/web-development/',
        'info_dict': {
            'id': 'web-development',
            'title': 'Introduction to Web Development',
            'description': 'md5:9317e6e842098bf725d62360e52d49a6',
        },
        'playlist_count': 81,
        'skip': 'Requires FrontendMasters account credentials',
    }

    @classmethod
    def suitable(cls, url):
        return False if FrontendMastersLessonIE.suitable(url) else super(
            FrontendMastersBaseIE, cls).suitable(url)

    def _real_extract(self, url):
        course_name = self._match_id(url)

        course = self._download_course(course_name, url)

        chapters = self._extract_chapters(course)

        lessons = sorted(
            course['lessonData'].values(), key=lambda data: data['index'])

        entries = []
        for lesson in lessons:
            lesson_name = lesson.get('slug')
            lesson_id = lesson.get('hash') or lesson.get('statsId')
            if not lesson_id or not lesson_name:
                continue
            entries.append(self._extract_lesson(chapters, lesson_id, lesson))

        title = course.get('title')
        description = course.get('description')

        return self.playlist_result(entries, course_name, title, description)
Commit	Line	Data
69fcdb84 S	1	import re
	2
	3	from .common import InfoExtractor
	4	from ..compat import (
	5	compat_str,
	6	compat_urlparse,
	7	)
	8	from ..utils import (
	9	ExtractorError,
	10	parse_duration,
3052a30d	11	url_or_none,
69fcdb84 S	12	urlencode_postdata,
	13	)
	14
	15
	16	class FrontendMastersBaseIE(InfoExtractor):
	17	_API_BASE = 'https://api.frontendmasters.com/v1/kabuki'
	18	_LOGIN_URL = 'https://frontendmasters.com/login/'
	19
	20	_NETRC_MACHINE = 'frontendmasters'
	21
	22	_QUALITIES = {
	23	'low': {'width': 480, 'height': 360},
	24	'mid': {'width': 1280, 'height': 720},
	25	'high': {'width': 1920, 'height': 1080}
	26	}
	27
52efa4b3	28	def _perform_login(self, username, password):
69fcdb84 S	29	login_page = self._download_webpage(
	30	self._LOGIN_URL, None, 'Downloading login page')
	31
	32	login_form = self._hidden_inputs(login_page)
	33
	34	login_form.update({
	35	'username': username,
	36	'password': password
	37	})
	38
	39	post_url = self._search_regex(
	40	r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
	41	'post_url', default=self._LOGIN_URL, group='url')
	42
	43	if not post_url.startswith('http'):
	44	post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
	45
	46	response = self._download_webpage(
	47	post_url, None, 'Logging in', data=urlencode_postdata(login_form),
	48	headers={'Content-Type': 'application/x-www-form-urlencoded'})
	49
	50	# Successful login
	51	if any(p in response for p in (
	52	'wp-login.php?action=logout', '>Logout')):
	53	return
	54
	55	error = self._html_search_regex(
	56	r'class=(["\'])(?:(?!\1).)\bMessageAlert\b(?:(?!\1).)\1[^>]*>(?P<error>[^<]+)<',
	57	response, 'error message', default=None, group='error')
	58	if error:
	59	raise ExtractorError('Unable to login: %s' % error, expected=True)
	60	raise ExtractorError('Unable to log in')
	61
	62
	63	class FrontendMastersPageBaseIE(FrontendMastersBaseIE):
	64	def _download_course(self, course_name, url):
	65	return self._download_json(
	66	'%s/courses/%s' % (self._API_BASE, course_name), course_name,
	67	'Downloading course JSON', headers={'Referer': url})
	68
	69	@staticmethod
	70	def _extract_chapters(course):
	71	chapters = []
	72	lesson_elements = course.get('lessonElements')
	73	if isinstance(lesson_elements, list):
3052a30d	74	chapters = [url_or_none(e) for e in lesson_elements if url_or_none(e)]
69fcdb84 S	75	return chapters
	76
	77	@staticmethod
	78	def _extract_lesson(chapters, lesson_id, lesson):
	79	title = lesson.get('title') or lesson_id
	80	display_id = lesson.get('slug')
	81	description = lesson.get('description')
	82	thumbnail = lesson.get('thumbnail')
	83
	84	chapter_number = None
	85	index = lesson.get('index')
	86	element_index = lesson.get('elementIndex')
3089bc74 S	87	if (isinstance(index, int) and isinstance(element_index, int)
3089bc74 S	88	and index < element_index):
69fcdb84 S	89	chapter_number = element_index - index
	90	chapter = (chapters[chapter_number - 1]
	91	if chapter_number - 1 < len(chapters) else None)
	92
	93	duration = None
	94	timestamp = lesson.get('timestamp')
	95	if isinstance(timestamp, compat_str):
	96	mobj = re.search(
	97	r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s-(?P<end>\s\d{1,2}:\d{1,2}:\d{1,2})',
	98	timestamp)
	99	if mobj:
	100	duration = parse_duration(mobj.group('end')) - parse_duration(
	101	mobj.group('start'))
	102
	103	return {
	104	'_type': 'url_transparent',
	105	'url': 'frontendmasters:%s' % lesson_id,
	106	'ie_key': FrontendMastersIE.ie_key(),
	107	'id': lesson_id,
	108	'display_id': display_id,
	109	'title': title,
	110	'description': description,
	111	'thumbnail': thumbnail,
	112	'duration': duration,
	113	'chapter': chapter,
	114	'chapter_number': chapter_number,
	115	}
	116
	117
	118	class FrontendMastersIE(FrontendMastersBaseIE):
	119	_VALID_URL = r'(?:frontendmasters:\|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)'
	120	_TESTS = [{
	121	'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba',
	122	'md5': '7f161159710d6b7016a4f4af6fcb05e2',
	123	'info_dict': {
	124	'id': 'a2qogef6ba',
	125	'ext': 'mp4',
	126	'title': 'a2qogef6ba',
	127	},
	128	'skip': 'Requires FrontendMasters account credentials',
	129	}, {
	130	'url': 'frontendmasters:a2qogef6ba',
	131	'only_matching': True,
	132	}]
	133
	134	def _real_extract(self, url):
	135	lesson_id = self._match_id(url)
	136
	137	source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id)
	138
	139	formats = []
	140	for ext in ('webm', 'mp4'):
	141	for quality in ('low', 'mid', 'high'):
	142	resolution = self._QUALITIES[quality].copy()
	143	format_id = '%s-%s' % (ext, quality)
	144	format_url = self._download_json(
	145	source_url, lesson_id,
	146	'Downloading %s source JSON' % format_id, query={
	147	'f': ext,
	148	'r': resolution['height'],
	149	}, headers={
	150	'Referer': url,
	151	}, fatal=False)['url']
	152
153	if not format_url:
154	continue
155
156	f = resolution.copy()
157	f.update({
158	'url': format_url,
159	'ext': ext,
160	'format_id': format_id,
161	})
162	formats.append(f)
69fcdb84 S	163
	164	subtitles = {
	165	'en': [{
	166	'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id),
	167	}]
	168	}
	169
	170	return {
	171	'id': lesson_id,
	172	'title': lesson_id,
	173	'formats': formats,
	174	'subtitles': subtitles
	175	}
	176
	177
	178	class FrontendMastersLessonIE(FrontendMastersPageBaseIE):
	179	_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)'
	180	_TEST = {
	181	'url': 'https://frontendmasters.com/courses/web-development/tools',
	182	'info_dict': {
	183	'id': 'a2qogef6ba',
	184	'display_id': 'tools',
	185	'ext': 'mp4',
	186	'title': 'Tools',
	187	'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7',
	188	'thumbnail': r're:^https?://.*\.jpg$',
	189	'chapter': 'Introduction',
	190	'chapter_number': 1,
	191	},
	192	'params': {
	193	'skip_download': True,
	194	},
	195	'skip': 'Requires FrontendMasters account credentials',
	196	}
	197
	198	def _real_extract(self, url):
5ad28e7f	199	mobj = self._match_valid_url(url)
69fcdb84 S	200	course_name, lesson_name = mobj.group('course_name', 'lesson_name')
	201
	202	course = self._download_course(course_name, url)
	203
	204	lesson_id, lesson = next(
	205	(video_id, data)
	206	for video_id, data in course['lessonData'].items()
	207	if data.get('slug') == lesson_name)
	208
	209	chapters = self._extract_chapters(course)
	210	return self._extract_lesson(chapters, lesson_id, lesson)
	211
	212
	213	class FrontendMastersCourseIE(FrontendMastersPageBaseIE):
	214	_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)'
	215	_TEST = {
	216	'url': 'https://frontendmasters.com/courses/web-development/',
	217	'info_dict': {
	218	'id': 'web-development',
	219	'title': 'Introduction to Web Development',
	220	'description': 'md5:9317e6e842098bf725d62360e52d49a6',
	221	},
	222	'playlist_count': 81,
	223	'skip': 'Requires FrontendMasters account credentials',
	224	}
	225
	226	@classmethod
	227	def suitable(cls, url):
	228	return False if FrontendMastersLessonIE.suitable(url) else super(
	229	FrontendMastersBaseIE, cls).suitable(url)
	230
	231	def _real_extract(self, url):
	232	course_name = self._match_id(url)
	233
	234	course = self._download_course(course_name, url)
	235
	236	chapters = self._extract_chapters(course)
	237
	238	lessons = sorted(
	239	course['lessonData'].values(), key=lambda data: data['index'])
	240
	241	entries = []
	242	for lesson in lessons:
	243	lesson_name = lesson.get('slug')
69fcdb84	244	lesson_id = lesson.get('hash') or lesson.get('statsId')
08d30158	245	if not lesson_id or not lesson_name:
08d30158	246	continue
69fcdb84 S	247	entries.append(self._extract_lesson(chapters, lesson_id, lesson))
	248
	249	title = course.get('title')
	250	description = course.get('description')
	251
	252	return self.playlist_result(entries, course_name, title, description)