[yt-dlp.git] / youtube_dl / extractor / pluralsight.py

from __future__ import unicode_literals

import json

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urllib_parse,
    compat_urllib_request,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_duration,
)


class PluralsightIE(InfoExtractor):
    IE_NAME = 'pluralsight'
    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
    _LOGIN_URL = 'https://app.pluralsight.com/id/'
    _API_BASE = 'http://app.pluralsight.com'
    _NETRC_MACHINE = 'pluralsight'

    _TESTS = [{
        'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
        'md5': '4d458cf5cf4c593788672419a8dd4cf8',
        'info_dict': {
            'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
            'ext': 'mp4',
            'title': 'Management of SQL Server - Demo Monitoring',
            'duration': 338,
        },
        'skip': 'Requires pluralsight account credentials',
    }, {
        'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
        'only_matching': True,
    }, {
        # available without pluralsight account
        'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
        'only_matching': True,
    }]

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'Username': username.encode('utf-8'),
            'Password': password.encode('utf-8'),
        })

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post url', default=self._LOGIN_URL, group='url')

        if not post_url.startswith('http'):
            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)

        request = compat_urllib_request.Request(
            post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
        request.add_header('Content-Type', 'application/x-www-form-urlencoded')

        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        error = self._search_regex(
            r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
            response, 'error message', default=None)
        if error:
            raise ExtractorError('Unable to login: %s' % error, expected=True)

    def _real_extract(self, url):
        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)

        author = qs.get('author', [None])[0]
        name = qs.get('name', [None])[0]
        clip_id = qs.get('clip', [None])[0]
        course = qs.get('course', [None])[0]

        if any(not f for f in (author, name, clip_id, course,)):
            raise ExtractorError('Invalid URL', expected=True)

        display_id = '%s-%s' % (name, clip_id)

        webpage = self._download_webpage(url, display_id)

        collection = self._parse_json(
            self._search_regex(
                r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
                webpage, 'modules'),
            display_id)

        module, clip = None, None

        for module_ in collection:
            if module_.get('moduleName') == name:
                module = module_
                for clip_ in module_.get('clips', []):
                    clip_index = clip_.get('clipIndex')
                    if clip_index is None:
                        continue
                    if compat_str(clip_index) == clip_id:
                        clip = clip_
                        break

        if not clip:
            raise ExtractorError('Unable to resolve clip')

        QUALITIES = {
            'low': {'width': 640, 'height': 480},
            'medium': {'width': 848, 'height': 640},
            'high': {'width': 1024, 'height': 768},
        }

        ALLOWED_QUALITIES = (
            ('webm', ('high',)),
            ('mp4', ('low', 'medium', 'high',)),
        )

        formats = []
        for ext, qualities in ALLOWED_QUALITIES:
            for quality in qualities:
                f = QUALITIES[quality].copy()
                clip_post = {
                    'a': author,
                    'cap': 'false',
                    'cn': clip_id,
                    'course': course,
                    'lc': 'en',
                    'm': name,
                    'mt': ext,
                    'q': '%dx%d' % (f['width'], f['height']),
                }
                request = compat_urllib_request.Request(
                    '%s/training/Player/ViewClip' % self._API_BASE,
                    json.dumps(clip_post).encode('utf-8'))
                request.add_header('Content-Type', 'application/json;charset=utf-8')
                format_id = '%s-%s' % (ext, quality)
                clip_url = self._download_webpage(
                    request, display_id, 'Downloading %s URL' % format_id, fatal=False)
                if not clip_url:
                    continue
                f.update({
                    'url': clip_url,
                    'ext': ext,
                    'format_id': format_id,
                })
                formats.append(f)
        self._sort_formats(formats)

        # TODO: captions
        # http://www.pluralsight.com/training/Player/ViewClip + cap = true
        # or
        # http://www.pluralsight.com/training/Player/Captions
        # { a = author, cn = clip_id, lc = end, m = name }

        return {
            'id': clip['clipName'],
            'title': '%s - %s' % (module['title'], clip['title']),
            'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
            'creator': author,
            'formats': formats
        }


class PluralsightCourseIE(InfoExtractor):
    IE_NAME = 'pluralsight:course'
    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
    _TESTS = [{
        # Free course from Pluralsight Starter Subscription for Microsoft TechNet
        # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
        'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
        'info_dict': {
            'id': 'hosting-sql-server-windows-azure-iaas',
            'title': 'Hosting SQL Server in Microsoft Azure IaaS Fundamentals',
            'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
        },
        'playlist_count': 31,
    }, {
        # available without pluralsight account
        'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
        'only_matching': True,
    }, {
        'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        course_id = self._match_id(url)

        # TODO: PSM cookie

        course = self._download_json(
            '%s/data/course/%s' % (self._API_BASE, course_id),
            course_id, 'Downloading course JSON')

        title = course['title']
        description = course.get('description') or course.get('shortDescription')

        course_data = self._download_json(
            '%s/data/course/content/%s' % (self._API_BASE, course_id),
            course_id, 'Downloading course data JSON')

        entries = []
        for module in course_data:
            for clip in module.get('clips', []):
                player_parameters = clip.get('playerParameters')
                if not player_parameters:
                    continue
                entries.append(self.url_result(
                    '%s/training/player?%s' % (self._API_BASE, player_parameters),
                    'Pluralsight'))

        return self.playlist_result(entries, course_id, title, description)
Commit	Line	Data
483fc223 S	1	from __future__ import unicode_literals
483fc223 S	2
483fc223 S	3	import json
	4
	5	from .common import InfoExtractor
	6	from ..compat import (
	7	compat_str,
	8	compat_urllib_parse,
	9	compat_urllib_request,
	10	compat_urlparse,
	11	)
	12	from ..utils import (
	13	ExtractorError,
	14	int_or_none,
	15	parse_duration,
	16	)
	17
	18
	19	class PluralsightIE(InfoExtractor):
	20	IE_NAME = 'pluralsight'
71bd93b8	21	_VALID_URL = r'https?://(?:(?:www\|app)\.)?pluralsight\.com/training/player\?'
c3a227d1	22	_LOGIN_URL = 'https://app.pluralsight.com/id/'
0533915a	23	_API_BASE = 'http://app.pluralsight.com'
483fc223 S	24	_NETRC_MACHINE = 'pluralsight'
483fc223 S	25
71bd93b8	26	_TESTS = [{
483fc223 S	27	'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
	28	'md5': '4d458cf5cf4c593788672419a8dd4cf8',
	29	'info_dict': {
	30	'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
	31	'ext': 'mp4',
	32	'title': 'Management of SQL Server - Demo Monitoring',
	33	'duration': 338,
	34	},
	35	'skip': 'Requires pluralsight account credentials',
71bd93b8 S	36	}, {
	37	'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
	38	'only_matching': True,
c23e2664 S	39	}, {
	40	# available without pluralsight account
	41	'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
	42	'only_matching': True,
71bd93b8	43	}]
483fc223 S	44
	45	def _real_initialize(self):
	46	self._login()
	47
	48	def _login(self):
	49	(username, password) = self._get_login_info()
	50	if username is None:
c23e2664	51	return
483fc223 S	52
	53	login_page = self._download_webpage(
	54	self._LOGIN_URL, None, 'Downloading login page')
	55
	56	login_form = self._hidden_inputs(login_page)
	57
	58	login_form.update({
	59	'Username': username.encode('utf-8'),
	60	'Password': password.encode('utf-8'),
	61	})
	62
	63	post_url = self._search_regex(
	64	r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
	65	'post url', default=self._LOGIN_URL, group='url')
	66
	67	if not post_url.startswith('http'):
	68	post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
	69
	70	request = compat_urllib_request.Request(
	71	post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
	72	request.add_header('Content-Type', 'application/x-www-form-urlencoded')
	73
	74	response = self._download_webpage(
	75	request, None, 'Logging in as %s' % username)
	76
	77	error = self._search_regex(
	78	r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
	79	response, 'error message', default=None)
	80	if error:
	81	raise ExtractorError('Unable to login: %s' % error, expected=True)
	82
	83	def _real_extract(self, url):
71bd93b8 S	84	qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
	85
	86	author = qs.get('author', [None])[0]
	87	name = qs.get('name', [None])[0]
	88	clip_id = qs.get('clip', [None])[0]
	89	course = qs.get('course', [None])[0]
	90
	91	if any(not f for f in (author, name, clip_id, course,)):
	92	raise ExtractorError('Invalid URL', expected=True)
483fc223 S	93
	94	display_id = '%s-%s' % (name, clip_id)
	95
	96	webpage = self._download_webpage(url, display_id)
	97
	98	collection = self._parse_json(
	99	self._search_regex(
	100	r'moduleCollection\s:\snew\s+ModuleCollection\((\[.+?\])\s,\s\$rootScope\)',
	101	webpage, 'modules'),
	102	display_id)
	103
	104	module, clip = None, None
	105
	106	for module_ in collection:
	107	if module_.get('moduleName') == name:
	108	module = module_
	109	for clip_ in module_.get('clips', []):
	110	clip_index = clip_.get('clipIndex')
	111	if clip_index is None:
	112	continue
	113	if compat_str(clip_index) == clip_id:
	114	clip = clip_
	115	break
	116
	117	if not clip:
	118	raise ExtractorError('Unable to resolve clip')
	119
	120	QUALITIES = {
	121	'low': {'width': 640, 'height': 480},
	122	'medium': {'width': 848, 'height': 640},
	123	'high': {'width': 1024, 'height': 768},
	124	}
	125
	126	ALLOWED_QUALITIES = (
	127	('webm', ('high',)),
	128	('mp4', ('low', 'medium', 'high',)),
	129	)
	130
	131	formats = []
	132	for ext, qualities in ALLOWED_QUALITIES:
	133	for quality in qualities:
	134	f = QUALITIES[quality].copy()
	135	clip_post = {
	136	'a': author,
	137	'cap': 'false',
	138	'cn': clip_id,
	139	'course': course,
	140	'lc': 'en',
	141	'm': name,
	142	'mt': ext,
	143	'q': '%dx%d' % (f['width'], f['height']),
	144	}
	145	request = compat_urllib_request.Request(
0533915a	146	'%s/training/Player/ViewClip' % self._API_BASE,
483fc223 S	147	json.dumps(clip_post).encode('utf-8'))
	148	request.add_header('Content-Type', 'application/json;charset=utf-8')
	149	format_id = '%s-%s' % (ext, quality)
	150	clip_url = self._download_webpage(
	151	request, display_id, 'Downloading %s URL' % format_id, fatal=False)
	152	if not clip_url:
	153	continue
	154	f.update({
	155	'url': clip_url,
	156	'ext': ext,
	157	'format_id': format_id,
	158	})
	159	formats.append(f)
	160	self._sort_formats(formats)
	161
	162	# TODO: captions
	163	# http://www.pluralsight.com/training/Player/ViewClip + cap = true
	164	# or
	165	# http://www.pluralsight.com/training/Player/Captions
	166	# { a = author, cn = clip_id, lc = end, m = name }
	167
	168	return {
	169	'id': clip['clipName'],
	170	'title': '%s - %s' % (module['title'], clip['title']),
	171	'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
	172	'creator': author,
	173	'formats': formats
	174	}
	175
	176
	177	class PluralsightCourseIE(InfoExtractor):
	178	IE_NAME = 'pluralsight:course'
a5cd0eb8	179	_VALID_URL = r'https?://(?:(?:www\|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
c23e2664	180	_TESTS = [{
483fc223 S	181	# Free course from Pluralsight Starter Subscription for Microsoft TechNet
	182	# https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
	183	'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
	184	'info_dict': {
	185	'id': 'hosting-sql-server-windows-azure-iaas',
	186	'title': 'Hosting SQL Server in Microsoft Azure IaaS Fundamentals',
	187	'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
	188	},
	189	'playlist_count': 31,
c23e2664 S	190	}, {
	191	# available without pluralsight account
	192	'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
	193	'only_matching': True,
a5cd0eb8 S	194	}, {
	195	'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
	196	'only_matching': True,
c23e2664	197	}]
483fc223 S	198
	199	def _real_extract(self, url):
	200	course_id = self._match_id(url)
	201
2b6bda1e S	202	# TODO: PSM cookie
2b6bda1e S	203
483fc223	204	course = self._download_json(
0533915a	205	'%s/data/course/%s' % (self._API_BASE, course_id),
483fc223 S	206	course_id, 'Downloading course JSON')
	207
	208	title = course['title']
	209	description = course.get('description') or course.get('shortDescription')
	210
	211	course_data = self._download_json(
0533915a	212	'%s/data/course/content/%s' % (self._API_BASE, course_id),
483fc223 S	213	course_id, 'Downloading course data JSON')
483fc223 S	214
483fc223 S	215	entries = []
	216	for module in course_data:
	217	for clip in module.get('clips', []):
483fc223 S	218	player_parameters = clip.get('playerParameters')
	219	if not player_parameters:
	220	continue
	221	entries.append(self.url_result(
0533915a	222	'%s/training/player?%s' % (self._API_BASE, player_parameters),
483fc223 S	223	'Pluralsight'))
483fc223 S	224
483fc223	225	return self.playlist_result(entries, course_id, title, description)