[yt-dlp.git] / youtube_dl / extractor / pluralsight.py

from __future__ import unicode_literals

import collections
import json
import os
import random

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urlparse,
)
from ..utils import (
    dict_get,
    ExtractorError,
    float_or_none,
    int_or_none,
    parse_duration,
    qualities,
    srt_subtitles_timecode,
    try_get,
    update_url_query,
    urlencode_postdata,
)


class PluralsightBaseIE(InfoExtractor):
    _API_BASE = 'https://app.pluralsight.com'

    def _download_course(self, course_id, url, display_id):
        try:
            return self._download_course_rpc(course_id, url, display_id)
        except ExtractorError:
            # Old API fallback
            return self._download_json(
                'https://app.pluralsight.com/player/user/api/v1/player/payload',
                display_id, data=urlencode_postdata({'courseId': course_id}),
                headers={'Referer': url})

    def _download_course_rpc(self, course_id, url, display_id):
        response = self._download_json(
            '%s/player/functions/rpc' % self._API_BASE, display_id,
            'Downloading course JSON',
            data=json.dumps({
                'fn': 'bootstrapPlayer',
                'payload': {
                    'courseId': course_id,
                },
            }).encode('utf-8'),
            headers={
                'Content-Type': 'application/json;charset=utf-8',
                'Referer': url,
            })

        course = try_get(response, lambda x: x['payload']['course'], dict)
        if course:
            return course

        raise ExtractorError(
            '%s said: %s' % (self.IE_NAME, response['error']['message']),
            expected=True)


class PluralsightIE(PluralsightBaseIE):
    IE_NAME = 'pluralsight'
    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:training/)?player\?'
    _LOGIN_URL = 'https://app.pluralsight.com/id/'

    _NETRC_MACHINE = 'pluralsight'

    _TESTS = [{
        'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
        'md5': '4d458cf5cf4c593788672419a8dd4cf8',
        'info_dict': {
            'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
            'ext': 'mp4',
            'title': 'Demo Monitoring',
            'duration': 338,
        },
        'skip': 'Requires pluralsight account credentials',
    }, {
        'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
        'only_matching': True,
    }, {
        # available without pluralsight account
        'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
        'only_matching': True,
    }, {
        'url': 'https://app.pluralsight.com/player?course=ccna-intro-networking&author=ross-bagurdes&name=ccna-intro-networking-m06&clip=0',
        'only_matching': True,
    }]

    def _real_initialize(self):
        self._login()

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'Username': username,
            'Password': password,
        })

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post url', default=self._LOGIN_URL, group='url')

        if not post_url.startswith('http'):
            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)

        response = self._download_webpage(
            post_url, None, 'Logging in',
            data=urlencode_postdata(login_form),
            headers={'Content-Type': 'application/x-www-form-urlencoded'})

        error = self._search_regex(
            r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
            response, 'error message', default=None)
        if error:
            raise ExtractorError('Unable to login: %s' % error, expected=True)

        if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
            BLOCKED = 'Your account has been blocked due to suspicious activity'
            if BLOCKED in response:
                raise ExtractorError(
                    'Unable to login: %s' % BLOCKED, expected=True)
            MUST_AGREE = 'To continue using Pluralsight, you must agree to'
            if any(p in response for p in (MUST_AGREE, '>Disagree<', '>Agree<')):
                raise ExtractorError(
                    'Unable to login: %s some documents. Go to pluralsight.com, '
                    'log in and agree with what Pluralsight requires.'
                    % MUST_AGREE, expected=True)

            raise ExtractorError('Unable to log in')

    def _get_subtitles(self, author, clip_idx, lang, name, duration, video_id):
        captions_post = {
            'a': author,
            'cn': clip_idx,
            'lc': lang,
            'm': name,
        }
        captions = self._download_json(
            '%s/player/retrieve-captions' % self._API_BASE, video_id,
            'Downloading captions JSON', 'Unable to download captions JSON',
            fatal=False, data=json.dumps(captions_post).encode('utf-8'),
            headers={'Content-Type': 'application/json;charset=utf-8'})
        if captions:
            return {
                lang: [{
                    'ext': 'json',
                    'data': json.dumps(captions),
                }, {
                    'ext': 'srt',
                    'data': self._convert_subtitles(duration, captions),
                }]
            }

    @staticmethod
    def _convert_subtitles(duration, subs):
        srt = ''
        TIME_OFFSET_KEYS = ('displayTimeOffset', 'DisplayTimeOffset')
        TEXT_KEYS = ('text', 'Text')
        for num, current in enumerate(subs):
            current = subs[num]
            start, text = (
                float_or_none(dict_get(current, TIME_OFFSET_KEYS, skip_false_values=False)),
                dict_get(current, TEXT_KEYS))
            if start is None or text is None:
                continue
            end = duration if num == len(subs) - 1 else float_or_none(
                dict_get(subs[num + 1], TIME_OFFSET_KEYS, skip_false_values=False))
            if end is None:
                continue
            srt += os.linesep.join(
                (
                    '%d' % num,
                    '%s --> %s' % (
                        srt_subtitles_timecode(start),
                        srt_subtitles_timecode(end)),
                    text,
                    os.linesep,
                ))
        return srt

    def _real_extract(self, url):
        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)

        author = qs.get('author', [None])[0]
        name = qs.get('name', [None])[0]
        clip_idx = qs.get('clip', [None])[0]
        course_name = qs.get('course', [None])[0]

        if any(not f for f in (author, name, clip_idx, course_name,)):
            raise ExtractorError('Invalid URL', expected=True)

        display_id = '%s-%s' % (name, clip_idx)

        course = self._download_course(course_name, url, display_id)

        collection = course['modules']

        clip = None

        for module_ in collection:
            if name in (module_.get('moduleName'), module_.get('name')):
                for clip_ in module_.get('clips', []):
                    clip_index = clip_.get('clipIndex')
                    if clip_index is None:
                        clip_index = clip_.get('index')
                    if clip_index is None:
                        continue
                    if compat_str(clip_index) == clip_idx:
                        clip = clip_
                        break

        if not clip:
            raise ExtractorError('Unable to resolve clip')

        title = clip['title']
        clip_id = clip.get('clipName') or clip.get('name') or clip['clipId']

        QUALITIES = {
            'low': {'width': 640, 'height': 480},
            'medium': {'width': 848, 'height': 640},
            'high': {'width': 1024, 'height': 768},
            'high-widescreen': {'width': 1280, 'height': 720},
        }

        QUALITIES_PREFERENCE = ('low', 'medium', 'high', 'high-widescreen',)
        quality_key = qualities(QUALITIES_PREFERENCE)

        AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])

        ALLOWED_QUALITIES = (
            AllowedQuality('webm', ['high', ]),
            AllowedQuality('mp4', ['low', 'medium', 'high', ]),
        )

        # Some courses also offer widescreen resolution for high quality (see
        # https://github.com/rg3/youtube-dl/issues/7766)
        widescreen = course.get('supportsWideScreenVideoFormats') is True
        best_quality = 'high-widescreen' if widescreen else 'high'
        if widescreen:
            for allowed_quality in ALLOWED_QUALITIES:
                allowed_quality.qualities.append(best_quality)

        # In order to minimize the number of calls to ViewClip API and reduce
        # the probability of being throttled or banned by Pluralsight we will request
        # only single format until formats listing was explicitly requested.
        if self._downloader.params.get('listformats', False):
            allowed_qualities = ALLOWED_QUALITIES
        else:
            def guess_allowed_qualities():
                req_format = self._downloader.params.get('format') or 'best'
                req_format_split = req_format.split('-', 1)
                if len(req_format_split) > 1:
                    req_ext, req_quality = req_format_split
                    req_quality = '-'.join(req_quality.split('-')[:2])
                    for allowed_quality in ALLOWED_QUALITIES:
                        if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
                            return (AllowedQuality(req_ext, (req_quality, )), )
                req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
                return (AllowedQuality(req_ext, (best_quality, )), )
            allowed_qualities = guess_allowed_qualities()

        formats = []
        for ext, qualities_ in allowed_qualities:
            for quality in qualities_:
                f = QUALITIES[quality].copy()
                clip_post = {
                    'author': author,
                    'includeCaptions': False,
                    'clipIndex': int(clip_idx),
                    'courseName': course_name,
                    'locale': 'en',
                    'moduleName': name,
                    'mediaType': ext,
                    'quality': '%dx%d' % (f['width'], f['height']),
                }
                format_id = '%s-%s' % (ext, quality)
                viewclip = self._download_json(
                    '%s/video/clips/viewclip' % self._API_BASE, display_id,
                    'Downloading %s viewclip JSON' % format_id, fatal=False,
                    data=json.dumps(clip_post).encode('utf-8'),
                    headers={'Content-Type': 'application/json;charset=utf-8'})

                # Pluralsight tracks multiple sequential calls to ViewClip API and start
                # to return 429 HTTP errors after some time (see
                # https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
                # to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
                # To somewhat reduce the probability of these consequences
                # we will sleep random amount of time before each call to ViewClip.
                self._sleep(
                    random.randint(2, 5), display_id,
                    '%(video_id)s: Waiting for %(timeout)s seconds to avoid throttling')

                if not viewclip:
                    continue

                clip_urls = viewclip.get('urls')
                if not isinstance(clip_urls, list):
                    continue

                for clip_url_data in clip_urls:
                    clip_url = clip_url_data.get('url')
                    if not clip_url:
                        continue
                    cdn = clip_url_data.get('cdn')
                    clip_f = f.copy()
                    clip_f.update({
                        'url': clip_url,
                        'ext': ext,
                        'format_id': '%s-%s' % (format_id, cdn) if cdn else format_id,
                        'quality': quality_key(quality),
                        'source_preference': int_or_none(clip_url_data.get('rank')),
                    })
                    formats.append(clip_f)

        self._sort_formats(formats)

        duration = int_or_none(
            clip.get('duration')) or parse_duration(clip.get('formattedDuration'))

        # TODO: other languages?
        subtitles = self.extract_subtitles(
            author, clip_idx, 'en', name, duration, display_id)

        return {
            'id': clip_id,
            'title': title,
            'duration': duration,
            'creator': author,
            'formats': formats,
            'subtitles': subtitles,
        }


class PluralsightCourseIE(PluralsightBaseIE):
    IE_NAME = 'pluralsight:course'
    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
    _TESTS = [{
        # Free course from Pluralsight Starter Subscription for Microsoft TechNet
        # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
        'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
        'info_dict': {
            'id': 'hosting-sql-server-windows-azure-iaas',
            'title': 'Hosting SQL Server in Microsoft Azure IaaS Fundamentals',
            'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
        },
        'playlist_count': 31,
    }, {
        # available without pluralsight account
        'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
        'only_matching': True,
    }, {
        'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        course_id = self._match_id(url)

        # TODO: PSM cookie

        course = self._download_course(course_id, url, course_id)

        title = course['title']
        course_name = course['name']
        course_data = course['modules']
        description = course.get('description') or course.get('shortDescription')

        entries = []
        for num, module in enumerate(course_data, 1):
            author = module.get('author')
            module_name = module.get('name')
            if not author or not module_name:
                continue
            for clip in module.get('clips', []):
                clip_index = int_or_none(clip.get('index'))
                if clip_index is None:
                    continue
                clip_url = update_url_query(
                    '%s/player' % self._API_BASE, query={
                        'mode': 'live',
                        'course': course_name,
                        'author': author,
                        'name': module_name,
                        'clip': clip_index,
                    })
                entries.append({
                    '_type': 'url_transparent',
                    'url': clip_url,
                    'ie_key': PluralsightIE.ie_key(),
                    'chapter': module.get('title'),
                    'chapter_number': num,
                    'chapter_id': module.get('moduleRef'),
                })

        return self.playlist_result(entries, course_id, title, description)
Commit	Line	Data
483fc223 S	1	from __future__ import unicode_literals
483fc223 S	2
8c3e35dd	3	import collections
483fc223	4	import json
8c3e35dd	5	import os
38eb2968	6	import random
483fc223 S	7
	8	from .common import InfoExtractor
	9	from ..compat import (
	10	compat_str,
483fc223 S	11	compat_urlparse,
	12	)
	13	from ..utils import (
425f3fdf	14	dict_get,
483fc223	15	ExtractorError,
8c3e35dd	16	float_or_none,
483fc223 S	17	int_or_none,
483fc223 S	18	parse_duration,
756926ff	19	qualities,
8c3e35dd	20	srt_subtitles_timecode,
93d0583e	21	try_get,
3d7e3aaa	22	update_url_query,
6e6bc8da	23	urlencode_postdata,
483fc223 S	24	)
	25
	26
563772ed	27	class PluralsightBaseIE(InfoExtractor):
9df6b03c	28	_API_BASE = 'https://app.pluralsight.com'
563772ed	29
93d0583e S	30	def _download_course(self, course_id, url, display_id):
	31	try:
	32	return self._download_course_rpc(course_id, url, display_id)
	33	except ExtractorError:
	34	# Old API fallback
	35	return self._download_json(
	36	'https://app.pluralsight.com/player/user/api/v1/player/payload',
	37	display_id, data=urlencode_postdata({'courseId': course_id}),
	38	headers={'Referer': url})
	39
	40	def _download_course_rpc(self, course_id, url, display_id):
	41	response = self._download_json(
	42	'%s/player/functions/rpc' % self._API_BASE, display_id,
	43	'Downloading course JSON',
	44	data=json.dumps({
	45	'fn': 'bootstrapPlayer',
	46	'payload': {
	47	'courseId': course_id,
	48	},
	49	}).encode('utf-8'),
	50	headers={
	51	'Content-Type': 'application/json;charset=utf-8',
	52	'Referer': url,
	53	})
	54
	55	course = try_get(response, lambda x: x['payload']['course'], dict)
	56	if course:
	57	return course
	58
	59	raise ExtractorError(
	60	'%s said: %s' % (self.IE_NAME, response['error']['message']),
	61	expected=True)
	62
563772ed S	63
563772ed S	64	class PluralsightIE(PluralsightBaseIE):
483fc223	65	IE_NAME = 'pluralsight'
b0dfcab6	66	_VALID_URL = r'https?://(?:(?:www\|app)\.)?pluralsight\.com/(?:training/)?player\?'
c3a227d1	67	_LOGIN_URL = 'https://app.pluralsight.com/id/'
563772ed	68
483fc223 S	69	_NETRC_MACHINE = 'pluralsight'
483fc223 S	70
71bd93b8	71	_TESTS = [{
483fc223 S	72	'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
	73	'md5': '4d458cf5cf4c593788672419a8dd4cf8',
	74	'info_dict': {
	75	'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
	76	'ext': 'mp4',
8e5a7c5e	77	'title': 'Demo Monitoring',
483fc223 S	78	'duration': 338,
	79	},
	80	'skip': 'Requires pluralsight account credentials',
71bd93b8 S	81	}, {
	82	'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
	83	'only_matching': True,
c23e2664 S	84	}, {
	85	# available without pluralsight account
	86	'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
	87	'only_matching': True,
b0dfcab6 S	88	}, {
	89	'url': 'https://app.pluralsight.com/player?course=ccna-intro-networking&author=ross-bagurdes&name=ccna-intro-networking-m06&clip=0',
	90	'only_matching': True,
71bd93b8	91	}]
483fc223 S	92
	93	def _real_initialize(self):
	94	self._login()
	95
	96	def _login(self):
68217024	97	username, password = self._get_login_info()
483fc223	98	if username is None:
c23e2664	99	return
483fc223 S	100
	101	login_page = self._download_webpage(
	102	self._LOGIN_URL, None, 'Downloading login page')
	103
	104	login_form = self._hidden_inputs(login_page)
	105
	106	login_form.update({
244cd042 S	107	'Username': username,
244cd042 S	108	'Password': password,
483fc223 S	109	})
	110
	111	post_url = self._search_regex(
	112	r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
	113	'post url', default=self._LOGIN_URL, group='url')
	114
	115	if not post_url.startswith('http'):
	116	post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
	117
483fc223	118	response = self._download_webpage(
e4d95865	119	post_url, None, 'Logging in',
30317f48 S	120	data=urlencode_postdata(login_form),
30317f48 S	121	headers={'Content-Type': 'application/x-www-form-urlencoded'})
483fc223 S	122
	123	error = self._search_regex(
	124	r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
	125	response, 'error message', default=None)
	126	if error:
	127	raise ExtractorError('Unable to login: %s' % error, expected=True)
	128
7e508ff2	129	if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
9dd5408c S	130	BLOCKED = 'Your account has been blocked due to suspicious activity'
	131	if BLOCKED in response:
	132	raise ExtractorError(
	133	'Unable to login: %s' % BLOCKED, expected=True)
c94427dd S	134	MUST_AGREE = 'To continue using Pluralsight, you must agree to'
	135	if any(p in response for p in (MUST_AGREE, '>Disagree<', '>Agree<')):
	136	raise ExtractorError(
	137	'Unable to login: %s some documents. Go to pluralsight.com, '
	138	'log in and agree with what Pluralsight requires.'
	139	% MUST_AGREE, expected=True)
	140
7e508ff2 S	141	raise ExtractorError('Unable to log in')
7e508ff2 S	142
a3f86160	143	def _get_subtitles(self, author, clip_idx, lang, name, duration, video_id):
8c3e35dd S	144	captions_post = {
8c3e35dd S	145	'a': author,
a3f86160	146	'cn': clip_idx,
8c3e35dd S	147	'lc': lang,
	148	'm': name,
	149	}
	150	captions = self._download_json(
9df6b03c	151	'%s/player/retrieve-captions' % self._API_BASE, video_id,
8c3e35dd S	152	'Downloading captions JSON', 'Unable to download captions JSON',
	153	fatal=False, data=json.dumps(captions_post).encode('utf-8'),
	154	headers={'Content-Type': 'application/json;charset=utf-8'})
	155	if captions:
	156	return {
	157	lang: [{
	158	'ext': 'json',
	159	'data': json.dumps(captions),
	160	}, {
	161	'ext': 'srt',
	162	'data': self._convert_subtitles(duration, captions),
	163	}]
	164	}
	165
	166	@staticmethod
	167	def _convert_subtitles(duration, subs):
	168	srt = ''
425f3fdf S	169	TIME_OFFSET_KEYS = ('displayTimeOffset', 'DisplayTimeOffset')
425f3fdf S	170	TEXT_KEYS = ('text', 'Text')
8c3e35dd S	171	for num, current in enumerate(subs):
8c3e35dd S	172	current = subs[num]
425f3fdf	173	start, text = (
2c8e11b4	174	float_or_none(dict_get(current, TIME_OFFSET_KEYS, skip_false_values=False)),
425f3fdf	175	dict_get(current, TEXT_KEYS))
8c3e35dd S	176	if start is None or text is None:
	177	continue
	178	end = duration if num == len(subs) - 1 else float_or_none(
2c8e11b4	179	dict_get(subs[num + 1], TIME_OFFSET_KEYS, skip_false_values=False))
30317f48 S	180	if end is None:
30317f48 S	181	continue
8c3e35dd S	182	srt += os.linesep.join(
	183	(
	184	'%d' % num,
	185	'%s --> %s' % (
	186	srt_subtitles_timecode(start),
	187	srt_subtitles_timecode(end)),
	188	text,
	189	os.linesep,
	190	))
	191	return srt
	192
483fc223	193	def _real_extract(self, url):
71bd93b8 S	194	qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
	195
	196	author = qs.get('author', [None])[0]
	197	name = qs.get('name', [None])[0]
a3f86160	198	clip_idx = qs.get('clip', [None])[0]
9df6b03c	199	course_name = qs.get('course', [None])[0]
71bd93b8	200
a3f86160	201	if any(not f for f in (author, name, clip_idx, course_name,)):
71bd93b8	202	raise ExtractorError('Invalid URL', expected=True)
483fc223	203
a3f86160	204	display_id = '%s-%s' % (name, clip_idx)
483fc223	205
93d0583e	206	course = self._download_course(course_name, url, display_id)
9df6b03c S	207
9df6b03c S	208	collection = course['modules']
483fc223	209
d212c93d	210	clip = None
483fc223 S	211
483fc223 S	212	for module_ in collection:
02f0da20	213	if name in (module_.get('moduleName'), module_.get('name')):
483fc223 S	214	for clip_ in module_.get('clips', []):
483fc223 S	215	clip_index = clip_.get('clipIndex')
02f0da20 S	216	if clip_index is None:
02f0da20 S	217	clip_index = clip_.get('index')
483fc223 S	218	if clip_index is None:
483fc223 S	219	continue
a3f86160	220	if compat_str(clip_index) == clip_idx:
483fc223 S	221	clip = clip_
	222	break
	223
	224	if not clip:
	225	raise ExtractorError('Unable to resolve clip')
	226
8e5a7c5e	227	title = clip['title']
a3f86160	228	clip_id = clip.get('clipName') or clip.get('name') or clip['clipId']
8c3e35dd	229
483fc223 S	230	QUALITIES = {
	231	'low': {'width': 640, 'height': 480},
	232	'medium': {'width': 848, 'height': 640},
	233	'high': {'width': 1024, 'height': 768},
756926ff	234	'high-widescreen': {'width': 1280, 'height': 720},
483fc223 S	235	}
483fc223 S	236
756926ff S	237	QUALITIES_PREFERENCE = ('low', 'medium', 'high', 'high-widescreen',)
	238	quality_key = qualities(QUALITIES_PREFERENCE)
	239
4c57b485 S	240	AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])
4c57b485 S	241
483fc223	242	ALLOWED_QUALITIES = (
756926ff S	243	AllowedQuality('webm', ['high', ]),
756926ff S	244	AllowedQuality('mp4', ['low', 'medium', 'high', ]),
483fc223 S	245	)
483fc223 S	246
756926ff S	247	# Some courses also offer widescreen resolution for high quality (see
756926ff S	248	# https://github.com/rg3/youtube-dl/issues/7766)
9df6b03c	249	widescreen = course.get('supportsWideScreenVideoFormats') is True
756926ff S	250	best_quality = 'high-widescreen' if widescreen else 'high'
	251	if widescreen:
	252	for allowed_quality in ALLOWED_QUALITIES:
	253	allowed_quality.qualities.append(best_quality)
	254
cf186b77 S	255	# In order to minimize the number of calls to ViewClip API and reduce
cf186b77 S	256	# the probability of being throttled or banned by Pluralsight we will request
0eebf34d	257	# only single format until formats listing was explicitly requested.
4c57b485 S	258	if self._downloader.params.get('listformats', False):
	259	allowed_qualities = ALLOWED_QUALITIES
	260	else:
	261	def guess_allowed_qualities():
	262	req_format = self._downloader.params.get('format') or 'best'
edc70f4a	263	req_format_split = req_format.split('-', 1)
4c57b485 S	264	if len(req_format_split) > 1:
4c57b485 S	265	req_ext, req_quality = req_format_split
fac188c6	266	req_quality = '-'.join(req_quality.split('-')[:2])
4c57b485 S	267	for allowed_quality in ALLOWED_QUALITIES:
	268	if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
	269	return (AllowedQuality(req_ext, (req_quality, )), )
	270	req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
756926ff	271	return (AllowedQuality(req_ext, (best_quality, )), )
4c57b485 S	272	allowed_qualities = guess_allowed_qualities()
4c57b485 S	273
483fc223	274	formats = []
756926ff S	275	for ext, qualities_ in allowed_qualities:
756926ff S	276	for quality in qualities_:
483fc223 S	277	f = QUALITIES[quality].copy()
483fc223 S	278	clip_post = {
9df6b03c S	279	'author': author,
9df6b03c S	280	'includeCaptions': False,
a3f86160	281	'clipIndex': int(clip_idx),
9df6b03c S	282	'courseName': course_name,
	283	'locale': 'en',
	284	'moduleName': name,
	285	'mediaType': ext,
	286	'quality': '%dx%d' % (f['width'], f['height']),
483fc223	287	}
483fc223	288	format_id = '%s-%s' % (ext, quality)
f8ae2c7f	289	viewclip = self._download_json(
9df6b03c	290	'%s/video/clips/viewclip' % self._API_BASE, display_id,
f8ae2c7f	291	'Downloading %s viewclip JSON' % format_id, fatal=False,
30317f48 S	292	data=json.dumps(clip_post).encode('utf-8'),
30317f48 S	293	headers={'Content-Type': 'application/json;charset=utf-8'})
38eb2968 S	294
	295	# Pluralsight tracks multiple sequential calls to ViewClip API and start
	296	# to return 429 HTTP errors after some time (see
	297	# https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
	298	# to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
	299	# To somewhat reduce the probability of these consequences
	300	# we will sleep random amount of time before each call to ViewClip.
	301	self._sleep(
	302	random.randint(2, 5), display_id,
	303	'%(video_id)s: Waiting for %(timeout)s seconds to avoid throttling')
	304
f8ae2c7f	305	if not viewclip:
483fc223	306	continue
f8ae2c7f S	307
	308	clip_urls = viewclip.get('urls')
	309	if not isinstance(clip_urls, list):
	310	continue
	311
	312	for clip_url_data in clip_urls:
	313	clip_url = clip_url_data.get('url')
	314	if not clip_url:
	315	continue
	316	cdn = clip_url_data.get('cdn')
	317	clip_f = f.copy()
	318	clip_f.update({
	319	'url': clip_url,
	320	'ext': ext,
	321	'format_id': '%s-%s' % (format_id, cdn) if cdn else format_id,
	322	'quality': quality_key(quality),
	323	'source_preference': int_or_none(clip_url_data.get('rank')),
	324	})
	325	formats.append(clip_f)
	326
483fc223 S	327	self._sort_formats(formats)
483fc223 S	328
8c3e35dd S	329	duration = int_or_none(
	330	clip.get('duration')) or parse_duration(clip.get('formattedDuration'))
	331
	332	# TODO: other languages?
	333	subtitles = self.extract_subtitles(
a3f86160	334	author, clip_idx, 'en', name, duration, display_id)
483fc223 S	335
483fc223 S	336	return {
a3f86160	337	'id': clip_id,
8c3e35dd S	338	'title': title,
8c3e35dd S	339	'duration': duration,
483fc223	340	'creator': author,
8c3e35dd S	341	'formats': formats,
8c3e35dd S	342	'subtitles': subtitles,
483fc223 S	343	}
	344
	345
563772ed	346	class PluralsightCourseIE(PluralsightBaseIE):
483fc223	347	IE_NAME = 'pluralsight:course'
a5cd0eb8	348	_VALID_URL = r'https?://(?:(?:www\|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
c23e2664	349	_TESTS = [{
483fc223 S	350	# Free course from Pluralsight Starter Subscription for Microsoft TechNet
	351	# https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
	352	'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
	353	'info_dict': {
	354	'id': 'hosting-sql-server-windows-azure-iaas',
	355	'title': 'Hosting SQL Server in Microsoft Azure IaaS Fundamentals',
	356	'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
	357	},
	358	'playlist_count': 31,
c23e2664 S	359	}, {
	360	# available without pluralsight account
	361	'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
	362	'only_matching': True,
a5cd0eb8 S	363	}, {
	364	'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
	365	'only_matching': True,
c23e2664	366	}]
483fc223 S	367
	368	def _real_extract(self, url):
	369	course_id = self._match_id(url)
	370
2b6bda1e S	371	# TODO: PSM cookie
2b6bda1e S	372
93d0583e	373	course = self._download_course(course_id, url, course_id)
483fc223 S	374
483fc223 S	375	title = course['title']
3d7e3aaa S	376	course_name = course['name']
3d7e3aaa S	377	course_data = course['modules']
483fc223 S	378	description = course.get('description') or course.get('shortDescription')
483fc223 S	379
483fc223	380	entries = []
8018028d	381	for num, module in enumerate(course_data, 1):
3d7e3aaa S	382	author = module.get('author')
	383	module_name = module.get('name')
	384	if not author or not module_name:
	385	continue
483fc223	386	for clip in module.get('clips', []):
3d7e3aaa S	387	clip_index = int_or_none(clip.get('index'))
3d7e3aaa S	388	if clip_index is None:
483fc223	389	continue
3d7e3aaa S	390	clip_url = update_url_query(
	391	'%s/player' % self._API_BASE, query={
	392	'mode': 'live',
	393	'course': course_name,
	394	'author': author,
	395	'name': module_name,
	396	'clip': clip_index,
	397	})
8018028d S	398	entries.append({
8018028d S	399	'_type': 'url_transparent',
3d7e3aaa	400	'url': clip_url,
8018028d S	401	'ie_key': PluralsightIE.ie_key(),
	402	'chapter': module.get('title'),
	403	'chapter_number': num,
	404	'chapter_id': module.get('moduleRef'),
	405	})
483fc223	406
483fc223	407	return self.playlist_result(entries, course_id, title, description)