[yt-dlp.git] / youtube_dl / extractor / pluralsight.py

from __future__ import unicode_literals

import json
import random
import collections

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urllib_parse,
    compat_urllib_request,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    int_or_none,
    parse_duration,
)


class PluralsightBaseIE(InfoExtractor):
    _API_BASE = 'http://app.pluralsight.com'


class PluralsightIE(PluralsightBaseIE):
    IE_NAME = 'pluralsight'
    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
    _LOGIN_URL = 'https://app.pluralsight.com/id/'

    _NETRC_MACHINE = 'pluralsight'

    _TESTS = [{
        'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
        'md5': '4d458cf5cf4c593788672419a8dd4cf8',
        'info_dict': {
            'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
            'ext': 'mp4',
            'title': 'Management of SQL Server - Demo Monitoring',
            'duration': 338,
        },
        'skip': 'Requires pluralsight account credentials',
    }, {
        'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
        'only_matching': True,
    }, {
        # available without pluralsight account
        'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
        'only_matching': True,
    }]

    def _real_initialize(self):
        self._login()

    def _login(self):
        (username, password) = self._get_login_info()
        if username is None:
            return

        login_page = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login page')

        login_form = self._hidden_inputs(login_page)

        login_form.update({
            'Username': username.encode('utf-8'),
            'Password': password.encode('utf-8'),
        })

        post_url = self._search_regex(
            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
            'post url', default=self._LOGIN_URL, group='url')

        if not post_url.startswith('http'):
            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)

        request = compat_urllib_request.Request(
            post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
        request.add_header('Content-Type', 'application/x-www-form-urlencoded')

        response = self._download_webpage(
            request, None, 'Logging in as %s' % username)

        error = self._search_regex(
            r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
            response, 'error message', default=None)
        if error:
            raise ExtractorError('Unable to login: %s' % error, expected=True)

        if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
            raise ExtractorError('Unable to log in')

    def _real_extract(self, url):
        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)

        author = qs.get('author', [None])[0]
        name = qs.get('name', [None])[0]
        clip_id = qs.get('clip', [None])[0]
        course = qs.get('course', [None])[0]

        if any(not f for f in (author, name, clip_id, course,)):
            raise ExtractorError('Invalid URL', expected=True)

        display_id = '%s-%s' % (name, clip_id)

        webpage = self._download_webpage(url, display_id)

        collection = self._parse_json(
            self._search_regex(
                r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
                webpage, 'modules'),
            display_id)

        module, clip = None, None

        for module_ in collection:
            if module_.get('moduleName') == name:
                module = module_
                for clip_ in module_.get('clips', []):
                    clip_index = clip_.get('clipIndex')
                    if clip_index is None:
                        continue
                    if compat_str(clip_index) == clip_id:
                        clip = clip_
                        break

        if not clip:
            raise ExtractorError('Unable to resolve clip')

        QUALITIES = {
            'low': {'width': 640, 'height': 480},
            'medium': {'width': 848, 'height': 640},
            'high': {'width': 1024, 'height': 768},
        }

        AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])

        ALLOWED_QUALITIES = (
            AllowedQuality('webm', ('high',)),
            AllowedQuality('mp4', ('low', 'medium', 'high',)),
        )

        # In order to minimize the number of calls to ViewClip API and reduce
        # the probability of being throttled or banned by Pluralsight we will request
        # only single format until explicit listformats was requested.
        if self._downloader.params.get('listformats', False):
            allowed_qualities = ALLOWED_QUALITIES
        else:
            def guess_allowed_qualities():
                req_format = self._downloader.params.get('format') or 'best'
                req_format_split = req_format.split('-')
                if len(req_format_split) > 1:
                    req_ext, req_quality = req_format_split
                    for allowed_quality in ALLOWED_QUALITIES:
                        if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
                            return (AllowedQuality(req_ext, (req_quality, )), )
                req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
                return (AllowedQuality(req_ext, ('high', )), )
            allowed_qualities = guess_allowed_qualities()

        formats = []
        for ext, qualities in allowed_qualities:
            for quality in qualities:
                f = QUALITIES[quality].copy()
                clip_post = {
                    'a': author,
                    'cap': 'false',
                    'cn': clip_id,
                    'course': course,
                    'lc': 'en',
                    'm': name,
                    'mt': ext,
                    'q': '%dx%d' % (f['width'], f['height']),
                }
                request = compat_urllib_request.Request(
                    '%s/training/Player/ViewClip' % self._API_BASE,
                    json.dumps(clip_post).encode('utf-8'))
                request.add_header('Content-Type', 'application/json;charset=utf-8')
                format_id = '%s-%s' % (ext, quality)
                clip_url = self._download_webpage(
                    request, display_id, 'Downloading %s URL' % format_id, fatal=False)

                # Pluralsight tracks multiple sequential calls to ViewClip API and start
                # to return 429 HTTP errors after some time (see
                # https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
                # to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
                # To somewhat reduce the probability of these consequences
                # we will sleep random amount of time before each call to ViewClip.
                self._sleep(
                    random.randint(2, 5), display_id,
                    '%(video_id)s: Waiting for %(timeout)s seconds to avoid throttling')

                if not clip_url:
                    continue
                f.update({
                    'url': clip_url,
                    'ext': ext,
                    'format_id': format_id,
                })
                formats.append(f)
        self._sort_formats(formats)

        # TODO: captions
        # http://www.pluralsight.com/training/Player/ViewClip + cap = true
        # or
        # http://www.pluralsight.com/training/Player/Captions
        # { a = author, cn = clip_id, lc = end, m = name }

        return {
            'id': clip['clipName'],
            'title': '%s - %s' % (module['title'], clip['title']),
            'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
            'creator': author,
            'formats': formats
        }


class PluralsightCourseIE(PluralsightBaseIE):
    IE_NAME = 'pluralsight:course'
    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
    _TESTS = [{
        # Free course from Pluralsight Starter Subscription for Microsoft TechNet
        # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
        'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
        'info_dict': {
            'id': 'hosting-sql-server-windows-azure-iaas',
            'title': 'Hosting SQL Server in Microsoft Azure IaaS Fundamentals',
            'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
        },
        'playlist_count': 31,
    }, {
        # available without pluralsight account
        'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
        'only_matching': True,
    }, {
        'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        course_id = self._match_id(url)

        # TODO: PSM cookie

        course = self._download_json(
            '%s/data/course/%s' % (self._API_BASE, course_id),
            course_id, 'Downloading course JSON')

        title = course['title']
        description = course.get('description') or course.get('shortDescription')

        course_data = self._download_json(
            '%s/data/course/content/%s' % (self._API_BASE, course_id),
            course_id, 'Downloading course data JSON')

        entries = []
        for module in course_data:
            for clip in module.get('clips', []):
                player_parameters = clip.get('playerParameters')
                if not player_parameters:
                    continue
                entries.append(self.url_result(
                    '%s/training/player?%s' % (self._API_BASE, player_parameters),
                    'Pluralsight'))

        return self.playlist_result(entries, course_id, title, description)
Commit	Line	Data
483fc223 S	1	from __future__ import unicode_literals
483fc223 S	2
483fc223	3	import json
38eb2968	4	import random
4c57b485	5	import collections
483fc223 S	6
	7	from .common import InfoExtractor
	8	from ..compat import (
	9	compat_str,
	10	compat_urllib_parse,
	11	compat_urllib_request,
	12	compat_urlparse,
	13	)
	14	from ..utils import (
	15	ExtractorError,
	16	int_or_none,
	17	parse_duration,
	18	)
	19
	20
563772ed S	21	class PluralsightBaseIE(InfoExtractor):
	22	_API_BASE = 'http://app.pluralsight.com'
	23
	24
	25	class PluralsightIE(PluralsightBaseIE):
483fc223	26	IE_NAME = 'pluralsight'
71bd93b8	27	_VALID_URL = r'https?://(?:(?:www\|app)\.)?pluralsight\.com/training/player\?'
c3a227d1	28	_LOGIN_URL = 'https://app.pluralsight.com/id/'
563772ed	29
483fc223 S	30	_NETRC_MACHINE = 'pluralsight'
483fc223 S	31
71bd93b8	32	_TESTS = [{
483fc223 S	33	'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
	34	'md5': '4d458cf5cf4c593788672419a8dd4cf8',
	35	'info_dict': {
	36	'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
	37	'ext': 'mp4',
	38	'title': 'Management of SQL Server - Demo Monitoring',
	39	'duration': 338,
	40	},
	41	'skip': 'Requires pluralsight account credentials',
71bd93b8 S	42	}, {
	43	'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
	44	'only_matching': True,
c23e2664 S	45	}, {
	46	# available without pluralsight account
	47	'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
	48	'only_matching': True,
71bd93b8	49	}]
483fc223 S	50
	51	def _real_initialize(self):
	52	self._login()
	53
	54	def _login(self):
	55	(username, password) = self._get_login_info()
	56	if username is None:
c23e2664	57	return
483fc223 S	58
	59	login_page = self._download_webpage(
	60	self._LOGIN_URL, None, 'Downloading login page')
	61
	62	login_form = self._hidden_inputs(login_page)
	63
	64	login_form.update({
	65	'Username': username.encode('utf-8'),
	66	'Password': password.encode('utf-8'),
	67	})
	68
	69	post_url = self._search_regex(
	70	r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
	71	'post url', default=self._LOGIN_URL, group='url')
	72
	73	if not post_url.startswith('http'):
	74	post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
	75
	76	request = compat_urllib_request.Request(
	77	post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
	78	request.add_header('Content-Type', 'application/x-www-form-urlencoded')
	79
	80	response = self._download_webpage(
	81	request, None, 'Logging in as %s' % username)
	82
	83	error = self._search_regex(
	84	r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
	85	response, 'error message', default=None)
	86	if error:
	87	raise ExtractorError('Unable to login: %s' % error, expected=True)
	88
7e508ff2 S	89	if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
	90	raise ExtractorError('Unable to log in')
	91
483fc223	92	def _real_extract(self, url):
71bd93b8 S	93	qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
	94
	95	author = qs.get('author', [None])[0]
	96	name = qs.get('name', [None])[0]
	97	clip_id = qs.get('clip', [None])[0]
	98	course = qs.get('course', [None])[0]
	99
	100	if any(not f for f in (author, name, clip_id, course,)):
	101	raise ExtractorError('Invalid URL', expected=True)
483fc223 S	102
	103	display_id = '%s-%s' % (name, clip_id)
	104
	105	webpage = self._download_webpage(url, display_id)
	106
	107	collection = self._parse_json(
	108	self._search_regex(
	109	r'moduleCollection\s:\snew\s+ModuleCollection\((\[.+?\])\s,\s\$rootScope\)',
	110	webpage, 'modules'),
	111	display_id)
	112
	113	module, clip = None, None
	114
	115	for module_ in collection:
	116	if module_.get('moduleName') == name:
	117	module = module_
	118	for clip_ in module_.get('clips', []):
	119	clip_index = clip_.get('clipIndex')
	120	if clip_index is None:
	121	continue
	122	if compat_str(clip_index) == clip_id:
	123	clip = clip_
	124	break
	125
	126	if not clip:
	127	raise ExtractorError('Unable to resolve clip')
	128
	129	QUALITIES = {
	130	'low': {'width': 640, 'height': 480},
	131	'medium': {'width': 848, 'height': 640},
	132	'high': {'width': 1024, 'height': 768},
	133	}
	134
4c57b485 S	135	AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])
4c57b485 S	136
483fc223	137	ALLOWED_QUALITIES = (
4c57b485 S	138	AllowedQuality('webm', ('high',)),
4c57b485 S	139	AllowedQuality('mp4', ('low', 'medium', 'high',)),
483fc223 S	140	)
483fc223 S	141
cf186b77 S	142	# In order to minimize the number of calls to ViewClip API and reduce
	143	# the probability of being throttled or banned by Pluralsight we will request
	144	# only single format until explicit listformats was requested.
4c57b485 S	145	if self._downloader.params.get('listformats', False):
	146	allowed_qualities = ALLOWED_QUALITIES
	147	else:
	148	def guess_allowed_qualities():
	149	req_format = self._downloader.params.get('format') or 'best'
	150	req_format_split = req_format.split('-')
	151	if len(req_format_split) > 1:
	152	req_ext, req_quality = req_format_split
	153	for allowed_quality in ALLOWED_QUALITIES:
	154	if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
	155	return (AllowedQuality(req_ext, (req_quality, )), )
	156	req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
	157	return (AllowedQuality(req_ext, ('high', )), )
	158	allowed_qualities = guess_allowed_qualities()
	159
483fc223	160	formats = []
4c57b485	161	for ext, qualities in allowed_qualities:
483fc223 S	162	for quality in qualities:
	163	f = QUALITIES[quality].copy()
	164	clip_post = {
	165	'a': author,
	166	'cap': 'false',
	167	'cn': clip_id,
	168	'course': course,
	169	'lc': 'en',
	170	'm': name,
	171	'mt': ext,
	172	'q': '%dx%d' % (f['width'], f['height']),
	173	}
	174	request = compat_urllib_request.Request(
0533915a	175	'%s/training/Player/ViewClip' % self._API_BASE,
483fc223 S	176	json.dumps(clip_post).encode('utf-8'))
	177	request.add_header('Content-Type', 'application/json;charset=utf-8')
	178	format_id = '%s-%s' % (ext, quality)
	179	clip_url = self._download_webpage(
	180	request, display_id, 'Downloading %s URL' % format_id, fatal=False)
38eb2968 S	181
	182	# Pluralsight tracks multiple sequential calls to ViewClip API and start
	183	# to return 429 HTTP errors after some time (see
	184	# https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
	185	# to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
	186	# To somewhat reduce the probability of these consequences
	187	# we will sleep random amount of time before each call to ViewClip.
	188	self._sleep(
	189	random.randint(2, 5), display_id,
	190	'%(video_id)s: Waiting for %(timeout)s seconds to avoid throttling')
	191
483fc223 S	192	if not clip_url:
	193	continue
	194	f.update({
	195	'url': clip_url,
	196	'ext': ext,
	197	'format_id': format_id,
	198	})
	199	formats.append(f)
	200	self._sort_formats(formats)
	201
	202	# TODO: captions
	203	# http://www.pluralsight.com/training/Player/ViewClip + cap = true
	204	# or
	205	# http://www.pluralsight.com/training/Player/Captions
	206	# { a = author, cn = clip_id, lc = end, m = name }
	207
	208	return {
	209	'id': clip['clipName'],
	210	'title': '%s - %s' % (module['title'], clip['title']),
	211	'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
	212	'creator': author,
	213	'formats': formats
	214	}
	215
	216
563772ed	217	class PluralsightCourseIE(PluralsightBaseIE):
483fc223	218	IE_NAME = 'pluralsight:course'
a5cd0eb8	219	_VALID_URL = r'https?://(?:(?:www\|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
c23e2664	220	_TESTS = [{
483fc223 S	221	# Free course from Pluralsight Starter Subscription for Microsoft TechNet
	222	# https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
	223	'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
	224	'info_dict': {
	225	'id': 'hosting-sql-server-windows-azure-iaas',
	226	'title': 'Hosting SQL Server in Microsoft Azure IaaS Fundamentals',
	227	'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
	228	},
	229	'playlist_count': 31,
c23e2664 S	230	}, {
	231	# available without pluralsight account
	232	'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
	233	'only_matching': True,
a5cd0eb8 S	234	}, {
	235	'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
	236	'only_matching': True,
c23e2664	237	}]
483fc223 S	238
	239	def _real_extract(self, url):
	240	course_id = self._match_id(url)
	241
2b6bda1e S	242	# TODO: PSM cookie
2b6bda1e S	243
483fc223	244	course = self._download_json(
0533915a	245	'%s/data/course/%s' % (self._API_BASE, course_id),
483fc223 S	246	course_id, 'Downloading course JSON')
	247
	248	title = course['title']
	249	description = course.get('description') or course.get('shortDescription')
	250
	251	course_data = self._download_json(
0533915a	252	'%s/data/course/content/%s' % (self._API_BASE, course_id),
483fc223 S	253	course_id, 'Downloading course data JSON')
483fc223 S	254
483fc223 S	255	entries = []
	256	for module in course_data:
	257	for clip in module.get('clips', []):
483fc223 S	258	player_parameters = clip.get('playerParameters')
	259	if not player_parameters:
	260	continue
	261	entries.append(self.url_result(
0533915a	262	'%s/training/player?%s' % (self._API_BASE, player_parameters),
483fc223 S	263	'Pluralsight'))
483fc223 S	264
483fc223	265	return self.playlist_result(entries, course_id, title, description)