[yt-dlp.git] / yt_dlp / extractor / safari.py

# coding: utf-8
from __future__ import unicode_literals

import json
import re

from .common import InfoExtractor

from ..compat import (
    compat_parse_qs,
    compat_urlparse,
)
from ..utils import (
    ExtractorError,
    update_url_query,
)


class SafariBaseIE(InfoExtractor):
    _LOGIN_URL = 'https://learning.oreilly.com/accounts/login/'
    _NETRC_MACHINE = 'safari'

    _API_BASE = 'https://learning.oreilly.com/api/v1'
    _API_FORMAT = 'json'

    LOGGED_IN = False

    def _perform_login(self, username, password):
        _, urlh = self._download_webpage_handle(
            'https://learning.oreilly.com/accounts/login-check/', None,
            'Downloading login page')

        def is_logged(urlh):
            return 'learning.oreilly.com/home/' in urlh.geturl()

        if is_logged(urlh):
            self.LOGGED_IN = True
            return

        redirect_url = urlh.geturl()
        parsed_url = compat_urlparse.urlparse(redirect_url)
        qs = compat_parse_qs(parsed_url.query)
        next_uri = compat_urlparse.urljoin(
            'https://api.oreilly.com', qs['next'][0])

        auth, urlh = self._download_json_handle(
            'https://www.oreilly.com/member/auth/login/', None, 'Logging in',
            data=json.dumps({
                'email': username,
                'password': password,
                'redirect_uri': next_uri,
            }).encode(), headers={
                'Content-Type': 'application/json',
                'Referer': redirect_url,
            }, expected_status=400)

        credentials = auth.get('credentials')
        if (not auth.get('logged_in') and not auth.get('redirect_uri')
                and credentials):
            raise ExtractorError(
                'Unable to login: %s' % credentials, expected=True)

        # oreilly serves two same instances of the following cookies
        # in Set-Cookie header and expects first one to be actually set
        for cookie in ('groot_sessionid', 'orm-jwt', 'orm-rt'):
            self._apply_first_set_cookie_header(urlh, cookie)

        _, urlh = self._download_webpage_handle(
            auth.get('redirect_uri') or next_uri, None, 'Completing login',)

        if is_logged(urlh):
            self.LOGGED_IN = True
            return

        raise ExtractorError('Unable to log in')


class SafariIE(SafariBaseIE):
    IE_NAME = 'safari'
    IE_DESC = 'safaribooksonline.com online video'
    _VALID_URL = r'''(?x)
                        https?://
                            (?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/
                            (?:
                                library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?\#&]+)\.html|
                                videos/[^/]+/[^/]+/(?P<reference_id>[^-]+-[^/?\#&]+)
                            )
                    '''

    _TESTS = [{
        'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
        'md5': 'dcc5a425e79f2564148652616af1f2a3',
        'info_dict': {
            'id': '0_qbqx90ic',
            'ext': 'mp4',
            'title': 'Introduction to Hadoop Fundamentals LiveLessons',
            'timestamp': 1437758058,
            'upload_date': '20150724',
            'uploader_id': 'stork',
        },
    }, {
        # non-digits in course id
        'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
        'only_matching': True,
    }, {
        'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
        'only_matching': True,
    }, {
        'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314/9780134217314-PYMC_13_00',
        'only_matching': True,
    }, {
        'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838/9780133392838-00_SeriesIntro',
        'only_matching': True,
    }, {
        'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/00_SeriesIntro.html',
        'only_matching': True,
    }]

    _PARTNER_ID = '1926081'
    _UICONF_ID = '29375172'

    def _real_extract(self, url):
        mobj = self._match_valid_url(url)

        reference_id = mobj.group('reference_id')
        if reference_id:
            video_id = reference_id
            partner_id = self._PARTNER_ID
            ui_id = self._UICONF_ID
        else:
            video_id = '%s-%s' % (mobj.group('course_id'), mobj.group('part'))

            webpage, urlh = self._download_webpage_handle(url, video_id)

            mobj = re.match(self._VALID_URL, urlh.geturl())
            reference_id = mobj.group('reference_id')
            if not reference_id:
                reference_id = self._search_regex(
                    r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
                    webpage, 'kaltura reference id', group='id')
            partner_id = self._search_regex(
                r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
                webpage, 'kaltura widget id', default=self._PARTNER_ID,
                group='id')
            ui_id = self._search_regex(
                r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
                webpage, 'kaltura uiconf id', default=self._UICONF_ID,
                group='id')

        query = {
            'wid': '_%s' % partner_id,
            'uiconf_id': ui_id,
            'flashvars[referenceId]': reference_id,
        }

        if self.LOGGED_IN:
            kaltura_session = self._download_json(
                '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
                video_id, 'Downloading kaltura session JSON',
                'Unable to download kaltura session JSON', fatal=False,
                headers={'Accept': 'application/json'})
            if kaltura_session:
                session = kaltura_session.get('session')
                if session:
                    query['flashvars[ks]'] = session

        return self.url_result(update_url_query(
            'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
            'Kaltura')


class SafariApiIE(SafariBaseIE):
    IE_NAME = 'safari:api'
    _VALID_URL = r'https?://(?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'

    _TESTS = [{
        'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
        'only_matching': True,
    }, {
        'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        mobj = self._match_valid_url(url)
        part = self._download_json(
            url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
            'Downloading part JSON')
        web_url = part['web_url']
        if 'library/view' in web_url:
            web_url = web_url.replace('library/view', 'videos')
            natural_keys = part['natural_key']
            web_url = f'{web_url.rsplit("/", 1)[0]}/{natural_keys[0]}-{natural_keys[1][:-5]}'
        return self.url_result(web_url, SafariIE.ie_key())


class SafariCourseIE(SafariBaseIE):
    IE_NAME = 'safari:course'
    IE_DESC = 'safaribooksonline.com online courses'

    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/
                            (?:
                                library/view/[^/]+|
                                api/v1/book|
                                videos/[^/]+
                            )|
                            techbus\.safaribooksonline\.com
                        )
                        /(?P<id>[^/]+)
                    '''

    _TESTS = [{
        'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
        'info_dict': {
            'id': '9780133392838',
            'title': 'Hadoop Fundamentals LiveLessons',
        },
        'playlist_count': 22,
        'skip': 'Requires safaribooksonline account credentials',
    }, {
        'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
        'only_matching': True,
    }, {
        'url': 'http://techbus.safaribooksonline.com/9780134426365',
        'only_matching': True,
    }, {
        'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314',
        'only_matching': True,
    }, {
        'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838',
        'only_matching': True,
    }, {
        'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return (False if SafariIE.suitable(url) or SafariApiIE.suitable(url)
                else super(SafariCourseIE, cls).suitable(url))

    def _real_extract(self, url):
        course_id = self._match_id(url)

        course_json = self._download_json(
            '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
            course_id, 'Downloading course JSON')

        if 'chapters' not in course_json:
            raise ExtractorError(
                'No chapters found for course %s' % course_id, expected=True)

        entries = [
            self.url_result(chapter, SafariApiIE.ie_key())
            for chapter in course_json['chapters']]

        course_title = course_json['title']

        return self.playlist_result(entries, course_id, course_title)
Commit	Line	Data
dcdb292f	1	# coding: utf-8
32d687f5	2	from __future__ import unicode_literals
32d687f5	3
a9e03736	4	import json
32d687f5	5	import re
32d687f5	6
32d687f5	7	from .common import InfoExtractor
32d687f5	8
a9e03736 S	9	from ..compat import (
a9e03736 S	10	compat_parse_qs,
a9e03736 S	11	compat_urlparse,
a9e03736 S	12	)
32d687f5	13	from ..utils import (
32d687f5	14	ExtractorError,
bcb668de	15	update_url_query,
32d687f5	16	)
	17
	18
	19	class SafariBaseIE(InfoExtractor):
7f41a598	20	_LOGIN_URL = 'https://learning.oreilly.com/accounts/login/'
31c48098 S	21	_NETRC_MACHINE = 'safari'
31c48098 S	22
7f41a598	23	_API_BASE = 'https://learning.oreilly.com/api/v1'
31c48098	24	_API_FORMAT = 'json'
32d687f5	25
	26	LOGGED_IN = False
	27
52efa4b3	28	def _perform_login(self, username, password):
a9e03736 S	29	_, urlh = self._download_webpage_handle(
	30	'https://learning.oreilly.com/accounts/login-check/', None,
	31	'Downloading login page')
4244a13a	32
a9e03736	33	def is_logged(urlh):
7947a1f7	34	return 'learning.oreilly.com/home/' in urlh.geturl()
4244a13a	35
a9e03736	36	if is_logged(urlh):
4244a13a S	37	self.LOGGED_IN = True
4244a13a S	38	return
32d687f5	39
7947a1f7	40	redirect_url = urlh.geturl()
a9e03736 S	41	parsed_url = compat_urlparse.urlparse(redirect_url)
	42	qs = compat_parse_qs(parsed_url.query)
	43	next_uri = compat_urlparse.urljoin(
	44	'https://api.oreilly.com', qs['next'][0])
	45
	46	auth, urlh = self._download_json_handle(
	47	'https://www.oreilly.com/member/auth/login/', None, 'Logging in',
	48	data=json.dumps({
	49	'email': username,
	50	'password': password,
	51	'redirect_uri': next_uri,
	52	}).encode(), headers={
	53	'Content-Type': 'application/json',
	54	'Referer': redirect_url,
	55	}, expected_status=400)
	56
	57	credentials = auth.get('credentials')
	58	if (not auth.get('logged_in') and not auth.get('redirect_uri')
	59	and credentials):
	60	raise ExtractorError(
	61	'Unable to login: %s' % credentials, expected=True)
32d687f5	62
d1fcf255	63	# oreilly serves two same instances of the following cookies
	64	# in Set-Cookie header and expects first one to be actually set
	65	for cookie in ('groot_sessionid', 'orm-jwt', 'orm-rt'):
	66	self._apply_first_set_cookie_header(urlh, cookie)
32d687f5	67
a9e03736 S	68	_, urlh = self._download_webpage_handle(
a9e03736 S	69	auth.get('redirect_uri') or next_uri, None, 'Completing login',)
32d687f5	70
a9e03736 S	71	if is_logged(urlh):
	72	self.LOGGED_IN = True
	73	return
32d687f5	74
a9e03736	75	raise ExtractorError('Unable to log in')
e9c8999e	76
32d687f5	77
	78	class SafariIE(SafariBaseIE):
	79	IE_NAME = 'safari'
	80	IE_DESC = 'safaribooksonline.com online video'
003fe73c S	81	_VALID_URL = r'''(?x)
003fe73c S	82	https?://
a9e03736	83	(?:www\.)?(?:safaribooksonline\|(?:learning\.)?oreilly)\.com/
003fe73c S	84	(?:
	85	library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?\#&]+)\.html\|
	86	videos/[^/]+/[^/]+/(?P<reference_id>[^-]+-[^/?\#&]+)
	87	)
	88	'''
31c48098 S	89
	90	_TESTS = [{
	91	'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
bcb668de	92	'md5': 'dcc5a425e79f2564148652616af1f2a3',
32d687f5	93	'info_dict': {
bcb668de	94	'id': '0_qbqx90ic',
32d687f5	95	'ext': 'mp4',
bcb668de	96	'title': 'Introduction to Hadoop Fundamentals LiveLessons',
	97	'timestamp': 1437758058,
	98	'upload_date': '20150724',
	99	'uploader_id': 'stork',
31c48098	100	},
4fd35ee0 S	101	}, {
	102	# non-digits in course id
	103	'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
	104	'only_matching': True,
697655a7 S	105	}, {
	106	'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
	107	'only_matching': True,
003fe73c S	108	}, {
	109	'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314/9780134217314-PYMC_13_00',
	110	'only_matching': True,
7f41a598 S	111	}, {
	112	'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838/9780133392838-00_SeriesIntro',
	113	'only_matching': True,
a9e03736 S	114	}, {
	115	'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/00_SeriesIntro.html',
	116	'only_matching': True,
31c48098	117	}]
32d687f5	118
003fe73c S	119	_PARTNER_ID = '1926081'
	120	_UICONF_ID = '29375172'
	121
32d687f5	122	def _real_extract(self, url):
5ad28e7f	123	mobj = self._match_valid_url(url)
003fe73c S	124
	125	reference_id = mobj.group('reference_id')
	126	if reference_id:
	127	video_id = reference_id
	128	partner_id = self._PARTNER_ID
	129	ui_id = self._UICONF_ID
	130	else:
	131	video_id = '%s-%s' % (mobj.group('course_id'), mobj.group('part'))
	132
	133	webpage, urlh = self._download_webpage_handle(url, video_id)
	134
	135	mobj = re.match(self._VALID_URL, urlh.geturl())
	136	reference_id = mobj.group('reference_id')
	137	if not reference_id:
	138	reference_id = self._search_regex(
	139	r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
	140	webpage, 'kaltura reference id', group='id')
	141	partner_id = self._search_regex(
	142	r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
	143	webpage, 'kaltura widget id', default=self._PARTNER_ID,
	144	group='id')
	145	ui_id = self._search_regex(
	146	r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
	147	webpage, 'kaltura uiconf id', default=self._UICONF_ID,
	148	group='id')
31c48098	149
73cbd709	150	query = {
bcb668de	151	'wid': '_%s' % partner_id,
	152	'uiconf_id': ui_id,
	153	'flashvars[referenceId]': reference_id,
73cbd709 S	154	}
	155
	156	if self.LOGGED_IN:
	157	kaltura_session = self._download_json(
	158	'%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
3aec7176	159	video_id, 'Downloading kaltura session JSON',
3fdf5731	160	'Unable to download kaltura session JSON', fatal=False,
3fdf5731	161	headers={'Accept': 'application/json'})
73cbd709 S	162	if kaltura_session:
	163	session = kaltura_session.get('session')
	164	if session:
	165	query['flashvars[ks]'] = session
	166
	167	return self.url_result(update_url_query(
	168	'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
	169	'Kaltura')
32d687f5	170
32d687f5	171
3aec7176 S	172	class SafariApiIE(SafariBaseIE):
3aec7176 S	173	IE_NAME = 'safari:api'
a9e03736	174	_VALID_URL = r'https?://(?:www\.)?(?:safaribooksonline\|(?:learning\.)?oreilly)\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
3aec7176	175
697655a7	176	_TESTS = [{
3aec7176 S	177	'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
3aec7176 S	178	'only_matching': True,
697655a7 S	179	}, {
	180	'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
	181	'only_matching': True,
	182	}]
3aec7176 S	183
3aec7176 S	184	def _real_extract(self, url):
5ad28e7f	185	mobj = self._match_valid_url(url)
3aec7176 S	186	part = self._download_json(
	187	url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
	188	'Downloading part JSON')
7738bd32 MKA	189	web_url = part['web_url']
	190	if 'library/view' in web_url:
	191	web_url = web_url.replace('library/view', 'videos')
	192	natural_keys = part['natural_key']
9c1c3ec0	193	web_url = f'{web_url.rsplit("/", 1)[0]}/{natural_keys[0]}-{natural_keys[1][:-5]}'
7738bd32	194	return self.url_result(web_url, SafariIE.ie_key())
3aec7176 S	195
3aec7176 S	196
32d687f5	197	class SafariCourseIE(SafariBaseIE):
	198	IE_NAME = 'safari:course'
	199	IE_DESC = 'safaribooksonline.com online courses'
	200
a26b174c S	201	_VALID_URL = r'''(?x)
	202	https?://
	203	(?:
a9e03736	204	(?:www\.)?(?:safaribooksonline\|(?:learning\.)?oreilly)\.com/
003fe73c S	205	(?:
	206	library/view/[^/]+\|
	207	api/v1/book\|
	208	videos/[^/]+
	209	)\|
a26b174c S	210	techbus\.safaribooksonline\.com
a26b174c S	211	)
003fe73c	212	/(?P<id>[^/]+)
a26b174c	213	'''
32d687f5	214
31c48098 S	215	_TESTS = [{
	216	'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
	217	'info_dict': {
	218	'id': '9780133392838',
	219	'title': 'Hadoop Fundamentals LiveLessons',
	220	},
	221	'playlist_count': 22,
	222	'skip': 'Requires safaribooksonline account credentials',
	223	}, {
	224	'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
	225	'only_matching': True,
a26b174c S	226	}, {
	227	'url': 'http://techbus.safaribooksonline.com/9780134426365',
	228	'only_matching': True,
003fe73c S	229	}, {
	230	'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314',
	231	'only_matching': True,
7f41a598 S	232	}, {
	233	'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838',
	234	'only_matching': True,
a9e03736 S	235	}, {
	236	'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
	237	'only_matching': True,
31c48098	238	}]
32d687f5	239
003fe73c S	240	@classmethod
	241	def suitable(cls, url):
	242	return (False if SafariIE.suitable(url) or SafariApiIE.suitable(url)
	243	else super(SafariCourseIE, cls).suitable(url))
	244
32d687f5	245	def _real_extract(self, url):
31c48098	246	course_id = self._match_id(url)
32d687f5	247
31c48098	248	course_json = self._download_json(
73cbd709	249	'%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
31c48098	250	course_id, 'Downloading course JSON')
32d687f5	251
32d687f5	252	if 'chapters' not in course_json:
31c48098 S	253	raise ExtractorError(
31c48098 S	254	'No chapters found for course %s' % course_id, expected=True)
32d687f5	255
32d687f5	256	entries = [
3aec7176	257	self.url_result(chapter, SafariApiIE.ie_key())
31c48098	258	for chapter in course_json['chapters']]
32d687f5	259
	260	course_title = course_json['title']
	261
	262	return self.playlist_result(entries, course_id, course_title)