[yt-dlp.git] / yt_dlp / extractor / udemy.py

import re
import urllib.parse

from .common import InfoExtractor
from ..networking import Request
from ..networking.exceptions import HTTPError
from ..utils import (
    ExtractorError,
    determine_ext,
    extract_attributes,
    float_or_none,
    int_or_none,
    js_to_json,
    smuggle_url,
    try_get,
    unescapeHTML,
    unsmuggle_url,
    url_or_none,
    urlencode_postdata,
)


class UdemyIE(InfoExtractor):
    IE_NAME = 'udemy'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:[^/]+\.)?udemy\.com/
                        (?:
                            [^#]+\#/lecture/|
                            lecture/view/?\?lectureId=|
                            [^/]+/learn/v4/t/lecture/
                        )
                        (?P<id>\d+)
                    '''
    _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
    _ORIGIN_URL = 'https://www.udemy.com'
    _NETRC_MACHINE = 'udemy'

    _TESTS = [{
        'url': 'https://www.udemy.com/java-tutorial/#/lecture/172757',
        'md5': '98eda5b657e752cf945d8445e261b5c5',
        'info_dict': {
            'id': '160614',
            'ext': 'mp4',
            'title': 'Introduction and Installation',
            'description': 'md5:c0d51f6f21ef4ec65f091055a5eef876',
            'duration': 579.29,
        },
        'skip': 'Requires udemy account credentials',
    }, {
        # new URL schema
        'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
        'only_matching': True,
    }, {
        # no url in outputs format entry
        'url': 'https://www.udemy.com/learn-web-development-complete-step-by-step-guide-to-success/learn/v4/t/lecture/4125812',
        'only_matching': True,
    }, {
        # only outputs rendition
        'url': 'https://www.udemy.com/how-you-can-help-your-local-community-5-amazing-examples/learn/v4/t/lecture/3225750?start=0',
        'only_matching': True,
    }, {
        'url': 'https://wipro.udemy.com/java-tutorial/#/lecture/172757',
        'only_matching': True,
    }]

    def _extract_course_info(self, webpage, video_id):
        course = self._parse_json(
            unescapeHTML(self._search_regex(
                r'ng-init=["\'].*\bcourse=({.+?})[;"\']',
                webpage, 'course', default='{}')),
            video_id, fatal=False) or {}
        course_id = course.get('id') or self._search_regex(
            [
                r'data-course-id=["\'](\d+)',
                r'&quot;courseId&quot;\s*:\s*(\d+)',
            ], webpage, 'course id')
        return course_id, course.get('title')

    def _enroll_course(self, base_url, webpage, course_id):
        def combine_url(base_url, url):
            return urllib.parse.urljoin(base_url, url) if not url.startswith('http') else url

        checkout_url = unescapeHTML(self._search_regex(
            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/(?:payment|cart)/checkout/.+?)\1',
            webpage, 'checkout url', group='url', default=None))
        if checkout_url:
            raise ExtractorError(
                f'Course {course_id} is not free. You have to pay for it before you can download. '
                f'Use this URL to confirm purchase: {combine_url(base_url, checkout_url)}',
                expected=True)

        enroll_url = unescapeHTML(self._search_regex(
            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/course/subscribe/.+?)\1',
            webpage, 'enroll url', group='url', default=None))
        if enroll_url:
            webpage = self._download_webpage(
                combine_url(base_url, enroll_url),
                course_id, 'Enrolling in the course',
                headers={'Referer': base_url})
            if '>You have enrolled in' in webpage:
                self.to_screen(f'{course_id}: Successfully enrolled in the course')

    def _download_lecture(self, course_id, lecture_id):
        return self._download_json(
            f'https://www.udemy.com/api-2.0/users/me/subscribed-courses/{course_id}/lectures/{lecture_id}?',
            lecture_id, 'Downloading lecture JSON', query={
                'fields[lecture]': 'title,description,view_html,asset',
                'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,stream_urls,captions,data,course_is_drmed',
            })

    def _handle_error(self, response):
        if not isinstance(response, dict):
            return
        error = response.get('error')
        if error:
            error_str = 'Udemy returned error #{}: {}'.format(error.get('code'), error.get('message'))
            error_data = error.get('data')
            if error_data:
                error_str += ' - {}'.format(error_data.get('formErrors'))
            raise ExtractorError(error_str, expected=True)

    def _download_webpage_handle(self, *args, **kwargs):
        headers = kwargs.get('headers', {}).copy()
        headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'
        kwargs['headers'] = headers
        ret = super()._download_webpage_handle(
            *args, **kwargs)
        if not ret:
            return ret
        webpage, _ = ret
        if any(p in webpage for p in (
                '>Please verify you are a human',
                'Access to this page has been denied because we believe you are using automation tools to browse the website',
                '"_pxCaptcha"')):
            raise ExtractorError(
                'Udemy asks you to solve a CAPTCHA. Login with browser, '
                'solve CAPTCHA, then export cookies and pass cookie file to '
                'yt-dlp with --cookies.', expected=True)
        return ret

    def _download_json(self, url_or_request, *args, **kwargs):
        headers = {
            'X-Udemy-Snail-Case': 'true',
            'X-Requested-With': 'XMLHttpRequest',
        }
        for cookie in self.cookiejar:
            if cookie.name == 'client_id':
                headers['X-Udemy-Client-Id'] = cookie.value
            elif cookie.name == 'access_token':
                headers['X-Udemy-Bearer-Token'] = cookie.value
                headers['X-Udemy-Authorization'] = f'Bearer {cookie.value}'

        if isinstance(url_or_request, Request):
            url_or_request.headers.update(headers)
        else:
            url_or_request = Request(url_or_request, headers=headers)

        response = super()._download_json(url_or_request, *args, **kwargs)
        self._handle_error(response)
        return response

    def _perform_login(self, username, password):
        login_popup = self._download_webpage(
            self._LOGIN_URL, None, 'Downloading login popup')

        def is_logged(webpage):
            return any(re.search(p, webpage) for p in (
                r'href=["\'](?:https://www\.udemy\.com)?/user/logout/',
                r'>Logout<'))

        # already logged in
        if is_logged(login_popup):
            return

        login_form = self._form_hidden_inputs('login-form', login_popup)

        login_form.update({
            'email': username,
            'password': password,
        })

        response = self._download_webpage(
            self._LOGIN_URL, None, 'Logging in',
            data=urlencode_postdata(login_form),
            headers={
                'Referer': self._ORIGIN_URL,
                'Origin': self._ORIGIN_URL,
            })

        if not is_logged(response):
            error = self._html_search_regex(
                r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
                response, 'error message', default=None)
            if error:
                raise ExtractorError(f'Unable to login: {error}', expected=True)
            raise ExtractorError('Unable to log in')

    def _real_extract(self, url):
        lecture_id = self._match_id(url)
        course_id = unsmuggle_url(url, {})[1].get('course_id')

        webpage = None
        if not course_id:
            webpage = self._download_webpage(url, lecture_id)
            course_id, _ = self._extract_course_info(webpage, lecture_id)

        try:
            lecture = self._download_lecture(course_id, lecture_id)
        except ExtractorError as e:
            # Error could possibly mean we are not enrolled in the course
            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                webpage = webpage or self._download_webpage(url, lecture_id)
                self._enroll_course(url, webpage, course_id)
                lecture = self._download_lecture(course_id, lecture_id)
            else:
                raise

        title = lecture['title']
        description = lecture.get('description')

        asset = lecture['asset']

        asset_type = asset.get('asset_type') or asset.get('assetType')
        if asset_type != 'Video':
            raise ExtractorError(
                f'Lecture {lecture_id} is not a video', expected=True)

        stream_url = asset.get('stream_url') or asset.get('streamUrl')
        if stream_url:
            youtube_url = self._search_regex(
                r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
            if youtube_url:
                return self.url_result(youtube_url, 'Youtube')

        video_id = str(asset['id'])
        thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
        duration = float_or_none(asset.get('data', {}).get('duration'))

        subtitles = {}
        automatic_captions = {}

        formats = []

        def extract_output_format(src, f_id):
            return {
                'url': src.get('url'),
                'format_id': '%sp' % (src.get('height') or f_id),
                'width': int_or_none(src.get('width')),
                'height': int_or_none(src.get('height')),
                'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
                'vcodec': src.get('video_codec'),
                'fps': int_or_none(src.get('frame_rate')),
                'abr': int_or_none(src.get('audio_bitrate_in_kbps')),
                'acodec': src.get('audio_codec'),
                'asr': int_or_none(src.get('audio_sample_rate')),
                'tbr': int_or_none(src.get('total_bitrate_in_kbps')),
                'filesize': int_or_none(src.get('file_size_in_bytes')),
            }

        outputs = asset.get('data', {}).get('outputs')
        if not isinstance(outputs, dict):
            outputs = {}

        def add_output_format_meta(f, key):
            output = outputs.get(key)
            if isinstance(output, dict):
                output_format = extract_output_format(output, key)
                output_format.update(f)
                return output_format
            return f

        def extract_formats(source_list):
            if not isinstance(source_list, list):
                return
            for source in source_list:
                video_url = url_or_none(source.get('file') or source.get('src'))
                if not video_url:
                    continue
                if source.get('type') == 'application/x-mpegURL' or determine_ext(video_url) == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id='hls', fatal=False))
                    continue
                format_id = source.get('label')
                f = {
                    'url': video_url,
                    'format_id': f'{format_id}p',
                    'height': int_or_none(format_id),
                }
                if format_id:
                    # Some videos contain additional metadata (e.g.
                    # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
                    f = add_output_format_meta(f, format_id)
                formats.append(f)

        def extract_subtitles(track_list):
            if not isinstance(track_list, list):
                return
            for track in track_list:
                if not isinstance(track, dict):
                    continue
                if track.get('kind') != 'captions':
                    continue
                src = url_or_none(track.get('src'))
                if not src:
                    continue
                lang = track.get('language') or track.get(
                    'srclang') or track.get('label')
                sub_dict = automatic_captions if track.get(
                    'autogenerated') is True else subtitles
                sub_dict.setdefault(lang, []).append({
                    'url': src,
                })

        for url_kind in ('download', 'stream'):
            urls = asset.get(f'{url_kind}_urls')
            if isinstance(urls, dict):
                extract_formats(urls.get('Video'))

        captions = asset.get('captions')
        if isinstance(captions, list):
            for cc in captions:
                if not isinstance(cc, dict):
                    continue
                cc_url = url_or_none(cc.get('url'))
                if not cc_url:
                    continue
                lang = try_get(cc, lambda x: x['locale']['locale'], str)
                sub_dict = (automatic_captions if cc.get('source') == 'auto'
                            else subtitles)
                sub_dict.setdefault(lang or 'en', []).append({
                    'url': cc_url,
                })

        view_html = lecture.get('view_html')
        if view_html:
            view_html_urls = set()
            for source in re.findall(r'<source[^>]+>', view_html):
                attributes = extract_attributes(source)
                src = attributes.get('src')
                if not src:
                    continue
                res = attributes.get('data-res')
                height = int_or_none(res)
                if src in view_html_urls:
                    continue
                view_html_urls.add(src)
                if attributes.get('type') == 'application/x-mpegURL' or determine_ext(src) == 'm3u8':
                    m3u8_formats = self._extract_m3u8_formats(
                        src, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id='hls', fatal=False)
                    for f in m3u8_formats:
                        m = re.search(r'/hls_(?P<height>\d{3,4})_(?P<tbr>\d{2,})/', f['url'])
                        if m:
                            if not f.get('height'):
                                f['height'] = int(m.group('height'))
                            if not f.get('tbr'):
                                f['tbr'] = int(m.group('tbr'))
                    formats.extend(m3u8_formats)
                else:
                    formats.append(add_output_format_meta({
                        'url': src,
                        'format_id': f'{height}p' if height else None,
                        'height': height,
                    }, res))

            # react rendition since 2017.04.15 (see
            # https://github.com/ytdl-org/youtube-dl/issues/12744)
            data = self._parse_json(
                self._search_regex(
                    r'videojs-setup-data=(["\'])(?P<data>{.+?})\1', view_html,
                    'setup data', default='{}', group='data'), video_id,
                transform_source=unescapeHTML, fatal=False)
            if data and isinstance(data, dict):
                extract_formats(data.get('sources'))
                if not duration:
                    duration = int_or_none(data.get('duration'))
                extract_subtitles(data.get('tracks'))

            if not subtitles and not automatic_captions:
                text_tracks = self._parse_json(
                    self._search_regex(
                        r'text-tracks=(["\'])(?P<data>\[.+?\])\1', view_html,
                        'text tracks', default='{}', group='data'), video_id,
                    transform_source=lambda s: js_to_json(unescapeHTML(s)),
                    fatal=False)
                extract_subtitles(text_tracks)

        if not formats and outputs:
            for format_id, output in outputs.items():
                f = extract_output_format(output, format_id)
                if f.get('url'):
                    formats.append(f)

        if not formats and asset.get('course_is_drmed'):
            self.report_drm(video_id)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
            'automatic_captions': automatic_captions,
        }


class UdemyCourseIE(UdemyIE):  # XXX: Do not subclass from concrete IE
    IE_NAME = 'udemy:course'
    _VALID_URL = r'https?://(?:[^/]+\.)?udemy\.com/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.udemy.com/java-tutorial/',
        'only_matching': True,
    }, {
        'url': 'https://wipro.udemy.com/java-tutorial/',
        'only_matching': True,
    }]

    @classmethod
    def suitable(cls, url):
        return False if UdemyIE.suitable(url) else super().suitable(url)

    def _real_extract(self, url):
        course_path = self._match_id(url)

        webpage = self._download_webpage(url, course_path)

        course_id, title = self._extract_course_info(webpage, course_path)

        self._enroll_course(url, webpage, course_id)

        response = self._download_json(
            f'https://www.udemy.com/api-2.0/courses/{course_id}/cached-subscriber-curriculum-items',
            course_id, 'Downloading course curriculum', query={
                'fields[chapter]': 'title,object_index',
                'fields[lecture]': 'title,asset',
                'page_size': '1000',
            })

        entries = []
        chapter, chapter_number = [None] * 2
        for entry in response['results']:
            clazz = entry.get('_class')
            if clazz == 'lecture':
                asset = entry.get('asset')
                if isinstance(asset, dict):
                    asset_type = asset.get('asset_type') or asset.get('assetType')
                    if asset_type != 'Video':
                        continue
                lecture_id = entry.get('id')
                if lecture_id:
                    entry = {
                        '_type': 'url_transparent',
                        'url': smuggle_url(
                            f'https://www.udemy.com/{course_path}/learn/v4/t/lecture/{entry["id"]}',
                            {'course_id': course_id}),
                        'title': entry.get('title'),
                        'ie_key': UdemyIE.ie_key(),
                    }
                    if chapter_number:
                        entry['chapter_number'] = chapter_number
                    if chapter:
                        entry['chapter'] = chapter
                    entries.append(entry)
            elif clazz == 'chapter':
                chapter_number = entry.get('object_index')
                chapter = entry.get('title')

        return self.playlist_result(entries, course_id, title)
Commit	Line	Data
efcba804	1	import re
add96eb9	2	import urllib.parse
efcba804	3
e5de3f6c	4	from .common import InfoExtractor
3d2623a8	5	from ..networking import Request
3d2623a8	6	from ..networking.exceptions import HTTPError
1cc79574	7	from ..utils import (
ac668111	8	ExtractorError,
efcba804 S	9	determine_ext,
efcba804 S	10	extract_attributes,
328f82d5	11	float_or_none,
3b35c342	12	int_or_none,
57a38a38	13	js_to_json,
8d1ddb08	14	smuggle_url,
2fbd8635	15	try_get,
17b2d7ca	16	unescapeHTML,
8d1ddb08	17	unsmuggle_url,
3052a30d	18	url_or_none,
6e6bc8da	19	urlencode_postdata,
e5de3f6c S	20	)
	21
	22
	23	class UdemyIE(InfoExtractor):
	24	IE_NAME = 'udemy'
5eb7db4e S	25	_VALID_URL = r'''(?x)
5eb7db4e S	26	https?://
d7d51389	27	(?:[^/]+\.)?udemy\.com/
5eb7db4e S	28	(?:
	29	[^#]+\#/lecture/\|
	30	lecture/view/?\?lectureId=\|
	31	[^/]+/learn/v4/t/lecture/
	32	)
	33	(?P<id>\d+)
	34	'''
dcd4d95c S	35	_LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
dcd4d95c S	36	_ORIGIN_URL = 'https://www.udemy.com'
e5de3f6c S	37	_NETRC_MACHINE = 'udemy'
e5de3f6c S	38
6563837e	39	_TESTS = [{
e5de3f6c S	40	'url': 'https://www.udemy.com/java-tutorial/#/lecture/172757',
	41	'md5': '98eda5b657e752cf945d8445e261b5c5',
	42	'info_dict': {
	43	'id': '160614',
	44	'ext': 'mp4',
	45	'title': 'Introduction and Installation',
	46	'description': 'md5:c0d51f6f21ef4ec65f091055a5eef876',
	47	'duration': 579.29,
	48	},
	49	'skip': 'Requires udemy account credentials',
5eb7db4e S	50	}, {
	51	# new URL schema
	52	'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
	53	'only_matching': True,
b7f87493 S	54	}, {
	55	# no url in outputs format entry
	56	'url': 'https://www.udemy.com/learn-web-development-complete-step-by-step-guide-to-success/learn/v4/t/lecture/4125812',
	57	'only_matching': True,
85139634 S	58	}, {
	59	# only outputs rendition
	60	'url': 'https://www.udemy.com/how-you-can-help-your-local-community-5-amazing-examples/learn/v4/t/lecture/3225750?start=0',
	61	'only_matching': True,
d7d51389 S	62	}, {
	63	'url': 'https://wipro.udemy.com/java-tutorial/#/lecture/172757',
	64	'only_matching': True,
6563837e	65	}]
e5de3f6c	66
81da8cbc S	67	def _extract_course_info(self, webpage, video_id):
	68	course = self._parse_json(
	69	unescapeHTML(self._search_regex(
6f1ec339 S	70	r'ng-init=["\'].*\bcourse=({.+?})[;"\']',
6f1ec339 S	71	webpage, 'course', default='{}')),
81da8cbc S	72	video_id, fatal=False) or {}
81da8cbc S	73	course_id = course.get('id') or self._search_regex(
5de53878 RA	74	[
5de53878 RA	75	r'data-course-id=["\'](\d+)',
add96eb9	76	r'"courseId"\s:\s(\d+)',
5de53878	77	], webpage, 'course id')
81da8cbc S	78	return course_id, course.get('title')
81da8cbc S	79
ff9d5d09	80	def _enroll_course(self, base_url, webpage, course_id):
b24ab3e3	81	def combine_url(base_url, url):
add96eb9	82	return urllib.parse.urljoin(base_url, url) if not url.startswith('http') else url
b24ab3e3	83
17b2d7ca	84	checkout_url = unescapeHTML(self._search_regex(
5f5c7b92	85	r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/(?:payment\|cart)/checkout/.+?)\1',
17b2d7ca S	86	webpage, 'checkout url', group='url', default=None))
	87	if checkout_url:
	88	raise ExtractorError(
add96eb9	89	f'Course {course_id} is not free. You have to pay for it before you can download. '
add96eb9	90	f'Use this URL to confirm purchase: {combine_url(base_url, checkout_url)}',
b24ab3e3	91	expected=True)
17b2d7ca S	92
17b2d7ca S	93	enroll_url = unescapeHTML(self._search_regex(
ff9d5d09	94	r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/course/subscribe/.+?)\1',
17b2d7ca S	95	webpage, 'enroll url', group='url', default=None))
17b2d7ca S	96	if enroll_url:
b24ab3e3 S	97	webpage = self._download_webpage(
b24ab3e3 S	98	combine_url(base_url, enroll_url),
aabdc83d S	99	course_id, 'Enrolling in the course',
aabdc83d S	100	headers={'Referer': base_url})
17b2d7ca	101	if '>You have enrolled in' in webpage:
add96eb9	102	self.to_screen(f'{course_id}: Successfully enrolled in the course')
328f82d5 S	103
	104	def _download_lecture(self, course_id, lecture_id):
	105	return self._download_json(
add96eb9	106	f'https://www.udemy.com/api-2.0/users/me/subscribed-courses/{course_id}/lectures/{lecture_id}?',
75b81df3 S	107	lecture_id, 'Downloading lecture JSON', query={
75b81df3 S	108	'fields[lecture]': 'title,description,view_html,asset',
8d1ddb08	109	'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,stream_urls,captions,data,course_is_drmed',
75b81df3	110	})
328f82d5	111
e5de3f6c S	112	def _handle_error(self, response):
	113	if not isinstance(response, dict):
	114	return
	115	error = response.get('error')
	116	if error:
add96eb9	117	error_str = 'Udemy returned error #{}: {}'.format(error.get('code'), error.get('message'))
e5de3f6c S	118	error_data = error.get('data')
e5de3f6c S	119	if error_data:
add96eb9	120	error_str += ' - {}'.format(error_data.get('formErrors'))
e5de3f6c S	121	raise ExtractorError(error_str, expected=True)
e5de3f6c S	122
e5eadfa8	123	def _download_webpage_handle(self, args, *kwargs):
c8f6ab8c	124	headers = kwargs.get('headers', {}).copy()
ae65c93a	125	headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'
c8f6ab8c	126	kwargs['headers'] = headers
add96eb9	127	ret = super()._download_webpage_handle(
f9934b96	128	args, *kwargs)
ae65c93a S	129	if not ret:
	130	return ret
	131	webpage, _ = ret
	132	if any(p in webpage for p in (
	133	'>Please verify you are a human',
	134	'Access to this page has been denied because we believe you are using automation tools to browse the website',
	135	'"_pxCaptcha"')):
	136	raise ExtractorError(
	137	'Udemy asks you to solve a CAPTCHA. Login with browser, '
	138	'solve CAPTCHA, then export cookies and pass cookie file to '
7a5c1cfe	139	'yt-dlp with --cookies.', expected=True)
ae65c93a	140	return ret
abe8766c	141
81da8cbc	142	def _download_json(self, url_or_request, args, *kwargs):
e2937118 S	143	headers = {
	144	'X-Udemy-Snail-Case': 'true',
	145	'X-Requested-With': 'XMLHttpRequest',
	146	}
9809740b	147	for cookie in self.cookiejar:
e2937118 S	148	if cookie.name == 'client_id':
	149	headers['X-Udemy-Client-Id'] = cookie.value
	150	elif cookie.name == 'access_token':
	151	headers['X-Udemy-Bearer-Token'] = cookie.value
add96eb9	152	headers['X-Udemy-Authorization'] = f'Bearer {cookie.value}'
61c0663c	153
3d2623a8	154	if isinstance(url_or_request, Request):
3d2623a8	155	url_or_request.headers.update(headers)
61c0663c	156	else:
3d2623a8	157	url_or_request = Request(url_or_request, headers=headers)
61c0663c	158
add96eb9	159	response = super()._download_json(url_or_request, args, *kwargs)
61c0663c S	160	self._handle_error(response)
61c0663c S	161	return response
e2937118	162
52efa4b3	163	def _perform_login(self, username, password):
e5de3f6c	164	login_popup = self._download_webpage(
dcd4d95c	165	self._LOGIN_URL, None, 'Downloading login popup')
e5de3f6c	166
d609edf4	167	def is_logged(webpage):
807cf7b0 S	168	return any(re.search(p, webpage) for p in (
	169	r'href=["\'](?:https://www\.udemy\.com)?/user/logout/',
	170	r'>Logout<'))
d609edf4 S	171
	172	# already logged in
	173	if is_logged(login_popup):
e5de3f6c S	174	return
e5de3f6c S	175
dcd4d95c	176	login_form = self._form_hidden_inputs('login-form', login_popup)
e5de3f6c	177
dcd4d95c	178	login_form.update({
a3373823 S	179	'email': username,
a3373823 S	180	'password': password,
dcd4d95c S	181	})
dcd4d95c S	182
dcd4d95c	183	response = self._download_webpage(
e4d95865	184	self._LOGIN_URL, None, 'Logging in',
75b81df3 S	185	data=urlencode_postdata(login_form),
	186	headers={
	187	'Referer': self._ORIGIN_URL,
	188	'Origin': self._ORIGIN_URL,
	189	})
e5de3f6c	190
d609edf4	191	if not is_logged(response):
dcd4d95c S	192	error = self._html_search_regex(
	193	r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
	194	response, 'error message', default=None)
	195	if error:
add96eb9	196	raise ExtractorError(f'Unable to login: {error}', expected=True)
e5de3f6c S	197	raise ExtractorError('Unable to log in')
	198
	199	def _real_extract(self, url):
810fb84d	200	lecture_id = self._match_id(url)
8d1ddb08	201	course_id = unsmuggle_url(url, {})[1].get('course_id')
e5de3f6c	202
8d1ddb08	203	webpage = None
	204	if not course_id:
	205	webpage = self._download_webpage(url, lecture_id)
	206	course_id, _ = self._extract_course_info(webpage, lecture_id)
e5de3f6c	207
328f82d5 S	208	try:
	209	lecture = self._download_lecture(course_id, lecture_id)
	210	except ExtractorError as e:
	211	# Error could possibly mean we are not enrolled in the course
3d2623a8	212	if isinstance(e.cause, HTTPError) and e.cause.status == 403:
8d1ddb08	213	webpage = webpage or self._download_webpage(url, lecture_id)
ff9d5d09	214	self._enroll_course(url, webpage, course_id)
3092fc40	215	lecture = self._download_lecture(course_id, lecture_id)
328f82d5 S	216	else:
	217	raise
	218
	219	title = lecture['title']
	220	description = lecture.get('description')
	221
	222	asset = lecture['asset']
	223
03caa463	224	asset_type = asset.get('asset_type') or asset.get('assetType')
e2937118 S	225	if asset_type != 'Video':
e2937118 S	226	raise ExtractorError(
add96eb9	227	f'Lecture {lecture_id} is not a video', expected=True)
e5de3f6c	228
03caa463	229	stream_url = asset.get('stream_url') or asset.get('streamUrl')
328f82d5 S	230	if stream_url:
	231	youtube_url = self._search_regex(
	232	r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
	233	if youtube_url:
	234	return self.url_result(youtube_url, 'Youtube')
e5de3f6c	235
add96eb9	236	video_id = str(asset['id'])
03caa463	237	thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
328f82d5	238	duration = float_or_none(asset.get('data', {}).get('duration'))
328f82d5	239
3dfceb28 S	240	subtitles = {}
	241	automatic_captions = {}
	242
328f82d5	243	formats = []
f0e83681	244
3dfceb28	245	def extract_output_format(src, f_id):
f0e83681	246	return {
b7f87493	247	'url': src.get('url'),
3dfceb28	248	'format_id': '%sp' % (src.get('height') or f_id),
f0e83681 S	249	'width': int_or_none(src.get('width')),
	250	'height': int_or_none(src.get('height')),
	251	'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
	252	'vcodec': src.get('video_codec'),
	253	'fps': int_or_none(src.get('frame_rate')),
	254	'abr': int_or_none(src.get('audio_bitrate_in_kbps')),
	255	'acodec': src.get('audio_codec'),
	256	'asr': int_or_none(src.get('audio_sample_rate')),
	257	'tbr': int_or_none(src.get('total_bitrate_in_kbps')),
	258	'filesize': int_or_none(src.get('file_size_in_bytes')),
328f82d5	259	}
f0e83681 S	260
	261	outputs = asset.get('data', {}).get('outputs')
	262	if not isinstance(outputs, dict):
	263	outputs = {}
	264
af4116f4	265	def add_output_format_meta(f, key):
62f55aa6 S	266	output = outputs.get(key)
62f55aa6 S	267	if isinstance(output, dict):
3dfceb28	268	output_format = extract_output_format(output, key)
62f55aa6 S	269	output_format.update(f)
62f55aa6 S	270	return output_format
af4116f4	271	return f
62f55aa6	272
3dfceb28 S	273	def extract_formats(source_list):
	274	if not isinstance(source_list, list):
	275	return
	276	for source in source_list:
3052a30d S	277	video_url = url_or_none(source.get('file') or source.get('src'))
3052a30d S	278	if not video_url:
3dfceb28	279	continue
913b61ee S	280	if source.get('type') == 'application/x-mpegURL' or determine_ext(video_url) == 'm3u8':
	281	formats.extend(self._extract_m3u8_formats(
	282	video_url, video_id, 'mp4', entry_protocol='m3u8_native',
	283	m3u8_id='hls', fatal=False))
	284	continue
3dfceb28 S	285	format_id = source.get('label')
	286	f = {
	287	'url': video_url,
add96eb9	288	'format_id': f'{format_id}p',
3dfceb28 S	289	'height': int_or_none(format_id),
	290	}
	291	if format_id:
	292	# Some videos contain additional metadata (e.g.
	293	# https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
	294	f = add_output_format_meta(f, format_id)
	295	formats.append(f)
	296
57a38a38 S	297	def extract_subtitles(track_list):
	298	if not isinstance(track_list, list):
	299	return
	300	for track in track_list:
	301	if not isinstance(track, dict):
	302	continue
	303	if track.get('kind') != 'captions':
	304	continue
3052a30d S	305	src = url_or_none(track.get('src'))
3052a30d S	306	if not src:
57a38a38 S	307	continue
	308	lang = track.get('language') or track.get(
	309	'srclang') or track.get('label')
	310	sub_dict = automatic_captions if track.get(
	311	'autogenerated') is True else subtitles
	312	sub_dict.setdefault(lang, []).append({
	313	'url': src,
	314	})
	315
0ce76801	316	for url_kind in ('download', 'stream'):
add96eb9	317	urls = asset.get(f'{url_kind}_urls')
0ce76801 S	318	if isinstance(urls, dict):
0ce76801 S	319	extract_formats(urls.get('Video'))
3b35c342	320
2fbd8635 S	321	captions = asset.get('captions')
	322	if isinstance(captions, list):
	323	for cc in captions:
	324	if not isinstance(cc, dict):
	325	continue
3052a30d S	326	cc_url = url_or_none(cc.get('url'))
3052a30d S	327	if not cc_url:
2fbd8635	328	continue
add96eb9	329	lang = try_get(cc, lambda x: x['locale']['locale'], str)
2fbd8635 S	330	sub_dict = (automatic_captions if cc.get('source') == 'auto'
	331	else subtitles)
	332	sub_dict.setdefault(lang or 'en', []).append({
	333	'url': cc_url,
	334	})
	335
efcba804 S	336	view_html = lecture.get('view_html')
	337	if view_html:
	338	view_html_urls = set()
	339	for source in re.findall(r'<source[^>]+>', view_html):
	340	attributes = extract_attributes(source)
	341	src = attributes.get('src')
	342	if not src:
	343	continue
	344	res = attributes.get('data-res')
	345	height = int_or_none(res)
	346	if src in view_html_urls:
	347	continue
	348	view_html_urls.add(src)
	349	if attributes.get('type') == 'application/x-mpegURL' or determine_ext(src) == 'm3u8':
	350	m3u8_formats = self._extract_m3u8_formats(
	351	src, video_id, 'mp4', entry_protocol='m3u8_native',
	352	m3u8_id='hls', fatal=False)
	353	for f in m3u8_formats:
	354	m = re.search(r'/hls_(?P<height>\d{3,4})_(?P<tbr>\d{2,})/', f['url'])
	355	if m:
	356	if not f.get('height'):
	357	f['height'] = int(m.group('height'))
	358	if not f.get('tbr'):
	359	f['tbr'] = int(m.group('tbr'))
	360	formats.extend(m3u8_formats)
	361	else:
62f55aa6	362	formats.append(add_output_format_meta({
efcba804	363	'url': src,
add96eb9	364	'format_id': f'{height}p' if height else None,
efcba804	365	'height': height,
af4116f4	366	}, res))
efcba804	367
3dfceb28	368	# react rendition since 2017.04.15 (see
067aa17e	369	# https://github.com/ytdl-org/youtube-dl/issues/12744)
3dfceb28 S	370	data = self._parse_json(
	371	self._search_regex(
	372	r'videojs-setup-data=(["\'])(?P<data>{.+?})\1', view_html,
	373	'setup data', default='{}', group='data'), video_id,
	374	transform_source=unescapeHTML, fatal=False)
	375	if data and isinstance(data, dict):
	376	extract_formats(data.get('sources'))
	377	if not duration:
	378	duration = int_or_none(data.get('duration'))
57a38a38 S	379	extract_subtitles(data.get('tracks'))
	380
	381	if not subtitles and not automatic_captions:
	382	text_tracks = self._parse_json(
	383	self._search_regex(
	384	r'text-tracks=(["\'])(?P<data>\[.+?\])\1', view_html,
	385	'text tracks', default='{}', group='data'), video_id,
	386	transform_source=lambda s: js_to_json(unescapeHTML(s)),
	387	fatal=False)
	388	extract_subtitles(text_tracks)
3dfceb28	389
85139634 S	390	if not formats and outputs:
	391	for format_id, output in outputs.items():
	392	f = extract_output_format(output, format_id)
	393	if f.get('url'):
	394	formats.append(f)
	395
8d1ddb08	396	if not formats and asset.get('course_is_drmed'):
	397	self.report_drm(video_id)
	398
e5de3f6c S	399	return {
	400	'id': video_id,
	401	'title': title,
	402	'description': description,
	403	'thumbnail': thumbnail,
	404	'duration': duration,
3dfceb28 S	405	'formats': formats,
	406	'subtitles': subtitles,
	407	'automatic_captions': automatic_captions,
e5de3f6c S	408	}
	409
	410
6368e2e6	411	class UdemyCourseIE(UdemyIE): # XXX: Do not subclass from concrete IE
e5de3f6c	412	IE_NAME = 'udemy:course'
d7d51389 S	413	_VALID_URL = r'https?://(?:[^/]+\.)?udemy\.com/(?P<id>[^/?#&]+)'
	414	_TESTS = [{
	415	'url': 'https://www.udemy.com/java-tutorial/',
	416	'only_matching': True,
	417	}, {
	418	'url': 'https://wipro.udemy.com/java-tutorial/',
	419	'only_matching': True,
	420	}]
e5de3f6c S	421
	422	@classmethod
	423	def suitable(cls, url):
add96eb9	424	return False if UdemyIE.suitable(url) else super().suitable(url)
e5de3f6c S	425
e5de3f6c S	426	def _real_extract(self, url):
328f82d5 S	427	course_path = self._match_id(url)
	428
	429	webpage = self._download_webpage(url, course_path)
e5de3f6c	430
81da8cbc	431	course_id, title = self._extract_course_info(webpage, course_path)
e5de3f6c	432
ff9d5d09	433	self._enroll_course(url, webpage, course_id)
e5de3f6c	434
6bb46007	435	response = self._download_json(
add96eb9	436	f'https://www.udemy.com/api-2.0/courses/{course_id}/cached-subscriber-curriculum-items',
6bb46007	437	course_id, 'Downloading course curriculum', query={
81da8cbc	438	'fields[chapter]': 'title,object_index',
03caa463	439	'fields[lecture]': 'title,asset',
81da8cbc S	440	'page_size': '1000',
	441	})
	442
4d402db5	443	entries = []
81da8cbc S	444	chapter, chapter_number = [None] * 2
	445	for entry in response['results']:
	446	clazz = entry.get('_class')
	447	if clazz == 'lecture':
03caa463 S	448	asset = entry.get('asset')
	449	if isinstance(asset, dict):
	450	asset_type = asset.get('asset_type') or asset.get('assetType')
	451	if asset_type != 'Video':
	452	continue
81da8cbc S	453	lecture_id = entry.get('id')
81da8cbc S	454	if lecture_id:
4d402db5 S	455	entry = {
4d402db5 S	456	'_type': 'url_transparent',
8d1ddb08	457	'url': smuggle_url(
	458	f'https://www.udemy.com/{course_path}/learn/v4/t/lecture/{entry["id"]}',
	459	{'course_id': course_id}),
81da8cbc	460	'title': entry.get('title'),
4d402db5 S	461	'ie_key': UdemyIE.ie_key(),
4d402db5 S	462	}
5bafcf65 S	463	if chapter_number:
5bafcf65 S	464	entry['chapter_number'] = chapter_number
4d402db5 S	465	if chapter:
	466	entry['chapter'] = chapter
	467	entries.append(entry)
81da8cbc S	468	elif clazz == 'chapter':
	469	chapter_number = entry.get('object_index')
	470	chapter = entry.get('title')
e5de3f6c	471
81da8cbc	472	return self.playlist_result(entries, course_id, title)