1 from __future__
import unicode_literals
5 from .common
import InfoExtractor
25 class UdemyIE(InfoExtractor
):
32 lecture/view/?\?lectureId=|
33 [^/]+/learn/v4/t/lecture/
37 _LOGIN_URL
= 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
38 _ORIGIN_URL
= 'https://www.udemy.com'
39 _NETRC_MACHINE
= 'udemy'
42 'url': 'https://www.udemy.com/java-tutorial/#/lecture/172757',
43 'md5': '98eda5b657e752cf945d8445e261b5c5',
47 'title': 'Introduction and Installation',
48 'description': 'md5:c0d51f6f21ef4ec65f091055a5eef876',
51 'skip': 'Requires udemy account credentials',
54 'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
55 'only_matching': True,
57 # no url in outputs format entry
58 'url': 'https://www.udemy.com/learn-web-development-complete-step-by-step-guide-to-success/learn/v4/t/lecture/4125812',
59 'only_matching': True,
62 def _extract_course_info(self
, webpage
, video_id
):
63 course
= self
._parse
_json
(
64 unescapeHTML(self
._search
_regex
(
65 r
'ng-init=["\'].*\bcourse
=({.+?}
);', webpage, 'course
', default='{}')),
66 video_id, fatal=False) or {}
67 course_id = course.get('id') or self._search_regex(
68 (r'"
;id"
;\s
*:\s
*(\d
+)', r'data
-course
-id=["\'](\d+)'),
70 return course_id, course.get('title')
72 def _enroll_course(self, base_url, webpage, course_id):
73 def combine_url(base_url, url):
74 return compat_urlparse.urljoin(base_url, url) if not url.startswith('http') else url
76 checkout_url = unescapeHTML(self._search_regex(
77 r'href=(["\'])(?P
<url
>(?
:https?
://(?
:www\
.)?udemy\
.com
)?
/(?
:payment|cart
)/checkout
/.+?
)\
1',
78 webpage, 'checkout url
', group='url
', default=None))
81 'Course
%s is not free
. You have to pay
for it before you can download
. '
82 'Use this URL to confirm purchase
: %s'
83 % (course_id, combine_url(base_url, checkout_url)),
86 enroll_url = unescapeHTML(self._search_regex(
87 r'href
=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/course/subscribe/.+?)\1',
88 webpage, 'enroll url', group='url', default=None))
90 webpage = self._download_webpage(
91 combine_url(base_url, enroll_url),
92 course_id, 'Enrolling in the course',
93 headers={'Referer': base_url})
94 if '>You have enrolled in' in webpage:
95 self.to_screen('%s: Successfully enrolled in the course' % course_id)
97 def _download_lecture(self, course_id, lecture_id):
98 return self._download_json(
99 'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?'
100 % (course_id, lecture_id),
101 lecture_id, 'Downloading lecture JSON', query={
102 'fields[lecture]': 'title,description,view_html,asset',
103 'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
106 def _handle_error(self, response):
107 if not isinstance(response, dict):
109 error = response.get('error')
111 error_str = 'Udemy returned error #%s: %s' % (error.get('code'), error.get('message'))
112 error_data = error.get('data')
114 error_str += ' - %s' % error_data.get('formErrors')
115 raise ExtractorError(error_str, expected=True)
117 def _download_json(self, url_or_request, *args, **kwargs):
119 'X-Udemy-Snail-Case': 'true',
120 'X-Requested-With': 'XMLHttpRequest',
122 for cookie in self._downloader.cookiejar:
123 if cookie.name == 'client_id':
124 headers['X-Udemy-Client-Id'] = cookie.value
125 elif cookie.name == 'access_token':
126 headers['X-Udemy-Bearer-Token'] = cookie.value
127 headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value
129 if isinstance(url_or_request, compat_urllib_request.Request):
130 for header, value in headers.items():
131 url_or_request.add_header(header, value)
133 url_or_request = sanitized_Request(url_or_request, headers=headers)
135 response = super(UdemyIE, self)._download_json(url_or_request, *args, **kwargs)
136 self._handle_error(response)
139 def _real_initialize(self):
143 (username, password) = self._get_login_info()
147 login_popup = self._download_webpage(
148 self._LOGIN_URL, None, 'Downloading login popup')
150 def is_logged(webpage):
151 return any(re.search(p, webpage) for p in (
152 r'href=["\'](?
:https
://www\
.udemy\
.com
)?
/user
/logout
/',
156 if is_logged(login_popup):
159 login_form = self._form_hidden_inputs('login
-form
', login_popup)
163 'password
': password,
166 response = self._download_webpage(
167 self._LOGIN_URL, None, 'Logging
in as %s' % username,
168 data=urlencode_postdata(login_form),
170 'Referer
': self._ORIGIN_URL,
171 'Origin
': self._ORIGIN_URL,
174 if not is_logged(response):
175 error = self._html_search_regex(
176 r'(?s
)<div
[^
>]+class="form-errors[^"]*">(.+?)</div>',
177 response, 'error message', default=None)
179 raise ExtractorError('Unable to login: %s' % error, expected=True)
180 raise ExtractorError('Unable to log in')
182 def _real_extract(self, url):
183 lecture_id = self._match_id(url)
185 webpage = self._download_webpage(url, lecture_id)
187 course_id, _ = self._extract_course_info(webpage, lecture_id)
190 lecture = self._download_lecture(course_id, lecture_id)
191 except ExtractorError as e:
192 # Error could possibly mean we are not enrolled in the course
193 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
194 self._enroll_course(url, webpage, course_id)
195 lecture = self._download_lecture(course_id, lecture_id)
199 title = lecture['title']
200 description = lecture.get('description')
202 asset = lecture['asset']
204 asset_type = asset.get('asset_type') or asset.get('assetType')
205 if asset_type != 'Video':
206 raise ExtractorError(
207 'Lecture %s is not a video' % lecture_id, expected=True)
209 stream_url = asset.get('stream_url') or asset.get('streamUrl')
211 youtube_url = self._search_regex(
212 r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
214 return self.url_result(youtube_url, 'Youtube')
216 video_id = compat_str(asset['id'])
217 thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
218 duration = float_or_none(asset.get('data', {}).get('duration'))
221 automatic_captions = {}
225 def extract_output_format(src, f_id):
227 'url': src.get('url'),
228 'format_id': '%sp' % (src.get('height') or f_id),
229 'width': int_or_none(src.get('width')),
230 'height': int_or_none(src.get('height')),
231 'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
232 'vcodec': src.get('video_codec'),
233 'fps': int_or_none(src.get('frame_rate')),
234 'abr': int_or_none(src.get('audio_bitrate_in_kbps')),
235 'acodec': src.get('audio_codec'),
236 'asr': int_or_none(src.get('audio_sample_rate')),
237 'tbr': int_or_none(src.get('total_bitrate_in_kbps')),
238 'filesize': int_or_none(src.get('file_size_in_bytes')),
241 outputs = asset.get('data', {}).get('outputs')
242 if not isinstance(outputs, dict):
245 def add_output_format_meta(f, key):
246 output = outputs.get(key)
247 if isinstance(output, dict):
248 output_format = extract_output_format(output, key)
249 output_format.update(f)
253 def extract_formats(source_list):
254 if not isinstance(source_list, list):
256 for source in source_list:
257 video_url = source.get('file') or source.get('src')
258 if not video_url or not isinstance(video_url, compat_str):
260 format_id = source.get('label')
263 'format_id': '%sp' % format_id,
264 'height': int_or_none(format_id),
267 # Some videos contain additional metadata (e.g.
268 # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
269 f = add_output_format_meta(f, format_id)
272 def extract_subtitles(track_list):
273 if not isinstance(track_list, list):
275 for track in track_list:
276 if not isinstance(track, dict):
278 if track.get('kind') != 'captions':
280 src = track.get('src')
281 if not src or not isinstance(src, compat_str):
283 lang = track.get('language') or track.get(
284 'srclang') or track.get('label')
285 sub_dict = automatic_captions if track.get(
286 'autogenerated') is True else subtitles
287 sub_dict.setdefault(lang, []).append({
291 download_urls = asset.get('download_urls')
292 if isinstance(download_urls, dict):
293 extract_formats(download_urls.get('Video'))
295 view_html = lecture.get('view_html')
297 view_html_urls = set()
298 for source in re.findall(r'<source[^>]+>', view_html):
299 attributes = extract_attributes(source)
300 src = attributes.get('src')
303 res = attributes.get('data-res')
304 height = int_or_none(res)
305 if src in view_html_urls:
307 view_html_urls.add(src)
308 if attributes.get('type') == 'application/x-mpegURL' or determine_ext(src) == 'm3u8':
309 m3u8_formats = self._extract_m3u8_formats(
310 src, video_id, 'mp4', entry_protocol='m3u8_native',
311 m3u8_id='hls', fatal=False)
312 for f in m3u8_formats:
313 m = re.search(r'/hls_(?P<height>\d{3,4})_(?P<tbr>\d{2,})/', f['url'])
315 if not f.get('height'):
316 f['height'] = int(m.group('height'))
318 f['tbr'] = int(m.group('tbr'))
319 formats.extend(m3u8_formats)
321 formats.append(add_output_format_meta({
323 'format_id': '%dp' % height if height else None,
327 # react rendition since 2017.04.15 (see
328 # https://github.com/rg3/youtube-dl/issues/12744)
329 data = self._parse_json(
331 r'videojs-setup-data=(["\'])(?P
<data
>{.+?}
)\
1', view_html,
332 'setup data
', default='{}', group='data
'), video_id,
333 transform_source=unescapeHTML, fatal=False)
334 if data and isinstance(data, dict):
335 extract_formats(data.get('sources
'))
337 duration = int_or_none(data.get('duration
'))
338 extract_subtitles(data.get('tracks
'))
340 if not subtitles and not automatic_captions:
341 text_tracks = self._parse_json(
343 r'text
-tracks
=(["\'])(?P<data>\[.+?\])\1', view_html,
344 'text tracks', default='{}', group='data'), video_id,
345 transform_source=lambda s: js_to_json(unescapeHTML(s)),
347 extract_subtitles(text_tracks)
349 self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
354 'description': description,
355 'thumbnail': thumbnail,
356 'duration': duration,
358 'subtitles': subtitles,
359 'automatic_captions': automatic_captions,
363 class UdemyCourseIE(UdemyIE):
364 IE_NAME = 'udemy:course'
365 _VALID_URL = r'https?://(?:www\.)?udemy\.com/(?P<id>[^/?#&]+)'
369 def suitable(cls, url):
370 return False if UdemyIE.suitable(url) else super(UdemyCourseIE, cls).suitable(url)
372 def _real_extract(self, url):
373 course_path = self._match_id(url)
375 webpage = self._download_webpage(url, course_path)
377 course_id, title = self._extract_course_info(webpage, course_path)
379 self._enroll_course(url, webpage, course_id)
381 response = self._download_json(
382 'https://www.udemy.com/api-2.0/courses/%s/cached-subscriber-curriculum-items' % course_id,
383 course_id, 'Downloading course curriculum', query={
384 'fields[chapter]': 'title,object_index',
385 'fields[lecture]': 'title,asset',
390 chapter, chapter_number = [None] * 2
391 for entry in response['results']:
392 clazz = entry.get('_class')
393 if clazz == 'lecture':
394 asset = entry.get('asset')
395 if isinstance(asset, dict):
396 asset_type = asset.get('asset_type') or asset.get('assetType')
397 if asset_type != 'Video':
399 lecture_id = entry.get('id')
402 '_type': 'url_transparent',
403 'url': 'https://www.udemy.com/%s/learn/v4/t/lecture/%s' % (course_path, entry['id']),
404 'title': entry.get('title'),
405 'ie_key': UdemyIE.ie_key(),
408 entry['chapter_number'] = chapter_number
410 entry['chapter'] = chapter
411 entries.append(entry)
412 elif clazz == 'chapter':
413 chapter_number = entry.get('object_index')
414 chapter = entry.get('title')
416 return self.playlist_result(entries, course_id, title)