yt_dlp/extractor/linkedin.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 from itertools import zip_longest
   5 import re
   6
   7 from .common import InfoExtractor
   8 from ..utils import (
   9     ExtractorError,
  10     float_or_none,
  11     int_or_none,
  12     srt_subtitles_timecode,
  13     try_get,
  14     urlencode_postdata,
  15     urljoin,
  16 )
  17
  18
  19 class LinkedInLearningBaseIE(InfoExtractor):
  20     _NETRC_MACHINE = 'linkedin'
  21     _LOGIN_URL = 'https://www.linkedin.com/uas/login?trk=learning'
  22
  23     def _call_api(self, course_slug, fields, video_slug=None, resolution=None):
  24         query = {
  25             'courseSlug': course_slug,
  26             'fields': fields,
  27             'q': 'slugs',
  28         }
  29         sub = ''
  30         if video_slug:
  31             query.update({
  32                 'videoSlug': video_slug,
  33                 'resolution': '_%s' % resolution,
  34             })
  35             sub = ' %dp' % resolution
  36         api_url = 'https://www.linkedin.com/learning-api/detailedCourses'
  37         return self._download_json(
  38             api_url, video_slug, 'Downloading%s JSON metadata' % sub, headers={
  39                 'Csrf-Token': self._get_cookies(api_url)['JSESSIONID'].value,
  40             }, query=query)['elements'][0]
  41
  42     def _get_urn_id(self, video_data):
  43         urn = video_data.get('urn')
  44         if urn:
  45             mobj = re.search(r'urn:li:lyndaCourse:\d+,(\d+)', urn)
  46             if mobj:
  47                 return mobj.group(1)
  48
  49     def _get_video_id(self, video_data, course_slug, video_slug):
  50         return self._get_urn_id(video_data) or '%s/%s' % (course_slug, video_slug)
  51
  52     def _real_initialize(self):
  53         email, password = self._get_login_info()
  54         if email is None:
  55             return
  56
  57         login_page = self._download_webpage(
  58             self._LOGIN_URL, None, 'Downloading login page')
  59         action_url = urljoin(self._LOGIN_URL, self._search_regex(
  60             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, 'post url',
  61             default='https://www.linkedin.com/uas/login-submit', group='url'))
  62         data = self._hidden_inputs(login_page)
  63         data.update({
  64             'session_key': email,
  65             'session_password': password,
  66         })
  67         login_submit_page = self._download_webpage(
  68             action_url, None, 'Logging in',
  69             data=urlencode_postdata(data))
  70         error = self._search_regex(
  71             r'<span[^>]+class="error"[^>]*>\s*(.+?)\s*</span>',
  72             login_submit_page, 'error', default=None)
  73         if error:
  74             raise ExtractorError(error, expected=True)
  75
  76
  77 class LinkedInLearningIE(LinkedInLearningBaseIE):
  78     IE_NAME = 'linkedin:learning'
  79     _VALID_URL = r'https?://(?:www\.)?linkedin\.com/learning/(?P<course_slug>[^/]+)/(?P<id>[^/?#]+)'
  80     _TEST = {
  81         'url': 'https://www.linkedin.com/learning/programming-foundations-fundamentals/welcome?autoplay=true',
  82         'md5': 'a1d74422ff0d5e66a792deb996693167',
  83         'info_dict': {
  84             'id': '90426',
  85             'ext': 'mp4',
  86             'title': 'Welcome',
  87             'timestamp': 1430396150.82,
  88             'upload_date': '20150430',
  89         },
  90     }
  91
  92     def json2srt(self, transcript_lines, duration=None):
  93         srt_data = ''
  94         for line, (line_dict, next_dict) in enumerate(zip_longest(transcript_lines, transcript_lines[1:])):
  95             start_time, caption = line_dict['transcriptStartAt'] / 1000, line_dict['caption']
  96             end_time = next_dict['transcriptStartAt'] / 1000 if next_dict else duration or start_time + 1
  97             srt_data += '%d\n%s --> %s\n%s\n\n' % (line + 1, srt_subtitles_timecode(start_time),
  98                                                    srt_subtitles_timecode(end_time),
  99                                                    caption)
 100         return srt_data
 101
 102     def _real_extract(self, url):
 103         course_slug, video_slug = self._match_valid_url(url).groups()
 104
 105         video_data = None
 106         formats = []
 107         for width, height in ((640, 360), (960, 540), (1280, 720)):
 108             video_data = self._call_api(
 109                 course_slug, 'selectedVideo', video_slug, height)['selectedVideo']
 110
 111             video_url_data = video_data.get('url') or {}
 112             progressive_url = video_url_data.get('progressiveUrl')
 113             if progressive_url:
 114                 formats.append({
 115                     'format_id': 'progressive-%dp' % height,
 116                     'url': progressive_url,
 117                     'ext': 'mp4',
 118                     'height': height,
 119                     'width': width,
 120                     'source_preference': 1,
 121                 })
 122
 123         title = video_data['title']
 124
 125         audio_url = video_data.get('audio', {}).get('progressiveUrl')
 126         if audio_url:
 127             formats.append({
 128                 'abr': 64,
 129                 'ext': 'm4a',
 130                 'format_id': 'audio',
 131                 'url': audio_url,
 132                 'vcodec': 'none',
 133             })
 134
 135         streaming_url = video_url_data.get('streamingUrl')
 136         if streaming_url:
 137             formats.extend(self._extract_m3u8_formats(
 138                 streaming_url, video_slug, 'mp4',
 139                 'm3u8_native', m3u8_id='hls', fatal=False))
 140
 141         # It seems like this would be correctly handled by default
 142         # However, unless someone can confirm this, the old
 143         # behaviour is being kept as-is
 144         self._sort_formats(formats, ('res', 'source_preference'))
 145         subtitles = {}
 146         duration = int_or_none(video_data.get('durationInSeconds'))
 147         transcript_lines = try_get(video_data, lambda x: x['transcript']['lines'], expected_type=list)
 148         if transcript_lines:
 149             subtitles['en'] = [{
 150                 'ext': 'srt',
 151                 'data': self.json2srt(transcript_lines, duration)
 152             }]
 153
 154         return {
 155             'id': self._get_video_id(video_data, course_slug, video_slug),
 156             'title': title,
 157             'formats': formats,
 158             'thumbnail': video_data.get('defaultThumbnail'),
 159             'timestamp': float_or_none(video_data.get('publishedOn'), 1000),
 160             'duration': duration,
 161             'subtitles': subtitles,
 162         }
 163
 164
 165 class LinkedInLearningCourseIE(LinkedInLearningBaseIE):
 166     IE_NAME = 'linkedin:learning:course'
 167     _VALID_URL = r'https?://(?:www\.)?linkedin\.com/learning/(?P<id>[^/?#]+)'
 168     _TEST = {
 169         'url': 'https://www.linkedin.com/learning/programming-foundations-fundamentals',
 170         'info_dict': {
 171             'id': 'programming-foundations-fundamentals',
 172             'title': 'Programming Foundations: Fundamentals',
 173             'description': 'md5:76e580b017694eb89dc8e8923fff5c86',
 174         },
 175         'playlist_mincount': 61,
 176     }
 177
 178     @classmethod
 179     def suitable(cls, url):
 180         return False if LinkedInLearningIE.suitable(url) else super(LinkedInLearningCourseIE, cls).suitable(url)
 181
 182     def _real_extract(self, url):
 183         course_slug = self._match_id(url)
 184         course_data = self._call_api(course_slug, 'chapters,description,title')
 185
 186         entries = []
 187         for chapter_number, chapter in enumerate(course_data.get('chapters', []), 1):
 188             chapter_title = chapter.get('title')
 189             chapter_id = self._get_urn_id(chapter)
 190             for video in chapter.get('videos', []):
 191                 video_slug = video.get('slug')
 192                 if not video_slug:
 193                     continue
 194                 entries.append({
 195                     '_type': 'url_transparent',
 196                     'id': self._get_video_id(video, course_slug, video_slug),
 197                     'title': video.get('title'),
 198                     'url': 'https://www.linkedin.com/learning/%s/%s' % (course_slug, video_slug),
 199                     'chapter': chapter_title,
 200                     'chapter_number': chapter_number,
 201                     'chapter_id': chapter_id,
 202                     'ie_key': LinkedInLearningIE.ie_key(),
 203                 })
 204
 205         return self.playlist_result(
 206             entries, course_slug,
 207             course_data.get('title'),
 208             course_data.get('description'))