yt_dlp/extractor/packtpub.py

   1 import json
   2
   3 from .common import InfoExtractor
   4 from ..networking.exceptions import HTTPError
   5 from ..utils import (
   6     clean_html,
   7     ExtractorError,
   8     # remove_end,
   9     str_or_none,
  10     strip_or_none,
  11     unified_timestamp,
  12     # urljoin,
  13 )
  14
  15
  16 class PacktPubBaseIE(InfoExtractor):
  17     # _PACKT_BASE = 'https://www.packtpub.com'
  18     _STATIC_PRODUCTS_BASE = 'https://static.packt-cdn.com/products/'
  19
  20
  21 class PacktPubIE(PacktPubBaseIE):
  22     _VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>[^/]+)/(?P<id>[^/]+)(?:/(?P<display_id>[^/?&#]+))?'
  23
  24     _TESTS = [{
  25         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
  26         'md5': '1e74bd6cfd45d7d07666f4684ef58f70',
  27         'info_dict': {
  28             'id': '20530',
  29             'ext': 'mp4',
  30             'title': 'Project Intro',
  31             'thumbnail': r're:(?i)^https?://.*\.jpg',
  32             'timestamp': 1490918400,
  33             'upload_date': '20170331',
  34         },
  35     }, {
  36         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215/20528/20530/project-intro',
  37         'only_matching': True,
  38     }, {
  39         'url': 'https://subscription.packtpub.com/video/programming/9781838988906/p1/video1_1/business-card-project',
  40         'only_matching': True,
  41     }]
  42     _NETRC_MACHINE = 'packtpub'
  43     _TOKEN = None
  44
  45     def _perform_login(self, username, password):
  46         try:
  47             self._TOKEN = self._download_json(
  48                 'https://services.packtpub.com/auth-v1/users/tokens', None,
  49                 'Downloading Authorization Token', data=json.dumps({
  50                     'username': username,
  51                     'password': password,
  52                 }).encode())['data']['access']
  53         except ExtractorError as e:
  54             if isinstance(e.cause, HTTPError) and e.cause.status in (400, 401, 404):
  55                 message = self._parse_json(e.cause.response.read().decode(), None)['message']
  56                 raise ExtractorError(message, expected=True)
  57             raise
  58
  59     def _real_extract(self, url):
  60         course_id, chapter_id, video_id, display_id = self._match_valid_url(url).groups()
  61
  62         headers = {}
  63         if self._TOKEN:
  64             headers['Authorization'] = 'Bearer ' + self._TOKEN
  65         try:
  66             video_url = self._download_json(
  67                 'https://services.packtpub.com/products-v1/products/%s/%s/%s' % (course_id, chapter_id, video_id), video_id,
  68                 'Downloading JSON video', headers=headers)['data']
  69         except ExtractorError as e:
  70             if isinstance(e.cause, HTTPError) and e.cause.status == 400:
  71                 self.raise_login_required('This video is locked')
  72             raise
  73
  74         # TODO: find a better way to avoid duplicating course requests
  75         # metadata = self._download_json(
  76         #     '%s/products/%s/chapters/%s/sections/%s/metadata'
  77         #     % (self._MAPT_REST, course_id, chapter_id, video_id),
  78         #     video_id)['data']
  79
  80         # title = metadata['pageTitle']
  81         # course_title = metadata.get('title')
  82         # if course_title:
  83         #     title = remove_end(title, ' - %s' % course_title)
  84         # timestamp = unified_timestamp(metadata.get('publicationDate'))
  85         # thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
  86
  87         return {
  88             'id': video_id,
  89             'url': video_url,
  90             'title': display_id or video_id,  # title,
  91             # 'thumbnail': thumbnail,
  92             # 'timestamp': timestamp,
  93         }
  94
  95
  96 class PacktPubCourseIE(PacktPubBaseIE):
  97     _VALID_URL = r'(?P<url>https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<id>\d+))'
  98     _TESTS = [{
  99         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215',
 100         'info_dict': {
 101             'id': '9781787122215',
 102             'title': 'Learn Nodejs by building 12 projects [Video]',
 103             'description': 'md5:489da8d953f416e51927b60a1c7db0aa',
 104         },
 105         'playlist_count': 90,
 106     }, {
 107         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215',
 108         'only_matching': True,
 109     }]
 110
 111     @classmethod
 112     def suitable(cls, url):
 113         return False if PacktPubIE.suitable(url) else super(
 114             PacktPubCourseIE, cls).suitable(url)
 115
 116     def _real_extract(self, url):
 117         mobj = self._match_valid_url(url)
 118         url, course_id = mobj.group('url', 'id')
 119
 120         course = self._download_json(
 121             self._STATIC_PRODUCTS_BASE + '%s/toc' % course_id, course_id)
 122         metadata = self._download_json(
 123             self._STATIC_PRODUCTS_BASE + '%s/summary' % course_id,
 124             course_id, fatal=False) or {}
 125
 126         entries = []
 127         for chapter_num, chapter in enumerate(course['chapters'], 1):
 128             chapter_id = str_or_none(chapter.get('id'))
 129             sections = chapter.get('sections')
 130             if not chapter_id or not isinstance(sections, list):
 131                 continue
 132             chapter_info = {
 133                 'chapter': chapter.get('title'),
 134                 'chapter_number': chapter_num,
 135                 'chapter_id': chapter_id,
 136             }
 137             for section in sections:
 138                 section_id = str_or_none(section.get('id'))
 139                 if not section_id or section.get('contentType') != 'video':
 140                     continue
 141                 entry = {
 142                     '_type': 'url_transparent',
 143                     'url': '/'.join([url, chapter_id, section_id]),
 144                     'title': strip_or_none(section.get('title')),
 145                     'description': clean_html(section.get('summary')),
 146                     'thumbnail': metadata.get('coverImage'),
 147                     'timestamp': unified_timestamp(metadata.get('publicationDate')),
 148                     'ie_key': PacktPubIE.ie_key(),
 149                 }
 150                 entry.update(chapter_info)
 151                 entries.append(entry)
 152
 153         return self.playlist_result(
 154             entries, course_id, metadata.get('title'),
 155             clean_html(metadata.get('about')))