yt_dlp/extractor/packtpub.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4
   5 from .common import InfoExtractor
   6 from ..compat import (
   7     # compat_str,
   8     compat_HTTPError,
   9 )
  10 from ..utils import (
  11     clean_html,
  12     ExtractorError,
  13     # remove_end,
  14     str_or_none,
  15     strip_or_none,
  16     unified_timestamp,
  17     # urljoin,
  18 )
  19
  20
  21 class PacktPubBaseIE(InfoExtractor):
  22     # _PACKT_BASE = 'https://www.packtpub.com'
  23     _STATIC_PRODUCTS_BASE = 'https://static.packt-cdn.com/products/'
  24
  25
  26 class PacktPubIE(PacktPubBaseIE):
  27     _VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>[^/]+)/(?P<id>[^/]+)(?:/(?P<display_id>[^/?&#]+))?'
  28
  29     _TESTS = [{
  30         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
  31         'md5': '1e74bd6cfd45d7d07666f4684ef58f70',
  32         'info_dict': {
  33             'id': '20530',
  34             'ext': 'mp4',
  35             'title': 'Project Intro',
  36             'thumbnail': r're:(?i)^https?://.*\.jpg',
  37             'timestamp': 1490918400,
  38             'upload_date': '20170331',
  39         },
  40     }, {
  41         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215/20528/20530/project-intro',
  42         'only_matching': True,
  43     }, {
  44         'url': 'https://subscription.packtpub.com/video/programming/9781838988906/p1/video1_1/business-card-project',
  45         'only_matching': True,
  46     }]
  47     _NETRC_MACHINE = 'packtpub'
  48     _TOKEN = None
  49
  50     def _perform_login(self, username, password):
  51         try:
  52             self._TOKEN = self._download_json(
  53                 'https://services.packtpub.com/auth-v1/users/tokens', None,
  54                 'Downloading Authorization Token', data=json.dumps({
  55                     'username': username,
  56                     'password': password,
  57                 }).encode())['data']['access']
  58         except ExtractorError as e:
  59             if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 404):
  60                 message = self._parse_json(e.cause.read().decode(), None)['message']
  61                 raise ExtractorError(message, expected=True)
  62             raise
  63
  64     def _real_extract(self, url):
  65         course_id, chapter_id, video_id, display_id = self._match_valid_url(url).groups()
  66
  67         headers = {}
  68         if self._TOKEN:
  69             headers['Authorization'] = 'Bearer ' + self._TOKEN
  70         try:
  71             video_url = self._download_json(
  72                 'https://services.packtpub.com/products-v1/products/%s/%s/%s' % (course_id, chapter_id, video_id), video_id,
  73                 'Downloading JSON video', headers=headers)['data']
  74         except ExtractorError as e:
  75             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
  76                 self.raise_login_required('This video is locked')
  77             raise
  78
  79         # TODO: find a better way to avoid duplicating course requests
  80         # metadata = self._download_json(
  81         #     '%s/products/%s/chapters/%s/sections/%s/metadata'
  82         #     % (self._MAPT_REST, course_id, chapter_id, video_id),
  83         #     video_id)['data']
  84
  85         # title = metadata['pageTitle']
  86         # course_title = metadata.get('title')
  87         # if course_title:
  88         #     title = remove_end(title, ' - %s' % course_title)
  89         # timestamp = unified_timestamp(metadata.get('publicationDate'))
  90         # thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
  91
  92         return {
  93             'id': video_id,
  94             'url': video_url,
  95             'title': display_id or video_id,  # title,
  96             # 'thumbnail': thumbnail,
  97             # 'timestamp': timestamp,
  98         }
  99
 100
 101 class PacktPubCourseIE(PacktPubBaseIE):
 102     _VALID_URL = r'(?P<url>https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<id>\d+))'
 103     _TESTS = [{
 104         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215',
 105         'info_dict': {
 106             'id': '9781787122215',
 107             'title': 'Learn Nodejs by building 12 projects [Video]',
 108             'description': 'md5:489da8d953f416e51927b60a1c7db0aa',
 109         },
 110         'playlist_count': 90,
 111     }, {
 112         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215',
 113         'only_matching': True,
 114     }]
 115
 116     @classmethod
 117     def suitable(cls, url):
 118         return False if PacktPubIE.suitable(url) else super(
 119             PacktPubCourseIE, cls).suitable(url)
 120
 121     def _real_extract(self, url):
 122         mobj = self._match_valid_url(url)
 123         url, course_id = mobj.group('url', 'id')
 124
 125         course = self._download_json(
 126             self._STATIC_PRODUCTS_BASE + '%s/toc' % course_id, course_id)
 127         metadata = self._download_json(
 128             self._STATIC_PRODUCTS_BASE + '%s/summary' % course_id,
 129             course_id, fatal=False) or {}
 130
 131         entries = []
 132         for chapter_num, chapter in enumerate(course['chapters'], 1):
 133             chapter_id = str_or_none(chapter.get('id'))
 134             sections = chapter.get('sections')
 135             if not chapter_id or not isinstance(sections, list):
 136                 continue
 137             chapter_info = {
 138                 'chapter': chapter.get('title'),
 139                 'chapter_number': chapter_num,
 140                 'chapter_id': chapter_id,
 141             }
 142             for section in sections:
 143                 section_id = str_or_none(section.get('id'))
 144                 if not section_id or section.get('contentType') != 'video':
 145                     continue
 146                 entry = {
 147                     '_type': 'url_transparent',
 148                     'url': '/'.join([url, chapter_id, section_id]),
 149                     'title': strip_or_none(section.get('title')),
 150                     'description': clean_html(section.get('summary')),
 151                     'thumbnail': metadata.get('coverImage'),
 152                     'timestamp': unified_timestamp(metadata.get('publicationDate')),
 153                     'ie_key': PacktPubIE.ie_key(),
 154                 }
 155                 entry.update(chapter_info)
 156                 entries.append(entry)
 157
 158         return self.playlist_result(
 159             entries, course_id, metadata.get('title'),
 160             clean_html(metadata.get('about')))