yt_dlp/extractor/packtpub.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4
   5 from .common import InfoExtractor
   6 from ..compat import (
   7     # compat_str,
   8     compat_HTTPError,
   9 )
  10 from ..utils import (
  11     clean_html,
  12     ExtractorError,
  13     # remove_end,
  14     str_or_none,
  15     strip_or_none,
  16     unified_timestamp,
  17     # urljoin,
  18 )
  19
  20
  21 class PacktPubBaseIE(InfoExtractor):
  22     # _PACKT_BASE = 'https://www.packtpub.com'
  23     _STATIC_PRODUCTS_BASE = 'https://static.packt-cdn.com/products/'
  24
  25
  26 class PacktPubIE(PacktPubBaseIE):
  27     _VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>[^/]+)/(?P<id>[^/]+)(?:/(?P<display_id>[^/?&#]+))?'
  28
  29     _TESTS = [{
  30         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
  31         'md5': '1e74bd6cfd45d7d07666f4684ef58f70',
  32         'info_dict': {
  33             'id': '20530',
  34             'ext': 'mp4',
  35             'title': 'Project Intro',
  36             'thumbnail': r're:(?i)^https?://.*\.jpg',
  37             'timestamp': 1490918400,
  38             'upload_date': '20170331',
  39         },
  40     }, {
  41         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215/20528/20530/project-intro',
  42         'only_matching': True,
  43     }, {
  44         'url': 'https://subscription.packtpub.com/video/programming/9781838988906/p1/video1_1/business-card-project',
  45         'only_matching': True,
  46     }]
  47     _NETRC_MACHINE = 'packtpub'
  48     _TOKEN = None
  49
  50     def _real_initialize(self):
  51         username, password = self._get_login_info()
  52         if username is None:
  53             return
  54         try:
  55             self._TOKEN = self._download_json(
  56                 'https://services.packtpub.com/auth-v1/users/tokens', None,
  57                 'Downloading Authorization Token', data=json.dumps({
  58                     'username': username,
  59                     'password': password,
  60                 }).encode())['data']['access']
  61         except ExtractorError as e:
  62             if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 404):
  63                 message = self._parse_json(e.cause.read().decode(), None)['message']
  64                 raise ExtractorError(message, expected=True)
  65             raise
  66
  67     def _real_extract(self, url):
  68         course_id, chapter_id, video_id, display_id = self._match_valid_url(url).groups()
  69
  70         headers = {}
  71         if self._TOKEN:
  72             headers['Authorization'] = 'Bearer ' + self._TOKEN
  73         try:
  74             video_url = self._download_json(
  75                 'https://services.packtpub.com/products-v1/products/%s/%s/%s' % (course_id, chapter_id, video_id), video_id,
  76                 'Downloading JSON video', headers=headers)['data']
  77         except ExtractorError as e:
  78             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
  79                 self.raise_login_required('This video is locked')
  80             raise
  81
  82         # TODO: find a better way to avoid duplicating course requests
  83         # metadata = self._download_json(
  84         #     '%s/products/%s/chapters/%s/sections/%s/metadata'
  85         #     % (self._MAPT_REST, course_id, chapter_id, video_id),
  86         #     video_id)['data']
  87
  88         # title = metadata['pageTitle']
  89         # course_title = metadata.get('title')
  90         # if course_title:
  91         #     title = remove_end(title, ' - %s' % course_title)
  92         # timestamp = unified_timestamp(metadata.get('publicationDate'))
  93         # thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
  94
  95         return {
  96             'id': video_id,
  97             'url': video_url,
  98             'title': display_id or video_id,  # title,
  99             # 'thumbnail': thumbnail,
 100             # 'timestamp': timestamp,
 101         }
 102
 103
 104 class PacktPubCourseIE(PacktPubBaseIE):
 105     _VALID_URL = r'(?P<url>https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<id>\d+))'
 106     _TESTS = [{
 107         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215',
 108         'info_dict': {
 109             'id': '9781787122215',
 110             'title': 'Learn Nodejs by building 12 projects [Video]',
 111             'description': 'md5:489da8d953f416e51927b60a1c7db0aa',
 112         },
 113         'playlist_count': 90,
 114     }, {
 115         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215',
 116         'only_matching': True,
 117     }]
 118
 119     @classmethod
 120     def suitable(cls, url):
 121         return False if PacktPubIE.suitable(url) else super(
 122             PacktPubCourseIE, cls).suitable(url)
 123
 124     def _real_extract(self, url):
 125         mobj = self._match_valid_url(url)
 126         url, course_id = mobj.group('url', 'id')
 127
 128         course = self._download_json(
 129             self._STATIC_PRODUCTS_BASE + '%s/toc' % course_id, course_id)
 130         metadata = self._download_json(
 131             self._STATIC_PRODUCTS_BASE + '%s/summary' % course_id,
 132             course_id, fatal=False) or {}
 133
 134         entries = []
 135         for chapter_num, chapter in enumerate(course['chapters'], 1):
 136             chapter_id = str_or_none(chapter.get('id'))
 137             sections = chapter.get('sections')
 138             if not chapter_id or not isinstance(sections, list):
 139                 continue
 140             chapter_info = {
 141                 'chapter': chapter.get('title'),
 142                 'chapter_number': chapter_num,
 143                 'chapter_id': chapter_id,
 144             }
 145             for section in sections:
 146                 section_id = str_or_none(section.get('id'))
 147                 if not section_id or section.get('contentType') != 'video':
 148                     continue
 149                 entry = {
 150                     '_type': 'url_transparent',
 151                     'url': '/'.join([url, chapter_id, section_id]),
 152                     'title': strip_or_none(section.get('title')),
 153                     'description': clean_html(section.get('summary')),
 154                     'thumbnail': metadata.get('coverImage'),
 155                     'timestamp': unified_timestamp(metadata.get('publicationDate')),
 156                     'ie_key': PacktPubIE.ie_key(),
 157                 }
 158                 entry.update(chapter_info)
 159                 entries.append(entry)
 160
 161         return self.playlist_result(
 162             entries, course_id, metadata.get('title'),
 163             clean_html(metadata.get('about')))