youtube_dl/extractor/dramafever.py

   1 # encoding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import itertools
   5
   6 from .common import InfoExtractor
   7 from ..compat import (
   8     compat_HTTPError,
   9     compat_urllib_parse,
  10     compat_urllib_request,
  11     compat_urlparse,
  12 )
  13 from ..utils import (
  14     ExtractorError,
  15     clean_html,
  16     determine_ext,
  17     int_or_none,
  18     parse_iso8601,
  19 )
  20
  21
  22 class DramaFeverBaseIE(InfoExtractor):
  23     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
  24     _NETRC_MACHINE = 'dramafever'
  25
  26     def _real_initialize(self):
  27         self._login()
  28
  29     def _login(self):
  30         (username, password) = self._get_login_info()
  31         if username is None:
  32             return
  33
  34         login_form = {
  35             'username': username,
  36             'password': password,
  37         }
  38
  39         request = compat_urllib_request.Request(
  40             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
  41         response = self._download_webpage(
  42             request, None, 'Logging in as %s' % username)
  43
  44         if all(logout_pattern not in response
  45                for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
  46             error = self._html_search_regex(
  47                 r'(?s)class="hidden-xs prompt"[^>]*>(.+?)<',
  48                 response, 'error message', default=None)
  49             if error:
  50                 raise ExtractorError('Unable to login: %s' % error, expected=True)
  51             raise ExtractorError('Unable to log in')
  52
  53
  54 class DramaFeverIE(DramaFeverBaseIE):
  55     IE_NAME = 'dramafever'
  56     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
  57     _TEST = {
  58         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
  59         'info_dict': {
  60             'id': '4512.1',
  61             'ext': 'flv',
  62             'title': 'Cooking with Shin 4512.1',
  63             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
  64             'thumbnail': 're:^https?://.*\.jpg',
  65             'timestamp': 1404336058,
  66             'upload_date': '20140702',
  67             'duration': 343,
  68         }
  69     }
  70
  71     def _real_extract(self, url):
  72         video_id = self._match_id(url).replace('/', '.')
  73
  74         try:
  75             feed = self._download_json(
  76                 'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id,
  77                 video_id, 'Downloading episode JSON')['channel']['item']
  78         except ExtractorError as e:
  79             if isinstance(e.cause, compat_HTTPError):
  80                 raise ExtractorError(
  81                     'Currently unavailable in your country.', expected=True)
  82             raise
  83
  84         media_group = feed.get('media-group', {})
  85
  86         formats = []
  87         for media_content in media_group['media-content']:
  88             src = media_content.get('@attributes', {}).get('url')
  89             if not src:
  90                 continue
  91             ext = determine_ext(src)
  92             if ext == 'f4m':
  93                 formats.extend(self._extract_f4m_formats(
  94                     src, video_id, f4m_id='hds'))
  95             elif ext == 'm3u8':
  96                 formats.extend(self._extract_m3u8_formats(
  97                     src, video_id, 'mp4', m3u8_id='hls'))
  98             else:
  99                 formats.append({
 100                     'url': src,
 101                 })
 102         self._sort_formats(formats)
 103
 104         title = media_group.get('media-title')
 105         description = media_group.get('media-description')
 106         duration = int_or_none(media_group['media-content'][0].get('@attributes', {}).get('duration'))
 107         thumbnail = self._proto_relative_url(
 108             media_group.get('media-thumbnail', {}).get('@attributes', {}).get('url'))
 109         timestamp = parse_iso8601(feed.get('pubDate'), ' ')
 110
 111         subtitles = {}
 112         for media_subtitle in media_group.get('media-subTitle', []):
 113             lang = media_subtitle.get('@attributes', {}).get('lang')
 114             href = media_subtitle.get('@attributes', {}).get('href')
 115             if not lang or not href:
 116                 continue
 117             subtitles[lang] = [{
 118                 'ext': 'ttml',
 119                 'url': href,
 120             }]
 121
 122         return {
 123             'id': video_id,
 124             'title': title,
 125             'description': description,
 126             'thumbnail': thumbnail,
 127             'timestamp': timestamp,
 128             'duration': duration,
 129             'formats': formats,
 130             'subtitles': subtitles,
 131         }
 132
 133
 134 class DramaFeverSeriesIE(DramaFeverBaseIE):
 135     IE_NAME = 'dramafever:series'
 136     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
 137     _TESTS = [{
 138         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
 139         'info_dict': {
 140             'id': '4512',
 141             'title': 'Cooking with Shin',
 142             'description': 'md5:84a3f26e3cdc3fb7f500211b3593b5c1',
 143         },
 144         'playlist_count': 4,
 145     }, {
 146         'url': 'http://www.dramafever.com/drama/124/IRIS/',
 147         'info_dict': {
 148             'id': '124',
 149             'title': 'IRIS',
 150             'description': 'md5:b3a30e587cf20c59bd1c01ec0ee1b862',
 151         },
 152         'playlist_count': 20,
 153     }]
 154
 155     _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
 156     _PAGE_SIZE = 60  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
 157
 158     def _get_consumer_secret(self, video_id):
 159         mainjs = self._download_webpage(
 160             'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
 161             video_id, 'Downloading main.js', fatal=False)
 162         if not mainjs:
 163             return self._CONSUMER_SECRET
 164         return self._search_regex(
 165             r"var\s+cs\s*=\s*'([^']+)'", mainjs,
 166             'consumer secret', default=self._CONSUMER_SECRET)
 167
 168     def _real_extract(self, url):
 169         series_id = self._match_id(url)
 170
 171         consumer_secret = self._get_consumer_secret(series_id)
 172
 173         series = self._download_json(
 174             'http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s'
 175             % (consumer_secret, series_id),
 176             series_id, 'Downloading series JSON')['series'][series_id]
 177
 178         title = clean_html(series['name'])
 179         description = clean_html(series.get('description') or series.get('description_short'))
 180
 181         entries = []
 182         for page_num in itertools.count(1):
 183             episodes = self._download_json(
 184                 'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d'
 185                 % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
 186                 series_id, 'Downloading episodes JSON page #%d' % page_num)
 187             for episode in episodes.get('value', []):
 188                 episode_url = episode.get('episode_url')
 189                 if not episode_url:
 190                     continue
 191                 entries.append(self.url_result(
 192                     compat_urlparse.urljoin(url, episode_url),
 193                     'DramaFever', episode.get('guid')))
 194             if page_num == episodes['num_pages']:
 195                 break
 196
 197         return self.playlist_result(entries, series_id, title, description)