yt_dlp/extractor/asiancrush.py

   1 import functools
   2 import re
   3
   4 from .common import InfoExtractor
   5 from .kaltura import KalturaIE
   6 from ..utils import (
   7     extract_attributes,
   8     int_or_none,
   9     OnDemandPagedList,
  10     parse_age_limit,
  11     strip_or_none,
  12     try_get,
  13 )
  14
  15
  16 class AsianCrushBaseIE(InfoExtractor):
  17     _VALID_URL_BASE = r'https?://(?:www\.)?(?P<host>(?:(?:asiancrush|yuyutv|midnightpulp)\.com|(?:cocoro|retrocrush)\.tv))'
  18     _KALTURA_KEYS = [
  19         'video_url', 'progressive_url', 'download_url', 'thumbnail_url',
  20         'widescreen_thumbnail_url', 'screencap_widescreen',
  21     ]
  22     _API_SUFFIX = {'retrocrush.tv': '-ott'}
  23
  24     def _call_api(self, host, endpoint, video_id, query, resource):
  25         return self._download_json(
  26             'https://api%s.%s/%s' % (self._API_SUFFIX.get(host, ''), host, endpoint), video_id,
  27             'Downloading %s JSON metadata' % resource, query=query,
  28             headers=self.geo_verification_headers())['objects']
  29
  30     def _download_object_data(self, host, object_id, resource):
  31         return self._call_api(
  32             host, 'search', object_id, {'id': object_id}, resource)[0]
  33
  34     def _get_object_description(self, obj):
  35         return strip_or_none(obj.get('long_description') or obj.get('short_description'))
  36
  37     def _parse_video_data(self, video):
  38         title = video['name']
  39
  40         entry_id, partner_id = [None] * 2
  41         for k in self._KALTURA_KEYS:
  42             k_url = video.get(k)
  43             if k_url:
  44                 mobj = re.search(r'/p/(\d+)/.+?/entryId/([^/]+)/', k_url)
  45                 if mobj:
  46                     partner_id, entry_id = mobj.groups()
  47                     break
  48
  49         meta_categories = try_get(video, lambda x: x['meta']['categories'], list) or []
  50         categories = list(filter(None, [c.get('name') for c in meta_categories]))
  51
  52         show_info = video.get('show_info') or {}
  53
  54         return {
  55             '_type': 'url_transparent',
  56             'url': 'kaltura:%s:%s' % (partner_id, entry_id),
  57             'ie_key': KalturaIE.ie_key(),
  58             'id': entry_id,
  59             'title': title,
  60             'description': self._get_object_description(video),
  61             'age_limit': parse_age_limit(video.get('mpaa_rating') or video.get('tv_rating')),
  62             'categories': categories,
  63             'series': show_info.get('show_name'),
  64             'season_number': int_or_none(show_info.get('season_num')),
  65             'season_id': show_info.get('season_id'),
  66             'episode_number': int_or_none(show_info.get('episode_num')),
  67         }
  68
  69
  70 class AsianCrushIE(AsianCrushBaseIE):
  71     _VALID_URL = r'%s/video/(?:[^/]+/)?0+(?P<id>\d+)v\b' % AsianCrushBaseIE._VALID_URL_BASE
  72     _TESTS = [{
  73         'url': 'https://www.asiancrush.com/video/004289v/women-who-flirt',
  74         'md5': 'c3b740e48d0ba002a42c0b72857beae6',
  75         'info_dict': {
  76             'id': '1_y4tmjm5r',
  77             'ext': 'mp4',
  78             'title': 'Women Who Flirt',
  79             'description': 'md5:b65c7e0ae03a85585476a62a186f924c',
  80             'timestamp': 1496936429,
  81             'upload_date': '20170608',
  82             'uploader_id': 'craig@crifkin.com',
  83             'age_limit': 13,
  84             'categories': 'count:5',
  85             'duration': 5812,
  86         },
  87     }, {
  88         'url': 'https://www.asiancrush.com/video/she-was-pretty/011886v-pretty-episode-3/',
  89         'only_matching': True,
  90     }, {
  91         'url': 'https://www.yuyutv.com/video/013886v/the-act-of-killing/',
  92         'only_matching': True,
  93     }, {
  94         'url': 'https://www.yuyutv.com/video/peep-show/013922v-warring-factions/',
  95         'only_matching': True,
  96     }, {
  97         'url': 'https://www.midnightpulp.com/video/010400v/drifters/',
  98         'only_matching': True,
  99     }, {
 100         'url': 'https://www.midnightpulp.com/video/mononoke/016378v-zashikiwarashi-part-1/',
 101         'only_matching': True,
 102     }, {
 103         'url': 'https://www.cocoro.tv/video/the-wonderful-wizard-of-oz/008878v-the-wonderful-wizard-of-oz-ep01/',
 104         'only_matching': True,
 105     }, {
 106         'url': 'https://www.retrocrush.tv/video/true-tears/012328v-i...gave-away-my-tears',
 107         'only_matching': True,
 108     }]
 109
 110     def _real_extract(self, url):
 111         host, video_id = self._match_valid_url(url).groups()
 112
 113         if host == 'cocoro.tv':
 114             webpage = self._download_webpage(url, video_id)
 115             embed_vars = self._parse_json(self._search_regex(
 116                 r'iEmbedVars\s*=\s*({.+?})', webpage, 'embed vars',
 117                 default='{}'), video_id, fatal=False) or {}
 118             video_id = embed_vars.get('entry_id') or video_id
 119
 120         video = self._download_object_data(host, video_id, 'video')
 121         return self._parse_video_data(video)
 122
 123
 124 class AsianCrushPlaylistIE(AsianCrushBaseIE):
 125     _VALID_URL = r'%s/series/0+(?P<id>\d+)s\b' % AsianCrushBaseIE._VALID_URL_BASE
 126     _TESTS = [{
 127         'url': 'https://www.asiancrush.com/series/006447s/fruity-samurai',
 128         'info_dict': {
 129             'id': '6447',
 130             'title': 'Fruity Samurai',
 131             'description': 'md5:7535174487e4a202d3872a7fc8f2f154',
 132         },
 133         'playlist_count': 13,
 134     }, {
 135         'url': 'https://www.yuyutv.com/series/013920s/peep-show/',
 136         'only_matching': True,
 137     }, {
 138         'url': 'https://www.midnightpulp.com/series/016375s/mononoke/',
 139         'only_matching': True,
 140     }, {
 141         'url': 'https://www.cocoro.tv/series/008549s/the-wonderful-wizard-of-oz/',
 142         'only_matching': True,
 143     }, {
 144         'url': 'https://www.retrocrush.tv/series/012355s/true-tears',
 145         'only_matching': True,
 146     }]
 147     _PAGE_SIZE = 1000000000
 148
 149     def _fetch_page(self, domain, parent_id, page):
 150         videos = self._call_api(
 151             domain, 'getreferencedobjects', parent_id, {
 152                 'max': self._PAGE_SIZE,
 153                 'object_type': 'video',
 154                 'parent_id': parent_id,
 155                 'start': page * self._PAGE_SIZE,
 156             }, 'page %d' % (page + 1))
 157         for video in videos:
 158             yield self._parse_video_data(video)
 159
 160     def _real_extract(self, url):
 161         host, playlist_id = self._match_valid_url(url).groups()
 162
 163         if host == 'cocoro.tv':
 164             webpage = self._download_webpage(url, playlist_id)
 165
 166             entries = []
 167
 168             for mobj in re.finditer(
 169                     r'<a[^>]+href=(["\'])(?P<url>%s.*?)\1[^>]*>' % AsianCrushIE._VALID_URL,
 170                     webpage):
 171                 attrs = extract_attributes(mobj.group(0))
 172                 if attrs.get('class') == 'clearfix':
 173                     entries.append(self.url_result(
 174                         mobj.group('url'), ie=AsianCrushIE.ie_key()))
 175
 176             title = self._html_search_regex(
 177                 r'(?s)<h1\b[^>]\bid=["\']movieTitle[^>]+>(.+?)</h1>', webpage,
 178                 'title', default=None) or self._og_search_title(
 179                 webpage, default=None) or self._html_search_meta(
 180                 'twitter:title', webpage, 'title',
 181                 default=None) or self._html_extract_title(webpage)
 182             if title:
 183                 title = re.sub(r'\s*\|\s*.+?$', '', title)
 184
 185             description = self._og_search_description(
 186                 webpage, default=None) or self._html_search_meta(
 187                 'twitter:description', webpage, 'description', fatal=False)
 188         else:
 189             show = self._download_object_data(host, playlist_id, 'show')
 190             title = show.get('name')
 191             description = self._get_object_description(show)
 192             entries = OnDemandPagedList(
 193                 functools.partial(self._fetch_page, host, playlist_id),
 194                 self._PAGE_SIZE)
 195
 196         return self.playlist_result(entries, playlist_id, title, description)