yt_dlp/extractor/spankwire.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     float_or_none,
   6     int_or_none,
   7     merge_dicts,
   8     str_or_none,
   9     str_to_int,
  10     url_or_none,
  11 )
  12
  13
  14 class SpankwireIE(InfoExtractor):
  15     _VALID_URL = r'''(?x)
  16                     https?://
  17                         (?:www\.)?spankwire\.com/
  18                         (?:
  19                             [^/]+/video|
  20                             EmbedPlayer\.aspx/?\?.*?\bArticleId=
  21                         )
  22                         (?P<id>\d+)
  23                     '''
  24     _TESTS = [{
  25         # download URL pattern: */<height>P_<tbr>K_<video_id>.mp4
  26         'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
  27         'md5': '5aa0e4feef20aad82cbcae3aed7ab7cd',
  28         'info_dict': {
  29             'id': '103545',
  30             'ext': 'mp4',
  31             'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
  32             'description': 'Crazy Bitch X rated music video.',
  33             'duration': 222,
  34             'uploader': 'oreusz',
  35             'uploader_id': '124697',
  36             'timestamp': 1178587885,
  37             'upload_date': '20070508',
  38             'average_rating': float,
  39             'view_count': int,
  40             'comment_count': int,
  41             'age_limit': 18,
  42             'categories': list,
  43             'tags': list,
  44         },
  45     }, {
  46         # download URL pattern: */mp4_<format_id>_<video_id>.mp4
  47         'url': 'http://www.spankwire.com/Titcums-Compiloation-I/video1921551/',
  48         'md5': '09b3c20833308b736ae8902db2f8d7e6',
  49         'info_dict': {
  50             'id': '1921551',
  51             'ext': 'mp4',
  52             'title': 'Titcums Compiloation I',
  53             'description': 'cum on tits',
  54             'uploader': 'dannyh78999',
  55             'uploader_id': '3056053',
  56             'upload_date': '20150822',
  57             'age_limit': 18,
  58         },
  59         'params': {
  60             'proxy': '127.0.0.1:8118'
  61         },
  62         'skip': 'removed',
  63     }, {
  64         'url': 'https://www.spankwire.com/EmbedPlayer.aspx/?ArticleId=156156&autostart=true',
  65         'only_matching': True,
  66     }]
  67
  68     @staticmethod
  69     def _extract_urls(webpage):
  70         return re.findall(
  71             r'<iframe[^>]+\bsrc=["\']((?:https?:)?//(?:www\.)?spankwire\.com/EmbedPlayer\.aspx/?\?.*?\bArticleId=\d+)',
  72             webpage)
  73
  74     def _real_extract(self, url):
  75         video_id = self._match_id(url)
  76
  77         video = self._download_json(
  78             'https://www.spankwire.com/api/video/%s.json' % video_id, video_id)
  79
  80         title = video['title']
  81
  82         formats = []
  83         videos = video.get('videos')
  84         if isinstance(videos, dict):
  85             for format_id, format_url in videos.items():
  86                 video_url = url_or_none(format_url)
  87                 if not format_url:
  88                     continue
  89                 height = int_or_none(self._search_regex(
  90                     r'(\d+)[pP]', format_id, 'height', default=None))
  91                 m = re.search(
  92                     r'/(?P<height>\d+)[pP]_(?P<tbr>\d+)[kK]', video_url)
  93                 if m:
  94                     tbr = int(m.group('tbr'))
  95                     height = height or int(m.group('height'))
  96                 else:
  97                     tbr = None
  98                 formats.append({
  99                     'url': video_url,
 100                     'format_id': '%dp' % height if height else format_id,
 101                     'height': height,
 102                     'tbr': tbr,
 103                 })
 104         m3u8_url = url_or_none(video.get('HLS'))
 105         if m3u8_url:
 106             formats.extend(self._extract_m3u8_formats(
 107                 m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
 108                 m3u8_id='hls', fatal=False))
 109         self._sort_formats(formats)
 110
 111         view_count = str_to_int(video.get('viewed'))
 112
 113         thumbnails = []
 114         for preference, t in enumerate(('', '2x'), start=0):
 115             thumbnail_url = url_or_none(video.get('poster%s' % t))
 116             if not thumbnail_url:
 117                 continue
 118             thumbnails.append({
 119                 'url': thumbnail_url,
 120                 'preference': preference,
 121             })
 122
 123         def extract_names(key):
 124             entries_list = video.get(key)
 125             if not isinstance(entries_list, list):
 126                 return
 127             entries = []
 128             for entry in entries_list:
 129                 name = str_or_none(entry.get('name'))
 130                 if name:
 131                     entries.append(name)
 132             return entries
 133
 134         categories = extract_names('categories')
 135         tags = extract_names('tags')
 136
 137         uploader = None
 138         info = {}
 139
 140         webpage = self._download_webpage(
 141             'https://www.spankwire.com/_/video%s/' % video_id, video_id,
 142             fatal=False)
 143         if webpage:
 144             info = self._search_json_ld(webpage, video_id, default={})
 145             thumbnail_url = None
 146             if 'thumbnail' in info:
 147                 thumbnail_url = url_or_none(info['thumbnail'])
 148                 del info['thumbnail']
 149             if not thumbnail_url:
 150                 thumbnail_url = self._og_search_thumbnail(webpage)
 151             if thumbnail_url:
 152                 thumbnails.append({
 153                     'url': thumbnail_url,
 154                     'preference': 10,
 155                 })
 156             uploader = self._html_search_regex(
 157                 r'(?s)by\s*<a[^>]+\bclass=["\']uploaded__by[^>]*>(.+?)</a>',
 158                 webpage, 'uploader', fatal=False)
 159             if not view_count:
 160                 view_count = str_to_int(self._search_regex(
 161                     r'data-views=["\']([\d,.]+)', webpage, 'view count',
 162                     fatal=False))
 163
 164         return merge_dicts({
 165             'id': video_id,
 166             'title': title,
 167             'description': video.get('description'),
 168             'duration': int_or_none(video.get('duration')),
 169             'thumbnails': thumbnails,
 170             'uploader': uploader,
 171             'uploader_id': str_or_none(video.get('userId')),
 172             'timestamp': int_or_none(video.get('time_approved_on')),
 173             'average_rating': float_or_none(video.get('rating')),
 174             'view_count': view_count,
 175             'comment_count': int_or_none(video.get('comments')),
 176             'age_limit': 18,
 177             'categories': categories,
 178             'tags': tags,
 179             'formats': formats,
 180         }, info)