yt_dlp/extractor/motherless.py

   1 import datetime
   2 import re
   3
   4 from .common import InfoExtractor
   5 from ..compat import compat_urlparse
   6 from ..utils import (
   7     ExtractorError,
   8     InAdvancePagedList,
   9     orderedSet,
  10     str_to_int,
  11     unified_strdate,
  12 )
  13
  14
  15 class MotherlessIE(InfoExtractor):
  16     _VALID_URL = r'https?://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
  17     _TESTS = [{
  18         'url': 'http://motherless.com/AC3FFE1',
  19         'md5': '310f62e325a9fafe64f68c0bccb6e75f',
  20         'info_dict': {
  21             'id': 'AC3FFE1',
  22             'ext': 'mp4',
  23             'title': 'Fucked in the ass while playing PS3',
  24             'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
  25             'upload_date': '20100913',
  26             'uploader_id': 'famouslyfuckedup',
  27             'thumbnail': r're:https?://.*\.jpg',
  28             'age_limit': 18,
  29         }
  30     }, {
  31         'url': 'http://motherless.com/532291B',
  32         'md5': 'bc59a6b47d1f958e61fbd38a4d31b131',
  33         'info_dict': {
  34             'id': '532291B',
  35             'ext': 'mp4',
  36             'title': 'Amazing girl playing the omegle game, PERFECT!',
  37             'categories': ['Amateur', 'webcam', 'omegle', 'pink', 'young', 'masturbate', 'teen',
  38                            'game', 'hairy'],
  39             'upload_date': '20140622',
  40             'uploader_id': 'Sulivana7x',
  41             'thumbnail': r're:https?://.*\.jpg',
  42             'age_limit': 18,
  43         },
  44         'skip': '404',
  45     }, {
  46         'url': 'http://motherless.com/g/cosplay/633979F',
  47         'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
  48         'info_dict': {
  49             'id': '633979F',
  50             'ext': 'mp4',
  51             'title': 'Turtlette',
  52             'categories': ['superheroine heroine  superher'],
  53             'upload_date': '20140827',
  54             'uploader_id': 'shade0230',
  55             'thumbnail': r're:https?://.*\.jpg',
  56             'age_limit': 18,
  57         }
  58     }, {
  59         # no keywords
  60         'url': 'http://motherless.com/8B4BBC1',
  61         'only_matching': True,
  62     }, {
  63         # see https://motherless.com/videos/recent for recent videos with
  64         # uploaded date in "ago" format
  65         'url': 'https://motherless.com/3C3E2CF',
  66         'info_dict': {
  67             'id': '3C3E2CF',
  68             'ext': 'mp4',
  69             'title': 'a/ Hot Teens',
  70             'categories': list,
  71             'upload_date': '20210104',
  72             'uploader_id': 'yonbiw',
  73             'thumbnail': r're:https?://.*\.jpg',
  74             'age_limit': 18,
  75         },
  76         'params': {
  77             'skip_download': True,
  78         },
  79     }]
  80
  81     def _real_extract(self, url):
  82         video_id = self._match_id(url)
  83         webpage = self._download_webpage(url, video_id)
  84
  85         if any(p in webpage for p in (
  86                 '<title>404 - MOTHERLESS.COM<',
  87                 ">The page you're looking for cannot be found.<")):
  88             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
  89
  90         if '>The content you are trying to view is for friends only.' in webpage:
  91             raise ExtractorError('Video %s is for friends only' % video_id, expected=True)
  92
  93         title = self._html_search_regex(
  94             (r'(?s)<div[^>]+\bclass=["\']media-meta-title[^>]+>(.+?)</div>',
  95              r'id="view-upload-title">\s+([^<]+)<'), webpage, 'title')
  96         video_url = (self._html_search_regex(
  97             (r'setup\(\{\s*["\']file["\']\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1',
  98              r'fileurl\s*=\s*(["\'])(?P<url>(?:(?!\1).)+)\1'),
  99             webpage, 'video URL', default=None, group='url')
 100             or 'http://cdn4.videos.motherlessmedia.com/videos/%s.mp4?fs=opencloud' % video_id)
 101         age_limit = self._rta_search(webpage)
 102         view_count = str_to_int(self._html_search_regex(
 103             (r'>([\d,.]+)\s+Views<', r'<strong>Views</strong>\s+([^<]+)<'),
 104             webpage, 'view count', fatal=False))
 105         like_count = str_to_int(self._html_search_regex(
 106             (r'>([\d,.]+)\s+Favorites<',
 107              r'<strong>Favorited</strong>\s+([^<]+)<'),
 108             webpage, 'like count', fatal=False))
 109
 110         upload_date = unified_strdate(self._search_regex(
 111             r'class=["\']count[^>]+>(\d+\s+[a-zA-Z]{3}\s+\d{4})<', webpage,
 112             'upload date', default=None))
 113         if not upload_date:
 114             uploaded_ago = self._search_regex(
 115                 r'>\s*(\d+[hd])\s+[aA]go\b', webpage, 'uploaded ago',
 116                 default=None)
 117             if uploaded_ago:
 118                 delta = int(uploaded_ago[:-1])
 119                 _AGO_UNITS = {
 120                     'h': 'hours',
 121                     'd': 'days',
 122                 }
 123                 kwargs = {_AGO_UNITS.get(uploaded_ago[-1]): delta}
 124                 upload_date = (datetime.datetime.utcnow() - datetime.timedelta(**kwargs)).strftime('%Y%m%d')
 125
 126         comment_count = webpage.count('class="media-comment-contents"')
 127         uploader_id = self._html_search_regex(
 128             (r'"media-meta-member">\s+<a href="/m/([^"]+)"',
 129              r'<span\b[^>]+\bclass="username">([^<]+)</span>'),
 130             webpage, 'uploader_id', fatal=False)
 131         categories = self._html_search_meta('keywords', webpage, default=None)
 132         if categories:
 133             categories = [cat.strip() for cat in categories.split(',')]
 134
 135         return {
 136             'id': video_id,
 137             'title': title,
 138             'upload_date': upload_date,
 139             'uploader_id': uploader_id,
 140             'thumbnail': self._og_search_thumbnail(webpage),
 141             'categories': categories,
 142             'view_count': view_count,
 143             'like_count': like_count,
 144             'comment_count': comment_count,
 145             'age_limit': age_limit,
 146             'url': video_url,
 147         }
 148
 149
 150 class MotherlessGroupIE(InfoExtractor):
 151     _VALID_URL = r'https?://(?:www\.)?motherless\.com/gv?/(?P<id>[a-z0-9_]+)'
 152     _TESTS = [{
 153         'url': 'http://motherless.com/g/movie_scenes',
 154         'info_dict': {
 155             'id': 'movie_scenes',
 156             'title': 'Movie Scenes',
 157             'description': 'Hot and sexy scenes from "regular" movies... '
 158                            'Beautiful actresses fully nude... A looot of '
 159                            'skin! :)Enjoy!',
 160         },
 161         'playlist_mincount': 662,
 162     }, {
 163         'url': 'http://motherless.com/gv/sex_must_be_funny',
 164         'info_dict': {
 165             'id': 'sex_must_be_funny',
 166             'title': 'Sex must be funny',
 167             'description': 'Sex can be funny. Wide smiles,laugh, games, fun of '
 168                            'any kind!'
 169         },
 170         'playlist_mincount': 0,
 171         'expected_warnings': [
 172             'This group has no videos.',
 173         ]
 174     }, {
 175         'url': 'https://motherless.com/g/beautiful_cock',
 176         'info_dict': {
 177             'id': 'beautiful_cock',
 178             'title': 'Beautiful Cock',
 179             'description': 'Group for lovely cocks yours, mine, a friends anything human',
 180         },
 181         'playlist_mincount': 2500,
 182     }]
 183
 184     @classmethod
 185     def suitable(cls, url):
 186         return (False if MotherlessIE.suitable(url)
 187                 else super(MotherlessGroupIE, cls).suitable(url))
 188
 189     def _extract_entries(self, webpage, base):
 190         entries = []
 191         for mobj in re.finditer(
 192                 r'href="(?P<href>/[^"]+)"[^>]*>(?:\s*<img[^>]+alt="[^-]+-\s(?P<title>[^"]+)")?',
 193                 webpage):
 194             video_url = compat_urlparse.urljoin(base, mobj.group('href'))
 195             if not MotherlessIE.suitable(video_url):
 196                 continue
 197             video_id = MotherlessIE._match_id(video_url)
 198             title = mobj.group('title')
 199             entries.append(self.url_result(
 200                 video_url, ie=MotherlessIE.ie_key(), video_id=video_id,
 201                 video_title=title))
 202         # Alternative fallback
 203         if not entries:
 204             entries = [
 205                 self.url_result(
 206                     compat_urlparse.urljoin(base, '/' + entry_id),
 207                     ie=MotherlessIE.ie_key(), video_id=entry_id)
 208                 for entry_id in orderedSet(re.findall(
 209                     r'data-codename=["\']([A-Z0-9]+)', webpage))]
 210         return entries
 211
 212     def _real_extract(self, url):
 213         group_id = self._match_id(url)
 214         page_url = compat_urlparse.urljoin(url, '/gv/%s' % group_id)
 215         webpage = self._download_webpage(page_url, group_id)
 216         title = self._search_regex(
 217             r'<title>([\w\s]+\w)\s+-', webpage, 'title', fatal=False)
 218         description = self._html_search_meta(
 219             'description', webpage, fatal=False)
 220         page_count = self._int(self._search_regex(
 221             r'(\d+)</(?:a|span)><(?:a|span)[^>]+rel="next">',
 222             webpage, 'page_count', default=0), 'page_count')
 223         if not page_count:
 224             message = self._search_regex(
 225                 r'class="error-page"[^>]*>\s*<p[^>]*>\s*(?P<error_msg>[^<]+)(?<=\S)\s*',
 226                 webpage, 'error_msg', default=None) or 'This group has no videos.'
 227             self.report_warning(message, group_id)
 228         PAGE_SIZE = 80
 229
 230         def _get_page(idx):
 231             if not page_count:
 232                 return
 233             webpage = self._download_webpage(
 234                 page_url, group_id, query={'page': idx + 1},
 235                 note='Downloading page %d/%d' % (idx + 1, page_count)
 236             )
 237             for entry in self._extract_entries(webpage, url):
 238                 yield entry
 239
 240         playlist = InAdvancePagedList(_get_page, page_count, PAGE_SIZE)
 241
 242         return {
 243             '_type': 'playlist',
 244             'id': group_id,
 245             'title': title,
 246             'description': description,
 247             'entries': playlist
 248         }