yt_dlp/extractor/xfileshare.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     ExtractorError,
   6     decode_packed_codes,
   7     determine_ext,
   8     int_or_none,
   9     js_to_json,
  10     urlencode_postdata,
  11 )
  12
  13
  14 # based on openload_decode from 2bfeee69b976fe049761dd3012e30b637ee05a58
  15 def aa_decode(aa_code):
  16     symbol_table = [
  17         ('7', '((ﾟｰﾟ) + (o^_^o))'),
  18         ('6', '((o^_^o) +(o^_^o))'),
  19         ('5', '((ﾟｰﾟ) + (ﾟΘﾟ))'),
  20         ('2', '((o^_^o) - (ﾟΘﾟ))'),
  21         ('4', '(ﾟｰﾟ)'),
  22         ('3', '(o^_^o)'),
  23         ('1', '(ﾟΘﾟ)'),
  24         ('0', '(c^_^o)'),
  25     ]
  26     delim = '(ﾟДﾟ)[ﾟεﾟ]+'
  27     ret = ''
  28     for aa_char in aa_code.split(delim):
  29         for val, pat in symbol_table:
  30             aa_char = aa_char.replace(pat, val)
  31         aa_char = aa_char.replace('+ ', '')
  32         m = re.match(r'^\d+', aa_char)
  33         if m:
  34             ret += chr(int(m.group(0), 8))
  35         else:
  36             m = re.match(r'^u([\da-f]+)', aa_char)
  37             if m:
  38                 ret += chr(int(m.group(1), 16))
  39     return ret
  40
  41
  42 class XFileShareIE(InfoExtractor):
  43     _SITES = (
  44         (r'aparat\.cam', 'Aparat'),
  45         (r'clipwatching\.com', 'ClipWatching'),
  46         (r'gounlimited\.to', 'GoUnlimited'),
  47         (r'govid\.me', 'GoVid'),
  48         (r'holavid\.com', 'HolaVid'),
  49         (r'streamty\.com', 'Streamty'),
  50         (r'thevideobee\.to', 'TheVideoBee'),
  51         (r'uqload\.com', 'Uqload'),
  52         (r'vidbom\.com', 'VidBom'),
  53         (r'vidlo\.us', 'vidlo'),
  54         (r'vidlocker\.xyz', 'VidLocker'),
  55         (r'vidshare\.tv', 'VidShare'),
  56         (r'vup\.to', 'VUp'),
  57         (r'wolfstream\.tv', 'WolfStream'),
  58         (r'xvideosharing\.com', 'XVideoSharing'),
  59     )
  60
  61     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
  62     _VALID_URL = (r'https?://(?:www\.)?(?P<host>%s)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
  63                   % '|'.join(site for site in list(zip(*_SITES))[0]))
  64
  65     _FILE_NOT_FOUND_REGEXES = (
  66         r'>(?:404 - )?File Not Found<',
  67         r'>The file was removed by administrator<',
  68     )
  69
  70     _TESTS = [{
  71         'url': 'http://xvideosharing.com/fq65f94nd2ve',
  72         'md5': '4181f63957e8fe90ac836fa58dc3c8a6',
  73         'info_dict': {
  74             'id': 'fq65f94nd2ve',
  75             'ext': 'mp4',
  76             'title': 'sample',
  77             'thumbnail': r're:http://.*\.jpg',
  78         },
  79     }, {
  80         'url': 'https://aparat.cam/n4d6dh0wvlpr',
  81         'only_matching': True,
  82     }, {
  83         'url': 'https://wolfstream.tv/nthme29v9u2x',
  84         'only_matching': True,
  85     }]
  86
  87     @staticmethod
  88     def _extract_urls(webpage):
  89         return [
  90             mobj.group('url')
  91             for mobj in re.finditer(
  92                 r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1'
  93                 % '|'.join(site for site in list(zip(*XFileShareIE._SITES))[0]),
  94                 webpage)]
  95
  96     def _real_extract(self, url):
  97         host, video_id = self._match_valid_url(url).groups()
  98
  99         url = 'https://%s/' % host + ('embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
 100         webpage = self._download_webpage(url, video_id)
 101
 102         if any(re.search(p, webpage) for p in self._FILE_NOT_FOUND_REGEXES):
 103             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 104
 105         fields = self._hidden_inputs(webpage)
 106
 107         if fields.get('op') == 'download1':
 108             countdown = int_or_none(self._search_regex(
 109                 r'<span id="countdown_str">(?:[Ww]ait)?\s*<span id="cxc">(\d+)</span>\s*(?:seconds?)?</span>',
 110                 webpage, 'countdown', default=None))
 111             if countdown:
 112                 self._sleep(countdown, video_id)
 113
 114             webpage = self._download_webpage(
 115                 url, video_id, 'Downloading video page',
 116                 data=urlencode_postdata(fields), headers={
 117                     'Referer': url,
 118                     'Content-type': 'application/x-www-form-urlencoded',
 119                 })
 120
 121         title = (self._search_regex(
 122             (r'style="z-index: [0-9]+;">([^<]+)</span>',
 123              r'<td nowrap>([^<]+)</td>',
 124              r'h4-fine[^>]*>([^<]+)<',
 125              r'>Watch (.+)[ <]',
 126              r'<h2 class="video-page-head">([^<]+)</h2>',
 127              r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<',  # streamin.to
 128              r'title\s*:\s*"([^"]+)"'),  # govid.me
 129             webpage, 'title', default=None) or self._og_search_title(
 130             webpage, default=None) or video_id).strip()
 131
 132         for regex, func in (
 133                 (r'(eval\(function\(p,a,c,k,e,d\){.+)', decode_packed_codes),
 134                 (r'(ﾟ.+)', aa_decode)):
 135             obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
 136             if obf_code:
 137                 webpage = webpage.replace(obf_code, func(obf_code))
 138
 139         formats = []
 140
 141         jwplayer_data = self._search_regex(
 142             [
 143                 r'jwplayer\("[^"]+"\)\.load\(\[({.+?})\]\);',
 144                 r'jwplayer\("[^"]+"\)\.setup\(({.+?})\);',
 145             ], webpage,
 146             'jwplayer data', default=None)
 147         if jwplayer_data:
 148             jwplayer_data = self._parse_json(
 149                 jwplayer_data.replace(r"\'", "'"), video_id, js_to_json)
 150             if jwplayer_data:
 151                 formats = self._parse_jwplayer_data(
 152                     jwplayer_data, video_id, False,
 153                     m3u8_id='hls', mpd_id='dash')['formats']
 154
 155         if not formats:
 156             urls = []
 157             for regex in (
 158                     r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
 159                     r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
 160                     r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
 161                     r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1'):
 162                 for mobj in re.finditer(regex, webpage):
 163                     video_url = mobj.group('url')
 164                     if video_url not in urls:
 165                         urls.append(video_url)
 166
 167             sources = self._search_regex(
 168                 r'sources\s*:\s*(\[(?!{)[^\]]+\])', webpage, 'sources', default=None)
 169             if sources:
 170                 urls.extend(self._parse_json(sources, video_id))
 171
 172             formats = []
 173             for video_url in urls:
 174                 if determine_ext(video_url) == 'm3u8':
 175                     formats.extend(self._extract_m3u8_formats(
 176                         video_url, video_id, 'mp4',
 177                         entry_protocol='m3u8_native', m3u8_id='hls',
 178                         fatal=False))
 179                 else:
 180                     formats.append({
 181                         'url': video_url,
 182                         'format_id': 'sd',
 183                     })
 184         self._sort_formats(formats)
 185
 186         thumbnail = self._search_regex(
 187             [
 188                 r'<video[^>]+poster="([^"]+)"',
 189                 r'(?:image|poster)\s*:\s*["\'](http[^"\']+)["\'],',
 190             ], webpage, 'thumbnail', default=None)
 191
 192         return {
 193             'id': video_id,
 194             'title': title,
 195             'thumbnail': thumbnail,
 196             'formats': formats,
 197         }