yt_dlp/extractor/cda.py

   1 import codecs
   2 import re
   3 import json
   4
   5 from .common import InfoExtractor
   6 from ..compat import (
   7     compat_chr,
   8     compat_ord,
   9     compat_urllib_parse_unquote,
  10 )
  11 from ..utils import (
  12     ExtractorError,
  13     float_or_none,
  14     int_or_none,
  15     merge_dicts,
  16     multipart_encode,
  17     parse_duration,
  18     random_birthday,
  19     urljoin,
  20     try_get,
  21 )
  22
  23
  24 class CDAIE(InfoExtractor):
  25     _VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
  26     _BASE_URL = 'http://www.cda.pl/'
  27     _TESTS = [{
  28         'url': 'http://www.cda.pl/video/5749950c',
  29         'md5': '6f844bf51b15f31fae165365707ae970',
  30         'info_dict': {
  31             'id': '5749950c',
  32             'ext': 'mp4',
  33             'height': 720,
  34             'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
  35             'description': 'md5:269ccd135d550da90d1662651fcb9772',
  36             'thumbnail': r're:^https?://.*\.jpg$',
  37             'average_rating': float,
  38             'duration': 39,
  39             'age_limit': 0,
  40             'upload_date': '20160221',
  41             'timestamp': 1456078244,
  42         }
  43     }, {
  44         'url': 'http://www.cda.pl/video/57413289',
  45         'md5': 'a88828770a8310fc00be6c95faf7f4d5',
  46         'info_dict': {
  47             'id': '57413289',
  48             'ext': 'mp4',
  49             'title': 'Lądowanie na lotnisku na Maderze',
  50             'description': 'md5:60d76b71186dcce4e0ba6d4bbdb13e1a',
  51             'thumbnail': r're:^https?://.*\.jpg$',
  52             'uploader': 'crash404',
  53             'view_count': int,
  54             'average_rating': float,
  55             'duration': 137,
  56             'age_limit': 0,
  57         }
  58     }, {
  59         # Age-restricted
  60         'url': 'http://www.cda.pl/video/1273454c4',
  61         'info_dict': {
  62             'id': '1273454c4',
  63             'ext': 'mp4',
  64             'title': 'Bronson (2008) napisy HD 1080p',
  65             'description': 'md5:1b6cb18508daf2dc4e0fa4db77fec24c',
  66             'height': 1080,
  67             'uploader': 'boniek61',
  68             'thumbnail': r're:^https?://.*\.jpg$',
  69             'duration': 5554,
  70             'age_limit': 18,
  71             'view_count': int,
  72             'average_rating': float,
  73         },
  74     }, {
  75         'url': 'http://ebd.cda.pl/0x0/5749950c',
  76         'only_matching': True,
  77     }]
  78
  79     def _download_age_confirm_page(self, url, video_id, *args, **kwargs):
  80         form_data = random_birthday('rok', 'miesiac', 'dzien')
  81         form_data.update({'return': url, 'module': 'video', 'module_id': video_id})
  82         data, content_type = multipart_encode(form_data)
  83         return self._download_webpage(
  84             urljoin(url, '/a/validatebirth'), video_id, *args,
  85             data=data, headers={
  86                 'Referer': url,
  87                 'Content-Type': content_type,
  88             }, **kwargs)
  89
  90     def _real_extract(self, url):
  91         video_id = self._match_id(url)
  92         self._set_cookie('cda.pl', 'cda.player', 'html5')
  93         webpage = self._download_webpage(
  94             self._BASE_URL + '/video/' + video_id, video_id)
  95
  96         if 'Ten film jest dostępny dla użytkowników premium' in webpage:
  97             raise ExtractorError('This video is only available for premium users.', expected=True)
  98
  99         if re.search(r'niedostępn[ey] w(?:&nbsp;|\s+)Twoim kraju\s*<', webpage):
 100             self.raise_geo_restricted()
 101
 102         need_confirm_age = False
 103         if self._html_search_regex(r'(<form[^>]+action="[^"]*/a/validatebirth[^"]*")',
 104                                    webpage, 'birthday validate form', default=None):
 105             webpage = self._download_age_confirm_page(
 106                 url, video_id, note='Confirming age')
 107             need_confirm_age = True
 108
 109         formats = []
 110
 111         uploader = self._search_regex(r'''(?x)
 112             <(span|meta)[^>]+itemprop=(["\'])author\2[^>]*>
 113             (?:<\1[^>]*>[^<]*</\1>|(?!</\1>)(?:.|\n))*?
 114             <(span|meta)[^>]+itemprop=(["\'])name\4[^>]*>(?P<uploader>[^<]+)</\3>
 115         ''', webpage, 'uploader', default=None, group='uploader')
 116         view_count = self._search_regex(
 117             r'Odsłony:(?:\s|&nbsp;)*([0-9]+)', webpage,
 118             'view_count', default=None)
 119         average_rating = self._search_regex(
 120             (r'<(?:span|meta)[^>]+itemprop=(["\'])ratingValue\1[^>]*>(?P<rating_value>[0-9.]+)',
 121              r'<span[^>]+\bclass=["\']rating["\'][^>]*>(?P<rating_value>[0-9.]+)'), webpage, 'rating', fatal=False,
 122             group='rating_value')
 123
 124         info_dict = {
 125             'id': video_id,
 126             'title': self._og_search_title(webpage),
 127             'description': self._og_search_description(webpage),
 128             'uploader': uploader,
 129             'view_count': int_or_none(view_count),
 130             'average_rating': float_or_none(average_rating),
 131             'thumbnail': self._og_search_thumbnail(webpage),
 132             'formats': formats,
 133             'duration': None,
 134             'age_limit': 18 if need_confirm_age else 0,
 135         }
 136
 137         info = self._search_json_ld(webpage, video_id, default={})
 138
 139         # Source: https://www.cda.pl/js/player.js?t=1606154898
 140         def decrypt_file(a):
 141             for p in ('_XDDD', '_CDA', '_ADC', '_CXD', '_QWE', '_Q5', '_IKSDE'):
 142                 a = a.replace(p, '')
 143             a = compat_urllib_parse_unquote(a)
 144             b = []
 145             for c in a:
 146                 f = compat_ord(c)
 147                 b.append(compat_chr(33 + (f + 14) % 94) if 33 <= f <= 126 else compat_chr(f))
 148             a = ''.join(b)
 149             a = a.replace('.cda.mp4', '')
 150             for p in ('.2cda.pl', '.3cda.pl'):
 151                 a = a.replace(p, '.cda.pl')
 152             if '/upstream' in a:
 153                 a = a.replace('/upstream', '.mp4/upstream')
 154                 return 'https://' + a
 155             return 'https://' + a + '.mp4'
 156
 157         def extract_format(page, version):
 158             json_str = self._html_search_regex(
 159                 r'player_data=(\\?["\'])(?P<player_data>.+?)\1', page,
 160                 '%s player_json' % version, fatal=False, group='player_data')
 161             if not json_str:
 162                 return
 163             player_data = self._parse_json(
 164                 json_str, '%s player_data' % version, fatal=False)
 165             if not player_data:
 166                 return
 167             video = player_data.get('video')
 168             if not video or 'file' not in video:
 169                 self.report_warning('Unable to extract %s version information' % version)
 170                 return
 171             if video['file'].startswith('uggc'):
 172                 video['file'] = codecs.decode(video['file'], 'rot_13')
 173                 if video['file'].endswith('adc.mp4'):
 174                     video['file'] = video['file'].replace('adc.mp4', '.mp4')
 175             elif not video['file'].startswith('http'):
 176                 video['file'] = decrypt_file(video['file'])
 177             video_quality = video.get('quality')
 178             qualities = video.get('qualities', {})
 179             video_quality = next((k for k, v in qualities.items() if v == video_quality), video_quality)
 180             info_dict['formats'].append({
 181                 'url': video['file'],
 182                 'format_id': video_quality,
 183                 'height': int_or_none(video_quality[:-1]),
 184             })
 185             for quality, cda_quality in qualities.items():
 186                 if quality == video_quality:
 187                     continue
 188                 data = {'jsonrpc': '2.0', 'method': 'videoGetLink', 'id': 2,
 189                         'params': [video_id, cda_quality, video.get('ts'), video.get('hash2'), {}]}
 190                 data = json.dumps(data).encode('utf-8')
 191                 video_url = self._download_json(
 192                     f'https://www.cda.pl/video/{video_id}', video_id, headers={
 193                         'Content-Type': 'application/json',
 194                         'X-Requested-With': 'XMLHttpRequest'
 195                     }, data=data, note=f'Fetching {quality} url',
 196                     errnote=f'Failed to fetch {quality} url', fatal=False)
 197                 if try_get(video_url, lambda x: x['result']['status']) == 'ok':
 198                     video_url = try_get(video_url, lambda x: x['result']['resp'])
 199                     info_dict['formats'].append({
 200                         'url': video_url,
 201                         'format_id': quality,
 202                         'height': int_or_none(quality[:-1])
 203                     })
 204
 205             if not info_dict['duration']:
 206                 info_dict['duration'] = parse_duration(video.get('duration'))
 207
 208         extract_format(webpage, 'default')
 209
 210         for href, resolution in re.findall(
 211                 r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
 212                 webpage):
 213             if need_confirm_age:
 214                 handler = self._download_age_confirm_page
 215             else:
 216                 handler = self._download_webpage
 217
 218             webpage = handler(
 219                 urljoin(self._BASE_URL, href), video_id,
 220                 'Downloading %s version information' % resolution, fatal=False)
 221             if not webpage:
 222                 # Manually report warning because empty page is returned when
 223                 # invalid version is requested.
 224                 self.report_warning('Unable to download %s version information' % resolution)
 225                 continue
 226
 227             extract_format(webpage, resolution)
 228
 229         self._sort_formats(formats)
 230
 231         return merge_dicts(info_dict, info)