yt_dlp/extractor/cda.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import codecs
   5 import re
   6 import json
   7
   8 from .common import InfoExtractor
   9 from ..compat import (
  10     compat_chr,
  11     compat_ord,
  12     compat_urllib_parse_unquote,
  13 )
  14 from ..utils import (
  15     ExtractorError,
  16     float_or_none,
  17     int_or_none,
  18     merge_dicts,
  19     multipart_encode,
  20     parse_duration,
  21     random_birthday,
  22     urljoin,
  23     try_get,
  24 )
  25
  26
  27 class CDAIE(InfoExtractor):
  28     _VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
  29     _BASE_URL = 'http://www.cda.pl/'
  30     _TESTS = [{
  31         'url': 'http://www.cda.pl/video/5749950c',
  32         'md5': '6f844bf51b15f31fae165365707ae970',
  33         'info_dict': {
  34             'id': '5749950c',
  35             'ext': 'mp4',
  36             'height': 720,
  37             'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
  38             'description': 'md5:269ccd135d550da90d1662651fcb9772',
  39             'thumbnail': r're:^https?://.*\.jpg$',
  40             'average_rating': float,
  41             'duration': 39,
  42             'age_limit': 0,
  43             'upload_date': '20160221',
  44             'timestamp': 1456078244,
  45         }
  46     }, {
  47         'url': 'http://www.cda.pl/video/57413289',
  48         'md5': 'a88828770a8310fc00be6c95faf7f4d5',
  49         'info_dict': {
  50             'id': '57413289',
  51             'ext': 'mp4',
  52             'title': 'Lądowanie na lotnisku na Maderze',
  53             'description': 'md5:60d76b71186dcce4e0ba6d4bbdb13e1a',
  54             'thumbnail': r're:^https?://.*\.jpg$',
  55             'uploader': 'crash404',
  56             'view_count': int,
  57             'average_rating': float,
  58             'duration': 137,
  59             'age_limit': 0,
  60         }
  61     }, {
  62         # Age-restricted
  63         'url': 'http://www.cda.pl/video/1273454c4',
  64         'info_dict': {
  65             'id': '1273454c4',
  66             'ext': 'mp4',
  67             'title': 'Bronson (2008) napisy HD 1080p',
  68             'description': 'md5:1b6cb18508daf2dc4e0fa4db77fec24c',
  69             'height': 1080,
  70             'uploader': 'boniek61',
  71             'thumbnail': r're:^https?://.*\.jpg$',
  72             'duration': 5554,
  73             'age_limit': 18,
  74             'view_count': int,
  75             'average_rating': float,
  76         },
  77     }, {
  78         'url': 'http://ebd.cda.pl/0x0/5749950c',
  79         'only_matching': True,
  80     }]
  81
  82     def _download_age_confirm_page(self, url, video_id, *args, **kwargs):
  83         form_data = random_birthday('rok', 'miesiac', 'dzien')
  84         form_data.update({'return': url, 'module': 'video', 'module_id': video_id})
  85         data, content_type = multipart_encode(form_data)
  86         return self._download_webpage(
  87             urljoin(url, '/a/validatebirth'), video_id, *args,
  88             data=data, headers={
  89                 'Referer': url,
  90                 'Content-Type': content_type,
  91             }, **kwargs)
  92
  93     def _real_extract(self, url):
  94         video_id = self._match_id(url)
  95         self._set_cookie('cda.pl', 'cda.player', 'html5')
  96         webpage = self._download_webpage(
  97             self._BASE_URL + '/video/' + video_id, video_id)
  98
  99         if 'Ten film jest dostępny dla użytkowników premium' in webpage:
 100             raise ExtractorError('This video is only available for premium users.', expected=True)
 101
 102         if re.search(r'niedostępn[ey] w(?:&nbsp;|\s+)Twoim kraju\s*<', webpage):
 103             self.raise_geo_restricted()
 104
 105         need_confirm_age = False
 106         if self._html_search_regex(r'(<form[^>]+action="[^"]*/a/validatebirth[^"]*")',
 107                                    webpage, 'birthday validate form', default=None):
 108             webpage = self._download_age_confirm_page(
 109                 url, video_id, note='Confirming age')
 110             need_confirm_age = True
 111
 112         formats = []
 113
 114         uploader = self._search_regex(r'''(?x)
 115             <(span|meta)[^>]+itemprop=(["\'])author\2[^>]*>
 116             (?:<\1[^>]*>[^<]*</\1>|(?!</\1>)(?:.|\n))*?
 117             <(span|meta)[^>]+itemprop=(["\'])name\4[^>]*>(?P<uploader>[^<]+)</\3>
 118         ''', webpage, 'uploader', default=None, group='uploader')
 119         view_count = self._search_regex(
 120             r'Odsłony:(?:\s|&nbsp;)*([0-9]+)', webpage,
 121             'view_count', default=None)
 122         average_rating = self._search_regex(
 123             (r'<(?:span|meta)[^>]+itemprop=(["\'])ratingValue\1[^>]*>(?P<rating_value>[0-9.]+)',
 124              r'<span[^>]+\bclass=["\']rating["\'][^>]*>(?P<rating_value>[0-9.]+)'), webpage, 'rating', fatal=False,
 125             group='rating_value')
 126
 127         info_dict = {
 128             'id': video_id,
 129             'title': self._og_search_title(webpage),
 130             'description': self._og_search_description(webpage),
 131             'uploader': uploader,
 132             'view_count': int_or_none(view_count),
 133             'average_rating': float_or_none(average_rating),
 134             'thumbnail': self._og_search_thumbnail(webpage),
 135             'formats': formats,
 136             'duration': None,
 137             'age_limit': 18 if need_confirm_age else 0,
 138         }
 139
 140         info = self._search_json_ld(webpage, video_id, default={})
 141
 142         # Source: https://www.cda.pl/js/player.js?t=1606154898
 143         def decrypt_file(a):
 144             for p in ('_XDDD', '_CDA', '_ADC', '_CXD', '_QWE', '_Q5', '_IKSDE'):
 145                 a = a.replace(p, '')
 146             a = compat_urllib_parse_unquote(a)
 147             b = []
 148             for c in a:
 149                 f = compat_ord(c)
 150                 b.append(compat_chr(33 + (f + 14) % 94) if 33 <= f <= 126 else compat_chr(f))
 151             a = ''.join(b)
 152             a = a.replace('.cda.mp4', '')
 153             for p in ('.2cda.pl', '.3cda.pl'):
 154                 a = a.replace(p, '.cda.pl')
 155             if '/upstream' in a:
 156                 a = a.replace('/upstream', '.mp4/upstream')
 157                 return 'https://' + a
 158             return 'https://' + a + '.mp4'
 159
 160         def extract_format(page, version):
 161             json_str = self._html_search_regex(
 162                 r'player_data=(\\?["\'])(?P<player_data>.+?)\1', page,
 163                 '%s player_json' % version, fatal=False, group='player_data')
 164             if not json_str:
 165                 return
 166             player_data = self._parse_json(
 167                 json_str, '%s player_data' % version, fatal=False)
 168             if not player_data:
 169                 return
 170             video = player_data.get('video')
 171             if not video or 'file' not in video:
 172                 self.report_warning('Unable to extract %s version information' % version)
 173                 return
 174             if video['file'].startswith('uggc'):
 175                 video['file'] = codecs.decode(video['file'], 'rot_13')
 176                 if video['file'].endswith('adc.mp4'):
 177                     video['file'] = video['file'].replace('adc.mp4', '.mp4')
 178             elif not video['file'].startswith('http'):
 179                 video['file'] = decrypt_file(video['file'])
 180             video_quality = video.get('quality')
 181             qualities = video.get('qualities', {})
 182             video_quality = next((k for k, v in qualities.items() if v == video_quality), video_quality)
 183             info_dict['formats'].append({
 184                 'url': video['file'],
 185                 'format_id': video_quality,
 186                 'height': int_or_none(video_quality[:-1]),
 187             })
 188             for quality, cda_quality in qualities.items():
 189                 if quality == video_quality:
 190                     continue
 191                 data = {'jsonrpc': '2.0', 'method': 'videoGetLink', 'id': 2,
 192                         'params': [video_id, cda_quality, video.get('ts'), video.get('hash2'), {}]}
 193                 data = json.dumps(data).encode('utf-8')
 194                 video_url = self._download_json(
 195                     f'https://www.cda.pl/video/{video_id}', video_id, headers={
 196                         'Content-Type': 'application/json',
 197                         'X-Requested-With': 'XMLHttpRequest'
 198                     }, data=data, note=f'Fetching {quality} url',
 199                     errnote=f'Failed to fetch {quality} url', fatal=False)
 200                 if try_get(video_url, lambda x: x['result']['status']) == 'ok':
 201                     video_url = try_get(video_url, lambda x: x['result']['resp'])
 202                     info_dict['formats'].append({
 203                         'url': video_url,
 204                         'format_id': quality,
 205                         'height': int_or_none(quality[:-1])
 206                     })
 207
 208             if not info_dict['duration']:
 209                 info_dict['duration'] = parse_duration(video.get('duration'))
 210
 211         extract_format(webpage, 'default')
 212
 213         for href, resolution in re.findall(
 214                 r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
 215                 webpage):
 216             if need_confirm_age:
 217                 handler = self._download_age_confirm_page
 218             else:
 219                 handler = self._download_webpage
 220
 221             webpage = handler(
 222                 urljoin(self._BASE_URL, href), video_id,
 223                 'Downloading %s version information' % resolution, fatal=False)
 224             if not webpage:
 225                 # Manually report warning because empty page is returned when
 226                 # invalid version is requested.
 227                 self.report_warning('Unable to download %s version information' % resolution)
 228                 continue
 229
 230             extract_format(webpage, resolution)
 231
 232         self._sort_formats(formats)
 233
 234         return merge_dicts(info_dict, info)