youtube_dlc/extractor/cda.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import codecs
   5 import re
   6
   7 from .common import InfoExtractor
   8 from ..compat import (
   9     compat_chr,
  10     compat_ord,
  11     compat_urllib_parse_unquote,
  12 )
  13 from ..utils import (
  14     ExtractorError,
  15     float_or_none,
  16     int_or_none,
  17     merge_dicts,
  18     multipart_encode,
  19     parse_duration,
  20     random_birthday,
  21     urljoin,
  22 )
  23
  24
  25 class CDAIE(InfoExtractor):
  26     _VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
  27     _BASE_URL = 'http://www.cda.pl/'
  28     _TESTS = [{
  29         'url': 'http://www.cda.pl/video/5749950c',
  30         'md5': '6f844bf51b15f31fae165365707ae970',
  31         'info_dict': {
  32             'id': '5749950c',
  33             'ext': 'mp4',
  34             'height': 720,
  35             'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
  36             'description': 'md5:269ccd135d550da90d1662651fcb9772',
  37             'thumbnail': r're:^https?://.*\.jpg$',
  38             'average_rating': float,
  39             'duration': 39,
  40             'age_limit': 0,
  41         }
  42     }, {
  43         'url': 'http://www.cda.pl/video/57413289',
  44         'md5': 'a88828770a8310fc00be6c95faf7f4d5',
  45         'info_dict': {
  46             'id': '57413289',
  47             'ext': 'mp4',
  48             'title': 'Lądowanie na lotnisku na Maderze',
  49             'description': 'md5:60d76b71186dcce4e0ba6d4bbdb13e1a',
  50             'thumbnail': r're:^https?://.*\.jpg$',
  51             'uploader': 'crash404',
  52             'view_count': int,
  53             'average_rating': float,
  54             'duration': 137,
  55             'age_limit': 0,
  56         }
  57     }, {
  58         # Age-restricted
  59         'url': 'http://www.cda.pl/video/1273454c4',
  60         'info_dict': {
  61             'id': '1273454c4',
  62             'ext': 'mp4',
  63             'title': 'Bronson (2008) napisy HD 1080p',
  64             'description': 'md5:1b6cb18508daf2dc4e0fa4db77fec24c',
  65             'height': 1080,
  66             'uploader': 'boniek61',
  67             'thumbnail': r're:^https?://.*\.jpg$',
  68             'duration': 5554,
  69             'age_limit': 18,
  70             'view_count': int,
  71             'average_rating': float,
  72         },
  73     }, {
  74         'url': 'http://ebd.cda.pl/0x0/5749950c',
  75         'only_matching': True,
  76     }]
  77
  78     def _download_age_confirm_page(self, url, video_id, *args, **kwargs):
  79         form_data = random_birthday('rok', 'miesiac', 'dzien')
  80         form_data.update({'return': url, 'module': 'video', 'module_id': video_id})
  81         data, content_type = multipart_encode(form_data)
  82         return self._download_webpage(
  83             urljoin(url, '/a/validatebirth'), video_id, *args,
  84             data=data, headers={
  85                 'Referer': url,
  86                 'Content-Type': content_type,
  87             }, **kwargs)
  88
  89     def _real_extract(self, url):
  90         video_id = self._match_id(url)
  91         self._set_cookie('cda.pl', 'cda.player', 'html5')
  92         webpage = self._download_webpage(
  93             self._BASE_URL + '/video/' + video_id, video_id)
  94
  95         if 'Ten film jest dostępny dla użytkowników premium' in webpage:
  96             raise ExtractorError('This video is only available for premium users.', expected=True)
  97
  98         need_confirm_age = False
  99         if self._html_search_regex(r'(<form[^>]+action="[^"]*/a/validatebirth[^"]*")',
 100                                    webpage, 'birthday validate form', default=None):
 101             webpage = self._download_age_confirm_page(
 102                 url, video_id, note='Confirming age')
 103             need_confirm_age = True
 104
 105         formats = []
 106
 107         uploader = self._search_regex(r'''(?x)
 108             <(span|meta)[^>]+itemprop=(["\'])author\2[^>]*>
 109             (?:<\1[^>]*>[^<]*</\1>|(?!</\1>)(?:.|\n))*?
 110             <(span|meta)[^>]+itemprop=(["\'])name\4[^>]*>(?P<uploader>[^<]+)</\3>
 111         ''', webpage, 'uploader', default=None, group='uploader')
 112         view_count = self._search_regex(
 113             r'Odsłony:(?:\s|&nbsp;)*([0-9]+)', webpage,
 114             'view_count', default=None)
 115         average_rating = self._search_regex(
 116             (r'<(?:span|meta)[^>]+itemprop=(["\'])ratingValue\1[^>]*>(?P<rating_value>[0-9.]+)',
 117              r'<span[^>]+\bclass=["\']rating["\'][^>]*>(?P<rating_value>[0-9.]+)'), webpage, 'rating', fatal=False,
 118             group='rating_value')
 119
 120         info_dict = {
 121             'id': video_id,
 122             'title': self._og_search_title(webpage),
 123             'description': self._og_search_description(webpage),
 124             'uploader': uploader,
 125             'view_count': int_or_none(view_count),
 126             'average_rating': float_or_none(average_rating),
 127             'thumbnail': self._og_search_thumbnail(webpage),
 128             'formats': formats,
 129             'duration': None,
 130             'age_limit': 18 if need_confirm_age else 0,
 131         }
 132
 133         # Source: https://www.cda.pl/js/player.js?t=1606154898
 134         def decrypt_file(a):
 135             for p in ('_XDDD', '_CDA', '_ADC', '_CXD', '_QWE', '_Q5', '_IKSDE'):
 136                 a = a.replace(p, '')
 137             a = compat_urllib_parse_unquote(a)
 138             b = []
 139             for c in a:
 140                 f = compat_ord(c)
 141                 b.append(compat_chr(33 + (f + 14) % 94) if 33 <= f and 126 >= f else compat_chr(f))
 142             a = ''.join(b)
 143             a = a.replace('.cda.mp4', '')
 144             for p in ('.2cda.pl', '.3cda.pl'):
 145                 a = a.replace(p, '.cda.pl')
 146             if '/upstream' in a:
 147                 a = a.replace('/upstream', '.mp4/upstream')
 148                 return 'https://' + a
 149             return 'https://' + a + '.mp4'
 150
 151         def extract_format(page, version):
 152             json_str = self._html_search_regex(
 153                 r'player_data=(\\?["\'])(?P<player_data>.+?)\1', page,
 154                 '%s player_json' % version, fatal=False, group='player_data')
 155             if not json_str:
 156                 return
 157             player_data = self._parse_json(
 158                 json_str, '%s player_data' % version, fatal=False)
 159             if not player_data:
 160                 return
 161             video = player_data.get('video')
 162             if not video or 'file' not in video:
 163                 self.report_warning('Unable to extract %s version information' % version)
 164                 return
 165             if video['file'].startswith('uggc'):
 166                 video['file'] = codecs.decode(video['file'], 'rot_13')
 167                 if video['file'].endswith('adc.mp4'):
 168                     video['file'] = video['file'].replace('adc.mp4', '.mp4')
 169             elif not video['file'].startswith('http'):
 170                 video['file'] = decrypt_file(video['file'])
 171             f = {
 172                 'url': video['file'],
 173             }
 174             m = re.search(
 175                 r'<a[^>]+data-quality="(?P<format_id>[^"]+)"[^>]+href="[^"]+"[^>]+class="[^"]*quality-btn-active[^"]*">(?P<height>[0-9]+)p',
 176                 page)
 177             if m:
 178                 f.update({
 179                     'format_id': m.group('format_id'),
 180                     'height': int(m.group('height')),
 181                 })
 182             info_dict['formats'].append(f)
 183             if not info_dict['duration']:
 184                 info_dict['duration'] = parse_duration(video.get('duration'))
 185
 186         extract_format(webpage, 'default')
 187
 188         for href, resolution in re.findall(
 189                 r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
 190                 webpage):
 191             if need_confirm_age:
 192                 handler = self._download_age_confirm_page
 193             else:
 194                 handler = self._download_webpage
 195
 196             webpage = handler(
 197                 self._BASE_URL + href, video_id,
 198                 'Downloading %s version information' % resolution, fatal=False)
 199             if not webpage:
 200                 # Manually report warning because empty page is returned when
 201                 # invalid version is requested.
 202                 self.report_warning('Unable to download %s version information' % resolution)
 203                 continue
 204
 205             extract_format(webpage, resolution)
 206
 207         self._sort_formats(formats)
 208
 209         info = self._search_json_ld(webpage, video_id, default={})
 210
 211         return merge_dicts(info_dict, info)