[yt-dlp.git] / yt_dlp / extractor / cda.py

# coding: utf-8
from __future__ import unicode_literals

import codecs
import re

from .common import InfoExtractor
from ..compat import (
    compat_chr,
    compat_ord,
    compat_urllib_parse_unquote,
)
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
    merge_dicts,
    multipart_encode,
    parse_duration,
    random_birthday,
    urljoin,
)


class CDAIE(InfoExtractor):
    _VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
    _BASE_URL = 'http://www.cda.pl/'
    _TESTS = [{
        'url': 'http://www.cda.pl/video/5749950c',
        'md5': '6f844bf51b15f31fae165365707ae970',
        'info_dict': {
            'id': '5749950c',
            'ext': 'mp4',
            'height': 720,
            'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
            'description': 'md5:269ccd135d550da90d1662651fcb9772',
            'thumbnail': r're:^https?://.*\.jpg$',
            'average_rating': float,
            'duration': 39,
            'age_limit': 0,
        }
    }, {
        'url': 'http://www.cda.pl/video/57413289',
        'md5': 'a88828770a8310fc00be6c95faf7f4d5',
        'info_dict': {
            'id': '57413289',
            'ext': 'mp4',
            'title': 'Lądowanie na lotnisku na Maderze',
            'description': 'md5:60d76b71186dcce4e0ba6d4bbdb13e1a',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader': 'crash404',
            'view_count': int,
            'average_rating': float,
            'duration': 137,
            'age_limit': 0,
        }
    }, {
        # Age-restricted
        'url': 'http://www.cda.pl/video/1273454c4',
        'info_dict': {
            'id': '1273454c4',
            'ext': 'mp4',
            'title': 'Bronson (2008) napisy HD 1080p',
            'description': 'md5:1b6cb18508daf2dc4e0fa4db77fec24c',
            'height': 1080,
            'uploader': 'boniek61',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 5554,
            'age_limit': 18,
            'view_count': int,
            'average_rating': float,
        },
    }, {
        'url': 'http://ebd.cda.pl/0x0/5749950c',
        'only_matching': True,
    }]

    def _download_age_confirm_page(self, url, video_id, *args, **kwargs):
        form_data = random_birthday('rok', 'miesiac', 'dzien')
        form_data.update({'return': url, 'module': 'video', 'module_id': video_id})
        data, content_type = multipart_encode(form_data)
        return self._download_webpage(
            urljoin(url, '/a/validatebirth'), video_id, *args,
            data=data, headers={
                'Referer': url,
                'Content-Type': content_type,
            }, **kwargs)

    def _real_extract(self, url):
        video_id = self._match_id(url)
        self._set_cookie('cda.pl', 'cda.player', 'html5')
        webpage = self._download_webpage(
            self._BASE_URL + '/video/' + video_id, video_id)

        if 'Ten film jest dostępny dla użytkowników premium' in webpage:
            raise ExtractorError('This video is only available for premium users.', expected=True)

        if re.search(r'niedostępn[ey] w(?:&nbsp;|\s+)Twoim kraju\s*<', webpage):
            self.raise_geo_restricted()

        need_confirm_age = False
        if self._html_search_regex(r'(<form[^>]+action="[^"]*/a/validatebirth[^"]*")',
                                   webpage, 'birthday validate form', default=None):
            webpage = self._download_age_confirm_page(
                url, video_id, note='Confirming age')
            need_confirm_age = True

        formats = []

        uploader = self._search_regex(r'''(?x)
            <(span|meta)[^>]+itemprop=(["\'])author\2[^>]*>
            (?:<\1[^>]*>[^<]*</\1>|(?!</\1>)(?:.|\n))*?
            <(span|meta)[^>]+itemprop=(["\'])name\4[^>]*>(?P<uploader>[^<]+)</\3>
        ''', webpage, 'uploader', default=None, group='uploader')
        view_count = self._search_regex(
            r'Odsłony:(?:\s|&nbsp;)*([0-9]+)', webpage,
            'view_count', default=None)
        average_rating = self._search_regex(
            (r'<(?:span|meta)[^>]+itemprop=(["\'])ratingValue\1[^>]*>(?P<rating_value>[0-9.]+)',
             r'<span[^>]+\bclass=["\']rating["\'][^>]*>(?P<rating_value>[0-9.]+)'), webpage, 'rating', fatal=False,
            group='rating_value')

        info_dict = {
            'id': video_id,
            'title': self._og_search_title(webpage),
            'description': self._og_search_description(webpage),
            'uploader': uploader,
            'view_count': int_or_none(view_count),
            'average_rating': float_or_none(average_rating),
            'thumbnail': self._og_search_thumbnail(webpage),
            'formats': formats,
            'duration': None,
            'age_limit': 18 if need_confirm_age else 0,
        }

        # Source: https://www.cda.pl/js/player.js?t=1606154898
        def decrypt_file(a):
            for p in ('_XDDD', '_CDA', '_ADC', '_CXD', '_QWE', '_Q5', '_IKSDE'):
                a = a.replace(p, '')
            a = compat_urllib_parse_unquote(a)
            b = []
            for c in a:
                f = compat_ord(c)
                b.append(compat_chr(33 + (f + 14) % 94) if 33 <= f and 126 >= f else compat_chr(f))
            a = ''.join(b)
            a = a.replace('.cda.mp4', '')
            for p in ('.2cda.pl', '.3cda.pl'):
                a = a.replace(p, '.cda.pl')
            if '/upstream' in a:
                a = a.replace('/upstream', '.mp4/upstream')
                return 'https://' + a
            return 'https://' + a + '.mp4'

        def extract_format(page, version):
            json_str = self._html_search_regex(
                r'player_data=(\\?["\'])(?P<player_data>.+?)\1', page,
                '%s player_json' % version, fatal=False, group='player_data')
            if not json_str:
                return
            player_data = self._parse_json(
                json_str, '%s player_data' % version, fatal=False)
            if not player_data:
                return
            video = player_data.get('video')
            if not video or 'file' not in video:
                self.report_warning('Unable to extract %s version information' % version)
                return
            if video['file'].startswith('uggc'):
                video['file'] = codecs.decode(video['file'], 'rot_13')
                if video['file'].endswith('adc.mp4'):
                    video['file'] = video['file'].replace('adc.mp4', '.mp4')
            elif not video['file'].startswith('http'):
                video['file'] = decrypt_file(video['file'])
            f = {
                'url': video['file'],
            }
            m = re.search(
                r'<a[^>]+data-quality="(?P<format_id>[^"]+)"[^>]+href="[^"]+"[^>]+class="[^"]*quality-btn-active[^"]*">(?P<height>[0-9]+)p',
                page)
            if m:
                f.update({
                    'format_id': m.group('format_id'),
                    'height': int(m.group('height')),
                })
            info_dict['formats'].append(f)
            if not info_dict['duration']:
                info_dict['duration'] = parse_duration(video.get('duration'))

        extract_format(webpage, 'default')

        for href, resolution in re.findall(
                r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
                webpage):
            if need_confirm_age:
                handler = self._download_age_confirm_page
            else:
                handler = self._download_webpage

            webpage = handler(
                self._BASE_URL + href, video_id,
                'Downloading %s version information' % resolution, fatal=False)
            if not webpage:
                # Manually report warning because empty page is returned when
                # invalid version is requested.
                self.report_warning('Unable to download %s version information' % resolution)
                continue

            extract_format(webpage, resolution)

        self._sort_formats(formats)

        info = self._search_json_ld(webpage, video_id, default={})

        return merge_dicts(info_dict, info)
Commit	Line	Data
8b0d7a66 KM	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
fdeea726	4	import codecs
8b0d7a66 KM	5	import re
	6
	7	from .common import InfoExtractor
38d70284	8	from ..compat import (
	9	compat_chr,
	10	compat_ord,
	11	compat_urllib_parse_unquote,
	12	)
8b0d7a66	13	from ..utils import (
8b0d7a66	14	ExtractorError,
577281b0 KM	15	float_or_none,
577281b0 KM	16	int_or_none,
38d70284	17	merge_dicts,
0c265486	18	multipart_encode,
577281b0	19	parse_duration,
0c265486 YCH	20	random_birthday,
0c265486 YCH	21	urljoin,
8b0d7a66 KM	22	)
	23
	24
	25	class CDAIE(InfoExtractor):
f1ced6df	26	_VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video\|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
577281b0	27	_BASE_URL = 'http://www.cda.pl/'
f1ced6df S	28	_TESTS = [{
	29	'url': 'http://www.cda.pl/video/5749950c',
	30	'md5': '6f844bf51b15f31fae165365707ae970',
	31	'info_dict': {
	32	'id': '5749950c',
	33	'ext': 'mp4',
	34	'height': 720,
	35	'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
577281b0	36	'description': 'md5:269ccd135d550da90d1662651fcb9772',
ec85ded8	37	'thumbnail': r're:^https?://.*\.jpg$',
577281b0	38	'average_rating': float,
0c265486 YCH	39	'duration': 39,
0c265486 YCH	40	'age_limit': 0,
f1ced6df S	41	}
	42	}, {
	43	'url': 'http://www.cda.pl/video/57413289',
	44	'md5': 'a88828770a8310fc00be6c95faf7f4d5',
	45	'info_dict': {
	46	'id': '57413289',
	47	'ext': 'mp4',
	48	'title': 'Lądowanie na lotnisku na Maderze',
577281b0	49	'description': 'md5:60d76b71186dcce4e0ba6d4bbdb13e1a',
ec85ded8	50	'thumbnail': r're:^https?://.*\.jpg$',
577281b0 KM	51	'uploader': 'crash404',
	52	'view_count': int,
	53	'average_rating': float,
0c265486 YCH	54	'duration': 137,
0c265486 YCH	55	'age_limit': 0,
8b0d7a66	56	}
0c265486 YCH	57	}, {
	58	# Age-restricted
	59	'url': 'http://www.cda.pl/video/1273454c4',
	60	'info_dict': {
	61	'id': '1273454c4',
	62	'ext': 'mp4',
	63	'title': 'Bronson (2008) napisy HD 1080p',
	64	'description': 'md5:1b6cb18508daf2dc4e0fa4db77fec24c',
	65	'height': 1080,
	66	'uploader': 'boniek61',
	67	'thumbnail': r're:^https?://.*\.jpg$',
	68	'duration': 5554,
	69	'age_limit': 18,
	70	'view_count': int,
	71	'average_rating': float,
	72	},
f1ced6df S	73	}, {
	74	'url': 'http://ebd.cda.pl/0x0/5749950c',
	75	'only_matching': True,
	76	}]
8b0d7a66	77
0c265486 YCH	78	def _download_age_confirm_page(self, url, video_id, args, *kwargs):
	79	form_data = random_birthday('rok', 'miesiac', 'dzien')
	80	form_data.update({'return': url, 'module': 'video', 'module_id': video_id})
	81	data, content_type = multipart_encode(form_data)
	82	return self._download_webpage(
	83	urljoin(url, '/a/validatebirth'), video_id, *args,
	84	data=data, headers={
	85	'Referer': url,
	86	'Content-Type': content_type,
	87	}, **kwargs)
	88
8b0d7a66 KM	89	def _real_extract(self, url):
8b0d7a66 KM	90	video_id = self._match_id(url)
577281b0 KM	91	self._set_cookie('cda.pl', 'cda.player', 'html5')
	92	webpage = self._download_webpage(
	93	self._BASE_URL + '/video/' + video_id, video_id)
8b0d7a66 KM	94
	95	if 'Ten film jest dostępny dla użytkowników premium' in webpage:
	96	raise ExtractorError('This video is only available for premium users.', expected=True)
	97
cc2db878	98	if re.search(r'niedostępn[ey] w(?: \|\s+)Twoim kraju\s*<', webpage):
	99	self.raise_geo_restricted()
	100
0c265486	101	need_confirm_age = False
2181983a	102	if self._html_search_regex(r'(<form[^>]+action="[^"]/a/validatebirth[^"]")',
0c265486 YCH	103	webpage, 'birthday validate form', default=None):
	104	webpage = self._download_age_confirm_page(
	105	url, video_id, note='Confirming age')
	106	need_confirm_age = True
	107
8b0d7a66 KM	108	formats = []
8b0d7a66 KM	109
577281b0 KM	110	uploader = self._search_regex(r'''(?x)
	111	<(span\|meta)[^>]+itemprop=(["\'])author\2[^>]*>
	112	(?:<\1[^>]>[^<]</\1>\|(?!</\1>)(?:.\|\n))*?
	113	<(span\|meta)[^>]+itemprop=(["\'])name\4[^>]*>(?P<uploader>[^<]+)</\3>
	114	''', webpage, 'uploader', default=None, group='uploader')
	115	view_count = self._search_regex(
	116	r'Odsłony:(?:\s\| )*([0-9]+)', webpage,
	117	'view_count', default=None)
	118	average_rating = self._search_regex(
38d70284	119	(r'<(?:span\|meta)[^>]+itemprop=(["\'])ratingValue\1[^>]*>(?P<rating_value>[0-9.]+)',
	120	r'<span[^>]+\bclass=["\']rating["\'][^>]*>(?P<rating_value>[0-9.]+)'), webpage, 'rating', fatal=False,
	121	group='rating_value')
577281b0	122
f1ced6df S	123	info_dict = {
f1ced6df S	124	'id': video_id,
577281b0 KM	125	'title': self._og_search_title(webpage),
	126	'description': self._og_search_description(webpage),
	127	'uploader': uploader,
	128	'view_count': int_or_none(view_count),
	129	'average_rating': float_or_none(average_rating),
	130	'thumbnail': self._og_search_thumbnail(webpage),
f1ced6df S	131	'formats': formats,
f1ced6df S	132	'duration': None,
0c265486	133	'age_limit': 18 if need_confirm_age else 0,
f1ced6df	134	}
8b0d7a66	135
38d70284	136	# Source: https://www.cda.pl/js/player.js?t=1606154898
	137	def decrypt_file(a):
	138	for p in ('_XDDD', '_CDA', '_ADC', '_CXD', '_QWE', '_Q5', '_IKSDE'):
	139	a = a.replace(p, '')
	140	a = compat_urllib_parse_unquote(a)
	141	b = []
	142	for c in a:
	143	f = compat_ord(c)
	144	b.append(compat_chr(33 + (f + 14) % 94) if 33 <= f and 126 >= f else compat_chr(f))
	145	a = ''.join(b)
	146	a = a.replace('.cda.mp4', '')
	147	for p in ('.2cda.pl', '.3cda.pl'):
	148	a = a.replace(p, '.cda.pl')
	149	if '/upstream' in a:
	150	a = a.replace('/upstream', '.mp4/upstream')
	151	return 'https://' + a
	152	return 'https://' + a + '.mp4'
	153
f1ced6df	154	def extract_format(page, version):
f8f18f33	155	json_str = self._html_search_regex(
577281b0 KM	156	r'player_data=(\\?["\'])(?P<player_data>.+?)\1', page,
	157	'%s player_json' % version, fatal=False, group='player_data')
	158	if not json_str:
	159	return
	160	player_data = self._parse_json(
	161	json_str, '%s player_data' % version, fatal=False)
	162	if not player_data:
	163	return
	164	video = player_data.get('video')
	165	if not video or 'file' not in video:
	166	self.report_warning('Unable to extract %s version information' % version)
f1ced6df	167	return
fdeea726 AS	168	if video['file'].startswith('uggc'):
	169	video['file'] = codecs.decode(video['file'], 'rot_13')
	170	if video['file'].endswith('adc.mp4'):
	171	video['file'] = video['file'].replace('adc.mp4', '.mp4')
38d70284	172	elif not video['file'].startswith('http'):
38d70284	173	video['file'] = decrypt_file(video['file'])
f1ced6df	174	f = {
577281b0	175	'url': video['file'],
f1ced6df S	176	}
	177	m = re.search(
	178	r'<a[^>]+data-quality="(?P<format_id>[^"]+)"[^>]+href="[^"]+"[^>]+class="[^"]quality-btn-active[^"]">(?P<height>[0-9]+)p',
	179	page)
	180	if m:
	181	f.update({
	182	'format_id': m.group('format_id'),
	183	'height': int(m.group('height')),
	184	})
	185	info_dict['formats'].append(f)
	186	if not info_dict['duration']:
577281b0	187	info_dict['duration'] = parse_duration(video.get('duration'))
f1ced6df S	188
	189	extract_format(webpage, 'default')
	190
	191	for href, resolution in re.findall(
	192	r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
	193	webpage):
0c265486 YCH	194	if need_confirm_age:
	195	handler = self._download_age_confirm_page
	196	else:
	197	handler = self._download_webpage
	198
	199	webpage = handler(
577281b0 KM	200	self._BASE_URL + href, video_id,
577281b0 KM	201	'Downloading %s version information' % resolution, fatal=False)
8b0d7a66	202	if not webpage:
f1ced6df S	203	# Manually report warning because empty page is returned when
	204	# invalid version is requested.
	205	self.report_warning('Unable to download %s version information' % resolution)
8b0d7a66	206	continue
0c265486	207
f1ced6df	208	extract_format(webpage, resolution)
8b0d7a66 KM	209
	210	self._sort_formats(formats)
	211
38d70284	212	info = self._search_json_ld(webpage, video_id, default={})
	213
	214	return merge_dicts(info_dict, info)