[yt-dlp.git] / youtube_dl / extractor / ceskatelevize.py

# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    float_or_none,
    sanitized_Request,
    unescapeHTML,
    update_url_query,
    urlencode_postdata,
    USER_AGENTS,
)


class CeskaTelevizeIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/ivysilani/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
    _TESTS = [{
        'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
        'info_dict': {
            'id': '61924494877246241',
            'ext': 'mp4',
            'title': 'Hyde Park Civilizace: Život v Grónsku',
            'description': 'md5:3fec8f6bb497be5cdb0c9e8781076626',
            'thumbnail': r're:^https?://.*\.jpg',
            'duration': 3350,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
        'info_dict': {
            'id': '61924494877028507',
            'ext': 'mp4',
            'title': 'Hyde Park Civilizace: Bonus 01 - En',
            'description': 'English Subtittles',
            'thumbnail': r're:^https?://.*\.jpg',
            'duration': 81.3,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # live stream
        'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/',
        'info_dict': {
            'id': 402,
            'ext': 'mp4',
            'title': r're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
            'is_live': True,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'Georestricted to Czech Republic',
    }, {
        'url': 'http://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php?hash=d6a3e1370d2e4fa76296b90bad4dfc19673b641e&IDEC=217 562 22150/0004&channelID=1&width=100%25',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        webpage = self._download_webpage(url, playlist_id)

        NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
        if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
            raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)

        type_ = None
        episode_id = None

        playlist = self._parse_json(
            self._search_regex(
                r'getPlaylistUrl\(\[({.+?})\]', webpage, 'playlist',
                default='{}'), playlist_id)
        if playlist:
            type_ = playlist.get('type')
            episode_id = playlist.get('id')

        if not type_:
            type_ = self._html_search_regex(
                r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],',
                webpage, 'type')
        if not episode_id:
            episode_id = self._html_search_regex(
                r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],',
                webpage, 'episode_id')

        data = {
            'playlist[0][type]': type_,
            'playlist[0][id]': episode_id,
            'requestUrl': compat_urllib_parse_urlparse(url).path,
            'requestSource': 'iVysilani',
        }

        entries = []

        for user_agent in (None, USER_AGENTS['Safari']):
            req = sanitized_Request(
                'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
                data=urlencode_postdata(data))

            req.add_header('Content-type', 'application/x-www-form-urlencoded')
            req.add_header('x-addr', '127.0.0.1')
            req.add_header('X-Requested-With', 'XMLHttpRequest')
            if user_agent:
                req.add_header('User-Agent', user_agent)
            req.add_header('Referer', url)

            playlistpage = self._download_json(req, playlist_id, fatal=False)

            if not playlistpage:
                continue

            playlist_url = playlistpage['url']
            if playlist_url == 'error_region':
                raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)

            req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
            req.add_header('Referer', url)

            playlist_title = self._og_search_title(webpage, default=None)
            playlist_description = self._og_search_description(webpage, default=None)

            playlist = self._download_json(req, playlist_id, fatal=False)
            if not playlist:
                continue

            playlist = playlist.get('playlist')
            if not isinstance(playlist, list):
                continue

            playlist_len = len(playlist)

            for num, item in enumerate(playlist):
                is_live = item.get('type') == 'LIVE'
                formats = []
                for format_id, stream_url in item.get('streamUrls', {}).items():
                    if 'playerType=flash' in stream_url:
                        stream_formats = self._extract_m3u8_formats(
                            stream_url, playlist_id, 'mp4', 'm3u8_native',
                            m3u8_id='hls-%s' % format_id, fatal=False)
                    else:
                        stream_formats = self._extract_mpd_formats(
                            stream_url, playlist_id,
                            mpd_id='dash-%s' % format_id, fatal=False)
                    # See https://github.com/ytdl-org/youtube-dl/issues/12119#issuecomment-280037031
                    if format_id == 'audioDescription':
                        for f in stream_formats:
                            f['source_preference'] = -10
                    formats.extend(stream_formats)

                if user_agent and len(entries) == playlist_len:
                    entries[num]['formats'].extend(formats)
                    continue

                item_id = item.get('id') or item['assetId']
                title = item['title']

                duration = float_or_none(item.get('duration'))
                thumbnail = item.get('previewImageUrl')

                subtitles = {}
                if item.get('type') == 'VOD':
                    subs = item.get('subtitles')
                    if subs:
                        subtitles = self.extract_subtitles(episode_id, subs)

                if playlist_len == 1:
                    final_title = playlist_title or title
                    if is_live:
                        final_title = self._live_title(final_title)
                else:
                    final_title = '%s (%s)' % (playlist_title, title)

                entries.append({
                    'id': item_id,
                    'title': final_title,
                    'description': playlist_description if playlist_len == 1 else None,
                    'thumbnail': thumbnail,
                    'duration': duration,
                    'formats': formats,
                    'subtitles': subtitles,
                    'is_live': is_live,
                })

        for e in entries:
            self._sort_formats(e['formats'])

        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

    def _get_subtitles(self, episode_id, subs):
        original_subtitles = self._download_webpage(
            subs[0]['url'], episode_id, 'Downloading subtitles')
        srt_subs = self._fix_subtitles(original_subtitles)
        return {
            'cs': [{
                'ext': 'srt',
                'data': srt_subs,
            }]
        }

    @staticmethod
    def _fix_subtitles(subtitles):
        """ Convert millisecond-based subtitles to SRT """

        def _msectotimecode(msec):
            """ Helper utility to convert milliseconds to timecode """
            components = []
            for divider in [1000, 60, 60, 100]:
                components.append(msec % divider)
                msec //= divider
            return '{3:02}:{2:02}:{1:02},{0:03}'.format(*components)

        def _fix_subtitle(subtitle):
            for line in subtitle.splitlines():
                m = re.match(r'^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$', line)
                if m:
                    yield m.group(1)
                    start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
                    yield '{0} --> {1}'.format(start, stop)
                else:
                    yield line

        return '\r\n'.join(_fix_subtitle(subtitles))


class CeskaTelevizePoradyIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/porady/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
    _TESTS = [{
        # video with 18+ caution trailer
        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
        'info_dict': {
            'id': '215562210900007-bogotart',
            'title': 'Queer: Bogotart',
            'description': 'Alternativní průvodce současným queer světem',
        },
        'playlist': [{
            'info_dict': {
                'id': '61924494876844842',
                'ext': 'mp4',
                'title': 'Queer: Bogotart (Varování 18+)',
                'duration': 10.2,
            },
        }, {
            'info_dict': {
                'id': '61924494877068022',
                'ext': 'mp4',
                'title': 'Queer: Bogotart (Queer)',
                'thumbnail': r're:^https?://.*\.jpg',
                'duration': 1558.3,
            },
        }],
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }, {
        # iframe embed
        'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        data_url = update_url_query(unescapeHTML(self._search_regex(
            (r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
             r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?ceskatelevize\.cz/ivysilani/embed/iFramePlayer\.php.*?)\1'),
            webpage, 'iframe player url', group='url')), query={
                'autoStart': 'true',
        })

        return self.url_result(data_url, ie=CeskaTelevizeIE.ie_key())
Commit	Line	Data
dcdb292f	1	# coding: utf-8
8cfb5bbf	2	from __future__ import unicode_literals
	3
	4	import re
8cfb5bbf	5
48246541	6	from .common import InfoExtractor
1cc79574	7	from ..compat import (
3e72f5f1	8	compat_urllib_parse_unquote,
39201787	9	compat_urllib_parse_urlparse,
1cc79574 PH	10	)
1cc79574 PH	11	from ..utils import (
39201787	12	ExtractorError,
02ec32a1	13	float_or_none,
5c2266df	14	sanitized_Request,
28b674ca	15	unescapeHTML,
9a054fcb	16	update_url_query,
6e6bc8da	17	urlencode_postdata,
5cb2d36c	18	USER_AGENTS,
8cfb5bbf	19	)
	20
	21
48246541	22	class CeskaTelevizeIE(InfoExtractor):
e18f1da9	23	_VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/ivysilani/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
97243fe3 S	24	_TESTS = [{
	25	'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
	26	'info_dict': {
fcca0d53	27	'id': '61924494877246241',
97243fe3	28	'ext': 'mp4',
fcca0d53 OC	29	'title': 'Hyde Park Civilizace: Život v Grónsku',
fcca0d53 OC	30	'description': 'md5:3fec8f6bb497be5cdb0c9e8781076626',
ec85ded8	31	'thumbnail': r're:^https?://.*\.jpg',
97243fe3 S	32	'duration': 3350,
	33	},
	34	'params': {
	35	# m3u8 download
	36	'skip_download': True,
	37	},
3951e7eb S	38	}, {
	39	'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
	40	'info_dict': {
	41	'id': '61924494877028507',
	42	'ext': 'mp4',
	43	'title': 'Hyde Park Civilizace: Bonus 01 - En',
	44	'description': 'English Subtittles',
ec85ded8	45	'thumbnail': r're:^https?://.*\.jpg',
3951e7eb S	46	'duration': 81.3,
	47	},
	48	'params': {
	49	# m3u8 download
	50	'skip_download': True,
	51	},
97243fe3	52	}, {
f1f6f5aa TČ	53	# live stream
f1f6f5aa TČ	54	'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/',
97243fe3	55	'info_dict': {
f1f6f5aa	56	'id': 402,
97243fe3	57	'ext': 'mp4',
ec85ded8	58	'title': r're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
f1f6f5aa	59	'is_live': True,
97243fe3 S	60	},
	61	'params': {
	62	# m3u8 download
	63	'skip_download': True,
	64	},
3951e7eb	65	'skip': 'Georestricted to Czech Republic',
97243fe3	66	}, {
e18f1da9 S	67	'url': 'http://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php?hash=d6a3e1370d2e4fa76296b90bad4dfc19673b641e&IDEC=217 562 22150/0004&channelID=1&width=100%25',
e18f1da9 S	68	'only_matching': True,
97243fe3	69	}]
8cfb5bbf	70
8cfb5bbf	71	def _real_extract(self, url):
e18f1da9	72	playlist_id = self._match_id(url)
8cfb5bbf	73
97243fe3	74	webpage = self._download_webpage(url, playlist_id)
8cfb5bbf	75
7d78f0cc S	76	NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
	77	if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
	78	raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
39201787	79
e18f1da9 S	80	type_ = None
	81	episode_id = None
	82
	83	playlist = self._parse_json(
	84	self._search_regex(
	85	r'getPlaylistUrl\(\[({.+?})\]', webpage, 'playlist',
	86	default='{}'), playlist_id)
	87	if playlist:
	88	type_ = playlist.get('type')
	89	episode_id = playlist.get('id')
	90
	91	if not type_:
	92	type_ = self._html_search_regex(
	93	r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],',
	94	webpage, 'type')
	95	if not episode_id:
	96	episode_id = self._html_search_regex(
	97	r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],',
	98	webpage, 'episode_id')
8cfb5bbf	99
8cfb5bbf	100	data = {
e18f1da9	101	'playlist[0][type]': type_,
8cfb5bbf	102	'playlist[0][id]': episode_id,
	103	'requestUrl': compat_urllib_parse_urlparse(url).path,
	104	'requestSource': 'iVysilani',
	105	}
	106
97243fe3	107	entries = []
5cb2d36c S	108
	109	for user_agent in (None, USER_AGENTS['Safari']):
	110	req = sanitized_Request(
1a88fc5a	111	'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
5cb2d36c S	112	data=urlencode_postdata(data))
	113
	114	req.add_header('Content-type', 'application/x-www-form-urlencoded')
	115	req.add_header('x-addr', '127.0.0.1')
	116	req.add_header('X-Requested-With', 'XMLHttpRequest')
	117	if user_agent:
	118	req.add_header('User-Agent', user_agent)
	119	req.add_header('Referer', url)
	120
	121	playlistpage = self._download_json(req, playlist_id, fatal=False)
	122
	123	if not playlistpage:
	124	continue
	125
	126	playlist_url = playlistpage['url']
	127	if playlist_url == 'error_region':
	128	raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
	129
	130	req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
	131	req.add_header('Referer', url)
	132
	133	playlist_title = self._og_search_title(webpage, default=None)
	134	playlist_description = self._og_search_description(webpage, default=None)
	135
	136	playlist = self._download_json(req, playlist_id, fatal=False)
	137	if not playlist:
	138	continue
	139
	140	playlist = playlist.get('playlist')
	141	if not isinstance(playlist, list):
	142	continue
	143
	144	playlist_len = len(playlist)
	145
	146	for num, item in enumerate(playlist):
	147	is_live = item.get('type') == 'LIVE'
	148	formats = []
	149	for format_id, stream_url in item.get('streamUrls', {}).items():
	150	if 'playerType=flash' in stream_url:
eafaeb22	151	stream_formats = self._extract_m3u8_formats(
fb4fc449	152	stream_url, playlist_id, 'mp4', 'm3u8_native',
eafaeb22	153	m3u8_id='hls-%s' % format_id, fatal=False)
5cb2d36c	154	else:
eafaeb22 S	155	stream_formats = self._extract_mpd_formats(
	156	stream_url, playlist_id,
	157	mpd_id='dash-%s' % format_id, fatal=False)
067aa17e	158	# See https://github.com/ytdl-org/youtube-dl/issues/12119#issuecomment-280037031
eafaeb22 S	159	if format_id == 'audioDescription':
	160	for f in stream_formats:
	161	f['source_preference'] = -10
	162	formats.extend(stream_formats)
5cb2d36c S	163
	164	if user_agent and len(entries) == playlist_len:
	165	entries[num]['formats'].extend(formats)
	166	continue
	167
	168	item_id = item.get('id') or item['assetId']
	169	title = item['title']
	170
	171	duration = float_or_none(item.get('duration'))
	172	thumbnail = item.get('previewImageUrl')
	173
	174	subtitles = {}
	175	if item.get('type') == 'VOD':
	176	subs = item.get('subtitles')
	177	if subs:
	178	subtitles = self.extract_subtitles(episode_id, subs)
	179
	180	if playlist_len == 1:
	181	final_title = playlist_title or title
	182	if is_live:
	183	final_title = self._live_title(final_title)
	184	else:
	185	final_title = '%s (%s)' % (playlist_title, title)
	186
	187	entries.append({
	188	'id': item_id,
	189	'title': final_title,
	190	'description': playlist_description if playlist_len == 1 else None,
	191	'thumbnail': thumbnail,
	192	'duration': duration,
	193	'formats': formats,
	194	'subtitles': subtitles,
	195	'is_live': is_live,
	196	})
	197
	198	for e in entries:
	199	self._sort_formats(e['formats'])
97243fe3 S	200
97243fe3 S	201	return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
c067545c	202
48246541 JMF	203	def _get_subtitles(self, episode_id, subs):
	204	original_subtitles = self._download_webpage(
	205	subs[0]['url'], episode_id, 'Downloading subtitles')
	206	srt_subs = self._fix_subtitles(original_subtitles)
	207	return {
	208	'cs': [{
	209	'ext': 'srt',
	210	'data': srt_subs,
	211	}]
	212	}
	213
27a82a1b S	214	@staticmethod
	215	def _fix_subtitles(subtitles):
	216	""" Convert millisecond-based subtitles to SRT """
c067545c OC	217
c067545c OC	218	def _msectotimecode(msec):
27a82a1b	219	""" Helper utility to convert milliseconds to timecode """
c067545c OC	220	components = []
	221	for divider in [1000, 60, 60, 100]:
	222	components.append(msec % divider)
	223	msec //= divider
611c1dd9	224	return '{3:02}:{2:02}:{1:02},{0:03}'.format(*components)
c067545c OC	225
	226	def _fix_subtitle(subtitle):
	227	for line in subtitle.splitlines():
611c1dd9	228	m = re.match(r'^\s([0-9]+);\s([0-9]+)\s+([0-9]+)\s*$', line)
c067545c OC	229	if m:
	230	yield m.group(1)
	231	start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
611c1dd9	232	yield '{0} --> {1}'.format(start, stop)
c067545c OC	233	else:
	234	yield line
	235
611c1dd9	236	return '\r\n'.join(_fix_subtitle(subtitles))
28b674ca S	237
	238
	239	class CeskaTelevizePoradyIE(InfoExtractor):
	240	_VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/porady/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
	241	_TESTS = [{
	242	# video with 18+ caution trailer
	243	'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
	244	'info_dict': {
	245	'id': '215562210900007-bogotart',
	246	'title': 'Queer: Bogotart',
	247	'description': 'Alternativní průvodce současným queer světem',
	248	},
	249	'playlist': [{
	250	'info_dict': {
	251	'id': '61924494876844842',
	252	'ext': 'mp4',
	253	'title': 'Queer: Bogotart (Varování 18+)',
	254	'duration': 10.2,
	255	},
	256	}, {
	257	'info_dict': {
	258	'id': '61924494877068022',
	259	'ext': 'mp4',
	260	'title': 'Queer: Bogotart (Queer)',
	261	'thumbnail': r're:^https?://.*\.jpg',
	262	'duration': 1558.3,
	263	},
	264	}],
	265	'params': {
	266	# m3u8 download
	267	'skip_download': True,
	268	},
9a054fcb S	269	}, {
	270	# iframe embed
	271	'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
	272	'only_matching': True,
28b674ca S	273	}]
	274
	275	def _real_extract(self, url):
	276	video_id = self._match_id(url)
	277
	278	webpage = self._download_webpage(url, video_id)
	279
9a054fcb S	280	data_url = update_url_query(unescapeHTML(self._search_regex(
	281	(r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
	282	r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?ceskatelevize\.cz/ivysilani/embed/iFramePlayer\.php.*?)\1'),
	283	webpage, 'iframe player url', group='url')), query={
	284	'autoStart': 'true',
	285	})
28b674ca S	286
28b674ca S	287	return self.url_result(data_url, ie=CeskaTelevizeIE.ie_key())