[yt-dlp.git] / yt_dlp / extractor / puhutv.py

from .common import InfoExtractor
from ..compat import compat_str
from ..networking.exceptions import HTTPError
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
    parse_resolution,
    str_or_none,
    try_get,
    unified_timestamp,
    url_or_none,
    urljoin,
)


class PuhuTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-izle'
    IE_NAME = 'puhutv'
    _TESTS = [{
        # film
        'url': 'https://puhutv.com/sut-kardesler-izle',
        'md5': 'a347470371d56e1585d1b2c8dab01c96',
        'info_dict': {
            'id': '5085',
            'display_id': 'sut-kardesler',
            'ext': 'mp4',
            'title': 'Süt Kardeşler',
            'description': 'md5:ca09da25b7e57cbb5a9280d6e48d17aa',
            'thumbnail': r're:^https?://.*\.jpg$',
            'duration': 4832.44,
            'creator': 'Arzu Film',
            'timestamp': 1561062602,
            'upload_date': '20190620',
            'release_year': 1976,
            'view_count': int,
            'tags': list,
        },
    }, {
        # episode, geo restricted, bypassable with --geo-verification-proxy
        'url': 'https://puhutv.com/jet-sosyete-1-bolum-izle',
        'only_matching': True,
    }, {
        # 4k, with subtitles
        'url': 'https://puhutv.com/dip-1-bolum-izle',
        'only_matching': True,
    }]
    _SUBTITLE_LANGS = {
        'English': 'en',
        'Deutsch': 'de',
        'عربى': 'ar'
    }

    def _real_extract(self, url):
        display_id = self._match_id(url)

        info = self._download_json(
            urljoin(url, '/api/slug/%s-izle' % display_id),
            display_id)['data']

        video_id = compat_str(info['id'])
        show = info.get('title') or {}
        title = info.get('name') or show['name']
        if info.get('display_name'):
            title = '%s %s' % (title, info['display_name'])

        try:
            videos = self._download_json(
                'https://puhutv.com/api/assets/%s/videos' % video_id,
                display_id, 'Downloading video JSON',
                headers=self.geo_verification_headers())
        except ExtractorError as e:
            if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                self.raise_geo_restricted()
            raise

        urls = []
        formats = []

        for video in videos['data']['videos']:
            media_url = url_or_none(video.get('url'))
            if not media_url or media_url in urls:
                continue
            urls.append(media_url)

            playlist = video.get('is_playlist')
            if (video.get('stream_type') == 'hls' and playlist is True) or 'playlist.m3u8' in media_url:
                formats.extend(self._extract_m3u8_formats(
                    media_url, video_id, 'mp4', entry_protocol='m3u8_native',
                    m3u8_id='hls', fatal=False))
                continue

            quality = int_or_none(video.get('quality'))
            f = {
                'url': media_url,
                'ext': 'mp4',
                'height': quality
            }
            video_format = video.get('video_format')
            is_hls = (video_format == 'hls' or '/hls/' in media_url or '/chunklist.m3u8' in media_url) and playlist is False
            if is_hls:
                format_id = 'hls'
                f['protocol'] = 'm3u8_native'
            elif video_format == 'mp4':
                format_id = 'http'
            else:
                continue
            if quality:
                format_id += '-%sp' % quality
            f['format_id'] = format_id
            formats.append(f)

        creator = try_get(
            show, lambda x: x['producer']['name'], compat_str)

        content = info.get('content') or {}

        images = try_get(
            content, lambda x: x['images']['wide'], dict) or {}
        thumbnails = []
        for image_id, image_url in images.items():
            if not isinstance(image_url, compat_str):
                continue
            if not image_url.startswith(('http', '//')):
                image_url = 'https://%s' % image_url
            t = parse_resolution(image_id)
            t.update({
                'id': image_id,
                'url': image_url
            })
            thumbnails.append(t)

        tags = []
        for genre in show.get('genres') or []:
            if not isinstance(genre, dict):
                continue
            genre_name = genre.get('name')
            if genre_name and isinstance(genre_name, compat_str):
                tags.append(genre_name)

        subtitles = {}
        for subtitle in content.get('subtitles') or []:
            if not isinstance(subtitle, dict):
                continue
            lang = subtitle.get('language')
            sub_url = url_or_none(subtitle.get('url') or subtitle.get('file'))
            if not lang or not isinstance(lang, compat_str) or not sub_url:
                continue
            subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = [{
                'url': sub_url
            }]

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': info.get('description') or show.get('description'),
            'season_id': str_or_none(info.get('season_id')),
            'season_number': int_or_none(info.get('season_number')),
            'episode_number': int_or_none(info.get('episode_number')),
            'release_year': int_or_none(show.get('released_at')),
            'timestamp': unified_timestamp(info.get('created_at')),
            'creator': creator,
            'view_count': int_or_none(content.get('watch_count')),
            'duration': float_or_none(content.get('duration_in_ms'), 1000),
            'tags': tags,
            'subtitles': subtitles,
            'thumbnails': thumbnails,
            'formats': formats
        }


class PuhuTVSerieIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-detay'
    IE_NAME = 'puhutv:serie'
    _TESTS = [{
        'url': 'https://puhutv.com/deniz-yildizi-detay',
        'info_dict': {
            'title': 'Deniz Yıldızı',
            'id': 'deniz-yildizi',
        },
        'playlist_mincount': 205,
    }, {
        # a film detail page which is using same url with serie page
        'url': 'https://puhutv.com/kaybedenler-kulubu-detay',
        'only_matching': True,
    }]

    def _extract_entries(self, seasons):
        for season in seasons:
            season_id = season.get('id')
            if not season_id:
                continue
            page = 1
            has_more = True
            while has_more is True:
                season = self._download_json(
                    'https://galadriel.puhutv.com/seasons/%s' % season_id,
                    season_id, 'Downloading page %s' % page, query={
                        'page': page,
                        'per': 40,
                    })
                episodes = season.get('episodes')
                if isinstance(episodes, list):
                    for ep in episodes:
                        slug_path = str_or_none(ep.get('slugPath'))
                        if not slug_path:
                            continue
                        video_id = str_or_none(int_or_none(ep.get('id')))
                        yield self.url_result(
                            'https://puhutv.com/%s' % slug_path,
                            ie=PuhuTVIE.ie_key(), video_id=video_id,
                            video_title=ep.get('name') or ep.get('eventLabel'))
                page += 1
                has_more = season.get('hasMore')

    def _real_extract(self, url):
        playlist_id = self._match_id(url)

        info = self._download_json(
            urljoin(url, '/api/slug/%s-detay' % playlist_id),
            playlist_id)['data']

        seasons = info.get('seasons')
        if seasons:
            return self.playlist_result(
                self._extract_entries(seasons), playlist_id, info.get('name'))

        # For films, these are using same url with series
        video_id = info.get('slug') or info['assets'][0]['slug']
        return self.url_result(
            'https://puhutv.com/%s-izle' % video_id,
            PuhuTVIE.ie_key(), video_id)
Commit	Line	Data
6de82b44	1	from .common import InfoExtractor
3d2623a8	2	from ..compat import compat_str
3d2623a8	3	from ..networking.exceptions import HTTPError
6de82b44	4	from ..utils import (
8fd2a7be	5	ExtractorError,
6de82b44	6	float_or_none,
e897bd82	7	int_or_none,
8fd2a7be	8	parse_resolution,
6de82b44	9	str_or_none,
6de82b44	10	try_get,
8fd2a7be S	11	unified_timestamp,
	12	url_or_none,
	13	urljoin,
6de82b44 E	14	)
	15
	16
	17	class PuhuTVIE(InfoExtractor):
8fd2a7be	18	_VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-izle'
6de82b44	19	IE_NAME = 'puhutv'
8fd2a7be S	20	_TESTS = [{
	21	# film
	22	'url': 'https://puhutv.com/sut-kardesler-izle',
162bcc68	23	'md5': 'a347470371d56e1585d1b2c8dab01c96',
8fd2a7be S	24	'info_dict': {
	25	'id': '5085',
	26	'display_id': 'sut-kardesler',
	27	'ext': 'mp4',
	28	'title': 'Süt Kardeşler',
162bcc68	29	'description': 'md5:ca09da25b7e57cbb5a9280d6e48d17aa',
8fd2a7be S	30	'thumbnail': r're:^https?://.*\.jpg$',
	31	'duration': 4832.44,
	32	'creator': 'Arzu Film',
162bcc68 RA	33	'timestamp': 1561062602,
162bcc68 RA	34	'upload_date': '20190620',
8fd2a7be S	35	'release_year': 1976,
8fd2a7be S	36	'view_count': int,
162bcc68	37	'tags': list,
6de82b44	38	},
8fd2a7be S	39	}, {
	40	# episode, geo restricted, bypassable with --geo-verification-proxy
	41	'url': 'https://puhutv.com/jet-sosyete-1-bolum-izle',
	42	'only_matching': True,
	43	}, {
	44	# 4k, with subtitles
	45	'url': 'https://puhutv.com/dip-1-bolum-izle',
	46	'only_matching': True,
	47	}]
6de82b44 E	48	_SUBTITLE_LANGS = {
	49	'English': 'en',
	50	'Deutsch': 'de',
	51	'عربى': 'ar'
	52	}
	53
	54	def _real_extract(self, url):
8fd2a7be S	55	display_id = self._match_id(url)
8fd2a7be S	56
6de82b44	57	info = self._download_json(
8fd2a7be S	58	urljoin(url, '/api/slug/%s-izle' % display_id),
8fd2a7be S	59	display_id)['data']
6de82b44	60
8fd2a7be	61	video_id = compat_str(info['id'])
162bcc68 RA	62	show = info.get('title') or {}
162bcc68 RA	63	title = info.get('name') or show['name']
6de82b44	64	if info.get('display_name'):
162bcc68	65	title = '%s %s' % (title, info['display_name'])
6de82b44	66
8fd2a7be S	67	try:
	68	videos = self._download_json(
	69	'https://puhutv.com/api/assets/%s/videos' % video_id,
	70	display_id, 'Downloading video JSON',
	71	headers=self.geo_verification_headers())
	72	except ExtractorError as e:
3d2623a8	73	if isinstance(e.cause, HTTPError) and e.cause.status == 403:
8fd2a7be S	74	self.raise_geo_restricted()
	75	raise
	76
162bcc68	77	urls = []
8fd2a7be	78	formats = []
162bcc68	79
8fd2a7be S	80	for video in videos['data']['videos']:
8fd2a7be S	81	media_url = url_or_none(video.get('url'))
162bcc68	82	if not media_url or media_url in urls:
8fd2a7be	83	continue
162bcc68 RA	84	urls.append(media_url)
162bcc68 RA	85
8fd2a7be	86	playlist = video.get('is_playlist')
162bcc68	87	if (video.get('stream_type') == 'hls' and playlist is True) or 'playlist.m3u8' in media_url:
f7f30491	88	formats.extend(self._extract_m3u8_formats(
8fd2a7be	89	media_url, video_id, 'mp4', entry_protocol='m3u8_native',
f7f30491	90	m3u8_id='hls', fatal=False))
8fd2a7be	91	continue
162bcc68	92
8fd2a7be S	93	quality = int_or_none(video.get('quality'))
	94	f = {
	95	'url': media_url,
	96	'ext': 'mp4',
	97	'height': quality
	98	}
	99	video_format = video.get('video_format')
162bcc68 RA	100	is_hls = (video_format == 'hls' or '/hls/' in media_url or '/chunklist.m3u8' in media_url) and playlist is False
162bcc68 RA	101	if is_hls:
8fd2a7be S	102	format_id = 'hls'
	103	f['protocol'] = 'm3u8_native'
	104	elif video_format == 'mp4':
	105	format_id = 'http'
8fd2a7be S	106	else:
	107	continue
	108	if quality:
	109	format_id += '-%sp' % quality
	110	f['format_id'] = format_id
	111	formats.append(f)
8fd2a7be	112
8fd2a7be	113	creator = try_get(
162bcc68	114	show, lambda x: x['producer']['name'], compat_str)
8fd2a7be	115
162bcc68	116	content = info.get('content') or {}
8fd2a7be S	117
8fd2a7be S	118	images = try_get(
162bcc68	119	content, lambda x: x['images']['wide'], dict) or {}
8fd2a7be S	120	thumbnails = []
	121	for image_id, image_url in images.items():
	122	if not isinstance(image_url, compat_str):
	123	continue
	124	if not image_url.startswith(('http', '//')):
	125	image_url = 'https://%s' % image_url
	126	t = parse_resolution(image_id)
	127	t.update({
	128	'id': image_id,
	129	'url': image_url
	130	})
	131	thumbnails.append(t)
	132
6de82b44	133	tags = []
162bcc68	134	for genre in show.get('genres') or []:
8fd2a7be	135	if not isinstance(genre, dict):
6de82b44	136	continue
8fd2a7be S	137	genre_name = genre.get('name')
	138	if genre_name and isinstance(genre_name, compat_str):
	139	tags.append(genre_name)
6de82b44 E	140
6de82b44 E	141	subtitles = {}
162bcc68	142	for subtitle in content.get('subtitles') or []:
6de82b44 E	143	if not isinstance(subtitle, dict):
	144	continue
	145	lang = subtitle.get('language')
162bcc68	146	sub_url = url_or_none(subtitle.get('url') or subtitle.get('file'))
6de82b44 E	147	if not lang or not isinstance(lang, compat_str) or not sub_url:
	148	continue
	149	subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = [{
	150	'url': sub_url
	151	}]
	152
6de82b44 E	153	return {
	154	'id': video_id,
	155	'display_id': display_id,
	156	'title': title,
162bcc68 RA	157	'description': info.get('description') or show.get('description'),
	158	'season_id': str_or_none(info.get('season_id')),
	159	'season_number': int_or_none(info.get('season_number')),
	160	'episode_number': int_or_none(info.get('episode_number')),
	161	'release_year': int_or_none(show.get('released_at')),
	162	'timestamp': unified_timestamp(info.get('created_at')),
8fd2a7be	163	'creator': creator,
162bcc68 RA	164	'view_count': int_or_none(content.get('watch_count')),
162bcc68 RA	165	'duration': float_or_none(content.get('duration_in_ms'), 1000),
6de82b44 E	166	'tags': tags,
6de82b44 E	167	'subtitles': subtitles,
6de82b44 E	168	'thumbnails': thumbnails,
	169	'formats': formats
	170	}
	171
	172
	173	class PuhuTVSerieIE(InfoExtractor):
8fd2a7be	174	_VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-detay'
6de82b44	175	IE_NAME = 'puhutv:serie'
8fd2a7be S	176	_TESTS = [{
	177	'url': 'https://puhutv.com/deniz-yildizi-detay',
	178	'info_dict': {
	179	'title': 'Deniz Yıldızı',
	180	'id': 'deniz-yildizi',
6de82b44	181	},
8fd2a7be S	182	'playlist_mincount': 205,
	183	}, {
	184	# a film detail page which is using same url with serie page
	185	'url': 'https://puhutv.com/kaybedenler-kulubu-detay',
	186	'only_matching': True,
	187	}]
	188
	189	def _extract_entries(self, seasons):
6de82b44	190	for season in seasons:
8fd2a7be S	191	season_id = season.get('id')
	192	if not season_id:
	193	continue
	194	page = 1
6de82b44 E	195	has_more = True
6de82b44 E	196	while has_more is True:
8fd2a7be	197	season = self._download_json(
6de82b44	198	'https://galadriel.puhutv.com/seasons/%s' % season_id,
8fd2a7be S	199	season_id, 'Downloading page %s' % page, query={
8fd2a7be S	200	'page': page,
6de82b44 E	201	'per': 40,
6de82b44 E	202	})
8fd2a7be S	203	episodes = season.get('episodes')
	204	if isinstance(episodes, list):
	205	for ep in episodes:
	206	slug_path = str_or_none(ep.get('slugPath'))
	207	if not slug_path:
	208	continue
	209	video_id = str_or_none(int_or_none(ep.get('id')))
	210	yield self.url_result(
	211	'https://puhutv.com/%s' % slug_path,
	212	ie=PuhuTVIE.ie_key(), video_id=video_id,
	213	video_title=ep.get('name') or ep.get('eventLabel'))
	214	page += 1
	215	has_more = season.get('hasMore')
6de82b44 E	216
	217	def _real_extract(self, url):
	218	playlist_id = self._match_id(url)
	219
	220	info = self._download_json(
8fd2a7be S	221	urljoin(url, '/api/slug/%s-detay' % playlist_id),
8fd2a7be S	222	playlist_id)['data']
6de82b44	223
6de82b44 E	224	seasons = info.get('seasons')
6de82b44 E	225	if seasons:
8fd2a7be S	226	return self.playlist_result(
	227	self._extract_entries(seasons), playlist_id, info.get('name'))
	228
	229	# For films, these are using same url with series
	230	video_id = info.get('slug') or info['assets'][0]['slug']
	231	return self.url_result(
	232	'https://puhutv.com/%s-izle' % video_id,
	233	PuhuTVIE.ie_key(), video_id)