[yt-dlp.git] / yt_dlp / extractor / agora.py

import functools
import uuid

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    OnDemandPagedList,
    int_or_none,
    month_by_name,
    parse_duration,
    try_call,
)


class WyborczaVideoIE(InfoExtractor):
    # this id is not an article id, it has to be extracted from the article
    _VALID_URL = r'(?:wyborcza:video:|https?://wyborcza\.pl/(?:api-)?video/)(?P<id>\d+)'
    IE_NAME = 'wyborcza:video'
    _TESTS = [{
        'url': 'wyborcza:video:26207634',
        'info_dict': {
            'id': '26207634',
            'ext': 'mp4',
            'title': '- Polska w 2020 r. jest innym państwem niż w 2015 r. Nie zmieniła się konstytucja, ale jest to już inny ustrój - mówi Adam Bodnar',
            'description': ' ',
            'uploader': 'Dorota Roman',
            'duration': 2474,
            'thumbnail': r're:https://.+\.jpg',
        },
    }, {
        'url': 'https://wyborcza.pl/video/26207634',
        'only_matching': True,
    }, {
        'url': 'https://wyborcza.pl/api-video/26207634',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        meta = self._download_json(f'https://wyborcza.pl/api-video/{video_id}', video_id)

        formats = []
        base_url = meta['redirector'].replace('http://', 'https://') + meta['basePath']
        for quality in ('standard', 'high'):
            if not meta['files'].get(quality):
                continue
            formats.append({
                'url': base_url + meta['files'][quality],
                'height': int_or_none(
                    self._search_regex(
                        r'p(\d+)[a-z]+\.mp4$', meta['files'][quality],
                        'mp4 video height', default=None)),
                'format_id': quality,
            })
        if meta['files'].get('dash'):
            formats.extend(self._extract_mpd_formats(base_url + meta['files']['dash'], video_id))

        return {
            'id': video_id,
            'formats': formats,
            'title': meta.get('title'),
            'description': meta.get('lead'),
            'uploader': meta.get('signature'),
            'thumbnail': meta.get('imageUrl'),
            'duration': meta.get('duration'),
        }


class WyborczaPodcastIE(InfoExtractor):
    _VALID_URL = r'''(?x)
        https?://(?:www\.)?(?:
            wyborcza\.pl/podcast(?:/0,172673\.html)?|
            wysokieobcasy\.pl/wysokie-obcasy/0,176631\.html
        )(?:\?(?:[^&#]+?&)*podcast=(?P<id>\d+))?
    '''
    _TESTS = [{
        'url': 'https://wyborcza.pl/podcast/0,172673.html?podcast=100720#S.main_topic-K.C-B.6-L.1.podcast',
        'info_dict': {
            'id': '100720',
            'ext': 'mp3',
            'title': 'Cyfrodziewczyny. Kim były pionierki polskiej informatyki ',
            'uploader': 'Michał Nogaś ',
            'upload_date': '20210117',
            'description': 'md5:49f0a06ffc4c1931210d3ab1416a651d',
            'duration': 3684.0,
            'thumbnail': r're:https://.+\.jpg',
        },
    }, {
        'url': 'https://www.wysokieobcasy.pl/wysokie-obcasy/0,176631.html?podcast=100673',
        'info_dict': {
            'id': '100673',
            'ext': 'mp3',
            'title': 'Czym jest ubóstwo menstruacyjne i dlaczego dotyczy każdej i każdego z nas?',
            'uploader': 'Agnieszka Urazińska ',
            'upload_date': '20210115',
            'description': 'md5:c161dc035f8dbb60077011fc41274899',
            'duration': 1803.0,
            'thumbnail': r're:https://.+\.jpg',
        },
    }, {
        'url': 'https://wyborcza.pl/podcast',
        'info_dict': {
            'id': '334',
            'title': 'Gościnnie: Wyborcza, 8:10',
            'series': 'Gościnnie: Wyborcza, 8:10',
        },
        'playlist_mincount': 370,
    }, {
        'url': 'https://www.wysokieobcasy.pl/wysokie-obcasy/0,176631.html',
        'info_dict': {
            'id': '395',
            'title': 'Gościnnie: Wysokie Obcasy',
            'series': 'Gościnnie: Wysokie Obcasy',
        },
        'playlist_mincount': 12,
    }]

    def _real_extract(self, url):
        podcast_id = self._match_id(url)

        if not podcast_id:  # playlist
            podcast_id = '395' if 'wysokieobcasy.pl/' in url else '334'
            return self.url_result(TokFMAuditionIE._create_url(podcast_id), TokFMAuditionIE, podcast_id)

        meta = self._download_json('https://wyborcza.pl/api/podcast', podcast_id,
                                   query={'guid': podcast_id, 'type': 'wo' if 'wysokieobcasy.pl/' in url else None})

        day, month, year = self._search_regex(r'^(\d\d?) (\w+) (\d{4})$', meta.get('publishedDate'),
                                              'upload date', group=(1, 2, 3), default=(None, None, None))
        return {
            'id': podcast_id,
            'url': meta['url'],
            'title': meta.get('title'),
            'description': meta.get('description'),
            'thumbnail': meta.get('imageUrl'),
            'duration': parse_duration(meta.get('duration')),
            'uploader': meta.get('author'),
            'upload_date': try_call(lambda: f'{year}{month_by_name(month, lang="pl"):0>2}{day:0>2}'),
        }


class TokFMPodcastIE(InfoExtractor):
    _VALID_URL = r'(?:https?://audycje\.tokfm\.pl/podcast/|tokfm:podcast:)(?P<id>\d+),?'
    IE_NAME = 'tokfm:podcast'
    _TESTS = [{
        'url': 'https://audycje.tokfm.pl/podcast/91275,-Systemowy-rasizm-Czy-zamieszki-w-USA-po-morderstwie-w-Minneapolis-doprowadza-do-zmian-w-sluzbach-panstwowych',
        'info_dict': {
            'id': '91275',
            'ext': 'aac',
            'title': 'md5:a9b15488009065556900169fb8061cce',
            'episode': 'md5:a9b15488009065556900169fb8061cce',
            'series': 'Analizy',
        },
    }]

    def _real_extract(self, url):
        media_id = self._match_id(url)

        # in case it breaks see this but it returns a lot of useless data
        # https://api.podcast.radioagora.pl/api4/getPodcasts?podcast_id=100091&with_guests=true&with_leaders_for_mobile=true
        metadata = self._download_json(
            f'https://audycje.tokfm.pl/getp/3{media_id}', media_id, 'Downloading podcast metadata')
        if not metadata:
            raise ExtractorError('No such podcast', expected=True)
        metadata = metadata[0]

        formats = []
        for ext in ('aac', 'mp3'):
            url_data = self._download_json(
                f'https://api.podcast.radioagora.pl/api4/getSongUrl?podcast_id={media_id}&device_id={uuid.uuid4()}&ppre=false&audio={ext}',
                media_id, f'Downloading podcast {ext} URL')
            # prevents inserting the mp3 (default) multiple times
            if 'link_ssl' in url_data and f'.{ext}' in url_data['link_ssl']:
                formats.append({
                    'url': url_data['link_ssl'],
                    'ext': ext,
                    'vcodec': 'none',
                    'acodec': ext,
                })

        return {
            'id': media_id,
            'formats': formats,
            'title': metadata.get('podcast_name'),
            'series': metadata.get('series_name'),
            'episode': metadata.get('podcast_name'),
        }


class TokFMAuditionIE(InfoExtractor):
    _VALID_URL = r'(?:https?://audycje\.tokfm\.pl/audycja/|tokfm:audition:)(?P<id>\d+),?'
    IE_NAME = 'tokfm:audition'
    _TESTS = [{
        'url': 'https://audycje.tokfm.pl/audycja/218,Analizy',
        'info_dict': {
            'id': '218',
            'title': 'Analizy',
            'series': 'Analizy',
        },
        'playlist_count': 1635,
    }]

    _PAGE_SIZE = 30
    _HEADERS = {
        'User-Agent': 'Mozilla/5.0 (Linux; Android 9; Redmi 3S Build/PQ3A.190801.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/87.0.4280.101 Mobile Safari/537.36',
    }

    @staticmethod
    def _create_url(video_id):
        return f'https://audycje.tokfm.pl/audycja/{video_id}'

    def _real_extract(self, url):
        audition_id = self._match_id(url)

        data = self._download_json(
            f'https://api.podcast.radioagora.pl/api4/getSeries?series_id={audition_id}',
            audition_id, 'Downloading audition metadata', headers=self._HEADERS)
        if not data:
            raise ExtractorError('No such audition', expected=True)
        data = data[0]

        entries = OnDemandPagedList(functools.partial(
            self._fetch_page, audition_id, data), self._PAGE_SIZE)

        return {
            '_type': 'playlist',
            'id': audition_id,
            'title': data.get('series_name'),
            'series': data.get('series_name'),
            'entries': entries,
        }

    def _fetch_page(self, audition_id, data, page):
        for retry in self.RetryManager():
            podcast_page = self._download_json(
                f'https://api.podcast.radioagora.pl/api4/getPodcasts?series_id={audition_id}&limit=30&offset={page}&with_guests=true&with_leaders_for_mobile=true',
                audition_id, f'Downloading podcast list page {page + 1}', headers=self._HEADERS)
            if not podcast_page:
                retry.error = ExtractorError('Agora returned empty page', expected=True)

        for podcast in podcast_page:
            yield {
                '_type': 'url_transparent',
                'url': podcast['podcast_sharing_url'],
                'ie_key': TokFMPodcastIE.ie_key(),
                'title': podcast.get('podcast_name'),
                'episode': podcast.get('podcast_name'),
                'description': podcast.get('podcast_description'),
                'timestamp': int_or_none(podcast.get('podcast_timestamp')),
                'series': data.get('series_name'),
            }
Commit	Line	Data
78545664	1	import functools
	2	import uuid
	3
	4	from .common import InfoExtractor
	5	from ..utils import (
	6	ExtractorError,
	7	OnDemandPagedList,
	8	int_or_none,
	9	month_by_name,
	10	parse_duration,
	11	try_call,
	12	)
	13
	14
	15	class WyborczaVideoIE(InfoExtractor):
	16	# this id is not an article id, it has to be extracted from the article
	17	_VALID_URL = r'(?:wyborcza:video:\|https?://wyborcza\.pl/(?:api-)?video/)(?P<id>\d+)'
	18	IE_NAME = 'wyborcza:video'
	19	_TESTS = [{
	20	'url': 'wyborcza:video:26207634',
	21	'info_dict': {
	22	'id': '26207634',
	23	'ext': 'mp4',
	24	'title': '- Polska w 2020 r. jest innym państwem niż w 2015 r. Nie zmieniła się konstytucja, ale jest to już inny ustrój - mówi Adam Bodnar',
	25	'description': ' ',
	26	'uploader': 'Dorota Roman',
	27	'duration': 2474,
	28	'thumbnail': r're:https://.+\.jpg',
	29	},
	30	}, {
	31	'url': 'https://wyborcza.pl/video/26207634',
	32	'only_matching': True,
	33	}, {
	34	'url': 'https://wyborcza.pl/api-video/26207634',
	35	'only_matching': True,
	36	}]
	37
	38	def _real_extract(self, url):
	39	video_id = self._match_id(url)
	40	meta = self._download_json(f'https://wyborcza.pl/api-video/{video_id}', video_id)
	41
	42	formats = []
	43	base_url = meta['redirector'].replace('http://', 'https://') + meta['basePath']
	44	for quality in ('standard', 'high'):
	45	if not meta['files'].get(quality):
	46	continue
	47	formats.append({
	48	'url': base_url + meta['files'][quality],
	49	'height': int_or_none(
	50	self._search_regex(
	51	r'p(\d+)[a-z]+\.mp4$', meta['files'][quality],
	52	'mp4 video height', default=None)),
	53	'format_id': quality,
	54	})
	55	if meta['files'].get('dash'):
	56	formats.extend(self._extract_mpd_formats(base_url + meta['files']['dash'], video_id))
	57
78545664	58	return {
	59	'id': video_id,
	60	'formats': formats,
	61	'title': meta.get('title'),
	62	'description': meta.get('lead'),
	63	'uploader': meta.get('signature'),
	64	'thumbnail': meta.get('imageUrl'),
	65	'duration': meta.get('duration'),
	66	}
	67
	68
	69	class WyborczaPodcastIE(InfoExtractor):
	70	_VALID_URL = r'''(?x)
	71	https?://(?:www\.)?(?:
	72	wyborcza\.pl/podcast(?:/0,172673\.html)?\|
	73	wysokieobcasy\.pl/wysokie-obcasy/0,176631\.html
	74	)(?:\?(?:[^&#]+?&)*podcast=(?P<id>\d+))?
	75	'''
	76	_TESTS = [{
	77	'url': 'https://wyborcza.pl/podcast/0,172673.html?podcast=100720#S.main_topic-K.C-B.6-L.1.podcast',
	78	'info_dict': {
	79	'id': '100720',
	80	'ext': 'mp3',
	81	'title': 'Cyfrodziewczyny. Kim były pionierki polskiej informatyki ',
	82	'uploader': 'Michał Nogaś ',
	83	'upload_date': '20210117',
	84	'description': 'md5:49f0a06ffc4c1931210d3ab1416a651d',
	85	'duration': 3684.0,
	86	'thumbnail': r're:https://.+\.jpg',
	87	},
	88	}, {
	89	'url': 'https://www.wysokieobcasy.pl/wysokie-obcasy/0,176631.html?podcast=100673',
	90	'info_dict': {
	91	'id': '100673',
	92	'ext': 'mp3',
	93	'title': 'Czym jest ubóstwo menstruacyjne i dlaczego dotyczy każdej i każdego z nas?',
	94	'uploader': 'Agnieszka Urazińska ',
	95	'upload_date': '20210115',
	96	'description': 'md5:c161dc035f8dbb60077011fc41274899',
	97	'duration': 1803.0,
	98	'thumbnail': r're:https://.+\.jpg',
	99	},
	100	}, {
	101	'url': 'https://wyborcza.pl/podcast',
	102	'info_dict': {
	103	'id': '334',
	104	'title': 'Gościnnie: Wyborcza, 8:10',
	105	'series': 'Gościnnie: Wyborcza, 8:10',
	106	},
	107	'playlist_mincount': 370,
	108	}, {
	109	'url': 'https://www.wysokieobcasy.pl/wysokie-obcasy/0,176631.html',
	110	'info_dict': {
	111	'id': '395',
	112	'title': 'Gościnnie: Wysokie Obcasy',
	113	'series': 'Gościnnie: Wysokie Obcasy',
	114	},
	115	'playlist_mincount': 12,
	116	}]
	117
	118	def _real_extract(self, url):
	119	podcast_id = self._match_id(url)
	120
	121	if not podcast_id: # playlist
122	podcast_id = '395' if 'wysokieobcasy.pl/' in url else '334'
123	return self.url_result(TokFMAuditionIE._create_url(podcast_id), TokFMAuditionIE, podcast_id)
124
125	meta = self._download_json('https://wyborcza.pl/api/podcast', podcast_id,
126	query={'guid': podcast_id, 'type': 'wo' if 'wysokieobcasy.pl/' in url else None})
127
128	day, month, year = self._search_regex(r'^(\d\d?) (\w+) (\d{4})$', meta.get('publishedDate'),
129	'upload date', group=(1, 2, 3), default=(None, None, None))
130	return {
131	'id': podcast_id,
132	'url': meta['url'],
133	'title': meta.get('title'),
134	'description': meta.get('description'),
135	'thumbnail': meta.get('imageUrl'),
136	'duration': parse_duration(meta.get('duration')),
137	'uploader': meta.get('author'),
138	'upload_date': try_call(lambda: f'{year}{month_by_name(month, lang="pl"):0>2}{day:0>2}'),
139	}
140
141
142	class TokFMPodcastIE(InfoExtractor):
143	_VALID_URL = r'(?:https?://audycje\.tokfm\.pl/podcast/\|tokfm:podcast:)(?P<id>\d+),?'
144	IE_NAME = 'tokfm:podcast'
145	_TESTS = [{
146	'url': 'https://audycje.tokfm.pl/podcast/91275,-Systemowy-rasizm-Czy-zamieszki-w-USA-po-morderstwie-w-Minneapolis-doprowadza-do-zmian-w-sluzbach-panstwowych',
147	'info_dict': {
148	'id': '91275',
149	'ext': 'aac',
150	'title': 'md5:a9b15488009065556900169fb8061cce',
151	'episode': 'md5:a9b15488009065556900169fb8061cce',
152	'series': 'Analizy',
153	},
154	}]
155
156	def _real_extract(self, url):
157	media_id = self._match_id(url)
158
159	# in case it breaks see this but it returns a lot of useless data
160	# https://api.podcast.radioagora.pl/api4/getPodcasts?podcast_id=100091&with_guests=true&with_leaders_for_mobile=true
161	metadata = self._download_json(
162	f'https://audycje.tokfm.pl/getp/3{media_id}', media_id, 'Downloading podcast metadata')
163	if not metadata:
164	raise ExtractorError('No such podcast', expected=True)
165	metadata = metadata[0]
166
167	formats = []
168	for ext in ('aac', 'mp3'):
169	url_data = self._download_json(
170	f'https://api.podcast.radioagora.pl/api4/getSongUrl?podcast_id={media_id}&device_id={uuid.uuid4()}&ppre=false&audio={ext}',
add96eb9	171	media_id, f'Downloading podcast {ext} URL')
78545664	172	# prevents inserting the mp3 (default) multiple times
	173	if 'link_ssl' in url_data and f'.{ext}' in url_data['link_ssl']:
	174	formats.append({
	175	'url': url_data['link_ssl'],
	176	'ext': ext,
	177	'vcodec': 'none',
	178	'acodec': ext,
	179	})
	180
78545664	181	return {
	182	'id': media_id,
	183	'formats': formats,
	184	'title': metadata.get('podcast_name'),
	185	'series': metadata.get('series_name'),
	186	'episode': metadata.get('podcast_name'),
	187	}
	188
	189
	190	class TokFMAuditionIE(InfoExtractor):
	191	_VALID_URL = r'(?:https?://audycje\.tokfm\.pl/audycja/\|tokfm:audition:)(?P<id>\d+),?'
	192	IE_NAME = 'tokfm:audition'
	193	_TESTS = [{
	194	'url': 'https://audycje.tokfm.pl/audycja/218,Analizy',
	195	'info_dict': {
	196	'id': '218',
	197	'title': 'Analizy',
	198	'series': 'Analizy',
	199	},
	200	'playlist_count': 1635,
	201	}]
	202
	203	_PAGE_SIZE = 30
	204	_HEADERS = {
	205	'User-Agent': 'Mozilla/5.0 (Linux; Android 9; Redmi 3S Build/PQ3A.190801.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/87.0.4280.101 Mobile Safari/537.36',
	206	}
	207
	208	@staticmethod
add96eb9	209	def _create_url(video_id):
add96eb9	210	return f'https://audycje.tokfm.pl/audycja/{video_id}'
78545664	211
	212	def _real_extract(self, url):
	213	audition_id = self._match_id(url)
	214
	215	data = self._download_json(
	216	f'https://api.podcast.radioagora.pl/api4/getSeries?series_id={audition_id}',
	217	audition_id, 'Downloading audition metadata', headers=self._HEADERS)
	218	if not data:
	219	raise ExtractorError('No such audition', expected=True)
	220	data = data[0]
	221
	222	entries = OnDemandPagedList(functools.partial(
	223	self._fetch_page, audition_id, data), self._PAGE_SIZE)
	224
	225	return {
	226	'_type': 'playlist',
	227	'id': audition_id,
	228	'title': data.get('series_name'),
	229	'series': data.get('series_name'),
	230	'entries': entries,
	231	}
	232
	233	def _fetch_page(self, audition_id, data, page):
	234	for retry in self.RetryManager():
	235	podcast_page = self._download_json(
	236	f'https://api.podcast.radioagora.pl/api4/getPodcasts?series_id={audition_id}&limit=30&offset={page}&with_guests=true&with_leaders_for_mobile=true',
	237	audition_id, f'Downloading podcast list page {page + 1}', headers=self._HEADERS)
	238	if not podcast_page:
	239	retry.error = ExtractorError('Agora returned empty page', expected=True)
	240
	241	for podcast in podcast_page:
	242	yield {
	243	'_type': 'url_transparent',
	244	'url': podcast['podcast_sharing_url'],
	245	'ie_key': TokFMPodcastIE.ie_key(),
	246	'title': podcast.get('podcast_name'),
	247	'episode': podcast.get('podcast_name'),
	248	'description': podcast.get('podcast_description'),
	249	'timestamp': int_or_none(podcast.get('podcast_timestamp')),
	250	'series': data.get('series_name'),
	251	}