[yt-dlp.git] / yt_dlp / extractor / tunein.py

import urllib.parse

from .common import InfoExtractor
from ..utils import (
    OnDemandPagedList,
    determine_ext,
    parse_iso8601,
    traverse_obj,
)


class TuneInBaseIE(InfoExtractor):
    _VALID_URL_BASE = r'https?://(?:www\.)?tunein\.com'

    def _extract_metadata(self, webpage, content_id):
        return self._search_json(r'window.INITIAL_STATE=', webpage, 'hydration', content_id, fatal=False)

    def _extract_formats_and_subtitles(self, content_id):
        streams = self._download_json(
            f'https://opml.radiotime.com/Tune.ashx?render=json&formats=mp3,aac,ogg,flash,hls&id={content_id}',
            content_id)['body']

        formats, subtitles = [], {}
        for stream in streams:
            if stream.get('media_type') == 'hls':
                fmts, subs = self._extract_m3u8_formats_and_subtitles(stream['url'], content_id, fatal=False)
                formats.extend(fmts)
                self._merge_subtitles(subs, target=subtitles)
            elif determine_ext(stream['url']) == 'pls':
                playlist_content = self._download_webpage(stream['url'], content_id)
                formats.append({
                    'url': self._search_regex(r'File1=(.*)', playlist_content, 'url', fatal=False),
                    'abr': stream.get('bitrate'),
                    'ext': stream.get('media_type'),
                })
            else:
                formats.append({
                    'url': stream['url'],
                    'abr': stream.get('bitrate'),
                    'ext': stream.get('media_type'),
                })

        return formats, subtitles


class TuneInStationIE(TuneInBaseIE):
    _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'(?:/radio/[^?#]+-|/embed/player/)(?P<id>s\d+)'
    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/s\d+)']

    _TESTS = [{
        'url': 'https://tunein.com/radio/Jazz24-885-s34682/',
        'info_dict': {
            'id': 's34682',
            'title': 're:^Jazz24',
            'description': 'md5:d6d0b89063fd68d529fa7058ee98619b',
            'thumbnail': 're:^https?://[^?&]+/s34682',
            'location': 'Seattle-Tacoma, US',
            'ext': 'mp3',
            'live_status': 'is_live',
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://tunein.com/embed/player/s6404/',
        'only_matching': True,
    }, {
        'url': 'https://tunein.com/radio/BBC-Radio-1-988-s24939/',
        'info_dict': {
            'id': 's24939',
            'title': 're:^BBC Radio 1',
            'description': 'md5:f3f75f7423398d87119043c26e7bfb84',
            'thumbnail': 're:^https?://[^?&]+/s24939',
            'location': 'London, UK',
            'ext': 'mp3',
            'live_status': 'is_live',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _real_extract(self, url):
        station_id = self._match_id(url)

        webpage = self._download_webpage(url, station_id)
        metadata = self._extract_metadata(webpage, station_id)

        formats, subtitles = self._extract_formats_and_subtitles(station_id)
        return {
            'id': station_id,
            'title': traverse_obj(metadata, ('profiles', station_id, 'title')),
            'description': traverse_obj(metadata, ('profiles', station_id, 'description')),
            'thumbnail': traverse_obj(metadata, ('profiles', station_id, 'image')),
            'timestamp': parse_iso8601(
                traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'publishTime'))),
            'location': traverse_obj(
                metadata, ('profiles', station_id, 'metadata', 'properties', 'location', 'displayName'),
                ('profiles', station_id, 'properties', 'location', 'displayName')),
            'formats': formats,
            'subtitles': subtitles,
            'is_live': traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'isLive')),
        }


class TuneInPodcastIE(TuneInBaseIE):
    _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/(?:podcasts/[^?#]+-|embed/player/)(?P<id>p\d+)/?(?:#|$)'
    _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/p\d+)']

    _TESTS = [{
        'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019',
        'info_dict': {
            'id': 'p1153019',
            'title': 'Lex Fridman Podcast',
            'description': 'md5:bedc4e5f1c94f7dec6e4317b5654b00d',
        },
        'playlist_mincount': 200,
    }, {
        'url': 'https://tunein.com/embed/player/p191660/',
        'only_matching': True,
    }, {
        'url': 'https://tunein.com/podcasts/World-News/BBC-News-p14/',
        'info_dict': {
            'id': 'p14',
            'title': 'BBC News',
            'description': 'md5:1218e575eeaff75f48ed978261fa2068',
        },
        'playlist_mincount': 200,
    }]

    _PAGE_SIZE = 30

    def _real_extract(self, url):
        podcast_id = self._match_id(url)

        webpage = self._download_webpage(url, podcast_id, fatal=False)
        metadata = self._extract_metadata(webpage, podcast_id)

        def page_func(page_num):
            api_response = self._download_json(
                f'https://api.tunein.com/profiles/{podcast_id}/contents', podcast_id,
                note=f'Downloading page {page_num + 1}', query={
                    'filter': 't:free',
                    'offset': page_num * self._PAGE_SIZE,
                    'limit': self._PAGE_SIZE,
                })

            return [
                self.url_result(
                    f'https://tunein.com/podcasts/{podcast_id}?topicId={episode["GuideId"][1:]}',
                    TuneInPodcastEpisodeIE, title=episode.get('Title'))
                for episode in api_response['Items']]

        entries = OnDemandPagedList(page_func, self._PAGE_SIZE)
        return self.playlist_result(
            entries, playlist_id=podcast_id, title=traverse_obj(metadata, ('profiles', podcast_id, 'title')),
            description=traverse_obj(metadata, ('profiles', podcast_id, 'description')))


class TuneInPodcastEpisodeIE(TuneInBaseIE):
    _VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/podcasts/(?:[^?&]+-)?(?P<podcast_id>p\d+)/?\?topicId=(?P<id>\w\d+)'

    _TESTS = [{
        'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019/?topicId=236404354',
        'info_dict': {
            'id': 't236404354',
            'title': '#351 \u2013 MrBeast: Future of YouTube, Twitter, TikTok, and Instagram',
            'description': 'md5:e1734db6f525e472c0c290d124a2ad77',
            'thumbnail': 're:^https?://[^?&]+/p1153019',
            'timestamp': 1673458571,
            'upload_date': '20230111',
            'series_id': 'p1153019',
            'series': 'Lex Fridman Podcast',
            'ext': 'mp3',
        },
    }]

    def _real_extract(self, url):
        podcast_id, episode_id = self._match_valid_url(url).group('podcast_id', 'id')
        episode_id = f't{episode_id}'

        webpage = self._download_webpage(url, episode_id)
        metadata = self._extract_metadata(webpage, episode_id)

        formats, subtitles = self._extract_formats_and_subtitles(episode_id)
        return {
            'id': episode_id,
            'title': traverse_obj(metadata, ('profiles', episode_id, 'title')),
            'description': traverse_obj(metadata, ('profiles', episode_id, 'description')),
            'thumbnail': traverse_obj(metadata, ('profiles', episode_id, 'image')),
            'timestamp': parse_iso8601(
                traverse_obj(metadata, ('profiles', episode_id, 'actions', 'play', 'publishTime'))),
            'series_id': podcast_id,
            'series': traverse_obj(metadata, ('profiles', podcast_id, 'title')),
            'formats': formats,
            'subtitles': subtitles,
        }


class TuneInShortenerIE(InfoExtractor):
    IE_NAME = 'tunein:shortener'
    IE_DESC = False  # Do not list
    _VALID_URL = r'https?://tun\.in/(?P<id>[A-Za-z0-9]+)'

    _TEST = {
        # test redirection
        'url': 'http://tun.in/ser7s',
        'info_dict': {
            'id': 's34682',
            'title': 're:^Jazz24',
            'description': 'md5:d6d0b89063fd68d529fa7058ee98619b',
            'thumbnail': 're:^https?://[^?&]+/s34682',
            'location': 'Seattle-Tacoma, US',
            'ext': 'mp3',
            'live_status': 'is_live',
        },
        'params': {
            'skip_download': True,  # live stream
        },
    }

    def _real_extract(self, url):
        redirect_id = self._match_id(url)
        # The server doesn't support HEAD requests
        urlh = self._request_webpage(
            url, redirect_id, note='Downloading redirect page')

        url = urlh.url
        url_parsed = urllib.parse.urlparse(url)
        if url_parsed.port == 443:
            url = url_parsed._replace(netloc=url_parsed.hostname).url

        self.to_screen(f'Following redirect: {url}')
        return self.url_result(url)
Commit	Line	Data
46580ced	1	import urllib.parse
2c25a2bd NJ	2
2c25a2bd NJ	3	from .common import InfoExtractor
46580ced E	4	from ..utils import (
	5	OnDemandPagedList,
	6	determine_ext,
	7	parse_iso8601,
	8	traverse_obj,
	9	)
2c25a2bd NJ	10
2c25a2bd NJ	11
bd3f9eca	12	class TuneInBaseIE(InfoExtractor):
46580ced	13	_VALID_URL_BASE = r'https?://(?:www\.)?tunein\.com'
2c25a2bd	14
46580ced E	15	def _extract_metadata(self, webpage, content_id):
46580ced E	16	return self._search_json(r'window.INITIAL_STATE=', webpage, 'hydration', content_id, fatal=False)
bd3f9eca	17
46580ced	18	def _extract_formats_and_subtitles(self, content_id):
882c6992	19	streams = self._download_json(
46580ced E	20	f'https://opml.radiotime.com/Tune.ashx?render=json&formats=mp3,aac,ogg,flash,hls&id={content_id}',
46580ced E	21	content_id)['body']
2c25a2bd	22
46580ced	23	formats, subtitles = [], {}
2c25a2bd	24	for stream in streams:
46580ced E	25	if stream.get('media_type') == 'hls':
	26	fmts, subs = self._extract_m3u8_formats_and_subtitles(stream['url'], content_id, fatal=False)
	27	formats.extend(fmts)
	28	self._merge_subtitles(subs, target=subtitles)
	29	elif determine_ext(stream['url']) == 'pls':
	30	playlist_content = self._download_webpage(stream['url'], content_id)
	31	formats.append({
	32	'url': self._search_regex(r'File1=(.*)', playlist_content, 'url', fatal=False),
	33	'abr': stream.get('bitrate'),
	34	'ext': stream.get('media_type'),
	35	})
	36	else:
	37	formats.append({
	38	'url': stream['url'],
	39	'abr': stream.get('bitrate'),
	40	'ext': stream.get('media_type'),
	41	})
	42
	43	return formats, subtitles
bd3f9eca	44
	45
	46	class TuneInStationIE(TuneInBaseIE):
46580ced E	47	_VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'(?:/radio/[^?#]+-\|/embed/player/)(?P<id>s\d+)'
46580ced E	48	_EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/s\d+)']
bd3f9eca	49
027e2312	50	_TESTS = [{
46580ced	51	'url': 'https://tunein.com/radio/Jazz24-885-s34682/',
027e2312	52	'info_dict': {
46580ced E	53	'id': 's34682',
	54	'title': 're:^Jazz24',
	55	'description': 'md5:d6d0b89063fd68d529fa7058ee98619b',
	56	'thumbnail': 're:^https?://[^?&]+/s34682',
	57	'location': 'Seattle-Tacoma, US',
027e2312	58	'ext': 'mp3',
46580ced	59	'live_status': 'is_live',
027e2312 S	60	},
027e2312 S	61	'params': {
46580ced	62	'skip_download': True,
bd3f9eca	63	},
027e2312	64	}, {
46580ced	65	'url': 'https://tunein.com/embed/player/s6404/',
027e2312	66	'only_matching': True,
46580ced E	67	}, {
	68	'url': 'https://tunein.com/radio/BBC-Radio-1-988-s24939/',
	69	'info_dict': {
	70	'id': 's24939',
	71	'title': 're:^BBC Radio 1',
	72	'description': 'md5:f3f75f7423398d87119043c26e7bfb84',
	73	'thumbnail': 're:^https?://[^?&]+/s24939',
	74	'location': 'London, UK',
	75	'ext': 'mp3',
	76	'live_status': 'is_live',
	77	},
	78	'params': {
	79	'skip_download': True,
	80	},
027e2312	81	}]
bd3f9eca	82
46580ced E	83	def _real_extract(self, url):
	84	station_id = self._match_id(url)
	85
	86	webpage = self._download_webpage(url, station_id)
	87	metadata = self._extract_metadata(webpage, station_id)
	88
	89	formats, subtitles = self._extract_formats_and_subtitles(station_id)
	90	return {
	91	'id': station_id,
	92	'title': traverse_obj(metadata, ('profiles', station_id, 'title')),
	93	'description': traverse_obj(metadata, ('profiles', station_id, 'description')),
	94	'thumbnail': traverse_obj(metadata, ('profiles', station_id, 'image')),
	95	'timestamp': parse_iso8601(
	96	traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'publishTime'))),
	97	'location': traverse_obj(
	98	metadata, ('profiles', station_id, 'metadata', 'properties', 'location', 'displayName'),
	99	('profiles', station_id, 'properties', 'location', 'displayName')),
	100	'formats': formats,
	101	'subtitles': subtitles,
	102	'is_live': traverse_obj(metadata, ('profiles', station_id, 'actions', 'play', 'isLive')),
	103	}
	104
bd3f9eca	105
46580ced E	106	class TuneInPodcastIE(TuneInBaseIE):
	107	_VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/(?:podcasts/[^?#]+-\|embed/player/)(?P<id>p\d+)/?(?:#\|$)'
	108	_EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?://)?tunein\.com/embed/player/p\d+)']
bd3f9eca	109
027e2312	110	_TESTS = [{
46580ced	111	'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019',
027e2312	112	'info_dict': {
46580ced E	113	'id': 'p1153019',
	114	'title': 'Lex Fridman Podcast',
	115	'description': 'md5:bedc4e5f1c94f7dec6e4317b5654b00d',
027e2312	116	},
46580ced	117	'playlist_mincount': 200,
027e2312	118	}, {
46580ced	119	'url': 'https://tunein.com/embed/player/p191660/',
add96eb9	120	'only_matching': True,
46580ced E	121	}, {
	122	'url': 'https://tunein.com/podcasts/World-News/BBC-News-p14/',
	123	'info_dict': {
	124	'id': 'p14',
	125	'title': 'BBC News',
	126	'description': 'md5:1218e575eeaff75f48ed978261fa2068',
	127	},
	128	'playlist_mincount': 200,
027e2312	129	}]
bd3f9eca	130
46580ced E	131	_PAGE_SIZE = 30
	132
	133	def _real_extract(self, url):
	134	podcast_id = self._match_id(url)
	135
	136	webpage = self._download_webpage(url, podcast_id, fatal=False)
	137	metadata = self._extract_metadata(webpage, podcast_id)
	138
	139	def page_func(page_num):
	140	api_response = self._download_json(
	141	f'https://api.tunein.com/profiles/{podcast_id}/contents', podcast_id,
	142	note=f'Downloading page {page_num + 1}', query={
	143	'filter': 't:free',
	144	'offset': page_num * self._PAGE_SIZE,
	145	'limit': self._PAGE_SIZE,
	146	})
bd3f9eca	147
46580ced E	148	return [
	149	self.url_result(
	150	f'https://tunein.com/podcasts/{podcast_id}?topicId={episode["GuideId"][1:]}',
	151	TuneInPodcastEpisodeIE, title=episode.get('Title'))
	152	for episode in api_response['Items']]
	153
	154	entries = OnDemandPagedList(page_func, self._PAGE_SIZE)
	155	return self.playlist_result(
	156	entries, playlist_id=podcast_id, title=traverse_obj(metadata, ('profiles', podcast_id, 'title')),
	157	description=traverse_obj(metadata, ('profiles', podcast_id, 'description')))
	158
	159
	160	class TuneInPodcastEpisodeIE(TuneInBaseIE):
	161	_VALID_URL = TuneInBaseIE._VALID_URL_BASE + r'/podcasts/(?:[^?&]+-)?(?P<podcast_id>p\d+)/?\?topicId=(?P<id>\w\d+)'
bd3f9eca	162
027e2312	163	_TESTS = [{
46580ced	164	'url': 'https://tunein.com/podcasts/Technology-Podcasts/Artificial-Intelligence-p1153019/?topicId=236404354',
027e2312	165	'info_dict': {
46580ced E	166	'id': 't236404354',
	167	'title': '#351 \u2013 MrBeast: Future of YouTube, Twitter, TikTok, and Instagram',
	168	'description': 'md5:e1734db6f525e472c0c290d124a2ad77',
	169	'thumbnail': 're:^https?://[^?&]+/p1153019',
	170	'timestamp': 1673458571,
	171	'upload_date': '20230111',
	172	'series_id': 'p1153019',
	173	'series': 'Lex Fridman Podcast',
027e2312	174	'ext': 'mp3',
bd3f9eca	175	},
027e2312	176	}]
bd3f9eca	177
46580ced E	178	def _real_extract(self, url):
	179	podcast_id, episode_id = self._match_valid_url(url).group('podcast_id', 'id')
	180	episode_id = f't{episode_id}'
	181
	182	webpage = self._download_webpage(url, episode_id)
	183	metadata = self._extract_metadata(webpage, episode_id)
	184
	185	formats, subtitles = self._extract_formats_and_subtitles(episode_id)
	186	return {
	187	'id': episode_id,
	188	'title': traverse_obj(metadata, ('profiles', episode_id, 'title')),
	189	'description': traverse_obj(metadata, ('profiles', episode_id, 'description')),
	190	'thumbnail': traverse_obj(metadata, ('profiles', episode_id, 'image')),
	191	'timestamp': parse_iso8601(
	192	traverse_obj(metadata, ('profiles', episode_id, 'actions', 'play', 'publishTime'))),
	193	'series_id': podcast_id,
	194	'series': traverse_obj(metadata, ('profiles', podcast_id, 'title')),
	195	'formats': formats,
	196	'subtitles': subtitles,
	197	}
	198
bd3f9eca	199
	200	class TuneInShortenerIE(InfoExtractor):
	201	IE_NAME = 'tunein:shortener'
	202	IE_DESC = False # Do not list
	203	_VALID_URL = r'https?://tun\.in/(?P<id>[A-Za-z0-9]+)'
	204
	205	_TEST = {
	206	# test redirection
	207	'url': 'http://tun.in/ser7s',
	208	'info_dict': {
46580ced E	209	'id': 's34682',
	210	'title': 're:^Jazz24',
	211	'description': 'md5:d6d0b89063fd68d529fa7058ee98619b',
	212	'thumbnail': 're:^https?://[^?&]+/s34682',
	213	'location': 'Seattle-Tacoma, US',
bd3f9eca	214	'ext': 'mp3',
46580ced	215	'live_status': 'is_live',
bd3f9eca	216	},
	217	'params': {
	218	'skip_download': True, # live stream
	219	},
	220	}
	221
	222	def _real_extract(self, url):
	223	redirect_id = self._match_id(url)
	224	# The server doesn't support HEAD requests
	225	urlh = self._request_webpage(
	226	url, redirect_id, note='Downloading redirect page')
46580ced	227
3d2623a8	228	url = urlh.url
46580ced E	229	url_parsed = urllib.parse.urlparse(url)
46580ced E	230	if url_parsed.port == 443:
3d2623a8	231	url = url_parsed._replace(netloc=url_parsed.hostname).url
46580ced	232
add96eb9	233	self.to_screen(f'Following redirect: {url}')
bd3f9eca	234	return self.url_result(url)