[yt-dlp.git] / yt_dlp / extractor / callin.py

from .common import InfoExtractor
from ..utils import float_or_none, int_or_none, make_archive_id, traverse_obj


class CallinIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?callin\.com/(episode)/(?P<id>[-a-zA-Z]+)'
    _TESTS = [{
        'url': 'https://www.callin.com/episode/the-title-ix-regime-and-the-long-march-through-EBfXYSrsjc',
        'info_dict': {
            'id': '218b979630a35ead12c6fd096f2996c56c37e4d0dc1f6dc0feada32dcf7b31cd',
            'title': 'The Title IX Regime and the Long March Through and Beyond the Institutions',
            'ext': 'ts',
            'display_id': 'the-title-ix-regime-and-the-long-march-through-EBfXYSrsjc',
            'thumbnail': 're:https://.+\\.png',
            'description': 'First episode',
            'uploader': 'Wesley Yang',
            'timestamp': 1639404128.65,
            'upload_date': '20211213',
            'uploader_id': 'wesyang',
            'uploader_url': 'http://wesleyyang.substack.com',
            'channel': 'Conversations in Year Zero',
            'channel_id': '436d1f82ddeb30cd2306ea9156044d8d2cfdc3f1f1552d245117a42173e78553',
            'channel_url': 'https://callin.com/show/conversations-in-year-zero-oJNllRFSfx',
            'duration': 9951.936,
            'view_count': int,
            'categories': ['News & Politics', 'History', 'Technology'],
            'cast': ['Wesley Yang', 'KC Johnson', 'Gabi Abramovich'],
            'series': 'Conversations in Year Zero',
            'series_id': '436d1f82ddeb30cd2306ea9156044d8d2cfdc3f1f1552d245117a42173e78553',
            'episode': 'The Title IX Regime and the Long March Through and Beyond the Institutions',
            'episode_number': 1,
            'episode_id': '218b979630a35ead12c6fd096f2996c56c37e4d0dc1f6dc0feada32dcf7b31cd',
        },
    }, {
        'url': 'https://www.callin.com/episode/fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
        'md5': '14ede27ee2c957b7e4db93140fc0745c',
        'info_dict': {
            'id': 'c3dab47f237bf953d180d3f243477a84302798be0e0b29bc9ade6d60a69f04f5',
            'ext': 'ts',
            'title': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
            'description': 'Or, why the government doesn’t like SpaceX',
            'channel': 'The Pull Request',
            'channel_url': 'https://callin.com/show/the-pull-request-ucnDJmEKAa',
            'duration': 3182.472,
            'series_id': '7e9c23156e4aecfdcaef46bfb2ed7ca268509622ec006c0f0f25d90e34496638',
            'uploader_url': 'http://thepullrequest.com',
            'upload_date': '20220902',
            'episode': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
            'display_id': 'fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
            'series': 'The Pull Request',
            'channel_id': '7e9c23156e4aecfdcaef46bfb2ed7ca268509622ec006c0f0f25d90e34496638',
            'view_count': int,
            'uploader': 'Antonio García Martínez',
            'thumbnail': 'https://d1z76fhpoqkd01.cloudfront.net/shows/legacy/1ade9142625344045dc17cf523469ced1d93610762f4c886d06aa190a2f979e8.png',
            'episode_id': 'c3dab47f237bf953d180d3f243477a84302798be0e0b29bc9ade6d60a69f04f5',
            'timestamp': 1662100688.005,
        },
    }, {
        'url': 'https://www.callin.com/episode/episode-81-elites-melt-down-over-student-debt-lzxMidUnjA',
        'md5': '16f704ddbf82a27e3930533b12062f07',
        'info_dict': {
            'id': '8d06f869798f93a7814e380bceabea72d501417e620180416ff6bd510596e83c',
            'ext': 'ts',
            'title': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
            'description': 'Let’s talk todays episode about the primary election shake up in NYC and the elites melting down over student debt cancelation.',
            'channel': 'The DEBRIEF With Briahna Joy Gray',
            'channel_url': 'https://callin.com/show/the-debrief-with-briahna-joy-gray-siiFDzGegm',
            'duration': 10043.16,
            'series_id': '61cea58444465fd26674069703bd8322993bc9e5b4f1a6d0872690554a046ff7',
            'uploader_url': 'http://patreon.com/badfaithpodcast',
            'upload_date': '20220826',
            'episode': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
            'display_id': 'episode-',
            'series': 'The DEBRIEF With Briahna Joy Gray',
            'channel_id': '61cea58444465fd26674069703bd8322993bc9e5b4f1a6d0872690554a046ff7',
            'view_count': int,
            'uploader': 'Briahna Gray',
            'thumbnail': 'https://d1z76fhpoqkd01.cloudfront.net/shows/legacy/461ea0d86172cb6aff7d6c80fd49259cf5e64bdf737a4650f8bc24cf392ca218.png',
            'episode_id': '8d06f869798f93a7814e380bceabea72d501417e620180416ff6bd510596e83c',
            'timestamp': 1661476708.282,
        },
    }]

    def try_get_user_name(self, d):
        names = [d.get(n) for n in ('first', 'last')]
        if None in names:
            return next((n for n in names if n), default=None)
        return ' '.join(names)

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)

        next_data = self._search_nextjs_data(webpage, display_id)
        episode = next_data['props']['pageProps']['episode']

        video_id = episode['id']
        title = episode.get('title') or self._generic_title('', webpage)
        url = episode['m3u8']
        formats = self._extract_m3u8_formats(url, display_id, ext='ts')

        show = traverse_obj(episode, ('show', 'title'))
        show_id = traverse_obj(episode, ('show', 'id'))

        show_json = None
        app_slug = (self._html_search_regex(
            '<script\\s+src=["\']/_next/static/([-_a-zA-Z0-9]+)/_',
            webpage, 'app slug', fatal=False) or next_data.get('buildId'))
        show_slug = traverse_obj(episode, ('show', 'linkObj', 'resourceUrl'))
        if app_slug and show_slug and '/' in show_slug:
            show_slug = show_slug.rsplit('/', 1)[1]
            show_json_url = f'https://www.callin.com/_next/data/{app_slug}/show/{show_slug}.json'
            show_json = self._download_json(show_json_url, display_id, fatal=False)

        host = (traverse_obj(show_json, ('pageProps', 'show', 'hosts', 0))
                or traverse_obj(episode, ('speakers', 0)))

        host_nick = traverse_obj(host, ('linkObj', 'resourceUrl'))
        host_nick = host_nick.rsplit('/', 1)[1] if (host_nick and '/' in host_nick) else None

        cast = list(filter(None, [
            self.try_get_user_name(u) for u in
            traverse_obj(episode, (('speakers', 'callerTags'), ...)) or []
        ]))

        episode_list = traverse_obj(show_json, ('pageProps', 'show', 'episodes')) or []
        episode_number = next(
            (len(episode_list) - i for i, e in enumerate(episode_list) if e.get('id') == video_id),
            None)

        return {
            'id': video_id,
            '_old_archive_ids': [make_archive_id(self, display_id.rsplit('-', 1)[-1])],
            'display_id': display_id,
            'title': title,
            'formats': formats,
            'thumbnail': traverse_obj(episode, ('show', 'photo')),
            'description': episode.get('description'),
            'uploader': self.try_get_user_name(host) if host else None,
            'timestamp': episode.get('publishedAt'),
            'uploader_id': host_nick,
            'uploader_url': traverse_obj(show_json, ('pageProps', 'show', 'url')),
            'channel': show,
            'channel_id': show_id,
            'channel_url': traverse_obj(episode, ('show', 'linkObj', 'resourceUrl')),
            'duration': float_or_none(episode.get('runtime')),
            'view_count': int_or_none(episode.get('plays')),
            'categories': traverse_obj(episode, ('show', 'categorizations', ..., 'name')),
            'cast': cast if cast else None,
            'series': show,
            'series_id': show_id,
            'episode': title,
            'episode_number': episode_number,
            'episode_id': video_id,
        }
Commit	Line	Data
97a6b117	1	from .common import InfoExtractor
45b2ee6f	2	from ..utils import float_or_none, int_or_none, make_archive_id, traverse_obj
97a6b117	3
	4
	5	class CallinIE(InfoExtractor):
	6	_VALID_URL = r'https?://(?:www\.)?callin\.com/(episode)/(?P<id>[-a-zA-Z]+)'
	7	_TESTS = [{
	8	'url': 'https://www.callin.com/episode/the-title-ix-regime-and-the-long-march-through-EBfXYSrsjc',
	9	'info_dict': {
	10	'id': '218b979630a35ead12c6fd096f2996c56c37e4d0dc1f6dc0feada32dcf7b31cd',
	11	'title': 'The Title IX Regime and the Long March Through and Beyond the Institutions',
	12	'ext': 'ts',
	13	'display_id': 'the-title-ix-regime-and-the-long-march-through-EBfXYSrsjc',
	14	'thumbnail': 're:https://.+\\.png',
	15	'description': 'First episode',
	16	'uploader': 'Wesley Yang',
	17	'timestamp': 1639404128.65,
	18	'upload_date': '20211213',
	19	'uploader_id': 'wesyang',
	20	'uploader_url': 'http://wesleyyang.substack.com',
	21	'channel': 'Conversations in Year Zero',
	22	'channel_id': '436d1f82ddeb30cd2306ea9156044d8d2cfdc3f1f1552d245117a42173e78553',
	23	'channel_url': 'https://callin.com/show/conversations-in-year-zero-oJNllRFSfx',
	24	'duration': 9951.936,
	25	'view_count': int,
	26	'categories': ['News & Politics', 'History', 'Technology'],
	27	'cast': ['Wesley Yang', 'KC Johnson', 'Gabi Abramovich'],
	28	'series': 'Conversations in Year Zero',
	29	'series_id': '436d1f82ddeb30cd2306ea9156044d8d2cfdc3f1f1552d245117a42173e78553',
	30	'episode': 'The Title IX Regime and the Long March Through and Beyond the Institutions',
	31	'episode_number': 1,
add96eb9	32	'episode_id': '218b979630a35ead12c6fd096f2996c56c37e4d0dc1f6dc0feada32dcf7b31cd',
add96eb9	33	},
45b2ee6f	34	}, {
	35	'url': 'https://www.callin.com/episode/fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
	36	'md5': '14ede27ee2c957b7e4db93140fc0745c',
	37	'info_dict': {
	38	'id': 'c3dab47f237bf953d180d3f243477a84302798be0e0b29bc9ade6d60a69f04f5',
	39	'ext': 'ts',
	40	'title': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
	41	'description': 'Or, why the government doesn’t like SpaceX',
	42	'channel': 'The Pull Request',
	43	'channel_url': 'https://callin.com/show/the-pull-request-ucnDJmEKAa',
	44	'duration': 3182.472,
	45	'series_id': '7e9c23156e4aecfdcaef46bfb2ed7ca268509622ec006c0f0f25d90e34496638',
	46	'uploader_url': 'http://thepullrequest.com',
	47	'upload_date': '20220902',
	48	'episode': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
	49	'display_id': 'fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
	50	'series': 'The Pull Request',
	51	'channel_id': '7e9c23156e4aecfdcaef46bfb2ed7ca268509622ec006c0f0f25d90e34496638',
	52	'view_count': int,
	53	'uploader': 'Antonio García Martínez',
	54	'thumbnail': 'https://d1z76fhpoqkd01.cloudfront.net/shows/legacy/1ade9142625344045dc17cf523469ced1d93610762f4c886d06aa190a2f979e8.png',
	55	'episode_id': 'c3dab47f237bf953d180d3f243477a84302798be0e0b29bc9ade6d60a69f04f5',
	56	'timestamp': 1662100688.005,
add96eb9	57	},
45b2ee6f	58	}, {
	59	'url': 'https://www.callin.com/episode/episode-81-elites-melt-down-over-student-debt-lzxMidUnjA',
	60	'md5': '16f704ddbf82a27e3930533b12062f07',
	61	'info_dict': {
	62	'id': '8d06f869798f93a7814e380bceabea72d501417e620180416ff6bd510596e83c',
	63	'ext': 'ts',
	64	'title': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
	65	'description': 'Let’s talk todays episode about the primary election shake up in NYC and the elites melting down over student debt cancelation.',
	66	'channel': 'The DEBRIEF With Briahna Joy Gray',
	67	'channel_url': 'https://callin.com/show/the-debrief-with-briahna-joy-gray-siiFDzGegm',
	68	'duration': 10043.16,
	69	'series_id': '61cea58444465fd26674069703bd8322993bc9e5b4f1a6d0872690554a046ff7',
	70	'uploader_url': 'http://patreon.com/badfaithpodcast',
	71	'upload_date': '20220826',
	72	'episode': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
	73	'display_id': 'episode-',
	74	'series': 'The DEBRIEF With Briahna Joy Gray',
	75	'channel_id': '61cea58444465fd26674069703bd8322993bc9e5b4f1a6d0872690554a046ff7',
	76	'view_count': int,
	77	'uploader': 'Briahna Gray',
	78	'thumbnail': 'https://d1z76fhpoqkd01.cloudfront.net/shows/legacy/461ea0d86172cb6aff7d6c80fd49259cf5e64bdf737a4650f8bc24cf392ca218.png',
	79	'episode_id': '8d06f869798f93a7814e380bceabea72d501417e620180416ff6bd510596e83c',
	80	'timestamp': 1661476708.282,
add96eb9	81	},
97a6b117	82	}]
	83
	84	def try_get_user_name(self, d):
	85	names = [d.get(n) for n in ('first', 'last')]
	86	if None in names:
	87	return next((n for n in names if n), default=None)
	88	return ' '.join(names)
	89
	90	def _real_extract(self, url):
	91	display_id = self._match_id(url)
	92	webpage = self._download_webpage(url, display_id)
	93
	94	next_data = self._search_nextjs_data(webpage, display_id)
	95	episode = next_data['props']['pageProps']['episode']
	96
add96eb9	97	video_id = episode['id']
62b8dac4	98	title = episode.get('title') or self._generic_title('', webpage)
97a6b117	99	url = episode['m3u8']
97a6b117	100	formats = self._extract_m3u8_formats(url, display_id, ext='ts')
97a6b117	101
	102	show = traverse_obj(episode, ('show', 'title'))
	103	show_id = traverse_obj(episode, ('show', 'id'))
	104
	105	show_json = None
	106	app_slug = (self._html_search_regex(
	107	'<script\\s+src=["\']/_next/static/([-_a-zA-Z0-9]+)/_',
	108	webpage, 'app slug', fatal=False) or next_data.get('buildId'))
	109	show_slug = traverse_obj(episode, ('show', 'linkObj', 'resourceUrl'))
	110	if app_slug and show_slug and '/' in show_slug:
	111	show_slug = show_slug.rsplit('/', 1)[1]
	112	show_json_url = f'https://www.callin.com/_next/data/{app_slug}/show/{show_slug}.json'
	113	show_json = self._download_json(show_json_url, display_id, fatal=False)
	114
	115	host = (traverse_obj(show_json, ('pageProps', 'show', 'hosts', 0))
	116	or traverse_obj(episode, ('speakers', 0)))
	117
	118	host_nick = traverse_obj(host, ('linkObj', 'resourceUrl'))
	119	host_nick = host_nick.rsplit('/', 1)[1] if (host_nick and '/' in host_nick) else None
	120
	121	cast = list(filter(None, [
	122	self.try_get_user_name(u) for u in
	123	traverse_obj(episode, (('speakers', 'callerTags'), ...)) or []
	124	]))
	125
	126	episode_list = traverse_obj(show_json, ('pageProps', 'show', 'episodes')) or []
	127	episode_number = next(
add96eb9	128	(len(episode_list) - i for i, e in enumerate(episode_list) if e.get('id') == video_id),
97a6b117	129	None)
	130
	131	return {
add96eb9	132	'id': video_id,
45b2ee6f	133	'_old_archive_ids': [make_archive_id(self, display_id.rsplit('-', 1)[-1])],
97a6b117	134	'display_id': display_id,
	135	'title': title,
	136	'formats': formats,
	137	'thumbnail': traverse_obj(episode, ('show', 'photo')),
	138	'description': episode.get('description'),
	139	'uploader': self.try_get_user_name(host) if host else None,
	140	'timestamp': episode.get('publishedAt'),
	141	'uploader_id': host_nick,
	142	'uploader_url': traverse_obj(show_json, ('pageProps', 'show', 'url')),
	143	'channel': show,
	144	'channel_id': show_id,
	145	'channel_url': traverse_obj(episode, ('show', 'linkObj', 'resourceUrl')),
	146	'duration': float_or_none(episode.get('runtime')),
	147	'view_count': int_or_none(episode.get('plays')),
	148	'categories': traverse_obj(episode, ('show', 'categorizations', ..., 'name')),
	149	'cast': cast if cast else None,
	150	'series': show,
	151	'series_id': show_id,
	152	'episode': title,
	153	'episode_number': episode_number,
add96eb9	154	'episode_id': video_id,
97a6b117	155	}