[yt-dlp.git] / yt_dlp / extractor / dropout.py

from .common import InfoExtractor
from .vimeo import VHXEmbedIE
from ..utils import (
    clean_html,
    ExtractorError,
    get_element_by_class,
    get_element_by_id,
    get_elements_by_class,
    int_or_none,
    join_nonempty,
    unified_strdate,
    urlencode_postdata,
)


class DropoutIE(InfoExtractor):
    _LOGIN_URL = 'https://www.dropout.tv/login'
    _NETRC_MACHINE = 'dropout'

    _VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?:[^/]+/)*videos/(?P<id>[^/]+)/?$'
    _TESTS = [
        {
            'url': 'https://www.dropout.tv/game-changer/season:2/videos/yes-or-no',
            'note': 'Episode in a series',
            'md5': '5e000fdfd8d8fa46ff40456f1c2af04a',
            'info_dict': {
                'id': '738153',
                'display_id': 'yes-or-no',
                'ext': 'mp4',
                'title': 'Yes or No',
                'description': 'Ally, Brennan, and Zac are asked a simple question, but is there a correct answer?',
                'release_date': '20200508',
                'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/351e3f24-c4a3-459a-8b79-dc80f1e5b7fd.jpg',
                'series': 'Game Changer',
                'season_number': 2,
                'season': 'Season 2',
                'episode_number': 6,
                'episode': 'Yes or No',
                'duration': 1180,
                'uploader_id': 'user80538407',
                'uploader_url': 'https://vimeo.com/user80538407',
                'uploader': 'OTT Videos'
            },
            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest']
        },
        {
            'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1/videos/episode-1',
            'note': 'Episode in a series (missing release_date)',
            'md5': '712caf7c191f1c47c8f1879520c2fa5c',
            'info_dict': {
                'id': '320562',
                'display_id': 'episode-1',
                'ext': 'mp4',
                'title': 'The Beginning Begins',
                'description': 'The cast introduces their PCs, including a neurotic elf, a goblin PI, and a corn-worshipping cleric.',
                'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/4421ed0d-f630-4c88-9004-5251b2b8adfa.jpg',
                'series': 'Dimension 20: Fantasy High',
                'season_number': 1,
                'season': 'Season 1',
                'episode_number': 1,
                'episode': 'The Beginning Begins',
                'duration': 6838,
                'uploader_id': 'user80538407',
                'uploader_url': 'https://vimeo.com/user80538407',
                'uploader': 'OTT Videos'
            },
            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest']
        },
        {
            'url': 'https://www.dropout.tv/videos/misfits-magic-holiday-special',
            'note': 'Episode not in a series',
            'md5': 'c30fa18999c5880d156339f13c953a26',
            'info_dict': {
                'id': '1915774',
                'display_id': 'misfits-magic-holiday-special',
                'ext': 'mp4',
                'title': 'Misfits & Magic Holiday Special',
                'description': 'The magical misfits spend Christmas break at Gowpenny, with an unwelcome visitor.',
                'release_date': '20211215',
                'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/d91ea8a6-b250-42ed-907e-b30fb1c65176-8e24b8e5.jpg',
                'duration': 11698,
                'uploader_id': 'user80538407',
                'uploader_url': 'https://vimeo.com/user80538407',
                'uploader': 'OTT Videos'
            },
            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest']
        }
    ]

    def _get_authenticity_token(self, display_id):
        signin_page = self._download_webpage(
            self._LOGIN_URL, display_id, note='Getting authenticity token')
        return self._html_search_regex(
            r'name=["\']authenticity_token["\'] value=["\'](.+?)["\']',
            signin_page, 'authenticity_token')

    def _login(self, display_id):
        username, password = self._get_login_info()
        if not (username and password):
            self.raise_login_required(method='password')

        response = self._download_webpage(
            self._LOGIN_URL, display_id, note='Logging in', data=urlencode_postdata({
                'email': username,
                'password': password,
                'authenticity_token': self._get_authenticity_token(display_id),
                'utf8': True
            }))

        user_has_subscription = self._search_regex(
            r'user_has_subscription:\s*["\'](.+?)["\']', response, 'subscription status', default='none')
        if user_has_subscription.lower() == 'true':
            return response
        elif user_has_subscription.lower() == 'false':
            raise ExtractorError('Account is not subscribed')
        else:
            raise ExtractorError('Incorrect username/password')

    def _real_extract(self, url):
        display_id = self._match_id(url)
        try:
            self._login(display_id)
            webpage = self._download_webpage(url, display_id, note='Downloading video webpage')
        finally:
            self._download_webpage('https://www.dropout.tv/logout', display_id, note='Logging out', fatal=False)

        embed_url = self._search_regex(r'embed_url:\s*["\'](.+?)["\']', webpage, 'embed url')
        thumbnail = self._og_search_thumbnail(webpage)
        watch_info = get_element_by_id('watch-info', webpage) or ''

        title = clean_html(get_element_by_class('video-title', watch_info))
        season_episode = get_element_by_class(
            'site-font-secondary-color', get_element_by_class('text', watch_info))
        episode_number = int_or_none(self._search_regex(
            r'Episode (\d+)', season_episode or '', 'episode', default=None))

        return {
            '_type': 'url_transparent',
            'ie_key': VHXEmbedIE.ie_key(),
            'url': VHXEmbedIE._smuggle_referrer(embed_url, 'https://www.dropout.tv'),
            'id': self._search_regex(r'embed\.vhx\.tv/videos/(.+?)\?', embed_url, 'id'),
            'display_id': display_id,
            'title': title,
            'description': self._html_search_meta('description', webpage, fatal=False),
            'thumbnail': thumbnail.split('?')[0] if thumbnail else None,  # Ignore crop/downscale
            'series': clean_html(get_element_by_class('series-title', watch_info)),
            'episode_number': episode_number,
            'episode': title if episode_number else None,
            'season_number': int_or_none(self._search_regex(
                r'Season (\d+),', season_episode or '', 'season', default=None)),
            'release_date': unified_strdate(self._search_regex(
                r'data-meta-field-name=["\']release_dates["\'] data-meta-field-value=["\'](.+?)["\']',
                watch_info, 'release date', default=None)),
        }


class DropoutSeasonIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?P<id>[^\/$&?#]+)(?:/?$|/season:[0-9]+/?$)'
    _TESTS = [
        {
            'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1',
            'note': 'Multi-season series with the season in the url',
            'playlist_count': 17,
            'info_dict': {
                'id': 'dimension-20-fantasy-high-season-1',
                'title': 'Dimension 20 Fantasy High - Season 1'
            }
        },
        {
            'url': 'https://www.dropout.tv/dimension-20-fantasy-high',
            'note': 'Multi-season series with the season not in the url',
            'playlist_count': 17,
            'info_dict': {
                'id': 'dimension-20-fantasy-high-season-1',
                'title': 'Dimension 20 Fantasy High - Season 1'
            }
        },
        {
            'url': 'https://www.dropout.tv/dimension-20-shriek-week',
            'note': 'Single-season series',
            'playlist_count': 4,
            'info_dict': {
                'id': 'dimension-20-shriek-week-season-1',
                'title': 'Dimension 20 Shriek Week - Season 1'
            }
        }
    ]

    def _real_extract(self, url):
        season_id = self._match_id(url)
        season_title = season_id.replace('-', ' ').title()
        webpage = self._download_webpage(url, season_id)

        entries = [
            self.url_result(
                url=self._search_regex(r'<a href=["\'](.+?)["\'] class=["\']browse-item-link["\']',
                                       item, 'item_url'),
                ie=DropoutIE.ie_key()
            ) for item in get_elements_by_class('js-collection-item', webpage)
        ]

        seasons = (get_element_by_class('select-dropdown-wrapper', webpage) or '').strip().replace('\n', '')
        current_season = self._search_regex(r'<option[^>]+selected>([^<]+)</option>',
                                            seasons, 'current_season', default='').strip()

        return {
            '_type': 'playlist',
            'id': join_nonempty(season_id, current_season.lower().replace(' ', '-')),
            'title': join_nonempty(season_title, current_season, delim=' - '),
            'entries': entries
        }
Commit	Line	Data
e9efb99f S	1	from .common import InfoExtractor
	2	from .vimeo import VHXEmbedIE
	3	from ..utils import (
	4	clean_html,
	5	ExtractorError,
	6	get_element_by_class,
	7	get_element_by_id,
	8	get_elements_by_class,
	9	int_or_none,
	10	join_nonempty,
	11	unified_strdate,
	12	urlencode_postdata,
	13	)
	14
	15
	16	class DropoutIE(InfoExtractor):
	17	_LOGIN_URL = 'https://www.dropout.tv/login'
	18	_NETRC_MACHINE = 'dropout'
	19
	20	_VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?:[^/]+/)*videos/(?P<id>[^/]+)/?$'
	21	_TESTS = [
	22	{
	23	'url': 'https://www.dropout.tv/game-changer/season:2/videos/yes-or-no',
	24	'note': 'Episode in a series',
	25	'md5': '5e000fdfd8d8fa46ff40456f1c2af04a',
	26	'info_dict': {
	27	'id': '738153',
	28	'display_id': 'yes-or-no',
	29	'ext': 'mp4',
	30	'title': 'Yes or No',
	31	'description': 'Ally, Brennan, and Zac are asked a simple question, but is there a correct answer?',
	32	'release_date': '20200508',
	33	'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/351e3f24-c4a3-459a-8b79-dc80f1e5b7fd.jpg',
	34	'series': 'Game Changer',
	35	'season_number': 2,
	36	'season': 'Season 2',
	37	'episode_number': 6,
	38	'episode': 'Yes or No',
	39	'duration': 1180,
	40	'uploader_id': 'user80538407',
	41	'uploader_url': 'https://vimeo.com/user80538407',
	42	'uploader': 'OTT Videos'
	43	},
	44	'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest']
	45	},
	46	{
	47	'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1/videos/episode-1',
	48	'note': 'Episode in a series (missing release_date)',
	49	'md5': '712caf7c191f1c47c8f1879520c2fa5c',
	50	'info_dict': {
	51	'id': '320562',
	52	'display_id': 'episode-1',
	53	'ext': 'mp4',
	54	'title': 'The Beginning Begins',
	55	'description': 'The cast introduces their PCs, including a neurotic elf, a goblin PI, and a corn-worshipping cleric.',
	56	'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/4421ed0d-f630-4c88-9004-5251b2b8adfa.jpg',
	57	'series': 'Dimension 20: Fantasy High',
	58	'season_number': 1,
	59	'season': 'Season 1',
	60	'episode_number': 1,
	61	'episode': 'The Beginning Begins',
	62	'duration': 6838,
	63	'uploader_id': 'user80538407',
	64	'uploader_url': 'https://vimeo.com/user80538407',
65	'uploader': 'OTT Videos'
66	},
67	'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest']
68	},
69	{
70	'url': 'https://www.dropout.tv/videos/misfits-magic-holiday-special',
71	'note': 'Episode not in a series',
72	'md5': 'c30fa18999c5880d156339f13c953a26',
73	'info_dict': {
74	'id': '1915774',
75	'display_id': 'misfits-magic-holiday-special',
76	'ext': 'mp4',
77	'title': 'Misfits & Magic Holiday Special',
78	'description': 'The magical misfits spend Christmas break at Gowpenny, with an unwelcome visitor.',
79	'release_date': '20211215',
80	'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/d91ea8a6-b250-42ed-907e-b30fb1c65176-8e24b8e5.jpg',
81	'duration': 11698,
82	'uploader_id': 'user80538407',
83	'uploader_url': 'https://vimeo.com/user80538407',
84	'uploader': 'OTT Videos'
85	},
86	'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest']
87	}
88	]
89
90	def _get_authenticity_token(self, display_id):
91	signin_page = self._download_webpage(
92	self._LOGIN_URL, display_id, note='Getting authenticity token')
93	return self._html_search_regex(
94	r'name=["\']authenticity_token["\'] value=["\'](.+?)["\']',
95	signin_page, 'authenticity_token')
96
97	def _login(self, display_id):
98	username, password = self._get_login_info()
99	if not (username and password):
100	self.raise_login_required(method='password')
101
102	response = self._download_webpage(
103	self._LOGIN_URL, display_id, note='Logging in', data=urlencode_postdata({
104	'email': username,
105	'password': password,
106	'authenticity_token': self._get_authenticity_token(display_id),
107	'utf8': True
108	}))
109
110	user_has_subscription = self._search_regex(
111	r'user_has_subscription:\s*["\'](.+?)["\']', response, 'subscription status', default='none')
112	if user_has_subscription.lower() == 'true':
113	return response
114	elif user_has_subscription.lower() == 'false':
115	raise ExtractorError('Account is not subscribed')
116	else:
117	raise ExtractorError('Incorrect username/password')
118
119	def _real_extract(self, url):
120	display_id = self._match_id(url)
121	try:
122	self._login(display_id)
123	webpage = self._download_webpage(url, display_id, note='Downloading video webpage')
124	finally:
a44ca5a4	125	self._download_webpage('https://www.dropout.tv/logout', display_id, note='Logging out', fatal=False)
e9efb99f S	126
	127	embed_url = self._search_regex(r'embed_url:\s*["\'](.+?)["\']', webpage, 'embed url')
	128	thumbnail = self._og_search_thumbnail(webpage)
	129	watch_info = get_element_by_id('watch-info', webpage) or ''
	130
	131	title = clean_html(get_element_by_class('video-title', watch_info))
	132	season_episode = get_element_by_class(
	133	'site-font-secondary-color', get_element_by_class('text', watch_info))
	134	episode_number = int_or_none(self._search_regex(
	135	r'Episode (\d+)', season_episode or '', 'episode', default=None))
	136
	137	return {
	138	'_type': 'url_transparent',
	139	'ie_key': VHXEmbedIE.ie_key(),
aedaa455	140	'url': VHXEmbedIE._smuggle_referrer(embed_url, 'https://www.dropout.tv'),
a44ca5a4	141	'id': self._search_regex(r'embed\.vhx\.tv/videos/(.+?)\?', embed_url, 'id'),
e9efb99f S	142	'display_id': display_id,
	143	'title': title,
	144	'description': self._html_search_meta('description', webpage, fatal=False),
	145	'thumbnail': thumbnail.split('?')[0] if thumbnail else None, # Ignore crop/downscale
	146	'series': clean_html(get_element_by_class('series-title', watch_info)),
	147	'episode_number': episode_number,
	148	'episode': title if episode_number else None,
	149	'season_number': int_or_none(self._search_regex(
	150	r'Season (\d+),', season_episode or '', 'season', default=None)),
	151	'release_date': unified_strdate(self._search_regex(
	152	r'data-meta-field-name=["\']release_dates["\'] data-meta-field-value=["\'](.+?)["\']',
	153	watch_info, 'release date', default=None)),
	154	}
	155
	156
	157	class DropoutSeasonIE(InfoExtractor):
	158	_VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?P<id>[^\/$&?#]+)(?:/?$\|/season:[0-9]+/?$)'
	159	_TESTS = [
	160	{
	161	'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1',
	162	'note': 'Multi-season series with the season in the url',
	163	'playlist_count': 17,
	164	'info_dict': {
	165	'id': 'dimension-20-fantasy-high-season-1',
	166	'title': 'Dimension 20 Fantasy High - Season 1'
	167	}
	168	},
	169	{
	170	'url': 'https://www.dropout.tv/dimension-20-fantasy-high',
	171	'note': 'Multi-season series with the season not in the url',
	172	'playlist_count': 17,
	173	'info_dict': {
	174	'id': 'dimension-20-fantasy-high-season-1',
	175	'title': 'Dimension 20 Fantasy High - Season 1'
	176	}
	177	},
	178	{
	179	'url': 'https://www.dropout.tv/dimension-20-shriek-week',
	180	'note': 'Single-season series',
	181	'playlist_count': 4,
	182	'info_dict': {
	183	'id': 'dimension-20-shriek-week-season-1',
	184	'title': 'Dimension 20 Shriek Week - Season 1'
	185	}
	186	}
	187	]
	188
	189	def _real_extract(self, url):
	190	season_id = self._match_id(url)
	191	season_title = season_id.replace('-', ' ').title()
	192	webpage = self._download_webpage(url, season_id)
	193
	194	entries = [
	195	self.url_result(
	196	url=self._search_regex(r'<a href=["\'](.+?)["\'] class=["\']browse-item-link["\']',
	197	item, 'item_url'),
	198	ie=DropoutIE.ie_key()
	199	) for item in get_elements_by_class('js-collection-item', webpage)
	200	]
	201
	202	seasons = (get_element_by_class('select-dropdown-wrapper', webpage) or '').strip().replace('\n', '')
	203	current_season = self._search_regex(r'<option[^>]+selected>([^<]+)</option>',
	204	seasons, 'current_season', default='').strip()
	205
206	return {
207	'_type': 'playlist',
208	'id': join_nonempty(season_id, current_season.lower().replace(' ', '-')),
209	'title': join_nonempty(season_title, current_season, delim=' - '),
210	'entries': entries
211	}