[yt-dlp.git] / yt_dlp / extractor / redgifs.py

import functools

from .common import InfoExtractor
from ..compat import compat_parse_qs
from ..utils import (
    ExtractorError,
    int_or_none,
    qualities,
    try_get,
    OnDemandPagedList,
)


class RedGifsBaseInfoExtractor(InfoExtractor):
    _FORMATS = {
        'gif': 250,
        'sd': 480,
        'hd': None,
    }

    def _parse_gif_data(self, gif_data):
        video_id = gif_data.get('id')
        quality = qualities(tuple(self._FORMATS.keys()))

        orig_height = int_or_none(gif_data.get('height'))
        aspect_ratio = try_get(gif_data, lambda x: orig_height / x['width'])

        formats = []
        for format_id, height in self._FORMATS.items():
            video_url = gif_data['urls'].get(format_id)
            if not video_url:
                continue
            height = min(orig_height, height or orig_height)
            formats.append({
                'url': video_url,
                'format_id': format_id,
                'width': height * aspect_ratio if aspect_ratio else None,
                'height': height,
                'quality': quality(format_id),
            })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'webpage_url': f'https://redgifs.com/watch/{video_id}',
            'ie_key': RedGifsIE.ie_key(),
            'extractor': 'RedGifs',
            'title': ' '.join(gif_data.get('tags') or []) or 'RedGifs',
            'timestamp': int_or_none(gif_data.get('createDate')),
            'uploader': gif_data.get('userName'),
            'duration': int_or_none(gif_data.get('duration')),
            'view_count': int_or_none(gif_data.get('views')),
            'like_count': int_or_none(gif_data.get('likes')),
            'categories': gif_data.get('tags') or [],
            'tags': gif_data.get('tags'),
            'age_limit': 18,
            'formats': formats,
        }

    def _call_api(self, ep, video_id, *args, **kwargs):
        data = self._download_json(
            f'https://api.redgifs.com/v2/{ep}', video_id, *args, **kwargs)
        if 'error' in data:
            raise ExtractorError(f'RedGifs said: {data["error"]}', expected=True, video_id=video_id)
        return data

    def _fetch_page(self, ep, video_id, query, page):
        query['page'] = page + 1
        data = self._call_api(
            ep, video_id, query=query, note=f'Downloading JSON metadata page {page + 1}')

        for entry in data['gifs']:
            yield self._parse_gif_data(entry)

    def _prepare_api_query(self, query, fields):
        api_query = [
            (field_name, query.get(field_name, (default,))[0])
            for field_name, default in fields.items()]

        return {key: val for key, val in api_query if val is not None}

    def _paged_entries(self, ep, item_id, query, fields):
        page = int_or_none(query.get('page', (None,))[0])
        page_fetcher = functools.partial(
            self._fetch_page, ep, item_id, self._prepare_api_query(query, fields))
        return page_fetcher(page) if page else OnDemandPagedList(page_fetcher, self._PAGE_SIZE)


class RedGifsIE(RedGifsBaseInfoExtractor):
    _VALID_URL = r'https?://(?:(?:www\.)?redgifs\.com/watch/|thumbs2\.redgifs\.com/)(?P<id>[^-/?#\.]+)'
    _TESTS = [{
        'url': 'https://www.redgifs.com/watch/squeakyhelplesswisent',
        'info_dict': {
            'id': 'squeakyhelplesswisent',
            'ext': 'mp4',
            'title': 'Hotwife Legs Thick',
            'timestamp': 1636287915,
            'upload_date': '20211107',
            'uploader': 'ignored52',
            'duration': 16,
            'view_count': int,
            'like_count': int,
            'categories': list,
            'age_limit': 18,
        }
    }, {
        'url': 'https://thumbs2.redgifs.com/SqueakyHelplessWisent-mobile.mp4#t=0',
        'info_dict': {
            'id': 'squeakyhelplesswisent',
            'ext': 'mp4',
            'title': 'Hotwife Legs Thick',
            'timestamp': 1636287915,
            'upload_date': '20211107',
            'uploader': 'ignored52',
            'duration': 16,
            'view_count': int,
            'like_count': int,
            'categories': list,
            'age_limit': 18,
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url).lower()
        video_info = self._call_api(
            f'gifs/{video_id}', video_id, note='Downloading video info')
        return self._parse_gif_data(video_info['gif'])


class RedGifsSearchIE(RedGifsBaseInfoExtractor):
    IE_DESC = 'Redgifs search'
    _VALID_URL = r'https?://(?:www\.)?redgifs\.com/browse\?(?P<query>[^#]+)'
    _PAGE_SIZE = 80
    _TESTS = [
        {
            'url': 'https://www.redgifs.com/browse?tags=Lesbian',
            'info_dict': {
                'id': 'tags=Lesbian',
                'title': 'Lesbian',
                'description': 'RedGifs search for Lesbian, ordered by trending'
            },
            'playlist_mincount': 100,
        },
        {
            'url': 'https://www.redgifs.com/browse?type=g&order=latest&tags=Lesbian',
            'info_dict': {
                'id': 'type=g&order=latest&tags=Lesbian',
                'title': 'Lesbian',
                'description': 'RedGifs search for Lesbian, ordered by latest'
            },
            'playlist_mincount': 100,
        },
        {
            'url': 'https://www.redgifs.com/browse?type=g&order=latest&tags=Lesbian&page=2',
            'info_dict': {
                'id': 'type=g&order=latest&tags=Lesbian&page=2',
                'title': 'Lesbian',
                'description': 'RedGifs search for Lesbian, ordered by latest'
            },
            'playlist_count': 80,
        }
    ]

    def _real_extract(self, url):
        query_str = self._match_valid_url(url).group('query')
        query = compat_parse_qs(query_str)
        if not query.get('tags'):
            raise ExtractorError('Invalid query tags', expected=True)

        tags = query.get('tags')[0]
        order = query.get('order', ('trending',))[0]

        query['search_text'] = [tags]
        entries = self._paged_entries('gifs/search', query_str, query, {
            'search_text': None,
            'order': 'trending',
            'type': None,
        })

        return self.playlist_result(
            entries, query_str, tags, f'RedGifs search for {tags}, ordered by {order}')


class RedGifsUserIE(RedGifsBaseInfoExtractor):
    IE_DESC = 'Redgifs user'
    _VALID_URL = r'https?://(?:www\.)?redgifs\.com/users/(?P<username>[^/?#]+)(?:\?(?P<query>[^#]+))?'
    _PAGE_SIZE = 30
    _TESTS = [
        {
            'url': 'https://www.redgifs.com/users/lamsinka89',
            'info_dict': {
                'id': 'lamsinka89',
                'title': 'lamsinka89',
                'description': 'RedGifs user lamsinka89, ordered by recent'
            },
            'playlist_mincount': 100,
        },
        {
            'url': 'https://www.redgifs.com/users/lamsinka89?page=3',
            'info_dict': {
                'id': 'lamsinka89?page=3',
                'title': 'lamsinka89',
                'description': 'RedGifs user lamsinka89, ordered by recent'
            },
            'playlist_count': 30,
        },
        {
            'url': 'https://www.redgifs.com/users/lamsinka89?order=best&type=g',
            'info_dict': {
                'id': 'lamsinka89?order=best&type=g',
                'title': 'lamsinka89',
                'description': 'RedGifs user lamsinka89, ordered by best'
            },
            'playlist_mincount': 100,
        }
    ]

    def _real_extract(self, url):
        username, query_str = self._match_valid_url(url).group('username', 'query')
        playlist_id = f'{username}?{query_str}' if query_str else username

        query = compat_parse_qs(query_str)
        order = query.get('order', ('recent',))[0]

        entries = self._paged_entries(f'users/{username}/search', playlist_id, query, {
            'order': 'recent',
            'type': None,
        })

        return self.playlist_result(
            entries, playlist_id, username, f'RedGifs user {username}, ordered by {order}')
Commit	Line	Data
bf57cfa8	1	import functools
4e4ba1d7	2
4e4ba1d7	3	from .common import InfoExtractor
bf57cfa8	4	from ..compat import compat_parse_qs
4e4ba1d7	5	from ..utils import (
	6	ExtractorError,
	7	int_or_none,
	8	qualities,
	9	try_get,
bf57cfa8	10	OnDemandPagedList,
4e4ba1d7	11	)
	12
	13
bf57cfa8	14	class RedGifsBaseInfoExtractor(InfoExtractor):
4e4ba1d7	15	_FORMATS = {
	16	'gif': 250,
	17	'sd': 480,
	18	'hd': None,
	19	}
bf57cfa8 DS	20
	21	def _parse_gif_data(self, gif_data):
	22	video_id = gif_data.get('id')
	23	quality = qualities(tuple(self._FORMATS.keys()))
	24
	25	orig_height = int_or_none(gif_data.get('height'))
	26	aspect_ratio = try_get(gif_data, lambda x: orig_height / x['width'])
	27
	28	formats = []
	29	for format_id, height in self._FORMATS.items():
	30	video_url = gif_data['urls'].get(format_id)
	31	if not video_url:
	32	continue
	33	height = min(orig_height, height or orig_height)
	34	formats.append({
	35	'url': video_url,
	36	'format_id': format_id,
	37	'width': height * aspect_ratio if aspect_ratio else None,
	38	'height': height,
	39	'quality': quality(format_id),
	40	})
	41	self._sort_formats(formats)
	42
	43	return {
	44	'id': video_id,
	45	'webpage_url': f'https://redgifs.com/watch/{video_id}',
	46	'ie_key': RedGifsIE.ie_key(),
	47	'extractor': 'RedGifs',
	48	'title': ' '.join(gif_data.get('tags') or []) or 'RedGifs',
	49	'timestamp': int_or_none(gif_data.get('createDate')),
	50	'uploader': gif_data.get('userName'),
	51	'duration': int_or_none(gif_data.get('duration')),
	52	'view_count': int_or_none(gif_data.get('views')),
	53	'like_count': int_or_none(gif_data.get('likes')),
	54	'categories': gif_data.get('tags') or [],
	55	'tags': gif_data.get('tags'),
	56	'age_limit': 18,
	57	'formats': formats,
	58	}
	59
	60	def _call_api(self, ep, video_id, args, *kwargs):
	61	data = self._download_json(
	62	f'https://api.redgifs.com/v2/{ep}', video_id, args, *kwargs)
	63	if 'error' in data:
	64	raise ExtractorError(f'RedGifs said: {data["error"]}', expected=True, video_id=video_id)
	65	return data
	66
	67	def _fetch_page(self, ep, video_id, query, page):
	68	query['page'] = page + 1
	69	data = self._call_api(
	70	ep, video_id, query=query, note=f'Downloading JSON metadata page {page + 1}')
	71
	72	for entry in data['gifs']:
	73	yield self._parse_gif_data(entry)
	74
	75	def _prepare_api_query(self, query, fields):
	76	api_query = [
	77	(field_name, query.get(field_name, (default,))[0])
	78	for field_name, default in fields.items()]
	79
	80	return {key: val for key, val in api_query if val is not None}
	81
	82	def _paged_entries(self, ep, item_id, query, fields):
	83	page = int_or_none(query.get('page', (None,))[0])
84	page_fetcher = functools.partial(
85	self._fetch_page, ep, item_id, self._prepare_api_query(query, fields))
86	return page_fetcher(page) if page else OnDemandPagedList(page_fetcher, self._PAGE_SIZE)
87
88
89	class RedGifsIE(RedGifsBaseInfoExtractor):
90	_VALID_URL = r'https?://(?:(?:www\.)?redgifs\.com/watch/\|thumbs2\.redgifs\.com/)(?P<id>[^-/?#\.]+)'
4e4ba1d7	91	_TESTS = [{
	92	'url': 'https://www.redgifs.com/watch/squeakyhelplesswisent',
	93	'info_dict': {
	94	'id': 'squeakyhelplesswisent',
	95	'ext': 'mp4',
	96	'title': 'Hotwife Legs Thick',
	97	'timestamp': 1636287915,
	98	'upload_date': '20211107',
	99	'uploader': 'ignored52',
	100	'duration': 16,
	101	'view_count': int,
	102	'like_count': int,
	103	'categories': list,
	104	'age_limit': 18,
	105	}
	106	}, {
	107	'url': 'https://thumbs2.redgifs.com/SqueakyHelplessWisent-mobile.mp4#t=0',
	108	'info_dict': {
	109	'id': 'squeakyhelplesswisent',
	110	'ext': 'mp4',
	111	'title': 'Hotwife Legs Thick',
	112	'timestamp': 1636287915,
	113	'upload_date': '20211107',
	114	'uploader': 'ignored52',
	115	'duration': 16,
	116	'view_count': int,
	117	'like_count': int,
	118	'categories': list,
	119	'age_limit': 18,
	120	}
	121	}]
	122
	123	def _real_extract(self, url):
	124	video_id = self._match_id(url).lower()
bf57cfa8 DS	125	video_info = self._call_api(
	126	f'gifs/{video_id}', video_id, note='Downloading video info')
	127	return self._parse_gif_data(video_info['gif'])
4e4ba1d7	128
4e4ba1d7	129
bf57cfa8 DS	130	class RedGifsSearchIE(RedGifsBaseInfoExtractor):
	131	IE_DESC = 'Redgifs search'
	132	_VALID_URL = r'https?://(?:www\.)?redgifs\.com/browse\?(?P<query>[^#]+)'
	133	_PAGE_SIZE = 80
	134	_TESTS = [
	135	{
	136	'url': 'https://www.redgifs.com/browse?tags=Lesbian',
	137	'info_dict': {
	138	'id': 'tags=Lesbian',
	139	'title': 'Lesbian',
	140	'description': 'RedGifs search for Lesbian, ordered by trending'
	141	},
	142	'playlist_mincount': 100,
	143	},
	144	{
	145	'url': 'https://www.redgifs.com/browse?type=g&order=latest&tags=Lesbian',
	146	'info_dict': {
	147	'id': 'type=g&order=latest&tags=Lesbian',
	148	'title': 'Lesbian',
	149	'description': 'RedGifs search for Lesbian, ordered by latest'
	150	},
	151	'playlist_mincount': 100,
	152	},
	153	{
	154	'url': 'https://www.redgifs.com/browse?type=g&order=latest&tags=Lesbian&page=2',
	155	'info_dict': {
	156	'id': 'type=g&order=latest&tags=Lesbian&page=2',
	157	'title': 'Lesbian',
	158	'description': 'RedGifs search for Lesbian, ordered by latest'
	159	},
	160	'playlist_count': 80,
	161	}
	162	]
4e4ba1d7	163
bf57cfa8 DS	164	def _real_extract(self, url):
	165	query_str = self._match_valid_url(url).group('query')
	166	query = compat_parse_qs(query_str)
	167	if not query.get('tags'):
	168	raise ExtractorError('Invalid query tags', expected=True)
4e4ba1d7	169
bf57cfa8 DS	170	tags = query.get('tags')[0]
bf57cfa8 DS	171	order = query.get('order', ('trending',))[0]
4e4ba1d7	172
bf57cfa8 DS	173	query['search_text'] = [tags]
	174	entries = self._paged_entries('gifs/search', query_str, query, {
	175	'search_text': None,
	176	'order': 'trending',
	177	'type': None,
	178	})
4e4ba1d7	179
bf57cfa8 DS	180	return self.playlist_result(
	181	entries, query_str, tags, f'RedGifs search for {tags}, ordered by {order}')
	182
	183
	184	class RedGifsUserIE(RedGifsBaseInfoExtractor):
	185	IE_DESC = 'Redgifs user'
	186	_VALID_URL = r'https?://(?:www\.)?redgifs\.com/users/(?P<username>[^/?#]+)(?:\?(?P<query>[^#]+))?'
	187	_PAGE_SIZE = 30
	188	_TESTS = [
	189	{
	190	'url': 'https://www.redgifs.com/users/lamsinka89',
	191	'info_dict': {
	192	'id': 'lamsinka89',
	193	'title': 'lamsinka89',
	194	'description': 'RedGifs user lamsinka89, ordered by recent'
	195	},
	196	'playlist_mincount': 100,
	197	},
	198	{
	199	'url': 'https://www.redgifs.com/users/lamsinka89?page=3',
	200	'info_dict': {
	201	'id': 'lamsinka89?page=3',
	202	'title': 'lamsinka89',
	203	'description': 'RedGifs user lamsinka89, ordered by recent'
	204	},
	205	'playlist_count': 30,
	206	},
	207	{
	208	'url': 'https://www.redgifs.com/users/lamsinka89?order=best&type=g',
	209	'info_dict': {
	210	'id': 'lamsinka89?order=best&type=g',
	211	'title': 'lamsinka89',
	212	'description': 'RedGifs user lamsinka89, ordered by best'
	213	},
	214	'playlist_mincount': 100,
4e4ba1d7	215	}
bf57cfa8 DS	216	]
	217
	218	def _real_extract(self, url):
	219	username, query_str = self._match_valid_url(url).group('username', 'query')
	220	playlist_id = f'{username}?{query_str}' if query_str else username
	221
	222	query = compat_parse_qs(query_str)
	223	order = query.get('order', ('recent',))[0]
	224
	225	entries = self._paged_entries(f'users/{username}/search', playlist_id, query, {
	226	'order': 'recent',
	227	'type': None,
	228	})
	229
	230	return self.playlist_result(
	231	entries, playlist_id, username, f'RedGifs user {username}, ordered by {order}')