[yt-dlp.git] / yt_dlp / extractor / allstar.py

import functools
import json

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    OnDemandPagedList,
    int_or_none,
    join_nonempty,
    parse_qs,
    urljoin,
)
from ..utils.traversal import traverse_obj

_FIELDS = '''
    _id
    clipImageSource
    clipImageThumb
    clipLink
    clipTitle
    createdDate
    shareId
    user { _id }
    username
    views'''

_EXTRA_FIELDS = '''
    clipLength
    clipSizeBytes'''

_QUERIES = {
    'clip': '''query ($id: String!) {
        video: getClip(clipIdentifier: $id) {
            %s %s
        }
    }''' % (_FIELDS, _EXTRA_FIELDS),
    'montage': '''query ($id: String!) {
        video: getMontage(clipIdentifier: $id) {
            %s
        }
    }''' % _FIELDS,
    'Clips': '''query ($page: Int!, $user: String!, $game: Int) {
        videos: clips(search: createdDate, page: $page, user: $user, mobile: false, game: $game) {
            data { %s %s }
        }
    }''' % (_FIELDS, _EXTRA_FIELDS),
    'Montages': '''query ($page: Int!, $user: String!) {
        videos: montages(search: createdDate, page: $page, user: $user) {
            data { %s }
        }
    }''' % _FIELDS,
    'Mobile Clips': '''query ($page: Int!, $user: String!) {
        videos: clips(search: createdDate, page: $page, user: $user, mobile: true) {
            data { %s %s }
        }
    }''' % (_FIELDS, _EXTRA_FIELDS),
}


class AllstarBaseIE(InfoExtractor):
    @staticmethod
    def _parse_video_data(video_data):
        def media_url_or_none(path):
            return urljoin('https://media.allstar.gg/', path)

        info = traverse_obj(video_data, {
            'id': ('_id', {str}),
            'display_id': ('shareId', {str}),
            'title': ('clipTitle', {str}),
            'url': ('clipLink', {media_url_or_none}),
            'thumbnails': (('clipImageThumb', 'clipImageSource'), {'url': {media_url_or_none}}),
            'duration': ('clipLength', {int_or_none}),
            'filesize': ('clipSizeBytes', {int_or_none}),
            'timestamp': ('createdDate', {functools.partial(int_or_none, scale=1000)}),
            'uploader': ('username', {str}),
            'uploader_id': ('user', '_id', {str}),
            'view_count': ('views', {int_or_none}),
        })

        if info.get('id') and info.get('url'):
            basename = 'clip' if '/clips/' in info['url'] else 'montage'
            info['webpage_url'] = f'https://allstar.gg/{basename}?{basename}={info["id"]}'

        info.update({
            'extractor_key': AllstarIE.ie_key(),
            'extractor': AllstarIE.IE_NAME,
            'uploader_url': urljoin('https://allstar.gg/u/', info.get('uploader_id')),
        })

        return info

    def _call_api(self, query, variables, path, video_id=None, note=None):
        response = self._download_json(
            'https://a1.allstar.gg/graphql', video_id, note=note,
            headers={'content-type': 'application/json'},
            data=json.dumps({'variables': variables, 'query': query}).encode())

        errors = traverse_obj(response, ('errors', ..., 'message', {str}))
        if errors:
            raise ExtractorError('; '.join(errors))

        return traverse_obj(response, path)


class AllstarIE(AllstarBaseIE):
    _VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?P<type>(?:clip|montage))\?(?P=type)=(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'https://allstar.gg/clip?clip=64482c2da9eec30008a67d1b',
        'info_dict': {
            'id': '64482c2da9eec30008a67d1b',
            'title': '4K on Inferno',
            'url': 'md5:66befb5381eef0c9456026386c25fa55',
            'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
            'uploader': 'chrk.',
            'ext': 'mp4',
            'duration': 20,
            'filesize': 21199257,
            'timestamp': 1682451501,
            'uploader_id': '62b8bdfc9021052f7905882d',
            'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
            'upload_date': '20230425',
            'view_count': int,
        }
    }, {
        'url': 'https://allstar.gg/clip?clip=8LJLY4JKB',
        'info_dict': {
            'id': '64a1ec6b887f4c0008dc50b8',
            'display_id': '8LJLY4JKB',
            'title': 'AK-47 3K on Mirage',
            'url': 'md5:dde224fd12f035c0e2529a4ae34c4283',
            'ext': 'mp4',
            'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
            'duration': 16,
            'filesize': 30175859,
            'timestamp': 1688333419,
            'uploader': 'cherokee',
            'uploader_id': '62b8bdfc9021052f7905882d',
            'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
            'upload_date': '20230702',
            'view_count': int,
        }
    }, {
        'url': 'https://allstar.gg/montage?montage=643e64089da7e9363e1fa66c',
        'info_dict': {
            'id': '643e64089da7e9363e1fa66c',
            'display_id': 'APQLGM2IMXW',
            'title': 'cherokee Rapid Fire Snipers Montage',
            'url': 'md5:a3ee356022115db2b27c81321d195945',
            'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
            'ext': 'mp4',
            'timestamp': 1681810448,
            'uploader': 'cherokee',
            'uploader_id': '62b8bdfc9021052f7905882d',
            'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
            'upload_date': '20230418',
            'view_count': int,
        }
    }, {
        'url': 'https://allstar.gg/montage?montage=RILJMH6QOS',
        'info_dict': {
            'id': '64a2697372ce3703de29e868',
            'display_id': 'RILJMH6QOS',
            'title': 'cherokee Rapid Fire Snipers Montage',
            'url': 'md5:d5672e6f88579730c2310a80fdbc4030',
            'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
            'ext': 'mp4',
            'timestamp': 1688365434,
            'uploader': 'cherokee',
            'uploader_id': '62b8bdfc9021052f7905882d',
            'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
            'upload_date': '20230703',
            'view_count': int,
        }
    }]

    def _real_extract(self, url):
        query_id, video_id = self._match_valid_url(url).group('type', 'id')

        return self._parse_video_data(
            self._call_api(
                _QUERIES.get(query_id), {'id': video_id}, ('data', 'video'), video_id))


class AllstarProfileIE(AllstarBaseIE):
    _VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?:profile\?user=|u/)(?P<id>[^/?#&]+)'

    _TESTS = [{
        'url': 'https://allstar.gg/profile?user=62b8bdfc9021052f7905882d',
        'info_dict': {
            'id': '62b8bdfc9021052f7905882d-clips',
            'title': 'cherokee - Clips',
        },
        'playlist_mincount': 15
    }, {
        'url': 'https://allstar.gg/u/cherokee?game=730&view=Clips',
        'info_dict': {
            'id': '62b8bdfc9021052f7905882d-clips-730',
            'title': 'cherokee - Clips - 730',
        },
        'playlist_mincount': 15
    }, {
        'url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d?view=Montages',
        'info_dict': {
            'id': '62b8bdfc9021052f7905882d-montages',
            'title': 'cherokee - Montages',
        },
        'playlist_mincount': 4
    }, {
        'url': 'https://allstar.gg/profile?user=cherokee&view=Mobile Clips',
        'info_dict': {
            'id': '62b8bdfc9021052f7905882d-mobile',
            'title': 'cherokee - Mobile Clips',
        },
        'playlist_mincount': 1
    }]

    _PAGE_SIZE = 10

    def _get_page(self, user_id, display_id, game, query, page_num):
        page_num += 1

        for video_data in self._call_api(
                query, {
                    'user': user_id,
                    'page': page_num,
                    'game': game,
                }, ('data', 'videos', 'data'), display_id, f'Downloading page {page_num}'):
            yield self._parse_video_data(video_data)

    def _real_extract(self, url):
        display_id = self._match_id(url)
        profile_data = self._download_json(
            urljoin('https://api.allstar.gg/v1/users/profile/', display_id), display_id)
        user_id = traverse_obj(profile_data, ('data', ('_id'), {str}))
        if not user_id:
            raise ExtractorError('Unable to extract the user id')

        username = traverse_obj(profile_data, ('data', 'profile', ('username'), {str}))
        url_query = parse_qs(url)
        game = traverse_obj(url_query, ('game', 0, {int_or_none}))
        query_id = traverse_obj(url_query, ('view', 0), default='Clips')

        if query_id not in ('Clips', 'Montages', 'Mobile Clips'):
            raise ExtractorError(f'Unsupported playlist URL type {query_id!r}')

        return self.playlist_result(
            OnDemandPagedList(
                functools.partial(
                    self._get_page, user_id, display_id, game, _QUERIES.get(query_id)), self._PAGE_SIZE),
            playlist_id=join_nonempty(user_id, query_id.lower().split()[0], game),
            playlist_title=join_nonempty((username or display_id), query_id, game, delim=' - '))
Commit	Line	Data
3237f8ba SA	1	import functools
	2	import json
	3
	4	from .common import InfoExtractor
	5	from ..utils import (
	6	ExtractorError,
	7	OnDemandPagedList,
	8	int_or_none,
	9	join_nonempty,
	10	parse_qs,
	11	urljoin,
	12	)
	13	from ..utils.traversal import traverse_obj
	14
3237f8ba SA	15	_FIELDS = '''
	16	_id
	17	clipImageSource
	18	clipImageThumb
	19	clipLink
	20	clipTitle
	21	createdDate
	22	shareId
	23	user { _id }
	24	username
	25	views'''
	26
	27	_EXTRA_FIELDS = '''
	28	clipLength
	29	clipSizeBytes'''
	30
	31	_QUERIES = {
	32	'clip': '''query ($id: String!) {
	33	video: getClip(clipIdentifier: $id) {
	34	%s %s
	35	}
	36	}''' % (_FIELDS, _EXTRA_FIELDS),
	37	'montage': '''query ($id: String!) {
	38	video: getMontage(clipIdentifier: $id) {
	39	%s
	40	}
	41	}''' % _FIELDS,
	42	'Clips': '''query ($page: Int!, $user: String!, $game: Int) {
	43	videos: clips(search: createdDate, page: $page, user: $user, mobile: false, game: $game) {
	44	data { %s %s }
	45	}
	46	}''' % (_FIELDS, _EXTRA_FIELDS),
	47	'Montages': '''query ($page: Int!, $user: String!) {
	48	videos: montages(search: createdDate, page: $page, user: $user) {
	49	data { %s }
	50	}
	51	}''' % _FIELDS,
	52	'Mobile Clips': '''query ($page: Int!, $user: String!) {
	53	videos: clips(search: createdDate, page: $page, user: $user, mobile: true) {
	54	data { %s %s }
	55	}
	56	}''' % (_FIELDS, _EXTRA_FIELDS),
	57	}
	58
	59
	60	class AllstarBaseIE(InfoExtractor):
	61	@staticmethod
	62	def _parse_video_data(video_data):
	63	def media_url_or_none(path):
	64	return urljoin('https://media.allstar.gg/', path)
	65
	66	info = traverse_obj(video_data, {
	67	'id': ('_id', {str}),
	68	'display_id': ('shareId', {str}),
	69	'title': ('clipTitle', {str}),
	70	'url': ('clipLink', {media_url_or_none}),
	71	'thumbnails': (('clipImageThumb', 'clipImageSource'), {'url': {media_url_or_none}}),
	72	'duration': ('clipLength', {int_or_none}),
	73	'filesize': ('clipSizeBytes', {int_or_none}),
	74	'timestamp': ('createdDate', {functools.partial(int_or_none, scale=1000)}),
	75	'uploader': ('username', {str}),
	76	'uploader_id': ('user', '_id', {str}),
	77	'view_count': ('views', {int_or_none}),
	78	})
79
80	if info.get('id') and info.get('url'):
81	basename = 'clip' if '/clips/' in info['url'] else 'montage'
82	info['webpage_url'] = f'https://allstar.gg/{basename}?{basename}={info["id"]}'
83
84	info.update({
85	'extractor_key': AllstarIE.ie_key(),
86	'extractor': AllstarIE.IE_NAME,
87	'uploader_url': urljoin('https://allstar.gg/u/', info.get('uploader_id')),
88	})
89
90	return info
91
92	def _call_api(self, query, variables, path, video_id=None, note=None):
93	response = self._download_json(
94	'https://a1.allstar.gg/graphql', video_id, note=note,
95	headers={'content-type': 'application/json'},
96	data=json.dumps({'variables': variables, 'query': query}).encode())
97
98	errors = traverse_obj(response, ('errors', ..., 'message', {str}))
99	if errors:
100	raise ExtractorError('; '.join(errors))
101
102	return traverse_obj(response, path)
103
104
105	class AllstarIE(AllstarBaseIE):
106	_VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?P<type>(?:clip\|montage))\?(?P=type)=(?P<id>[^/?#&]+)'
107
108	_TESTS = [{
109	'url': 'https://allstar.gg/clip?clip=64482c2da9eec30008a67d1b',
110	'info_dict': {
111	'id': '64482c2da9eec30008a67d1b',
112	'title': '4K on Inferno',
113	'url': 'md5:66befb5381eef0c9456026386c25fa55',
114	'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png\|jpg)$',
115	'uploader': 'chrk.',
116	'ext': 'mp4',
117	'duration': 20,
118	'filesize': 21199257,
119	'timestamp': 1682451501,
120	'uploader_id': '62b8bdfc9021052f7905882d',
121	'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
122	'upload_date': '20230425',
123	'view_count': int,
124	}
125	}, {
126	'url': 'https://allstar.gg/clip?clip=8LJLY4JKB',
127	'info_dict': {
128	'id': '64a1ec6b887f4c0008dc50b8',
129	'display_id': '8LJLY4JKB',
130	'title': 'AK-47 3K on Mirage',
131	'url': 'md5:dde224fd12f035c0e2529a4ae34c4283',
132	'ext': 'mp4',
133	'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png\|jpg)$',
134	'duration': 16,
135	'filesize': 30175859,
136	'timestamp': 1688333419,
137	'uploader': 'cherokee',
138	'uploader_id': '62b8bdfc9021052f7905882d',
139	'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
140	'upload_date': '20230702',
141	'view_count': int,
142	}
143	}, {
144	'url': 'https://allstar.gg/montage?montage=643e64089da7e9363e1fa66c',
145	'info_dict': {
146	'id': '643e64089da7e9363e1fa66c',
147	'display_id': 'APQLGM2IMXW',
148	'title': 'cherokee Rapid Fire Snipers Montage',
149	'url': 'md5:a3ee356022115db2b27c81321d195945',
150	'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png\|jpg)$',
151	'ext': 'mp4',
152	'timestamp': 1681810448,
153	'uploader': 'cherokee',
154	'uploader_id': '62b8bdfc9021052f7905882d',
155	'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
156	'upload_date': '20230418',
157	'view_count': int,
158	}
159	}, {
160	'url': 'https://allstar.gg/montage?montage=RILJMH6QOS',
161	'info_dict': {
162	'id': '64a2697372ce3703de29e868',
163	'display_id': 'RILJMH6QOS',
164	'title': 'cherokee Rapid Fire Snipers Montage',
165	'url': 'md5:d5672e6f88579730c2310a80fdbc4030',
166	'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png\|jpg)$',
167	'ext': 'mp4',
168	'timestamp': 1688365434,
169	'uploader': 'cherokee',
170	'uploader_id': '62b8bdfc9021052f7905882d',
171	'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
172	'upload_date': '20230703',
173	'view_count': int,
174	}
175	}]
176
177	def _real_extract(self, url):
178	query_id, video_id = self._match_valid_url(url).group('type', 'id')
179
180	return self._parse_video_data(
181	self._call_api(
182	_QUERIES.get(query_id), {'id': video_id}, ('data', 'video'), video_id))
183
184
185	class AllstarProfileIE(AllstarBaseIE):
186	_VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?:profile\?user=\|u/)(?P<id>[^/?#&]+)'
187
188	_TESTS = [{
189	'url': 'https://allstar.gg/profile?user=62b8bdfc9021052f7905882d',
190	'info_dict': {
191	'id': '62b8bdfc9021052f7905882d-clips',
192	'title': 'cherokee - Clips',
193	},
194	'playlist_mincount': 15
195	}, {
196	'url': 'https://allstar.gg/u/cherokee?game=730&view=Clips',
197	'info_dict': {
198	'id': '62b8bdfc9021052f7905882d-clips-730',
199	'title': 'cherokee - Clips - 730',
200	},
201	'playlist_mincount': 15
202	}, {
203	'url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d?view=Montages',
204	'info_dict': {
205	'id': '62b8bdfc9021052f7905882d-montages',
206	'title': 'cherokee - Montages',
207	},
208	'playlist_mincount': 4
209	}, {
210	'url': 'https://allstar.gg/profile?user=cherokee&view=Mobile Clips',
211	'info_dict': {
212	'id': '62b8bdfc9021052f7905882d-mobile',
213	'title': 'cherokee - Mobile Clips',
214	},
215	'playlist_mincount': 1
216	}]
217
218	_PAGE_SIZE = 10
219
220	def _get_page(self, user_id, display_id, game, query, page_num):
221	page_num += 1
222
223	for video_data in self._call_api(
224	query, {
225	'user': user_id,
226	'page': page_num,
227	'game': game,
228	}, ('data', 'videos', 'data'), display_id, f'Downloading page {page_num}'):
229	yield self._parse_video_data(video_data)
230
231	def _real_extract(self, url):
232	display_id = self._match_id(url)
233	profile_data = self._download_json(
234	urljoin('https://api.allstar.gg/v1/users/profile/', display_id), display_id)
235	user_id = traverse_obj(profile_data, ('data', ('_id'), {str}))
236	if not user_id:
237	raise ExtractorError('Unable to extract the user id')
238
239	username = traverse_obj(profile_data, ('data', 'profile', ('username'), {str}))
240	url_query = parse_qs(url)
241	game = traverse_obj(url_query, ('game', 0, {int_or_none}))
242	query_id = traverse_obj(url_query, ('view', 0), default='Clips')
243
244	if query_id not in ('Clips', 'Montages', 'Mobile Clips'):
245	raise ExtractorError(f'Unsupported playlist URL type {query_id!r}')
246
247	return self.playlist_result(
248	OnDemandPagedList(
249	functools.partial(
250	self._get_page, user_id, display_id, game, _QUERIES.get(query_id)), self._PAGE_SIZE),
251	playlist_id=join_nonempty(user_id, query_id.lower().split()[0], game),
252	playlist_title=join_nonempty((username or display_id), query_id, game, delim=' - '))