]>
Commit | Line | Data |
---|---|---|
1 | import itertools | |
2 | import json | |
3 | import math | |
4 | ||
5 | from .common import InfoExtractor | |
6 | from ..compat import compat_urllib_parse_unquote | |
7 | from ..utils import ( | |
8 | determine_ext, | |
9 | format_field, | |
10 | int_or_none, | |
11 | str_or_none, | |
12 | traverse_obj, | |
13 | try_get | |
14 | ) | |
15 | ||
16 | ||
17 | class GameJoltBaseIE(InfoExtractor): | |
18 | _API_BASE = 'https://gamejolt.com/site-api/' | |
19 | ||
20 | def _call_api(self, endpoint, *args, **kwargs): | |
21 | kwargs.setdefault('headers', {}).update({'Accept': 'image/webp,*/*'}) | |
22 | return self._download_json(self._API_BASE + endpoint, *args, **kwargs)['payload'] | |
23 | ||
24 | def _parse_content_as_text(self, content): | |
25 | outer_contents, joined_contents = content.get('content') or [], [] | |
26 | for outer_content in outer_contents: | |
27 | if outer_content.get('type') != 'paragraph': | |
28 | joined_contents.append(self._parse_content_as_text(outer_content)) | |
29 | continue | |
30 | inner_contents, inner_content_text = outer_content.get('content') or [], '' | |
31 | for inner_content in inner_contents: | |
32 | if inner_content.get('text'): | |
33 | inner_content_text += inner_content['text'] | |
34 | elif inner_content.get('type') == 'hardBreak': | |
35 | inner_content_text += '\n' | |
36 | joined_contents.append(inner_content_text) | |
37 | ||
38 | return '\n'.join(joined_contents) | |
39 | ||
40 | def _get_comments(self, post_num_id, post_hash_id): | |
41 | sort_by, scroll_id = self._configuration_arg('comment_sort', ['hot'], ie_key=GameJoltIE.ie_key())[0], -1 | |
42 | is_scrolled = sort_by in ('new', 'you') | |
43 | for page in itertools.count(1): | |
44 | comments_data = self._call_api( | |
45 | 'comments/Fireside_Post/%s/%s?%s=%d' % ( | |
46 | post_num_id, sort_by, | |
47 | 'scroll_id' if is_scrolled else 'page', scroll_id if is_scrolled else page), | |
48 | post_hash_id, note='Downloading comments list page %d' % page) | |
49 | if not comments_data.get('comments'): | |
50 | break | |
51 | for comment in traverse_obj(comments_data, (('comments', 'childComments'), ...), expected_type=dict): | |
52 | yield { | |
53 | 'id': comment['id'], | |
54 | 'text': self._parse_content_as_text( | |
55 | self._parse_json(comment['comment_content'], post_hash_id)), | |
56 | 'timestamp': int_or_none(comment.get('posted_on'), scale=1000), | |
57 | 'like_count': comment.get('votes'), | |
58 | 'author': traverse_obj(comment, ('user', ('display_name', 'name')), expected_type=str_or_none, get_all=False), | |
59 | 'author_id': traverse_obj(comment, ('user', 'username'), expected_type=str_or_none), | |
60 | 'author_thumbnail': traverse_obj(comment, ('user', 'image_avatar'), expected_type=str_or_none), | |
61 | 'parent': comment.get('parent_id') or None, | |
62 | } | |
63 | scroll_id = int_or_none(comments_data['comments'][-1].get('posted_on')) | |
64 | ||
65 | def _parse_post(self, post_data): | |
66 | post_id = post_data['hash'] | |
67 | lead_content = self._parse_json(post_data.get('lead_content') or '{}', post_id, fatal=False) or {} | |
68 | description, full_description = post_data.get('leadStr') or self._parse_content_as_text( | |
69 | self._parse_json(post_data.get('lead_content'), post_id)), None | |
70 | if post_data.get('has_article'): | |
71 | article_content = self._parse_json( | |
72 | post_data.get('article_content') | |
73 | or self._call_api(f'web/posts/article/{post_data.get("id", post_id)}', post_id, | |
74 | note='Downloading article metadata', errnote='Unable to download article metadata', fatal=False).get('article'), | |
75 | post_id, fatal=False) | |
76 | full_description = self._parse_content_as_text(article_content) | |
77 | ||
78 | user_data = post_data.get('user') or {} | |
79 | info_dict = { | |
80 | 'extractor_key': GameJoltIE.ie_key(), | |
81 | 'extractor': 'GameJolt', | |
82 | 'webpage_url': str_or_none(post_data.get('url')) or f'https://gamejolt.com/p/{post_id}', | |
83 | 'id': post_id, | |
84 | 'title': description, | |
85 | 'description': full_description or description, | |
86 | 'display_id': post_data.get('slug'), | |
87 | 'uploader': user_data.get('display_name') or user_data.get('name'), | |
88 | 'uploader_id': user_data.get('username'), | |
89 | 'uploader_url': format_field(user_data, 'url', 'https://gamejolt.com%s'), | |
90 | 'categories': [try_get(category, lambda x: '%s - %s' % (x['community']['name'], x['channel'].get('display_title') or x['channel']['title'])) | |
91 | for category in post_data.get('communities' or [])], | |
92 | 'tags': traverse_obj( | |
93 | lead_content, ('content', ..., 'content', ..., 'marks', ..., 'attrs', 'tag'), expected_type=str_or_none), | |
94 | 'like_count': int_or_none(post_data.get('like_count')), | |
95 | 'comment_count': int_or_none(post_data.get('comment_count'), default=0), | |
96 | 'timestamp': int_or_none(post_data.get('added_on'), scale=1000), | |
97 | 'release_timestamp': int_or_none(post_data.get('published_on'), scale=1000), | |
98 | '__post_extractor': self.extract_comments(post_data.get('id'), post_id) | |
99 | } | |
100 | ||
101 | # TODO: Handle multiple videos/embeds? | |
102 | video_data = traverse_obj(post_data, ('videos', ...), expected_type=dict, get_all=False) or {} | |
103 | formats, subtitles, thumbnails = [], {}, [] | |
104 | for media in video_data.get('media') or []: | |
105 | media_url, mimetype, ext, media_id = media['img_url'], media.get('filetype', ''), determine_ext(media['img_url']), media.get('type') | |
106 | if mimetype == 'application/vnd.apple.mpegurl' or ext == 'm3u8': | |
107 | hls_formats, hls_subs = self._extract_m3u8_formats_and_subtitles(media_url, post_id, 'mp4', m3u8_id=media_id) | |
108 | formats.extend(hls_formats) | |
109 | subtitles.update(hls_subs) | |
110 | elif mimetype == 'application/dash+xml' or ext == 'mpd': | |
111 | dash_formats, dash_subs = self._extract_mpd_formats_and_subtitles(media_url, post_id, mpd_id=media_id) | |
112 | formats.extend(dash_formats) | |
113 | subtitles.update(dash_subs) | |
114 | elif 'image' in mimetype: | |
115 | thumbnails.append({ | |
116 | 'id': media_id, | |
117 | 'url': media_url, | |
118 | 'width': media.get('width'), | |
119 | 'height': media.get('height'), | |
120 | 'filesize': media.get('filesize'), | |
121 | }) | |
122 | else: | |
123 | formats.append({ | |
124 | 'format_id': media_id, | |
125 | 'url': media_url, | |
126 | 'width': media.get('width'), | |
127 | 'height': media.get('height'), | |
128 | 'filesize': media.get('filesize'), | |
129 | 'acodec': 'none' if 'video-card' in media_url else None, | |
130 | }) | |
131 | ||
132 | if formats: | |
133 | return { | |
134 | **info_dict, | |
135 | 'formats': formats, | |
136 | 'subtitles': subtitles, | |
137 | 'thumbnails': thumbnails, | |
138 | 'view_count': int_or_none(video_data.get('view_count')), | |
139 | } | |
140 | ||
141 | gif_entries = [] | |
142 | for media in post_data.get('media', []): | |
143 | if determine_ext(media['img_url']) != 'gif' or 'gif' not in media.get('filetype', ''): | |
144 | continue | |
145 | gif_entries.append({ | |
146 | 'id': media['hash'], | |
147 | 'title': media['filename'].split('.')[0], | |
148 | 'formats': [{ | |
149 | 'format_id': url_key, | |
150 | 'url': media[url_key], | |
151 | 'width': media.get('width') if url_key == 'img_url' else None, | |
152 | 'height': media.get('height') if url_key == 'img_url' else None, | |
153 | 'filesize': media.get('filesize') if url_key == 'img_url' else None, | |
154 | 'acodec': 'none', | |
155 | } for url_key in ('img_url', 'mediaserver_url', 'mediaserver_url_mp4', 'mediaserver_url_webm') if media.get(url_key)] | |
156 | }) | |
157 | if gif_entries: | |
158 | return { | |
159 | '_type': 'playlist', | |
160 | **info_dict, | |
161 | 'entries': gif_entries, | |
162 | } | |
163 | ||
164 | embed_url = traverse_obj(post_data, ('embeds', ..., 'url'), expected_type=str_or_none, get_all=False) | |
165 | if embed_url: | |
166 | return self.url_result(embed_url) | |
167 | return info_dict | |
168 | ||
169 | ||
170 | class GameJoltIE(GameJoltBaseIE): | |
171 | _VALID_URL = r'https?://(?:www\.)?gamejolt\.com/p/(?:[\w-]*-)?(?P<id>\w{8})' | |
172 | _TESTS = [{ | |
173 | # No audio | |
174 | 'url': 'https://gamejolt.com/p/introducing-ramses-jackson-some-fnf-himbo-i-ve-been-animating-fo-c6achnzu', | |
175 | 'md5': 'cd5f733258f6678b0ce500dd88166d86', | |
176 | 'info_dict': { | |
177 | 'id': 'c6achnzu', | |
178 | 'ext': 'mp4', | |
179 | 'display_id': 'introducing-ramses-jackson-some-fnf-himbo-i-ve-been-animating-fo-c6achnzu', | |
180 | 'title': 'Introducing Ramses Jackson, some FNF himbo I’ve been animating for the past few days, hehe.\n#fnfmod #fridaynightfunkin', | |
181 | 'description': 'Introducing Ramses Jackson, some FNF himbo I’ve been animating for the past few days, hehe.\n#fnfmod #fridaynightfunkin', | |
182 | 'uploader': 'Jakeneutron', | |
183 | 'uploader_id': 'Jakeneutron', | |
184 | 'uploader_url': 'https://gamejolt.com/@Jakeneutron', | |
185 | 'categories': ['Friday Night Funkin\' - Videos'], | |
186 | 'tags': ['fnfmod', 'fridaynightfunkin'], | |
187 | 'timestamp': 1633499590, | |
188 | 'upload_date': '20211006', | |
189 | 'release_timestamp': 1633499655, | |
190 | 'release_date': '20211006', | |
191 | 'thumbnail': 're:^https?://.+wgch9mhq.png$', | |
192 | 'like_count': int, | |
193 | 'comment_count': int, | |
194 | 'view_count': int, | |
195 | } | |
196 | }, { | |
197 | # YouTube embed | |
198 | 'url': 'https://gamejolt.com/p/hey-hey-if-there-s-anyone-who-s-looking-to-get-into-learning-a-n6g4jzpq', | |
199 | 'md5': '79a931ff500a5c783ef6c3bda3272e32', | |
200 | 'info_dict': { | |
201 | 'id': 'XsNA_mzC0q4', | |
202 | 'title': 'Adobe Animate CC 2021 Tutorial || Part 1 - The Basics', | |
203 | 'description': 'md5:9d1ab9e2625b3fe1f42b2a44c67fdd13', | |
204 | 'uploader': 'Jakeneutron', | |
205 | 'uploader_id': 'Jakeneutron', | |
206 | 'uploader_url': 'http://www.youtube.com/user/Jakeneutron', | |
207 | 'ext': 'mp4', | |
208 | 'duration': 1749, | |
209 | 'tags': ['Adobe Animate CC', 'Tutorial', 'Animation', 'The Basics', 'For Beginners'], | |
210 | 'like_count': int, | |
211 | 'playable_in_embed': True, | |
212 | 'categories': ['Education'], | |
213 | 'availability': 'public', | |
214 | 'thumbnail': 'https://i.ytimg.com/vi_webp/XsNA_mzC0q4/maxresdefault.webp', | |
215 | 'age_limit': 0, | |
216 | 'live_status': 'not_live', | |
217 | 'channel_url': 'https://www.youtube.com/channel/UC6_L7fnczNalFZyBthUE9oA', | |
218 | 'channel': 'Jakeneutron', | |
219 | 'channel_id': 'UC6_L7fnczNalFZyBthUE9oA', | |
220 | 'upload_date': '20211015', | |
221 | 'view_count': int, | |
222 | 'chapters': 'count:18', | |
223 | } | |
224 | }, { | |
225 | # Article | |
226 | 'url': 'https://gamejolt.com/p/i-fuckin-broke-chaos-d56h3eue', | |
227 | 'md5': '786c1ccf98fde02c03a2768acb4258d0', | |
228 | 'info_dict': { | |
229 | 'id': 'd56h3eue', | |
230 | 'ext': 'mp4', | |
231 | 'display_id': 'i-fuckin-broke-chaos-d56h3eue', | |
232 | 'title': 'I fuckin broke Chaos.', | |
233 | 'description': 'I moved my tab durning the cutscene so now it\'s stuck like this.', | |
234 | 'uploader': 'Jeff____________', | |
235 | 'uploader_id': 'The_Nyesh_Man', | |
236 | 'uploader_url': 'https://gamejolt.com/@The_Nyesh_Man', | |
237 | 'categories': ['Friday Night Funkin\' - Videos'], | |
238 | 'timestamp': 1639800264, | |
239 | 'upload_date': '20211218', | |
240 | 'release_timestamp': 1639800330, | |
241 | 'release_date': '20211218', | |
242 | 'thumbnail': 're:^https?://.+euksy8bd.png$', | |
243 | 'like_count': int, | |
244 | 'comment_count': int, | |
245 | 'view_count': int, | |
246 | } | |
247 | }, { | |
248 | # Single GIF | |
249 | 'url': 'https://gamejolt.com/p/hello-everyone-i-m-developing-a-pixel-art-style-mod-for-fnf-and-i-vs4gdrd8', | |
250 | 'info_dict': { | |
251 | 'id': 'vs4gdrd8', | |
252 | 'display_id': 'hello-everyone-i-m-developing-a-pixel-art-style-mod-for-fnf-and-i-vs4gdrd8', | |
253 | 'title': 'md5:cc3d8b031d9bc7ec2ec5a9ffc707e1f9', | |
254 | 'description': 'md5:cc3d8b031d9bc7ec2ec5a9ffc707e1f9', | |
255 | 'uploader': 'Quesoguy', | |
256 | 'uploader_id': 'CheeseguyDev', | |
257 | 'uploader_url': 'https://gamejolt.com/@CheeseguyDev', | |
258 | 'categories': ['Game Dev - General', 'Arts n\' Crafts - Creations', 'Pixel Art - showcase', | |
259 | 'Friday Night Funkin\' - Mods', 'Newgrounds - Friday Night Funkin (13+)'], | |
260 | 'timestamp': 1639517122, | |
261 | 'release_timestamp': 1639519966, | |
262 | 'like_count': int, | |
263 | 'comment_count': int, | |
264 | }, | |
265 | 'playlist': [{ | |
266 | 'info_dict': { | |
267 | 'id': 'dszyjnwi', | |
268 | 'ext': 'webm', | |
269 | 'title': 'gif-presentacion-mejorado-dszyjnwi', | |
270 | } | |
271 | }], | |
272 | 'playlist_count': 1, | |
273 | }, { | |
274 | # Multiple GIFs | |
275 | 'url': 'https://gamejolt.com/p/gif-yhsqkumq', | |
276 | 'playlist_count': 35, | |
277 | 'info_dict': { | |
278 | 'id': 'yhsqkumq', | |
279 | 'display_id': 'gif-yhsqkumq', | |
280 | 'title': 'GIF', | |
281 | 'description': 'GIF', | |
282 | 'uploader': 'DaniilTvman', | |
283 | 'uploader_id': 'DaniilTvman', | |
284 | 'uploader_url': 'https://gamejolt.com/@DaniilTvman', | |
285 | 'categories': ['Five Nights At The AGK Studio Comunity - NEWS game'], | |
286 | 'timestamp': 1638721559, | |
287 | 'release_timestamp': 1638722276, | |
288 | 'like_count': int, | |
289 | 'comment_count': int, | |
290 | }, | |
291 | }] | |
292 | ||
293 | def _real_extract(self, url): | |
294 | post_id = self._match_id(url) | |
295 | post_data = self._call_api( | |
296 | f'web/posts/view/{post_id}', post_id)['post'] | |
297 | return self._parse_post(post_data) | |
298 | ||
299 | ||
300 | class GameJoltPostListBaseIE(GameJoltBaseIE): | |
301 | def _entries(self, endpoint, list_id, note='Downloading post list', errnote='Unable to download post list', initial_items=[]): | |
302 | page_num, scroll_id = 1, None | |
303 | items = initial_items or self._call_api(endpoint, list_id, note=note, errnote=errnote)['items'] | |
304 | while items: | |
305 | for item in items: | |
306 | yield self._parse_post(item['action_resource_model']) | |
307 | scroll_id = items[-1]['scroll_id'] | |
308 | page_num += 1 | |
309 | items = self._call_api( | |
310 | endpoint, list_id, note=f'{note} page {page_num}', errnote=errnote, data=json.dumps({ | |
311 | 'scrollDirection': 'from', | |
312 | 'scrollId': scroll_id, | |
313 | }).encode('utf-8')).get('items') | |
314 | ||
315 | ||
316 | class GameJoltUserIE(GameJoltPostListBaseIE): | |
317 | _VALID_URL = r'https?://(?:www\.)?gamejolt\.com/@(?P<id>[\w-]+)' | |
318 | _TESTS = [{ | |
319 | 'url': 'https://gamejolt.com/@BlazikenSuperStar', | |
320 | 'playlist_mincount': 1, | |
321 | 'info_dict': { | |
322 | 'id': '6116784', | |
323 | 'title': 'S. Blaze', | |
324 | 'description': 'md5:5ba7fbbb549e8ea2545aafbfe22eb03a', | |
325 | }, | |
326 | 'params': { | |
327 | 'ignore_no_formats_error': True, | |
328 | }, | |
329 | 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'], | |
330 | }] | |
331 | ||
332 | def _real_extract(self, url): | |
333 | user_id = self._match_id(url) | |
334 | user_data = self._call_api( | |
335 | f'web/profile/@{user_id}', user_id, note='Downloading user info', errnote='Unable to download user info')['user'] | |
336 | bio = self._parse_content_as_text( | |
337 | self._parse_json(user_data.get('bio_content', '{}'), user_id, fatal=False) or {}) | |
338 | return self.playlist_result( | |
339 | self._entries(f'web/posts/fetch/user/@{user_id}?tab=active', user_id, 'Downloading user posts', 'Unable to download user posts'), | |
340 | str_or_none(user_data.get('id')), user_data.get('display_name') or user_data.get('name'), bio) | |
341 | ||
342 | ||
343 | class GameJoltGameIE(GameJoltPostListBaseIE): | |
344 | _VALID_URL = r'https?://(?:www\.)?gamejolt\.com/games/[\w-]+/(?P<id>\d+)' | |
345 | _TESTS = [{ | |
346 | 'url': 'https://gamejolt.com/games/Friday4Fun/655124', | |
347 | 'playlist_mincount': 2, | |
348 | 'info_dict': { | |
349 | 'id': '655124', | |
350 | 'title': 'Friday Night Funkin\': Friday 4 Fun', | |
351 | 'description': 'md5:576a7dd87912a2dcf33c50d2bd3966d3' | |
352 | }, | |
353 | 'params': { | |
354 | 'ignore_no_formats_error': True, | |
355 | }, | |
356 | 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'], | |
357 | }] | |
358 | ||
359 | def _real_extract(self, url): | |
360 | game_id = self._match_id(url) | |
361 | game_data = self._call_api( | |
362 | f'web/discover/games/{game_id}', game_id, note='Downloading game info', errnote='Unable to download game info')['game'] | |
363 | description = self._parse_content_as_text( | |
364 | self._parse_json(game_data.get('description_content', '{}'), game_id, fatal=False) or {}) | |
365 | return self.playlist_result( | |
366 | self._entries(f'web/posts/fetch/game/{game_id}', game_id, 'Downloading game posts', 'Unable to download game posts'), | |
367 | game_id, game_data.get('title'), description) | |
368 | ||
369 | ||
370 | class GameJoltGameSoundtrackIE(GameJoltBaseIE): | |
371 | _VALID_URL = r'https?://(?:www\.)?gamejolt\.com/get/soundtrack(?:\?|\#!?)(?:.*?[&;])??game=(?P<id>(?:\d+)+)' | |
372 | _TESTS = [{ | |
373 | 'url': 'https://gamejolt.com/get/soundtrack?foo=bar&game=657899', | |
374 | 'info_dict': { | |
375 | 'id': '657899', | |
376 | 'title': 'Friday Night Funkin\': Vs Oswald', | |
377 | }, | |
378 | 'playlist': [{ | |
379 | 'info_dict': { | |
380 | 'id': '184434', | |
381 | 'ext': 'mp3', | |
382 | 'title': 'Gettin\' Lucky (Menu Music)', | |
383 | 'url': r're:^https://.+vs-oswald-menu-music\.mp3$', | |
384 | 'release_timestamp': 1635190816, | |
385 | 'release_date': '20211025', | |
386 | } | |
387 | }, { | |
388 | 'info_dict': { | |
389 | 'id': '184435', | |
390 | 'ext': 'mp3', | |
391 | 'title': 'Rabbit\'s Luck (Extended Version)', | |
392 | 'url': r're:^https://.+rabbit-s-luck--full-version-\.mp3$', | |
393 | 'release_timestamp': 1635190841, | |
394 | 'release_date': '20211025', | |
395 | } | |
396 | }, { | |
397 | 'info_dict': { | |
398 | 'id': '185228', | |
399 | 'ext': 'mp3', | |
400 | 'title': 'Last Straw', | |
401 | 'url': r're:^https://.+last-straw\.mp3$', | |
402 | 'release_timestamp': 1635881104, | |
403 | 'release_date': '20211102', | |
404 | } | |
405 | }], | |
406 | 'playlist_count': 3, | |
407 | }] | |
408 | ||
409 | def _real_extract(self, url): | |
410 | game_id = self._match_id(url) | |
411 | game_overview = self._call_api( | |
412 | f'web/discover/games/overview/{game_id}', game_id, note='Downloading soundtrack info', errnote='Unable to download soundtrack info') | |
413 | return self.playlist_result([{ | |
414 | 'id': str_or_none(song.get('id')), | |
415 | 'title': str_or_none(song.get('title')), | |
416 | 'url': str_or_none(song.get('url')), | |
417 | 'release_timestamp': int_or_none(song.get('posted_on'), scale=1000), | |
418 | } for song in game_overview.get('songs') or []], game_id, traverse_obj( | |
419 | game_overview, ('microdata', 'name'), (('twitter', 'fb'), 'title'), expected_type=str_or_none, get_all=False)) | |
420 | ||
421 | ||
422 | class GameJoltCommunityIE(GameJoltPostListBaseIE): | |
423 | _VALID_URL = r'https?://(?:www\.)?gamejolt\.com/c/(?P<id>(?P<community>[\w-]+)(?:/(?P<channel>[\w-]+))?)(?:(?:\?|\#!?)(?:.*?[&;])??sort=(?P<sort>\w+))?' | |
424 | _TESTS = [{ | |
425 | 'url': 'https://gamejolt.com/c/fnf/videos', | |
426 | 'playlist_mincount': 50, | |
427 | 'info_dict': { | |
428 | 'id': 'fnf/videos', | |
429 | 'title': 'Friday Night Funkin\' - Videos', | |
430 | 'description': 'md5:6d8c06f27460f7d35c1554757ffe53c8' | |
431 | }, | |
432 | 'params': { | |
433 | 'playlistend': 50, | |
434 | 'ignore_no_formats_error': True, | |
435 | }, | |
436 | 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'], | |
437 | }, { | |
438 | 'url': 'https://gamejolt.com/c/youtubers', | |
439 | 'playlist_mincount': 50, | |
440 | 'info_dict': { | |
441 | 'id': 'youtubers/featured', | |
442 | 'title': 'Youtubers - featured', | |
443 | 'description': 'md5:53e5582c93dcc467ab597bfca4db17d4' | |
444 | }, | |
445 | 'params': { | |
446 | 'playlistend': 50, | |
447 | 'ignore_no_formats_error': True, | |
448 | }, | |
449 | 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'], | |
450 | }] | |
451 | ||
452 | def _real_extract(self, url): | |
453 | display_id, community_id, channel_id, sort_by = self._match_valid_url(url).group('id', 'community', 'channel', 'sort') | |
454 | channel_id, sort_by = channel_id or 'featured', sort_by or 'new' | |
455 | ||
456 | community_data = self._call_api( | |
457 | f'web/communities/view/{community_id}', display_id, | |
458 | note='Downloading community info', errnote='Unable to download community info')['community'] | |
459 | channel_data = traverse_obj(self._call_api( | |
460 | f'web/communities/view-channel/{community_id}/{channel_id}', display_id, | |
461 | note='Downloading channel info', errnote='Unable to download channel info', fatal=False), 'channel') or {} | |
462 | ||
463 | title = f'{community_data.get("name") or community_id} - {channel_data.get("display_title") or channel_id}' | |
464 | description = self._parse_content_as_text( | |
465 | self._parse_json(community_data.get('description_content') or '{}', display_id, fatal=False) or {}) | |
466 | return self.playlist_result( | |
467 | self._entries( | |
468 | f'web/posts/fetch/community/{community_id}?channels[]={sort_by}&channels[]={channel_id}', | |
469 | display_id, 'Downloading community posts', 'Unable to download community posts'), | |
470 | f'{community_id}/{channel_id}', title, description) | |
471 | ||
472 | ||
473 | class GameJoltSearchIE(GameJoltPostListBaseIE): | |
474 | _VALID_URL = r'https?://(?:www\.)?gamejolt\.com/search(?:/(?P<filter>communities|users|games))?(?:\?|\#!?)(?:.*?[&;])??q=(?P<id>(?:[^&#]+)+)' | |
475 | _URL_FORMATS = { | |
476 | 'users': 'https://gamejolt.com/@{username}', | |
477 | 'communities': 'https://gamejolt.com/c/{path}', | |
478 | 'games': 'https://gamejolt.com/games/{slug}/{id}', | |
479 | } | |
480 | _TESTS = [{ | |
481 | 'url': 'https://gamejolt.com/search?foo=bar&q=%23fnf', | |
482 | 'playlist_mincount': 50, | |
483 | 'info_dict': { | |
484 | 'id': '#fnf', | |
485 | 'title': '#fnf', | |
486 | }, | |
487 | 'params': { | |
488 | 'playlistend': 50, | |
489 | 'ignore_no_formats_error': True, | |
490 | }, | |
491 | 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'], | |
492 | }, { | |
493 | 'url': 'https://gamejolt.com/search/communities?q=cookie%20run', | |
494 | 'playlist_mincount': 10, | |
495 | 'info_dict': { | |
496 | 'id': 'cookie run', | |
497 | 'title': 'cookie run', | |
498 | }, | |
499 | }, { | |
500 | 'url': 'https://gamejolt.com/search/users?q=mlp', | |
501 | 'playlist_mincount': 278, | |
502 | 'info_dict': { | |
503 | 'id': 'mlp', | |
504 | 'title': 'mlp', | |
505 | }, | |
506 | }, { | |
507 | 'url': 'https://gamejolt.com/search/games?q=roblox', | |
508 | 'playlist_mincount': 688, | |
509 | 'info_dict': { | |
510 | 'id': 'roblox', | |
511 | 'title': 'roblox', | |
512 | }, | |
513 | }] | |
514 | ||
515 | def _search_entries(self, query, filter_mode, display_query): | |
516 | initial_search_data = self._call_api( | |
517 | f'web/search/{filter_mode}?q={query}', display_query, | |
518 | note=f'Downloading {filter_mode} list', errnote=f'Unable to download {filter_mode} list') | |
519 | entries_num = traverse_obj(initial_search_data, 'count', f'{filter_mode}Count') | |
520 | if not entries_num: | |
521 | return | |
522 | for page in range(1, math.ceil(entries_num / initial_search_data['perPage']) + 1): | |
523 | search_results = self._call_api( | |
524 | f'web/search/{filter_mode}?q={query}&page={page}', display_query, | |
525 | note=f'Downloading {filter_mode} list page {page}', errnote=f'Unable to download {filter_mode} list') | |
526 | for result in search_results[filter_mode]: | |
527 | yield self.url_result(self._URL_FORMATS[filter_mode].format(**result)) | |
528 | ||
529 | def _real_extract(self, url): | |
530 | filter_mode, query = self._match_valid_url(url).group('filter', 'id') | |
531 | display_query = compat_urllib_parse_unquote(query) | |
532 | return self.playlist_result( | |
533 | self._search_entries(query, filter_mode, display_query) if filter_mode else self._entries( | |
534 | f'web/posts/fetch/search/{query}', display_query, initial_items=self._call_api( | |
535 | f'web/search?q={query}', display_query, | |
536 | note='Downloading initial post list', errnote='Unable to download initial post list')['posts']), | |
537 | display_query, display_query) |