5 from .common
import InfoExtractor
6 from ..compat
import compat_urllib_parse_unquote
17 class GameJoltBaseIE(InfoExtractor
):
18 _API_BASE
= 'https://gamejolt.com/site-api/'
20 def _call_api(self
, endpoint
, *args
, **kwargs
):
21 kwargs
.setdefault('headers', {}).update({'Accept': 'image/webp,*/*'}
)
22 return self
._download
_json
(self
._API
_BASE
+ endpoint
, *args
, **kwargs
)['payload']
24 def _parse_content_as_text(self
, content
):
25 outer_contents
, joined_contents
= content
.get('content') or [], []
26 for outer_content
in outer_contents
:
27 if outer_content
.get('type') != 'paragraph':
28 joined_contents
.append(self
._parse
_content
_as
_text
(outer_content
))
30 inner_contents
, inner_content_text
= outer_content
.get('content') or [], ''
31 for inner_content
in inner_contents
:
32 if inner_content
.get('text'):
33 inner_content_text
+= inner_content
['text']
34 elif inner_content
.get('type') == 'hardBreak':
35 inner_content_text
+= '\n'
36 joined_contents
.append(inner_content_text
)
38 return '\n'.join(joined_contents
)
40 def _get_comments(self
, post_num_id
, post_hash_id
):
41 sort_by
, scroll_id
= self
._configuration
_arg
('comment_sort', ['hot'], ie_key
=GameJoltIE
.ie_key())[0], -1
42 is_scrolled
= sort_by
in ('new', 'you')
43 for page
in itertools
.count(1):
44 comments_data
= self
._call
_api
(
45 'comments/Fireside_Post/%s/%s?%s=%d' % (
47 'scroll_id' if is_scrolled
else 'page', scroll_id
if is_scrolled
else page
),
48 post_hash_id
, note
='Downloading comments list page %d' % page
)
49 if not comments_data
.get('comments'):
51 for comment
in traverse_obj(comments_data
, (('comments', 'childComments'), ...), expected_type
=dict, default
=[]):
54 'text': self
._parse
_content
_as
_text
(
55 self
._parse
_json
(comment
['comment_content'], post_hash_id
)),
56 'timestamp': int_or_none(comment
.get('posted_on'), scale
=1000),
57 'like_count': comment
.get('votes'),
58 'author': traverse_obj(comment
, ('user', ('display_name', 'name')), expected_type
=str_or_none
, get_all
=False),
59 'author_id': traverse_obj(comment
, ('user', 'username'), expected_type
=str_or_none
),
60 'author_thumbnail': traverse_obj(comment
, ('user', 'image_avatar'), expected_type
=str_or_none
),
61 'parent': comment
.get('parent_id') or None,
63 scroll_id
= int_or_none(comments_data
['comments'][-1].get('posted_on'))
65 def _parse_post(self
, post_data
):
66 post_id
= post_data
['hash']
67 lead_content
= self
._parse
_json
(post_data
.get('lead_content') or '{}', post_id, fatal=False) or {}
68 description, full_description = post_data.get('leadStr
') or self._parse_content_as_text(
69 self._parse_json(post_data.get('lead_content
'), post_id)), None
70 if post_data.get('has_article
'):
71 article_content = self._parse_json(
72 post_data.get('article_content
')
73 or self._call_api(f'web
/posts
/article
/{post_data.get("id", post_id)}
', post_id,
74 note='Downloading article metadata
', errnote='Unable to download article metadata
', fatal=False).get('article
'),
76 full_description = self._parse_content_as_text(article_content)
78 user_data = post_data.get('user
') or {}
80 'extractor_key
': GameJoltIE.ie_key(),
81 'extractor
': 'GameJolt
',
82 'webpage_url
': str_or_none(post_data.get('url
')) or f'https
://gamejolt
.com
/p
/{post_id}
',
85 'description
': full_description or description,
86 'display_id
': post_data.get('slug
'),
87 'uploader
': user_data.get('display_name
') or user_data.get('name
'),
88 'uploader_id
': user_data.get('username
'),
89 'uploader_url
': format_field(user_data, 'url
', 'https
://gamejolt
.com
%s'),
90 'categories
': [try_get(category, lambda x: '%s - %s' % (x['community
']['name
'], x['channel
'].get('display_title
') or x['channel
']['title
']))
91 for category in post_data.get('communities
' or [])],
93 lead_content, ('content
', ..., 'content
', ..., 'marks
', ..., 'attrs
', 'tag
'), expected_type=str_or_none),
94 'like_count
': int_or_none(post_data.get('like_count
')),
95 'comment_count
': int_or_none(post_data.get('comment_count
'), default=0),
96 'timestamp
': int_or_none(post_data.get('added_on
'), scale=1000),
97 'release_timestamp
': int_or_none(post_data.get('published_on
'), scale=1000),
98 '__post_extractor
': self.extract_comments(post_data.get('id'), post_id)
101 # TODO: Handle multiple videos/embeds?
102 video_data = traverse_obj(post_data, ('videos
', ...), expected_type=dict, get_all=False) or {}
103 formats, subtitles, thumbnails = [], {}, []
104 for media in video_data.get('media
') or []:
105 media_url, mimetype, ext, media_id = media['img_url
'], media.get('filetype
', ''), determine_ext(media['img_url
']), media.get('type')
106 if mimetype == 'application
/vnd
.apple
.mpegurl
' or ext == 'm3u8
':
107 hls_formats, hls_subs = self._extract_m3u8_formats_and_subtitles(media_url, post_id, 'mp4
', m3u8_id=media_id)
108 formats.extend(hls_formats)
109 subtitles.update(hls_subs)
110 elif mimetype == 'application
/dash
+xml
' or ext == 'mpd
':
111 dash_formats, dash_subs = self._extract_mpd_formats_and_subtitles(media_url, post_id, mpd_id=media_id)
112 formats.extend(dash_formats)
113 subtitles.update(dash_subs)
114 elif 'image
' in mimetype:
118 'width
': media.get('width
'),
119 'height
': media.get('height
'),
120 'filesize
': media.get('filesize
'),
124 'format_id
': media_id,
126 'width
': media.get('width
'),
127 'height
': media.get('height
'),
128 'filesize
': media.get('filesize
'),
129 'acodec
': 'none
' if 'video
-card
' in media_url else None,
136 'subtitles
': subtitles,
137 'thumbnails
': thumbnails,
138 'view_count
': int_or_none(video_data.get('view_count
')),
142 for media in post_data.get('media
', []):
143 if determine_ext(media['img_url
']) != 'gif
' or 'gif
' not in media.get('filetype
', ''):
147 'title
': media['filename
'].split('.')[0],
149 'format_id
': url_key,
150 'url
': media[url_key],
151 'width
': media.get('width
') if url_key == 'img_url
' else None,
152 'height
': media.get('height
') if url_key == 'img_url
' else None,
153 'filesize
': media.get('filesize
') if url_key == 'img_url
' else None,
155 } for url_key in ('img_url
', 'mediaserver_url
', 'mediaserver_url_mp4
', 'mediaserver_url_webm
') if media.get(url_key)]
161 'entries
': gif_entries,
164 embed_url = traverse_obj(post_data, ('embeds
', ..., 'url
'), expected_type=str_or_none, get_all=False)
166 return self.url_result(embed_url)
170 class GameJoltIE(GameJoltBaseIE):
171 _VALID_URL = r'https?
://(?
:www\
.)?gamejolt\
.com
/p
/(?
:[\w
-]*-)?
(?P
<id>\w{8}
)'
174 'url
': 'https
://gamejolt
.com
/p
/introducing
-ramses
-jackson
-some
-fnf
-himbo
-i
-ve
-been
-animating
-fo
-c6achnzu
',
175 'md5
': 'cd5f733258f6678b0ce500dd88166d86
',
179 'display_id
': 'introducing
-ramses
-jackson
-some
-fnf
-himbo
-i
-ve
-been
-animating
-fo
-c6achnzu
',
180 'title
': 'Introducing Ramses Jackson
, some FNF himbo I’ve been animating
for the past few days
, hehe
.\n#fnfmod #fridaynightfunkin',
181 'description': 'Introducing Ramses Jackson, some FNF himbo I’ve been animating for the past few days, hehe.\n#fnfmod #fridaynightfunkin',
182 'uploader': 'Jakeneutron',
183 'uploader_id': 'Jakeneutron',
184 'uploader_url': 'https://gamejolt.com/@Jakeneutron',
185 'categories': ['Friday Night Funkin\' - Videos'],
186 'tags': ['fnfmod', 'fridaynightfunkin'],
187 'timestamp': 1633499590,
188 'upload_date': '20211006',
189 'release_timestamp': 1633499655,
190 'release_date': '20211006',
191 'thumbnail': 're:^https?://.+wgch9mhq.png$',
193 'comment_count': int,
198 'url': 'https://gamejolt.com/p/hey-hey-if-there-s-anyone-who-s-looking-to-get-into-learning-a-n6g4jzpq',
199 'md5': '79a931ff500a5c783ef6c3bda3272e32',
202 'title': 'Adobe Animate CC 2021 Tutorial || Part 1 - The Basics',
203 'description': 'md5:9d1ab9e2625b3fe1f42b2a44c67fdd13',
204 'uploader': 'Jakeneutron',
205 'uploader_id': 'Jakeneutron',
206 'uploader_url': 'http://www.youtube.com/user/Jakeneutron',
209 'tags': ['Adobe Animate CC', 'Tutorial', 'Animation', 'The Basics', 'For Beginners'],
211 'playable_in_embed': True,
212 'categories': ['Education'],
213 'availability': 'public',
214 'thumbnail': 'https://i.ytimg.com/vi_webp/XsNA_mzC0q4/maxresdefault.webp',
216 'live_status': 'not_live',
217 'channel_url': 'https://www.youtube.com/channel/UC6_L7fnczNalFZyBthUE9oA',
218 'channel': 'Jakeneutron',
219 'channel_id': 'UC6_L7fnczNalFZyBthUE9oA',
220 'upload_date': '20211015',
222 'chapters': 'count:18',
226 'url': 'https://gamejolt.com/p/i-fuckin-broke-chaos-d56h3eue',
227 'md5': '786c1ccf98fde02c03a2768acb4258d0',
231 'display_id': 'i-fuckin-broke-chaos-d56h3eue',
232 'title': 'I fuckin broke Chaos.',
233 'description': 'I moved my tab durning the cutscene so now it\'s stuck like this.',
234 'uploader': 'Jeff____________',
235 'uploader_id': 'The_Nyesh_Man',
236 'uploader_url': 'https://gamejolt.com/@The_Nyesh_Man',
237 'categories': ['Friday Night Funkin\' - Videos'],
238 'timestamp': 1639800264,
239 'upload_date': '20211218',
240 'release_timestamp': 1639800330,
241 'release_date': '20211218',
242 'thumbnail': 're:^https?://.+euksy8bd.png$',
244 'comment_count': int,
249 'url': 'https://gamejolt.com/p/hello-everyone-i-m-developing-a-pixel-art-style-mod-for-fnf-and-i-vs4gdrd8',
252 'display_id': 'hello-everyone-i-m-developing-a-pixel-art-style-mod-for-fnf-and-i-vs4gdrd8',
253 'title': 'md5:cc3d8b031d9bc7ec2ec5a9ffc707e1f9',
254 'description': 'md5:cc3d8b031d9bc7ec2ec5a9ffc707e1f9',
255 'uploader': 'Quesoguy',
256 'uploader_id': 'CheeseguyDev',
257 'uploader_url': 'https://gamejolt.com/@CheeseguyDev',
258 'categories': ['Game Dev - General', 'Arts n\' Crafts - Creations', 'Pixel Art - showcase',
259 'Friday Night Funkin\' - Mods', 'Newgrounds - Friday Night Funkin (13+)'],
260 'timestamp': 1639517122,
261 'release_timestamp': 1639519966,
263 'comment_count': int,
269 'title': 'gif-presentacion-mejorado-dszyjnwi',
275 'url': 'https://gamejolt.com/p/gif-yhsqkumq',
276 'playlist_count': 35,
279 'display_id': 'gif-yhsqkumq',
281 'description': 'GIF',
282 'uploader': 'DaniilTvman',
283 'uploader_id': 'DaniilTvman',
284 'uploader_url': 'https://gamejolt.com/@DaniilTvman',
285 'categories': ['Five Nights At The AGK Studio Comunity - NEWS game'],
286 'timestamp': 1638721559,
287 'release_timestamp': 1638722276,
289 'comment_count': int,
293 def _real_extract(self
, url
):
294 post_id
= self
._match
_id
(url
)
295 post_data
= self
._call
_api
(
296 f
'web/posts/view/{post_id}', post_id
)['post']
297 return self
._parse
_post
(post_data
)
300 class GameJoltPostListBaseIE(GameJoltBaseIE
):
301 def _entries(self
, endpoint
, list_id
, note
='Downloading post list', errnote
='Unable to download post list', initial_items
=[]):
302 page_num
, scroll_id
= 1, None
303 items
= initial_items
or self
._call
_api
(endpoint
, list_id
, note
=note
, errnote
=errnote
)['items']
306 yield self
._parse
_post
(item
['action_resource_model'])
307 scroll_id
= items
[-1]['scroll_id']
309 items
= self
._call
_api
(
310 endpoint
, list_id
, note
=f
'{note} page {page_num}', errnote
=errnote
, data
=json
.dumps({
311 'scrollDirection': 'from',
312 'scrollId': scroll_id
,
313 }).encode('utf-8')).get('items')
316 class GameJoltUserIE(GameJoltPostListBaseIE
):
317 _VALID_URL
= r
'https?://(?:www\.)?gamejolt\.com/@(?P<id>[\w-]+)'
319 'url': 'https://gamejolt.com/@BlazikenSuperStar',
320 'playlist_mincount': 1,
324 'description': 'md5:5ba7fbbb549e8ea2545aafbfe22eb03a',
327 'ignore_no_formats_error': True,
329 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
332 def _real_extract(self
, url
):
333 user_id
= self
._match
_id
(url
)
334 user_data
= self
._call
_api
(
335 f
'web/profile/@{user_id}', user_id
, note
='Downloading user info', errnote
='Unable to download user info')['user']
336 bio
= self
._parse
_content
_as
_text
(
337 self
._parse
_json
(user_data
.get('bio_content', '{}'), user_id, fatal=False) or {})
338 return self.playlist_result(
339 self._entries(f'web
/posts
/fetch
/user
/@{user_id}?tab
=active
', user_id, 'Downloading user posts
', 'Unable to download user posts
'),
340 str_or_none(user_data.get('id')), user_data.get('display_name
') or user_data.get('name
'), bio)
343 class GameJoltGameIE(GameJoltPostListBaseIE):
344 _VALID_URL = r'https?
://(?
:www\
.)?gamejolt\
.com
/games
/[\w
-]+/(?P
<id>\d
+)'
346 'url
': 'https
://gamejolt
.com
/games
/Friday4Fun
/655124',
347 'playlist_mincount
': 2,
350 'title
': 'Friday Night Funkin
\': Friday
4 Fun
',
351 'description
': 'md5
:576a7dd87912a2dcf33c50d2bd3966d3
'
354 'ignore_no_formats_error
': True,
356 'expected_warnings
': ['skipping format
', 'No video formats found
', 'Requested format
is not available
'],
359 def _real_extract(self, url):
360 game_id = self._match_id(url)
361 game_data = self._call_api(
362 f'web
/discover
/games
/{game_id}
', game_id, note='Downloading game info
', errnote='Unable to download game info
')['game
']
363 description = self._parse_content_as_text(
364 self._parse_json(game_data.get('description_content
', '{}'), game_id, fatal=False) or {}
)
365 return self
.playlist_result(
366 self
._entries
(f
'web/posts/fetch/game/{game_id}', game_id
, 'Downloading game posts', 'Unable to download game posts'),
367 game_id
, game_data
.get('title'), description
)
370 class GameJoltGameSoundtrackIE(GameJoltBaseIE
):
371 _VALID_URL
= r
'https?://(?:www\.)?gamejolt\.com/get/soundtrack(?:\?|\#!?)(?:.*?[&;])??game=(?P<id>(?:\d+)+)'
373 'url': 'https://gamejolt.com/get/soundtrack?foo=bar&game=657899',
376 'title': 'Friday Night Funkin\': Vs Oswald',
383 'title': 'Gettin\' Lucky (Menu Music)',
384 'url': r
're:^https://.+vs-oswald-menu-music\.mp3$',
385 'release_timestamp': 1635190816,
386 'release_date': '20211025',
393 'title': 'Rabbit\'s Luck (Extended Version)',
394 'url': r
're:^https://.+rabbit-s-luck--full-version-\.mp3$',
395 'release_timestamp': 1635190841,
396 'release_date': '20211025',
403 'title': 'Last Straw',
404 'url': r
're:^https://.+last-straw\.mp3$',
405 'release_timestamp': 1635881104,
406 'release_date': '20211102',
412 def _real_extract(self
, url
):
413 game_id
= self
._match
_id
(url
)
414 game_overview
= self
._call
_api
(
415 f
'web/discover/games/overview/{game_id}', game_id
, note
='Downloading soundtrack info', errnote
='Unable to download soundtrack info')
416 return self
.playlist_result([{
417 'id': str_or_none(song
.get('id')),
418 'title': str_or_none(song
.get('title')),
419 'url': str_or_none(song
.get('url')),
420 'release_timestamp': int_or_none(song
.get('posted_on'), scale
=1000),
421 } for song
in game_overview
.get('songs') or []], game_id
, traverse_obj(
422 game_overview
, ('microdata', 'name'), (('twitter', 'fb'), 'title'), expected_type
=str_or_none
, get_all
=False))
425 class GameJoltCommunityIE(GameJoltPostListBaseIE
):
426 _VALID_URL
= r
'https?://(?:www\.)?gamejolt\.com/c/(?P<id>(?P<community>[\w-]+)(?:/(?P<channel>[\w-]+))?)(?:(?:\?|\#!?)(?:.*?[&;])??sort=(?P<sort>\w+))?'
428 'url': 'https://gamejolt.com/c/fnf/videos',
429 'playlist_mincount': 50,
432 'title': 'Friday Night Funkin\' - Videos',
433 'description': 'md5:6d8c06f27460f7d35c1554757ffe53c8'
437 'ignore_no_formats_error': True,
439 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
441 'url': 'https://gamejolt.com/c/youtubers',
442 'playlist_mincount': 50,
444 'id': 'youtubers/featured',
445 'title': 'Youtubers - featured',
446 'description': 'md5:53e5582c93dcc467ab597bfca4db17d4'
450 'ignore_no_formats_error': True,
452 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
455 def _real_extract(self
, url
):
456 display_id
, community_id
, channel_id
, sort_by
= self
._match
_valid
_url
(url
).group('id', 'community', 'channel', 'sort')
457 channel_id
, sort_by
= channel_id
or 'featured', sort_by
or 'new'
459 community_data
= self
._call
_api
(
460 f
'web/communities/view/{community_id}', display_id
,
461 note
='Downloading community info', errnote
='Unable to download community info')['community']
462 channel_data
= traverse_obj(self
._call
_api
(
463 f
'web/communities/view-channel/{community_id}/{channel_id}', display_id
,
464 note
='Downloading channel info', errnote
='Unable to download channel info', fatal
=False), 'channel') or {}
466 title
= f
'{community_data.get("name") or community_id} - {channel_data.get("display_title") or channel_id}'
467 description
= self
._parse
_content
_as
_text
(
468 self
._parse
_json
(community_data
.get('description_content') or '{}', display_id, fatal=False) or {})
469 return self.playlist_result(
471 f'web
/posts
/fetch
/community
/{community_id}?channels
[]={sort_by}
&channels
[]={channel_id}
',
472 display_id, 'Downloading community posts
', 'Unable to download community posts
'),
473 f'{community_id}
/{channel_id}
', title, description)
476 class GameJoltSearchIE(GameJoltPostListBaseIE):
477 _VALID_URL = r'https?
://(?
:www\
.)?gamejolt\
.com
/search(?
:/(?P
<filter>communities|users|games
))?
(?
:\?|\
#!?)(?:.*?[&;])??q=(?P<id>(?:[^&#]+)+)'
479 'users': 'https://gamejolt.com/@{username}',
480 'communities': 'https://gamejolt.com/c/{path}',
481 'games': 'https://gamejolt.com/games/{slug}/{id}',
484 'url': 'https://gamejolt.com/search?foo=bar&q=%23fnf',
485 'playlist_mincount': 50,
492 'ignore_no_formats_error': True,
494 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
496 'url': 'https://gamejolt.com/search/communities?q=cookie%20run',
497 'playlist_mincount': 10,
500 'title': 'cookie run',
503 'url': 'https://gamejolt.com/search/users?q=mlp',
504 'playlist_mincount': 278,
510 'url': 'https://gamejolt.com/search/games?q=roblox',
511 'playlist_mincount': 688,
518 def _search_entries(self
, query
, filter_mode
, display_query
):
519 initial_search_data
= self
._call
_api
(
520 f
'web/search/{filter_mode}?q={query}', display_query
,
521 note
=f
'Downloading {filter_mode} list', errnote
=f
'Unable to download {filter_mode} list')
522 entries_num
= traverse_obj(initial_search_data
, 'count', f
'{filter_mode}Count')
525 for page
in range(1, math
.ceil(entries_num
/ initial_search_data
['perPage']) + 1):
526 search_results
= self
._call
_api
(
527 f
'web/search/{filter_mode}?q={query}&page={page}', display_query
,
528 note
=f
'Downloading {filter_mode} list page {page}', errnote
=f
'Unable to download {filter_mode} list')
529 for result
in search_results
[filter_mode
]:
530 yield self
.url_result(self
._URL
_FORMATS
[filter_mode
].format(**result
))
532 def _real_extract(self
, url
):
533 filter_mode
, query
= self
._match
_valid
_url
(url
).group('filter', 'id')
534 display_query
= compat_urllib_parse_unquote(query
)
535 return self
.playlist_result(
536 self
._search
_entries
(query
, filter_mode
, display_query
) if filter_mode
else self
._entries
(
537 f
'web/posts/fetch/search/{query}', display_query
, initial_items
=self
._call
_api
(
538 f
'web/search?q={query}', display_query
,
539 note
='Downloading initial post list', errnote
='Unable to download initial post list')['posts']),
540 display_query
, display_query
)