6 from .common
import InfoExtractor
7 from ..compat
import compat_urllib_parse_unquote
18 class GameJoltBaseIE(InfoExtractor
):
19 _API_BASE
= 'https://gamejolt.com/site-api/'
21 def _call_api(self
, endpoint
, *args
, **kwargs
):
22 kwargs
.setdefault('headers', {}).update({'Accept': 'image/webp,*/*'}
)
23 return self
._download
_json
(self
._API
_BASE
+ endpoint
, *args
, **kwargs
)['payload']
25 def _parse_content_as_text(self
, content
):
26 outer_contents
, joined_contents
= content
.get('content') or [], []
27 for outer_content
in outer_contents
:
28 if outer_content
.get('type') != 'paragraph':
29 joined_contents
.append(self
._parse
_content
_as
_text
(outer_content
))
31 inner_contents
, inner_content_text
= outer_content
.get('content') or [], ''
32 for inner_content
in inner_contents
:
33 if inner_content
.get('text'):
34 inner_content_text
+= inner_content
['text']
35 elif inner_content
.get('type') == 'hardBreak':
36 inner_content_text
+= '\n'
37 joined_contents
.append(inner_content_text
)
39 return '\n'.join(joined_contents
)
41 def _get_comments(self
, post_num_id
, post_hash_id
):
42 sort_by
, scroll_id
= self
._configuration
_arg
('comment_sort', ['hot'], ie_key
=GameJoltIE
.ie_key())[0], -1
43 is_scrolled
= sort_by
in ('new', 'you')
44 for page
in itertools
.count(1):
45 comments_data
= self
._call
_api
(
46 'comments/Fireside_Post/%s/%s?%s=%d' % (
48 'scroll_id' if is_scrolled
else 'page', scroll_id
if is_scrolled
else page
),
49 post_hash_id
, note
='Downloading comments list page %d' % page
)
50 if not comments_data
.get('comments'):
52 for comment
in traverse_obj(comments_data
, (('comments', 'childComments'), ...), expected_type
=dict, default
=[]):
55 'text': self
._parse
_content
_as
_text
(
56 self
._parse
_json
(comment
['comment_content'], post_hash_id
)),
57 'timestamp': int_or_none(comment
.get('posted_on'), scale
=1000),
58 'like_count': comment
.get('votes'),
59 'author': traverse_obj(comment
, ('user', ('display_name', 'name')), expected_type
=str_or_none
, get_all
=False),
60 'author_id': traverse_obj(comment
, ('user', 'username'), expected_type
=str_or_none
),
61 'author_thumbnail': traverse_obj(comment
, ('user', 'image_avatar'), expected_type
=str_or_none
),
62 'parent': comment
.get('parent_id') or None,
64 scroll_id
= int_or_none(comments_data
['comments'][-1].get('posted_on'))
66 def _parse_post(self
, post_data
):
67 post_id
= post_data
['hash']
68 lead_content
= self
._parse
_json
(post_data
.get('lead_content') or '{}', post_id, fatal=False) or {}
69 description, full_description = post_data.get('leadStr
') or self._parse_content_as_text(
70 self._parse_json(post_data.get('lead_content
'), post_id)), None
71 if post_data.get('has_article
'):
72 article_content = self._parse_json(
73 post_data.get('article_content
')
74 or self._call_api(f'web
/posts
/article
/{post_data.get("id", post_id)}
', post_id,
75 note='Downloading article metadata
', errnote='Unable to download article metadata
', fatal=False).get('article
'),
77 full_description = self._parse_content_as_text(article_content)
79 user_data = post_data.get('user
') or {}
81 'extractor_key
': GameJoltIE.ie_key(),
82 'extractor
': 'GameJolt
',
83 'webpage_url
': str_or_none(post_data.get('url
')) or f'https
://gamejolt
.com
/p
/{post_id}
',
86 'description
': full_description or description,
87 'display_id
': post_data.get('slug
'),
88 'uploader
': user_data.get('display_name
') or user_data.get('name
'),
89 'uploader_id
': user_data.get('username
'),
90 'uploader_url
': format_field(user_data, 'url
', 'https
://gamejolt
.com
%s'),
91 'categories
': [try_get(category, lambda x: '%s - %s' % (x['community
']['name
'], x['channel
'].get('display_title
') or x['channel
']['title
']))
92 for category in post_data.get('communities
' or [])],
94 lead_content, ('content
', ..., 'content
', ..., 'marks
', ..., 'attrs
', 'tag
'), expected_type=str_or_none),
95 'like_count
': int_or_none(post_data.get('like_count
')),
96 'comment_count
': int_or_none(post_data.get('comment_count
'), default=0),
97 'timestamp
': int_or_none(post_data.get('added_on
'), scale=1000),
98 'release_timestamp
': int_or_none(post_data.get('published_on
'), scale=1000),
99 '__post_extractor
': self.extract_comments(post_data.get('id'), post_id)
102 # TODO: Handle multiple videos/embeds?
103 video_data = traverse_obj(post_data, ('videos
', ...), expected_type=dict, get_all=False) or {}
104 formats, subtitles, thumbnails = [], {}, []
105 for media in video_data.get('media
') or []:
106 media_url, mimetype, ext, media_id = media['img_url
'], media.get('filetype
', ''), determine_ext(media['img_url
']), media.get('type')
107 if mimetype == 'application
/vnd
.apple
.mpegurl
' or ext == 'm3u8
':
108 hls_formats, hls_subs = self._extract_m3u8_formats_and_subtitles(media_url, post_id, 'mp4
', m3u8_id=media_id)
109 formats.extend(hls_formats)
110 subtitles.update(hls_subs)
111 elif mimetype == 'application
/dash
+xml
' or ext == 'mpd
':
112 dash_formats, dash_subs = self._extract_mpd_formats_and_subtitles(media_url, post_id, mpd_id=media_id)
113 formats.extend(dash_formats)
114 subtitles.update(dash_subs)
115 elif 'image
' in mimetype:
119 'width
': media.get('width
'),
120 'height
': media.get('height
'),
121 'filesize
': media.get('filesize
'),
125 'format_id
': media_id,
127 'width
': media.get('width
'),
128 'height
': media.get('height
'),
129 'filesize
': media.get('filesize
'),
130 'acodec
': 'none
' if 'video
-card
' in media_url else None,
137 'subtitles
': subtitles,
138 'thumbnails
': thumbnails,
139 'view_count
': int_or_none(video_data.get('view_count
')),
143 for media in post_data.get('media
', []):
144 if determine_ext(media['img_url
']) != 'gif
' or 'gif
' not in media.get('filetype
', ''):
148 'title
': media['filename
'].split('.')[0],
150 'format_id
': url_key,
151 'url
': media[url_key],
152 'width
': media.get('width
') if url_key == 'img_url
' else None,
153 'height
': media.get('height
') if url_key == 'img_url
' else None,
154 'filesize
': media.get('filesize
') if url_key == 'img_url
' else None,
156 } for url_key in ('img_url
', 'mediaserver_url
', 'mediaserver_url_mp4
', 'mediaserver_url_webm
') if media.get(url_key)]
162 'entries
': gif_entries,
165 embed_url = traverse_obj(post_data, ('embeds
', ..., 'url
'), expected_type=str_or_none, get_all=False)
167 return self.url_result(embed_url)
171 class GameJoltIE(GameJoltBaseIE):
172 _VALID_URL = r'https?
://(?
:www\
.)?gamejolt\
.com
/p
/(?
:[\w
-]*-)?
(?P
<id>\w{8}
)'
175 'url
': 'https
://gamejolt
.com
/p
/introducing
-ramses
-jackson
-some
-fnf
-himbo
-i
-ve
-been
-animating
-fo
-c6achnzu
',
176 'md5
': 'cd5f733258f6678b0ce500dd88166d86
',
180 'display_id
': 'introducing
-ramses
-jackson
-some
-fnf
-himbo
-i
-ve
-been
-animating
-fo
-c6achnzu
',
181 'title
': 'Introducing Ramses Jackson
, some FNF himbo I’ve been animating
for the past few days
, hehe
.\n#fnfmod #fridaynightfunkin',
182 'description': 'Introducing Ramses Jackson, some FNF himbo I’ve been animating for the past few days, hehe.\n#fnfmod #fridaynightfunkin',
183 'uploader': 'Jakeneutron',
184 'uploader_id': 'Jakeneutron',
185 'uploader_url': 'https://gamejolt.com/@Jakeneutron',
186 'categories': ['Friday Night Funkin\' - Videos'],
187 'tags': ['fnfmod', 'fridaynightfunkin'],
188 'timestamp': 1633499590,
189 'upload_date': '20211006',
190 'release_timestamp': 1633499655,
191 'release_date': '20211006',
192 'thumbnail': 're:^https?://.+wgch9mhq.png$',
194 'comment_count': int,
199 'url': 'https://gamejolt.com/p/hey-hey-if-there-s-anyone-who-s-looking-to-get-into-learning-a-n6g4jzpq',
200 'md5': '79a931ff500a5c783ef6c3bda3272e32',
203 'title': 'Adobe Animate CC 2021 Tutorial || Part 1 - The Basics',
204 'description': 'md5:9d1ab9e2625b3fe1f42b2a44c67fdd13',
205 'uploader': 'Jakeneutron',
206 'uploader_id': 'Jakeneutron',
207 'uploader_url': 'http://www.youtube.com/user/Jakeneutron',
210 'tags': ['Adobe Animate CC', 'Tutorial', 'Animation', 'The Basics', 'For Beginners'],
212 'playable_in_embed': True,
213 'categories': ['Education'],
214 'availability': 'public',
215 'thumbnail': 'https://i.ytimg.com/vi_webp/XsNA_mzC0q4/maxresdefault.webp',
217 'live_status': 'not_live',
218 'channel_url': 'https://www.youtube.com/channel/UC6_L7fnczNalFZyBthUE9oA',
219 'channel': 'Jakeneutron',
220 'channel_id': 'UC6_L7fnczNalFZyBthUE9oA',
221 'upload_date': '20211015',
223 'chapters': 'count:18',
227 'url': 'https://gamejolt.com/p/i-fuckin-broke-chaos-d56h3eue',
228 'md5': '786c1ccf98fde02c03a2768acb4258d0',
232 'display_id': 'i-fuckin-broke-chaos-d56h3eue',
233 'title': 'I fuckin broke Chaos.',
234 'description': 'I moved my tab durning the cutscene so now it\'s stuck like this.',
235 'uploader': 'Jeff____________',
236 'uploader_id': 'The_Nyesh_Man',
237 'uploader_url': 'https://gamejolt.com/@The_Nyesh_Man',
238 'categories': ['Friday Night Funkin\' - Videos'],
239 'timestamp': 1639800264,
240 'upload_date': '20211218',
241 'release_timestamp': 1639800330,
242 'release_date': '20211218',
243 'thumbnail': 're:^https?://.+euksy8bd.png$',
245 'comment_count': int,
250 'url': 'https://gamejolt.com/p/hello-everyone-i-m-developing-a-pixel-art-style-mod-for-fnf-and-i-vs4gdrd8',
253 'display_id': 'hello-everyone-i-m-developing-a-pixel-art-style-mod-for-fnf-and-i-vs4gdrd8',
254 'title': 'md5:cc3d8b031d9bc7ec2ec5a9ffc707e1f9',
255 'description': 'md5:cc3d8b031d9bc7ec2ec5a9ffc707e1f9',
256 'uploader': 'Quesoguy',
257 'uploader_id': 'CheeseguyDev',
258 'uploader_url': 'https://gamejolt.com/@CheeseguyDev',
259 'categories': ['Game Dev - General', 'Arts n\' Crafts - Creations', 'Pixel Art - showcase',
260 'Friday Night Funkin\' - Mods', 'Newgrounds - Friday Night Funkin (13+)'],
261 'timestamp': 1639517122,
262 'release_timestamp': 1639519966,
264 'comment_count': int,
270 'title': 'gif-presentacion-mejorado-dszyjnwi',
276 'url': 'https://gamejolt.com/p/gif-yhsqkumq',
277 'playlist_count': 35,
280 'display_id': 'gif-yhsqkumq',
282 'description': 'GIF',
283 'uploader': 'DaniilTvman',
284 'uploader_id': 'DaniilTvman',
285 'uploader_url': 'https://gamejolt.com/@DaniilTvman',
286 'categories': ['Five Nights At The AGK Studio Comunity - NEWS game'],
287 'timestamp': 1638721559,
288 'release_timestamp': 1638722276,
290 'comment_count': int,
294 def _real_extract(self
, url
):
295 post_id
= self
._match
_id
(url
)
296 post_data
= self
._call
_api
(
297 f
'web/posts/view/{post_id}', post_id
)['post']
298 return self
._parse
_post
(post_data
)
301 class GameJoltPostListBaseIE(GameJoltBaseIE
):
302 def _entries(self
, endpoint
, list_id
, note
='Downloading post list', errnote
='Unable to download post list', initial_items
=[]):
303 page_num
, scroll_id
= 1, None
304 items
= initial_items
or self
._call
_api
(endpoint
, list_id
, note
=note
, errnote
=errnote
)['items']
307 yield self
._parse
_post
(item
['action_resource_model'])
308 scroll_id
= items
[-1]['scroll_id']
310 items
= self
._call
_api
(
311 endpoint
, list_id
, note
=f
'{note} page {page_num}', errnote
=errnote
, data
=json
.dumps({
312 'scrollDirection': 'from',
313 'scrollId': scroll_id
,
314 }).encode('utf-8')).get('items')
317 class GameJoltUserIE(GameJoltPostListBaseIE
):
318 _VALID_URL
= r
'https?://(?:www\.)?gamejolt\.com/@(?P<id>[\w-]+)'
320 'url': 'https://gamejolt.com/@BlazikenSuperStar',
321 'playlist_mincount': 1,
325 'description': 'md5:5ba7fbbb549e8ea2545aafbfe22eb03a',
328 'ignore_no_formats_error': True,
330 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
333 def _real_extract(self
, url
):
334 user_id
= self
._match
_id
(url
)
335 user_data
= self
._call
_api
(
336 f
'web/profile/@{user_id}', user_id
, note
='Downloading user info', errnote
='Unable to download user info')['user']
337 bio
= self
._parse
_content
_as
_text
(
338 self
._parse
_json
(user_data
.get('bio_content', '{}'), user_id, fatal=False) or {})
339 return self.playlist_result(
340 self._entries(f'web
/posts
/fetch
/user
/@{user_id}?tab
=active
', user_id, 'Downloading user posts
', 'Unable to download user posts
'),
341 str_or_none(user_data.get('id')), user_data.get('display_name
') or user_data.get('name
'), bio)
344 class GameJoltGameIE(GameJoltPostListBaseIE):
345 _VALID_URL = r'https?
://(?
:www\
.)?gamejolt\
.com
/games
/[\w
-]+/(?P
<id>\d
+)'
347 'url
': 'https
://gamejolt
.com
/games
/Friday4Fun
/655124',
348 'playlist_mincount
': 2,
351 'title
': 'Friday Night Funkin
\': Friday
4 Fun
',
352 'description
': 'md5
:576a7dd87912a2dcf33c50d2bd3966d3
'
355 'ignore_no_formats_error
': True,
357 'expected_warnings
': ['skipping format
', 'No video formats found
', 'Requested format
is not available
'],
360 def _real_extract(self, url):
361 game_id = self._match_id(url)
362 game_data = self._call_api(
363 f'web
/discover
/games
/{game_id}
', game_id, note='Downloading game info
', errnote='Unable to download game info
')['game
']
364 description = self._parse_content_as_text(
365 self._parse_json(game_data.get('description_content
', '{}'), game_id, fatal=False) or {}
)
366 return self
.playlist_result(
367 self
._entries
(f
'web/posts/fetch/game/{game_id}', game_id
, 'Downloading game posts', 'Unable to download game posts'),
368 game_id
, game_data
.get('title'), description
)
371 class GameJoltGameSoundtrackIE(GameJoltBaseIE
):
372 _VALID_URL
= r
'https?://(?:www\.)?gamejolt\.com/get/soundtrack(?:\?|\#!?)(?:.*?[&;])??game=(?P<id>(?:\d+)+)'
374 'url': 'https://gamejolt.com/get/soundtrack?foo=bar&game=657899',
377 'title': 'Friday Night Funkin\': Vs Oswald',
384 'title': 'Gettin\' Lucky (Menu Music)',
385 'url': r
're:^https://.+vs-oswald-menu-music\.mp3$',
386 'release_timestamp': 1635190816,
387 'release_date': '20211025',
394 'title': 'Rabbit\'s Luck (Extended Version)',
395 'url': r
're:^https://.+rabbit-s-luck--full-version-\.mp3$',
396 'release_timestamp': 1635190841,
397 'release_date': '20211025',
404 'title': 'Last Straw',
405 'url': r
're:^https://.+last-straw\.mp3$',
406 'release_timestamp': 1635881104,
407 'release_date': '20211102',
413 def _real_extract(self
, url
):
414 game_id
= self
._match
_id
(url
)
415 game_overview
= self
._call
_api
(
416 f
'web/discover/games/overview/{game_id}', game_id
, note
='Downloading soundtrack info', errnote
='Unable to download soundtrack info')
417 return self
.playlist_result([{
418 'id': str_or_none(song
.get('id')),
419 'title': str_or_none(song
.get('title')),
420 'url': str_or_none(song
.get('url')),
421 'release_timestamp': int_or_none(song
.get('posted_on'), scale
=1000),
422 } for song
in game_overview
.get('songs') or []], game_id
, traverse_obj(
423 game_overview
, ('microdata', 'name'), (('twitter', 'fb'), 'title'), expected_type
=str_or_none
, get_all
=False))
426 class GameJoltCommunityIE(GameJoltPostListBaseIE
):
427 _VALID_URL
= r
'https?://(?:www\.)?gamejolt\.com/c/(?P<id>(?P<community>[\w-]+)(?:/(?P<channel>[\w-]+))?)(?:(?:\?|\#!?)(?:.*?[&;])??sort=(?P<sort>\w+))?'
429 'url': 'https://gamejolt.com/c/fnf/videos',
430 'playlist_mincount': 50,
433 'title': 'Friday Night Funkin\' - Videos',
434 'description': 'md5:6d8c06f27460f7d35c1554757ffe53c8'
438 'ignore_no_formats_error': True,
440 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
442 'url': 'https://gamejolt.com/c/youtubers',
443 'playlist_mincount': 50,
445 'id': 'youtubers/featured',
446 'title': 'Youtubers - featured',
447 'description': 'md5:53e5582c93dcc467ab597bfca4db17d4'
451 'ignore_no_formats_error': True,
453 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
456 def _real_extract(self
, url
):
457 display_id
, community_id
, channel_id
, sort_by
= self
._match
_valid
_url
(url
).group('id', 'community', 'channel', 'sort')
458 channel_id
, sort_by
= channel_id
or 'featured', sort_by
or 'new'
460 community_data
= self
._call
_api
(
461 f
'web/communities/view/{community_id}', display_id
,
462 note
='Downloading community info', errnote
='Unable to download community info')['community']
463 channel_data
= traverse_obj(self
._call
_api
(
464 f
'web/communities/view-channel/{community_id}/{channel_id}', display_id
,
465 note
='Downloading channel info', errnote
='Unable to download channel info', fatal
=False), 'channel') or {}
467 title
= f
'{community_data.get("name") or community_id} - {channel_data.get("display_title") or channel_id}'
468 description
= self
._parse
_content
_as
_text
(
469 self
._parse
_json
(community_data
.get('description_content') or '{}', display_id, fatal=False) or {})
470 return self.playlist_result(
472 f'web
/posts
/fetch
/community
/{community_id}?channels
[]={sort_by}
&channels
[]={channel_id}
',
473 display_id, 'Downloading community posts
', 'Unable to download community posts
'),
474 f'{community_id}
/{channel_id}
', title, description)
477 class GameJoltSearchIE(GameJoltPostListBaseIE):
478 _VALID_URL = r'https?
://(?
:www\
.)?gamejolt\
.com
/search(?
:/(?P
<filter>communities|users|games
))?
(?
:\?|\
#!?)(?:.*?[&;])??q=(?P<id>(?:[^&#]+)+)'
480 'users': 'https://gamejolt.com/@{username}',
481 'communities': 'https://gamejolt.com/c/{path}',
482 'games': 'https://gamejolt.com/games/{slug}/{id}',
485 'url': 'https://gamejolt.com/search?foo=bar&q=%23fnf',
486 'playlist_mincount': 50,
493 'ignore_no_formats_error': True,
495 'expected_warnings': ['skipping format', 'No video formats found', 'Requested format is not available'],
497 'url': 'https://gamejolt.com/search/communities?q=cookie%20run',
498 'playlist_mincount': 10,
501 'title': 'cookie run',
504 'url': 'https://gamejolt.com/search/users?q=mlp',
505 'playlist_mincount': 278,
511 'url': 'https://gamejolt.com/search/games?q=roblox',
512 'playlist_mincount': 688,
519 def _search_entries(self
, query
, filter_mode
, display_query
):
520 initial_search_data
= self
._call
_api
(
521 f
'web/search/{filter_mode}?q={query}', display_query
,
522 note
=f
'Downloading {filter_mode} list', errnote
=f
'Unable to download {filter_mode} list')
523 entries_num
= traverse_obj(initial_search_data
, 'count', f
'{filter_mode}Count')
526 for page
in range(1, math
.ceil(entries_num
/ initial_search_data
['perPage']) + 1):
527 search_results
= self
._call
_api
(
528 f
'web/search/{filter_mode}?q={query}&page={page}', display_query
,
529 note
=f
'Downloading {filter_mode} list page {page}', errnote
=f
'Unable to download {filter_mode} list')
530 for result
in search_results
[filter_mode
]:
531 yield self
.url_result(self
._URL
_FORMATS
[filter_mode
].format(**result
))
533 def _real_extract(self
, url
):
534 filter_mode
, query
= self
._match
_valid
_url
(url
).group('filter', 'id')
535 display_query
= compat_urllib_parse_unquote(query
)
536 return self
.playlist_result(
537 self
._search
_entries
(query
, filter_mode
, display_query
) if filter_mode
else self
._entries
(
538 f
'web/posts/fetch/search/{query}', display_query
, initial_items
=self
._call
_api
(
539 f
'web/search?q={query}', display_query
,
540 note
='Downloading initial post list', errnote
='Unable to download initial post list')['posts']),
541 display_query
, display_query
)