[erebus.git] / modules / reddark.py

# Reddark streaming module
# vim: fileencoding=utf-8

# module info
modinfo = {
	'author': 'Multiple',
	'license': 'unknown',
	'compatible': [0], # compatible module API versions
	'depends': [], # other modules required to work properly?
	'softdeps': ['help'], # modules which are preferred but not required
}
# note: softdeps will be loaded before this module, IF not disabled in the configuration (autoload.module = 0) (and if it exists)
# however, if it is disabled it will be silently ignored, and if it is unloaded at runtime it won't cause this one to unload.
#
# basically, softdeps are things this module will use if available, but does not require (no errors will occur if it's not loaded)
# for example, @lib.help() will attempt to use the help module, but swallow errors if it is not loaded


# global variables
wants_to_stop = False
runner = None
last_update = 0
last_topic = ""

# preamble
import modlib
lib = modlib.modlib(__name__)
def modstart(parent, *args, **kwargs):
	gotParent(parent)
	return lib.modstart(parent, *args, **kwargs)
def modstop(*args, **kwargs):
	global wants_to_stop, runner
	if runner:
		debug("Stopping runner")
		wants_to_stop = True # tell the SSE thread to stop the next time it wakes up
		runner.join() # blocks the main thread til the next chunk received (or timeout)! oh well.
		debug("runner stopped")
		wants_to_stop = False
	return lib.modstop(*args, **kwargs)

# module code
from modules.contrib.sseclient import SSEClient
import requests
from bs4 import BeautifulSoup
import threading, json, time, collections, re

def chan():
	return lib.parent.channel(lib.parent.cfg.get('reddark', 'channel', default="##.test"))

def bot():
	return chan().bot

def chanmsg(message):
	return chan().msg(message, truncate=True, msgtype="NOTICE")

def debug(message, send_to_owner=True):
	if lib.parent is None:
		print(message)
		return
	if lib.parent.cfg.getboolean('reddark', 'debug', True):
		lib.parent.log('Reddark', 'R', message)
	if send_to_owner and lib.parent.cfg.get('debug', 'owner'):
		bot().fastmsg(lib.parent.cfg.get('debug', 'owner'), message)

def getText(subreddit):
	r = None
	e = None
	try:
		r = requests.get('https://old.reddit.com/' + subreddit, headers={'User-Agent': 'better-see-reason-bot/1.0'})
	except Exception as e2:
		e = e2
	if r is None:
		debug("Error getting text: " + repr(e), False)
		return ''
	if r.status_code != 403:
		debug("Error getting text: " + str(r.status_code) + " (403 expected)", False)
		return ''
	soup = BeautifulSoup(r.text, 'html.parser')
	elements = soup.find_all(class_='interstitial-subreddit-description')
	if len(elements):
		text = elements[0].get_text()
		text = re.sub(re.escape(subreddit), '', text, re.IGNORECASE).replace("\n", " ")
		return ' - ' + text
	return ''

def handleDelta(message):
	message['state'] = message['state'].lower()
	message['previous_state'] = message['previous_state'].lower()
	if message['state'] == 'private':
		message['text'] = getText(message['name'])
		print(repr(message))
		chanmsg('[%(section)s] %(name)s went %(state)s (was: %(previous_state)s) (https://old.reddit.com/%(name)s)%(text)s' % message)
	elif message['state'] == 'public':
		chanmsg('[%(section)s] %(name)s went \x02%(state)s\x02 (was: %(previous_state)s) (https://old.reddit.com/%(name)s)' % message)
	else:
		chanmsg('[%(section)s] %(name)s went %(state)s (was: %(previous_state)s) (https://old.reddit.com/%(name)s)' % message)

def handleState(message):
	global last_update, last_topic
	if time.time() < last_update + lib.parent.cfg.getint('reddark', 'update_interval', 600):
		return
	output = collections.defaultdict(int)
	output['totalSubs'] = len(message['subreddits'])
	for sub in message['subreddits']:
		if sub['state'] == 'PRIVATE':
			output['privateSubs'] += 1
			output['protestingSubs'] += 1
		elif sub['state'] == 'PUBLIC':
			output['publicSubs'] += 1
		else:
			output['restrictedSubs'] += 1
			output['protestingSubs'] += 1
	output['pct'] = round(output['protestingSubs']/output['totalSubs']*100, 2)
	output['pctPublic'] = round(output['publicSubs']/output['totalSubs']*100, 2)
	output['pctPrivate'] = round(output['privateSubs']/output['totalSubs']*100, 2)
	output['pctRestricted'] = round(output['restrictedSubs']/output['totalSubs']*100, 2)
	newTopic = 'subreddits protesting: %(protestingSubs)s out of %(totalSubs)s pledged (%(pct)s%%) | private: %(privateSubs)s (%(pctPrivate)s%%), restricted: %(restrictedSubs)s (%(pctRestricted)s%%), public: %(publicSubs)s (%(pctPublic)s%%)' % dict(output)
	debug(newTopic, False)
	if last_topic != newTopic:
		last_topic = newTopic
		last_update = time.time()
		bot().conn.send("TOPIC %(chan)s :%(topic)s %(suffix)s" % {'chan': chan(), 'topic': newTopic, 'suffix': lib.parent.cfg.get('reddark', 'topicsuffix', '| Reddark That Works™ https://reddark.rewby.archivete.am/ (https://github.com/reddark-remix/reddark-remix) | https://www.youtube.com/watch?v=xMaE6toi4mk')})

def gotParent(parent):
	global runner
	def loop_messages():
		global wants_to_stop
		messages = SSEClient(parent.cfg.get('reddark', 'sse', default="https://reddark.rewby.archivete.am/sse"), timeout=60)
		debug("Connected to SSE", False)
		for msg in messages:
			try:
				if wants_to_stop:
					return
				if len(msg.data) == 0:
					continue
				data = json.loads(msg.data)
				debug(repr(data)[0:500], False)
				if data['type'] == 'Delta':
					handleDelta(data['content'])
				elif data['type'] == 'CurrentStateUpdate':
					handleState(data['content'])
				else:
					debug("Unknown event: " + msg, False)
			except Exception as e:
				debug("Failed to parse an event: " + repr(e))
#{"type":"Delta","content":{"name":"r/TrainCrashSeries","section":"1k+","previous_state":"PRIVATE","state":"RESTRICTED"}}
#{"type":"CurrentStateUpdate","content":{"sections":["40+ million","30+ million","20+ million","10+ million","5+ million","1+ million","500k+","250k+","100k+","50k+","5k+","1k+","1k and below"],"subreddits":[{"name":"r/032r4r","section":"1k+","state":"PRIVATE"},{"name":"r/0sanitymemes","section":"5k+","state":"PRIVATE"},{"name":"r/1022","section":"5k+","state":"PRIVATE"},{"name":"r/11foot8","section":"100k+","state":"PRIVATE"},{"name":"r/1200isjerky","section":"50k+","state":"PRIVATE"},{"name":"r

	runner = threading.Thread(target=loop_messages)
	runner.daemon = True
	runner.start()

@lib.hook(needchan=False, glevel=50)
@lib.help('[<suffix>]', 'sets reddark topic suffix')
def topicsuffix(bot, user, chan, realtarget, *args):
	if chan is not None: replyto = chan
	else: replyto = user

	lib.parent.cfg.set('reddark', 'topicsuffix', ' '.join(args))

	bot.msg(replyto, "Updated topic suffix")

@lib.hook(needchan=False, glevel=50)
@lib.help('<seconds>', 'sets reddark topic max update interval')
@lib.argsEQ(1)
def updateinterval(bot, user, chan, realtarget, *args):
	if chan is not None: replyto = chan
	else: replyto = user

	lib.parent.cfg.set('reddark', 'update_interval', int(' '.join(args)))

	bot.msg(replyto, "Updated topic interval")
Commit	Line	Data
72669360 JR	1	# Reddark streaming module
	2	# vim: fileencoding=utf-8
	3
	4	# module info
	5	modinfo = {
	6	'author': 'Multiple',
	7	'license': 'unknown',
	8	'compatible': [0], # compatible module API versions
	9	'depends': [], # other modules required to work properly?
	10	'softdeps': ['help'], # modules which are preferred but not required
	11	}
	12	# note: softdeps will be loaded before this module, IF not disabled in the configuration (autoload.module = 0) (and if it exists)
	13	# however, if it is disabled it will be silently ignored, and if it is unloaded at runtime it won't cause this one to unload.
	14	#
	15	# basically, softdeps are things this module will use if available, but does not require (no errors will occur if it's not loaded)
	16	# for example, @lib.help() will attempt to use the help module, but swallow errors if it is not loaded
	17
	18
	19	# global variables
	20	wants_to_stop = False
	21	runner = None
587ee5ca	22	last_update = 0
587ee5ca	23	last_topic = ""
72669360 JR	24
	25	# preamble
	26	import modlib
	27	lib = modlib.modlib(__name__)
	28	def modstart(parent, args, *kwargs):
	29	gotParent(parent)
	30	return lib.modstart(parent, args, *kwargs)
	31	def modstop(args, *kwargs):
	32	global wants_to_stop, runner
	33	if runner:
	34	debug("Stopping runner")
85267bc7 JR	35	wants_to_stop = True # tell the SSE thread to stop the next time it wakes up
85267bc7 JR	36	runner.join() # blocks the main thread til the next chunk received (or timeout)! oh well.
72669360 JR	37	debug("runner stopped")
	38	wants_to_stop = False
	39	return lib.modstop(args, *kwargs)
	40
	41	# module code
	42	from modules.contrib.sseclient import SSEClient
587ee5ca JR	43	import requests
	44	from bs4 import BeautifulSoup
	45	import threading, json, time, collections, re
72669360 JR	46
	47	def chan():
	48	return lib.parent.channel(lib.parent.cfg.get('reddark', 'channel', default="##.test"))
	49
	50	def bot():
	51	return chan().bot
	52
e669bde0 JR	53	def chanmsg(message):
	54	return chan().msg(message, truncate=True, msgtype="NOTICE")
	55
72669360 JR	56	def debug(message, send_to_owner=True):
	57	if lib.parent is None:
	58	print(message)
	59	return
	60	if lib.parent.cfg.getboolean('reddark', 'debug', True):
	61	lib.parent.log('Reddark', 'R', message)
	62	if send_to_owner and lib.parent.cfg.get('debug', 'owner'):
587ee5ca	63	bot().fastmsg(lib.parent.cfg.get('debug', 'owner'), message)
72669360 JR	64
72669360 JR	65	def getText(subreddit):
587ee5ca JR	66	r = None
	67	e = None
	68	try:
	69	r = requests.get('https://old.reddit.com/' + subreddit, headers={'User-Agent': 'better-see-reason-bot/1.0'})
	70	except Exception as e2:
	71	e = e2
	72	if r is None:
36c4d5fc	73	debug("Error getting text: " + repr(e), False)
b93d6f64	74	return ''
587ee5ca	75	if r.status_code != 403:
36c4d5fc	76	debug("Error getting text: " + str(r.status_code) + " (403 expected)", False)
b93d6f64	77	return ''
587ee5ca JR	78	soup = BeautifulSoup(r.text, 'html.parser')
587ee5ca JR	79	elements = soup.find_all(class_='interstitial-subreddit-description')
b7ecdf32	80	if len(elements):
587ee5ca JR	81	text = elements[0].get_text()
587ee5ca JR	82	text = re.sub(re.escape(subreddit), '', text, re.IGNORECASE).replace("\n", " ")
81a6a6e2 JR	83	return ' - ' + text
81a6a6e2 JR	84	return ''
72669360 JR	85
	86	def handleDelta(message):
	87	message['state'] = message['state'].lower()
	88	message['previous_state'] = message['previous_state'].lower()
	89	if message['state'] == 'private':
	90	message['text'] = getText(message['name'])
587ee5ca	91	print(repr(message))
e669bde0	92	chanmsg('[%(section)s] %(name)s went %(state)s (was: %(previous_state)s) (https://old.reddit.com/%(name)s)%(text)s' % message)
36c4d5fc	93	elif message['state'] == 'public':
e669bde0	94	chanmsg('[%(section)s] %(name)s went \x02%(state)s\x02 (was: %(previous_state)s) (https://old.reddit.com/%(name)s)' % message)
36c4d5fc JR	95	else:
36c4d5fc JR	96	chanmsg('[%(section)s] %(name)s went %(state)s (was: %(previous_state)s) (https://old.reddit.com/%(name)s)' % message)
72669360 JR	97
72669360 JR	98	def handleState(message):
85267bc7 JR	99	global last_update, last_topic
85267bc7 JR	100	if time.time() < last_update + lib.parent.cfg.getint('reddark', 'update_interval', 600):
587ee5ca JR	101	return
	102	output = collections.defaultdict(int)
	103	output['totalSubs'] = len(message['subreddits'])
	104	for sub in message['subreddits']:
	105	if sub['state'] == 'PRIVATE':
	106	output['privateSubs'] += 1
	107	output['protestingSubs'] += 1
587ee5ca JR	108	elif sub['state'] == 'PUBLIC':
587ee5ca JR	109	output['publicSubs'] += 1
36c4d5fc JR	110	else:
	111	output['restrictedSubs'] += 1
	112	output['protestingSubs'] += 1
587ee5ca JR	113	output['pct'] = round(output['protestingSubs']/output['totalSubs']*100, 2)
	114	output['pctPublic'] = round(output['publicSubs']/output['totalSubs']*100, 2)
	115	output['pctPrivate'] = round(output['privateSubs']/output['totalSubs']*100, 2)
	116	output['pctRestricted'] = round(output['restrictedSubs']/output['totalSubs']*100, 2)
	117	newTopic = 'subreddits protesting: %(protestingSubs)s out of %(totalSubs)s pledged (%(pct)s%%) \| private: %(privateSubs)s (%(pctPrivate)s%%), restricted: %(restrictedSubs)s (%(pctRestricted)s%%), public: %(publicSubs)s (%(pctPublic)s%%)' % dict(output)
	118	debug(newTopic, False)
	119	if last_topic != newTopic:
	120	last_topic = newTopic
	121	last_update = time.time()
85267bc7	122	bot().conn.send("TOPIC %(chan)s :%(topic)s %(suffix)s" % {'chan': chan(), 'topic': newTopic, 'suffix': lib.parent.cfg.get('reddark', 'topicsuffix', '\| Reddark That Works™ https://reddark.rewby.archivete.am/ (https://github.com/reddark-remix/reddark-remix) \| https://www.youtube.com/watch?v=xMaE6toi4mk')})
72669360 JR	123
	124	def gotParent(parent):
	125	global runner
	126	def loop_messages():
	127	global wants_to_stop
	128	messages = SSEClient(parent.cfg.get('reddark', 'sse', default="https://reddark.rewby.archivete.am/sse"), timeout=60)
	129	debug("Connected to SSE", False)
	130	for msg in messages:
b7ecdf32 JR	131	try:
	132	if wants_to_stop:
	133	return
	134	if len(msg.data) == 0:
	135	continue
	136	data = json.loads(msg.data)
587ee5ca	137	debug(repr(data)[0:500], False)
b7ecdf32 JR	138	if data['type'] == 'Delta':
	139	handleDelta(data['content'])
	140	elif data['type'] == 'CurrentStateUpdate':
	141	handleState(data['content'])
	142	else:
	143	debug("Unknown event: " + msg, False)
	144	except Exception as e:
ddea07bb	145	debug("Failed to parse an event: " + repr(e))
72669360 JR	146	#{"type":"Delta","content":{"name":"r/TrainCrashSeries","section":"1k+","previous_state":"PRIVATE","state":"RESTRICTED"}}
	147	#{"type":"CurrentStateUpdate","content":{"sections":["40+ million","30+ million","20+ million","10+ million","5+ million","1+ million","500k+","250k+","100k+","50k+","5k+","1k+","1k and below"],"subreddits":[{"name":"r/032r4r","section":"1k+","state":"PRIVATE"},{"name":"r/0sanitymemes","section":"5k+","state":"PRIVATE"},{"name":"r/1022","section":"5k+","state":"PRIVATE"},{"name":"r/11foot8","section":"100k+","state":"PRIVATE"},{"name":"r/1200isjerky","section":"50k+","state":"PRIVATE"},{"name":"r
	148
	149	runner = threading.Thread(target=loop_messages)
	150	runner.daemon = True
	151	runner.start()
	152
	153	@lib.hook(needchan=False, glevel=50)
85267bc7	154	@lib.help('[<suffix>]', 'sets reddark topic suffix')
72669360 JR	155	def topicsuffix(bot, user, chan, realtarget, *args):
	156	if chan is not None: replyto = chan
	157	else: replyto = user
	158
	159	lib.parent.cfg.set('reddark', 'topicsuffix', ' '.join(args))
	160
	161	bot.msg(replyto, "Updated topic suffix")
85267bc7 JR	162
	163	@lib.hook(needchan=False, glevel=50)
	164	@lib.help('<seconds>', 'sets reddark topic max update interval')
	165	@lib.argsEQ(1)
	166	def updateinterval(bot, user, chan, realtarget, *args):
	167	if chan is not None: replyto = chan
	168	else: replyto = user
	169
	170	lib.parent.cfg.set('reddark', 'update_interval', int(' '.join(args)))
	171
	172	bot.msg(replyto, "Updated topic interval")