X-Git-Url: https://jfr.im/git/erebus.git/blobdiff_plain/11919f17b1336b06d38a23734a0f158cc0e864ab..3c63ef0547be2798c4ed4f55ad9ee7d5e14eb013:/modules/urls.py

diff --git a/modules/urls.py b/modules/urls.py
index 5793ade..4935d68 100644
--- a/modules/urls.py
+++ b/modules/urls.py
@@ -38,6 +38,20 @@ else:
 
 import re, json, datetime
 
+try:
+	import aia
+	aia_session = aia.AIASession()
+	# aia is broken on capath systems, needs cafile to work
+	aia_session._context.load_verify_locations(cafile='/etc/ssl/certs/ca-certificates.crt')
+	aia_session._trusted = {
+		aia.openssl_get_cert_info(ca_der)["subject"]: ca_der
+		for ca_der in aia_session._context.get_ca_certs(True)
+	}
+	print("aia loaded")
+except ImportError as e:
+	print(repr(e))
+	aia = None
+
 hostmask_regex = re.compile(r'^(.*)!(.*)@(.*)$')
 
 def parser_hostmask(hostmask):
@@ -223,14 +237,13 @@ def _humanize_bytes(b):
 	else:
 		return "%.2f%siB" % (b, table[i])
 
-def goturl(url):
-	output = []
-	for _, group in other_regexes:
-		for regex in group:
-			if regex.match(url):
-				return None
+def _do_request(url, try_aia=False):
+	"""Returns the HTTPResponse object, or a string on error"""
 	request = urllib2.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'})
-	opener = urllib2.build_opener(SmartRedirectHandler())
+	if try_aia:
+		opener = urllib2.build_opener(urllib2.HTTPSHandler(context=aia_session.ssl_context_from_url(url)), SmartRedirectHandler())
+	else:
+		opener = urllib2.build_opener(SmartRedirectHandler())
 
 	# Send request and handle errors
 	try:
@@ -238,12 +251,33 @@ def goturl(url):
 	except urllib2.HTTPError as e:
 		return 'Request error: %s %s' % (e.code, e.reason)
 	except urllib2.URLError as e:
-		return 'Request error: %s' % (e.reason)
+		if "certificate verify failed: unable to get local issuer certificate" in str(e.reason):
+			if aia: # Retry with AIA enabled
+				return _do_request(url, True)
+			else:
+				lib.parent.log('urls', '?', 'If the site is not serving the certificate chain, installing the aia library might make this request work: pip install aia')
+				return 'Request error: site may have broken TLS configuration (%s)' % (e.reason)
+		else:
+			return 'Request error: %s' % (e.reason)
 	except TimeoutError as e:
 		return 'Request error: request timed out'
 	except Exception as e:
 		return 'Unknown error: %s %r' % (type(e).__name__, e.args)
 
+	return response
+
+
+def goturl(url):
+	output = []
+	for _, group in other_regexes:
+		for regex in group:
+			if regex.match(url):
+				return None
+
+	response = _do_request(url)
+	if isinstance(response, stringbase):
+		return response
+
 	# Try to add type and length headers to reply
 	c_type = response.getheader('Content-Type', '').split(';', 1)[0]
 	c_len = response.getheader('Content-Length')
@@ -283,6 +317,7 @@ url_regex = (
 	re.compile(r'https?://(?:[^/\s.]+\.)+[^/\s.]+(?:/\S+)?'),
 )
 other_regexes = (
+	(lambda x: '', (re.compile(r"""https?://(?:www\.)?(?:twitter|x)\.com/""", re.I),)), # skip twitter
 )
 regexes = other_regexes + (
 	(goturl, url_regex),