+ response = opener.open(request, timeout=2)
+ except urllib2.HTTPError as e:
+ return 'Request error: %s %s' % (e.code, e.reason)
+ except urllib2.URLError as e:
+ if "certificate verify failed: unable to get local issuer certificate" in str(e.reason):
+ if aia: # Retry with AIA enabled
+ return _do_request(url, True)
+ else:
+ lib.parent.log('urls', '?', 'If the site is not serving the certificate chain, installing the aia library might make this request work: pip install aia')
+ return 'Request error: site may have broken TLS configuration (%s)' % (e.reason)
+ else:
+ return 'Request error: %s' % (e.reason)
+ except TimeoutError as e:
+ return 'Request error: request timed out'
+ except Exception as e:
+ return 'Unknown error: %s %r' % (type(e).__name__, e.args)
+
+ return response
+
+
+def goturl(url):
+ output = []
+ for _, group in other_regexes:
+ for regex in group:
+ if regex.match(url):
+ return None
+
+ response = _do_request(url)
+ if isinstance(response, stringbase):
+ return response
+
+ # Try to add type and length headers to reply
+ c_type = response.getheader('Content-Type', '').split(';', 1)[0]
+ c_len = response.getheader('Content-Length')
+ if c_type != '':
+ output.append("[%s] " % (c_type))
+ else:
+ output.append("[no type] ")
+ if c_type != "text/html": # else length will be provided by HTML code below
+ if c_len is not None:
+ output.append("[%s] " % (_humanize_bytes(c_len)))
+ else:
+ output.append("[no length] ")
+
+ # Try to add title if HTML
+ if c_type == 'text/html':
+ try:
+ responsebody = response.read(1024*1024)
+ except Exception as e:
+ output.append('Error reading response body: %s %r' % (type(e).__name__, e.args))
+ else:
+ if c_len is not None and len(responsebody) != int(c_len):
+ output.append("[actual %s; Content-Length %s] " % (_humanize_bytes(len(responsebody)), _humanize_bytes(c_len)))
+ else:
+ output.append("[%s] " % (_humanize_bytes(len(responsebody))))
+ try:
+ soup = BeautifulSoup(responsebody)
+ if soup.title:
+ output.append('Title: ' + unescape('%s' % (soup.find('title').string.strip())))
+ else:
+ output.append('No title')
+ except Exception as e:
+ output.append('Title error: %s %r ' % (type(e).__name__, e.args))
+
+ return ''.join(output)
+
+url_regex = (
+ re.compile(r'https?://(?:[^/\s.]+\.)+[^/\s.]+(?:/\S+)?'),
+)
+other_regexes = (
+ (lambda x: '', (re.compile(r"""https?://(?:www\.)?(?:twitter|x)\.com/""", re.I),)), # skip twitter
+)
+regexes = other_regexes + (
+ (goturl, url_regex),
+)