urls - remove broken APIs

[erebus.git] / modules / urls.py

diff --git a/modules/urls.py b/modules/urls.py
index 9ac8629fbbf3db90b39c55af769abb448805e18d..ae80a4e92e3c3424b9642300ff5d19fa7ae6a06b 100644 (file)
--- a/modules/urls.py
+++ b/modules/urls.py
@@ -23,12 +23,14 @@ modstop = lib.modstop
 # module code
 import sys
 if sys.version_info.major < 3:
+       stringbase = basestring
        import urllib2
        import urlparse
        import HTMLParser
        html = HTMLParser.HTMLParser()
        from BeautifulSoup import BeautifulSoup
 else:
+       stringbase = str
        import urllib.request as urllib2
        import urllib.parse as urlparse
        import html
@@ -86,9 +88,12 @@ def process_line(line):
                                        num_found += 1
                                        if num_found > limit:
                                                return responses
-                                       resp = action(match)
+                                       if isinstance(match, stringbase):
+                                               resp = action(match)
+                                       else:
+                                               resp = action(*match)
                                        if resp is not None:
-                                               responses.append("%s: %s" % (prefix, action(match)))
+                                               responses.append("%s: %s" % (prefix, resp))
        return responses
 
 @lib.hooknum("PRIVMSG")
@@ -206,36 +211,27 @@ def goturl(url):
                for regex in group:
                        if regex.match(url):
                                return None
-       request = urllib2.Request(url)
+       request = urllib2.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'})
        opener = urllib2.build_opener(SmartRedirectHandler())
        try:
-               soup = BeautifulSoup(opener.open(request, timeout=0.5))
+               soup = BeautifulSoup(opener.open(request, timeout=2))
                if soup.title:
                        return unescape('%s' % (soup.title.string))
                else:
                        return None
        except urllib2.HTTPError as e:
                return 'Error: %s %s' % (e.code, e.reason)
+       except urllib2.URLError as e:
+               return 'Error: %s' % (e.reason)
+       except TimeoutError as e:
+               return 'Error: request timed out'
        except Exception as e:
-               return 'Error: %r' % (e.message)
+               return 'Error: %s %r' % (type(e).__name__, e.args)
 
 url_regex = (
        re.compile(r'https?://[^/\s]+\.[^/\s]+(?:/\S+)?'),
 )
-spotify_regex = (
-       re.compile(r'spotify:(?P<type>\w+):(?P<track_id>\w{22})'),
-       re.compile(r'https?://open\.spotify\.com/(?P<type>\w+)/(?P<track_id>\w+)')
-)
-youtube_regex = (
-       re.compile(r'https?://(?:www\.)?youtube\.com/watch\?[a-zA-Z0-9=&_\-]+'),
-)
-twitch_regex = (
-       re.compile(r'https?:\/\/(?:www\.)?twitch.tv\/([A-Za-z0-9]*)'),
-)
 other_regexes = (
-       (gotspotify, spotify_regex, 'Spotify'),
-       (gotyoutube, youtube_regex, 'YouTube'),
-       (gottwitch, twitch_regex, 'Twitch'),
 )
 regexes = other_regexes + (
        (goturl, url_regex, 'Title'),