X-Git-Url: https://jfr.im/git/z_archive/twitter.git/blobdiff_plain/907402f64301f9f87555afa5c305c40504da7145..c0c3727024108dd12d96b7a3133a44f1fd252d1c:/twitter/util.py diff --git a/twitter/util.py b/twitter/util.py index 4396b07..8d66c4e 100644 --- a/twitter/util.py +++ b/twitter/util.py @@ -11,14 +11,16 @@ import contextlib import re import sys import time -import urllib2 -import urlparse try: from html.entities import name2codepoint unichr = chr + import urllib.request as urllib2 + import urllib.parse as urlparse except ImportError: from htmlentitydefs import name2codepoint + import urllib2 + import urlparse def htmlentitydecode(s): return re.sub( @@ -83,18 +85,18 @@ class Fail(object): def find_links(line): """Find all links in the given line. The function returns a sprintf style format string (with %s placeholders for the links) and a list of urls.""" - l = line.replace(u"%", u"%%") + l = line.replace("%", "%%") regex = "(https?://[^ )]+)" return ( - re.sub(regex, "%s", l), + re.sub(regex, "%s", l), [m.group(1) for m in re.finditer(regex, l)]) - + def follow_redirects(link, sites= None): """Follow directs for the link as long as the redirects are on the given sites and return the resolved link.""" def follow(url): return sites == None or urlparse.urlparse(url).hostname in sites - + class RedirectHandler(urllib2.HTTPRedirectHandler): def __init__(self): self.last_url = None @@ -106,7 +108,7 @@ def follow_redirects(link, sites= None): self, req, fp, code, msg, hdrs, newurl) r.get_method = lambda : 'HEAD' return r - + if not follow(link): return link redirect_handler = RedirectHandler() @@ -131,4 +133,4 @@ def parse_host_list(list_of_hosts): p = set( m.group(1) for m in re.finditer("\s*([^,\s]+)\s*,?\s*", list_of_hosts)) return p - +