Unit test for previous silly regression

[z_archive/twitter.git] / twitter / util.py

diff --git a/twitter/util.py b/twitter/util.py
index 4396b07dabc3408d4f6113edb26b7687c7b2d5db..8d3bd8acac7c4b165ad515e840cb49a90d367b83 100644 (file)
--- a/twitter/util.py
+++ b/twitter/util.py
@@ -10,15 +10,19 @@ from __future__ import print_function
 import contextlib
 import re
 import sys
+import textwrap
 import time
-import urllib2
-import urlparse
+import socket
 
 try:
     from html.entities import name2codepoint
     unichr = chr
+    import urllib.request as urllib2
+    import urllib.parse as urlparse
 except ImportError:
     from htmlentitydefs import name2codepoint
+    import urllib2
+    import urlparse
 
 def htmlentitydecode(s):
     return re.sub(
@@ -45,6 +49,8 @@ def printNicely(string):
     if hasattr(sys.stdout, 'buffer'):
         sys.stdout.buffer.write(string.encode('utf8'))
         print()
+        sys.stdout.buffer.flush()
+        sys.stdout.flush()
     else:
         print(string.encode('utf8'))
 
@@ -83,18 +89,18 @@ class Fail(object):
 def find_links(line):
     """Find all links in the given line. The function returns a sprintf style
     format string (with %s placeholders for the links) and a list of urls."""
-    l = line.replace(u"%", u"%%")
+    l = line.replace("%", "%%")
     regex = "(https?://[^ )]+)"
     return (
-        re.sub(regex, "%s", l), 
+        re.sub(regex, "%s", l),
         [m.group(1) for m in re.finditer(regex, l)])
-    
+
 def follow_redirects(link, sites= None):
     """Follow directs for the link as long as the redirects are on the given
     sites and return the resolved link."""
     def follow(url):
         return sites == None or urlparse.urlparse(url).hostname in sites
-                
+
     class RedirectHandler(urllib2.HTTPRedirectHandler):
         def __init__(self):
             self.last_url = None
@@ -106,7 +112,7 @@ def follow_redirects(link, sites= None):
                 self, req, fp, code, msg, hdrs, newurl)
             r.get_method = lambda : 'HEAD'
             return r
-            
+
     if not follow(link):
         return link
     redirect_handler = RedirectHandler()
@@ -114,21 +120,37 @@ def follow_redirects(link, sites= None):
     req = urllib2.Request(link)
     req.get_method = lambda : 'HEAD'
     try:
-        with contextlib.closing(opener.open(req)) as site:
+        with contextlib.closing(opener.open(req,timeout=1)) as site:
             return site.url
-    except (urllib2.HTTPError, urllib2.URLError):
+    except:
         return redirect_handler.last_url if redirect_handler.last_url else link
 
 def expand_line(line, sites):
     """Expand the links in the line for the given sites."""
-    l = line.strip()
-    msg_format, links = find_links(l)
-    args = tuple(follow_redirects(l, sites) for l in links)
-    return msg_format % args
+    try:
+        l = line.strip()
+        msg_format, links = find_links(l)
+        args = tuple(follow_redirects(l, sites) for l in links)
+        line = msg_format % args
+    except Exception as e:
+        try:
+            err("expanding line %s failed due to %s" % (line, unicode(e)))
+        except:
+            pass
+    return line
 
 def parse_host_list(list_of_hosts):
     """Parse the comma separated list of hosts."""
     p = set(
         m.group(1) for m in re.finditer("\s*([^,\s]+)\s*,?\s*", list_of_hosts))
     return p
-    
+
+
+def align_text(text, left_margin=17, max_width=160):
+    lines = []
+    for line in text.split('\n'):
+        temp_lines = textwrap.wrap(line, max_width - left_margin)
+        temp_lines = [(' ' * left_margin + line) for line in temp_lines]
+        lines.append('\n'.join(temp_lines))
+    ret = '\n'.join(lines)
+    return ret.lstrip()