Trapping additional exceptions causing archiver script failures on individual tweets

diff --git a/twitter/archiver.py b/twitter/archiver.py
index 4075409e62a1ede40c1784a49d6e53b11f5dde27..a161311652dbe3eac5e4b05bd1b46e6997073f9d 100644 (file)
--- a/twitter/archiver.py
+++ b/twitter/archiver.py
@@ -97,8 +97,11 @@ def load_tweets(filename):
 
     tweets = {}
     for line in archive.readlines():
-        tid, text = line.strip().split(" ", 1)
-        tweets[int(tid)] = text.decode("utf-8")
+        try:
+            tid, text = line.strip().split(" ", 1)
+            tweets[int(tid)] = text.decode("utf-8")
+        except Exception as e:
+            err("loading tweet %s failed due to %s" % (line, unicode(e)))
 
     archive.close()
     return tweets
@@ -125,7 +128,10 @@ def save_tweets(filename, tweets):
         return
 
     for k in sorted(tweets.keys()):
-        archive.write("%i %s\n" % (k, tweets[k].encode('utf-8')))
+        try:
+            archive.write("%i %s\n" % (k, tweets[k].encode('utf-8')))
+        except Exception as ex:
+            err("archiving tweet %s failed due to %s" % (k, unicode(ex)))
 
     archive.close()
 

diff --git a/twitter/util.py b/twitter/util.py
index 48c131f0dfe2a55c2976947873512c89398c6553..f4a5dd5bff7d3ec4a4af3e0394b278f4c33fe3aa 100644 (file)
--- a/twitter/util.py
+++ b/twitter/util.py
@@ -119,15 +119,22 @@ def follow_redirects(link, sites= None):
     try:
         with contextlib.closing(opener.open(req,timeout=1)) as site:
             return site.url
-    except (urllib2.HTTPError, urllib2.URLError, socket.timeout):
+    except:
         return redirect_handler.last_url if redirect_handler.last_url else link
 
 def expand_line(line, sites):
     """Expand the links in the line for the given sites."""
-    l = line.strip()
-    msg_format, links = find_links(l)
-    args = tuple(follow_redirects(l, sites) for l in links)
-    return msg_format % args
+    try:
+        l = line.strip()
+        msg_format, links = find_links(l)
+        args = tuple(follow_redirects(l, sites) for l in links)
+        line = msg_format % args
+    except Exception as e:
+        try:
+            err("expanding line %s failed due to %s" % (line, unicode(e)))
+        except:
+            pass
+    return line
 
 def parse_host_list(list_of_hosts):
     """Parse the comma separated list of hosts."""