From: Mike Verdone <redacted>
Date: Sat, 12 Mar 2011 18:03:18 +0000 (+0100)
Subject: By golly the logger works.
X-Git-Tag: twitter-1.6~16
X-Git-Url: https://jfr.im/git/z_archive/twitter.git/commitdiff_plain/7f9d60c6b9ada393eeb0d8e52b13ad78c7a2b9fc?hp=146b4f2989ef561333623008244a9513ceb25095

By golly the logger works.
---

diff --git a/twitter/logger.py b/twitter/logger.py
index 7ff61f4..331b921 100644
--- a/twitter/logger.py
+++ b/twitter/logger.py
@@ -21,32 +21,64 @@ DESCRIPTION:
 
 import sys
 import os
+from time import sleep
+
 
 OPTIONS = {
     'oauth_filename': os.environ.get('HOME', '') + os.sep + '.twitter_oauth',
 }
 
-from api import Twitter
+from api import Twitter, TwitterError
 from cmdline import CONSUMER_KEY, CONSUMER_SECRET
 from oauth import read_token_file, OAuth
 
-def main(args=sys.argv[1:]):
-    oauth_filename = OPTIONS['oauth_filename']
-    oauth_token, oauth_token_secret = read_token_file(oauth_filename)
-
-    twitter = Twitter(
-        auth=OAuth(
-            oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET),
-        api_version='1',
-        domain='api.twitter.com')
+def log_debug(msg):
+    print >> sys.stderr, msg
 
+def get_tweets(twitter, max_id=None):
     kwargs = dict(count=3200)
+    if max_id:
+        kwargs['max_id'] = max_id
 
+    n_tweets = 0
     tweets = twitter.statuses.user_timeline(**kwargs)
     for tweet in tweets:
+        if tweet['id'] == max_id:
+            continue
         print "%s %s %s" % (tweet['user']['screen_name'],
                             tweet['id'],
                             tweet['created_at'])
         for line in tweet['text'].splitlines():
             print '    ' + line.encode('utf-8')
         print
+        max_id = tweet['id']
+        n_tweets += 1
+    return n_tweets, max_id
+
+def main(args=sys.argv[1:]):
+    oauth_filename = OPTIONS['oauth_filename']
+    oauth_token, oauth_token_secret = read_token_file(oauth_filename)
+
+    twitter = Twitter(
+        auth=OAuth(
+            oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET),
+        api_version='1',
+        domain='api.twitter.com')
+
+    if args:
+        max_id = args[0]
+    else:
+        max_id = None
+
+    n_tweets = 0
+    while True:
+        try:
+            tweets_processed, max_id = get_tweets(twitter, max_id)
+            n_tweets += tweets_processed
+            log_debug("Processed %i tweets (max_id %s)" %(n_tweets, max_id))
+            if tweets_processed == 0:
+                log_debug("That's it, we got all the tweets. Done.")
+                break
+        except TwitterError, e:
+            log_debug("Twitter bailed out. I'm going to sleep a bit then try again")
+            sleep(3)