Ignore .pyo

[z_archive/twitter.git] / twitter / archiver.py

diff --git a/twitter/archiver.py b/twitter/archiver.py
index 587b63e6e4abbc4a34f22f9ff157b0e2213e441c..6f2468a6e03cec1120a88d15bc5bf32a4ba9f20a 100644 (file)
--- a/twitter/archiver.py
+++ b/twitter/archiver.py
@@ -21,6 +21,7 @@ OPTIONS
  -r --redirect-sites   follow redirects for this comma separated list of hosts
  -d --dms <file>       archive own direct messages (both received and
                        sent) into given file name.
  -r --redirect-sites   follow redirects for this comma separated list of hosts
  -d --dms <file>       archive own direct messages (both received and
                        sent) into given file name.

+ -i --isoformat        store dates in ISO format (specifically RFC 3339)

 
 AUTHENTICATION
     Authenticate to Twitter using OAuth to archive tweets of private profiles
 
 AUTHENTICATION
     Authenticate to Twitter using OAuth to archive tweets of private profiles


@@ -30,7 +31,8 @@ AUTHENTICATION
 
 from __future__ import print_function
 
 
 from __future__ import print_function
 

-import os, sys, time, calendar, functools
+import os, sys, time as _time, calendar, functools
+from datetime import time, date, datetime

 from getopt import gnu_getopt as getopt, GetoptError
 
 try:
 from getopt import gnu_getopt as getopt, GetoptError
 
 try:


@@ -51,11 +53,12 @@ from .oauth_dance import oauth_dance
 from .auth import NoAuth
 from .util import Fail, err, expand_line, parse_host_list
 from .follow import lookup
 from .auth import NoAuth
 from .util import Fail, err, expand_line, parse_host_list
 from .follow import lookup

+from .timezones import utc as UTC, Local

 
 def parse_args(args, options):
     """Parse arguments from command-line to set options."""
 
 def parse_args(args, options):
     """Parse arguments from command-line to set options."""

-    long_opts = ['help', 'oauth', 'save-dir=', 'api-rate', 'timeline=', 'mentions=', 'favorites', 'follow-redirects',"redirect-sites=", 'dms=']
-    short_opts = "hos:at:m:vfr:d:"
+    long_opts = ['help', 'oauth', 'save-dir=', 'api-rate', 'timeline=', 'mentions=', 'favorites', 'follow-redirects',"redirect-sites=", 'dms=', 'isoformat']
+    short_opts = "hos:at:m:vfr:d:i"

     opts, extra_args = getopt(args, short_opts, long_opts)
 
     for opt, arg in opts:
     opts, extra_args = getopt(args, short_opts, long_opts)
 
     for opt, arg in opts:


@@ -80,6 +83,8 @@ def parse_args(args, options):
             options['redirect-sites'] = arg
         elif opt in ('-d', '--dms'):
             options['dms'] = arg
             options['redirect-sites'] = arg
         elif opt in ('-d', '--dms'):
             options['dms'] = arg

+        elif opt in ('-i', '--isoformat'):
+            options['isoformat'] = True

 
     options['extra_args'] = extra_args
 
 
     options['extra_args'] = extra_args
 


@@ -124,14 +129,22 @@ def save_tweets(filename, tweets):
 
     archive.close()
 
 
     archive.close()
 

-def format_date(utc, to_localtime=True):
+def format_date(utc, isoformat=False):

     """Parse Twitter's UTC date into UTC or local time."""
     """Parse Twitter's UTC date into UTC or local time."""

-    u = time.strptime(utc.replace('+0000','UTC'), '%a %b %d %H:%M:%S %Z %Y')
-    if to_localtime and time.timezone != 0:
-        t = time.localtime(calendar.timegm(u))
-        return time.strftime("%Y-%m-%d %H:%M:%S", t) + " " + time.tzname[1]
+    u = datetime.strptime(utc.replace('+0000','UTC'), '%a %b %d %H:%M:%S %Z %Y')
+    # This is the least painful way I could find to create a non-naive
+    # datetime including a UTC timezone. Alternative suggestions
+    # welcome.
+    unew = datetime.combine(u.date(), time(u.time().hour,
+        u.time().minute, u.time().second, tzinfo=UTC))
+
+    # Convert to localtime
+    unew = unew.astimezone(Local)
+
+    if isoformat:
+        return unew.isoformat()

     else:
     else:

-        return time.strftime("%Y-%m-%d %H:%M:%S UTC", u)
+        return unew.strftime('%Y-%m-%d %H:%M:%S %Z')

 
 def expand_format_text(hosts, text):
     """Following redirects in links."""
 
 def expand_format_text(hosts, text):
     """Following redirects in links."""


@@ -169,7 +182,7 @@ def statuses_resolve_uids(twitter, tl):
 
     return new_tl
 
 
     return new_tl
 

-def statuses_portion(twitter, screen_name, max_id=None, mentions=False, favorites=False, received_dms=None):
+def statuses_portion(twitter, screen_name, max_id=None, mentions=False, favorites=False, received_dms=None, isoformat=False):

     """Get a portion of the statuses of a screen name."""
     kwargs = dict(count=200, include_rts=1, screen_name=screen_name)
     if max_id:
     """Get a portion of the statuses of a screen name."""
     kwargs = dict(count=200, include_rts=1, screen_name=screen_name)
     if max_id:


@@ -177,9 +190,9 @@ def statuses_portion(twitter, screen_name, max_id=None, mentions=False, favorite
 
     tweets = {}
     if mentions:
 
     tweets = {}
     if mentions:

-        tl = twitter.statuses.mentions(**kwargs)
+        tl = twitter.statuses.mentions_timeline(**kwargs)

     elif favorites:
     elif favorites:

-        tl = twitter.favorites(**kwargs) # API v1, favorites.list() in v1.1
+        tl = twitter.favorites.list(**kwargs)

     elif received_dms != None:
         if received_dms:
             tl = twitter.direct_messages(**kwargs)
     elif received_dms != None:
         if received_dms:
             tl = twitter.direct_messages(**kwargs)


@@ -206,24 +219,24 @@ def statuses_portion(twitter, screen_name, max_id=None, mentions=False, favorite
         # the recipient was, we synthesise a mention. If we're not
         # operating on DMs, behave as normal
         if received_dms == None:
         # the recipient was, we synthesise a mention. If we're not
         # operating on DMs, behave as normal
         if received_dms == None:

-          tweets[t['id']] = "%s <%s> %s" % (format_date(t['created_at']),
+          tweets[t['id']] = "%s <%s> %s" % (format_date(t['created_at'], isoformat=isoformat),

                                             t['user']['screen_name'],
                                             format_text(text))
         else:
                                             t['user']['screen_name'],
                                             format_text(text))
         else:

-          tweets[t['id']] = "%s <%s> @%s %s" % (format_date(t['created_at']),
+          tweets[t['id']] = "%s <%s> @%s %s" % (format_date(t['created_at'], isoformat=isoformat),

                                             t['sender_screen_name'],
                                             t['recipient']['screen_name'],
                                             format_text(text))
     return tweets
 
                                             t['sender_screen_name'],
                                             t['recipient']['screen_name'],
                                             format_text(text))
     return tweets
 

-def statuses(twitter, screen_name, tweets, mentions=False, favorites=False, received_dms=None):
+def statuses(twitter, screen_name, tweets, mentions=False, favorites=False, received_dms=None, isoformat=False):

     """Get all the statuses for a screen name."""
     max_id = None
     fail = Fail()
     # get portions of statuses, incrementing max id until no new tweets appear
     while True:
         try:
     """Get all the statuses for a screen name."""
     max_id = None
     fail = Fail()
     # get portions of statuses, incrementing max id until no new tweets appear
     while True:
         try:

-            portion = statuses_portion(twitter, screen_name, max_id, mentions, favorites, received_dms)
+            portion = statuses_portion(twitter, screen_name, max_id, mentions, favorites, received_dms, isoformat)

         except TwitterError as e:
             if e.e.code == 401:
                 err("Fail: %i Unauthorized (tweets of that user are protected)"
         except TwitterError as e:
             if e.e.code == 401:
                 err("Fail: %i Unauthorized (tweets of that user are protected)"


@@ -290,6 +303,7 @@ def main(args=sys.argv[1:]):
         'favorites': False,
         'follow-redirects': False,
         'redirect-sites': None,
         'favorites': False,
         'follow-redirects': False,
         'redirect-sites': None,

+        'isoformat': False,

     }
     try:
         parse_args(args, options)
     }
     try:
         parse_args(args, options)


@@ -319,7 +333,7 @@ def main(args=sys.argv[1:]):
     else:
         auth = NoAuth()
 
     else:
         auth = NoAuth()
 

-    twitter = Twitter(auth=auth, api_version='1', domain='api.twitter.com')
+    twitter = Twitter(auth=auth, api_version='1.1', domain='api.twitter.com')

 
     if options['api-rate']:
         rate_limit_status(twitter)
 
     if options['api-rate']:
         rate_limit_status(twitter)


@@ -356,7 +370,7 @@ def main(args=sys.argv[1:]):
                 % str(e))
 
         try:
                 % str(e))
 
         try:

-            statuses(twitter, "", tweets, options['mentions'], options['favorites'])
+            statuses(twitter, "", tweets, options['mentions'], options['favorites'], isoformat=options['isoformat'])

         except KeyboardInterrupt:
             err()
             err("Interrupted")
         except KeyboardInterrupt:
             err()
             err("Interrupted")


@@ -379,13 +393,13 @@ def main(args=sys.argv[1:]):
         dms = {}
         try:
             dms = load_tweets(filename)
         dms = {}
         try:
             dms = load_tweets(filename)

-        except Exception, e:
+        except Exception as e:

             err("Error when loading saved DMs: %s - continuing without"
                 % str(e))
 
         try:
             err("Error when loading saved DMs: %s - continuing without"
                 % str(e))
 
         try:

-            statuses(twitter, "", dms, received_dms=True)
-            statuses(twitter, "", dms, received_dms=False)
+            statuses(twitter, "", dms, received_dms=True, isoformat=options['isoformat'])
+            statuses(twitter, "", dms, received_dms=False, isoformat=options['isoformat'])

         except KeyboardInterrupt:
             err()
             err("Interrupted")
         except KeyboardInterrupt:
             err()
             err("Interrupted")


@@ -418,7 +432,7 @@ def main(args=sys.argv[1:]):
         new = 0
         before = len(tweets)
         try:
         new = 0
         before = len(tweets)
         try:

-            statuses(twitter, user, tweets, options['mentions'], options['favorites'])
+            statuses(twitter, user, tweets, options['mentions'], options['favorites'], isoformat=options['isoformat'])

         except KeyboardInterrupt:
             err()
             err("Interrupted")
         except KeyboardInterrupt:
             err()
             err("Interrupted")