Replace 'getheader' with 'get'.

[z_archive/twitter.git] / twitter / archiver.py

diff --git a/twitter/archiver.py b/twitter/archiver.py
index a5edb0533b5ac8ff7539c500aebe409ccc65e131..1984eaa18d37370cdaf57a32c25f34d3cbdf210f 100644 (file)
--- a/twitter/archiver.py
+++ b/twitter/archiver.py
@@ -14,6 +14,8 @@ OPTIONS
  -a --api-rate         see current API rate limit status
  -t --timeline <file>  archive own timeline into given file name (requires
                        OAuth, max 800 statuses).
  -a --api-rate         see current API rate limit status
  -t --timeline <file>  archive own timeline into given file name (requires
                        OAuth, max 800 statuses).

+ -f --follow-redirects follow redirects of urls
+ -r --redirect-sites   follow redirects for this comma separated list of hosts

 
 AUTHENTICATION
     Authenticate to Twitter using OAuth to archive tweets of private profiles
 
 AUTHENTICATION
     Authenticate to Twitter using OAuth to archive tweets of private profiles


@@ -23,9 +25,17 @@ AUTHENTICATION
 
 from __future__ import print_function
 
 
 from __future__ import print_function
 

-import os, sys, time, calendar, urllib2, httplib
+import os, sys, time, calendar, functools

 from getopt import gnu_getopt as getopt, GetoptError
 
 from getopt import gnu_getopt as getopt, GetoptError
 

+try:
+    import urllib.request as urllib2
+    import http.client as httplib
+except ImportError:
+    import urllib2
+    import httplib
+
+

 # T-Archiver (Twitter-Archiver) application registered by @stalkr_
 CONSUMER_KEY='d8hIyfzs7ievqeeZLjZrqQ'
 CONSUMER_SECRET='AnZmK0rnvaX7BoJ75l6XlilnbyMv7FoiDXWVmPD8'
 # T-Archiver (Twitter-Archiver) application registered by @stalkr_
 CONSUMER_KEY='d8hIyfzs7ievqeeZLjZrqQ'
 CONSUMER_SECRET='AnZmK0rnvaX7BoJ75l6XlilnbyMv7FoiDXWVmPD8'


@@ -34,13 +44,13 @@ from .api import Twitter, TwitterError
 from .oauth import OAuth, read_token_file
 from .oauth_dance import oauth_dance
 from .auth import NoAuth
 from .oauth import OAuth, read_token_file
 from .oauth_dance import oauth_dance
 from .auth import NoAuth

-from .util import Fail, err
+from .util import Fail, err, expand_line, parse_host_list

 from .follow import lookup
 
 def parse_args(args, options):
     """Parse arguments from command-line to set options."""
 from .follow import lookup
 
 def parse_args(args, options):
     """Parse arguments from command-line to set options."""

-    long_opts = ['help', 'oauth', 'save-dir=', 'api-rate', 'timeline=']
-    short_opts = "hos:at:"
+    long_opts = ['help', 'oauth', 'save-dir=', 'api-rate', 'timeline=', 'follow-redirects',"redirect-sites="]
+    short_opts = "hos:at:fr:"

     opts, extra_args = getopt(args, short_opts, long_opts)
 
     for opt, arg in opts:
     opts, extra_args = getopt(args, short_opts, long_opts)
 
     for opt, arg in opts:


@@ -55,6 +65,10 @@ def parse_args(args, options):
             options['api-rate' ] = True
         elif opt in ('-t', '--timeline'):
             options['timeline'] = arg
             options['api-rate' ] = True
         elif opt in ('-t', '--timeline'):
             options['timeline'] = arg

+        elif opt in ('-f', '--follow-redirects'):
+            options['follow-redirects'] = True
+        elif opt in ('-r', '--redirect-sites'):
+            options['redirect-sites'] = arg

 
     options['extra_args'] = extra_args
 
 
     options['extra_args'] = extra_args
 


@@ -108,7 +122,11 @@ def format_date(utc, to_localtime=True):
     else:
         return time.strftime("%Y-%m-%d %H:%M:%S UTC", u)
 
     else:
         return time.strftime("%Y-%m-%d %H:%M:%S UTC", u)
 

-def format_text(text):
+def expand_format_text(hosts, text):
+    """Following redirects in links."""
+    return direct_format_text(expand_line(text, hosts))
+
+def direct_format_text(text):

     """Transform special chars in text to have only one line."""
     return text.replace('\n','\\n').replace('\r','\\r')
 
     """Transform special chars in text to have only one line."""
     return text.replace('\n','\\n').replace('\r','\\r')
 


@@ -161,7 +179,6 @@ def timeline_portion(twitter, screen_name, max_id=None):
         tweets[t['id']] = "%s <%s> %s" % (format_date(t['created_at']),
                                           t['user']['screen_name'],
                                           format_text(text))
         tweets[t['id']] = "%s <%s> %s" % (format_date(t['created_at']),
                                           t['user']['screen_name'],
                                           format_text(text))

-

     return tweets
 
 def timeline(twitter, screen_name, tweets):
     return tweets
 
 def timeline(twitter, screen_name, tweets):


@@ -189,6 +206,9 @@ def timeline(twitter, screen_name, tweets):
                                                     reset, delay))
                 fail.wait(delay)
                 continue
                                                     reset, delay))
                 fail.wait(delay)
                 continue

+            elif e.e.code == 404:
+                err("Fail: %i This profile does not exist" % e.e.code)
+                break

             elif e.e.code == 502:
                 err("Fail: %i Service currently unavailable, retrying..."
                     % e.e.code)
             elif e.e.code == 502:
                 err("Fail: %i Service currently unavailable, retrying..."
                     % e.e.code)


@@ -229,7 +249,9 @@ def main(args=sys.argv[1:]):
         'oauth': False,
         'save-dir': ".",
         'api-rate': False,
         'oauth': False,
         'save-dir': ".",
         'api-rate': False,

-        'timeline': ""
+        'timeline': "",
+        'follow-redirects': False,
+        'redirect-sites': None,

     }
     try:
         parse_args(args, options)
     }
     try:
         parse_args(args, options)


@@ -263,6 +285,16 @@ def main(args=sys.argv[1:]):
         rate_limit_status(twitter)
         return
 
         rate_limit_status(twitter)
         return
 

+    global format_text
+    if options['follow-redirects'] or options['redirect-sites'] :
+        if options['redirect-sites']:
+            hosts = parse_host_list(options['redirect-sites'])
+        else:
+            hosts = None
+        format_text = functools.partial(expand_format_text, hosts)
+    else:
+        format_text = direct_format_text
+    

     # save own timeline (the user used in OAuth)
     if options['timeline']:
         if isinstance(auth, NoAuth):
     # save own timeline (the user used in OAuth)
     if options['timeline']:
         if isinstance(auth, NoAuth):


@@ -275,7 +307,7 @@ def main(args=sys.argv[1:]):
         tweets = {}
         try:
             tweets = load_tweets(filename)
         tweets = {}
         try:
             tweets = load_tweets(filename)

-        except Exception, e:
+        except Exception as e:

             err("Error when loading saved tweets: %s - continuing without"
                 % str(e))
 
             err("Error when loading saved tweets: %s - continuing without"
                 % str(e))
 


@@ -304,7 +336,7 @@ def main(args=sys.argv[1:]):
         tweets = {}
         try:
             tweets = load_tweets(filename)
         tweets = {}
         try:
             tweets = load_tweets(filename)

-        except Exception, e:
+        except Exception as e:

             err("Error when loading saved tweets: %s - continuing without"
                 % str(e))
 
             err("Error when loading saved tweets: %s - continuing without"
                 % str(e))