[z_archive/twitter.git] / twitter / follow.py

"""USAGE
    twitter-follow [options] <user>

DESCRIPTION
    Display all following/followers of a user, one user per line.

OPTIONS
 -o --oauth            authenticate to Twitter using OAuth (default no)
 -r --followers        display followers of the given user (default)
 -g --following        display users the given user is following
 -a --api-rate         see your current API rate limit status
 -i --ids              prepend user id to each line. useful to tracking renames

AUTHENTICATION
    Authenticate to Twitter using OAuth to see following/followers of private
    profiles and have higher API rate limits. OAuth authentication tokens
    are stored in the file .twitter-follow_oauth in your home directory.
"""

from __future__ import print_function

import os, sys, time, calendar
from getopt import gnu_getopt as getopt, GetoptError

try:
    import urllib.request as urllib2
    import http.client as httplib
except ImportError:
    import urllib2
    import httplib

# T-Follow (Twitter-Follow) application registered by @stalkr_
CONSUMER_KEY='USRZQfvFFjB6UvZIN2Edww'
CONSUMER_SECRET='AwGAaSzZa5r0TDL8RKCDtffnI9H9mooZUdOa95nw8'

from .api import Twitter, TwitterError
from .oauth import OAuth, read_token_file
from .oauth_dance import oauth_dance
from .auth import NoAuth
from .util import Fail, err


def parse_args(args, options):
    """Parse arguments from command-line to set options."""
    long_opts = ['help', 'oauth', 'followers', 'following', 'api-rate', 'ids']
    short_opts = "horgai"
    opts, extra_args = getopt(args, short_opts, long_opts)

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            print(__doc__)
            raise SystemExit(1)
        elif opt in ('-o', '--oauth'):
            options['oauth'] = True
        elif opt in ('-r', '--followers'):
            options['followers'] = True
        elif opt in ('-g', '--following'):
            options['followers'] = False
        elif opt in ('-a', '--api-rate'):
            options['api-rate' ] = True
        elif opt in ('-i', '--ids'):
            options['show_id'] = True

    options['extra_args'] = extra_args

def lookup_portion(twitter, user_ids):
    """Resolve a limited list of user ids to screen names."""
    users = {}
    kwargs = dict(user_id=",".join(map(str, user_ids)), skip_status=1)
    for u in twitter.users.lookup(**kwargs):
        users[int(u['id'])] = u['screen_name']
    return users

def lookup(twitter, user_ids):
    """Resolve an entire list of user ids to screen names."""
    users = {}
    api_limit = 100
    for i in range(0, len(user_ids), api_limit):
        fail = Fail()
        while True:
            try:
                portion = lookup_portion(twitter, user_ids[i:][:api_limit])
            except TwitterError as e:
                if e.e.code == 429:
                    err("Fail: %i API rate limit exceeded" % e.e.code)
                    rls = twitter.application.rate_limit_status()
                    reset = rls.rate_limit_reset
                    reset = time.asctime(time.localtime(reset))
                    delay = int(rls.rate_limit_reset
                                - time.time()) + 5 # avoid race
                    err("Interval limit of %i requests reached, next reset on "
                        "%s: going to sleep for %i secs"
                        % (rls.rate_limit_limit, reset, delay))
                    fail.wait(delay)
                    continue
                elif e.e.code == 502:
                    err("Fail: %i Service currently unavailable, retrying..."
                        % e.e.code)
                else:
                    err("Fail: %s\nRetrying..." % str(e)[:500])
                fail.wait(3)
            except urllib2.URLError as e:
                err("Fail: urllib2.URLError %s - Retrying..." % str(e))
                fail.wait(3)
            except httplib.error as e:
                err("Fail: httplib.error %s - Retrying..." % str(e))
                fail.wait(3)
            except KeyError as e:
                err("Fail: KeyError %s - Retrying..." % str(e))
                fail.wait(3)
            else:
                users.update(portion)
                err("Resolving user ids to screen names: %i/%i"
                    % (len(users), len(user_ids)))
                break
    return users

def follow_portion(twitter, screen_name, cursor=-1, followers=True):
    """Get a portion of followers/following for a user."""
    kwargs = dict(screen_name=screen_name, cursor=cursor)
    if followers:
        t = twitter.followers.ids(**kwargs)
    else: # following
        t = twitter.friends.ids(**kwargs)
    return t['ids'], t['next_cursor']

def follow(twitter, screen_name, followers=True):
    """Get the entire list of followers/following for a user."""
    user_ids = []
    cursor = -1
    fail = Fail()
    while True:
        try:
            portion, cursor = follow_portion(twitter, screen_name, cursor,
                                             followers)
        except TwitterError as e:
            if e.e.code == 401:
                reason = ("follow%s of that user are protected"
                          % ("ers" if followers else "ing"))
                err("Fail: %i Unauthorized (%s)" % (e.e.code, reason))
                break
            elif e.e.code == 429:
                err("Fail: %i API rate limit exceeded" % e.e.code)
                rls = twitter.application.rate_limit_status()
                reset = rls.rate_limit_reset
                reset = time.asctime(time.localtime(reset))
                delay = int(rls.rate_limit_reset
                            - time.time()) + 5 # avoid race
                err("Interval limit of %i requests reached, next reset on %s: "
                    "going to sleep for %i secs" % (rls.rate_limit_limit,
                                                    reset, delay))
                fail.wait(delay)
                continue
            elif e.e.code == 502:
                err("Fail: %i Service currently unavailable, retrying..."
                    % e.e.code)
            else:
                err("Fail: %s\nRetrying..." % str(e)[:500])
            fail.wait(3)
        except urllib2.URLError as e:
            err("Fail: urllib2.URLError %s - Retrying..." % str(e))
            fail.wait(3)
        except httplib.error as e:
            err("Fail: httplib.error %s - Retrying..." % str(e))
            fail.wait(3)
        except KeyError as e:
            err("Fail: KeyError %s - Retrying..." % str(e))
            fail.wait(3)
        else:
            new = -len(user_ids)
            user_ids = list(set(user_ids + portion))
            new += len(user_ids)
            what = "follow%s" % ("ers" if followers else "ing")
            err("Browsing %s %s, new: %i" % (screen_name, what, new))
            if cursor == 0:
                break
            fail = Fail()
    return user_ids


def rate_limit_status(twitter):
    """Print current Twitter API rate limit status."""
    rls = twitter.application.rate_limit_status()
    print("Remaining API requests: %i/%i (interval limit)"
          % (rls.rate_limit_remaining, rls.rate_limit_limit))
    print("Next reset in %is (%s)"
          % (int(rls.rate_limit_reset - time.time()),
             time.asctime(time.localtime(rls.rate_limit_reset))))

def main(args=sys.argv[1:]):
    options = {
        'oauth': False,
        'followers': True,
        'api-rate': False,
        'show_id': False
    }
    try:
        parse_args(args, options)
    except GetoptError as e:
        err("I can't do that, %s." % e)
        raise SystemExit(1)

    # exit if no user or given, except if asking for API rate
    if not options['extra_args'] and not options['api-rate']:
        print(__doc__)
        raise SystemExit(1)

    # authenticate using OAuth, asking for token if necessary
    if options['oauth']:
        oauth_filename = (os.getenv("HOME", "") + os.sep
                          + ".twitter-follow_oauth")
        if not os.path.exists(oauth_filename):
            oauth_dance("Twitter-Follow", CONSUMER_KEY, CONSUMER_SECRET,
                        oauth_filename)
        oauth_token, oauth_token_secret = read_token_file(oauth_filename)
        auth = OAuth(oauth_token, oauth_token_secret, CONSUMER_KEY,
                     CONSUMER_SECRET)
    else:
        auth = NoAuth()

    twitter = Twitter(auth=auth, api_version='1.1', domain='api.twitter.com')

    if options['api-rate']:
        rate_limit_status(twitter)
        return

    # obtain list of followers (or following) for every given user
    for user in options['extra_args']:
        user_ids, users = [], {}
        try:
            user_ids = follow(twitter, user, options['followers'])
            users = lookup(twitter, user_ids)
        except KeyboardInterrupt as e:
            err()
            err("Interrupted.")
            raise SystemExit(1)

        for uid in user_ids:
            if options['show_id']:
              try:
                print(str(uid) + "\t" + users[uid].encode("utf-8"))
              except KeyError:
                pass

            else:
              try:
                print(users[uid].encode("utf-8"))
              except KeyError:
                pass

        # print total on stderr to separate from user list on stdout
        if options['followers']:
            err("Total followers for %s: %i" % (user, len(user_ids)))
        else:
            err("Total users %s is following: %i" % (user, len(user_ids)))
Commit	Line	Data
a7282452 S	1	"""USAGE
	2	twitter-follow [options] <user>
	3
	4	DESCRIPTION
	5	Display all following/followers of a user, one user per line.
	6
	7	OPTIONS
	8	-o --oauth authenticate to Twitter using OAuth (default no)
	9	-r --followers display followers of the given user (default)
	10	-g --following display users the given user is following
	11	-a --api-rate see your current API rate limit status
fb431f15	12	-i --ids prepend user id to each line. useful to tracking renames
a7282452 S	13
	14	AUTHENTICATION
	15	Authenticate to Twitter using OAuth to see following/followers of private
	16	profiles and have higher API rate limits. OAuth authentication tokens
	17	are stored in the file .twitter-follow_oauth in your home directory.
	18	"""
	19
	20	from __future__ import print_function
	21
62ec1b07	22	import os, sys, time, calendar
a7282452 S	23	from getopt import gnu_getopt as getopt, GetoptError
a7282452 S	24
62ec1b07	25	try:
	26	import urllib.request as urllib2
	27	import http.client as httplib
	28	except ImportError:
	29	import urllib2
	30	import httplib
	31
a7282452 S	32	# T-Follow (Twitter-Follow) application registered by @stalkr_
	33	CONSUMER_KEY='USRZQfvFFjB6UvZIN2Edww'
	34	CONSUMER_SECRET='AwGAaSzZa5r0TDL8RKCDtffnI9H9mooZUdOa95nw8'
	35
	36	from .api import Twitter, TwitterError
	37	from .oauth import OAuth, read_token_file
	38	from .oauth_dance import oauth_dance
	39	from .auth import NoAuth
	40	from .util import Fail, err
	41
62ec1b07	42
a7282452 S	43	def parse_args(args, options):
a7282452 S	44	"""Parse arguments from command-line to set options."""
fb431f15 JS	45	long_opts = ['help', 'oauth', 'followers', 'following', 'api-rate', 'ids']
fb431f15 JS	46	short_opts = "horgai"
a7282452 S	47	opts, extra_args = getopt(args, short_opts, long_opts)
	48
	49	for opt, arg in opts:
	50	if opt in ('-h', '--help'):
	51	print(__doc__)
	52	raise SystemExit(1)
	53	elif opt in ('-o', '--oauth'):
	54	options['oauth'] = True
	55	elif opt in ('-r', '--followers'):
	56	options['followers'] = True
	57	elif opt in ('-g', '--following'):
	58	options['followers'] = False
	59	elif opt in ('-a', '--api-rate'):
	60	options['api-rate' ] = True
fb431f15 JS	61	elif opt in ('-i', '--ids'):
fb431f15 JS	62	options['show_id'] = True
a7282452 S	63
	64	options['extra_args'] = extra_args
	65
	66	def lookup_portion(twitter, user_ids):
	67	"""Resolve a limited list of user ids to screen names."""
	68	users = {}
	69	kwargs = dict(user_id=",".join(map(str, user_ids)), skip_status=1)
	70	for u in twitter.users.lookup(**kwargs):
	71	users[int(u['id'])] = u['screen_name']
	72	return users
	73
	74	def lookup(twitter, user_ids):
	75	"""Resolve an entire list of user ids to screen names."""
	76	users = {}
	77	api_limit = 100
	78	for i in range(0, len(user_ids), api_limit):
	79	fail = Fail()
	80	while True:
	81	try:
	82	portion = lookup_portion(twitter, user_ids[i:][:api_limit])
	83	except TwitterError as e:
780044e6	84	if e.e.code == 429:
a7282452	85	err("Fail: %i API rate limit exceeded" % e.e.code)
780044e6 EB	86	rls = twitter.application.rate_limit_status()
780044e6 EB	87	reset = rls.rate_limit_reset
a7282452	88	reset = time.asctime(time.localtime(reset))
780044e6	89	delay = int(rls.rate_limit_reset
a7282452	90	- time.time()) + 5 # avoid race
780044e6	91	err("Interval limit of %i requests reached, next reset on "
a7282452	92	"%s: going to sleep for %i secs"
780044e6	93	% (rls.rate_limit_limit, reset, delay))
a7282452 S	94	fail.wait(delay)
	95	continue
	96	elif e.e.code == 502:
	97	err("Fail: %i Service currently unavailable, retrying..."
	98	% e.e.code)
	99	else:
	100	err("Fail: %s\nRetrying..." % str(e)[:500])
	101	fail.wait(3)
	102	except urllib2.URLError as e:
	103	err("Fail: urllib2.URLError %s - Retrying..." % str(e))
	104	fail.wait(3)
	105	except httplib.error as e:
	106	err("Fail: httplib.error %s - Retrying..." % str(e))
	107	fail.wait(3)
	108	except KeyError as e:
	109	err("Fail: KeyError %s - Retrying..." % str(e))
	110	fail.wait(3)
	111	else:
	112	users.update(portion)
	113	err("Resolving user ids to screen names: %i/%i"
	114	% (len(users), len(user_ids)))
	115	break
	116	return users
	117
	118	def follow_portion(twitter, screen_name, cursor=-1, followers=True):
	119	"""Get a portion of followers/following for a user."""
	120	kwargs = dict(screen_name=screen_name, cursor=cursor)
	121	if followers:
	122	t = twitter.followers.ids(**kwargs)
	123	else: # following
	124	t = twitter.friends.ids(**kwargs)
	125	return t['ids'], t['next_cursor']
	126
	127	def follow(twitter, screen_name, followers=True):
	128	"""Get the entire list of followers/following for a user."""
	129	user_ids = []
	130	cursor = -1
	131	fail = Fail()
	132	while True:
	133	try:
	134	portion, cursor = follow_portion(twitter, screen_name, cursor,
	135	followers)
	136	except TwitterError as e:
	137	if e.e.code == 401:
	138	reason = ("follow%s of that user are protected"
	139	% ("ers" if followers else "ing"))
	140	err("Fail: %i Unauthorized (%s)" % (e.e.code, reason))
	141	break
780044e6	142	elif e.e.code == 429:
a7282452	143	err("Fail: %i API rate limit exceeded" % e.e.code)
780044e6 EB	144	rls = twitter.application.rate_limit_status()
780044e6 EB	145	reset = rls.rate_limit_reset
a7282452	146	reset = time.asctime(time.localtime(reset))
780044e6	147	delay = int(rls.rate_limit_reset
a7282452	148	- time.time()) + 5 # avoid race
780044e6 EB	149	err("Interval limit of %i requests reached, next reset on %s: "
780044e6 EB	150	"going to sleep for %i secs" % (rls.rate_limit_limit,
a7282452 S	151	reset, delay))
	152	fail.wait(delay)
	153	continue
	154	elif e.e.code == 502:
	155	err("Fail: %i Service currently unavailable, retrying..."
	156	% e.e.code)
	157	else:
	158	err("Fail: %s\nRetrying..." % str(e)[:500])
	159	fail.wait(3)
	160	except urllib2.URLError as e:
	161	err("Fail: urllib2.URLError %s - Retrying..." % str(e))
	162	fail.wait(3)
	163	except httplib.error as e:
	164	err("Fail: httplib.error %s - Retrying..." % str(e))
	165	fail.wait(3)
	166	except KeyError as e:
	167	err("Fail: KeyError %s - Retrying..." % str(e))
	168	fail.wait(3)
	169	else:
	170	new = -len(user_ids)
	171	user_ids = list(set(user_ids + portion))
	172	new += len(user_ids)
	173	what = "follow%s" % ("ers" if followers else "ing")
	174	err("Browsing %s %s, new: %i" % (screen_name, what, new))
	175	if cursor == 0:
	176	break
	177	fail = Fail()
	178	return user_ids
	179
	180
	181	def rate_limit_status(twitter):
	182	"""Print current Twitter API rate limit status."""
780044e6 EB	183	rls = twitter.application.rate_limit_status()
	184	print("Remaining API requests: %i/%i (interval limit)"
	185	% (rls.rate_limit_remaining, rls.rate_limit_limit))
a7282452	186	print("Next reset in %is (%s)"
780044e6 EB	187	% (int(rls.rate_limit_reset - time.time()),
780044e6 EB	188	time.asctime(time.localtime(rls.rate_limit_reset))))
a7282452 S	189
	190	def main(args=sys.argv[1:]):
	191	options = {
	192	'oauth': False,
	193	'followers': True,
fb431f15 JS	194	'api-rate': False,
fb431f15 JS	195	'show_id': False
a7282452 S	196	}
	197	try:
	198	parse_args(args, options)
	199	except GetoptError as e:
	200	err("I can't do that, %s." % e)
	201	raise SystemExit(1)
	202
	203	# exit if no user or given, except if asking for API rate
	204	if not options['extra_args'] and not options['api-rate']:
	205	print(__doc__)
	206	raise SystemExit(1)
	207
	208	# authenticate using OAuth, asking for token if necessary
	209	if options['oauth']:
	210	oauth_filename = (os.getenv("HOME", "") + os.sep
	211	+ ".twitter-follow_oauth")
	212	if not os.path.exists(oauth_filename):
	213	oauth_dance("Twitter-Follow", CONSUMER_KEY, CONSUMER_SECRET,
	214	oauth_filename)
	215	oauth_token, oauth_token_secret = read_token_file(oauth_filename)
	216	auth = OAuth(oauth_token, oauth_token_secret, CONSUMER_KEY,
	217	CONSUMER_SECRET)
	218	else:
	219	auth = NoAuth()
	220
7a5610b1	221	twitter = Twitter(auth=auth, api_version='1.1', domain='api.twitter.com')
a7282452 S	222
	223	if options['api-rate']:
	224	rate_limit_status(twitter)
	225	return
	226
	227	# obtain list of followers (or following) for every given user
	228	for user in options['extra_args']:
	229	user_ids, users = [], {}
	230	try:
	231	user_ids = follow(twitter, user, options['followers'])
	232	users = lookup(twitter, user_ids)
	233	except KeyboardInterrupt as e:
	234	err()
	235	err("Interrupted.")
	236	raise SystemExit(1)
	237
	238	for uid in user_ids:
fb431f15	239	if options['show_id']:
42da6c2b D	240	try:
	241	print(str(uid) + "\t" + users[uid].encode("utf-8"))
	242	except KeyError:
	243	pass
9580a9df	244
fb431f15	245	else:
42da6c2b D	246	try:
	247	print(users[uid].encode("utf-8"))
	248	except KeyError:
	249	pass
a7282452 S	250
	251	# print total on stderr to separate from user list on stdout
	252	if options['followers']:
	253	err("Total followers for %s: %i" % (user, len(user_ids)))
	254	else:
	255	err("Total users %s is following: %i" % (user, len(user_ids)))