twitter/api.py

   1 # encoding: utf-8
   2 from __future__ import unicode_literals, print_function
   3
   4 try:
   5     import urllib.request as urllib_request
   6     import urllib.error as urllib_error
   7 except ImportError:
   8     import urllib2 as urllib_request
   9     import urllib2 as urllib_error
  10
  11 try:
  12     from cStringIO import StringIO
  13 except ImportError:
  14     from io import BytesIO as StringIO
  15
  16 from .twitter_globals import POST_ACTIONS
  17 from .auth import NoAuth
  18
  19 import re
  20 import sys
  21 import gzip
  22 from time import sleep, time
  23
  24 try:
  25     import http.client as http_client
  26 except ImportError:
  27     import httplib as http_client
  28
  29 try:
  30     import json
  31 except ImportError:
  32     import simplejson as json
  33
  34
  35 class _DEFAULT(object):
  36     pass
  37
  38
  39 class TwitterError(Exception):
  40     """
  41     Base Exception thrown by the Twitter object when there is a
  42     general error interacting with the API.
  43     """
  44     pass
  45
  46
  47 class TwitterHTTPError(TwitterError):
  48     """
  49     Exception thrown by the Twitter object when there is an
  50     HTTP error interacting with twitter.com.
  51     """
  52     def __init__(self, e, uri, format, uriparts):
  53         self.e = e
  54         self.uri = uri
  55         self.format = format
  56         self.uriparts = uriparts
  57         try:
  58             data = self.e.fp.read()
  59         except http_client.IncompleteRead as e:
  60             # can't read the error text
  61             # let's try some of it
  62             data = e.partial
  63         if self.e.headers.get('Content-Encoding') == 'gzip':
  64             buf = StringIO(data)
  65             f = gzip.GzipFile(fileobj=buf)
  66             self.response_data = f.read()
  67         else:
  68             self.response_data = data
  69         super(TwitterHTTPError, self).__init__(str(self))
  70
  71     def __str__(self):
  72         fmt = ("." + self.format) if self.format else ""
  73         return (
  74             "Twitter sent status %i for URL: %s%s using parameters: "
  75             "(%s)\ndetails: %s" % (
  76                 self.e.code, self.uri, fmt, self.uriparts,
  77                 self.response_data))
  78
  79
  80 class TwitterResponse(object):
  81     """
  82     Response from a twitter request. Behaves like a list or a string
  83     (depending on requested format) but it has a few other interesting
  84     attributes.
  85
  86     `headers` gives you access to the response headers as an
  87     httplib.HTTPHeaders instance. You can do
  88     `response.headers.get('h')` to retrieve a header.
  89     """
  90
  91     @property
  92     def rate_limit_remaining(self):
  93         """
  94         Remaining requests in the current rate-limit.
  95         """
  96         return int(self.headers.get('X-Rate-Limit-Remaining', "0"))
  97
  98     @property
  99     def rate_limit_limit(self):
 100         """
 101         The rate limit ceiling for that given request.
 102         """
 103         return int(self.headers.get('X-Rate-Limit-Limit', "0"))
 104
 105     @property
 106     def rate_limit_reset(self):
 107         """
 108         Time in UTC epoch seconds when the rate limit will reset.
 109         """
 110         return int(self.headers.get('X-Rate-Limit-Reset', "0"))
 111
 112
 113 class TwitterDictResponse(dict, TwitterResponse):
 114     pass
 115
 116
 117 class TwitterListResponse(list, TwitterResponse):
 118     pass
 119
 120
 121 def wrap_response(response, headers):
 122     response_typ = type(response)
 123     if response_typ is dict:
 124         res = TwitterDictResponse(response)
 125         res.headers = headers
 126     elif response_typ is list:
 127         res = TwitterListResponse(response)
 128         res.headers = headers
 129     else:
 130         res = response
 131     return res
 132
 133 def method_for_uri(uri):
 134     method = "GET"
 135     for action in POST_ACTIONS:
 136         if re.search("%s(/\d+)?$" % action, uri):
 137             method = "POST"
 138             break
 139     return method
 140
 141 class TwitterCall(object):
 142
 143     TWITTER_UNAVAILABLE_WAIT = 30  # delay after HTTP codes 502, 503 or 504
 144
 145     def __init__(
 146             self, auth, format, domain, callable_cls, uri="",
 147             uriparts=None, secure=True, timeout=None, gzip=False, retry=False):
 148         self.auth = auth
 149         self.format = format
 150         self.domain = domain
 151         self.callable_cls = callable_cls
 152         self.uri = uri
 153         self.uriparts = uriparts
 154         self.secure = secure
 155         self.timeout = timeout
 156         self.gzip = gzip
 157         self.retry = retry
 158
 159     def __getattr__(self, k):
 160         try:
 161             return object.__getattr__(self, k)
 162         except AttributeError:
 163             def extend_call(arg):
 164                 return self.callable_cls(
 165                     auth=self.auth, format=self.format, domain=self.domain,
 166                     callable_cls=self.callable_cls, timeout=self.timeout,
 167                     secure=self.secure, gzip=self.gzip, retry=self.retry,
 168                     uriparts=self.uriparts + (arg,))
 169             if k == "_":
 170                 return extend_call
 171             else:
 172                 return extend_call(k)
 173
 174     def __call__(self, **kwargs):
 175         # Build the uri.
 176         uriparts = []
 177         for uripart in self.uriparts:
 178             # If this part matches a keyword argument, use the
 179             # supplied value otherwise, just use the part.
 180             uriparts.append(str(kwargs.pop(uripart, uripart)))
 181         uri = '/'.join(uriparts)
 182
 183         method = kwargs.pop('_method', None) or method_for_uri(uri)
 184
 185         # If an id kwarg is present and there is no id to fill in in
 186         # the list of uriparts, assume the id goes at the end.
 187         id = kwargs.pop('id', None)
 188         if id:
 189             uri += "/%s" % (id)
 190
 191         # If an _id kwarg is present, this is treated as id as a CGI
 192         # param.
 193         _id = kwargs.pop('_id', None)
 194         if _id:
 195             kwargs['id'] = _id
 196
 197         # If an _timeout is specified in kwargs, use it
 198         _timeout = kwargs.pop('_timeout', None)
 199
 200         secure_str = ''
 201         if self.secure:
 202             secure_str = 's'
 203         dot = ""
 204         if self.format:
 205             dot = "."
 206         uriBase = "http%s://%s/%s%s%s" % (
 207             secure_str, self.domain, uri, dot, self.format)
 208
 209         # Check if argument tells whether img is already base64 encoded
 210         b64_convert = True
 211         if "_base64" in kwargs:
 212             b64_convert = not kwargs.pop("_base64")
 213         if b64_convert:
 214             import base64
 215
 216         # Catch media arguments to handle oauth query differently for multipart
 217         media = None
 218         for arg in ['media[]']:
 219             if arg in kwargs:
 220                 media = kwargs.pop(arg)
 221                 if b64_convert:
 222                     media = base64.b64encode(media)
 223                 if sys.version_info >= (3, 0):
 224                     media = str(media, 'utf8')
 225                 mediafield = arg
 226                 break
 227
 228         # Catch media arguments that are not accepted through multipart
 229         # and are not yet base64 encoded
 230         if b64_convert:
 231             for arg in ['banner', 'image']:
 232                 if arg in kwargs:
 233                     kwargs[arg] = base64.b64encode(kwargs[arg])
 234
 235         headers = {'Accept-Encoding': 'gzip'} if self.gzip else dict()
 236         body = None
 237         arg_data = None
 238         if self.auth:
 239             headers.update(self.auth.generate_headers())
 240             # Use urlencoded oauth args with no params when sending media
 241             # via multipart and send it directly via uri even for post
 242             arg_data = self.auth.encode_params(
 243                 uriBase, method, {} if media else kwargs)
 244             if method == 'GET' or media:
 245                 uriBase += '?' + arg_data
 246             else:
 247                 body = arg_data.encode('utf8')
 248
 249         # Handle query as multipart when sending media
 250         if media:
 251             BOUNDARY = "###Python-Twitter###"
 252             bod = []
 253             bod.append('--' + BOUNDARY)
 254             bod.append(
 255                 'Content-Disposition: form-data; name="%s"' % mediafield)
 256             bod.append('Content-Transfer-Encoding: base64')
 257             bod.append('')
 258             bod.append(media)
 259             for k, v in kwargs.items():
 260                 bod.append('--' + BOUNDARY)
 261                 bod.append('Content-Disposition: form-data; name="%s"' % k)
 262                 bod.append('')
 263                 bod.append(v)
 264             bod.append('--' + BOUNDARY + '--')
 265             body = '\r\n'.join(bod).encode('utf8')
 266             headers['Content-Type'] = \
 267                 'multipart/form-data; boundary=%s' % BOUNDARY
 268
 269         req = urllib_request.Request(uriBase, body, headers)
 270         if self.retry:
 271             return self._handle_response_with_retry(req, uri, arg_data, _timeout)
 272         else:
 273             return self._handle_response(req, uri, arg_data, _timeout)
 274
 275     def _handle_response(self, req, uri, arg_data, _timeout=None):
 276         kwargs = {}
 277         if _timeout:
 278             kwargs['timeout'] = _timeout
 279         try:
 280             handle = urllib_request.urlopen(req, **kwargs)
 281             if handle.headers['Content-Type'] in ['image/jpeg', 'image/png']:
 282                 return handle
 283             try:
 284                 data = handle.read()
 285             except http_client.IncompleteRead as e:
 286                 # Even if we don't get all the bytes we should have there
 287                 # may be a complete response in e.partial
 288                 data = e.partial
 289             if handle.info().get('Content-Encoding') == 'gzip':
 290                 # Handle gzip decompression
 291                 buf = StringIO(data)
 292                 f = gzip.GzipFile(fileobj=buf)
 293                 data = f.read()
 294             if len(data) == 0:
 295                 return wrap_response({}, handle.headers)
 296             elif "json" == self.format:
 297                 res = json.loads(data.decode('utf8'))
 298                 return wrap_response(res, handle.headers)
 299             else:
 300                 return wrap_response(
 301                     data.decode('utf8'), handle.headers)
 302         except urllib_error.HTTPError as e:
 303             if (e.code == 304):
 304                 return []
 305             else:
 306                 raise TwitterHTTPError(e, uri, self.format, arg_data)
 307
 308     def _handle_response_with_retry(self, req, uri, arg_data, _timeout=None):
 309         while True:
 310             try:
 311                 return self._handle_response(req, uri, arg_data, _timeout)
 312             except TwitterHTTPError as e:
 313                 if e.e.code == 429:
 314                     # API rate limit reached
 315                     reset = int(e.e.headers.get('X-Rate-Limit-Reset', time() + 30))
 316                     delay = int(reset - time() + 2)  # add some extra margin
 317                     print("API rate limit reached; waiting for %ds..." % delay, file=sys.stderr)
 318                 elif e.e.code in (502, 503, 504):
 319                     delay = self.TWITTER_UNAVAILABLE_WAIT
 320                     print("Service unavailable; waiting for %ds..." % delay, file=sys.stderr)
 321                 else:
 322                     raise
 323                 sleep(delay)
 324
 325
 326 class Twitter(TwitterCall):
 327     """
 328     The minimalist yet fully featured Twitter API class.
 329
 330     Get RESTful data by accessing members of this class. The result
 331     is decoded python objects (lists and dicts).
 332
 333     The Twitter API is documented at:
 334
 335       http://dev.twitter.com/doc
 336
 337
 338     Examples::
 339
 340         from twitter import *
 341
 342         t = Twitter(
 343             auth=OAuth(token, token_key, con_secret, con_secret_key)))
 344
 345         # Get your "home" timeline
 346         t.statuses.home_timeline()
 347
 348         # Get a particular friend's timeline
 349         t.statuses.user_timeline(screen_name="billybob")
 350
 351         # to pass in GET/POST parameters, such as `count`
 352         t.statuses.home_timeline(count=5)
 353
 354         # to pass in the GET/POST parameter `id` you need to use `_id`
 355         t.statuses.oembed(_id=1234567890)
 356
 357         # Update your status
 358         t.statuses.update(
 359             status="Using @sixohsix's sweet Python Twitter Tools.")
 360
 361         # Send a direct message
 362         t.direct_messages.new(
 363             user="billybob",
 364             text="I think yer swell!")
 365
 366         # Get the members of tamtar's list "Things That Are Rad"
 367         t._("tamtar")._("things-that-are-rad").members()
 368
 369         # Note how the magic `_` method can be used to insert data
 370         # into the middle of a call. You can also use replacement:
 371         t.user.list.members(user="tamtar", list="things-that-are-rad")
 372
 373         # An *optional* `_timeout` parameter can also be used for API
 374         # calls which take much more time than normal or twitter stops
 375         # responding for some reason:
 376         t.users.lookup(
 377             screen_name=','.join(A_LIST_OF_100_SCREEN_NAMES), \
 378             _timeout=1)
 379
 380         # Overriding Method: GET/POST
 381         # you should not need to use this method as this library properly
 382         # detects whether GET or POST should be used, Nevertheless
 383         # to force a particular method, use `_method`
 384         t.statuses.oembed(_id=1234567890, _method='GET')
 385
 386         # Send a tweet with an image included (or set your banner or logo similarily)
 387         # by just reading your image from the web or a file in a string:
 388         with open("example.png", "rb") as imagefile:
 389             params = {"media[]": imagefile.read(), "status": "PTT"}
 390         t.statuses.update_with_media(**params)
 391
 392         # Or by sending a base64 encoded image:
 393         params = {"media[]": base64_image, "status": "PTT", "_base64": True}
 394         t.statuses.update_with_media(**params)
 395
 396
 397     Searching Twitter::
 398
 399         # Search for the latest tweets about #pycon
 400         t.search.tweets(q="#pycon")
 401
 402
 403     Using the data returned
 404     -----------------------
 405
 406     Twitter API calls return decoded JSON. This is converted into
 407     a bunch of Python lists, dicts, ints, and strings. For example::
 408
 409         x = twitter.statuses.home_timeline()
 410
 411         # The first 'tweet' in the timeline
 412         x[0]
 413
 414         # The screen name of the user who wrote the first 'tweet'
 415         x[0]['user']['screen_name']
 416
 417
 418     Getting raw XML data
 419     --------------------
 420
 421     If you prefer to get your Twitter data in XML format, pass
 422     format="xml" to the Twitter object when you instantiate it::
 423
 424         twitter = Twitter(format="xml")
 425
 426     The output will not be parsed in any way. It will be a raw string
 427     of XML.
 428
 429     """
 430     def __init__(
 431             self, format="json",
 432             domain="api.twitter.com", secure=True, auth=None,
 433             api_version=_DEFAULT, retry=False):
 434         """
 435         Create a new twitter API connector.
 436
 437         Pass an `auth` parameter to use the credentials of a specific
 438         user. Generally you'll want to pass an `OAuth`
 439         instance::
 440
 441             twitter = Twitter(auth=OAuth(
 442                     token, token_secret, consumer_key, consumer_secret))
 443
 444
 445         `domain` lets you change the domain you are connecting. By
 446         default it's `api.twitter.com`.
 447
 448         If `secure` is False you will connect with HTTP instead of
 449         HTTPS.
 450
 451         `api_version` is used to set the base uri. By default it's
 452         '1.1'.
 453
 454         If `retry` is True, API rate limits will automatically be
 455         handled by waiting until the next reset, as indicated by
 456         the X-Rate-Limit-Reset HTTP header.
 457         """
 458         if not auth:
 459             auth = NoAuth()
 460
 461         if (format not in ("json", "xml", "")):
 462             raise ValueError("Unknown data format '%s'" % (format))
 463
 464         if api_version is _DEFAULT:
 465             api_version = '1.1'
 466
 467         uriparts = ()
 468         if api_version:
 469             uriparts += (str(api_version),)
 470
 471         TwitterCall.__init__(
 472             self, auth=auth, format=format, domain=domain,
 473             callable_cls=TwitterCall,
 474             secure=secure, uriparts=uriparts, retry=retry)
 475
 476
 477 __all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]