twitter/api.py

   1 # encoding: utf-8
   2 from __future__ import unicode_literals, print_function
   3
   4 try:
   5     import urllib.request as urllib_request
   6     import urllib.error as urllib_error
   7 except ImportError:
   8     import urllib2 as urllib_request
   9     import urllib2 as urllib_error
  10
  11 try:
  12     from cStringIO import StringIO
  13 except ImportError:
  14     from io import BytesIO as StringIO
  15
  16 from .twitter_globals import POST_ACTIONS
  17 from .auth import NoAuth
  18
  19 import re
  20 import sys
  21 import gzip
  22 from time import sleep, time
  23
  24 try:
  25     import http.client as http_client
  26 except ImportError:
  27     import httplib as http_client
  28
  29 try:
  30     import json
  31 except ImportError:
  32     import simplejson as json
  33
  34
  35 class _DEFAULT(object):
  36     pass
  37
  38
  39 class TwitterError(Exception):
  40     """
  41     Base Exception thrown by the Twitter object when there is a
  42     general error interacting with the API.
  43     """
  44     pass
  45
  46
  47 class TwitterHTTPError(TwitterError):
  48     """
  49     Exception thrown by the Twitter object when there is an
  50     HTTP error interacting with twitter.com.
  51     """
  52     def __init__(self, e, uri, format, uriparts):
  53         self.e = e
  54         self.uri = uri
  55         self.format = format
  56         self.uriparts = uriparts
  57         try:
  58             data = self.e.fp.read()
  59         except http_client.IncompleteRead as e:
  60             # can't read the error text
  61             # let's try some of it
  62             data = e.partial
  63         if self.e.headers.get('Content-Encoding') == 'gzip':
  64             buf = StringIO(data)
  65             f = gzip.GzipFile(fileobj=buf)
  66             self.response_data = f.read()
  67         else:
  68             self.response_data = data
  69         super(TwitterHTTPError, self).__init__(str(self))
  70
  71     def __str__(self):
  72         fmt = ("." + self.format) if self.format else ""
  73         return (
  74             "Twitter sent status %i for URL: %s%s using parameters: "
  75             "(%s)\ndetails: %s" % (
  76                 self.e.code, self.uri, fmt, self.uriparts,
  77                 self.response_data))
  78
  79
  80 class TwitterResponse(object):
  81     """
  82     Response from a twitter request. Behaves like a list or a string
  83     (depending on requested format) but it has a few other interesting
  84     attributes.
  85
  86     `headers` gives you access to the response headers as an
  87     httplib.HTTPHeaders instance. You can do
  88     `response.headers.get('h')` to retrieve a header.
  89     """
  90
  91     @property
  92     def rate_limit_remaining(self):
  93         """
  94         Remaining requests in the current rate-limit.
  95         """
  96         return int(self.headers.get('X-Rate-Limit-Remaining', "0"))
  97
  98     @property
  99     def rate_limit_limit(self):
 100         """
 101         The rate limit ceiling for that given request.
 102         """
 103         return int(self.headers.get('X-Rate-Limit-Limit', "0"))
 104
 105     @property
 106     def rate_limit_reset(self):
 107         """
 108         Time in UTC epoch seconds when the rate limit will reset.
 109         """
 110         return int(self.headers.get('X-Rate-Limit-Reset', "0"))
 111
 112
 113 class TwitterDictResponse(dict, TwitterResponse):
 114     pass
 115
 116
 117 class TwitterListResponse(list, TwitterResponse):
 118     pass
 119
 120
 121 def wrap_response(response, headers):
 122     response_typ = type(response)
 123     if response_typ is dict:
 124         res = TwitterDictResponse(response)
 125         res.headers = headers
 126     elif response_typ is list:
 127         res = TwitterListResponse(response)
 128         res.headers = headers
 129     else:
 130         res = response
 131     return res
 132
 133 def method_for_uri(uri):
 134     method = "GET"
 135     for action in POST_ACTIONS:
 136         if re.search("%s(/\d+)?$" % action, uri):
 137             method = "POST"
 138             break
 139     return method
 140
 141 class TwitterCall(object):
 142
 143     TWITTER_UNAVAILABLE_WAIT = 30  # delay after HTTP codes 502, 503 or 504
 144
 145     def __init__(
 146             self, auth, format, domain, callable_cls, uri="",
 147             uriparts=None, secure=True, timeout=None, gzip=False, retry=False):
 148         self.auth = auth
 149         self.format = format
 150         self.domain = domain
 151         self.callable_cls = callable_cls
 152         self.uri = uri
 153         self.uriparts = uriparts
 154         self.secure = secure
 155         self.timeout = timeout
 156         self.gzip = gzip
 157         self.retry = retry
 158
 159     def __getattr__(self, k):
 160         try:
 161             return object.__getattr__(self, k)
 162         except AttributeError:
 163             def extend_call(arg):
 164                 return self.callable_cls(
 165                     auth=self.auth, format=self.format, domain=self.domain,
 166                     callable_cls=self.callable_cls, timeout=self.timeout,
 167                     secure=self.secure, gzip=self.gzip, retry=self.retry,
 168                     uriparts=self.uriparts + (arg,))
 169             if k == "_":
 170                 return extend_call
 171             else:
 172                 return extend_call(k)
 173
 174     def __call__(self, **kwargs):
 175         # Build the uri.
 176         uriparts = []
 177         for uripart in self.uriparts:
 178             # If this part matches a keyword argument, use the
 179             # supplied value otherwise, just use the part.
 180             uriparts.append(str(kwargs.pop(uripart, uripart)))
 181         uri = '/'.join(uriparts)
 182
 183         method = kwargs.pop('_method', None) or method_for_uri(uri)
 184
 185         # If an id kwarg is present and there is no id to fill in in
 186         # the list of uriparts, assume the id goes at the end.
 187         id = kwargs.pop('id', None)
 188         if id:
 189             uri += "/%s" % (id)
 190
 191         # If an _id kwarg is present, this is treated as id as a CGI
 192         # param.
 193         _id = kwargs.pop('_id', None)
 194         if _id:
 195             kwargs['id'] = _id
 196
 197         # If an _timeout is specified in kwargs, use it
 198         _timeout = kwargs.pop('_timeout', None)
 199
 200         secure_str = ''
 201         if self.secure:
 202             secure_str = 's'
 203         dot = ""
 204         if self.format:
 205             dot = "."
 206         uriBase = "http%s://%s/%s%s%s" % (
 207             secure_str, self.domain, uri, dot, self.format)
 208
 209         # Check if argument tells whether img is already base64 encoded
 210         b64_convert = True
 211         if "_base64" in kwargs:
 212             b64_convert = not kwargs.pop("_base64")
 213         if b64_convert:
 214             import base64
 215
 216         # Catch media arguments to handle oauth query differently for multipart
 217         media = None
 218         for arg in ['media[]']:
 219             if arg in kwargs:
 220                 media = kwargs.pop(arg)
 221                 if b64_convert:
 222                     media = base64.b64encode(media)
 223                 if sys.version_info >= (3, 0):
 224                     media = str(media, 'utf8')
 225                 mediafield = arg
 226                 break
 227
 228         # Catch media arguments that are not accepted through multipart
 229         # and are not yet base64 encoded
 230         if b64_convert:
 231             for arg in ['banner', 'image']:
 232                 if arg in kwargs:
 233                     kwargs[arg] = base64.b64encode(kwargs[arg])
 234
 235         headers = {'Accept-Encoding': 'gzip'} if self.gzip else dict()
 236         body = None
 237         arg_data = None
 238         if self.auth:
 239             headers.update(self.auth.generate_headers())
 240             # Use urlencoded oauth args with no params when sending media
 241             # via multipart and send it directly via uri even for post
 242             arg_data = self.auth.encode_params(
 243                 uriBase, method, {} if media else kwargs)
 244             if method == 'GET' or media:
 245                 uriBase += '?' + arg_data
 246             else:
 247                 body = arg_data.encode('utf8')
 248
 249         # Handle query as multipart when sending media
 250         if media:
 251             BOUNDARY = "###Python-Twitter###"
 252             bod = []
 253             bod.append('--' + BOUNDARY)
 254             bod.append(
 255                 'Content-Disposition: form-data; name="%s"' % mediafield)
 256             bod.append('Content-Transfer-Encoding: base64')
 257             bod.append('')
 258             bod.append(media)
 259             for k, v in kwargs.items():
 260                 bod.append('--' + BOUNDARY)
 261                 bod.append('Content-Disposition: form-data; name="%s"' % k)
 262                 bod.append('')
 263                 bod.append(v)
 264             bod.append('--' + BOUNDARY + '--')
 265             body = '\r\n'.join(bod).encode('utf8')
 266             headers['Content-Type'] = \
 267                 'multipart/form-data; boundary=%s' % BOUNDARY
 268
 269         req = urllib_request.Request(uriBase, body, headers)
 270         if self.retry:
 271             return self._handle_response_with_retry(req, uri, arg_data, _timeout)
 272         else:
 273             return self._handle_response(req, uri, arg_data, _timeout)
 274
 275     def _handle_response(self, req, uri, arg_data, _timeout=None):
 276         kwargs = {}
 277         if _timeout:
 278             kwargs['timeout'] = _timeout
 279         try:
 280             handle = urllib_request.urlopen(req, **kwargs)
 281             if handle.headers['Content-Type'] in ['image/jpeg', 'image/png']:
 282                 return handle
 283             try:
 284                 data = handle.read()
 285             except http_client.IncompleteRead as e:
 286                 # Even if we don't get all the bytes we should have there
 287                 # may be a complete response in e.partial
 288                 data = e.partial
 289             if handle.info().get('Content-Encoding') == 'gzip':
 290                 # Handle gzip decompression
 291                 buf = StringIO(data)
 292                 f = gzip.GzipFile(fileobj=buf)
 293                 data = f.read()
 294             if len(data) == 0:
 295                 return wrap_response({}, handle.headers)
 296             elif "json" == self.format:
 297                 res = json.loads(data.decode('utf8'))
 298                 return wrap_response(res, handle.headers)
 299             else:
 300                 return wrap_response(
 301                     data.decode('utf8'), handle.headers)
 302         except urllib_error.HTTPError as e:
 303             if (e.code == 304):
 304                 return []
 305             else:
 306                 raise TwitterHTTPError(e, uri, self.format, arg_data)
 307
 308     def _handle_response_with_retry(self, req, uri, arg_data, _timeout=None):
 309         retry = self.retry
 310         while retry:
 311             try:
 312                 return self._handle_response(req, uri, arg_data, _timeout)
 313             except TwitterHTTPError as e:
 314                 if e.e.code == 429:
 315                     # API rate limit reached
 316                     reset = int(e.e.headers.get('X-Rate-Limit-Reset', time() + 30))
 317                     delay = int(reset - time() + 2)  # add some extra margin
 318                     print("API rate limit reached; waiting for %ds..." % delay, file=sys.stderr)
 319                 elif e.e.code in (502, 503, 504):
 320                     delay = self.TWITTER_UNAVAILABLE_WAIT
 321                     print("Service unavailable; waiting for %ds..." % delay, file=sys.stderr)
 322                 else:
 323                     raise
 324                 if isinstance(retry, int):
 325                     if retry <= 0:
 326                         raise
 327                     retry -= 1
 328                 sleep(delay)
 329
 330
 331 class Twitter(TwitterCall):
 332     """
 333     The minimalist yet fully featured Twitter API class.
 334
 335     Get RESTful data by accessing members of this class. The result
 336     is decoded python objects (lists and dicts).
 337
 338     The Twitter API is documented at:
 339
 340       http://dev.twitter.com/doc
 341
 342
 343     Examples::
 344
 345         from twitter import *
 346
 347         t = Twitter(
 348             auth=OAuth(token, token_key, con_secret, con_secret_key)))
 349
 350         # Get your "home" timeline
 351         t.statuses.home_timeline()
 352
 353         # Get a particular friend's timeline
 354         t.statuses.user_timeline(screen_name="billybob")
 355
 356         # to pass in GET/POST parameters, such as `count`
 357         t.statuses.home_timeline(count=5)
 358
 359         # to pass in the GET/POST parameter `id` you need to use `_id`
 360         t.statuses.oembed(_id=1234567890)
 361
 362         # Update your status
 363         t.statuses.update(
 364             status="Using @sixohsix's sweet Python Twitter Tools.")
 365
 366         # Send a direct message
 367         t.direct_messages.new(
 368             user="billybob",
 369             text="I think yer swell!")
 370
 371         # Get the members of tamtar's list "Things That Are Rad"
 372         t._("tamtar")._("things-that-are-rad").members()
 373
 374         # Note how the magic `_` method can be used to insert data
 375         # into the middle of a call. You can also use replacement:
 376         t.user.list.members(user="tamtar", list="things-that-are-rad")
 377
 378         # An *optional* `_timeout` parameter can also be used for API
 379         # calls which take much more time than normal or twitter stops
 380         # responding for some reason:
 381         t.users.lookup(
 382             screen_name=','.join(A_LIST_OF_100_SCREEN_NAMES), \
 383             _timeout=1)
 384
 385         # Overriding Method: GET/POST
 386         # you should not need to use this method as this library properly
 387         # detects whether GET or POST should be used, Nevertheless
 388         # to force a particular method, use `_method`
 389         t.statuses.oembed(_id=1234567890, _method='GET')
 390
 391         # Send a tweet with an image included (or set your banner or logo similarily)
 392         # by just reading your image from the web or a file in a string:
 393         with open("example.png", "rb") as imagefile:
 394             params = {"media[]": imagefile.read(), "status": "PTT"}
 395         t.statuses.update_with_media(**params)
 396
 397         # Or by sending a base64 encoded image:
 398         params = {"media[]": base64_image, "status": "PTT", "_base64": True}
 399         t.statuses.update_with_media(**params)
 400
 401
 402     Searching Twitter::
 403
 404         # Search for the latest tweets about #pycon
 405         t.search.tweets(q="#pycon")
 406
 407
 408     Using the data returned
 409     -----------------------
 410
 411     Twitter API calls return decoded JSON. This is converted into
 412     a bunch of Python lists, dicts, ints, and strings. For example::
 413
 414         x = twitter.statuses.home_timeline()
 415
 416         # The first 'tweet' in the timeline
 417         x[0]
 418
 419         # The screen name of the user who wrote the first 'tweet'
 420         x[0]['user']['screen_name']
 421
 422
 423     Getting raw XML data
 424     --------------------
 425
 426     If you prefer to get your Twitter data in XML format, pass
 427     format="xml" to the Twitter object when you instantiate it::
 428
 429         twitter = Twitter(format="xml")
 430
 431     The output will not be parsed in any way. It will be a raw string
 432     of XML.
 433
 434     """
 435     def __init__(
 436             self, format="json",
 437             domain="api.twitter.com", secure=True, auth=None,
 438             api_version=_DEFAULT, retry=False):
 439         """
 440         Create a new twitter API connector.
 441
 442         Pass an `auth` parameter to use the credentials of a specific
 443         user. Generally you'll want to pass an `OAuth`
 444         instance::
 445
 446             twitter = Twitter(auth=OAuth(
 447                     token, token_secret, consumer_key, consumer_secret))
 448
 449
 450         `domain` lets you change the domain you are connecting. By
 451         default it's `api.twitter.com`.
 452
 453         If `secure` is False you will connect with HTTP instead of
 454         HTTPS.
 455
 456         `api_version` is used to set the base uri. By default it's
 457         '1.1'.
 458
 459         If `retry` is True, API rate limits will automatically be
 460         handled by waiting until the next reset, as indicated by
 461         the X-Rate-Limit-Reset HTTP header. If retry is an integer,
 462         it defines the number of retries attempted.
 463         """
 464         if not auth:
 465             auth = NoAuth()
 466
 467         if (format not in ("json", "xml", "")):
 468             raise ValueError("Unknown data format '%s'" % (format))
 469
 470         if api_version is _DEFAULT:
 471             api_version = '1.1'
 472
 473         uriparts = ()
 474         if api_version:
 475             uriparts += (str(api_version),)
 476
 477         TwitterCall.__init__(
 478             self, auth=auth, format=format, domain=domain,
 479             callable_cls=TwitterCall,
 480             secure=secure, uriparts=uriparts, retry=retry)
 481
 482
 483 __all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]