twitter/api.py

   1 # encoding: utf-8
   2 from __future__ import unicode_literals, print_function
   3
   4 from .util import PY_3_OR_HIGHER, actually_bytes
   5
   6 try:
   7     import urllib.request as urllib_request
   8     import urllib.error as urllib_error
   9 except ImportError:
  10     import urllib2 as urllib_request
  11     import urllib2 as urllib_error
  12
  13 try:
  14     from cStringIO import StringIO
  15 except ImportError:
  16     from io import BytesIO as StringIO
  17
  18 from .twitter_globals import POST_ACTIONS
  19 from .auth import NoAuth
  20
  21 import re
  22 import sys
  23 import gzip
  24 from time import sleep, time
  25
  26 try:
  27     import http.client as http_client
  28 except ImportError:
  29     import httplib as http_client
  30
  31 try:
  32     import json
  33 except ImportError:
  34     import simplejson as json
  35
  36
  37 class _DEFAULT(object):
  38     pass
  39
  40
  41 class TwitterError(Exception):
  42     """
  43     Base Exception thrown by the Twitter object when there is a
  44     general error interacting with the API.
  45     """
  46     pass
  47
  48
  49 class TwitterHTTPError(TwitterError):
  50     """
  51     Exception thrown by the Twitter object when there is an
  52     HTTP error interacting with twitter.com.
  53     """
  54     def __init__(self, e, uri, format, uriparts):
  55         self.e = e
  56         self.uri = uri
  57         self.format = format
  58         self.uriparts = uriparts
  59         try:
  60             data = self.e.fp.read()
  61         except http_client.IncompleteRead as e:
  62             # can't read the error text
  63             # let's try some of it
  64             data = e.partial
  65         if self.e.headers.get('Content-Encoding') == 'gzip':
  66             buf = StringIO(data)
  67             f = gzip.GzipFile(fileobj=buf)
  68             data = f.read()
  69         if len(data) == 0:
  70             data = {}
  71         elif "json" == self.format:
  72             data = json.loads(data.decode('utf8'))
  73         else:
  74             data = data.decode('utf8')
  75         self.response_data = data
  76         super(TwitterHTTPError, self).__init__(str(self))
  77
  78     def __str__(self):
  79         fmt = ("." + self.format) if self.format else ""
  80         return (
  81             "Twitter sent status %i for URL: %s%s using parameters: "
  82             "(%s)\ndetails: %s" % (
  83                 self.e.code, self.uri, fmt, self.uriparts,
  84                 self.response_data))
  85
  86
  87 class TwitterResponse(object):
  88     """
  89     Response from a twitter request. Behaves like a list or a string
  90     (depending on requested format) but it has a few other interesting
  91     attributes.
  92
  93     `headers` gives you access to the response headers as an
  94     httplib.HTTPHeaders instance. You can do
  95     `response.headers.get('h')` to retrieve a header.
  96     """
  97
  98     @property
  99     def rate_limit_remaining(self):
 100         """
 101         Remaining requests in the current rate-limit.
 102         """
 103         return int(self.headers.get('X-Rate-Limit-Remaining', "0"))
 104
 105     @property
 106     def rate_limit_limit(self):
 107         """
 108         The rate limit ceiling for that given request.
 109         """
 110         return int(self.headers.get('X-Rate-Limit-Limit', "0"))
 111
 112     @property
 113     def rate_limit_reset(self):
 114         """
 115         Time in UTC epoch seconds when the rate limit will reset.
 116         """
 117         return int(self.headers.get('X-Rate-Limit-Reset', "0"))
 118
 119
 120 class TwitterDictResponse(dict, TwitterResponse):
 121     pass
 122
 123
 124 class TwitterListResponse(list, TwitterResponse):
 125     pass
 126
 127
 128 def wrap_response(response, headers):
 129     response_typ = type(response)
 130     if response_typ is dict:
 131         res = TwitterDictResponse(response)
 132         res.headers = headers
 133     elif response_typ is list:
 134         res = TwitterListResponse(response)
 135         res.headers = headers
 136     else:
 137         res = response
 138     return res
 139
 140
 141 POST_ACTIONS_RE = re.compile('(' + '|'.join(POST_ACTIONS) + r')(/\d+)?$')
 142
 143 def method_for_uri(uri):
 144     if POST_ACTIONS_RE.search(uri):
 145         return "POST"
 146     return "GET"
 147
 148
 149 def build_uri(orig_uriparts, kwargs):
 150     """
 151     Build the URI from the original uriparts and kwargs. Modifies kwargs.
 152     """
 153     uriparts = []
 154     for uripart in orig_uriparts:
 155         # If this part matches a keyword argument (starting with _), use
 156         # the supplied value. Otherwise, just use the part.
 157         if uripart.startswith("_"):
 158             part = (str(kwargs.pop(uripart, uripart)))
 159         else:
 160             part = uripart
 161         uriparts.append(part)
 162     uri = '/'.join(uriparts)
 163
 164     # If an id kwarg is present and there is no id to fill in in
 165     # the list of uriparts, assume the id goes at the end.
 166     id = kwargs.pop('id', None)
 167     if id:
 168         uri += "/%s" % (id)
 169
 170     return uri
 171
 172
 173 class TwitterCall(object):
 174
 175     TWITTER_UNAVAILABLE_WAIT = 30  # delay after HTTP codes 502, 503 or 504
 176
 177     def __init__(
 178             self, auth, format, domain, callable_cls, uri="",
 179             uriparts=None, secure=True, timeout=None, gzip=False, retry=False):
 180         self.auth = auth
 181         self.format = format
 182         self.domain = domain
 183         self.callable_cls = callable_cls
 184         self.uri = uri
 185         self.uriparts = uriparts
 186         self.secure = secure
 187         self.timeout = timeout
 188         self.gzip = gzip
 189         self.retry = retry
 190
 191     def __getattr__(self, k):
 192         try:
 193             return object.__getattr__(self, k)
 194         except AttributeError:
 195             def extend_call(arg):
 196                 return self.callable_cls(
 197                     auth=self.auth, format=self.format, domain=self.domain,
 198                     callable_cls=self.callable_cls, timeout=self.timeout,
 199                     secure=self.secure, gzip=self.gzip, retry=self.retry,
 200                     uriparts=self.uriparts + (arg,))
 201             if k == "_":
 202                 return extend_call
 203             else:
 204                 return extend_call(k)
 205
 206     def __call__(self, **kwargs):
 207         kwargs = dict(kwargs)
 208         uri = build_uri(self.uriparts, kwargs)
 209         method = kwargs.pop('_method', None) or method_for_uri(uri)
 210         domain = self.domain
 211
 212         # If an _id kwarg is present, this is treated as id as a CGI
 213         # param.
 214         _id = kwargs.pop('_id', None)
 215         if _id:
 216             kwargs['id'] = _id
 217
 218         # If an _timeout is specified in kwargs, use it
 219         _timeout = kwargs.pop('_timeout', None)
 220
 221         secure_str = ''
 222         if self.secure:
 223             secure_str = 's'
 224         dot = ""
 225         if self.format:
 226             dot = "."
 227         url_base = "http%s://%s/%s%s%s" % (
 228             secure_str, domain, uri, dot, self.format)
 229
 230         # Check if argument tells whether img is already base64 encoded
 231         b64_convert = not kwargs.pop("_base64", False)
 232         if b64_convert:
 233             import base64
 234
 235         # Catch media arguments to handle oauth query differently for multipart
 236         media = None
 237         if 'media' in kwargs:
 238             mediafield = 'media'
 239             media = kwargs.pop('media')
 240             media_raw = True
 241         elif 'media[]' in kwargs:
 242             mediafield = 'media[]'
 243             media = kwargs.pop('media[]')
 244             if b64_convert:
 245                 media = base64.b64encode(media)
 246             media_raw = False
 247
 248         # Catch media arguments that are not accepted through multipart
 249         # and are not yet base64 encoded
 250         if b64_convert:
 251             for arg in ['banner', 'image']:
 252                 if arg in kwargs:
 253                     kwargs[arg] = base64.b64encode(kwargs[arg])
 254
 255         headers = {'Accept-Encoding': 'gzip'} if self.gzip else dict()
 256         body = None
 257         arg_data = None
 258         if self.auth:
 259             headers.update(self.auth.generate_headers())
 260             # Use urlencoded oauth args with no params when sending media
 261             # via multipart and send it directly via uri even for post
 262             arg_data = self.auth.encode_params(
 263                 url_base, method, {} if media else kwargs)
 264             if method == 'GET' or media:
 265                 url_base += '?' + arg_data
 266             else:
 267                 body = arg_data.encode('utf-8')
 268
 269         # Handle query as multipart when sending media
 270         if media:
 271             BOUNDARY = b"###Python-Twitter###"
 272             bod = []
 273             bod.append(b'--' + BOUNDARY)
 274             bod.append(
 275                 b'Content-Disposition: form-data; name="'
 276                 + actually_bytes(mediafield)
 277                 + b'"')
 278             bod.append(b'Content-Type: application/octet-stream')
 279             if not media_raw:
 280                 bod.append(b'Content-Transfer-Encoding: base64')
 281             bod.append(b'')
 282             bod.append(actually_bytes(media))
 283             for k, v in kwargs.items():
 284                 k = actually_bytes(k)
 285                 v = actually_bytes(v)
 286                 bod.append(b'--' + BOUNDARY)
 287                 bod.append(b'Content-Disposition: form-data; name="' + k + b'"')
 288                 bod.append(b'Content-Type: text/plain;charset=utf-8')
 289                 bod.append(b'')
 290                 bod.append(v)
 291             bod.append(b'--' + BOUNDARY + b'--')
 292             bod.append(b'')
 293             bod.append(b'')
 294             body = b'\r\n'.join(bod)
 295             # print(body.decode('utf-8', errors='ignore'))
 296             headers['Content-Type'] = \
 297                 b'multipart/form-data; boundary=' + BOUNDARY
 298
 299             if not PY_3_OR_HIGHER:
 300                 url_base = url_base.encode("utf-8")
 301                 for k in headers:
 302                     headers[actually_bytes(k)] = actually_bytes(headers.pop(k))
 303
 304         req = urllib_request.Request(url_base, data=body, headers=headers)
 305         if self.retry:
 306             return self._handle_response_with_retry(req, uri, arg_data, _timeout)
 307         else:
 308             return self._handle_response(req, uri, arg_data, _timeout)
 309
 310     def _handle_response(self, req, uri, arg_data, _timeout=None):
 311         kwargs = {}
 312         if _timeout:
 313             kwargs['timeout'] = _timeout
 314         try:
 315             handle = urllib_request.urlopen(req, **kwargs)
 316             if handle.headers['Content-Type'] in ['image/jpeg', 'image/png']:
 317                 return handle
 318             try:
 319                 data = handle.read()
 320             except http_client.IncompleteRead as e:
 321                 # Even if we don't get all the bytes we should have there
 322                 # may be a complete response in e.partial
 323                 data = e.partial
 324             if handle.info().get('Content-Encoding') == 'gzip':
 325                 # Handle gzip decompression
 326                 buf = StringIO(data)
 327                 f = gzip.GzipFile(fileobj=buf)
 328                 data = f.read()
 329             if len(data) == 0:
 330                 return wrap_response({}, handle.headers)
 331             elif "json" == self.format:
 332                 res = json.loads(data.decode('utf8'))
 333                 return wrap_response(res, handle.headers)
 334             else:
 335                 return wrap_response(
 336                     data.decode('utf8'), handle.headers)
 337         except urllib_error.HTTPError as e:
 338             if (e.code == 304):
 339                 return []
 340             else:
 341                 raise TwitterHTTPError(e, uri, self.format, arg_data)
 342
 343     def _handle_response_with_retry(self, req, uri, arg_data, _timeout=None):
 344         retry = self.retry
 345         while retry:
 346             try:
 347                 return self._handle_response(req, uri, arg_data, _timeout)
 348             except TwitterHTTPError as e:
 349                 if e.e.code == 429:
 350                     # API rate limit reached
 351                     reset = int(e.e.headers.get('X-Rate-Limit-Reset', time() + 30))
 352                     delay = int(reset - time() + 2)  # add some extra margin
 353                     print("API rate limit reached; waiting for %ds..." % delay, file=sys.stderr)
 354                 elif e.e.code in (502, 503, 504):
 355                     delay = self.TWITTER_UNAVAILABLE_WAIT
 356                     print("Service unavailable; waiting for %ds..." % delay, file=sys.stderr)
 357                 else:
 358                     raise
 359                 if isinstance(retry, int) and not isinstance(retry, bool):
 360                     if retry <= 0:
 361                         raise
 362                     retry -= 1
 363                 sleep(delay)
 364
 365
 366 class Twitter(TwitterCall):
 367     """
 368     The minimalist yet fully featured Twitter API class.
 369
 370     Get RESTful data by accessing members of this class. The result
 371     is decoded python objects (lists and dicts).
 372
 373     The Twitter API is documented at:
 374
 375       http://dev.twitter.com/doc
 376
 377
 378     Examples::
 379
 380         from twitter import *
 381
 382         t = Twitter(
 383             auth=OAuth(token, token_key, con_secret, con_secret_key))
 384
 385         # Get your "home" timeline
 386         t.statuses.home_timeline()
 387
 388         # Get a particular friend's timeline
 389         t.statuses.user_timeline(screen_name="billybob")
 390
 391         # to pass in GET/POST parameters, such as `count`
 392         t.statuses.home_timeline(count=5)
 393
 394         # to pass in the GET/POST parameter `id` you need to use `_id`
 395         t.statuses.oembed(_id=1234567890)
 396
 397         # Update your status
 398         t.statuses.update(
 399             status="Using @sixohsix's sweet Python Twitter Tools.")
 400
 401         # Send a direct message
 402         t.direct_messages.new(
 403             user="billybob",
 404             text="I think yer swell!")
 405
 406         # Get the members of tamtar's list "Things That Are Rad"
 407         t.lists.members(owner_screen_name="tamtar", slug="things-that-are-rad")
 408
 409         # An *optional* `_timeout` parameter can also be used for API
 410         # calls which take much more time than normal or twitter stops
 411         # responding for some reason:
 412         t.users.lookup(
 413             screen_name=','.join(A_LIST_OF_100_SCREEN_NAMES), \
 414             _timeout=1)
 415
 416         # Overriding Method: GET/POST
 417         # you should not need to use this method as this library properly
 418         # detects whether GET or POST should be used, Nevertheless
 419         # to force a particular method, use `_method`
 420         t.statuses.oembed(_id=1234567890, _method='GET')
 421
 422         # Send a tweet with an image included (or set your banner or logo similarily)
 423         # by just reading your image from the web or a file in a string:
 424         status = "PTT ★"
 425         with open("example.png", "rb") as imagefile:
 426             params = {"media[]": imagefile.read(), "status": status}
 427         t.statuses.update_with_media(**params)
 428
 429         # Or by sending a base64 encoded image:
 430         params = {"media[]": base64_image, "status": status, "_base64": True}
 431         t.statuses.update_with_media(**params)
 432
 433
 434     Searching Twitter::
 435
 436         # Search for the latest tweets about #pycon
 437         t.search.tweets(q="#pycon")
 438
 439
 440     Using the data returned
 441     -----------------------
 442
 443     Twitter API calls return decoded JSON. This is converted into
 444     a bunch of Python lists, dicts, ints, and strings. For example::
 445
 446         x = twitter.statuses.home_timeline()
 447
 448         # The first 'tweet' in the timeline
 449         x[0]
 450
 451         # The screen name of the user who wrote the first 'tweet'
 452         x[0]['user']['screen_name']
 453
 454
 455     Getting raw XML data
 456     --------------------
 457
 458     If you prefer to get your Twitter data in XML format, pass
 459     format="xml" to the Twitter object when you instantiate it::
 460
 461         twitter = Twitter(format="xml")
 462
 463     The output will not be parsed in any way. It will be a raw string
 464     of XML.
 465
 466     """
 467     def __init__(
 468             self, format="json",
 469             domain="api.twitter.com", secure=True, auth=None,
 470             api_version=_DEFAULT, retry=False):
 471         """
 472         Create a new twitter API connector.
 473
 474         Pass an `auth` parameter to use the credentials of a specific
 475         user. Generally you'll want to pass an `OAuth`
 476         instance::
 477
 478             twitter = Twitter(auth=OAuth(
 479                     token, token_secret, consumer_key, consumer_secret))
 480
 481
 482         `domain` lets you change the domain you are connecting. By
 483         default it's `api.twitter.com`.
 484
 485         If `secure` is False you will connect with HTTP instead of
 486         HTTPS.
 487
 488         `api_version` is used to set the base uri. By default it's
 489         '1.1'.
 490
 491         If `retry` is True, API rate limits will automatically be
 492         handled by waiting until the next reset, as indicated by
 493         the X-Rate-Limit-Reset HTTP header. If retry is an integer,
 494         it defines the number of retries attempted.
 495         """
 496         if not auth:
 497             auth = NoAuth()
 498
 499         if (format not in ("json", "xml", "")):
 500             raise ValueError("Unknown data format '%s'" % (format))
 501
 502         if api_version is _DEFAULT:
 503             api_version = '1.1'
 504
 505         uriparts = ()
 506         if api_version:
 507             uriparts += (str(api_version),)
 508
 509         TwitterCall.__init__(
 510             self, auth=auth, format=format, domain=domain,
 511             callable_cls=TwitterCall,
 512             secure=secure, uriparts=uriparts, retry=retry)
 513
 514
 515 __all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]