twitter/api.py

   1 # encoding: utf-8
   2 from __future__ import unicode_literals, print_function
   3
   4 from .util import PY_3_OR_HIGHER, actually_bytes
   5
   6 try:
   7     import urllib.request as urllib_request
   8     import urllib.error as urllib_error
   9 except ImportError:
  10     import urllib2 as urllib_request
  11     import urllib2 as urllib_error
  12
  13 try:
  14     from cStringIO import StringIO
  15 except ImportError:
  16     from io import BytesIO as StringIO
  17
  18 from .twitter_globals import POST_ACTIONS
  19 from .auth import NoAuth
  20
  21 import re
  22 import sys
  23 import gzip
  24 from time import sleep, time
  25
  26 try:
  27     import http.client as http_client
  28 except ImportError:
  29     import httplib as http_client
  30
  31 try:
  32     import json
  33 except ImportError:
  34     import simplejson as json
  35
  36
  37 class _DEFAULT(object):
  38     pass
  39
  40
  41 class TwitterError(Exception):
  42     """
  43     Base Exception thrown by the Twitter object when there is a
  44     general error interacting with the API.
  45     """
  46     pass
  47
  48
  49 class TwitterHTTPError(TwitterError):
  50     """
  51     Exception thrown by the Twitter object when there is an
  52     HTTP error interacting with twitter.com.
  53     """
  54     def __init__(self, e, uri, format, uriparts):
  55         self.e = e
  56         self.uri = uri
  57         self.format = format
  58         self.uriparts = uriparts
  59         try:
  60             data = self.e.fp.read()
  61         except http_client.IncompleteRead as e:
  62             # can't read the error text
  63             # let's try some of it
  64             data = e.partial
  65         if self.e.headers.get('Content-Encoding') == 'gzip':
  66             buf = StringIO(data)
  67             f = gzip.GzipFile(fileobj=buf)
  68             self.response_data = f.read()
  69         else:
  70             self.response_data = data
  71         super(TwitterHTTPError, self).__init__(str(self))
  72
  73     def __str__(self):
  74         fmt = ("." + self.format) if self.format else ""
  75         return (
  76             "Twitter sent status %i for URL: %s%s using parameters: "
  77             "(%s)\ndetails: %s" % (
  78                 self.e.code, self.uri, fmt, self.uriparts,
  79                 self.response_data))
  80
  81
  82 class TwitterResponse(object):
  83     """
  84     Response from a twitter request. Behaves like a list or a string
  85     (depending on requested format) but it has a few other interesting
  86     attributes.
  87
  88     `headers` gives you access to the response headers as an
  89     httplib.HTTPHeaders instance. You can do
  90     `response.headers.get('h')` to retrieve a header.
  91     """
  92
  93     @property
  94     def rate_limit_remaining(self):
  95         """
  96         Remaining requests in the current rate-limit.
  97         """
  98         return int(self.headers.get('X-Rate-Limit-Remaining', "0"))
  99
 100     @property
 101     def rate_limit_limit(self):
 102         """
 103         The rate limit ceiling for that given request.
 104         """
 105         return int(self.headers.get('X-Rate-Limit-Limit', "0"))
 106
 107     @property
 108     def rate_limit_reset(self):
 109         """
 110         Time in UTC epoch seconds when the rate limit will reset.
 111         """
 112         return int(self.headers.get('X-Rate-Limit-Reset', "0"))
 113
 114
 115 class TwitterDictResponse(dict, TwitterResponse):
 116     pass
 117
 118
 119 class TwitterListResponse(list, TwitterResponse):
 120     pass
 121
 122
 123 def wrap_response(response, headers):
 124     response_typ = type(response)
 125     if response_typ is dict:
 126         res = TwitterDictResponse(response)
 127         res.headers = headers
 128     elif response_typ is list:
 129         res = TwitterListResponse(response)
 130         res.headers = headers
 131     else:
 132         res = response
 133     return res
 134
 135
 136 POST_ACTIONS_RE = re.compile('(' + '|'.join(POST_ACTIONS) + r')(/\d+)?$')
 137
 138 def method_for_uri(uri):
 139     if POST_ACTIONS_RE.search(uri):
 140         return "POST"
 141     return "GET"
 142
 143
 144 def build_uri(orig_uriparts, kwargs):
 145     """
 146     Build the URI from the original uriparts and kwargs. Modifies kwargs.
 147     """
 148     uriparts = []
 149     for uripart in orig_uriparts:
 150         # If this part matches a keyword argument (starting with _), use
 151         # the supplied value. Otherwise, just use the part.
 152         if uripart.startswith("_"):
 153             part = (str(kwargs.pop(uripart, uripart)))
 154         else:
 155             part = uripart
 156         uriparts.append(part)
 157     uri = '/'.join(uriparts)
 158
 159     # If an id kwarg is present and there is no id to fill in in
 160     # the list of uriparts, assume the id goes at the end.
 161     id = kwargs.pop('id', None)
 162     if id:
 163         uri += "/%s" % (id)
 164
 165     return uri
 166
 167
 168 class TwitterCall(object):
 169
 170     TWITTER_UNAVAILABLE_WAIT = 30  # delay after HTTP codes 502, 503 or 504
 171
 172     def __init__(
 173             self, auth, format, domain, callable_cls, uri="",
 174             uriparts=None, secure=True, timeout=None, gzip=False, retry=False):
 175         self.auth = auth
 176         self.format = format
 177         self.domain = domain
 178         self.callable_cls = callable_cls
 179         self.uri = uri
 180         self.uriparts = uriparts
 181         self.secure = secure
 182         self.timeout = timeout
 183         self.gzip = gzip
 184         self.retry = retry
 185
 186     def __getattr__(self, k):
 187         try:
 188             return object.__getattr__(self, k)
 189         except AttributeError:
 190             def extend_call(arg):
 191                 return self.callable_cls(
 192                     auth=self.auth, format=self.format, domain=self.domain,
 193                     callable_cls=self.callable_cls, timeout=self.timeout,
 194                     secure=self.secure, gzip=self.gzip, retry=self.retry,
 195                     uriparts=self.uriparts + (arg,))
 196             if k == "_":
 197                 return extend_call
 198             else:
 199                 return extend_call(k)
 200
 201     def __call__(self, **kwargs):
 202         kwargs = dict(kwargs)
 203         uri = build_uri(self.uriparts, kwargs)
 204         method = kwargs.pop('_method', None) or method_for_uri(uri)
 205         domain = self.domain
 206
 207         # If an _id kwarg is present, this is treated as id as a CGI
 208         # param.
 209         _id = kwargs.pop('_id', None)
 210         if _id:
 211             kwargs['id'] = _id
 212
 213         # If an _timeout is specified in kwargs, use it
 214         _timeout = kwargs.pop('_timeout', None)
 215
 216         secure_str = ''
 217         if self.secure:
 218             secure_str = 's'
 219         dot = ""
 220         if self.format:
 221             dot = "."
 222         url_base = "http%s://%s/%s%s%s" % (
 223             secure_str, domain, uri, dot, self.format)
 224
 225         # Check if argument tells whether img is already base64 encoded
 226         b64_convert = not kwargs.pop("_base64", False)
 227         if b64_convert:
 228             import base64
 229
 230         # Catch media arguments to handle oauth query differently for multipart
 231         media = None
 232         if 'media' in kwargs:
 233             mediafield = 'media'
 234             media = kwargs.pop('media')
 235             media_raw = True
 236         elif 'media[]' in kwargs:
 237             mediafield = 'media[]'
 238             media = kwargs.pop('media[]')
 239             if b64_convert:
 240                 media = base64.b64encode(media)
 241             media_raw = False
 242
 243         # Catch media arguments that are not accepted through multipart
 244         # and are not yet base64 encoded
 245         if b64_convert:
 246             for arg in ['banner', 'image']:
 247                 if arg in kwargs:
 248                     kwargs[arg] = base64.b64encode(kwargs[arg])
 249
 250         headers = {'Accept-Encoding': 'gzip'} if self.gzip else dict()
 251         body = None
 252         arg_data = None
 253         if self.auth:
 254             headers.update(self.auth.generate_headers())
 255             # Use urlencoded oauth args with no params when sending media
 256             # via multipart and send it directly via uri even for post
 257             arg_data = self.auth.encode_params(
 258                 url_base, method, {} if media else kwargs)
 259             if method == 'GET' or media:
 260                 url_base += '?' + arg_data
 261             else:
 262                 body = arg_data.encode('utf-8')
 263
 264         # Handle query as multipart when sending media
 265         if media:
 266             BOUNDARY = b"###Python-Twitter###"
 267             bod = []
 268             bod.append(b'--' + BOUNDARY)
 269             bod.append(
 270                 b'Content-Disposition: form-data; name="'
 271                 + actually_bytes(mediafield)
 272                 + b'"')
 273             bod.append(b'Content-Type: application/octet-stream')
 274             if not media_raw:
 275                 bod.append(b'Content-Transfer-Encoding: base64')
 276             bod.append(b'')
 277             bod.append(actually_bytes(media))
 278             for k, v in kwargs.items():
 279                 k = actually_bytes(k)
 280                 v = actually_bytes(v)
 281                 bod.append(b'--' + BOUNDARY)
 282                 bod.append(b'Content-Disposition: form-data; name="' + k + b'"')
 283                 bod.append(b'Content-Type: text/plain;charset=utf-8')
 284                 bod.append(b'')
 285                 bod.append(v)
 286             bod.append(b'--' + BOUNDARY + b'--')
 287             bod.append(b'')
 288             bod.append(b'')
 289             body = b'\r\n'.join(bod)
 290             # print(body.decode('utf-8', errors='ignore'))
 291             headers['Content-Type'] = \
 292                 b'multipart/form-data; boundary=' + BOUNDARY
 293
 294             if not PY_3_OR_HIGHER:
 295                 url_base = url_base.encode("utf-8")
 296                 for k in headers:
 297                     headers[actually_bytes(k)] = actually_bytes(headers.pop(k))
 298
 299         req = urllib_request.Request(url_base, data=body, headers=headers)
 300         if self.retry:
 301             return self._handle_response_with_retry(req, uri, arg_data, _timeout)
 302         else:
 303             return self._handle_response(req, uri, arg_data, _timeout)
 304
 305     def _handle_response(self, req, uri, arg_data, _timeout=None):
 306         kwargs = {}
 307         if _timeout:
 308             kwargs['timeout'] = _timeout
 309         try:
 310             handle = urllib_request.urlopen(req, **kwargs)
 311             if handle.headers['Content-Type'] in ['image/jpeg', 'image/png']:
 312                 return handle
 313             try:
 314                 data = handle.read()
 315             except http_client.IncompleteRead as e:
 316                 # Even if we don't get all the bytes we should have there
 317                 # may be a complete response in e.partial
 318                 data = e.partial
 319             if handle.info().get('Content-Encoding') == 'gzip':
 320                 # Handle gzip decompression
 321                 buf = StringIO(data)
 322                 f = gzip.GzipFile(fileobj=buf)
 323                 data = f.read()
 324             if len(data) == 0:
 325                 return wrap_response({}, handle.headers)
 326             elif "json" == self.format:
 327                 res = json.loads(data.decode('utf8'))
 328                 return wrap_response(res, handle.headers)
 329             else:
 330                 return wrap_response(
 331                     data.decode('utf8'), handle.headers)
 332         except urllib_error.HTTPError as e:
 333             if (e.code == 304):
 334                 return []
 335             else:
 336                 raise TwitterHTTPError(e, uri, self.format, arg_data)
 337
 338     def _handle_response_with_retry(self, req, uri, arg_data, _timeout=None):
 339         retry = self.retry
 340         while retry:
 341             try:
 342                 return self._handle_response(req, uri, arg_data, _timeout)
 343             except TwitterHTTPError as e:
 344                 if e.e.code == 429:
 345                     # API rate limit reached
 346                     reset = int(e.e.headers.get('X-Rate-Limit-Reset', time() + 30))
 347                     delay = int(reset - time() + 2)  # add some extra margin
 348                     print("API rate limit reached; waiting for %ds..." % delay, file=sys.stderr)
 349                 elif e.e.code in (502, 503, 504):
 350                     delay = self.TWITTER_UNAVAILABLE_WAIT
 351                     print("Service unavailable; waiting for %ds..." % delay, file=sys.stderr)
 352                 else:
 353                     raise
 354                 if isinstance(retry, int) and not isinstance(retry, bool):
 355                     if retry <= 0:
 356                         raise
 357                     retry -= 1
 358                 sleep(delay)
 359
 360
 361 class Twitter(TwitterCall):
 362     """
 363     The minimalist yet fully featured Twitter API class.
 364
 365     Get RESTful data by accessing members of this class. The result
 366     is decoded python objects (lists and dicts).
 367
 368     The Twitter API is documented at:
 369
 370       http://dev.twitter.com/doc
 371
 372
 373     Examples::
 374
 375         from twitter import *
 376
 377         t = Twitter(
 378             auth=OAuth(token, token_key, con_secret, con_secret_key))
 379
 380         # Get your "home" timeline
 381         t.statuses.home_timeline()
 382
 383         # Get a particular friend's timeline
 384         t.statuses.user_timeline(screen_name="billybob")
 385
 386         # to pass in GET/POST parameters, such as `count`
 387         t.statuses.home_timeline(count=5)
 388
 389         # to pass in the GET/POST parameter `id` you need to use `_id`
 390         t.statuses.oembed(_id=1234567890)
 391
 392         # Update your status
 393         t.statuses.update(
 394             status="Using @sixohsix's sweet Python Twitter Tools.")
 395
 396         # Send a direct message
 397         t.direct_messages.new(
 398             user="billybob",
 399             text="I think yer swell!")
 400
 401         # Get the members of tamtar's list "Things That Are Rad"
 402         t._("tamtar")._("things-that-are-rad").members()
 403
 404         # Note how the magic `_` method can be used to insert data
 405         # into the middle of a call. You can also use replacement:
 406         t.user.list.members(user="tamtar", list="things-that-are-rad")
 407
 408         # An *optional* `_timeout` parameter can also be used for API
 409         # calls which take much more time than normal or twitter stops
 410         # responding for some reason:
 411         t.users.lookup(
 412             screen_name=','.join(A_LIST_OF_100_SCREEN_NAMES), \
 413             _timeout=1)
 414
 415         # Overriding Method: GET/POST
 416         # you should not need to use this method as this library properly
 417         # detects whether GET or POST should be used, Nevertheless
 418         # to force a particular method, use `_method`
 419         t.statuses.oembed(_id=1234567890, _method='GET')
 420
 421         # Send a tweet with an image included (or set your banner or logo similarily)
 422         # by just reading your image from the web or a file in a string:
 423         status = "PTT ★"
 424         with open("example.png", "rb") as imagefile:
 425             params = {"media[]": imagefile.read(), "status": status}
 426         t.statuses.update_with_media(**params)
 427
 428         # Or by sending a base64 encoded image:
 429         params = {"media[]": base64_image, "status": status, "_base64": True}
 430         t.statuses.update_with_media(**params)
 431
 432
 433     Searching Twitter::
 434
 435         # Search for the latest tweets about #pycon
 436         t.search.tweets(q="#pycon")
 437
 438
 439     Using the data returned
 440     -----------------------
 441
 442     Twitter API calls return decoded JSON. This is converted into
 443     a bunch of Python lists, dicts, ints, and strings. For example::
 444
 445         x = twitter.statuses.home_timeline()
 446
 447         # The first 'tweet' in the timeline
 448         x[0]
 449
 450         # The screen name of the user who wrote the first 'tweet'
 451         x[0]['user']['screen_name']
 452
 453
 454     Getting raw XML data
 455     --------------------
 456
 457     If you prefer to get your Twitter data in XML format, pass
 458     format="xml" to the Twitter object when you instantiate it::
 459
 460         twitter = Twitter(format="xml")
 461
 462     The output will not be parsed in any way. It will be a raw string
 463     of XML.
 464
 465     """
 466     def __init__(
 467             self, format="json",
 468             domain="api.twitter.com", secure=True, auth=None,
 469             api_version=_DEFAULT, retry=False):
 470         """
 471         Create a new twitter API connector.
 472
 473         Pass an `auth` parameter to use the credentials of a specific
 474         user. Generally you'll want to pass an `OAuth`
 475         instance::
 476
 477             twitter = Twitter(auth=OAuth(
 478                     token, token_secret, consumer_key, consumer_secret))
 479
 480
 481         `domain` lets you change the domain you are connecting. By
 482         default it's `api.twitter.com`.
 483
 484         If `secure` is False you will connect with HTTP instead of
 485         HTTPS.
 486
 487         `api_version` is used to set the base uri. By default it's
 488         '1.1'.
 489
 490         If `retry` is True, API rate limits will automatically be
 491         handled by waiting until the next reset, as indicated by
 492         the X-Rate-Limit-Reset HTTP header. If retry is an integer,
 493         it defines the number of retries attempted.
 494         """
 495         if not auth:
 496             auth = NoAuth()
 497
 498         if (format not in ("json", "xml", "")):
 499             raise ValueError("Unknown data format '%s'" % (format))
 500
 501         if api_version is _DEFAULT:
 502             api_version = '1.1'
 503
 504         uriparts = ()
 505         if api_version:
 506             uriparts += (str(api_version),)
 507
 508         TwitterCall.__init__(
 509             self, auth=auth, format=format, domain=domain,
 510             callable_cls=TwitterCall,
 511             secure=secure, uriparts=uriparts, retry=retry)
 512
 513
 514 __all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]