twitter/api.py

   1 # encoding: utf-8
   2 from __future__ import unicode_literals, print_function
   3
   4 from .util import PY_3_OR_HIGHER, actually_bytes
   5
   6 try:
   7     import urllib.request as urllib_request
   8     import urllib.error as urllib_error
   9 except ImportError:
  10     import urllib2 as urllib_request
  11     import urllib2 as urllib_error
  12
  13 try:
  14     from cStringIO import StringIO
  15 except ImportError:
  16     from io import BytesIO as StringIO
  17
  18 from .twitter_globals import POST_ACTIONS
  19 from .auth import NoAuth
  20
  21 import re
  22 import sys
  23 import gzip
  24 from time import sleep, time
  25
  26 try:
  27     import http.client as http_client
  28 except ImportError:
  29     import httplib as http_client
  30
  31 try:
  32     import json
  33 except ImportError:
  34     import simplejson as json
  35
  36
  37 class _DEFAULT(object):
  38     pass
  39
  40
  41 class TwitterError(Exception):
  42     """
  43     Base Exception thrown by the Twitter object when there is a
  44     general error interacting with the API.
  45     """
  46     pass
  47
  48
  49 class TwitterHTTPError(TwitterError):
  50     """
  51     Exception thrown by the Twitter object when there is an
  52     HTTP error interacting with twitter.com.
  53     """
  54     def __init__(self, e, uri, format, uriparts):
  55         self.e = e
  56         self.uri = uri
  57         self.format = format
  58         self.uriparts = uriparts
  59         try:
  60             data = self.e.fp.read()
  61         except http_client.IncompleteRead as e:
  62             # can't read the error text
  63             # let's try some of it
  64             data = e.partial
  65         if self.e.headers.get('Content-Encoding') == 'gzip':
  66             buf = StringIO(data)
  67             f = gzip.GzipFile(fileobj=buf)
  68             self.response_data = f.read()
  69         else:
  70             self.response_data = data
  71         super(TwitterHTTPError, self).__init__(str(self))
  72
  73     def __str__(self):
  74         fmt = ("." + self.format) if self.format else ""
  75         return (
  76             "Twitter sent status %i for URL: %s%s using parameters: "
  77             "(%s)\ndetails: %s" % (
  78                 self.e.code, self.uri, fmt, self.uriparts,
  79                 self.response_data))
  80
  81
  82 class TwitterResponse(object):
  83     """
  84     Response from a twitter request. Behaves like a list or a string
  85     (depending on requested format) but it has a few other interesting
  86     attributes.
  87
  88     `headers` gives you access to the response headers as an
  89     httplib.HTTPHeaders instance. You can do
  90     `response.headers.get('h')` to retrieve a header.
  91     """
  92
  93     @property
  94     def rate_limit_remaining(self):
  95         """
  96         Remaining requests in the current rate-limit.
  97         """
  98         return int(self.headers.get('X-Rate-Limit-Remaining', "0"))
  99
 100     @property
 101     def rate_limit_limit(self):
 102         """
 103         The rate limit ceiling for that given request.
 104         """
 105         return int(self.headers.get('X-Rate-Limit-Limit', "0"))
 106
 107     @property
 108     def rate_limit_reset(self):
 109         """
 110         Time in UTC epoch seconds when the rate limit will reset.
 111         """
 112         return int(self.headers.get('X-Rate-Limit-Reset', "0"))
 113
 114
 115 class TwitterDictResponse(dict, TwitterResponse):
 116     pass
 117
 118
 119 class TwitterListResponse(list, TwitterResponse):
 120     pass
 121
 122
 123 def wrap_response(response, headers):
 124     response_typ = type(response)
 125     if response_typ is dict:
 126         res = TwitterDictResponse(response)
 127         res.headers = headers
 128     elif response_typ is list:
 129         res = TwitterListResponse(response)
 130         res.headers = headers
 131     else:
 132         res = response
 133     return res
 134
 135
 136 POST_ACTIONS_RE = re.compile('(' + '|'.join(POST_ACTIONS) + r')(/\d+)?$')
 137
 138 def method_for_uri(uri):
 139     if POST_ACTIONS_RE.search(uri):
 140         return "POST"
 141     return "GET"
 142
 143
 144 def build_uri(orig_uriparts, kwargs):
 145     """
 146     Build the URI from the original uriparts and kwargs. Modifies kwargs.
 147     """
 148     uriparts = []
 149     for uripart in orig_uriparts:
 150         # If this part matches a keyword argument (starting with _), use
 151         # the supplied value. Otherwise, just use the part.
 152         if uripart.startswith("_"):
 153             part = (str(kwargs.pop(uripart, uripart)))
 154         else:
 155             part = uripart
 156         uriparts.append(part)
 157     uri = '/'.join(uriparts)
 158
 159     # If an id kwarg is present and there is no id to fill in in
 160     # the list of uriparts, assume the id goes at the end.
 161     id = kwargs.pop('id', None)
 162     if id:
 163         uri += "/%s" % (id)
 164
 165     return uri
 166
 167
 168 class TwitterCall(object):
 169
 170     TWITTER_UNAVAILABLE_WAIT = 30  # delay after HTTP codes 502, 503 or 504
 171
 172     def __init__(
 173             self, auth, format, domain, callable_cls, uri="",
 174             uriparts=None, secure=True, timeout=None, gzip=False, retry=False):
 175         self.auth = auth
 176         self.format = format
 177         self.domain = domain
 178         self.callable_cls = callable_cls
 179         self.uri = uri
 180         self.uriparts = uriparts
 181         self.secure = secure
 182         self.timeout = timeout
 183         self.gzip = gzip
 184         self.retry = retry
 185
 186     def __getattr__(self, k):
 187         try:
 188             return object.__getattr__(self, k)
 189         except AttributeError:
 190             def extend_call(arg):
 191                 return self.callable_cls(
 192                     auth=self.auth, format=self.format, domain=self.domain,
 193                     callable_cls=self.callable_cls, timeout=self.timeout,
 194                     secure=self.secure, gzip=self.gzip, retry=self.retry,
 195                     uriparts=self.uriparts + (arg,))
 196             if k == "_":
 197                 return extend_call
 198             else:
 199                 return extend_call(k)
 200
 201     def __call__(self, **kwargs):
 202         kwargs = dict(kwargs)
 203         uri = build_uri(self.uriparts, kwargs)
 204         method = kwargs.pop('_method', None) or method_for_uri(uri)
 205         domain = self.domain
 206
 207         # If an _id kwarg is present, this is treated as id as a CGI
 208         # param.
 209         _id = kwargs.pop('_id', None)
 210         if _id:
 211             kwargs['id'] = _id
 212
 213         # If an _timeout is specified in kwargs, use it
 214         _timeout = kwargs.pop('_timeout', None)
 215
 216         secure_str = ''
 217         if self.secure:
 218             secure_str = 's'
 219         dot = ""
 220         if self.format:
 221             dot = "."
 222         url_base = "http%s://%s/%s%s%s" % (
 223             secure_str, domain, uri, dot, self.format)
 224
 225         # Check if argument tells whether img is already base64 encoded
 226         b64_convert = not kwargs.pop("_base64", False)
 227         if b64_convert:
 228             import base64
 229
 230         # Catch media arguments to handle oauth query differently for multipart
 231         media = None
 232         if 'media' in kwargs:
 233             mediafield = 'media'
 234             media = kwargs.pop('media')
 235             media_raw = True
 236         elif 'media[]' in kwargs:
 237             mediafield = 'media[]'
 238             media = kwargs.pop('media[]')
 239             if b64_convert:
 240                 media = base64.b64encode(media)
 241             media_raw = False
 242
 243         # Catch media arguments that are not accepted through multipart
 244         # and are not yet base64 encoded
 245         if b64_convert:
 246             for arg in ['banner', 'image']:
 247                 if arg in kwargs:
 248                     kwargs[arg] = base64.b64encode(kwargs[arg])
 249
 250         headers = {'Accept-Encoding': 'gzip'} if self.gzip else dict()
 251         body = None
 252         arg_data = None
 253         if self.auth:
 254             headers.update(self.auth.generate_headers())
 255             # Use urlencoded oauth args with no params when sending media
 256             # via multipart and send it directly via uri even for post
 257             arg_data = self.auth.encode_params(
 258                 url_base, method, {} if media else kwargs)
 259             if method == 'GET' or media:
 260                 url_base += '?' + arg_data
 261             else:
 262                 body = arg_data.encode('utf-8')
 263
 264         # Handle query as multipart when sending media
 265         if media:
 266             BOUNDARY = b"###Python-Twitter###"
 267             bod = []
 268             bod.append(b'--' + BOUNDARY)
 269             bod.append(
 270                 b'Content-Disposition: form-data; name="'
 271                 + actually_bytes(mediafield)
 272                 + b'"')
 273             bod.append(b'Content-Type: application/octet-stream')
 274             if not media_raw:
 275                 bod.append(b'Content-Transfer-Encoding: base64')
 276             bod.append(b'')
 277             bod.append(actually_bytes(media))
 278             for k, v in kwargs.items():
 279                 k = actually_bytes(k)
 280                 v = actually_bytes(v)
 281                 bod.append(b'--' + BOUNDARY)
 282                 bod.append(b'Content-Disposition: form-data; name="' + k + b'"')
 283                 bod.append(b'Content-Type: text/plain;charset=utf-8')
 284                 bod.append(b'')
 285                 bod.append(v)
 286             bod.append(b'--' + BOUNDARY + b'--')
 287             bod.append(b'')
 288             bod.append(b'')
 289             body = b'\r\n'.join(bod)
 290             # print(body.decode('utf-8', errors='ignore'))
 291             headers[b'Content-Type'] = \
 292                 b'multipart/form-data; boundary=' + BOUNDARY
 293
 294             for k in headers:
 295                 headers[actually_bytes(k)] = actually_bytes(headers.pop(k))
 296             # print(headers)
 297
 298             if not PY_3_OR_HIGHER:
 299                 url_base = url_base.encode("utf-8")
 300
 301         req = urllib_request.Request(url_base, data=body, headers=headers)
 302         if self.retry:
 303             return self._handle_response_with_retry(req, uri, arg_data, _timeout)
 304         else:
 305             return self._handle_response(req, uri, arg_data, _timeout)
 306
 307     def _handle_response(self, req, uri, arg_data, _timeout=None):
 308         kwargs = {}
 309         if _timeout:
 310             kwargs['timeout'] = _timeout
 311         try:
 312             handle = urllib_request.urlopen(req, **kwargs)
 313             if handle.headers['Content-Type'] in ['image/jpeg', 'image/png']:
 314                 return handle
 315             try:
 316                 data = handle.read()
 317             except http_client.IncompleteRead as e:
 318                 # Even if we don't get all the bytes we should have there
 319                 # may be a complete response in e.partial
 320                 data = e.partial
 321             if handle.info().get('Content-Encoding') == 'gzip':
 322                 # Handle gzip decompression
 323                 buf = StringIO(data)
 324                 f = gzip.GzipFile(fileobj=buf)
 325                 data = f.read()
 326             if len(data) == 0:
 327                 return wrap_response({}, handle.headers)
 328             elif "json" == self.format:
 329                 res = json.loads(data.decode('utf8'))
 330                 return wrap_response(res, handle.headers)
 331             else:
 332                 return wrap_response(
 333                     data.decode('utf8'), handle.headers)
 334         except urllib_error.HTTPError as e:
 335             if (e.code == 304):
 336                 return []
 337             else:
 338                 raise TwitterHTTPError(e, uri, self.format, arg_data)
 339
 340     def _handle_response_with_retry(self, req, uri, arg_data, _timeout=None):
 341         retry = self.retry
 342         while retry:
 343             try:
 344                 return self._handle_response(req, uri, arg_data, _timeout)
 345             except TwitterHTTPError as e:
 346                 if e.e.code == 429:
 347                     # API rate limit reached
 348                     reset = int(e.e.headers.get('X-Rate-Limit-Reset', time() + 30))
 349                     delay = int(reset - time() + 2)  # add some extra margin
 350                     print("API rate limit reached; waiting for %ds..." % delay, file=sys.stderr)
 351                 elif e.e.code in (502, 503, 504):
 352                     delay = self.TWITTER_UNAVAILABLE_WAIT
 353                     print("Service unavailable; waiting for %ds..." % delay, file=sys.stderr)
 354                 else:
 355                     raise
 356                 if isinstance(retry, int):
 357                     if retry <= 0:
 358                         raise
 359                     retry -= 1
 360                 sleep(delay)
 361
 362
 363 class Twitter(TwitterCall):
 364     """
 365     The minimalist yet fully featured Twitter API class.
 366
 367     Get RESTful data by accessing members of this class. The result
 368     is decoded python objects (lists and dicts).
 369
 370     The Twitter API is documented at:
 371
 372       http://dev.twitter.com/doc
 373
 374
 375     Examples::
 376
 377         from twitter import *
 378
 379         t = Twitter(
 380             auth=OAuth(token, token_key, con_secret, con_secret_key))
 381
 382         # Get your "home" timeline
 383         t.statuses.home_timeline()
 384
 385         # Get a particular friend's timeline
 386         t.statuses.user_timeline(screen_name="billybob")
 387
 388         # to pass in GET/POST parameters, such as `count`
 389         t.statuses.home_timeline(count=5)
 390
 391         # to pass in the GET/POST parameter `id` you need to use `_id`
 392         t.statuses.oembed(_id=1234567890)
 393
 394         # Update your status
 395         t.statuses.update(
 396             status="Using @sixohsix's sweet Python Twitter Tools.")
 397
 398         # Send a direct message
 399         t.direct_messages.new(
 400             user="billybob",
 401             text="I think yer swell!")
 402
 403         # Get the members of tamtar's list "Things That Are Rad"
 404         t._("tamtar")._("things-that-are-rad").members()
 405
 406         # Note how the magic `_` method can be used to insert data
 407         # into the middle of a call. You can also use replacement:
 408         t.user.list.members(user="tamtar", list="things-that-are-rad")
 409
 410         # An *optional* `_timeout` parameter can also be used for API
 411         # calls which take much more time than normal or twitter stops
 412         # responding for some reason:
 413         t.users.lookup(
 414             screen_name=','.join(A_LIST_OF_100_SCREEN_NAMES), \
 415             _timeout=1)
 416
 417         # Overriding Method: GET/POST
 418         # you should not need to use this method as this library properly
 419         # detects whether GET or POST should be used, Nevertheless
 420         # to force a particular method, use `_method`
 421         t.statuses.oembed(_id=1234567890, _method='GET')
 422
 423         # Send a tweet with an image included (or set your banner or logo similarily)
 424         # by just reading your image from the web or a file in a string:
 425         status = "PTT ★"
 426         with open("example.png", "rb") as imagefile:
 427             params = {"media[]": imagefile.read(), "status": status}
 428         t.statuses.update_with_media(**params)
 429
 430         # Or by sending a base64 encoded image:
 431         params = {"media[]": base64_image, "status": status, "_base64": True}
 432         t.statuses.update_with_media(**params)
 433
 434
 435     Searching Twitter::
 436
 437         # Search for the latest tweets about #pycon
 438         t.search.tweets(q="#pycon")
 439
 440
 441     Using the data returned
 442     -----------------------
 443
 444     Twitter API calls return decoded JSON. This is converted into
 445     a bunch of Python lists, dicts, ints, and strings. For example::
 446
 447         x = twitter.statuses.home_timeline()
 448
 449         # The first 'tweet' in the timeline
 450         x[0]
 451
 452         # The screen name of the user who wrote the first 'tweet'
 453         x[0]['user']['screen_name']
 454
 455
 456     Getting raw XML data
 457     --------------------
 458
 459     If you prefer to get your Twitter data in XML format, pass
 460     format="xml" to the Twitter object when you instantiate it::
 461
 462         twitter = Twitter(format="xml")
 463
 464     The output will not be parsed in any way. It will be a raw string
 465     of XML.
 466
 467     """
 468     def __init__(
 469             self, format="json",
 470             domain="api.twitter.com", secure=True, auth=None,
 471             api_version=_DEFAULT, retry=False):
 472         """
 473         Create a new twitter API connector.
 474
 475         Pass an `auth` parameter to use the credentials of a specific
 476         user. Generally you'll want to pass an `OAuth`
 477         instance::
 478
 479             twitter = Twitter(auth=OAuth(
 480                     token, token_secret, consumer_key, consumer_secret))
 481
 482
 483         `domain` lets you change the domain you are connecting. By
 484         default it's `api.twitter.com`.
 485
 486         If `secure` is False you will connect with HTTP instead of
 487         HTTPS.
 488
 489         `api_version` is used to set the base uri. By default it's
 490         '1.1'.
 491
 492         If `retry` is True, API rate limits will automatically be
 493         handled by waiting until the next reset, as indicated by
 494         the X-Rate-Limit-Reset HTTP header. If retry is an integer,
 495         it defines the number of retries attempted.
 496         """
 497         if not auth:
 498             auth = NoAuth()
 499
 500         if (format not in ("json", "xml", "")):
 501             raise ValueError("Unknown data format '%s'" % (format))
 502
 503         if api_version is _DEFAULT:
 504             api_version = '1.1'
 505
 506         uriparts = ()
 507         if api_version:
 508             uriparts += (str(api_version),)
 509
 510         TwitterCall.__init__(
 511             self, auth=auth, format=format, domain=domain,
 512             callable_cls=TwitterCall,
 513             secure=secure, uriparts=uriparts, retry=retry)
 514
 515
 516 __all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]