twitter/api.py

   1 # encoding: utf-8
   2 from __future__ import unicode_literals, print_function
   3
   4 from .util import PY_3_OR_HIGHER, actually_bytes
   5
   6 try:
   7     import urllib.request as urllib_request
   8     import urllib.error as urllib_error
   9 except ImportError:
  10     import urllib2 as urllib_request
  11     import urllib2 as urllib_error
  12
  13 try:
  14     from cStringIO import StringIO
  15 except ImportError:
  16     from io import BytesIO as StringIO
  17
  18 from .twitter_globals import POST_ACTIONS
  19 from .auth import NoAuth
  20
  21 import re
  22 import sys
  23 import gzip
  24 from time import sleep, time
  25
  26 try:
  27     import http.client as http_client
  28 except ImportError:
  29     import httplib as http_client
  30
  31 try:
  32     import json
  33 except ImportError:
  34     import simplejson as json
  35
  36
  37 class _DEFAULT(object):
  38     pass
  39
  40
  41 class TwitterError(Exception):
  42     """
  43     Base Exception thrown by the Twitter object when there is a
  44     general error interacting with the API.
  45     """
  46     pass
  47
  48
  49 class TwitterHTTPError(TwitterError):
  50     """
  51     Exception thrown by the Twitter object when there is an
  52     HTTP error interacting with twitter.com.
  53     """
  54     def __init__(self, e, uri, format, uriparts):
  55         self.e = e
  56         self.uri = uri
  57         self.format = format
  58         self.uriparts = uriparts
  59         try:
  60             data = self.e.fp.read()
  61         except http_client.IncompleteRead as e:
  62             # can't read the error text
  63             # let's try some of it
  64             data = e.partial
  65         if self.e.headers.get('Content-Encoding') == 'gzip':
  66             buf = StringIO(data)
  67             f = gzip.GzipFile(fileobj=buf)
  68             data = f.read()
  69         if len(data) == 0:
  70             data = {}
  71         else:
  72             data = data.decode('utf8')
  73             if "json" == self.format:
  74                 try:
  75                     data = json.loads(data)
  76                 except ValueError:
  77                     # We try to load the response as json as a nicety; if it fails, carry on.
  78                     pass
  79         self.response_data = data
  80         super(TwitterHTTPError, self).__init__(str(self))
  81
  82     def __str__(self):
  83         fmt = ("." + self.format) if self.format else ""
  84         return (
  85             "Twitter sent status %i for URL: %s%s using parameters: "
  86             "(%s)\ndetails: %s" % (
  87                 self.e.code, self.uri, fmt, self.uriparts,
  88                 self.response_data))
  89
  90
  91 class TwitterResponse(object):
  92     """
  93     Response from a twitter request. Behaves like a list or a string
  94     (depending on requested format) but it has a few other interesting
  95     attributes.
  96
  97     `headers` gives you access to the response headers as an
  98     httplib.HTTPHeaders instance. You can do
  99     `response.headers.get('h')` to retrieve a header.
 100     """
 101
 102     @property
 103     def rate_limit_remaining(self):
 104         """
 105         Remaining requests in the current rate-limit.
 106         """
 107         return int(self.headers.get('X-Rate-Limit-Remaining', "0"))
 108
 109     @property
 110     def rate_limit_limit(self):
 111         """
 112         The rate limit ceiling for that given request.
 113         """
 114         return int(self.headers.get('X-Rate-Limit-Limit', "0"))
 115
 116     @property
 117     def rate_limit_reset(self):
 118         """
 119         Time in UTC epoch seconds when the rate limit will reset.
 120         """
 121         return int(self.headers.get('X-Rate-Limit-Reset', "0"))
 122
 123
 124 class TwitterDictResponse(dict, TwitterResponse):
 125     pass
 126
 127
 128 class TwitterListResponse(list, TwitterResponse):
 129     pass
 130
 131
 132 def wrap_response(response, headers):
 133     response_typ = type(response)
 134     if response_typ is dict:
 135         res = TwitterDictResponse(response)
 136         res.headers = headers
 137     elif response_typ is list:
 138         res = TwitterListResponse(response)
 139         res.headers = headers
 140     else:
 141         res = response
 142     return res
 143
 144
 145 POST_ACTIONS_RE = re.compile('(' + '|'.join(POST_ACTIONS) + r')(/\d+)?$')
 146
 147 def method_for_uri(uri):
 148     if POST_ACTIONS_RE.search(uri):
 149         return "POST"
 150     return "GET"
 151
 152
 153 def build_uri(orig_uriparts, kwargs):
 154     """
 155     Build the URI from the original uriparts and kwargs. Modifies kwargs.
 156     """
 157     uriparts = []
 158     for uripart in orig_uriparts:
 159         # If this part matches a keyword argument (starting with _), use
 160         # the supplied value. Otherwise, just use the part.
 161         if uripart.startswith("_"):
 162             part = (str(kwargs.pop(uripart, uripart)))
 163         else:
 164             part = uripart
 165         uriparts.append(part)
 166     uri = '/'.join(uriparts)
 167
 168     # If an id kwarg is present and there is no id to fill in in
 169     # the list of uriparts, assume the id goes at the end.
 170     id = kwargs.pop('id', None)
 171     if id:
 172         uri += "/%s" % (id)
 173
 174     return uri
 175
 176
 177 class TwitterCall(object):
 178
 179     TWITTER_UNAVAILABLE_WAIT = 30  # delay after HTTP codes 502, 503 or 504
 180
 181     def __init__(
 182             self, auth, format, domain, callable_cls, uri="",
 183             uriparts=None, secure=True, timeout=None, gzip=False, retry=False):
 184         self.auth = auth
 185         self.format = format
 186         self.domain = domain
 187         self.callable_cls = callable_cls
 188         self.uri = uri
 189         self.uriparts = uriparts
 190         self.secure = secure
 191         self.timeout = timeout
 192         self.gzip = gzip
 193         self.retry = retry
 194
 195     def __getattr__(self, k):
 196         try:
 197             return object.__getattr__(self, k)
 198         except AttributeError:
 199             def extend_call(arg):
 200                 return self.callable_cls(
 201                     auth=self.auth, format=self.format, domain=self.domain,
 202                     callable_cls=self.callable_cls, timeout=self.timeout,
 203                     secure=self.secure, gzip=self.gzip, retry=self.retry,
 204                     uriparts=self.uriparts + (arg,))
 205             if k == "_":
 206                 return extend_call
 207             else:
 208                 return extend_call(k)
 209
 210     def __call__(self, **kwargs):
 211         kwargs = dict(kwargs)
 212         uri = build_uri(self.uriparts, kwargs)
 213         method = kwargs.pop('_method', None) or method_for_uri(uri)
 214         domain = self.domain
 215
 216         # If an _id kwarg is present, this is treated as id as a CGI
 217         # param.
 218         _id = kwargs.pop('_id', None)
 219         if _id:
 220             kwargs['id'] = _id
 221
 222         # If an _timeout is specified in kwargs, use it
 223         _timeout = kwargs.pop('_timeout', None)
 224
 225         secure_str = ''
 226         if self.secure:
 227             secure_str = 's'
 228         dot = ""
 229         if self.format:
 230             dot = "."
 231         url_base = "http%s://%s/%s%s%s" % (
 232             secure_str, domain, uri, dot, self.format)
 233
 234         # Check if argument tells whether img is already base64 encoded
 235         b64_convert = not kwargs.pop("_base64", False)
 236         if b64_convert:
 237             import base64
 238
 239         # Catch media arguments to handle oauth query differently for multipart
 240         media = None
 241         if 'media' in kwargs:
 242             mediafield = 'media'
 243             media = kwargs.pop('media')
 244             media_raw = True
 245         elif 'media[]' in kwargs:
 246             mediafield = 'media[]'
 247             media = kwargs.pop('media[]')
 248             if b64_convert:
 249                 media = base64.b64encode(media)
 250             media_raw = False
 251
 252         # Catch media arguments that are not accepted through multipart
 253         # and are not yet base64 encoded
 254         if b64_convert:
 255             for arg in ['banner', 'image']:
 256                 if arg in kwargs:
 257                     kwargs[arg] = base64.b64encode(kwargs[arg])
 258
 259         headers = {'Accept-Encoding': 'gzip'} if self.gzip else dict()
 260         body = None
 261         arg_data = None
 262         if self.auth:
 263             headers.update(self.auth.generate_headers())
 264             # Use urlencoded oauth args with no params when sending media
 265             # via multipart and send it directly via uri even for post
 266             arg_data = self.auth.encode_params(
 267                 url_base, method, {} if media else kwargs)
 268             if method == 'GET' or media:
 269                 url_base += '?' + arg_data
 270             else:
 271                 body = arg_data.encode('utf-8')
 272
 273         # Handle query as multipart when sending media
 274         if media:
 275             BOUNDARY = b"###Python-Twitter###"
 276             bod = []
 277             bod.append(b'--' + BOUNDARY)
 278             bod.append(
 279                 b'Content-Disposition: form-data; name="'
 280                 + actually_bytes(mediafield)
 281                 + b'"')
 282             bod.append(b'Content-Type: application/octet-stream')
 283             if not media_raw:
 284                 bod.append(b'Content-Transfer-Encoding: base64')
 285             bod.append(b'')
 286             bod.append(actually_bytes(media))
 287             for k, v in kwargs.items():
 288                 k = actually_bytes(k)
 289                 v = actually_bytes(v)
 290                 bod.append(b'--' + BOUNDARY)
 291                 bod.append(b'Content-Disposition: form-data; name="' + k + b'"')
 292                 bod.append(b'Content-Type: text/plain;charset=utf-8')
 293                 bod.append(b'')
 294                 bod.append(v)
 295             bod.append(b'--' + BOUNDARY + b'--')
 296             bod.append(b'')
 297             bod.append(b'')
 298             body = b'\r\n'.join(bod)
 299             # print(body.decode('utf-8', errors='ignore'))
 300             headers['Content-Type'] = \
 301                 b'multipart/form-data; boundary=' + BOUNDARY
 302
 303             if not PY_3_OR_HIGHER:
 304                 url_base = url_base.encode("utf-8")
 305                 for k in headers:
 306                     headers[actually_bytes(k)] = actually_bytes(headers.pop(k))
 307
 308         req = urllib_request.Request(url_base, data=body, headers=headers)
 309         if self.retry:
 310             return self._handle_response_with_retry(req, uri, arg_data, _timeout)
 311         else:
 312             return self._handle_response(req, uri, arg_data, _timeout)
 313
 314     def _handle_response(self, req, uri, arg_data, _timeout=None):
 315         kwargs = {}
 316         if _timeout:
 317             kwargs['timeout'] = _timeout
 318         try:
 319             handle = urllib_request.urlopen(req, **kwargs)
 320             if handle.headers['Content-Type'] in ['image/jpeg', 'image/png']:
 321                 return handle
 322             try:
 323                 data = handle.read()
 324             except http_client.IncompleteRead as e:
 325                 # Even if we don't get all the bytes we should have there
 326                 # may be a complete response in e.partial
 327                 data = e.partial
 328             if handle.info().get('Content-Encoding') == 'gzip':
 329                 # Handle gzip decompression
 330                 buf = StringIO(data)
 331                 f = gzip.GzipFile(fileobj=buf)
 332                 data = f.read()
 333             if len(data) == 0:
 334                 return wrap_response({}, handle.headers)
 335             elif "json" == self.format:
 336                 res = json.loads(data.decode('utf8'))
 337                 return wrap_response(res, handle.headers)
 338             else:
 339                 return wrap_response(
 340                     data.decode('utf8'), handle.headers)
 341         except urllib_error.HTTPError as e:
 342             if (e.code == 304):
 343                 return []
 344             else:
 345                 raise TwitterHTTPError(e, uri, self.format, arg_data)
 346
 347     def _handle_response_with_retry(self, req, uri, arg_data, _timeout=None):
 348         retry = self.retry
 349         while retry:
 350             try:
 351                 return self._handle_response(req, uri, arg_data, _timeout)
 352             except TwitterHTTPError as e:
 353                 if e.e.code == 429:
 354                     # API rate limit reached
 355                     reset = int(e.e.headers.get('X-Rate-Limit-Reset', time() + 30))
 356                     delay = int(reset - time() + 2)  # add some extra margin
 357                     print("API rate limit reached; waiting for %ds..." % delay, file=sys.stderr)
 358                 elif e.e.code in (502, 503, 504):
 359                     delay = self.TWITTER_UNAVAILABLE_WAIT
 360                     print("Service unavailable; waiting for %ds..." % delay, file=sys.stderr)
 361                 else:
 362                     raise
 363                 if isinstance(retry, int) and not isinstance(retry, bool):
 364                     if retry <= 0:
 365                         raise
 366                     retry -= 1
 367                 sleep(delay)
 368
 369
 370 class Twitter(TwitterCall):
 371     """
 372     The minimalist yet fully featured Twitter API class.
 373
 374     Get RESTful data by accessing members of this class. The result
 375     is decoded python objects (lists and dicts).
 376
 377     The Twitter API is documented at:
 378
 379       http://dev.twitter.com/doc
 380
 381
 382     Examples::
 383
 384         from twitter import *
 385
 386         t = Twitter(
 387             auth=OAuth(token, token_key, con_secret, con_secret_key))
 388
 389         # Get your "home" timeline
 390         t.statuses.home_timeline()
 391
 392         # Get a particular friend's timeline
 393         t.statuses.user_timeline(screen_name="billybob")
 394
 395         # to pass in GET/POST parameters, such as `count`
 396         t.statuses.home_timeline(count=5)
 397
 398         # to pass in the GET/POST parameter `id` you need to use `_id`
 399         t.statuses.oembed(_id=1234567890)
 400
 401         # Update your status
 402         t.statuses.update(
 403             status="Using @sixohsix's sweet Python Twitter Tools.")
 404
 405         # Send a direct message
 406         t.direct_messages.new(
 407             user="billybob",
 408             text="I think yer swell!")
 409
 410         # Get the members of tamtar's list "Things That Are Rad"
 411         t.lists.members(owner_screen_name="tamtar", slug="things-that-are-rad")
 412
 413         # An *optional* `_timeout` parameter can also be used for API
 414         # calls which take much more time than normal or twitter stops
 415         # responding for some reason:
 416         t.users.lookup(
 417             screen_name=','.join(A_LIST_OF_100_SCREEN_NAMES), \
 418             _timeout=1)
 419
 420         # Overriding Method: GET/POST
 421         # you should not need to use this method as this library properly
 422         # detects whether GET or POST should be used, Nevertheless
 423         # to force a particular method, use `_method`
 424         t.statuses.oembed(_id=1234567890, _method='GET')
 425
 426         # Send images along with your tweets:
 427         # - first just read images from the web or from files the regular way:
 428         with open("example.png", "rb") as imagefile:
 429             imagedata = imagefile.read()
 430         # - then upload medias one by one on Twitter's dedicated server
 431         #   and collect each one's id:
 432         t_up = Twitter(domain='upload.twitter.com',
 433             auth=OAuth(token, token_key, con_secret, con_secret_key))
 434         id_img1 = t_up.media.upload(media=imagedata)["media_id_string"]
 435         id_img2 = t_up.media.upload(media=imagedata)["media_id_string"]
 436
 437         # - finally send your tweet with the list of media ids:
 438         t.statuses.update(status="PTT ★", media_ids=",".join([id_img1, id_img2]))
 439
 440         # Or send a tweet with an image (or set a logo/banner similarily)
 441         # using the old deprecated method that will probably disappear some day
 442         params = {"media[]": imagedata, "status": "PTT ★"}
 443         # Or for an image encoded as base64:
 444         params = {"media[]": base64_image, "status": "PTT ★", "_base64": True}
 445         t.statuses.update_with_media(**params)
 446
 447
 448
 449     Searching Twitter::
 450
 451         # Search for the latest tweets about #pycon
 452         t.search.tweets(q="#pycon")
 453
 454
 455     Using the data returned
 456     -----------------------
 457
 458     Twitter API calls return decoded JSON. This is converted into
 459     a bunch of Python lists, dicts, ints, and strings. For example::
 460
 461         x = twitter.statuses.home_timeline()
 462
 463         # The first 'tweet' in the timeline
 464         x[0]
 465
 466         # The screen name of the user who wrote the first 'tweet'
 467         x[0]['user']['screen_name']
 468
 469
 470     Getting raw XML data
 471     --------------------
 472
 473     If you prefer to get your Twitter data in XML format, pass
 474     format="xml" to the Twitter object when you instantiate it::
 475
 476         twitter = Twitter(format="xml")
 477
 478     The output will not be parsed in any way. It will be a raw string
 479     of XML.
 480
 481     """
 482     def __init__(
 483             self, format="json",
 484             domain="api.twitter.com", secure=True, auth=None,
 485             api_version=_DEFAULT, retry=False):
 486         """
 487         Create a new twitter API connector.
 488
 489         Pass an `auth` parameter to use the credentials of a specific
 490         user. Generally you'll want to pass an `OAuth`
 491         instance::
 492
 493             twitter = Twitter(auth=OAuth(
 494                     token, token_secret, consumer_key, consumer_secret))
 495
 496
 497         `domain` lets you change the domain you are connecting. By
 498         default it's `api.twitter.com`.
 499
 500         If `secure` is False you will connect with HTTP instead of
 501         HTTPS.
 502
 503         `api_version` is used to set the base uri. By default it's
 504         '1.1'.
 505
 506         If `retry` is True, API rate limits will automatically be
 507         handled by waiting until the next reset, as indicated by
 508         the X-Rate-Limit-Reset HTTP header. If retry is an integer,
 509         it defines the number of retries attempted.
 510         """
 511         if not auth:
 512             auth = NoAuth()
 513
 514         if (format not in ("json", "xml", "")):
 515             raise ValueError("Unknown data format '%s'" % (format))
 516
 517         if api_version is _DEFAULT:
 518             api_version = '1.1'
 519
 520         uriparts = ()
 521         if api_version:
 522             uriparts += (str(api_version),)
 523
 524         TwitterCall.__init__(
 525             self, auth=auth, format=format, domain=domain,
 526             callable_cls=TwitterCall,
 527             secure=secure, uriparts=uriparts, retry=retry)
 528
 529
 530 __all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]