Encode json more compactly.

Optimise json by attempting in the following order:
- using C simplejson module (faster than baseline by 3x)
- using python's built in json module (faster than baseline by 2x)

if both fail use pure Python embedded simplejson (baseline).

diff --git a/bin/dependencies_b.py b/bin/dependencies_b.py
index bd41f20a53e7d53d276b7df530e2303e763de831..e5ecda5f4c9837398e9cfe757bfcde7b42b4f9f0 100644 (file)
--- a/bin/dependencies_b.py
+++ b/bin/dependencies_b.py
@@ -16,6 +16,7 @@ def check_dependencies():
   \r
   check_twisted()\r
   check_win32()\r
+  i+=check_json()\r
   i+=check_java()\r
   i+=check_hg()\r
   \r
@@ -99,7 +100,16 @@ def check_twisted():
     import twisted.words\r
   except ImportError:\r
     twisted_fail("words")\r
-\r
+    \r
+def check_json():\r
+  import qwebirc.util.qjson\r
+  if qwebirc.util.qjson.slow:\r
+    warn("simplejson module with C speedups not installed.",\r
+         "using embedded module (slower); consider installing simplejson from:",\r
+         "http://pypi.python.org/pypi/simplejson/")\r
+    return 1\r
+  return 0\r
+  \r
 if __name__ == "__main__":\r
   import dependencies\r
   dependencies.check_dependencies()\r

diff --git a/bin/mkstatic.py b/bin/mkstatic.py
index b8b368fa444404fa5a283c0984b15d72e3447d99..6d6782b06f8ec97f0cb8c05b2f44ddf11340411c 100755 (executable)
--- a/bin/mkstatic.py
+++ b/bin/mkstatic.py
@@ -64,7 +64,7 @@ def main():
     
   compile.main(DEST, produce_debug=False)
   
-  for x in "authgate qwebirc simplejson twisted".split(" "):
+  for x in "authgate qwebirc esimplejson twisted".split(" "):
     copypydir(x, DEST)
   for x in "images panes sound".split(" "):
     copydir(os.path.join("static", x), DEST)

diff --git a/bin/optionsgen.py b/bin/optionsgen.py
index 52a8d37b29056fb5160781813c552a08d243b25d..6c22e26705aed3878f64533e2f1cee62cb508377 100644 (file)
--- a/bin/optionsgen.py
+++ b/bin/optionsgen.py
@@ -1,5 +1,6 @@
-import config, simplejson
+import config
+import qwebirc.util.qjson as json
 
 def get_options():
   options = dict(networkName=config.NETWORK_NAME, networkServices=[config.AUTH_SERVICE], loginRegex=config.  AUTH_OK_REGEX, appTitle=config.APP_TITLE, baseURL=config.BASE_URL)
-  return simplejson.dumps(options)
+  return json.dumps(options)

diff --git a/simplejson/__init__.py b/esimplejson/__init__.py
similarity index 99%
rename from simplejson/__init__.py
rename to esimplejson/__init__.py
index 1454ed6969663009aa8d3c406660c28ccd3034f8..5328990b5e36c96ecf7100f5595c91bad4f2507a 100644 (file)
--- a/simplejson/__init__.py
+++ b/esimplejson/__init__.py
@@ -106,8 +106,8 @@ __all__ = [
 ]
 
 if __name__ == '__main__':
-    from simplejson.decoder import JSONDecoder
-    from simplejson.encoder import JSONEncoder
+    from esimplejson.decoder import JSONDecoder
+    from esimplejson.encoder import JSONEncoder
 else:
     from decoder import JSONDecoder
     from encoder import JSONEncoder

diff --git a/simplejson/decoder.py b/esimplejson/decoder.py
similarity index 98%
rename from simplejson/decoder.py
rename to esimplejson/decoder.py
index baf10e99040223a5fe8bc110996f82abdaa88389..f144d675180dc94955b1523172fca5e18452f87a 100644 (file)
--- a/simplejson/decoder.py
+++ b/esimplejson/decoder.py
@@ -4,9 +4,9 @@ Implementation of JSONDecoder
 import re
 import sys
 
-from simplejson.scanner import Scanner, pattern
+from esimplejson.scanner import Scanner, pattern
 try:
-    from simplejson._speedups import scanstring as c_scanstring
+    from esimplejson._speedups import scanstring as c_scanstring
 except ImportError:
     pass
 

diff --git a/simplejson/encoder.py b/esimplejson/encoder.py
similarity index 99%
rename from simplejson/encoder.py
rename to esimplejson/encoder.py
index 772a2614bafa5d4535efe83c72b9b8ec8fca6a61..89e49c5e4ef0646bf787cf9d27a5a22a7abc89ac 100644 (file)
--- a/simplejson/encoder.py
+++ b/esimplejson/encoder.py
@@ -4,7 +4,7 @@ Implementation of JSONEncoder
 import re
 
 try:
-    from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii
+    from esimplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii
 except ImportError:
     pass
 

diff --git a/simplejson/scanner.py b/esimplejson/scanner.py
similarity index 100%
rename from simplejson/scanner.py
rename to esimplejson/scanner.py

diff --git a/qwebirc/engines/ajaxengine.py b/qwebirc/engines/ajaxengine.py
index a6aca3ebe39feab55db98cad71a814984fe7763d..324de11dd00c313bcb6b2bbcf9fb3db12356d6ea 100644 (file)
--- a/qwebirc/engines/ajaxengine.py
+++ b/qwebirc/engines/ajaxengine.py
@@ -2,11 +2,12 @@ from twisted.web import resource, server, static, error as http_error
 from twisted.names import client
 from twisted.internet import reactor, error
 from authgateengine import login_optional, getSessionData
-import simplejson, md5, sys, os, time, config, qwebirc.config_options as config_options, traceback, socket
+import md5, sys, os, time, config, qwebirc.config_options as config_options, traceback, socket
 import qwebirc.ircclient as ircclient
 from adminengine import AdminEngineAction
 from qwebirc.util import HitCounter
 import qwebirc.dns as qdns
+import qwebirc.util.qjson as json
 Sessions = {}
 
 def get_session_id():
@@ -25,6 +26,7 @@ class PassthruException(Exception):
   pass
   
 NOT_DONE_YET = None
+EMPTY_JSON_LIST = json.dumps([])
 
 def jsondump(fn):
   def decorator(*args, **kwargs):
@@ -38,7 +40,7 @@ def jsondump(fn):
     except PassthruException, e:
       return str(e)
       
-    return simplejson.dumps(x)
+    return json.dumps(x)
   return decorator
 
 def cleanupSession(id):
@@ -79,7 +81,7 @@ class IRCSession:
     if self.schedule:
       return
       
-    channel.write(simplejson.dumps([]))
+    channel.write(EMPTY_JSON_LIST)
     if channel in self.subscriptions:
       self.subscriptions.remove(channel)
       
@@ -105,7 +107,7 @@ class IRCSession:
         
     self.throttle = t + config.UPDATE_FREQ
 
-    encdata = simplejson.dumps(self.buffer)
+    encdata = json.dumps(self.buffer)
     self.buffer = []
     self.buflen = 0
 

diff --git a/qwebirc/util/qjson.py b/qwebirc/util/qjson.py
new file mode 100644 (file)
index 0000000..d4a4a49
--- /dev/null
+++ b/qwebirc/util/qjson.py
@@ -0,0 +1,24 @@
+slow = True
+try:
+  # first try the system module
+  import simplejson as json
+  try:
+    # try to see if the C module is available
+    json._speedups
+  except AttributeError:
+    pass
+  else:
+    slow = False
+except ImportError:
+  # try python 2.6's json library as
+  # it is 2x as fast as simplejson with no C
+  try:
+    import json
+    json.dumps # we don't want the silly third party version
+  except (ImportError, AttributeError):
+    # fallback to the embedded
+    import esimplejson as json
+  
+__SEPS = (',', ':')
+dumps = lambda x: json.dumps(x, encoding="utf8", ensure_ascii=True, check_circular=False, indent=None, separators=__SEPS)
+loads = lambda x: json.loads(x, encoding="utf8")