diff piecrust/fastpickle.py @ 989:8adc27285d93

bake: Big pass on bake performance. - Reduce the amount of data passed between processes. - Make inter-process data simple objects to make it easier to test with alternatives to pickle. - Make sources have the basic requirement to be able to find a content item from an item spec (path). - Make Hoedown the default Markdown formatter.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 19 Nov 2017 14:29:17 -0800
parents 4850f8c21b6e
children
line wrap: on
line diff
--- a/piecrust/fastpickle.py	Fri Nov 03 23:14:56 2017 -0700
+++ b/piecrust/fastpickle.py	Sun Nov 19 14:29:17 2017 -0800
@@ -1,54 +1,85 @@
+import io
 import sys
-import json
 import codecs
 import datetime
 import collections
 
 
-def pickle(obj):
-    data = _pickle_object(obj)
-    data = json.dumps(data, indent=None, separators=(',', ':'))
-    return data.encode('utf8')
+use_msgpack = False
+use_marshall = False
 
 
-def pickle_obj(obj):
-    if obj is not None:
-        return _pickle_object(obj)
-    return None
+if use_msgpack:
+    import msgpack
+
+    def _dumps_msgpack(obj, buf):
+        msgpack.pack(obj, buf)
+
+    def _loads_msgpack(buf, bufsize):
+        return msgpack.unpack(buf)
+
+    _dumps = _dumps_msgpack
+    _loads = _loads_msgpack
+
+elif use_marshall:
+    import marshal
+
+    def _dumps_marshal(obj, buf):
+        marshal.dump(obj, buf)
+
+    def _loads_marshal(buf, bufsize):
+        return marshal.load(buf)
+
+    _dumps = _dumps_marshal
+    _loads = _loads_marshal
+
+else:
+    import json
+
+    class _BufferWrapper:
+        def __init__(self, buf):
+            self._buf = buf
+
+        def write(self, data):
+            self._buf.write(data.encode('utf8'))
+
+        def read(self):
+            return self._buf.read().decode('utf8')
+
+    def _dumps_json(obj, buf):
+        buf = _BufferWrapper(buf)
+        json.dump(obj, buf, indent=None, separators=(',', ':'))
+
+    def _loads_json(buf, bufsize):
+        buf = _BufferWrapper(buf)
+        return json.load(buf)
+
+    _dumps = _dumps_json
+    _loads = _loads_json
+
+
+def pickle(obj):
+    with io.BytesIO() as buf:
+        pickle_intob(obj, buf)
+        return buf.getvalue()
 
 
 def pickle_intob(obj, buf):
     data = _pickle_object(obj)
-    buf = _WriteWrapper(buf)
-    json.dump(data, buf, indent=None, separators=(',', ':'))
+    _dumps(data, buf)
 
 
 def unpickle(data):
-    data = json.loads(data.decode('utf8'))
+    with io.BytesIO(data) as buf:
+        data = _loads(buf, len(data))
     return _unpickle_object(data)
 
 
-def unpickle_obj(data):
-    if data is not None:
-        return _unpickle_object(data)
-    return None
-
-
 def unpickle_fromb(buf, bufsize):
-    with buf.getbuffer() as innerbuf:
-        data = codecs.decode(innerbuf[:bufsize], 'utf8')
-    data = json.loads(data)
+    data = _loads(buf, bufsize)
     return _unpickle_object(data)
 
 
-class _WriteWrapper(object):
-    def __init__(self, buf):
-        self._buf = buf
-
-    def write(self, data):
-        self._buf.write(data.encode('utf8'))
-
-
 _PICKLING = 0
 _UNPICKLING = 1