Mercurial > piecrust2

comparison piecrust/fastpickle.py @ 989:8adc27285d93

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Big pass on bake performance. - Reduce the amount of data passed between processes. - Make inter-process data simple objects to make it easier to test with alternatives to pickle. - Make sources have the basic requirement to be able to find a content item from an item spec (path). - Make Hoedown the default Markdown formatter.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 19 Nov 2017 14:29:17 -0800
parents 4850f8c21b6e
children
comparison
equal deleted inserted replaced
988:f83ae0a5d793 989:8adc27285d93
1 import io
1 import sys 2 import sys
2 import json
3 import codecs 3 import codecs
4 import datetime 4 import datetime
5 import collections 5 import collections
6 6
7 7
8 use_msgpack = False
9 use_marshall = False
10
11
12 if use_msgpack:
13 import msgpack
14
15 def _dumps_msgpack(obj, buf):
16 msgpack.pack(obj, buf)
17
18 def _loads_msgpack(buf, bufsize):
19 return msgpack.unpack(buf)
20
21 _dumps = _dumps_msgpack
22 _loads = _loads_msgpack
23
24 elif use_marshall:
25 import marshal
26
27 def _dumps_marshal(obj, buf):
28 marshal.dump(obj, buf)
29
30 def _loads_marshal(buf, bufsize):
31 return marshal.load(buf)
32
33 _dumps = _dumps_marshal
34 _loads = _loads_marshal
35
36 else:
37 import json
38
39 class _BufferWrapper:
40 def __init__(self, buf):
41 self._buf = buf
42
43 def write(self, data):
44 self._buf.write(data.encode('utf8'))
45
46 def read(self):
47 return self._buf.read().decode('utf8')
48
49 def _dumps_json(obj, buf):
50 buf = _BufferWrapper(buf)
51 json.dump(obj, buf, indent=None, separators=(',', ':'))
52
53 def _loads_json(buf, bufsize):
54 buf = _BufferWrapper(buf)
55 return json.load(buf)
56
57 _dumps = _dumps_json
58 _loads = _loads_json
59
60
8 def pickle(obj): 61 def pickle(obj):
9 data = _pickle_object(obj) 62 with io.BytesIO() as buf:
10 data = json.dumps(data, indent=None, separators=(',', ':')) 63 pickle_intob(obj, buf)
11 return data.encode('utf8') 64 return buf.getvalue()
12
13
14 def pickle_obj(obj):
15 if obj is not None:
16 return _pickle_object(obj)
17 return None
18 65
19 66
20 def pickle_intob(obj, buf): 67 def pickle_intob(obj, buf):
21 data = _pickle_object(obj) 68 data = _pickle_object(obj)
22 buf = _WriteWrapper(buf) 69 _dumps(data, buf)
23 json.dump(data, buf, indent=None, separators=(',', ':'))
24 70
25 71
26 def unpickle(data): 72 def unpickle(data):
27 data = json.loads(data.decode('utf8')) 73 with io.BytesIO(data) as buf:
74 data = _loads(buf, len(data))
28 return _unpickle_object(data) 75 return _unpickle_object(data)
29 76
30 77
31 def unpickle_obj(data):
32 if data is not None:
33 return _unpickle_object(data)
34 return None
35
36
37 def unpickle_fromb(buf, bufsize): 78 def unpickle_fromb(buf, bufsize):
38 with buf.getbuffer() as innerbuf: 79 data = _loads(buf, bufsize)
39 data = codecs.decode(innerbuf[:bufsize], 'utf8')
40 data = json.loads(data)
41 return _unpickle_object(data) 80 return _unpickle_object(data)
42
43
44 class _WriteWrapper(object):
45 def __init__(self, buf):
46 self._buf = buf
47
48 def write(self, data):
49 self._buf.write(data.encode('utf8'))
50 81
51 82
52 _PICKLING = 0 83 _PICKLING = 0
53 _UNPICKLING = 1 84 _UNPICKLING = 1
54 85