Mercurial > piecrust2

annotate piecrust/fastpickle.py @ 701:066d6156525c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
showrecord: Don't print the record when you just want the stats.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 16 Apr 2016 22:48:57 -0700
parents 76a799eae824
children fcfbe103cfd1
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import sys
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
2 import json
697
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
3 import codecs
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import datetime
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import collections
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 def pickle(obj):
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
9 data = _pickle_object(obj)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
10 data = json.dumps(data, indent=None, separators=(',', ':'))
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
11 return data.encode('utf8')
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
697
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
14 def pickle_intob(obj, buf):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
15 data = _pickle_object(obj)
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
16 buf = _WriteWrapper(buf)
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
17 json.dump(data, buf, indent=None, separators=(',', ':'))
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
18
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
19
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 def unpickle(data):
697
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
21 data = json.loads(data.decode('utf8'))
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
22 return _unpickle_object(data)
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
23
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
24
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
25 def unpickle_fromb(buf, bufsize):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
26 with buf.getbuffer() as innerbuf:
700
76a799eae824 internal: Fix compatibility with older Python 3.x.
Ludovic Chabant <ludovic@chabant.com>
parents: 697
diff changeset
27 data = codecs.decode(innerbuf[:bufsize], 'utf8')
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
28 data = json.loads(data)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29 return _unpickle_object(data)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31
697
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
32 class _WriteWrapper(object):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
33 def __init__(self, buf):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
34 self._buf = buf
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
35
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
36 def write(self, data):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
37 self._buf.write(data.encode('utf8'))
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
38
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
39
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
40 _PICKLING = 0
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
41 _UNPICKLING = 1
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
42
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
43 _identity_dispatch = object()
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
46 def _tuple_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
47 if op == _PICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
48 return ['__type__:tuple'] + [func(c) for c in obj]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
49 elif op == _UNPICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
50 return tuple([func(c) for c in obj[1:]])
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
53 def _list_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
54 return [func(c) for c in obj]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
55
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
56
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
57 def _dict_convert(obj, func, op):
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 res = {}
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 for k, v in obj.items():
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 res[k] = func(v)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 return res
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
64 def _ordered_dict_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
65 if op == _PICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
66 res = {'__type__': 'OrderedDict'}
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
67 for k, v in obj.items():
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
68 res[k] = func(v)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
69 return res
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
70 elif op == _UNPICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
71 res = collections.OrderedDict()
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
72 for k, v in obj.items():
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
73 res[k] = func(v)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
74 return res
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
76
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
77 def _set_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
78 if op == _PICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
79 return ['__type__:set'] + [func(c) for c in obj]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
80 elif op == _UNPICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
81 return set([func(c) for c in obj[1:]])
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
82
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
83
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
84 def _date_convert(obj, func, op):
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
85 if op == _PICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
86 return {'__class__': 'date',
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
87 'year': obj.year,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
88 'month': obj.month,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
89 'day': obj.day}
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
90 elif op == _UNPICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
91 return datetime.date(
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
92 obj['year'], obj['month'], obj['day'])
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
93
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
94
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
95 def _datetime_convert(obj, func, op):
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
96 if op == _PICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
97 return {'__class__': 'datetime',
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
98 'year': obj.year,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
99 'month': obj.month,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
100 'day': obj.day,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
101 'hour': obj.hour,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
102 'minute': obj.minute,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
103 'second': obj.second,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
104 'microsecond': obj.microsecond}
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
105 elif op == _UNPICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
106 return datetime.datetime(
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
107 obj['year'], obj['month'], obj['day'],
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
108 obj['hour'], obj['minute'], obj['second'], obj['microsecond'])
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
109
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
110
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
111 def _time_convert(obj, func, op):
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
112 if op == _PICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
113 return {'__class__': 'time',
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
114 'hour': obj.hour,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
115 'minute': obj.minute,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
116 'second': obj.second,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
117 'microsecond': obj.microsecond}
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
118 elif op == _UNPICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
119 return datetime.time(
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
120 obj['hour'], obj['minute'], obj['second'], obj['microsecond'])
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
121
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
122
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
123 _type_convert = {
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
124 type(None): _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
125 bool: _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
126 int: _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
127 float: _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
128 str: _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
129 datetime.date: _date_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
130 datetime.datetime: _datetime_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
131 datetime.time: _time_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
132 tuple: _tuple_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
133 list: _list_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
134 dict: _dict_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
135 set: _set_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
136 collections.OrderedDict: _ordered_dict_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
137 }
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
138
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
139
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
140 _type_unconvert = {
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
141 type(None): _identity_dispatch,
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
142 bool: _identity_dispatch,
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143 int: _identity_dispatch,
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
144 float: _identity_dispatch,
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
145 str: _identity_dispatch,
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
146 'date': _date_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
147 'datetime': _datetime_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
148 'time': _time_convert,
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
149 }
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
150
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
151
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
152 _collection_unconvert = {
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
153 '__type__:tuple': _tuple_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
154 '__type__:set': _set_convert,
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
155 }
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
156
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
157
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
158 _mapping_unconvert = {
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
159 'OrderedDict': _ordered_dict_convert
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
160 }
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
161
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
162
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
163 def _pickle_object(obj):
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164 t = type(obj)
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
165 conv = _type_convert.get(t)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
166
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
167 # Object doesn't need conversion?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
168 if conv is _identity_dispatch:
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
169 return obj
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
170
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
171 # Object has special conversion?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
172 if conv is not None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
173 return conv(obj, _pickle_object, _PICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
174
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
175 # Use instance dictionary, or a custom state.
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
176 getter = getattr(obj, '__getstate__', None)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
177 if getter is not None:
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
178 state = getter()
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
179 else:
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
180 state = obj.__dict__
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
181
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
182 state = _dict_convert(state, _pickle_object, _PICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
183 state['__class__'] = obj.__class__.__name__
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
184 state['__module__'] = obj.__class__.__module__
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
185
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
186 return state
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
187
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
188
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
189 def _unpickle_object(state):
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
190 t = type(state)
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
191 conv = _type_unconvert.get(t)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
192
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
193 # Object doesn't need conversion?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
194 if conv is _identity_dispatch:
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
195 return state
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
196
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
197 # Try collection or mapping conversion.
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
198 if t is list:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
199 try:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
200 col_type = state[0]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
201 if not isinstance(col_type, str):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
202 col_type = None
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
203 except IndexError:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
204 col_type = None
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
205 if col_type is not None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
206 conv = _collection_unconvert.get(col_type)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
207 if conv is not None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
208 return conv(state, _unpickle_object, _UNPICKLING)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
209 return _list_convert(state, _unpickle_object, _UNPICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
210
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
211 assert t is dict
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
212
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
213 # Custom mapping type?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
214 map_type = state.get('__type__')
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
215 if map_type:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
216 conv = _mapping_unconvert.get(map_type)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
217 return conv(state, _unpickle_object, _UNPICKLING)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
218
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
219 # Class instance or other custom type.
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
220 class_name = state.get('__class__')
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
221 if class_name is None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
222 return _dict_convert(state, _unpickle_object, _UNPICKLING)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
223
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
224 conv = _type_unconvert.get(class_name)
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
225 if conv is not None:
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
226 return conv(state, _unpickle_object, _UNPICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
227
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
228 mod_name = state['__module__']
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
229 mod = sys.modules[mod_name]
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
230 class_def = getattr(mod, class_name)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
231 obj = class_def.__new__(class_def)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
232
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
233 del state['__class__']
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
234 del state['__module__']
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
235 attr_names = list(state.keys())
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
236 for name in attr_names:
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
237 state[name] = _unpickle_object(state[name])
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
238
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
239 obj.__dict__.update(state)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
240
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
241 return obj
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
242