Mercurial > piecrust2

annotate piecrust/fastpickle.py @ 469:f14796cbab0f 2.0.0a12

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cm: Fix wrong directory for utilities.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 14 Jul 2015 23:02:38 -0700
parents b015e38d4ee1
children 9e5393fcfab2
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import sys
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
2 import json
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import datetime
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import collections
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 def pickle(obj):
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
8 data = _pickle_object(obj)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
9 data = json.dumps(data, indent=None, separators=(',', ':'))
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
10 return data.encode('utf8')
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 def unpickle(data):
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
14 data = data.decode('utf8')
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
15 data = json.loads(data)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 return _unpickle_object(data)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
19 _PICKLING = 0
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
20 _UNPICKLING = 1
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
21
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
22 _identity_dispatch = object()
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
25 def _tuple_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
26 if op == _PICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
27 return ['__type__:tuple'] + [func(c) for c in obj]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
28 elif op == _UNPICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
29 return tuple([func(c) for c in obj[1:]])
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
30
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
32 def _list_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
33 return [func(c) for c in obj]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
34
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
35
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
36 def _dict_convert(obj, func, op):
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 res = {}
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 for k, v in obj.items():
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39 res[k] = func(v)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 return res
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
43 def _ordered_dict_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
44 if op == _PICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
45 res = {'__type__': 'OrderedDict'}
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
46 for k, v in obj.items():
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
47 res[k] = func(v)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
48 return res
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
49 elif op == _UNPICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
50 res = collections.OrderedDict()
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
51 for k, v in obj.items():
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
52 res[k] = func(v)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
53 return res
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
56 def _set_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
57 if op == _PICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
58 return ['__type__:set'] + [func(c) for c in obj]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
59 elif op == _UNPICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
60 return set([func(c) for c in obj[1:]])
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
61
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
62
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
63 def _date_convert(obj, func, op):
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
64 if op == _PICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
65 return {'__class__': 'date',
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
66 'year': obj.year,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
67 'month': obj.month,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
68 'day': obj.day}
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
69 elif op == _UNPICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
70 return datetime.date(
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
71 obj['year'], obj['month'], obj['day'])
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
72
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
73
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
74 def _datetime_convert(obj, func, op):
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
75 if op == _PICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
76 return {'__class__': 'datetime',
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
77 'year': obj.year,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
78 'month': obj.month,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
79 'day': obj.day,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
80 'hour': obj.hour,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
81 'minute': obj.minute,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
82 'second': obj.second,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
83 'microsecond': obj.microsecond}
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
84 elif op == _UNPICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
85 return datetime.datetime(
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
86 obj['year'], obj['month'], obj['day'],
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
87 obj['hour'], obj['minute'], obj['second'], obj['microsecond'])
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
88
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
89
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
90 def _time_convert(obj, func, op):
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
91 if op == _PICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
92 return {'__class__': 'time',
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
93 'hour': obj.hour,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
94 'minute': obj.minute,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
95 'second': obj.second,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
96 'microsecond': obj.microsecond}
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
97 elif op == _UNPICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
98 return datetime.time(
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
99 obj['hour'], obj['minute'], obj['second'], obj['microsecond'])
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
100
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
101
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
102 _type_convert = {
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
103 type(None): _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
104 bool: _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
105 int: _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
106 float: _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
107 str: _identity_dispatch,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
108 datetime.date: _date_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
109 datetime.datetime: _datetime_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
110 datetime.time: _time_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
111 tuple: _tuple_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
112 list: _list_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
113 dict: _dict_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
114 set: _set_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
115 collections.OrderedDict: _ordered_dict_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
116 }
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
118
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
119 _type_unconvert = {
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
120 type(None): _identity_dispatch,
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
121 bool: _identity_dispatch,
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
122 int: _identity_dispatch,
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
123 float: _identity_dispatch,
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
124 str: _identity_dispatch,
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
125 'date': _date_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
126 'datetime': _datetime_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
127 'time': _time_convert,
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
128 }
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
129
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
130
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
131 _collection_unconvert = {
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
132 '__type__:tuple': _tuple_convert,
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
133 '__type__:set': _set_convert,
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
134 }
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
135
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
136
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
137 _mapping_unconvert = {
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
138 'OrderedDict': _ordered_dict_convert
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
139 }
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
140
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
141
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
142 def _pickle_object(obj):
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
143 t = type(obj)
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
144 conv = _type_convert.get(t)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
145
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
146 # Object doesn't need conversion?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
147 if conv is _identity_dispatch:
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
148 return obj
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
149
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
150 # Object has special conversion?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
151 if conv is not None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
152 return conv(obj, _pickle_object, _PICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
153
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
154 # Use instance dictionary, or a custom state.
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
155 getter = getattr(obj, '__getstate__', None)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
156 if getter is not None:
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
157 state = getter()
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
158 else:
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
159 state = obj.__dict__
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
160
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
161 state = _dict_convert(state, _pickle_object, _PICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
162 state['__class__'] = obj.__class__.__name__
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
163 state['__module__'] = obj.__class__.__module__
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
165 return state
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
166
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
167
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
168 def _unpickle_object(state):
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
169 t = type(state)
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
170 conv = _type_unconvert.get(t)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
171
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
172 # Object doesn't need conversion?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
173 if conv is _identity_dispatch:
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
174 return state
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
175
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
176 # Try collection or mapping conversion.
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
177 if t is list:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
178 try:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
179 col_type = state[0]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
180 if not isinstance(col_type, str):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
181 col_type = None
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
182 except IndexError:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
183 col_type = None
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
184 if col_type is not None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
185 conv = _collection_unconvert.get(col_type)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
186 if conv is not None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
187 return conv(state, _unpickle_object, _UNPICKLING)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
188 return _list_convert(state, _unpickle_object, _UNPICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
189
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
190 assert t is dict
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
191
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
192 # Custom mapping type?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
193 map_type = state.get('__type__')
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
194 if map_type:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
195 conv = _mapping_unconvert.get(map_type)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
196 return conv(state, _unpickle_object, _UNPICKLING)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
197
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
198 # Class instance or other custom type.
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
199 class_name = state.get('__class__')
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
200 if class_name is None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
201 return _dict_convert(state, _unpickle_object, _UNPICKLING)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
202
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
203 conv = _type_unconvert.get(class_name)
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
204 if conv is not None:
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
205 return conv(state, _unpickle_object, _UNPICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
206
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
207 mod_name = state['__module__']
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
208 mod = sys.modules[mod_name]
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
209 class_def = getattr(mod, class_name)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
210 obj = class_def.__new__(class_def)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
211
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
212 del state['__class__']
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
213 del state['__module__']
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
214 attr_names = list(state.keys())
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
215 for name in attr_names:
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
216 state[name] = _unpickle_object(state[name])
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
217
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
218 obj.__dict__.update(state)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
219
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
220 return obj
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
221