Mercurial > piecrust2

annotate piecrust/fastpickle.py @ 882:acd9c3e8533f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Correctly setup unbaked entries for taxonomy pages.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 17 Jun 2017 09:14:36 -0700
parents 4850f8c21b6e
children 8adc27285d93
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import sys
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
2 import json
697
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
3 import codecs
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import datetime
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import collections
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 def pickle(obj):
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
9 data = _pickle_object(obj)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
10 data = json.dumps(data, indent=None, separators=(',', ':'))
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
11 return data.encode('utf8')
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
717
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
14 def pickle_obj(obj):
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
15 if obj is not None:
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
16 return _pickle_object(obj)
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
17 return None
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
18
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
19
697
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
20 def pickle_intob(obj, buf):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
21 data = _pickle_object(obj)
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
22 buf = _WriteWrapper(buf)
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
23 json.dump(data, buf, indent=None, separators=(',', ':'))
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
24
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
25
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26 def unpickle(data):
697
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
27 data = json.loads(data.decode('utf8'))
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
28 return _unpickle_object(data)
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
29
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
30
717
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
31 def unpickle_obj(data):
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
32 if data is not None:
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
33 return _unpickle_object(data)
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
34 return None
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
35
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
36
697
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
37 def unpickle_fromb(buf, bufsize):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
38 with buf.getbuffer() as innerbuf:
700
76a799eae824 internal: Fix compatibility with older Python 3.x.
Ludovic Chabant <ludovic@chabant.com>
parents: 697
diff changeset
39 data = codecs.decode(innerbuf[:bufsize], 'utf8')
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
40 data = json.loads(data)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 return _unpickle_object(data)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43
697
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
44 class _WriteWrapper(object):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
45 def __init__(self, buf):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
46 self._buf = buf
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
47
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
48 def write(self, data):
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
49 self._buf.write(data.encode('utf8'))
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
50
9e5393fcfab2 bake: Re-enable faster serialization between processes.
Ludovic Chabant <ludovic@chabant.com>
parents: 461
diff changeset
51
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
52 _PICKLING = 0
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
53 _UNPICKLING = 1
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
54
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
55 _identity_dispatch = object()
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
58 def _tuple_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
59 if op == _PICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
60 return ['__type__:tuple'] + [func(c) for c in obj]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
61 elif op == _UNPICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
62 return tuple([func(c) for c in obj[1:]])
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
65 def _list_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
66 return [func(c) for c in obj]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
67
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
68
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
69 def _dict_convert(obj, func, op):
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 res = {}
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 for k, v in obj.items():
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 res[k] = func(v)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 return res
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
76 def _ordered_dict_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
77 if op == _PICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
78 res = {'__type__': 'OrderedDict'}
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
79 for k, v in obj.items():
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
80 res[k] = func(v)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
81 return res
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
82 elif op == _UNPICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
83 res = collections.OrderedDict()
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
84 for k, v in obj.items():
717
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
85 if k != '__type__':
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
86 res[k] = func(v)
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
87 return res
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
90 def _set_convert(obj, func, op):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
91 if op == _PICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
92 return ['__type__:set'] + [func(c) for c in obj]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
93 elif op == _UNPICKLING:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
94 return set([func(c) for c in obj[1:]])
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
95
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
96
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
97 def _date_convert(obj, func, op):
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
98 if op == _PICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
99 return {'__class__': 'date',
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
100 'year': obj.year,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
101 'month': obj.month,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
102 'day': obj.day}
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
103 elif op == _UNPICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
104 return datetime.date(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
105 obj['year'], obj['month'], obj['day'])
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
106
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
107
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
108 def _datetime_convert(obj, func, op):
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
109 if op == _PICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
110 return {'__class__': 'datetime',
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
111 'year': obj.year,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
112 'month': obj.month,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
113 'day': obj.day,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
114 'hour': obj.hour,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
115 'minute': obj.minute,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
116 'second': obj.second,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
117 'microsecond': obj.microsecond}
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
118 elif op == _UNPICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
119 return datetime.datetime(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
120 obj['year'], obj['month'], obj['day'],
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
121 obj['hour'], obj['minute'], obj['second'], obj['microsecond'])
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
122
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
123
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
124 def _time_convert(obj, func, op):
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
125 if op == _PICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
126 return {'__class__': 'time',
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
127 'hour': obj.hour,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
128 'minute': obj.minute,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
129 'second': obj.second,
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
130 'microsecond': obj.microsecond}
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
131 elif op == _UNPICKLING:
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
132 return datetime.time(
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
133 obj['hour'], obj['minute'], obj['second'], obj['microsecond'])
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
134
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
135
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
136 _type_convert = {
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
137 type(None): _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
138 bool: _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
139 int: _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
140 float: _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
141 str: _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
142 datetime.date: _date_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
143 datetime.datetime: _datetime_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
144 datetime.time: _time_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
145 tuple: _tuple_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
146 list: _list_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
147 dict: _dict_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
148 set: _set_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
149 collections.OrderedDict: _ordered_dict_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
150 }
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
151
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
152
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
153 _type_unconvert = {
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
154 type(None): _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
155 bool: _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
156 int: _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
157 float: _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
158 str: _identity_dispatch,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
159 'date': _date_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
160 'datetime': _datetime_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
161 'time': _time_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
162 }
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
163
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
164
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
165 _collection_unconvert = {
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
166 '__type__:tuple': _tuple_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
167 '__type__:set': _set_convert,
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
168 }
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
169
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
170
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
171 _mapping_unconvert = {
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
172 'OrderedDict': _ordered_dict_convert
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents: 717
diff changeset
173 }
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
174
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
175
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
176 def _pickle_object(obj):
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
177 t = type(obj)
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
178 conv = _type_convert.get(t)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
179
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
180 # Object doesn't need conversion?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
181 if conv is _identity_dispatch:
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
182 return obj
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
183
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
184 # Object has special conversion?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
185 if conv is not None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
186 return conv(obj, _pickle_object, _PICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
187
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
188 # Use instance dictionary, or a custom state.
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
189 getter = getattr(obj, '__getstate__', None)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
190 if getter is not None:
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
191 state = getter()
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
192 else:
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
193 state = obj.__dict__
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
194
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
195 state = _dict_convert(state, _pickle_object, _PICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
196 state['__class__'] = obj.__class__.__name__
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
197 state['__module__'] = obj.__class__.__module__
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
198
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
199 return state
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
200
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
201
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
202 def _unpickle_object(state):
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
203 t = type(state)
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
204 conv = _type_unconvert.get(t)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
205
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
206 # Object doesn't need conversion?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
207 if conv is _identity_dispatch:
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
208 return state
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
209
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
210 # Try collection or mapping conversion.
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
211 if t is list:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
212 try:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
213 col_type = state[0]
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
214 if not isinstance(col_type, str):
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
215 col_type = None
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
216 except IndexError:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
217 col_type = None
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
218 if col_type is not None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
219 conv = _collection_unconvert.get(col_type)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
220 if conv is not None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
221 return conv(state, _unpickle_object, _UNPICKLING)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
222 return _list_convert(state, _unpickle_object, _UNPICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
223
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
224 assert t is dict
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
225
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
226 # Custom mapping type?
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
227 map_type = state.get('__type__')
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
228 if map_type:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
229 conv = _mapping_unconvert.get(map_type)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
230 return conv(state, _unpickle_object, _UNPICKLING)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
231
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
232 # Class instance or other custom type.
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
233 class_name = state.get('__class__')
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
234 if class_name is None:
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
235 return _dict_convert(state, _unpickle_object, _UNPICKLING)
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
236
459
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
237 conv = _type_unconvert.get(class_name)
2ef04e16f0b9 internal: Add support for fake pickling of date/time structures.
Ludovic Chabant <ludovic@chabant.com>
parents: 450
diff changeset
238 if conv is not None:
461
b015e38d4ee1 internal: Handle data serialization more under the hood.
Ludovic Chabant <ludovic@chabant.com>
parents: 459
diff changeset
239 return conv(state, _unpickle_object, _UNPICKLING)
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
240
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
241 mod_name = state['__module__']
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
242 mod = sys.modules[mod_name]
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
243 class_def = getattr(mod, class_name)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
244 obj = class_def.__new__(class_def)
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
245
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
246 attr_names = list(state.keys())
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
247 for name in attr_names:
717
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
248 if name == '__class__' or name == '__module__':
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
249 continue
fcfbe103cfd1 internal: Fix some bugs with the `fastpickle` module.
Ludovic Chabant <ludovic@chabant.com>
parents: 700
diff changeset
250 obj.__dict__[name] = _unpickle_object(state[name])
450
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
251
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
252 return obj
298f8f46432a internal: Add a `fastpickle` module to help with multiprocess serialization.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
253