Mercurial > piecrust2
diff piecrust/fastpickle.py @ 461:b015e38d4ee1
internal: Handle data serialization more under the hood.
Implement a new queue class that handles pickling of objects, add option
(currently disabled) to also do zlib compression before sending bytes.
author | Ludovic Chabant <ludovic@chabant.com> |
---|---|
date | Sat, 11 Jul 2015 17:51:56 -0700 |
parents | 2ef04e16f0b9 |
children | 9e5393fcfab2 |
line wrap: on
line diff
--- a/piecrust/fastpickle.py Sat Jul 11 00:45:35 2015 -0700 +++ b/piecrust/fastpickle.py Sat Jul 11 17:51:56 2015 -0700 @@ -1,49 +1,66 @@ import sys +import json import datetime import collections def pickle(obj): - return _pickle_object(obj) + data = _pickle_object(obj) + data = json.dumps(data, indent=None, separators=(',', ':')) + return data.encode('utf8') def unpickle(data): + data = data.decode('utf8') + data = json.loads(data) return _unpickle_object(data) _PICKLING = 0 _UNPICKLING = 1 - -def _tuple_dispatch(obj, func, op): - res = [None] * len(obj) - for i, c in enumerate(obj): - res[i] = func(c) - return tuple(res) +_identity_dispatch = object() -def _list_dispatch(obj, func, op): - res = [None] * len(obj) - for i, c in enumerate(obj): - res[i] = func(c) - return res +def _tuple_convert(obj, func, op): + if op == _PICKLING: + return ['__type__:tuple'] + [func(c) for c in obj] + elif op == _UNPICKLING: + return tuple([func(c) for c in obj[1:]]) -def _dict_dispatch(obj, func, op): +def _list_convert(obj, func, op): + return [func(c) for c in obj] + + +def _dict_convert(obj, func, op): res = {} for k, v in obj.items(): res[k] = func(v) return res -def _set_dispatch(obj, func, op): - res = set() - for v in obj: - res.add(func(v)) - return res +def _ordered_dict_convert(obj, func, op): + if op == _PICKLING: + res = {'__type__': 'OrderedDict'} + for k, v in obj.items(): + res[k] = func(v) + return res + elif op == _UNPICKLING: + res = collections.OrderedDict() + for k, v in obj.items(): + res[k] = func(v) + return res -def _date_convert(obj, op): +def _set_convert(obj, func, op): + if op == _PICKLING: + return ['__type__:set'] + [func(c) for c in obj] + elif op == _UNPICKLING: + return set([func(c) for c in obj[1:]]) + + +def _date_convert(obj, func, op): if op == _PICKLING: return {'__class__': 'date', 'year': obj.year, @@ -54,7 +71,7 @@ obj['year'], obj['month'], obj['day']) -def _datetime_convert(obj, op): +def _datetime_convert(obj, func, op): if op == _PICKLING: return {'__class__': 'datetime', 'year': obj.year, @@ -70,7 +87,7 @@ obj['hour'], obj['minute'], obj['second'], obj['microsecond']) -def _time_convert(obj, op): +def _time_convert(obj, func, op): if op == _PICKLING: return {'__class__': 'time', 'hour': obj.hour, @@ -82,56 +99,66 @@ obj['hour'], obj['minute'], obj['second'], obj['microsecond']) -_identity_dispatch = object() +_type_convert = { + type(None): _identity_dispatch, + bool: _identity_dispatch, + int: _identity_dispatch, + float: _identity_dispatch, + str: _identity_dispatch, + datetime.date: _date_convert, + datetime.datetime: _datetime_convert, + datetime.time: _time_convert, + tuple: _tuple_convert, + list: _list_convert, + dict: _dict_convert, + set: _set_convert, + collections.OrderedDict: _ordered_dict_convert, + } -_type_dispatch = { + +_type_unconvert = { type(None): _identity_dispatch, bool: _identity_dispatch, int: _identity_dispatch, float: _identity_dispatch, str: _identity_dispatch, - tuple: _tuple_dispatch, - list: _list_dispatch, - dict: _dict_dispatch, - collections.OrderedDict: _dict_dispatch, - set: _set_dispatch + 'date': _date_convert, + 'datetime': _datetime_convert, + 'time': _time_convert, } -_type_convert = { - datetime.date: _date_convert, - datetime.datetime: _datetime_convert, - datetime.time: _time_convert +_collection_unconvert = { + '__type__:tuple': _tuple_convert, + '__type__:set': _set_convert, } -_type_unconvert = { - 'date': _date_convert, - 'datetime': _datetime_convert, - 'time': _time_convert +_mapping_unconvert = { + 'OrderedDict': _ordered_dict_convert } def _pickle_object(obj): t = type(obj) - disp = _type_dispatch.get(t) - if disp is _identity_dispatch: + conv = _type_convert.get(t) + + # Object doesn't need conversion? + if conv is _identity_dispatch: return obj - if disp is not None: - return disp(obj, _pickle_object, _PICKLING) + # Object has special conversion? + if conv is not None: + return conv(obj, _pickle_object, _PICKLING) - conv = _type_convert.get(t) - if conv is not None: - return conv(obj, _PICKLING) - + # Use instance dictionary, or a custom state. getter = getattr(obj, '__getstate__', None) if getter is not None: state = getter() else: state = obj.__dict__ - state = _dict_dispatch(state, _pickle_object, _PICKLING) + state = _dict_convert(state, _pickle_object, _PICKLING) state['__class__'] = obj.__class__.__name__ state['__module__'] = obj.__class__.__module__ @@ -140,18 +167,42 @@ def _unpickle_object(state): t = type(state) - disp = _type_dispatch.get(t) - if disp is _identity_dispatch: + conv = _type_unconvert.get(t) + + # Object doesn't need conversion? + if conv is _identity_dispatch: return state - if (disp is not None and - (t != dict or '__class__' not in state)): - return disp(state, _unpickle_object, _UNPICKLING) + # Try collection or mapping conversion. + if t is list: + try: + col_type = state[0] + if not isinstance(col_type, str): + col_type = None + except IndexError: + col_type = None + if col_type is not None: + conv = _collection_unconvert.get(col_type) + if conv is not None: + return conv(state, _unpickle_object, _UNPICKLING) + return _list_convert(state, _unpickle_object, _UNPICKLING) - class_name = state['__class__'] + assert t is dict + + # Custom mapping type? + map_type = state.get('__type__') + if map_type: + conv = _mapping_unconvert.get(map_type) + return conv(state, _unpickle_object, _UNPICKLING) + + # Class instance or other custom type. + class_name = state.get('__class__') + if class_name is None: + return _dict_convert(state, _unpickle_object, _UNPICKLING) + conv = _type_unconvert.get(class_name) if conv is not None: - return conv(state, _UNPICKLING) + return conv(state, _unpickle_object, _UNPICKLING) mod_name = state['__module__'] mod = sys.modules[mod_name]