Mercurial > piecrust2
changeset 911:f2b75e4be981
internal: Use pickle for caching things on disk.
This is just easier and lets us use proper classes instead of converting
to/from dictionaries.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Sun, 23 Jul 2017 18:03:21 -0700 |
| parents | 371731b555ec |
| children | 9d804fa186a0 |
| files | piecrust/cache.py piecrust/rendering.py piecrust/workerpool.py |
| diffstat | 3 files changed, 35 insertions(+), 38 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/cache.py Sun Jul 23 18:01:58 2017 -0700 +++ b/piecrust/cache.py Sun Jul 23 18:03:21 2017 -0700 @@ -1,8 +1,7 @@ import os import os.path -import json import shutil -import codecs +import pickle import hashlib import logging import repoze.lru @@ -84,19 +83,23 @@ return os.path.isfile(cache_path) def read(self, path): - cache_path = self.getCachePath(path) - logger.debug("Reading cache: %s" % cache_path) - with codecs.open(cache_path, 'r', 'utf-8') as fp: + with self.openRead(path, mode='r', encoding='utf8') as fp: return fp.read() + def openRead(self, path, mode='r', encoding=None): + cache_path = self.getCachePath(path) + return open(cache_path, mode=mode, encoding=encoding) + def write(self, path, content): + with self.openWrite(path, mode='w', encoding='utf8') as fp: + fp.write(content) + + def openWrite(self, path, mode='w', encoding=None): cache_path = self.getCachePath(path) cache_dir = os.path.dirname(cache_path) if not os.path.isdir(cache_dir): os.makedirs(cache_dir, 0o755) - logger.debug("Writing cache: %s" % cache_path) - with codecs.open(cache_path, 'w', 'utf-8') as fp: - fp.write(content) + return open(cache_path, mode=mode, encoding=encoding) def getCachePath(self, path): if path.startswith('.'): @@ -154,7 +157,7 @@ class MemCache(object): """ Simple memory cache. It can be backed by a simple file-system - cache, but items need to be JSON-serializable to do this. + cache, but items need to be pickle-able to do this. """ def __init__(self, size=2048): self.cache = repoze.lru.LRUCache(size) @@ -181,8 +184,8 @@ self.cache.put(key, item) if self.fs_cache and save_to_fs: fs_key = _make_fs_cache_key(key) - item_raw = json.dumps(item) - self.fs_cache.write(fs_key, item_raw) + with self.fs_cache.openWrite(fs_key, mode='wb') as fp: + pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL) def get(self, key, item_maker, fs_cache_time=None, save_to_fs=True): self._last_access_hit = True @@ -201,10 +204,9 @@ fs_key = _make_fs_cache_key(key) if (fs_key not in self._invalidated_fs_items and self.fs_cache.isValid(fs_key, fs_cache_time)): - logger.debug("'%s' found in file-system cache." % - key) - item_raw = self.fs_cache.read(fs_key) - item = json.loads(item_raw) + logger.debug("'%s' found in file-system cache." % key) + with self.fs_cache.openRead(fs_key, mode='rb') as fp: + item = pickle.load(fp) self.cache.put(key, item) self._hits += 1 return item @@ -219,9 +221,8 @@ # Save to the file-system if needed. if self.fs_cache is not None and save_to_fs: - item_raw = json.dumps(item) - self.fs_cache.write(fs_key, item_raw) + with self.fs_cache.openWrite(fs_key, mode='wb') as fp: + pickle.dump(item, fp, pickle.HIGHEST_PROTOCOL) return item -
--- a/piecrust/rendering.py Sun Jul 23 18:01:58 2017 -0700 +++ b/piecrust/rendering.py Sun Jul 23 18:03:21 2017 -0700 @@ -4,7 +4,6 @@ import logging from piecrust.data.builder import ( DataBuildingContext, build_page_data, add_layout_data) -from piecrust.fastpickle import _pickle_object, _unpickle_object from piecrust.templating.base import TemplateNotFoundError, TemplatingError from piecrust.sources.base import AbortedSourceUseError @@ -202,24 +201,20 @@ null_names = ['', 'none', 'nil'] if layout_name not in null_names: with stats.timerScope("BuildRenderData"): - add_layout_data(page_data, render_result['segments']) + add_layout_data(page_data, render_result.segments) with stats.timerScope("PageRenderLayout"): layout_result = _do_render_layout( layout_name, page, page_data) else: - layout_result = { - 'content': render_result['segments']['content'], - 'pass_info': None} + layout_result = RenderedLayout( + render_result.segments['content'], None) rp = RenderedPage(page, ctx.sub_num) rp.data = page_data - rp.content = layout_result['content'] - rp.render_info[PASS_FORMATTING] = _unpickle_object( - render_result['pass_info']) - if layout_result['pass_info'] is not None: - rp.render_info[PASS_RENDERING] = _unpickle_object( - layout_result['pass_info']) + rp.content = layout_result.content + rp.render_info[PASS_FORMATTING] = render_result.render_pass_info + rp.render_info[PASS_RENDERING] = layout_result.render_pass_info return rp except AbortedSourceUseError: @@ -277,10 +272,7 @@ ctx.setCurrentPass(PASS_NONE) stack.popCtx() - rs = RenderedSegments( - render_result['segments'], - _unpickle_object(render_result['pass_info'])) - return rs + return render_result def _build_render_data(ctx): @@ -333,9 +325,7 @@ formatted_segments['content.abstract'] = content_abstract pass_info = ctx.render_passes[PASS_FORMATTING] - res = { - 'segments': formatted_segments, - 'pass_info': _pickle_object(pass_info)} + res = RenderedSegments(formatted_segments, pass_info) app.env.stats.stepCounter('PageRenderSegments') @@ -371,7 +361,7 @@ raise Exception(msg) from ex pass_info = cur_ctx.render_passes[PASS_RENDERING] - res = {'content': output, 'pass_info': _pickle_object(pass_info)} + res = RenderedLayout(output, pass_info) app.env.stats.stepCounter('PageRenderLayout') @@ -394,6 +384,12 @@ format_count = 0 format_name = format_name or app.config.get('site/default_format') + + auto_fmts = app.config.get('site/auto_formats') + redirect = auto_fmts.get(format_name) + if redirect is not None: + format_name = redirect + for fmt in app.plugin_loader.getFormatters(): if not fmt.enabled: continue