Mercurial > piecrust2

comparison piecrust/page.py @ 411:e7b865f8f335

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Enable multiprocess baking. Baking is now done by running a worker per CPU, and sending jobs to them. This changes several things across the codebase: * Ability to not cache things related to pages other than the 'main' page (i.e. the page at the bottom of the execution stack). * Decouple the baking process from the bake records, so only the main process keeps track (and modifies) the bake record. * Remove the need for 'batch page getters' and loading a page directly from the page factories. There are various smaller changes too included here, including support for scope performance timers that are saved with the bake record and can be printed out to the console. Yes I got carried away. For testing, the in-memory 'mock' file-system doesn't work anymore, since we're spawning processes, so this is replaced by a 'tmpfs' file-system which is saved in temporary files on disk and deleted after tests have run.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 12 Jun 2015 17:09:19 -0700
parents dd25bd3ce1f9
children 32c7c2d219d2
comparison
equal deleted inserted replaced
410:d1a472464e57 411:e7b865f8f335
7 import logging 7 import logging
8 import datetime 8 import datetime
9 import dateutil.parser 9 import dateutil.parser
10 import collections 10 import collections
11 from werkzeug.utils import cached_property 11 from werkzeug.utils import cached_property
12 from piecrust.configuration import (Configuration, ConfigurationError, 12 from piecrust.configuration import (
13 Configuration, ConfigurationError,
13 parse_config_header) 14 parse_config_header)
14 from piecrust.routing import IRouteMetadataProvider 15 from piecrust.routing import IRouteMetadataProvider
15 16
16 17
17 logger = logging.getLogger(__name__) 18 logger = logging.getLogger(__name__)
239 return data 240 return data
240 241
241 242
242 def load_page(app, path, path_mtime=None): 243 def load_page(app, path, path_mtime=None):
243 try: 244 try:
244 return _do_load_page(app, path, path_mtime) 245 with app.env.timerScope('PageLoad'):
246 return _do_load_page(app, path, path_mtime)
245 except Exception as e: 247 except Exception as e:
246 logger.exception("Error loading page: %s" % 248 logger.exception(
249 "Error loading page: %s" %
247 os.path.relpath(path, app.root_dir)) 250 os.path.relpath(path, app.root_dir))
248 _, __, traceback = sys.exc_info() 251 _, __, traceback = sys.exc_info()
249 raise PageLoadingError(path, e).with_traceback(traceback) 252 raise PageLoadingError(path, e).with_traceback(traceback)
250 253
251 254
253 # Check the cache first. 256 # Check the cache first.
254 cache = app.cache.getCache('pages') 257 cache = app.cache.getCache('pages')
255 cache_path = hashlib.md5(path.encode('utf8')).hexdigest() + '.json' 258 cache_path = hashlib.md5(path.encode('utf8')).hexdigest() + '.json'
256 page_time = path_mtime or os.path.getmtime(path) 259 page_time = path_mtime or os.path.getmtime(path)
257 if cache.isValid(cache_path, page_time): 260 if cache.isValid(cache_path, page_time):
258 cache_data = json.loads(cache.read(cache_path), 261 cache_data = json.loads(
262 cache.read(cache_path),
259 object_pairs_hook=collections.OrderedDict) 263 object_pairs_hook=collections.OrderedDict)
260 config = PageConfiguration(values=cache_data['config'], 264 config = PageConfiguration(
265 values=cache_data['config'],
261 validate=False) 266 validate=False)
262 content = json_load_segments(cache_data['content']) 267 content = json_load_segments(cache_data['content'])
263 return config, content, True 268 return config, content, True
264 269
265 # Nope, load the page from the source file. 270 # Nope, load the page from the source file.
266 logger.debug("Loading page configuration from: %s" % path) 271 logger.debug("Loading page configuration from: %s" % path)
267 with codecs.open(path, 'r', 'utf-8') as fp: 272 with codecs.open(path, 'r', 'utf-8') as fp:
268 raw = fp.read() 273 raw = fp.read()
269 header, offset = parse_config_header(raw) 274 header, offset = parse_config_header(raw)
270 275
271 if not 'format' in header: 276 if 'format' not in header:
272 auto_formats = app.config.get('site/auto_formats') 277 auto_formats = app.config.get('site/auto_formats')
273 name, ext = os.path.splitext(path) 278 name, ext = os.path.splitext(path)
274 header['format'] = auto_formats.get(ext, None) 279 header['format'] = auto_formats.get(ext, None)
275 280
276 config = PageConfiguration(header) 281 config = PageConfiguration(header)