Mercurial > piecrust2

comparison piecrust/appconfig.py @ 584:9ccc933ac2c7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
internal: Refactor the app configuration class. * Moved to its own module. * More extensible validation. * Allow easier setup of defaults so `showconfig` shows more useful stuff.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 01 Jan 2016 23:18:26 -0800
parents
children 25df894f9ab9
comparison
equal deleted inserted replaced
583:1eda551ee681 584:9ccc933ac2c7
1 import re
2 import os.path
3 import copy
4 import json
5 import urllib
6 import logging
7 import hashlib
8 import collections
9 import yaml
10 from piecrust import (
11 APP_VERSION, CACHE_VERSION,
12 DEFAULT_FORMAT, DEFAULT_TEMPLATE_ENGINE, DEFAULT_POSTS_FS,
13 DEFAULT_DATE_FORMAT, DEFAULT_THEME_SOURCE)
14 from piecrust.cache import NullCache
15 from piecrust.configuration import (
16 Configuration, ConfigurationError, ConfigurationLoader,
17 merge_dicts, visit_dict)
18 from piecrust.sources.base import REALM_USER, REALM_THEME
19
20
21 logger = logging.getLogger(__name__)
22
23
24 class VariantNotFoundError(Exception):
25 def __init__(self, variant_path, message=None):
26 super(VariantNotFoundError, self).__init__(
27 message or ("No such configuration variant: %s" %
28 variant_path))
29
30
31 class PieCrustConfiguration(Configuration):
32 def __init__(self, paths=None, cache=None, values=None, validate=True):
33 super(PieCrustConfiguration, self).__init__(values, validate)
34 self.paths = paths
35 self.cache = cache or NullCache()
36 self.fixups = []
37
38 def applyVariant(self, variant_path, raise_if_not_found=True):
39 variant = self.get(variant_path)
40 if variant is None:
41 if raise_if_not_found:
42 raise VariantNotFoundError(variant_path)
43 return
44 if not isinstance(variant, dict):
45 raise VariantNotFoundError(
46 variant_path,
47 "Configuration variant '%s' is not an array. "
48 "Check your configuration file." % variant_path)
49 self.merge(variant)
50
51 def _load(self):
52 if self.paths is None:
53 self._values = self._validateAll({})
54 return
55
56 path_times = [os.path.getmtime(p) for p in self.paths]
57
58 cache_key_hash = hashlib.md5(
59 ("version=%s&cache=%d" % (
60 APP_VERSION, CACHE_VERSION)).encode('utf8'))
61 for p in self.paths:
62 cache_key_hash.update(("&path=%s" % p).encode('utf8'))
63 cache_key = cache_key_hash.hexdigest()
64
65 if self.cache.isValid('config.json', path_times):
66 logger.debug("Loading configuration from cache...")
67 config_text = self.cache.read('config.json')
68 self._values = json.loads(
69 config_text,
70 object_pairs_hook=collections.OrderedDict)
71
72 actual_cache_key = self._values.get('__cache_key')
73 if actual_cache_key == cache_key:
74 self._values['__cache_valid'] = True
75 return
76 logger.debug("Outdated cache key '%s' (expected '%s')." % (
77 actual_cache_key, cache_key))
78
79 logger.debug("Loading configuration from: %s" % self.paths)
80 values = {}
81 for i, p in enumerate(self.paths):
82 with open(p, 'r', encoding='utf-8') as fp:
83 loaded_values = yaml.load(
84 fp.read(),
85 Loader=ConfigurationLoader)
86 if loaded_values is None:
87 loaded_values = {}
88 for fixup in self.fixups:
89 fixup(i, loaded_values)
90 merge_dicts(values, loaded_values)
91
92 for fixup in self.fixups:
93 fixup(len(self.paths), values)
94
95 self._values = self._validateAll(values)
96
97 logger.debug("Caching configuration...")
98 self._values['__cache_key'] = cache_key
99 config_text = json.dumps(self._values)
100 self.cache.write('config.json', config_text)
101
102 self._values['__cache_valid'] = False
103
104 def _validateAll(self, values):
105 if values is None:
106 values = {}
107
108 # Add the loaded values to the default configuration.
109 values = merge_dicts(copy.deepcopy(default_configuration), values)
110
111 # Figure out if we need to generate the configuration for the
112 # default content model.
113 sitec = values.setdefault('site', {})
114 if (
115 ('sources' not in sitec and
116 'routes' not in sitec and
117 'taxonomies' not in sitec) or
118 sitec.get('use_default_content')):
119 logger.debug("Generating default content model...")
120 values = self._generateDefaultContentModel(values)
121
122 # Add a section for our cached information.
123 cachec = collections.OrderedDict()
124 values['__cache'] = cachec
125 cache_writer = _ConfigCacheWriter(cachec)
126 globs = globals()
127
128 def _visitor(path, val, parent_val, parent_key):
129 callback_name = '_validate_' + path.replace('/', '_')
130 callback = globs.get(callback_name)
131 if callback:
132 val2 = callback(val, values, cache_writer)
133 if val2 is None:
134 raise Exception("Validator '%s' isn't returning a "
135 "coerced value." % callback_name)
136 parent_val[parent_key] = val2
137
138 visit_dict(values, _visitor)
139
140 return values
141
142 def _generateDefaultContentModel(self, values):
143 dcmcopy = copy.deepcopy(default_content_model_base)
144 values = merge_dicts(dcmcopy, values)
145
146 dcm = get_default_content_model(values)
147 values = merge_dicts(dcm, values)
148
149 blogsc = values['site'].get('blogs')
150 if blogsc is None:
151 blogsc = ['posts']
152 values['site']['blogs'] = blogsc
153
154 is_only_blog = (len(blogsc) == 1)
155 for blog_name in blogsc:
156 blog_cfg = get_default_content_model_for_blog(
157 blog_name, is_only_blog, values)
158 values = merge_dicts(blog_cfg, values)
159
160 return values
161
162
163 class _ConfigCacheWriter(object):
164 def __init__(self, cache_dict):
165 self._cache_dict = cache_dict
166
167 def write(self, name, val):
168 logger.debug("Caching configuration item '%s' = %s" % (name, val))
169 self._cache_dict[name] = val
170
171
172 default_configuration = collections.OrderedDict({
173 'site': collections.OrderedDict({
174 'title': "Untitled PieCrust website",
175 'root': '/',
176 'default_format': DEFAULT_FORMAT,
177 'default_template_engine': DEFAULT_TEMPLATE_ENGINE,
178 'enable_gzip': True,
179 'pretty_urls': False,
180 'trailing_slash': False,
181 'date_format': DEFAULT_DATE_FORMAT,
182 'auto_formats': collections.OrderedDict([
183 ('html', ''),
184 ('md', 'markdown'),
185 ('textile', 'textile')]),
186 'default_auto_format': 'md',
187 'pagination_suffix': '/%num%',
188 'slugify_mode': 'encode',
189 'themes_sources': [DEFAULT_THEME_SOURCE],
190 'cache_time': 28800,
191 'enable_debug_info': True,
192 'show_debug_info': False,
193 'use_default_content': True
194 }),
195 'baker': collections.OrderedDict({
196 'no_bake_setting': 'draft'
197 })
198 })
199
200
201 default_content_model_base = collections.OrderedDict({
202 'site': collections.OrderedDict({
203 'posts_fs': DEFAULT_POSTS_FS,
204 'date_format': DEFAULT_DATE_FORMAT,
205 'default_page_layout': 'default',
206 'default_post_layout': 'post',
207 'post_url': '%year%/%month%/%day%/%slug%',
208 'tag_url': 'tag/%tag%',
209 'category_url': '%category%',
210 'posts_per_page': 5
211 })
212 })
213
214
215 def get_default_content_model(values):
216 default_layout = values['site']['default_page_layout']
217 return collections.OrderedDict({
218 'site': collections.OrderedDict({
219 'sources': collections.OrderedDict({
220 'pages': {
221 'type': 'default',
222 'ignore_missing_dir': True,
223 'data_endpoint': 'site.pages',
224 'default_layout': default_layout,
225 'item_name': 'page'
226 }
227 }),
228 'routes': [
229 {
230 'url': '/%path:slug%',
231 'source': 'pages',
232 'func': 'pcurl(slug)'
233 }
234 ],
235 'taxonomies': collections.OrderedDict({
236 'tags': {
237 'multiple': True,
238 'term': 'tag'
239 },
240 'categories': {
241 'term': 'category'
242 }
243 })
244 })
245 })
246
247
248 def get_default_content_model_for_blog(blog_name, is_only_blog, values):
249 posts_fs = values['site']['posts_fs']
250 blog_cfg = values.get(blog_name, {})
251
252 if is_only_blog:
253 url_prefix = ''
254 tax_page_prefix = ''
255 fs_endpoint = 'posts'
256 data_endpoint = 'blog'
257 item_name = 'post'
258 else:
259 url_prefix = blog_name + '/'
260 tax_page_prefix = blog_name + '/'
261 fs_endpoint = 'posts/%s' % blog_name
262 data_endpoint = blog_name
263 item_name = '%s-post' % blog_name
264
265 items_per_page = blog_cfg.get(
266 'posts_per_page', values['site']['posts_per_page'])
267 date_format = blog_cfg.get(
268 'date_format', values['site']['date_format'])
269 default_layout = blog_cfg.get(
270 'default_layout', values['site']['default_post_layout'])
271
272 post_url = '/' + blog_cfg.get(
273 'post_url',
274 url_prefix + values['site']['post_url']).lstrip('/')
275 tag_url = '/' + blog_cfg.get(
276 'tag_url',
277 url_prefix + values['site']['tag_url']).lstrip('/')
278 category_url = '/' + blog_cfg.get(
279 'category_url',
280 url_prefix + values['site']['category_url']).lstrip('/')
281
282 return collections.OrderedDict({
283 'site': collections.OrderedDict({
284 'sources': collections.OrderedDict({
285 blog_name: collections.OrderedDict({
286 'type': 'posts/%s' % posts_fs,
287 'fs_endpoint': fs_endpoint,
288 'data_endpoint': data_endpoint,
289 'item_name': item_name,
290 'ignore_missing_dir': True,
291 'data_type': 'blog',
292 'items_per_page': items_per_page,
293 'date_format': date_format,
294 'default_layout': default_layout,
295 'taxonomy_pages': collections.OrderedDict({
296 'tags': ('pages:%s_tag.%%ext%%;'
297 'theme_pages:_tag.%%ext%%' %
298 tax_page_prefix),
299 'categories': ('pages:%s_category.%%ext%%;'
300 'theme_pages:_category.%%ext%%' %
301 tax_page_prefix)
302 })
303 })
304 }),
305 'routes': [
306 {
307 'url': post_url,
308 'source': blog_name,
309 'func': 'pcposturl(year,month,day,slug)'
310 },
311 {
312 'url': tag_url,
313 'source': blog_name,
314 'taxonomy': 'tags',
315 'func': 'pctagurl(tag)'
316 },
317 {
318 'url': category_url,
319 'source': blog_name,
320 'taxonomy': 'categories',
321 'func': 'pccaturl(category)'
322 }
323 ]
324 })
325 })
326
327
328 # Configuration value validators.
329 #
330 # Make sure we have basic site stuff.
331 def _validate_site(v, values, cache):
332 sources = v.get('sources')
333 if not sources:
334 raise ConfigurationError("No sources were defined.")
335 routes = v.get('routes')
336 if not routes:
337 raise ConfigurationError("No routes were defined.")
338 taxonomies = v.get('taxonomies')
339 if taxonomies is None:
340 v['taxonomies'] = {}
341 return v
342
343 # Make sure the site root starts and ends with a slash.
344 def _validate_site_root(v, values, cache):
345 if not v.startswith('/'):
346 raise ConfigurationError("The `site/root` setting must start "
347 "with a slash.")
348 root_url = urllib.parse.quote(v.rstrip('/') + '/')
349 return root_url
350
351
352 # Cache auto-format regexes, check that `.html` is in there.
353 def _validate_site_auto_formats(v, values, cache):
354 if not isinstance(v, dict):
355 raise ConfigurationError("The 'site/auto_formats' setting must be "
356 "a dictionary.")
357
358 v.setdefault('html', values['site']['default_format'])
359 auto_formats_re = r"\.(%s)$" % (
360 '|'.join(
361 [re.escape(i) for i in list(v.keys())]))
362 cache.write('auto_formats_re', auto_formats_re)
363 return v
364
365
366 # Check that the default auto-format is known.
367 def _validate_site_default_auto_format(v, values, cache):
368 if v not in values['site']['auto_formats']:
369 raise ConfigurationError(
370 "Default auto-format '%s' is not declared." % v)
371 return v
372
373
374 # Cache pagination suffix regex and format.
375 def _validate_site_pagination_suffix(v, values, cache):
376 if len(v) == 0 or v[0] != '/':
377 raise ConfigurationError("The 'site/pagination_suffix' setting "
378 "must start with a slash.")
379 if '%num%' not in v:
380 raise ConfigurationError("The 'site/pagination_suffix' setting "
381 "must contain the '%num%' placeholder.")
382
383 pgn_suffix_fmt = v.replace('%num%', '%(num)d')
384 cache.write('pagination_suffix_format', pgn_suffix_fmt)
385
386 pgn_suffix_re = re.escape(v)
387 pgn_suffix_re = (pgn_suffix_re.replace("\\%num\\%", "(?P<num>\\d+)") +
388 '$')
389 cache.write('pagination_suffix_re', pgn_suffix_re)
390 return v
391
392
393 # Make sure theme sources is a list.
394 def _validate_site_theme_sources(v, values, cache):
395 if not isinstance(v, list):
396 v = [v]
397 return v
398
399
400 def _validate_site_sources(v, values, cache):
401 # Basic checks.
402 if not v:
403 raise ConfigurationError("There are no sources defined.")
404 if not isinstance(v, dict):
405 raise ConfigurationError("The 'site/sources' setting must be a "
406 "dictionary.")
407
408 # Add the theme page source if no sources were defined in the theme
409 # configuration itself.
410 has_any_theme_source = False
411 for sn, sc in v.items():
412 if sc.get('realm') == REALM_THEME:
413 has_any_theme_source = True
414 break
415 if not has_any_theme_source:
416 v['theme_pages'] = {
417 'theme_source': True,
418 'fs_endpoint': 'pages',
419 'data_endpoint': 'site/pages',
420 'item_name': 'page',
421 'realm': REALM_THEME}
422 values['site']['routes'].append({
423 'url': '/%path:slug%',
424 'source': 'theme_pages',
425 'func': 'pcurl(slug)'})
426
427 # Sources have the `default` scanner by default, duh. Also, a bunch
428 # of other default values for other configuration stuff.
429 for sn, sc in v.items():
430 if not isinstance(sc, dict):
431 raise ConfigurationError("All sources in 'site/sources' must "
432 "be dictionaries.")
433 sc.setdefault('type', 'default')
434 sc.setdefault('fs_endpoint', sn)
435 sc.setdefault('ignore_missing_dir', False)
436 sc.setdefault('data_endpoint', sn)
437 sc.setdefault('data_type', 'iterator')
438 sc.setdefault('item_name', sn)
439 sc.setdefault('items_per_page', 5)
440 sc.setdefault('date_format', DEFAULT_DATE_FORMAT)
441 sc.setdefault('realm', REALM_USER)
442
443 return v
444
445
446 def _validate_site_routes(v, values, cache):
447 if not v:
448 raise ConfigurationError("There are no routes defined.")
449 if not isinstance(v, list):
450 raise ConfigurationError("The 'site/routes' setting must be a "
451 "list.")
452
453 # Check routes are referencing correct sources, have default
454 # values, etc.
455 for rc in v:
456 if not isinstance(rc, dict):
457 raise ConfigurationError("All routes in 'site/routes' must be "
458 "dictionaries.")
459 rc_url = rc.get('url')
460 if not rc_url:
461 raise ConfigurationError("All routes in 'site/routes' must "
462 "have an 'url'.")
463 if rc_url[0] != '/':
464 raise ConfigurationError("Route URLs must start with '/'.")
465 if rc.get('source') is None:
466 raise ConfigurationError("Routes must specify a source.")
467 if rc['source'] not in list(values['site']['sources'].keys()):
468 raise ConfigurationError("Route is referencing unknown "
469 "source: %s" % rc['source'])
470 rc.setdefault('taxonomy', None)
471 rc.setdefault('page_suffix', '/%num%')
472
473 return v
474
475
476 def _validate_site_taxonomies(v, values, cache):
477 for tn, tc in v.items():
478 tc.setdefault('multiple', False)
479 tc.setdefault('term', tn)
480 tc.setdefault('page', '_%s.%%ext%%' % tc['term'])
481
482 # Validate endpoints, and make sure the theme has a default source.
483 reserved_endpoints = set(['piecrust', 'site', 'page', 'route',
484 'assets', 'pagination', 'siblings',
485 'family'])
486 for name, src in values['site']['sources'].items():
487 endpoint = src['data_endpoint']
488 if endpoint in reserved_endpoints:
489 raise ConfigurationError(
490 "Source '%s' is using a reserved endpoint name: %s" %
491 (name, endpoint))
492
493 return v
494
495
496 def _validate_site_plugins(v, values, cache):
497 if isinstance(v, str):
498 v = v.split(',')
499 elif not isinstance(v, list):
500 raise ConfigurationError(
501 "The 'site/plugins' setting must be an array, or a "
502 "comma-separated list.")
503 return v
504