Mercurial > piecrust2

comparison piecrust/app.py @ 584:9ccc933ac2c7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
internal: Refactor the app configuration class. * Moved to its own module. * More extensible validation. * Allow easier setup of defaults so `showconfig` shows more useful stuff.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 01 Jan 2016 23:18:26 -0800
parents 6b6c5442c790
children 59268b4d8c71
comparison
equal deleted inserted replaced
583:1eda551ee681 584:9ccc933ac2c7
1 import re
2 import json
3 import time 1 import time
4 import os.path 2 import os.path
5 import urllib.parse
6 import codecs
7 import hashlib 3 import hashlib
8 import logging 4 import logging
9 import collections
10 import yaml
11 from werkzeug.utils import cached_property 5 from werkzeug.utils import cached_property
12 from piecrust import ( 6 from piecrust import (
13 APP_VERSION, RESOURCES_DIR, 7 RESOURCES_DIR,
14 CACHE_DIR, TEMPLATES_DIR, ASSETS_DIR, 8 CACHE_DIR, TEMPLATES_DIR, ASSETS_DIR,
15 THEME_DIR, 9 THEME_DIR,
16 CONFIG_PATH, THEME_CONFIG_PATH, 10 CONFIG_PATH, THEME_CONFIG_PATH)
17 DEFAULT_FORMAT, DEFAULT_TEMPLATE_ENGINE, DEFAULT_POSTS_FS, 11 from piecrust.appconfig import PieCrustConfiguration
18 DEFAULT_DATE_FORMAT, DEFAULT_THEME_SOURCE) 12 from piecrust.cache import ExtensibleCache, NullExtensibleCache
19 from piecrust.cache import ExtensibleCache, NullCache, NullExtensibleCache
20 from piecrust.plugins.base import PluginLoader 13 from piecrust.plugins.base import PluginLoader
21 from piecrust.environment import StandardEnvironment 14 from piecrust.environment import StandardEnvironment
22 from piecrust.configuration import ( 15 from piecrust.configuration import ConfigurationError
23 Configuration, ConfigurationError, ConfigurationLoader,
24 merge_dicts)
25 from piecrust.routing import Route 16 from piecrust.routing import Route
26 from piecrust.sources.base import REALM_USER, REALM_THEME 17 from piecrust.sources.base import REALM_THEME
27 from piecrust.taxonomies import Taxonomy 18 from piecrust.taxonomies import Taxonomy
28 19
29 20
30 logger = logging.getLogger(__name__) 21 logger = logging.getLogger(__name__)
31
32
33 CACHE_VERSION = 22
34
35
36 class VariantNotFoundError(Exception):
37 def __init__(self, variant_path, message=None):
38 super(VariantNotFoundError, self).__init__(
39 message or ("No such configuration variant: %s" % variant_path))
40
41
42 class PieCrustConfiguration(Configuration):
43 def __init__(self, paths=None, cache=None, values=None, validate=True):
44 super(PieCrustConfiguration, self).__init__(values, validate)
45 self.paths = paths
46 self.cache = cache or NullCache()
47 self.fixups = []
48
49 def applyVariant(self, variant_path, raise_if_not_found=True):
50 variant = self.get(variant_path)
51 if variant is None:
52 if raise_if_not_found:
53 raise VariantNotFoundError(variant_path)
54 return
55 if not isinstance(variant, dict):
56 raise VariantNotFoundError(variant_path,
57 "Configuration variant '%s' is not an array. "
58 "Check your configuration file." % variant_path)
59 self.merge(variant)
60
61 def _load(self):
62 if self.paths is None:
63 self._values = self._validateAll({})
64 return
65
66 path_times = [os.path.getmtime(p) for p in self.paths]
67 cache_key_hash = hashlib.md5(("version=%s&cache=%d" % (
68 APP_VERSION, CACHE_VERSION)).encode('utf8'))
69 for p in self.paths:
70 cache_key_hash.update(("&path=%s" % p).encode('utf8'))
71 cache_key = cache_key_hash.hexdigest()
72
73 if self.cache.isValid('config.json', path_times):
74 logger.debug("Loading configuration from cache...")
75 config_text = self.cache.read('config.json')
76 self._values = json.loads(config_text,
77 object_pairs_hook=collections.OrderedDict)
78
79 actual_cache_key = self._values.get('__cache_key')
80 if actual_cache_key == cache_key:
81 self._values['__cache_valid'] = True
82 return
83 logger.debug("Outdated cache key '%s' (expected '%s')." % (
84 actual_cache_key, cache_key))
85
86 values = {}
87 logger.debug("Loading configuration from: %s" % self.paths)
88 for i, p in enumerate(self.paths):
89 with codecs.open(p, 'r', 'utf-8') as fp:
90 loaded_values = yaml.load(fp.read(),
91 Loader=ConfigurationLoader)
92 if loaded_values is None:
93 loaded_values = {}
94 for fixup in self.fixups:
95 fixup(i, loaded_values)
96 merge_dicts(values, loaded_values)
97
98 for fixup in self.fixups:
99 fixup(len(self.paths), values)
100
101 self._values = self._validateAll(values)
102
103 logger.debug("Caching configuration...")
104 self._values['__cache_key'] = cache_key
105 config_text = json.dumps(self._values)
106 self.cache.write('config.json', config_text)
107 self._values['__cache_valid'] = False
108
109 def _validateAll(self, values):
110 # Put all the defaults in the `site` section.
111 default_sitec = collections.OrderedDict({
112 'title': "Untitled PieCrust website",
113 'root': '/',
114 'default_format': DEFAULT_FORMAT,
115 'default_template_engine': DEFAULT_TEMPLATE_ENGINE,
116 'enable_gzip': True,
117 'pretty_urls': False,
118 'trailing_slash': False,
119 'date_format': DEFAULT_DATE_FORMAT,
120 'auto_formats': collections.OrderedDict([
121 ('html', ''),
122 ('md', 'markdown'),
123 ('textile', 'textile')]),
124 'default_auto_format': 'md',
125 'pagination_suffix': '/%num%',
126 'slugify_mode': 'encode',
127 'plugins': None,
128 'themes_sources': [DEFAULT_THEME_SOURCE],
129 'cache_time': 28800,
130 'enable_debug_info': True,
131 'show_debug_info': False,
132 'use_default_content': True
133 })
134 sitec = values.get('site')
135 if sitec is None:
136 sitec = collections.OrderedDict()
137 for key, val in default_sitec.items():
138 sitec.setdefault(key, val)
139 values['site'] = sitec
140
141 # Add a section for our cached information.
142 cachec = collections.OrderedDict()
143 values['__cache'] = cachec
144
145 # Make sure the site root starts and ends with a slash.
146 if not sitec['root'].startswith('/'):
147 raise ConfigurationError("The `site/root` setting must start "
148 "with a slash.")
149 sitec['root'] = urllib.parse.quote(sitec['root'].rstrip('/') + '/')
150
151 # Cache auto-format regexes.
152 if not isinstance(sitec['auto_formats'], dict):
153 raise ConfigurationError("The 'site/auto_formats' setting must be "
154 "a dictionary.")
155 # Check that `.html` is in there.
156 sitec['auto_formats'].setdefault('html', sitec['default_format'])
157 cachec['auto_formats_re'] = r"\.(%s)$" % (
158 '|'.join(
159 [re.escape(i) for i in
160 list(sitec['auto_formats'].keys())]))
161 if sitec['default_auto_format'] not in sitec['auto_formats']:
162 raise ConfigurationError("Default auto-format '%s' is not "
163 "declared." %
164 sitec['default_auto_format'])
165
166 # Cache pagination suffix regex and format.
167 pgn_suffix = sitec['pagination_suffix']
168 if len(pgn_suffix) == 0 or pgn_suffix[0] != '/':
169 raise ConfigurationError("The 'site/pagination_suffix' setting "
170 "must start with a slash.")
171 if '%num%' not in pgn_suffix:
172 raise ConfigurationError("The 'site/pagination_suffix' setting "
173 "must contain the '%num%' placeholder.")
174
175 pgn_suffix_fmt = pgn_suffix.replace('%num%', '%(num)d')
176 cachec['pagination_suffix_format'] = pgn_suffix_fmt
177
178 pgn_suffix_re = re.escape(pgn_suffix)
179 pgn_suffix_re = (pgn_suffix_re.replace("\\%num\\%", "(?P<num>\\d+)") +
180 '$')
181 cachec['pagination_suffix_re'] = pgn_suffix_re
182
183 # Make sure theme sources is a list.
184 if not isinstance(sitec['themes_sources'], list):
185 sitec['themes_sources'] = [sitec['themes_sources']]
186
187 # Figure out if we need to validate sources/routes, or auto-generate
188 # them from simple blog settings.
189 orig_sources = sitec.get('sources')
190 orig_routes = sitec.get('routes')
191 orig_taxonomies = sitec.get('taxonomies')
192 use_default_content = sitec.get('use_default_content')
193 if (orig_sources is None or orig_routes is None or
194 orig_taxonomies is None or use_default_content):
195
196 # Setup defaults for various settings.
197 posts_fs = sitec.setdefault('posts_fs', DEFAULT_POSTS_FS)
198 blogsc = sitec.setdefault('blogs', ['posts'])
199
200 g_page_layout = sitec.get('default_page_layout', 'default')
201 g_post_layout = sitec.get('default_post_layout', 'post')
202 g_post_url = sitec.get('post_url', '%year%/%month%/%day%/%slug%')
203 g_tag_url = sitec.get('tag_url', 'tag/%tag%')
204 g_category_url = sitec.get('category_url', '%category%')
205 g_posts_per_page = sitec.get('posts_per_page', 5)
206 g_posts_filters = sitec.get('posts_filters')
207 g_date_format = sitec.get('date_format', DEFAULT_DATE_FORMAT)
208
209 # The normal pages and tags/categories.
210 sourcesc = collections.OrderedDict()
211 sourcesc['pages'] = {
212 'type': 'default',
213 'ignore_missing_dir': True,
214 'data_endpoint': 'site.pages',
215 'default_layout': g_page_layout,
216 'item_name': 'page'}
217 sitec['sources'] = sourcesc
218
219 routesc = []
220 routesc.append({
221 'url': '/%path:slug%',
222 'source': 'pages',
223 'func': 'pcurl(slug)'})
224 sitec['routes'] = routesc
225
226 taxonomiesc = collections.OrderedDict()
227 taxonomiesc['tags'] = {
228 'multiple': True,
229 'term': 'tag'}
230 taxonomiesc['categories'] = {
231 'term': 'category'}
232 sitec['taxonomies'] = taxonomiesc
233
234 # Setup sources/routes/taxonomies for each blog.
235 for blog_name in blogsc:
236 blogc = values.get(blog_name, {})
237 url_prefix = blog_name + '/'
238 fs_endpoint = 'posts/%s' % blog_name
239 data_endpoint = blog_name
240 item_name = '%s-post' % blog_name
241 items_per_page = blogc.get('posts_per_page', g_posts_per_page)
242 items_filters = blogc.get('posts_filters', g_posts_filters)
243 date_format = blogc.get('date_format', g_date_format)
244 if len(blogsc) == 1:
245 url_prefix = ''
246 fs_endpoint = 'posts'
247 data_endpoint = 'blog'
248 item_name = 'post'
249 sourcesc[blog_name] = {
250 'type': 'posts/%s' % posts_fs,
251 'fs_endpoint': fs_endpoint,
252 'data_endpoint': data_endpoint,
253 'ignore_missing_dir': True,
254 'data_type': 'blog',
255 'item_name': item_name,
256 'items_per_page': items_per_page,
257 'items_filters': items_filters,
258 'date_format': date_format,
259 'default_layout': g_post_layout}
260 tax_page_prefix = ''
261 if len(blogsc) > 1:
262 tax_page_prefix = blog_name + '/'
263 sourcesc[blog_name]['taxonomy_pages'] = {
264 'tags': ('pages:%s_tag.%%ext%%;'
265 'theme_pages:_tag.%%ext%%' %
266 tax_page_prefix),
267 'categories': ('pages:%s_category.%%ext%%;'
268 'theme_pages:_category.%%ext%%' %
269 tax_page_prefix)}
270
271 post_url = blogc.get('post_url', url_prefix + g_post_url)
272 post_url = '/' + post_url.lstrip('/')
273 tag_url = blogc.get('tag_url', url_prefix + g_tag_url)
274 tag_url = '/' + tag_url.lstrip('/')
275 category_url = blogc.get('category_url', url_prefix + g_category_url)
276 category_url = '/' + category_url.lstrip('/')
277 routesc.append({'url': post_url, 'source': blog_name,
278 'func': 'pcposturl(year,month,day,slug)'})
279 routesc.append({'url': tag_url, 'source': blog_name,
280 'taxonomy': 'tags',
281 'func': 'pctagurl(tag)'})
282 routesc.append({'url': category_url, 'source': blog_name,
283 'taxonomy': 'categories',
284 'func': 'pccaturl(category)'})
285
286 # If the user defined some additional sources/routes/taxonomies,
287 # add them to the default ones. For routes, the order matters,
288 # though, so we make sure to add the user routes at the front
289 # of the list so they're evaluated first.
290 if orig_sources:
291 sourcesc.update(orig_sources)
292 sitec['sources'] = sourcesc
293 if orig_routes:
294 routesc = orig_routes + routesc
295 sitec['routes'] = routesc
296 if orig_taxonomies:
297 taxonomiesc.update(orig_taxonomies)
298 sitec['taxonomies'] = taxonomiesc
299
300 # Validate sources/routes.
301 sourcesc = sitec.get('sources')
302 routesc = sitec.get('routes')
303 if not sourcesc:
304 raise ConfigurationError("There are no sources defined.")
305 if not routesc:
306 raise ConfigurationError("There are no routes defined.")
307 if not isinstance(sourcesc, dict):
308 raise ConfigurationError("The 'site/sources' setting must be a "
309 "dictionary.")
310 if not isinstance(routesc, list):
311 raise ConfigurationError("The 'site/routes' setting must be a "
312 "list.")
313
314 # Add the theme page source if no sources were defined in the theme
315 # configuration itself.
316 has_any_theme_source = False
317 for sn, sc in sourcesc.items():
318 if sc.get('realm') == REALM_THEME:
319 has_any_theme_source = True
320 break
321 if not has_any_theme_source:
322 sitec['sources']['theme_pages'] = {
323 'theme_source': True,
324 'fs_endpoint': 'pages',
325 'data_endpoint': 'site/pages',
326 'item_name': 'page',
327 'realm': REALM_THEME}
328 sitec['routes'].append({
329 'url': '/%path:slug%',
330 'source': 'theme_pages',
331 'func': 'pcurl(slug)'})
332
333 # Sources have the `default` scanner by default, duh. Also, a bunch
334 # of other default values for other configuration stuff.
335 for sn, sc in sourcesc.items():
336 if not isinstance(sc, dict):
337 raise ConfigurationError("All sources in 'site/sources' must "
338 "be dictionaries.")
339 sc.setdefault('type', 'default')
340 sc.setdefault('fs_endpoint', sn)
341 sc.setdefault('ignore_missing_dir', False)
342 sc.setdefault('data_endpoint', sn)
343 sc.setdefault('data_type', 'iterator')
344 sc.setdefault('item_name', sn)
345 sc.setdefault('items_per_page', 5)
346 sc.setdefault('date_format', DEFAULT_DATE_FORMAT)
347 sc.setdefault('realm', REALM_USER)
348
349 # Check routes are referencing correct routes, have default
350 # values, etc.
351 for rc in routesc:
352 if not isinstance(rc, dict):
353 raise ConfigurationError("All routes in 'site/routes' must be "
354 "dictionaries.")
355 rc_url = rc.get('url')
356 if not rc_url:
357 raise ConfigurationError("All routes in 'site/routes' must "
358 "have an 'url'.")
359 if rc_url[0] != '/':
360 raise ConfigurationError("Route URLs must start with '/'.")
361 if rc.get('source') is None:
362 raise ConfigurationError("Routes must specify a source.")
363 if rc['source'] not in list(sourcesc.keys()):
364 raise ConfigurationError("Route is referencing unknown "
365 "source: %s" % rc['source'])
366 rc.setdefault('taxonomy', None)
367 rc.setdefault('page_suffix', '/%num%')
368
369 # Validate taxonomies.
370 sitec.setdefault('taxonomies', {})
371 taxonomiesc = sitec.get('taxonomies')
372 for tn, tc in taxonomiesc.items():
373 tc.setdefault('multiple', False)
374 tc.setdefault('term', tn)
375 tc.setdefault('page', '_%s.%%ext%%' % tc['term'])
376
377 # Validate endpoints, and make sure the theme has a default source.
378 reserved_endpoints = set(['piecrust', 'site', 'page', 'route',
379 'assets', 'pagination', 'siblings',
380 'family'])
381 for name, src in sitec['sources'].items():
382 endpoint = src['data_endpoint']
383 if endpoint in reserved_endpoints:
384 raise ConfigurationError(
385 "Source '%s' is using a reserved endpoint name: %s" %
386 (name, endpoint))
387
388 # Make sure the `plugins` setting is a list.
389 user_plugins = sitec.get('plugins')
390 if user_plugins:
391 if isinstance(user_plugins, str):
392 sitec['plugins'] = user_plugins.split(',')
393 elif not isinstance(user_plugins, list):
394 raise ConfigurationError(
395 "The 'site/plugins' setting must be an array, or a "
396 "comma-separated list.")
397
398 # Done validating!
399 return values
400 22
401 23
402 class PieCrust(object): 24 class PieCrust(object):
403 def __init__(self, root_dir, cache=True, debug=False, theme_site=False, 25 def __init__(self, root_dir, cache=True, debug=False, theme_site=False,
404 env=None): 26 env=None):
452 self.env.stepTimer('SiteConfigLoad', time.perf_counter() - start_time) 74 self.env.stepTimer('SiteConfigLoad', time.perf_counter() - start_time)
453 return config 75 return config
454 76
455 @cached_property 77 @cached_property
456 def assets_dirs(self): 78 def assets_dirs(self):
457 assets_dirs = self._get_configurable_dirs(ASSETS_DIR, 79 assets_dirs = self._get_configurable_dirs(
458 'site/assets_dirs') 80 ASSETS_DIR, 'site/assets_dirs')
459 81
460 # Also add the theme directory, if any. 82 # Also add the theme directory, if any.
461 if self.theme_dir: 83 if self.theme_dir:
462 default_theme_dir = os.path.join(self.theme_dir, ASSETS_DIR) 84 default_theme_dir = os.path.join(self.theme_dir, ASSETS_DIR)
463 if os.path.isdir(default_theme_dir): 85 if os.path.isdir(default_theme_dir):
465 87
466 return assets_dirs 88 return assets_dirs
467 89
468 @cached_property 90 @cached_property
469 def templates_dirs(self): 91 def templates_dirs(self):
470 templates_dirs = self._get_configurable_dirs(TEMPLATES_DIR, 92 templates_dirs = self._get_configurable_dirs(
471 'site/templates_dirs') 93 TEMPLATES_DIR, 'site/templates_dirs')
472 94
473 # Also, add the theme directory, if any. 95 # Also, add the theme directory, if any.
474 if self.theme_dir: 96 if self.theme_dir:
475 default_theme_dir = os.path.join(self.theme_dir, TEMPLATES_DIR) 97 default_theme_dir = os.path.join(self.theme_dir, TEMPLATES_DIR)
476 if os.path.isdir(default_theme_dir): 98 if os.path.isdir(default_theme_dir):
503 125
504 sources = [] 126 sources = []
505 for n, s in self.config.get('site/sources').items(): 127 for n, s in self.config.get('site/sources').items():
506 cls = defs.get(s['type']) 128 cls = defs.get(s['type'])
507 if cls is None: 129 if cls is None:
508 raise ConfigurationError("No such page source type: %s" % s['type']) 130 raise ConfigurationError("No such page source type: %s" %
131 s['type'])
509 src = cls(self, n, s) 132 src = cls(self, n, s)
510 sources.append(src) 133 sources.append(src)
511 return sources 134 return sources
512 135
513 @cached_property 136 @cached_property
546 return route 169 return route
547 return None 170 return None
548 171
549 def getTaxonomyRoute(self, tax_name, source_name): 172 def getTaxonomyRoute(self, tax_name, source_name):
550 for route in self.routes: 173 for route in self.routes:
551 if route.taxonomy_name == tax_name and route.source_name == source_name: 174 if (route.taxonomy_name == tax_name and
175 route.source_name == source_name):
552 return route 176 return route
553 return None 177 return None
554 178
555 def getTaxonomy(self, tax_name): 179 def getTaxonomy(self, tax_name):
556 for tax in self.taxonomies: 180 for tax in self.taxonomies: