piecrust2: piecrust/app.py comparison

comparison piecrust/app.py @ 3:f485ba500df3

Gigantic change to basically make PieCrust 2 vaguely functional. - Serving works, with debug window. - Baking works, multi-threading, with dependency handling. - Various things not implemented yet.

author	Ludovic Chabant <ludovic@chabant.com>
date	Sun, 10 Aug 2014 23:43:16 -0700
parents	aaa8fb7c8918
children	474c9882decf

comparison

equal deleted inserted replaced

-:40fa08b261b9
+:f485ba500df3
+import re
 import json
 import os.path
 import types
 import codecs
 import hashlib
 import logging
 import yaml
-from cache import SimpleCache
+from werkzeug.utils import cached_property
-from decorators import lazy_property
+from piecrust import (APP_VERSION,
-from plugins.base import PluginLoader
+CACHE_DIR, TEMPLATES_DIR,
-from environment import StandardEnvironment
+PLUGINS_DIR, THEME_DIR,
-from configuration import Configuration, merge_dicts
+CONFIG_PATH, THEME_CONFIG_PATH,
+DEFAULT_FORMAT, DEFAULT_TEMPLATE_ENGINE, DEFAULT_POSTS_FS,
+DEFAULT_DATE_FORMAT, DEFAULT_PLUGIN_SOURCE, DEFAULT_THEME_SOURCE)
-APP_VERSION = '2.0.0alpha'
+from piecrust.cache import ExtensibleCache, NullCache, NullExtensibleCache
-CACHE_VERSION = '2.0'
+from piecrust.plugins.base import PluginLoader
+from piecrust.environment import StandardEnvironment
-CACHE_DIR = '_cache'
+from piecrust.configuration import Configuration, ConfigurationError, merge_dicts
-TEMPLATES_DIR = '_content/templates'
+from piecrust.routing import Route
-PAGES_DIR = '_content/pages'
+from piecrust.sources.base import REALM_USER, REALM_THEME
-POSTS_DIR = '_content/posts'
+from piecrust.taxonomies import Taxonomy
-PLUGINS_DIR = '_content/plugins'
-THEME_DIR = '_content/theme'
-CONFIG_PATH = '_content/config.yml'
-THEME_CONFIG_PATH = '_content/theme_config.yml'
 logger = logging.getLogger(__name__)
+CACHE_VERSION = 10
 class VariantNotFoundError(Exception):
 def __init__(self, variant_path, message=None):
 super(VariantNotFoundError, self).__init__(
 message or ("No such configuration variant: %s" % variant_path))
 class PieCrustConfiguration(Configuration):
-def __init__(self, paths=None, cache_dir=False):
+def __init__(self, paths=None, cache=None, values=None, validate=True):
-super(PieCrustConfiguration, self).__init__()
+super(PieCrustConfiguration, self).__init__(values, validate)
 self.paths = paths
-self.cache_dir = cache_dir
+self.cache = cache or NullCache()
 self.fixups = []
 def applyVariant(self, variant_path, raise_if_not_found=True):
 variant = self.get(variant_path)
 if variant is None:
 def _load(self):
 if self.paths is None:
 self._values = self._validateAll({})
 return
-path_times = filter(self.paths,
+path_times = map(lambda p: os.path.getmtime(p), self.paths)
-lambda p: os.path.getmtime(p))
+cache_key = hashlib.md5("version=%s&cache=%d" % (
-cache_key = hashlib.md5("version=%s&cache=%s" % (
+APP_VERSION, CACHE_VERSION)).hexdigest()
-APP_VERSION, CACHE_VERSION))
+if self.cache.isValid('config.json', path_times):
-cache = None
+logger.debug("Loading configuration from cache...")
-if self.cache_dir:
+config_text = self.cache.read('config.json')
-cache = SimpleCache(self.cache_dir)
+self._values = json.loads(config_text)
-if cache is not None:
+actual_cache_key = self._values.get('__cache_key')
-if cache.isValid('config.json', path_times):
+if actual_cache_key == cache_key:
-config_text = cache.read('config.json')
+return
-self._values = json.loads(config_text)
+logger.debug("Outdated cache key '%s' (expected '%s')." % (
+actual_cache_key, cache_key))
-actual_cache_key = self._values.get('__cache_key')
-if actual_cache_key == cache_key:
-return
 values = {}
+logger.debug("Loading configuration from: %s" % self.paths)
 for i, p in enumerate(self.paths):
 with codecs.open(p, 'r', 'utf-8') as fp:
 loaded_values = yaml.load(fp.read())
+if loaded_values is None:
+loaded_values = {}
 for fixup in self.fixups:
 fixup(i, loaded_values)
 merge_dicts(values, loaded_values)
 for fixup in self.fixups:
 fixup(len(self.paths), values)
 self._values = self._validateAll(values)
-if cache is not None:
+logger.debug("Caching configuration...")
 self._values['__cache_key'] = cache_key
 config_text = json.dumps(self._values)
-cache.write('config.json', config_text)
+self.cache.write('config.json', config_text)
+def _validateAll(self, values):
+# Put all the defaults in the `site` section.
+default_sitec = {
+'title': "Untitled PieCrust website",
+'root': '/',
+'default_format': DEFAULT_FORMAT,
+'default_template_engine': DEFAULT_TEMPLATE_ENGINE,
+'enable_gzip': True,
+'pretty_urls': False,
+'slugify': 'transliterate|lowercase',
+'timezone': False,
+'locale': False,
+'date_format': DEFAULT_DATE_FORMAT,
+'auto_formats': {
+'html': '',
+'md': 'markdown',
+'textile': 'textile'},
+'default_auto_format': 'md',
+'pagination_suffix': '/%num%',
+'plugins_sources': [DEFAULT_PLUGIN_SOURCE],
+'themes_sources': [DEFAULT_THEME_SOURCE],
+'cache_time': 28800,
+'display_errors': True,
+'enable_debug_info': True
+}
+sitec = values.get('site')
+if sitec is None:
+sitec = {}
+for key, val in default_sitec.iteritems():
+sitec.setdefault(key, val)
+values['site'] = sitec
+# Add a section for our cached information.
+cachec = {}
+values['__cache'] = cachec
+# Cache auto-format regexes.
+if not isinstance(sitec['auto_formats'], dict):
+raise ConfigurationError("The 'site/auto_formats' setting must be a dictionary.")
+cachec['auto_formats_re'] = r"\.(%s)$" % (
+'|'.join(
+map(lambda i: re.escape(i), sitec['auto_formats'].keys())))
+if sitec['default_auto_format'] not in sitec['auto_formats']:
+raise ConfigurationError("Default auto-format '%s' is not declared." % sitec['default_auto_format'])
+# Cache pagination suffix regex.
+pgn_suffix = re.escape(sitec['pagination_suffix'])
+pgn_suffix = pgn_suffix.replace("\\%num\\%", "(?P<num>\\d+)") + '$'
+cachec['pagination_suffix_re'] = pgn_suffix
+# Make sure plugins and theme sources are lists.
+if not isinstance(sitec['plugins_sources'], list):
+sitec['plugins_sources'] = [sitec['plugins_sources']]
+if not isinstance(sitec['themes_sources'], list):
+sitec['themes_sources'] = [sitec['themes_sources']]
+# Setup values for posts/items.
+ipp = sitec.get('posts_per_page')
+if ipp is not None:
+sitec.setdefault('items_per_page', ipp)
+pf = sitec.get('posts_filters')
+if pf is not None:
+sitec.setdefault('items_filters', pf)
+# Figure out if we need to validate sources/routes, or auto-generate
+# them from simple blog settings.
+if 'sources' not in sitec:
+posts_fs = sitec.setdefault('posts_fs', DEFAULT_POSTS_FS)
+blogsc = sitec.setdefault('blogs', ['posts'])
+g_post_url = sitec.get('post_url', '%year%/%month%/%slug%')
+g_tag_url = sitec.get('tag_url', 'tag/%tag%')
+g_category_url = sitec.get('category_url', '%category%')
+g_posts_per_page = sitec.get('items_per_page', 5)
+g_posts_filters = sitec.get('items_filters')
+g_date_format = sitec.get('date_format', DEFAULT_DATE_FORMAT)
+sourcesc = {}
+sourcesc['pages'] = {
+'type': 'default',
+'data_endpoint': 'site/pages',
+'item_name': 'page'}
+sitec['sources'] = sourcesc
+routesc = []
+sitec['routes'] = routesc
+taxonomiesc = {}
+taxonomiesc['tags'] = {
+'multiple': True,
+'term': 'tag'}
+taxonomiesc['categories'] = {
+'term': 'category'}
+sitec['taxonomies'] = taxonomiesc
+for blog_name in blogsc:
+blogc = values.get(blog_name, {})
+url_prefix = blog_name + '/'
+endpoint = 'posts/%s' % blog_name
+item_name = '%s-post' % blog_name
+items_per_page = blogc.get('posts_per_page', g_posts_per_page)
+items_filters = blogc.get('posts_filters', g_posts_filters)
+date_format = blogc.get('date_format', g_date_format)
+if len(blogsc) == 1:
+url_prefix = ''
+endpoint = 'posts'
+item_name = 'post'
+sourcesc[blog_name] = {
+'type': 'posts/%s' % posts_fs,
+'fs_endpoint': endpoint,
+'data_type': 'blog',
+'item_name': item_name,
+'items_per_page': items_per_page,
+'items_filters': items_filters,
+'date_format': date_format,
+'default_layout': 'post'}
+tax_page_prefix = ''
+if len(blogsc) > 1:
+tax_page_prefix = blog_name + '/'
+sourcesc[blog_name]['taxonomy_pages'] = {
+'tags': ('pages:%s_tag.%%ext%%;'
+'theme_pages:_tag.%%ext%%' %
+tax_page_prefix),
+'categories': ('pages:%s_category.%%ext%%;'
+'theme_pages:_category.%%ext%%' %
+tax_page_prefix)}
+post_url = blogc.get('post_url', url_prefix + g_post_url)
+post_url = '/' + post_url.lstrip('/')
+tag_url = blogc.get('tag_url', url_prefix + g_tag_url)
+tag_url = '/' + tag_url.lstrip('/')
+category_url = blogc.get('category_url', url_prefix + g_category_url)
+category_url = '/' + category_url.lstrip('/')
+routesc.append({'url': post_url, 'source': blog_name,
+'func': 'pcposturl(year,month,day,slug)'})
+routesc.append({'url': tag_url, 'source': blog_name,
+'taxonomy': 'tags',
+'func': 'pctagurl(tag)'})
+routesc.append({'url': category_url, 'source': blog_name,
+'taxonomy': 'categories',
+'func': 'pccaturl(category)'})
+routesc.append({'url': '/%path:path%', 'source': 'pages',
+'func': 'pcurl(path)'})
+# Validate sources/routes.
+sourcesc = sitec.get('sources')
+routesc = sitec.get('routes')
+if not sourcesc:
+raise ConfigurationError("There are no sources defined.")
+if not routesc:
+raise ConfigurationError("There are no routes defined.")
+if not isinstance(sourcesc, dict):
+raise ConfigurationError("The 'site/sources' setting must be a dictionary.")
+if not isinstance(routesc, list):
+raise ConfigurationError("The 'site/routes' setting must be a list.")
+# Add the theme page source if no sources were defined in the theme
+# configuration itself.
+has_any_theme_source = False
+for sn, sc in sourcesc.iteritems():
+if sc.get('realm') == REALM_THEME:
+has_any_theme_source = True
+break
+if not has_any_theme_source:
+sitec['sources']['theme_pages'] = {
+'theme_source': True,
+'fs_endpoint': 'pages',
+'data_endpoint': 'site/pages',
+'item_name': 'page',
+'realm': REALM_THEME}
+sitec['routes'].append({
+'url': '/%path:path%',
+'source': 'theme_pages',
+'func': 'pcurl(path)'})
+# Sources have the `default` scanner by default, duh. Also, a bunch
+# of other default values for other configuration stuff.
+for sn, sc in sourcesc.iteritems():
+if not isinstance(sc, dict):
+raise ConfigurationError("All sources in 'site/sources' must be dictionaries.")
+sc.setdefault('type', 'default')
+sc.setdefault('fs_endpoint', sn)
+sc.setdefault('data_endpoint', sn)
+sc.setdefault('data_type', 'iterator')
+sc.setdefault('item_name', sn)
+sc.setdefault('items_per_page', 5)
+sc.setdefault('date_format', DEFAULT_DATE_FORMAT)
+sc.setdefault('realm', REALM_USER)
+# Check routes are referencing correct routes, have default
+# values, etc.
+for rc in routesc:
+if not isinstance(rc, dict):
+raise ConfigurationError("All routes in 'site/routes' must be dictionaries.")
+rc_url = rc.get('url')
+if not rc_url:
+raise ConfigurationError("All routes in 'site/routes' must have an 'url'.")
+if rc_url[0] != '/':
+raise ConfigurationError("Route URLs must start with '/'.")
+if rc.get('source') is None:
+raise ConfigurationError("Routes must specify a source.")
+if rc['source'] not in sourcesc.keys():
+raise ConfigurationError("Route is referencing unknown source: %s" %
+rc['source'])
+rc.setdefault('taxonomy', None)
+rc.setdefault('page_suffix', '/%num%')
+# Validate taxonomies.
+sitec.setdefault('taxonomies', {})
+taxonomiesc = sitec.get('taxonomies')
+for tn, tc in taxonomiesc.iteritems():
+tc.setdefault('multiple', False)
+tc.setdefault('term', tn)
+tc.setdefault('page', '_%s.%%ext%%' % tc['term'])
+# Validate endpoints, and make sure the theme has a default source.
+reserved_endpoints = set(['piecrust', 'site', 'page', 'route',
+'assets', 'pagination', 'siblings',
+'family'])
+for name, src in sitec['sources'].iteritems():
+endpoint = src['data_endpoint']
+if endpoint in reserved_endpoints:
+raise ConfigurationError(
+"Source '%s' is using a reserved endpoint name: %s" %
+(name, endpoint))
+# Done validating!
+return values
 class PieCrust(object):
-def __init__(self, root, cache=True, debug=False, env=None):
+def __init__(self, root_dir, cache=True, debug=False, theme_site=False,
-self.root = root
+env=None):
+self.root_dir = root_dir
 self.debug = debug
-self.cache = cache
+self.theme_site = theme_site
 self.plugin_loader = PluginLoader(self)
+if cache:
+self.cache = ExtensibleCache(self.cache_dir)
+else:
+self.cache = NullExtensibleCache()
 self.env = env
 if self.env is None:
 self.env = StandardEnvironment()
 self.env.initialize(self)
-@lazy_property
+@cached_property
 def config(self):
-logger.debug("Loading site configuration...")
+logger.debug("Creating site configuration...")
 paths = []
 if self.theme_dir:
 paths.append(os.path.join(self.theme_dir, THEME_CONFIG_PATH))
-paths.append(os.path.join(self.root, CONFIG_PATH))
+paths.append(os.path.join(self.root_dir, CONFIG_PATH))
-config = PieCrustConfiguration(paths, self.cache_dir)
+config_cache = self.cache.getCache('app')
+config = PieCrustConfiguration(paths, config_cache)
 if self.theme_dir:
 # We'll need to patch the templates directories to be relative
 # to the site's root, and not the theme root.
 def _fixupThemeTemplatesDir(index, config):
-if index == 0:
+if index != 0:
-sitec = config.get('site')
+return
-if sitec:
+sitec = config.get('site')
-tplc = sitec.get('templates_dirs')
+if sitec is None:
-if tplc:
+return
-if isinstance(tplc, types.StringTypes):
+tplc = sitec.get('templates_dirs')
-tplc = [tplc]
+if tplc is None:
-sitec['templates_dirs'] = filter(tplc,
+return
-lambda p: os.path.join(self.theme_dir, p))
+if isinstance(tplc, types.StringTypes):
+tplc = [tplc]
+sitec['templates_dirs'] = filter(tplc,
+lambda p: os.path.join(self.theme_dir, p))
 config.fixups.append(_fixupThemeTemplatesDir)
+# We'll also need to flag all page sources as coming from
+# the theme.
+def _fixupThemeSources(index, config):
+if index != 0:
+return
+sitec = config.get('site')
+if sitec is None:
+sitec = {}
+config['site'] = sitec
+srcc = sitec.get('sources')
+if srcc is not None:
+for sn, sc in srcc.iteritems():
+sc['realm'] = REALM_THEME
+config.fixups.append(_fixupThemeSources)
 return config
-@lazy_property
+@cached_property
 def templates_dirs(self):
 templates_dirs = self._get_configurable_dirs(TEMPLATES_DIR,
 'site/templates_dirs')
 # Also, add the theme directory, if nay.
 if os.path.isdir(default_theme_dir):
 templates_dirs.append(default_theme_dir)
 return templates_dirs
-@lazy_property
+@cached_property
-def pages_dir(self):
-return self._get_dir(PAGES_DIR)
-@lazy_property
-def posts_dir(self):
-return self._get_dir(POSTS_DIR)
-@lazy_property
 def plugins_dirs(self):
 return self._get_configurable_dirs(PLUGINS_DIR,
 'site/plugins_dirs')
-@lazy_property
+@cached_property
 def theme_dir(self):
-return self._get_dir(THEME_DIR)
+td = self._get_dir(THEME_DIR)
+if td is not None:
-@lazy_property
+return td
+return os.path.join(os.path.dirname(__file__), 'resources', 'theme')
+@cached_property
 def cache_dir(self):
-if self.cache:
+return os.path.join(self.root_dir, CACHE_DIR)
-return os.path.join(self.root, CACHE_DIR)
-return False
+@cached_property
+def sources(self):
+defs = {}
+for cls in self.plugin_loader.getSources():
+defs[cls.SOURCE_NAME] = cls
+sources = []
+for n, s in self.config.get('site/sources').iteritems():
+cls = defs.get(s['type'])
+if cls is None:
+raise ConfigurationError("No such page source type: %s" % s['type'])
+src = cls(self, n, s)
+sources.append(src)
+return sources
+@cached_property
+def routes(self):
+routes = []
+for r in self.config.get('site/routes'):
+rte = Route(self, r)
+routes.append(rte)
+return routes
+@cached_property
+def taxonomies(self):
+taxonomies = []
+for tn, tc in self.config.get('site/taxonomies').iteritems():
+tax = Taxonomy(self, tn, tc)
+taxonomies.append(tax)
+return taxonomies
+def getSource(self, source_name):
+for source in self.sources:
+if source.name == source_name:
+return source
+return None
+def getRoutes(self, source_name, skip_taxonomies=False):
+for route in self.routes:
+if route.source_name == source_name:
+if not skip_taxonomies or route.taxonomy is None:
+yield route
+def getRoute(self, source_name, source_metadata):
+for route in self.getRoutes(source_name, True):
+if route.isMatch(source_metadata):
+return route
+return None
+def getTaxonomyRoute(self, tax_name, source_name):
+for route in self.routes:
+if route.taxonomy == tax_name and route.source_name == source_name:
+return route
+return None
+def getTaxonomy(self, tax_name):
+for tax in self.taxonomies:
+if tax.name == tax_name:
+return tax
+return None
 def _get_dir(self, default_rel_dir):
-abs_dir = os.path.join(self.root, default_rel_dir)
+abs_dir = os.path.join(self.root_dir, default_rel_dir)
 if os.path.isdir(abs_dir):
 return abs_dir
-return False
+return None
 def _get_configurable_dirs(self, default_rel_dir, conf_name):
 dirs = []
 # Add custom directories from the configuration.
 conf_dirs = self.config.get(conf_name)
 if conf_dirs is not None:
-dirs += filter(conf_dirs,
+if isinstance(conf_dirs, types.StringTypes):
-lambda p: os.path.join(self.root, p))
+dirs.append(os.path.join(self.root_dir, conf_dirs))
+else:
+dirs += filter(lambda p: os.path.join(self.root_dir, p),
+conf_dirs)
 # Add the default directory if it exists.
-default_dir = os.path.join(self.root, default_rel_dir)
+default_dir = os.path.join(self.root_dir, default_rel_dir)
 if os.path.isdir(default_dir):
 dirs.append(default_dir)
 return dirs

Mercurial > piecrust2

comparison piecrust/app.py @ 3:f485ba500df3