piecrust2: piecrust/sources/autoconfig.py comparison

comparison piecrust/sources/autoconfig.py @ 242:f130365568ff

internal: Code reorganization to put less stuff in `sources.base`. Interfaces that sources can implement are in `sources.interfaces`. The default page source is in `sources.default`. The `SimplePageSource` is gone since most subclasses only wanted to do *some* stuff the same, but *lots* of stuff slightly different. I may have to revisit the code to extract exactly the code that's in common.

author	Ludovic Chabant <ludovic@chabant.com>
date	Wed, 18 Feb 2015 18:35:03 -0800
parents	f43f19975671
children	3f740928043a

comparison

equal deleted inserted replaced

-:85a6c7ba5e3b
+:f130365568ff
 import re
 import os
 import os.path
-import glob
 import logging
 from piecrust.configuration import ConfigurationError
-from piecrust.data.iterators import SettingSortIterator
 from piecrust.sources.base import (
-SimplePageSource, IPreparingSource, SimplePaginationSourceMixin,
+PageSource, PageFactory, InvalidFileSystemEndpointError)
-PageNotFoundError, InvalidFileSystemEndpointError,
+from piecrust.sources.default import (
-PageFactory, MODE_CREATING, MODE_PARSING)
+filter_page_dirname, filter_page_filename)
+from piecrust.sources.interfaces import IListableSource
+from piecrust.sources.mixins import SimplePaginationSourceMixin
 logger = logging.getLogger(__name__)
-class AutoConfigSourceBase(SimplePageSource,
+class AutoConfigSourceBase(PageSource, SimplePaginationSourceMixin,
-SimplePaginationSourceMixin):
+IListableSource):
 """ Base class for page sources that automatically apply configuration
 settings to their generated pages based on those pages' paths.
 """
 def __init__(self, app, name, config):
 super(AutoConfigSourceBase, self).__init__(app, name, config)
+self.fs_endpoint = config.get('fs_endpoint', name)
+self.fs_endpoint_path = os.path.join(self.root_dir, self.fs_endpoint)
+self.supported_extensions = list(
+app.config.get('site/auto_formats').keys())
+self.default_auto_format = app.config.get('site/default_auto_format')
 self.capture_mode = config.get('capture_mode', 'path')
 if self.capture_mode not in ['path', 'dirname', 'filename']:
 raise ConfigurationError("Capture mode in source '%s' must be "
 "one of: path, dirname, filename" %
 name)
 def buildPageFactories(self):
+logger.debug("Scanning for pages in: %s" % self.fs_endpoint_path)
 if not os.path.isdir(self.fs_endpoint_path):
 raise InvalidFileSystemEndpointError(self.name,
 self.fs_endpoint_path)
 for dirpath, dirnames, filenames in os.walk(self.fs_endpoint_path):
-if not filenames:
-continue
 rel_dirpath = os.path.relpath(dirpath, self.fs_endpoint_path)
+dirnames[:] = list(filter(filter_page_dirname, dirnames))
 # If `capture_mode` is `dirname`, we don't need to recompute it
 # for each filename, so we do it here.
 if self.capture_mode == 'dirname':
-config = self.extractConfigFragment(rel_dirpath)
+config = self._extractConfigFragment(rel_dirpath)
-for f in filenames:
+for f in filter(filter_page_filename, filenames):
 if self.capture_mode == 'path':
 path = os.path.join(rel_dirpath, f)
-config = self.extractConfigFragment(path)
+config = self._extractConfigFragment(path)
 elif self.capture_mode == 'filename':
-config = self.extractConfigFragment(f)
+config = self._extractConfigFragment(f)
 fac_path = f
 if rel_dirpath != '.':
 fac_path = os.path.join(rel_dirpath, f)
-slug = self.makeSlug(rel_dirpath, f)
+slug = self._makeSlug(fac_path)
 metadata = {
 'slug': slug,
 'config': config}
 yield PageFactory(self, fac_path, metadata)
-def makeSlug(self, rel_dirpath, filename):
+def resolveRef(self, ref_path):
+return os.path.normpath(
+os.path.join(self.fs_endpoint_path, ref_path.lstrip("\\/")))
+def listPath(self, rel_path):
 raise NotImplementedError()
-def extractConfigFragment(self, rel_path):
+def getDirpath(self, rel_path):
+return os.path.dirname(rel_path)
+def getBasename(self, rel_path):
+filename = os.path.basename(rel_path)
+name, _ = os.path.splitext(filename)
+return name
+def _makeSlug(self, rel_path):
 raise NotImplementedError()
-def findPagePath(self, metadata, mode):
+def _extractConfigFragment(self, rel_path):
 raise NotImplementedError()
 class AutoConfigSource(AutoConfigSourceBase):
 """ Page source that extracts configuration settings from the sub-folders
 self.collapse_single_values = config.get('collapse_single_values',
 False)
 self.supported_extensions = list(
 app.config.get('site/auto_formats').keys())
-def makeSlug(self, rel_dirpath, filename):
+def _makeSlug(self, rel_path):
-slug, ext = os.path.splitext(filename)
+slug, ext = os.path.splitext(os.path.basename(rel_path))
 if ext.lstrip('.') not in self.supported_extensions:
 slug += ext
 return slug
-def extractConfigFragment(self, rel_path):
+def _extractConfigFragment(self, rel_path):
 if rel_path == '.':
 values = []
 else:
 values = rel_path.split(os.sep)
 for f in filenames:
 slug, _ = os.path.splitext(f)
 if slug == metadata['slug']:
 path = os.path.join(dirpath, f)
 rel_path = os.path.relpath(path, self.fs_endpoint_path)
-config = self.extractConfigFragment(dirpath)
+config = self._extractConfigFragment(rel_path)
 metadata = {'slug': slug, 'config': config}
 return rel_path, metadata
+def listPath(self, rel_path):
+rel_path = rel_path.lstrip('\\/')
+path = os.path.join(self.fs_endpoint_path, rel_path)
+names = sorted(os.listdir(path))
+items = []
+for name in names:
+if os.path.isdir(os.path.join(path, name)):
+if filter_page_dirname(name):
+rel_subdir = os.path.join(rel_path, name)
+items.append((True, name, rel_subdir))
+else:
+if filter_page_filename(name):
+cur_rel_path = os.path.join(rel_path, name)
+slug = self._makeSlug(cur_rel_path)
+config = self._extractConfigFragment(cur_rel_path)
+metadata = {'slug': slug, 'config': config}
+fac = PageFactory(self, cur_rel_path, metadata)
+name, _ = os.path.splitext(name)
+items.append((False, name, fac))
+return items
 class OrderedPageSource(AutoConfigSourceBase):
 """ A page source that assigns an "order" to its pages based on a
 numerical prefix in their filename. Page iterators will automatically
 SOURCE_NAME = 'ordered'
 re_pattern = re.compile(r'(^|/)(?P<num>\d+)_')
 def __init__(self, app, name, config):
-config['capture_mode'] = 'filename'
+config['capture_mode'] = 'path'
 super(OrderedPageSource, self).__init__(app, name, config)
 self.setting_name = config.get('setting_name', 'order')
 self.default_value = config.get('default_value', 0)
 self.supported_extensions = list(
 app.config.get('site/auto_formats').keys())
-def makeSlug(self, rel_dirpath, filename):
+def findPagePath(self, metadata, mode):
-slug, ext = os.path.splitext(filename)
+uri_path = metadata.get('slug', '')
+if uri_path == '':
+uri_path = '_index'
+path = self.fs_endpoint_path
+uri_parts = uri_path.split('/')
+for i, p in enumerate(uri_parts):
+if i == len(uri_parts) - 1:
+# Last part, this is the filename. We need to check for either
+# the name, or the name with the prefix, but also handle a
+# possible extension.
+p_pat = r'(\d+_)?' + re.escape(p)
+_, ext = os.path.splitext(uri_path)
+if ext == '':
+p_pat += r'\.[\w\d]+'
+found = False
+for name in os.listdir(path):
+if re.match(p_pat, name):
+path = os.path.join(path, name)
+found = True
+break
+if not found:
+return None, None
+else:
+# Find each sub-directory. It can either be a directory with
+# the name itself, or the name with a number prefix.
+p_pat = r'(\d+_)?' + re.escape(p)
+found = False
+for name in os.listdir(path):
+if re.match(p_pat, name):
+path = os.path.join(path, name)
+found = True
+break
+if not found:
+return None, None
+fac_path = os.path.relpath(path, self.fs_endpoint_path)
+config = self._extractConfigFragment(fac_path)
+metadata = {'slug': uri_path, 'config': config}
+return fac_path, metadata
+def getSorterIterator(self, it):
+accessor = self.getSettingAccessor()
+return OrderTrailSortIterator(it, self.setting_name + '_trail',
+value_accessor=accessor)
+def listPath(self, rel_path):
+rel_path = rel_path.lstrip('/')
+path = self.fs_endpoint_path
+if rel_path != '':
+parts = rel_path.split('/')
+for p in parts:
+p_pat = r'(\d+_)?' + re.escape(p)
+for name in os.listdir(path):
+if re.match(p_pat, name):
+path = os.path.join(path, name)
+break
+else:
+raise Exception("No such path: %s" % rel_path)
+items = []
+names = sorted(os.listdir(path))
+for name in names:
+if os.path.isdir(os.path.join(path, name)):
+if filter_page_dirname(name):
+rel_subdir = os.path.join(rel_path, name)
+items.append((True, name, rel_subdir))
+else:
+if filter_page_filename(name):
+slug = self._makeSlug(os.path.join(rel_path, name))
+fac_path = name
+if rel_path != '.':
+fac_path = os.path.join(rel_path, name)
+fac_path = fac_path.replace('\\', '/')
+config = self._extractConfigFragment(fac_path)
+metadata = {'slug': slug, 'config': config}
+fac = PageFactory(self, fac_path, metadata)
+name, _ = os.path.splitext(name)
+items.append((False, name, fac))
+return items
+def _makeSlug(self, rel_path):
+slug, ext = os.path.splitext(rel_path)
 if ext.lstrip('.') not in self.supported_extensions:
 slug += ext
 slug = self.re_pattern.sub(r'\1', slug)
-slug = os.path.join(rel_dirpath, slug).replace('\\', '/')
-if slug.startswith('./'):
-slug = slug[2:]
 return slug
-def extractConfigFragment(self, rel_path):
+def _extractConfigFragment(self, rel_path):
-m = self.re_pattern.match(rel_path)
+values = []
-if m is not None:
+for m in self.re_pattern.finditer(rel_path):
 val = int(m.group('num'))
-else:
+values.append(val)
-val = self.default_value
-return {self.setting_name: val}
+if len(values) == 0:
+values.append(self.default_value)
-def findPagePath(self, metadata, mode):
-uri_path = metadata.get('slug', '')
+return {
-if uri_path != '':
+self.setting_name: values[-1],
-uri_parts = ['*_%s' % p for p in uri_path.split('/')]
+self.setting_name + '_trail': values}
-else:
-uri_parts = ['*__index']
-uri_parts.insert(0, self.fs_endpoint_path)
-path = os.path.join(*uri_parts)
-_, ext = os.path.splitext(uri_path)
-if ext == '':
-path += '.*'
-possibles = glob.glob(path)
-if len(possibles) == 0:
-return None, None
-if len(possibles) > 1:
-raise Exception("More than one path matching: %s" % uri_path)
-path = possibles[0]
-fac_path = os.path.relpath(path, self.fs_endpoint_path)
-_, filename = os.path.split(path)
-config = self.extractConfigFragment(filename)
-metadata = {'slug': uri_path, 'config': config}
-return fac_path, metadata
-def getSorterIterator(self, it):
-accessor = self.getSettingAccessor()
-return SettingSortIterator(it, self.setting_name,
-value_accessor=accessor)
 def _populateMetadata(self, rel_path, metadata, mode=None):
 _, filename = os.path.split(rel_path)
-config = self.extractConfigFragment(filename)
+config = self._extractConfigFragment(filename)
 metadata['config'] = config
 slug = metadata['slug']
 metadata['slug'] = self.re_pattern.sub(r'\1', slug)
+class OrderTrailSortIterator(object):
+def __init__(self, it, trail_name, value_accessor):
+self.it = it
+self.trail_name = trail_name
+self.value_accessor = value_accessor
+def __iter__(self):
+return iter(sorted(self.it, key=self._key_getter))
+def _key_getter(self, item):
+values = self.value_accessor(item, self.trail_name)
+key = ''.join(values)
+return key

Mercurial > piecrust2

comparison piecrust/sources/autoconfig.py @ 242:f130365568ff