Mercurial > piecrust2
changeset 242:f130365568ff
internal: Code reorganization to put less stuff in `sources.base`.
Interfaces that sources can implement are in `sources.interfaces`. The default
page source is in `sources.default`. The `SimplePageSource` is gone since most
subclasses only wanted to do *some* stuff the same, but *lots* of stuff
slightly different. I may have to revisit the code to extract exactly the code
that's in common.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Wed, 18 Feb 2015 18:35:03 -0800 |
| parents | 85a6c7ba5e3b |
| children | 26e59f837558 d9d5c5de02a8 |
| files | piecrust/commands/builtin/scaffolding.py piecrust/data/base.py piecrust/data/iterators.py piecrust/data/linker.py piecrust/data/paginator.py piecrust/data/provider.py piecrust/plugins/builtin.py piecrust/sources/array.py piecrust/sources/autoconfig.py piecrust/sources/base.py piecrust/sources/default.py piecrust/sources/interfaces.py piecrust/sources/mixins.py piecrust/sources/pageref.py piecrust/sources/posts.py piecrust/sources/prose.py piecrust/taxonomies.py tests/test_data_paginator.py tests/test_sources_autoconfig.py tests/test_sources_base.py |
| diffstat | 20 files changed, 693 insertions(+), 471 deletions(-) [+] |
line wrap: on
line diff
--- a/piecrust/commands/builtin/scaffolding.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/commands/builtin/scaffolding.py Wed Feb 18 18:35:03 2015 -0800 @@ -9,7 +9,8 @@ from piecrust import RESOURCES_DIR from piecrust.chefutil import print_help_item from piecrust.commands.base import ExtendableChefCommand, ChefCommandExtension -from piecrust.sources.base import IPreparingSource, MODE_CREATING +from piecrust.sources.base import MODE_CREATING +from piecrust.sources.interfaces import IPreparingSource from piecrust.uriutil import multi_replace
--- a/piecrust/data/base.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/data/base.py Wed Feb 18 18:35:03 2015 -0800 @@ -7,29 +7,6 @@ logger = logging.getLogger(__name__) -class IPaginationSource(object): - """ Defines the interface for a source that can be used as the data - for an iterator or a pagination. - """ - def getItemsPerPage(self): - raise NotImplementedError() - - def getSourceIterator(self): - raise NotImplementedError() - - def getSorterIterator(self, it): - raise NotImplementedError() - - def getTailIterator(self, it): - raise NotImplementedError() - - def getPaginationFilter(self, page): - raise NotImplementedError() - - def getSettingAccessor(self): - raise NotImplementedError() - - class LazyPageConfigData(object): """ An object that represents the configuration header of a page, but also allows for additional data. It's meant to be exposed
--- a/piecrust/data/iterators.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/data/iterators.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,7 +1,7 @@ import logging -from piecrust.data.base import IPaginationSource from piecrust.data.filters import PaginationFilter from piecrust.events import Event +from piecrust.sources.interfaces import IPaginationSource logger = logging.getLogger(__name__)
--- a/piecrust/data/linker.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/data/linker.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,8 +1,9 @@ import logging import collections -from piecrust.data.base import PaginationData, IPaginationSource +from piecrust.data.base import PaginationData from piecrust.data.iterators import PageIterator -from piecrust.sources.base import IListableSource, build_pages +from piecrust.sources.base import build_pages +from piecrust.sources.interfaces import IPaginationSource, IListableSource logger = logging.getLogger(__name__)
--- a/piecrust/data/paginator.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/data/paginator.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,9 +1,9 @@ import math import logging from werkzeug.utils import cached_property -from piecrust.data.base import IPaginationSource from piecrust.data.filters import PaginationFilter from piecrust.data.iterators import PageIterator +from piecrust.sources.interfaces import IPaginationSource logger = logging.getLogger(__name__)
--- a/piecrust/data/provider.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/data/provider.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,6 +1,6 @@ import time from piecrust.data.iterators import PageIterator -from piecrust.sources.base import ArraySource +from piecrust.sources.array import ArraySource class DataProvider(object):
--- a/piecrust/plugins/builtin.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/plugins/builtin.py Wed Feb 18 18:35:03 2015 -0800 @@ -28,7 +28,7 @@ from piecrust.processing.sass import SassProcessor from piecrust.processing.sitemap import SitemapProcessor from piecrust.processing.util import ConcatProcessor -from piecrust.sources.base import DefaultPageSource +from piecrust.sources.default import DefaultPageSource from piecrust.sources.posts import ( FlatPostsSource, ShallowPostsSource, HierarchyPostsSource) from piecrust.sources.autoconfig import (
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/sources/array.py Wed Feb 18 18:35:03 2015 -0800 @@ -0,0 +1,43 @@ +from piecrust.sources.base import PageSource +from piecrust.sources.mixins import SimplePaginationSourceMixin + + +class CachedPageFactory(object): + """ A `PageFactory` (in appearance) that already has a page built. + """ + def __init__(self, page): + self._page = page + + @property + def rel_path(self): + return self._page.rel_path + + @property + def metadata(self): + return self._page.source_metadata + + @property + def ref_spec(self): + return self._page.ref_spec + + @property + def path(self): + return self._page.path + + def buildPage(self): + return self._page + + +class ArraySource(PageSource, SimplePaginationSourceMixin): + def __init__(self, app, inner_source, name='array', config=None): + super(ArraySource, self).__init__(app, name, config or {}) + self.inner_source = inner_source + + @property + def page_count(self): + return len(self.inner_source) + + def getPageFactories(self): + for p in self.inner_source: + yield CachedPageFactory(p) +
--- a/piecrust/sources/autoconfig.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/sources/autoconfig.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,26 +1,32 @@ import re import os import os.path -import glob import logging from piecrust.configuration import ConfigurationError -from piecrust.data.iterators import SettingSortIterator from piecrust.sources.base import ( - SimplePageSource, IPreparingSource, SimplePaginationSourceMixin, - PageNotFoundError, InvalidFileSystemEndpointError, - PageFactory, MODE_CREATING, MODE_PARSING) + PageSource, PageFactory, InvalidFileSystemEndpointError) +from piecrust.sources.default import ( + filter_page_dirname, filter_page_filename) +from piecrust.sources.interfaces import IListableSource +from piecrust.sources.mixins import SimplePaginationSourceMixin logger = logging.getLogger(__name__) -class AutoConfigSourceBase(SimplePageSource, - SimplePaginationSourceMixin): +class AutoConfigSourceBase(PageSource, SimplePaginationSourceMixin, + IListableSource): """ Base class for page sources that automatically apply configuration settings to their generated pages based on those pages' paths. """ def __init__(self, app, name, config): super(AutoConfigSourceBase, self).__init__(app, name, config) + self.fs_endpoint = config.get('fs_endpoint', name) + self.fs_endpoint_path = os.path.join(self.root_dir, self.fs_endpoint) + self.supported_extensions = list( + app.config.get('site/auto_formats').keys()) + self.default_auto_format = app.config.get('site/default_auto_format') + self.capture_mode = config.get('capture_mode', 'path') if self.capture_mode not in ['path', 'dirname', 'filename']: raise ConfigurationError("Capture mode in source '%s' must be " @@ -28,46 +34,57 @@ name) def buildPageFactories(self): + logger.debug("Scanning for pages in: %s" % self.fs_endpoint_path) if not os.path.isdir(self.fs_endpoint_path): raise InvalidFileSystemEndpointError(self.name, self.fs_endpoint_path) for dirpath, dirnames, filenames in os.walk(self.fs_endpoint_path): - if not filenames: - continue - rel_dirpath = os.path.relpath(dirpath, self.fs_endpoint_path) + dirnames[:] = list(filter(filter_page_dirname, dirnames)) # If `capture_mode` is `dirname`, we don't need to recompute it # for each filename, so we do it here. if self.capture_mode == 'dirname': - config = self.extractConfigFragment(rel_dirpath) + config = self._extractConfigFragment(rel_dirpath) - for f in filenames: + for f in filter(filter_page_filename, filenames): if self.capture_mode == 'path': path = os.path.join(rel_dirpath, f) - config = self.extractConfigFragment(path) + config = self._extractConfigFragment(path) elif self.capture_mode == 'filename': - config = self.extractConfigFragment(f) + config = self._extractConfigFragment(f) fac_path = f if rel_dirpath != '.': fac_path = os.path.join(rel_dirpath, f) - slug = self.makeSlug(rel_dirpath, f) + slug = self._makeSlug(fac_path) metadata = { 'slug': slug, 'config': config} yield PageFactory(self, fac_path, metadata) - def makeSlug(self, rel_dirpath, filename): + def resolveRef(self, ref_path): + return os.path.normpath( + os.path.join(self.fs_endpoint_path, ref_path.lstrip("\\/"))) + + def listPath(self, rel_path): raise NotImplementedError() - def extractConfigFragment(self, rel_path): + def getDirpath(self, rel_path): + return os.path.dirname(rel_path) + + def getBasename(self, rel_path): + filename = os.path.basename(rel_path) + name, _ = os.path.splitext(filename) + return name + + def _makeSlug(self, rel_path): raise NotImplementedError() - def findPagePath(self, metadata, mode): + def _extractConfigFragment(self, rel_path): raise NotImplementedError() @@ -88,13 +105,13 @@ self.supported_extensions = list( app.config.get('site/auto_formats').keys()) - def makeSlug(self, rel_dirpath, filename): - slug, ext = os.path.splitext(filename) + def _makeSlug(self, rel_path): + slug, ext = os.path.splitext(os.path.basename(rel_path)) if ext.lstrip('.') not in self.supported_extensions: slug += ext return slug - def extractConfigFragment(self, rel_path): + def _extractConfigFragment(self, rel_path): if rel_path == '.': values = [] else: @@ -126,10 +143,32 @@ if slug == metadata['slug']: path = os.path.join(dirpath, f) rel_path = os.path.relpath(path, self.fs_endpoint_path) - config = self.extractConfigFragment(dirpath) + config = self._extractConfigFragment(rel_path) metadata = {'slug': slug, 'config': config} return rel_path, metadata + def listPath(self, rel_path): + rel_path = rel_path.lstrip('\\/') + path = os.path.join(self.fs_endpoint_path, rel_path) + names = sorted(os.listdir(path)) + items = [] + for name in names: + if os.path.isdir(os.path.join(path, name)): + if filter_page_dirname(name): + rel_subdir = os.path.join(rel_path, name) + items.append((True, name, rel_subdir)) + else: + if filter_page_filename(name): + cur_rel_path = os.path.join(rel_path, name) + slug = self._makeSlug(cur_rel_path) + config = self._extractConfigFragment(cur_rel_path) + metadata = {'slug': slug, 'config': config} + fac = PageFactory(self, cur_rel_path, metadata) + + name, _ = os.path.splitext(name) + items.append((False, name, fac)) + return items + class OrderedPageSource(AutoConfigSourceBase): """ A page source that assigns an "order" to its pages based on a @@ -141,70 +180,140 @@ re_pattern = re.compile(r'(^|/)(?P<num>\d+)_') def __init__(self, app, name, config): - config['capture_mode'] = 'filename' + config['capture_mode'] = 'path' super(OrderedPageSource, self).__init__(app, name, config) self.setting_name = config.get('setting_name', 'order') self.default_value = config.get('default_value', 0) self.supported_extensions = list( app.config.get('site/auto_formats').keys()) - def makeSlug(self, rel_dirpath, filename): - slug, ext = os.path.splitext(filename) - if ext.lstrip('.') not in self.supported_extensions: - slug += ext - slug = self.re_pattern.sub(r'\1', slug) - slug = os.path.join(rel_dirpath, slug).replace('\\', '/') - if slug.startswith('./'): - slug = slug[2:] - return slug - - def extractConfigFragment(self, rel_path): - m = self.re_pattern.match(rel_path) - if m is not None: - val = int(m.group('num')) - else: - val = self.default_value - return {self.setting_name: val} - def findPagePath(self, metadata, mode): uri_path = metadata.get('slug', '') - if uri_path != '': - uri_parts = ['*_%s' % p for p in uri_path.split('/')] - else: - uri_parts = ['*__index'] - uri_parts.insert(0, self.fs_endpoint_path) - path = os.path.join(*uri_parts) + if uri_path == '': + uri_path = '_index' - _, ext = os.path.splitext(uri_path) - if ext == '': - path += '.*' + path = self.fs_endpoint_path + uri_parts = uri_path.split('/') + for i, p in enumerate(uri_parts): + if i == len(uri_parts) - 1: + # Last part, this is the filename. We need to check for either + # the name, or the name with the prefix, but also handle a + # possible extension. + p_pat = r'(\d+_)?' + re.escape(p) + + _, ext = os.path.splitext(uri_path) + if ext == '': + p_pat += r'\.[\w\d]+' - possibles = glob.glob(path) - - if len(possibles) == 0: - return None, None + found = False + for name in os.listdir(path): + if re.match(p_pat, name): + path = os.path.join(path, name) + found = True + break + if not found: + return None, None + else: + # Find each sub-directory. It can either be a directory with + # the name itself, or the name with a number prefix. + p_pat = r'(\d+_)?' + re.escape(p) + found = False + for name in os.listdir(path): + if re.match(p_pat, name): + path = os.path.join(path, name) + found = True + break + if not found: + return None, None - if len(possibles) > 1: - raise Exception("More than one path matching: %s" % uri_path) - - path = possibles[0] fac_path = os.path.relpath(path, self.fs_endpoint_path) - - _, filename = os.path.split(path) - config = self.extractConfigFragment(filename) + config = self._extractConfigFragment(fac_path) metadata = {'slug': uri_path, 'config': config} return fac_path, metadata def getSorterIterator(self, it): accessor = self.getSettingAccessor() - return SettingSortIterator(it, self.setting_name, - value_accessor=accessor) + return OrderTrailSortIterator(it, self.setting_name + '_trail', + value_accessor=accessor) + + def listPath(self, rel_path): + rel_path = rel_path.lstrip('/') + path = self.fs_endpoint_path + if rel_path != '': + parts = rel_path.split('/') + for p in parts: + p_pat = r'(\d+_)?' + re.escape(p) + for name in os.listdir(path): + if re.match(p_pat, name): + path = os.path.join(path, name) + break + else: + raise Exception("No such path: %s" % rel_path) + + items = [] + names = sorted(os.listdir(path)) + for name in names: + if os.path.isdir(os.path.join(path, name)): + if filter_page_dirname(name): + rel_subdir = os.path.join(rel_path, name) + items.append((True, name, rel_subdir)) + else: + if filter_page_filename(name): + slug = self._makeSlug(os.path.join(rel_path, name)) + + fac_path = name + if rel_path != '.': + fac_path = os.path.join(rel_path, name) + fac_path = fac_path.replace('\\', '/') + + config = self._extractConfigFragment(fac_path) + metadata = {'slug': slug, 'config': config} + fac = PageFactory(self, fac_path, metadata) + + name, _ = os.path.splitext(name) + items.append((False, name, fac)) + return items + + def _makeSlug(self, rel_path): + slug, ext = os.path.splitext(rel_path) + if ext.lstrip('.') not in self.supported_extensions: + slug += ext + slug = self.re_pattern.sub(r'\1', slug) + return slug + + def _extractConfigFragment(self, rel_path): + values = [] + for m in self.re_pattern.finditer(rel_path): + val = int(m.group('num')) + values.append(val) + + if len(values) == 0: + values.append(self.default_value) + + return { + self.setting_name: values[-1], + self.setting_name + '_trail': values} def _populateMetadata(self, rel_path, metadata, mode=None): _, filename = os.path.split(rel_path) - config = self.extractConfigFragment(filename) + config = self._extractConfigFragment(filename) metadata['config'] = config slug = metadata['slug'] metadata['slug'] = self.re_pattern.sub(r'\1', slug) + +class OrderTrailSortIterator(object): + def __init__(self, it, trail_name, value_accessor): + self.it = it + self.trail_name = trail_name + self.value_accessor = value_accessor + + def __iter__(self): + return iter(sorted(self.it, key=self._key_getter)) + + def _key_getter(self, item): + values = self.value_accessor(item, self.trail_name) + key = ''.join(values) + return key +
--- a/piecrust/sources/base.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/sources/base.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,11 +1,6 @@ -import re -import os -import os.path import logging from werkzeug.utils import cached_property from piecrust.configuration import ConfigurationError -from piecrust.data.base import IPaginationSource, PaginationData -from piecrust.data.filters import PaginationFilter from piecrust.page import Page @@ -23,19 +18,12 @@ logger = logging.getLogger(__name__) -page_ref_pattern = re.compile(r'(?P<src>[\w]+)\:(?P<path>.*?)(;|$)') - - def build_pages(app, factories): with app.env.page_repository.startBatchGet(): for f in factories: yield f.buildPage() -class PageNotFoundError(Exception): - pass - - class InvalidFileSystemEndpointError(Exception): def __init__(self, source_name, fs_endpoint): super(InvalidFileSystemEndpointError, self).__init__( @@ -75,120 +63,6 @@ return page -class CachedPageFactory(object): - """ A `PageFactory` (in appearance) that already has a page built. - """ - def __init__(self, page): - self._page = page - - @property - def rel_path(self): - return self._page.rel_path - - @property - def metadata(self): - return self._page.source_metadata - - @property - def ref_spec(self): - return self._page.ref_spec - - @property - def path(self): - return self._page.path - - def buildPage(self): - return self._page - - -class PageRef(object): - """ A reference to a page, with support for looking a page in different - realms. - """ - def __init__(self, app, page_ref): - self.app = app - self._page_ref = page_ref - self._paths = None - self._first_valid_path_index = -2 - self._exts = list(app.config.get('site/auto_formats').keys()) - - @property - def exists(self): - try: - self._checkPaths() - return True - except PageNotFoundError: - return False - - @property - def source_name(self): - self._checkPaths() - return self._paths[self._first_valid_path_index][0] - - @property - def source(self): - return self.app.getSource(self.source_name) - - @property - def rel_path(self): - self._checkPaths() - return self._paths[self._first_valid_path_index][1] - - @property - def path(self): - self._checkPaths() - return self._paths[self._first_valid_path_index][2] - - @property - def possible_rel_paths(self): - self._load() - return [p[1] for p in self._paths] - - @property - def possible_paths(self): - self._load() - return [p[2] for p in self._paths] - - def _load(self): - if self._paths is not None: - return - - it = list(page_ref_pattern.finditer(self._page_ref)) - if len(it) == 0: - raise Exception("Invalid page ref: %s" % self._page_ref) - - self._paths = [] - for m in it: - source_name = m.group('src') - source = self.app.getSource(source_name) - if source is None: - raise Exception("No such source: %s" % source_name) - rel_path = m.group('path') - path = source.resolveRef(rel_path) - if '%ext%' in rel_path: - for e in self._exts: - self._paths.append((source_name, - rel_path.replace('%ext%', e), - path.replace('%ext%', e))) - else: - self._paths.append((source_name, rel_path, path)) - - def _checkPaths(self): - if self._first_valid_path_index >= 0: - return - if self._first_valid_path_index == -1: - raise PageNotFoundError( - "No valid paths were found for page reference: %s" % - self._page_ref) - - self._load() - self._first_valid_path_index = -1 - for i, path_info in enumerate(self._paths): - if os.path.isfile(path_info[2]): - self._first_valid_path_index = i - break - - class PageSource(object): """ A source for pages, e.g. a directory with one file per page. """ @@ -235,11 +109,11 @@ def buildDataProvider(self, page, user_data): if self._provider_type is None: cls = next((pt for pt in self.app.plugin_loader.getDataProviders() - if pt.PROVIDER_NAME == self.data_type), - None) + if pt.PROVIDER_NAME == self.data_type), + None) if cls is None: - raise ConfigurationError("Unknown data provider type: %s" % - self.data_type) + raise ConfigurationError( + "Unknown data provider type: %s" % self.data_type) self._provider_type = cls return self._provider_type(self, page, user_data) @@ -250,230 +124,3 @@ return None return tax_pages.get(tax_name) - -class IPreparingSource: - def setupPrepareParser(self, parser, app): - raise NotImplementedError() - - def buildMetadata(self, args): - raise NotImplementedError() - - -class IListableSource: - def listPath(self, rel_path): - raise NotImplementedError() - - def getDirpath(self, rel_path): - raise NotImplementedError() - - def getBasename(self, rel_path): - raise NotImplementedError() - - -class SimplePaginationSourceMixin(IPaginationSource): - def getItemsPerPage(self): - return self.config['items_per_page'] - - def getSourceIterator(self): - return SourceFactoryIterator(self) - - def getSorterIterator(self, it): - return DateSortIterator(it) - - def getTailIterator(self, it): - return PaginationDataBuilderIterator(it) - - def getPaginationFilter(self, page): - conf = (page.config.get('items_filters') or - page.app.config.get('site/items_filters')) - if conf == 'none' or conf == 'nil' or conf == '': - conf = None - if conf is not None: - f = PaginationFilter() - f.addClausesFromConfig(conf) - return f - return None - - def getSettingAccessor(self): - return lambda i, n: i.config.get(n) - - -class ArraySource(PageSource, SimplePaginationSourceMixin): - def __init__(self, app, inner_source, name='array', config=None): - super(ArraySource, self).__init__(app, name, config or {}) - self.inner_source = inner_source - - @property - def page_count(self): - return len(self.inner_source) - - def getPageFactories(self): - for p in self.inner_source: - yield CachedPageFactory(p) - - -class SimplePageSource(PageSource, IListableSource, IPreparingSource, - SimplePaginationSourceMixin): - def __init__(self, app, name, config): - super(SimplePageSource, self).__init__(app, name, config) - self.fs_endpoint = config.get('fs_endpoint', name) - self.fs_endpoint_path = os.path.join(self.root_dir, self.fs_endpoint) - self.supported_extensions = list(app.config.get('site/auto_formats').keys()) - self.default_auto_format = app.config.get('site/default_auto_format') - - def buildPageFactories(self): - logger.debug("Scanning for pages in: %s" % self.fs_endpoint_path) - if not os.path.isdir(self.fs_endpoint_path): - if self.ignore_missing_dir: - return - raise InvalidFileSystemEndpointError(self.name, self.fs_endpoint_path) - - for dirpath, dirnames, filenames in os.walk(self.fs_endpoint_path): - rel_dirpath = os.path.relpath(dirpath, self.fs_endpoint_path) - dirnames[:] = list(filter(self._filterPageDirname, dirnames)) - for f in filter(self._filterPageFilename, filenames): - fac_path = f - if rel_dirpath != '.': - fac_path = os.path.join(rel_dirpath, f) - slug = self._makeSlug(fac_path) - metadata = {'slug': slug} - fac_path = fac_path.replace('\\', '/') - self._populateMetadata(fac_path, metadata) - yield PageFactory(self, fac_path, metadata) - - def resolveRef(self, ref_path): - return os.path.normpath( - os.path.join(self.fs_endpoint_path, ref_path.lstrip("\\/"))) - - def findPagePath(self, metadata, mode): - uri_path = metadata.get('slug', '') - if not uri_path: - uri_path = '_index' - path = os.path.join(self.fs_endpoint_path, uri_path) - _, ext = os.path.splitext(path) - - if mode == MODE_CREATING: - if ext == '': - path = '%s.%s' % (path, self.default_auto_format) - rel_path = os.path.relpath(path, self.fs_endpoint_path) - rel_path = rel_path.replace('\\', '/') - self._populateMetadata(rel_path, metadata, mode) - return rel_path, metadata - - if ext == '': - paths_to_check = [ - '%s.%s' % (path, e) - for e in self.supported_extensions] - else: - paths_to_check = [path] - for path in paths_to_check: - if os.path.isfile(path): - rel_path = os.path.relpath(path, self.fs_endpoint_path) - rel_path = rel_path.replace('\\', '/') - self._populateMetadata(rel_path, metadata, mode) - return rel_path, metadata - - return None, None - - def listPath(self, rel_path): - rel_path = rel_path.lstrip('\\/') - path = os.path.join(self.fs_endpoint_path, rel_path) - names = sorted(os.listdir(path)) - items = [] - for name in names: - if os.path.isdir(os.path.join(path, name)): - if self._filterPageDirname(name): - rel_subdir = os.path.join(rel_path, name) - items.append((True, name, rel_subdir)) - else: - if self._filterPageFilename(name): - slug = self._makeSlug(os.path.join(rel_path, name)) - metadata = {'slug': slug} - - fac_path = name - if rel_path != '.': - fac_path = os.path.join(rel_path, name) - fac_path = fac_path.replace('\\', '/') - - self._populateMetadata(fac_path, metadata) - fac = PageFactory(self, fac_path, metadata) - - name, _ = os.path.splitext(name) - items.append((False, name, fac)) - return items - - def getDirpath(self, rel_path): - return os.path.dirname(rel_path) - - def getBasename(self, rel_path): - filename = os.path.basename(rel_path) - name, _ = os.path.splitext(filename) - return name - - def setupPrepareParser(self, parser, app): - parser.add_argument('uri', help='The URI for the new page.') - - def buildMetadata(self, args): - return {'slug': args.uri} - - def _makeSlug(self, rel_path): - slug, ext = os.path.splitext(rel_path) - slug = slug.replace('\\', '/') - if ext.lstrip('.') not in self.supported_extensions: - slug += ext - if slug.startswith('./'): - slug = slug[2:] - if slug == '_index': - slug = '' - return slug - - def _filterPageDirname(self, d): - return not d.endswith('-assets') - - def _filterPageFilename(self, f): - return (f[0] != '.' and # .DS_store and other crap - f[-1] != '~' and # Vim temp files and what-not - f not in ['Thumbs.db']) # Windows bullshit - - def _populateMetadata(self, rel_path, metadata, mode=None): - pass - - -class DefaultPageSource(SimplePageSource): - SOURCE_NAME = 'default' - - def __init__(self, app, name, config): - super(DefaultPageSource, self).__init__(app, name, config) - - -class SourceFactoryIterator(object): - def __init__(self, source): - self.source = source - self.it = None # This is to permit recursive traversal of the - # iterator chain. It acts as the end. - - def __iter__(self): - return self.source.getPages() - - -class DateSortIterator(object): - def __init__(self, it, reverse=True): - self.it = it - self.reverse = reverse - - def __iter__(self): - return iter(sorted(self.it, - key=lambda x: x.datetime, reverse=self.reverse)) - - -class PaginationDataBuilderIterator(object): - def __init__(self, it): - self.it = it - - def __iter__(self): - for page in self.it: - if page is None: - yield None - else: - yield PaginationData(page) -
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/sources/default.py Wed Feb 18 18:35:03 2015 -0800 @@ -0,0 +1,145 @@ +import os +import os.path +import logging +from piecrust.sources.base import ( + PageFactory, PageSource, InvalidFileSystemEndpointError, + MODE_CREATING) +from piecrust.sources.interfaces import IListableSource, IPreparingSource +from piecrust.sources.mixins import SimplePaginationSourceMixin + + +logger = logging.getLogger(__name__) + + +def filter_page_dirname(d): + return not (d.startswith('.') or d.endswith('-assets')) + + +def filter_page_filename(f): + return (f[0] != '.' and # .DS_store and other crap + f[-1] != '~' and # Vim temp files and what-not + f not in ['Thumbs.db']) # Windows bullshit + + +class DefaultPageSource(PageSource, IListableSource, IPreparingSource, + SimplePaginationSourceMixin): + SOURCE_NAME = 'default' + + def __init__(self, app, name, config): + super(DefaultPageSource, self).__init__(app, name, config) + self.fs_endpoint = config.get('fs_endpoint', name) + self.fs_endpoint_path = os.path.join(self.root_dir, self.fs_endpoint) + self.supported_extensions = list( + app.config.get('site/auto_formats').keys()) + self.default_auto_format = app.config.get('site/default_auto_format') + + def buildPageFactories(self): + logger.debug("Scanning for pages in: %s" % self.fs_endpoint_path) + if not os.path.isdir(self.fs_endpoint_path): + if self.ignore_missing_dir: + return + raise InvalidFileSystemEndpointError(self.name, + self.fs_endpoint_path) + + for dirpath, dirnames, filenames in os.walk(self.fs_endpoint_path): + rel_dirpath = os.path.relpath(dirpath, self.fs_endpoint_path) + dirnames[:] = list(filter(filter_page_dirname, dirnames)) + for f in filter(filter_page_filename, filenames): + fac_path = f + if rel_dirpath != '.': + fac_path = os.path.join(rel_dirpath, f) + slug = self._makeSlug(fac_path) + metadata = {'slug': slug} + fac_path = fac_path.replace('\\', '/') + self._populateMetadata(fac_path, metadata) + yield PageFactory(self, fac_path, metadata) + + def resolveRef(self, ref_path): + return os.path.normpath( + os.path.join(self.fs_endpoint_path, ref_path.lstrip("\\/"))) + + def findPagePath(self, metadata, mode): + uri_path = metadata.get('slug', '') + if not uri_path: + uri_path = '_index' + path = os.path.join(self.fs_endpoint_path, uri_path) + _, ext = os.path.splitext(path) + + if mode == MODE_CREATING: + if ext == '': + path = '%s.%s' % (path, self.default_auto_format) + rel_path = os.path.relpath(path, self.fs_endpoint_path) + rel_path = rel_path.replace('\\', '/') + self._populateMetadata(rel_path, metadata, mode) + return rel_path, metadata + + if ext == '': + paths_to_check = [ + '%s.%s' % (path, e) + for e in self.supported_extensions] + else: + paths_to_check = [path] + for path in paths_to_check: + if os.path.isfile(path): + rel_path = os.path.relpath(path, self.fs_endpoint_path) + rel_path = rel_path.replace('\\', '/') + self._populateMetadata(rel_path, metadata, mode) + return rel_path, metadata + + return None, None + + def listPath(self, rel_path): + rel_path = rel_path.lstrip('\\/') + path = os.path.join(self.fs_endpoint_path, rel_path) + names = sorted(os.listdir(path)) + items = [] + for name in names: + if os.path.isdir(os.path.join(path, name)): + if filter_page_dirname(name): + rel_subdir = os.path.join(rel_path, name) + items.append((True, name, rel_subdir)) + else: + if filter_page_filename(name): + slug = self._makeSlug(os.path.join(rel_path, name)) + metadata = {'slug': slug} + + fac_path = name + if rel_path != '.': + fac_path = os.path.join(rel_path, name) + fac_path = fac_path.replace('\\', '/') + + self._populateMetadata(fac_path, metadata) + fac = PageFactory(self, fac_path, metadata) + + name, _ = os.path.splitext(name) + items.append((False, name, fac)) + return items + + def getDirpath(self, rel_path): + return os.path.dirname(rel_path) + + def getBasename(self, rel_path): + filename = os.path.basename(rel_path) + name, _ = os.path.splitext(filename) + return name + + def setupPrepareParser(self, parser, app): + parser.add_argument('uri', help='The URI for the new page.') + + def buildMetadata(self, args): + return {'slug': args.uri} + + def _makeSlug(self, rel_path): + slug, ext = os.path.splitext(rel_path) + slug = slug.replace('\\', '/') + if ext.lstrip('.') not in self.supported_extensions: + slug += ext + if slug.startswith('./'): + slug = slug[2:] + if slug == '_index': + slug = '' + return slug + + def _populateMetadata(self, rel_path, metadata, mode=None): + pass +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/sources/interfaces.py Wed Feb 18 18:35:03 2015 -0800 @@ -0,0 +1,50 @@ + + +class IPaginationSource(object): + """ Defines the interface for a source that can be used as the data + for an iterator or a pagination. + """ + def getItemsPerPage(self): + raise NotImplementedError() + + def getSourceIterator(self): + raise NotImplementedError() + + def getSorterIterator(self, it): + raise NotImplementedError() + + def getTailIterator(self, it): + raise NotImplementedError() + + def getPaginationFilter(self, page): + raise NotImplementedError() + + def getSettingAccessor(self): + raise NotImplementedError() + + +class IListableSource: + """ Defines the interface for a source that can be iterated on in a + hierarchical manner, for use with the `family` data endpoint. + """ + def listPath(self, rel_path): + raise NotImplementedError() + + def getDirpath(self, rel_path): + raise NotImplementedError() + + def getBasename(self, rel_path): + raise NotImplementedError() + + +class IPreparingSource: + """ Defines the interface for a source whose pages can be created by the + `chef prepare` command. + """ + def setupPrepareParser(self, parser, app): + raise NotImplementedError() + + def buildMetadata(self, args): + raise NotImplementedError() + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/sources/mixins.py Wed Feb 18 18:35:03 2015 -0800 @@ -0,0 +1,139 @@ +import os +import os.path +import logging +from piecrust.data.base import PaginationData +from piecrust.data.filters import PaginationFilter +from piecrust.sources.base import PageFactory +from piecrust.sources.interfaces import IPaginationSource, IListableSource + + +logger = logging.getLogger(__name__) + + +class SourceFactoryIterator(object): + def __init__(self, source): + self.source = source + + # This is to permit recursive traversal of the + # iterator chain. It acts as the end. + self.it = None + + def __iter__(self): + return self.source.getPages() + + +class DateSortIterator(object): + def __init__(self, it, reverse=True): + self.it = it + self.reverse = reverse + + def __iter__(self): + return iter(sorted(self.it, + key=lambda x: x.datetime, reverse=self.reverse)) + + +class PaginationDataBuilderIterator(object): + def __init__(self, it): + self.it = it + + def __iter__(self): + for page in self.it: + if page is None: + yield None + else: + yield PaginationData(page) + + +def page_setting_accessor(item, name): + return item.config.get(name) + + +class SimplePaginationSourceMixin(IPaginationSource): + """ Implements the `IPaginationSource` interface in a standard way that + should fit most page sources. + """ + def getItemsPerPage(self): + return self.config['items_per_page'] + + def getSourceIterator(self): + return SourceFactoryIterator(self) + + def getSorterIterator(self, it): + return DateSortIterator(it) + + def getTailIterator(self, it): + return PaginationDataBuilderIterator(it) + + def getPaginationFilter(self, page): + conf = (page.config.get('items_filters') or + page.app.config.get('site/items_filters')) + if conf == 'none' or conf == 'nil' or conf == '': + conf = None + if conf is not None: + f = PaginationFilter() + f.addClausesFromConfig(conf) + return f + return None + + def getSettingAccessor(self): + return page_setting_accessor + + +class SimpleListableSourceMixin(IListableSource): + """ Implements the `IListableSource` interface for sources that map to + simple file-system structures. + """ + def listPath(self, rel_path): + rel_path = rel_path.lstrip('\\/') + path = self._getFullPath(rel_path) + names = self._sortFilenames(os.listdir(path)) + + items = [] + for name in names: + if os.path.isdir(os.path.join(path, name)): + if self._filterPageDirname(name): + rel_subdir = os.path.join(rel_path, name) + items.append((True, name, rel_subdir)) + else: + if self._filterPageFilename(name): + slug = self._makeSlug(os.path.join(rel_path, name)) + metadata = {'slug': slug} + + fac_path = name + if rel_path != '.': + fac_path = os.path.join(rel_path, name) + fac_path = fac_path.replace('\\', '/') + + self._populateMetadata(fac_path, metadata) + fac = PageFactory(self, fac_path, metadata) + + name, _ = os.path.splitext(name) + items.append((False, name, fac)) + return items + + def getDirpath(self, rel_path): + return os.path.dirname(rel_path) + + def getBasename(self, rel_path): + filename = os.path.basename(rel_path) + name, _ = os.path.splitext(filename) + return name + + def _getFullPath(self, rel_path): + return os.path.join(self.fs_endpoint_path, rel_path) + + def _sortFilenames(self, names): + return sorted(names) + + def _filterPageDirname(self, name): + return True + + def _filterPageFilename(self, name): + return True + + def _makeSlug(self, rel_path): + return rel_path.replace('\\', '/') + + def _populateMetadata(self, rel_path, metadata, mode=None): + pass +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/piecrust/sources/pageref.py Wed Feb 18 18:35:03 2015 -0800 @@ -0,0 +1,99 @@ +import re +import os.path + + +page_ref_pattern = re.compile(r'(?P<src>[\w]+)\:(?P<path>.*?)(;|$)') + + +class PageNotFoundError(Exception): + pass + + +class PageRef(object): + """ A reference to a page, with support for looking a page in different + realms. + """ + def __init__(self, app, page_ref): + self.app = app + self._page_ref = page_ref + self._paths = None + self._first_valid_path_index = -2 + self._exts = list(app.config.get('site/auto_formats').keys()) + + @property + def exists(self): + try: + self._checkPaths() + return True + except PageNotFoundError: + return False + + @property + def source_name(self): + self._checkPaths() + return self._paths[self._first_valid_path_index][0] + + @property + def source(self): + return self.app.getSource(self.source_name) + + @property + def rel_path(self): + self._checkPaths() + return self._paths[self._first_valid_path_index][1] + + @property + def path(self): + self._checkPaths() + return self._paths[self._first_valid_path_index][2] + + @property + def possible_rel_paths(self): + self._load() + return [p[1] for p in self._paths] + + @property + def possible_paths(self): + self._load() + return [p[2] for p in self._paths] + + def _load(self): + if self._paths is not None: + return + + it = list(page_ref_pattern.finditer(self._page_ref)) + if len(it) == 0: + raise Exception("Invalid page ref: %s" % self._page_ref) + + self._paths = [] + for m in it: + source_name = m.group('src') + source = self.app.getSource(source_name) + if source is None: + raise Exception("No such source: %s" % source_name) + rel_path = m.group('path') + path = source.resolveRef(rel_path) + if '%ext%' in rel_path: + for e in self._exts: + self._paths.append( + (source_name, + rel_path.replace('%ext%', e), + path.replace('%ext%', e))) + else: + self._paths.append((source_name, rel_path, path)) + + def _checkPaths(self): + if self._first_valid_path_index >= 0: + return + if self._first_valid_path_index == -1: + raise PageNotFoundError( + "No valid paths were found for page reference: %s" % + self._page_ref) + + self._load() + self._first_valid_path_index = -1 + for i, path_info in enumerate(self._paths): + if os.path.isfile(path_info[2]): + self._first_valid_path_index = i + break +
--- a/piecrust/sources/posts.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/sources/posts.py Wed Feb 18 18:35:03 2015 -0800 @@ -4,10 +4,12 @@ import glob import logging import datetime -from piecrust.sources.base import (PageSource, IPreparingSource, - SimplePaginationSourceMixin, - PageNotFoundError, InvalidFileSystemEndpointError, - PageFactory, MODE_CREATING, MODE_PARSING) +from piecrust.sources.base import ( + PageSource, InvalidFileSystemEndpointError, PageFactory, + MODE_CREATING, MODE_PARSING) +from piecrust.sources.interfaces import IPreparingSource +from piecrust.sources.mixins import SimplePaginationSourceMixin +from piecrust.sources.pageref import PageNotFoundError logger = logging.getLogger(__name__)
--- a/piecrust/sources/prose.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/sources/prose.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,16 +1,14 @@ import os import os.path import logging -from piecrust.sources.base import ( - SimplePageSource, SimplePaginationSourceMixin, - MODE_CREATING) +from piecrust.sources.base import MODE_CREATING +from piecrust.sources.default import DefaultPageSource logger = logging.getLogger(__name__) -class ProseSource(SimplePageSource, - SimplePaginationSourceMixin): +class ProseSource(DefaultPageSource): SOURCE_NAME = 'prose' def __init__(self, app, name, config):
--- a/piecrust/taxonomies.py Mon Feb 16 08:25:08 2015 -0800 +++ b/piecrust/taxonomies.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,4 +1,4 @@ -from piecrust.sources.base import PageRef, PageNotFoundError +from piecrust.sources.pageref import PageRef, PageNotFoundError class Taxonomy(object):
--- a/tests/test_data_paginator.py Mon Feb 16 08:25:08 2015 -0800 +++ b/tests/test_data_paginator.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,7 +1,7 @@ import math import pytest -from piecrust.data.base import IPaginationSource from piecrust.data.paginator import Paginator +from piecrust.sources.interfaces import IPaginationSource class MockSource(list, IPaginationSource):
--- a/tests/test_sources_autoconfig.py Mon Feb 16 08:25:08 2015 -0800 +++ b/tests/test_sources_autoconfig.py Wed Feb 18 18:35:03 2015 -0800 @@ -88,10 +88,16 @@ (mock_fs(), [], []), (mock_fs().withPage('test/something.md'), ['something.md'], - [{'slug': 'something', 'config': {'foo': 0}}]), + [{'slug': 'something', + 'config': {'foo': 0, 'foo_trail': [0]}}]), (mock_fs().withPage('test/08_something.md'), ['08_something.md'], - [{'slug': 'something', 'config': {'foo': 8}}]) + [{'slug': 'something', + 'config': {'foo': 8, 'foo_trail': [8]}}]), + (mock_fs().withPage('test/02_there/08_something.md'), + ['02_there/08_something.md'], + [{'slug': 'there/something', + 'config': {'foo': 8, 'foo_trail': [2, 8]}}]), ]) def test_ordered_source_factories(fs, expected_paths, expected_metadata): site_config = { @@ -120,22 +126,27 @@ (mock_fs(), 'missing', None, None), (mock_fs().withPage('test/something.md'), 'something', 'something.md', - {'slug': 'something', 'config': {'foo': 0}}), + {'slug': 'something', + 'config': {'foo': 0, 'foo_trail': [0]}}), (mock_fs().withPage('test/bar/something.md'), 'bar/something', 'bar/something.md', - {'slug': 'bar/something', 'config': {'foo': 0}}), + {'slug': 'bar/something', + 'config': {'foo': 0, 'foo_trail': [0]}}), (mock_fs().withPage('test/42_something.md'), 'something', '42_something.md', - {'slug': 'something', 'config': {'foo': 42}}), + {'slug': 'something', + 'config': {'foo': 42, 'foo_trail': [42]}}), (mock_fs().withPage('test/bar/42_something.md'), 'bar/something', 'bar/42_something.md', - {'slug': 'bar/something', 'config': {'foo': 42}}), + {'slug': 'bar/something', + 'config': {'foo': 42, 'foo_trail': [42]}}), ((mock_fs() .withPage('test/42_something.md') .withPage('test/43_other_something.md')), 'something', '42_something.md', - {'slug': 'something', 'config': {'foo': 42}}), + {'slug': 'something', + 'config': {'foo': 42, 'foo_trail': [42]}}), ]) def test_ordered_source_find(fs, route_path, expected_path, expected_metadata): @@ -152,7 +163,7 @@ with mock_fs_scope(fs): app = fs.getApp() s = app.getSource('test') - route_metadata = {'path': route_path} + route_metadata = {'slug': route_path} fac_path, metadata = s.findPagePath(route_metadata, MODE_PARSING) assert fac_path == expected_path assert metadata == expected_metadata
--- a/tests/test_sources_base.py Mon Feb 16 08:25:08 2015 -0800 +++ b/tests/test_sources_base.py Wed Feb 18 18:35:03 2015 -0800 @@ -1,7 +1,7 @@ import os import pytest from piecrust.app import PieCrust -from piecrust.sources.base import PageRef, PageNotFoundError +from piecrust.sources.pageref import PageRef, PageNotFoundError from .mockutil import mock_fs, mock_fs_scope