Mercurial > piecrust2

changeset 239:f43f19975671

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
sources: Refactor `autoconfig` source, add `OrderedPageSource`. Also add unit tests.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 15 Feb 2015 22:48:42 -0800
parents 4dce0e61b48c
children c1d4e86a3918
files piecrust/plugins/builtin.py piecrust/sources/autoconfig.py tests/test_sources_autoconfig.py
diffstat 3 files changed, 331 insertions(+), 22 deletions(-) [+]
line wrap: on
line diff
--- a/piecrust/plugins/builtin.py	Sun Feb 15 22:47:46 2015 -0800
+++ b/piecrust/plugins/builtin.py	Sun Feb 15 22:48:42 2015 -0800
@@ -31,7 +31,8 @@
 from piecrust.sources.base import DefaultPageSource
 from piecrust.sources.posts import (
         FlatPostsSource, ShallowPostsSource, HierarchyPostsSource)
-from piecrust.sources.autoconfig import AutoConfigSource
+from piecrust.sources.autoconfig import (
+        AutoConfigSource, OrderedPageSource)
 from piecrust.sources.prose import ProseSource
 from piecrust.templating.jinjaengine import JinjaTemplateEngine
 from piecrust.templating.pystacheengine import PystacheTemplateEngine
@@ -72,6 +73,7 @@
                 ShallowPostsSource,
                 HierarchyPostsSource,
                 AutoConfigSource,
+                OrderedPageSource,
                 ProseSource]
 
     def getDataProviders(self):
--- a/piecrust/sources/autoconfig.py	Sun Feb 15 22:47:46 2015 -0800
+++ b/piecrust/sources/autoconfig.py	Sun Feb 15 22:48:42 2015 -0800
@@ -1,6 +1,10 @@
+import re
 import os
 import os.path
+import glob
 import logging
+from piecrust.configuration import ConfigurationError
+from piecrust.data.iterators import SettingSortIterator
 from piecrust.sources.base import (
         SimplePageSource, IPreparingSource, SimplePaginationSourceMixin,
         PageNotFoundError, InvalidFileSystemEndpointError,
@@ -10,53 +14,197 @@
 logger = logging.getLogger(__name__)
 
 
-class AutoConfigSource(SimplePageSource,
-                       SimplePaginationSourceMixin):
-    SOURCE_NAME = 'autoconfig'
-
+class AutoConfigSourceBase(SimplePageSource,
+                           SimplePaginationSourceMixin):
+    """ Base class for page sources that automatically apply configuration
+        settings to their generated pages based on those pages' paths.
+    """
     def __init__(self, app, name, config):
-        super(AutoConfigSource, self).__init__(app, name, config)
-        self.setting_name = config.get('setting_name', name)
-        self.collapse_single_values = config.get('collapse_single_values', False)
-        self.only_single_values = config.get('only_single_values', False)
+        super(AutoConfigSourceBase, self).__init__(app, name, config)
+        self.capture_mode = config.get('capture_mode', 'path')
+        if self.capture_mode not in ['path', 'dirname', 'filename']:
+            raise ConfigurationError("Capture mode in source '%s' must be "
+                                     "one of: path, dirname, filename" %
+                                     name)
 
     def buildPageFactories(self):
         if not os.path.isdir(self.fs_endpoint_path):
-            raise InvalidFileSystemEndpointError(self.name, self.fs_endpoint_path)
+            raise InvalidFileSystemEndpointError(self.name,
+                                                 self.fs_endpoint_path)
 
         for dirpath, dirnames, filenames in os.walk(self.fs_endpoint_path):
             if not filenames:
                 continue
-            config = self._extractConfigFragment(dirpath)
+
+            rel_dirpath = os.path.relpath(dirpath, self.fs_endpoint_path)
+
+            # If `capture_mode` is `dirname`, we don't need to recompute it
+            # for each filename, so we do it here.
+            if self.capture_mode == 'dirname':
+                config = self.extractConfigFragment(rel_dirpath)
+
             for f in filenames:
-                slug, ext = os.path.splitext(f)
-                path = os.path.join(dirpath, f)
+                if self.capture_mode == 'path':
+                    path = os.path.join(rel_dirpath, f)
+                    config = self.extractConfigFragment(path)
+                elif self.capture_mode == 'filename':
+                    config = self.extractConfigFragment(f)
+
+                fac_path = f
+                if rel_dirpath != '.':
+                    fac_path = os.path.join(rel_dirpath, f)
+
+                slug = self.makeSlug(rel_dirpath, f)
+
                 metadata = {
                         'slug': slug,
                         'config': config}
-                yield PageFactory(self, path, metadata)
+                yield PageFactory(self, fac_path, metadata)
+
+    def makeSlug(self, rel_dirpath, filename):
+        raise NotImplementedError()
+
+    def extractConfigFragment(self, rel_path):
+        raise NotImplementedError()
+
+    def findPagePath(self, metadata, mode):
+        raise NotImplementedError()
+
 
-    def _extractConfigFragment(self, path):
-        rel_path = os.path.relpath(path, self.fs_endpoint_path)
+class AutoConfigSource(AutoConfigSourceBase):
+    """ Page source that extracts configuration settings from the sub-folders
+        each page resides in. This is ideal for setting tags or categories
+        on pages based on the folders they're in.
+    """
+    SOURCE_NAME = 'autoconfig'
+
+    def __init__(self, app, name, config):
+        config['capture_mode'] = 'dirname'
+        super(AutoConfigSource, self).__init__(app, name, config)
+        self.setting_name = config.get('setting_name', name)
+        self.only_single_values = config.get('only_single_values', False)
+        self.collapse_single_values = config.get('collapse_single_values',
+                                                 False)
+        self.supported_extensions = list(
+                app.config.get('site/auto_formats').keys())
+
+    def makeSlug(self, rel_dirpath, filename):
+        slug, ext = os.path.splitext(filename)
+        if ext.lstrip('.') not in self.supported_extensions:
+            slug += ext
+        return slug
+
+    def extractConfigFragment(self, rel_path):
         if rel_path == '.':
             values = []
         else:
             values = rel_path.split(os.sep)
-        if self.only_single_values and len(values) > 1:
-            raise Exception("Only one folder level is allowed for pages "
-                            "in source '%s'." % self.name)
-        if self.collapse_single_values and len(values) == 1:
-            values = values[0]
+
+        if self.only_single_values:
+            if len(values) > 1:
+                raise Exception("Only one folder level is allowed for pages "
+                                "in source '%s'." % self.name)
+            elif len(values) == 1:
+                values = values[0]
+            else:
+                values = None
+
+        if self.collapse_single_values:
+            if len(values) == 1:
+                values = values[0]
+            elif len(values) == 0:
+                values = None
+
         return {self.setting_name: values}
 
     def findPagePath(self, metadata, mode):
+        # Pages from this source are effectively flattened, so we need to
+        # find pages using a brute-force kinda way.
         for dirpath, dirnames, filenames in os.walk(self.fs_endpoint_path):
             for f in filenames:
                 slug, _ = os.path.splitext(f)
                 if slug == metadata['slug']:
                     path = os.path.join(dirpath, f)
                     rel_path = os.path.relpath(path, self.fs_endpoint_path)
-                    config = self._extractConfigFragment(dirpath)
+                    config = self.extractConfigFragment(dirpath)
                     metadata = {'slug': slug, 'config': config}
                     return rel_path, metadata
 
+
+class OrderedPageSource(AutoConfigSourceBase):
+    """ A page source that assigns an "order" to its pages based on a
+        numerical prefix in their filename. Page iterators will automatically
+        sort pages using that order.
+    """
+    SOURCE_NAME = 'ordered'
+
+    re_pattern = re.compile(r'(^|/)(?P<num>\d+)_')
+
+    def __init__(self, app, name, config):
+        config['capture_mode'] = 'filename'
+        super(OrderedPageSource, self).__init__(app, name, config)
+        self.setting_name = config.get('setting_name', 'order')
+        self.default_value = config.get('default_value', 0)
+        self.supported_extensions = list(
+                app.config.get('site/auto_formats').keys())
+
+    def makeSlug(self, rel_dirpath, filename):
+        slug, ext = os.path.splitext(filename)
+        if ext.lstrip('.') not in self.supported_extensions:
+            slug += ext
+        slug = self.re_pattern.sub(r'\1', slug)
+        slug = os.path.join(rel_dirpath, slug).replace('\\', '/')
+        if slug.startswith('./'):
+            slug = slug[2:]
+        return slug
+
+    def extractConfigFragment(self, rel_path):
+        m = self.re_pattern.match(rel_path)
+        if m is not None:
+            val = int(m.group('num'))
+        else:
+            val = self.default_value
+        return {self.setting_name: val}
+
+    def findPagePath(self, metadata, mode):
+        uri_path = metadata.get('slug', '')
+        if uri_path != '':
+            uri_parts = ['*_%s' % p for p in uri_path.split('/')]
+        else:
+            uri_parts = ['*__index']
+        uri_parts.insert(0, self.fs_endpoint_path)
+        path = os.path.join(*uri_parts)
+
+        _, ext = os.path.splitext(uri_path)
+        if ext == '':
+            path += '.*'
+
+        possibles = glob.glob(path)
+
+        if len(possibles) == 0:
+            return None, None
+
+        if len(possibles) > 1:
+            raise Exception("More than one path matching: %s" % uri_path)
+
+        path = possibles[0]
+        fac_path = os.path.relpath(path, self.fs_endpoint_path)
+
+        _, filename = os.path.split(path)
+        config = self.extractConfigFragment(filename)
+        metadata = {'slug': uri_path, 'config': config}
+
+        return fac_path, metadata
+
+    def getSorterIterator(self, it):
+        accessor = self.getSettingAccessor()
+        return SettingSortIterator(it, self.setting_name,
+                                   value_accessor=accessor)
+
+    def _populateMetadata(self, rel_path, metadata, mode=None):
+        _, filename = os.path.split(rel_path)
+        config = self.extractConfigFragment(filename)
+        metadata['config'] = config
+        slug = metadata['slug']
+        metadata['slug'] = self.re_pattern.sub(r'\1', slug)
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test_sources_autoconfig.py	Sun Feb 15 22:48:42 2015 -0800
@@ -0,0 +1,159 @@
+import pytest
+from piecrust.sources.base import MODE_PARSING
+from .mockutil import mock_fs, mock_fs_scope
+
+
+@pytest.mark.parametrize(
+        'fs, src_config, expected_paths, expected_metadata',
+        [
+            (mock_fs(), {}, [], []),
+            (mock_fs().withPage('test/something.md'),
+                {},
+                ['something.md'],
+                [{'slug': 'something', 'config': {'foo': []}}]),
+            (mock_fs().withPage('test/bar/something.md'),
+                {},
+                ['bar/something.md'],
+                [{'slug': 'something', 'config': {'foo': ['bar']}}]),
+            (mock_fs().withPage('test/bar1/bar2/something.md'),
+                {},
+                ['bar1/bar2/something.md'],
+                [{'slug': 'something', 'config': {'foo': ['bar1', 'bar2']}}]),
+
+            (mock_fs().withPage('test/something.md'),
+                {'collapse_single_values': True},
+                ['something.md'],
+                [{'slug': 'something', 'config': {'foo': None}}]),
+            (mock_fs().withPage('test/bar/something.md'),
+                {'collapse_single_values': True},
+                ['bar/something.md'],
+                [{'slug': 'something', 'config': {'foo': 'bar'}}]),
+            (mock_fs().withPage('test/bar1/bar2/something.md'),
+                {'collapse_single_values': True},
+                ['bar1/bar2/something.md'],
+                [{'slug': 'something', 'config': {'foo': ['bar1', 'bar2']}}]),
+
+            (mock_fs().withPage('test/something.md'),
+                {'only_single_values': True},
+                ['something.md'],
+                [{'slug': 'something', 'config': {'foo': None}}]),
+            (mock_fs().withPage('test/bar/something.md'),
+                {'only_single_values': True},
+                ['bar/something.md'],
+                [{'slug': 'something', 'config': {'foo': 'bar'}}]),
+            ])
+def test_autoconfig_source_factories(fs, src_config, expected_paths,
+                                     expected_metadata):
+    site_config = {
+            'sources': {
+                'test': {'type': 'autoconfig',
+                         'setting_name': 'foo'}
+                },
+            'routes': [
+                {'url': '/%slug%', 'source': 'test'}]
+            }
+    site_config['sources']['test'].update(src_config)
+    fs.withConfig({'site': site_config})
+    fs.withDir('kitchen/test')
+    with mock_fs_scope(fs):
+        app = fs.getApp()
+        s = app.getSource('test')
+        facs = list(s.buildPageFactories())
+        paths = [f.rel_path for f in facs]
+        assert paths == expected_paths
+        metadata = [f.metadata for f in facs]
+        assert metadata == expected_metadata
+
+
+def test_autoconfig_fails_if_multiple_folders():
+    site_config = {
+            'sources': {
+                'test': {'type': 'autoconfig',
+                         'setting_name': 'foo',
+                         'only_single_values': True}
+                }
+            }
+    fs = mock_fs().withConfig({'site': site_config})
+    fs.withPage('test/bar1/bar2/something.md')
+    with mock_fs_scope(fs):
+        app = fs.getApp()
+        s = app.getSource('test')
+        with pytest.raises(Exception):
+            list(s.buildPageFactories())
+
+
+@pytest.mark.parametrize(
+        'fs, expected_paths, expected_metadata',
+        [
+            (mock_fs(), [], []),
+            (mock_fs().withPage('test/something.md'),
+                ['something.md'],
+                [{'slug': 'something', 'config': {'foo': 0}}]),
+            (mock_fs().withPage('test/08_something.md'),
+                ['08_something.md'],
+                [{'slug': 'something', 'config': {'foo': 8}}])
+            ])
+def test_ordered_source_factories(fs, expected_paths, expected_metadata):
+    site_config = {
+            'sources': {
+                'test': {'type': 'ordered',
+                         'setting_name': 'foo'}
+                },
+            'routes': [
+                {'url': '/%slug%', 'source': 'test'}]
+            }
+    fs.withConfig({'site': site_config})
+    fs.withDir('kitchen/test')
+    with mock_fs_scope(fs):
+        app = fs.getApp()
+        s = app.getSource('test')
+        facs = list(s.buildPageFactories())
+        paths = [f.rel_path for f in facs]
+        assert paths == expected_paths
+        metadata = [f.metadata for f in facs]
+        assert metadata == expected_metadata
+
+
+@pytest.mark.parametrize(
+        'fs, route_path, expected_path, expected_metadata',
+        [
+            (mock_fs(), 'missing', None, None),
+            (mock_fs().withPage('test/something.md'),
+                'something', 'something.md',
+                {'slug': 'something', 'config': {'foo': 0}}),
+            (mock_fs().withPage('test/bar/something.md'),
+                'bar/something', 'bar/something.md',
+                {'slug': 'bar/something', 'config': {'foo': 0}}),
+            (mock_fs().withPage('test/42_something.md'),
+                'something', '42_something.md',
+                {'slug': 'something', 'config': {'foo': 42}}),
+            (mock_fs().withPage('test/bar/42_something.md'),
+                'bar/something', 'bar/42_something.md',
+                {'slug': 'bar/something', 'config': {'foo': 42}}),
+
+            ((mock_fs()
+                .withPage('test/42_something.md')
+                .withPage('test/43_other_something.md')),
+                'something', '42_something.md',
+                {'slug': 'something', 'config': {'foo': 42}}),
+            ])
+def test_ordered_source_find(fs, route_path, expected_path,
+                             expected_metadata):
+    site_config = {
+            'sources': {
+                'test': {'type': 'ordered',
+                         'setting_name': 'foo'}
+                },
+            'routes': [
+                {'url': '/%slug%', 'source': 'test'}]
+            }
+    fs.withConfig({'site': site_config})
+    fs.withDir('kitchen/test')
+    with mock_fs_scope(fs):
+        app = fs.getApp()
+        s = app.getSource('test')
+        route_metadata = {'path': route_path}
+        fac_path, metadata = s.findPagePath(route_metadata, MODE_PARSING)
+        assert fac_path == expected_path
+        assert metadata == expected_metadata
+