Mercurial > piecrust2

comparison piecrust/sources/default.py @ 852:4850f8c21b6e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
core: Start of the big refactor for PieCrust 3.0. * Everything is a `ContentSource`, including assets directories. * Most content sources are subclasses of the base file-system source. * A source is processed by a "pipeline", and there are 2 built-in pipelines, one for assets and one for pages. The asset pipeline is vaguely functional, but the page pipeline is completely broken right now. * Rewrite the baking process as just running appropriate pipelines on each content item. This should allow for better parallelization.
author Ludovic Chabant <ludovic@chabant.com>
date Wed, 17 May 2017 00:11:48 -0700
parents 58ebf50235a5
children f070a4fc033c
comparison
equal deleted inserted replaced
851:2c7e57d80bba 852:4850f8c21b6e
1 import os.path 1 import os.path
2 import logging 2 import logging
3 from piecrust import osutil
4 from piecrust.routing import RouteParameter 3 from piecrust.routing import RouteParameter
5 from piecrust.sources.base import ( 4 from piecrust.sources.base import REL_ASSETS, ContentItem
6 PageFactory, PageSource, InvalidFileSystemEndpointError, 5 from piecrust.sources.fs import FSContentSource
7 MODE_CREATING)
8 from piecrust.sources.interfaces import ( 6 from piecrust.sources.interfaces import (
9 IListableSource, IPreparingSource, IInteractiveSource, 7 IPreparingSource, IInteractiveSource, InteractiveField)
10 InteractiveField) 8 from piecrust.sources.mixins import SimpleAssetsSubDirMixin
11 from piecrust.sources.mixins import SimplePaginationSourceMixin 9 from piecrust.uriutil import uri_to_title
12 10
13 11
14 logger = logging.getLogger(__name__) 12 logger = logging.getLogger(__name__)
15 13
16 14
17 def filter_page_dirname(d): 15 class DefaultContentSource(FSContentSource,
18 return not (d.startswith('.') or d.endswith('-assets')) 16 SimpleAssetsSubDirMixin,
19 17 IPreparingSource, IInteractiveSource):
20
21 def filter_page_filename(f):
22 return (f[0] != '.' and # .DS_store and other crap
23 f[-1] != '~' and # Vim temp files and what-not
24 f not in ['Thumbs.db']) # Windows bullshit
25
26
27 class DefaultPageSource(PageSource,
28 IListableSource, IPreparingSource, IInteractiveSource,
29 SimplePaginationSourceMixin):
30 SOURCE_NAME = 'default' 18 SOURCE_NAME = 'default'
31 19
32 def __init__(self, app, name, config): 20 def __init__(self, app, name, config):
33 super(DefaultPageSource, self).__init__(app, name, config) 21 super().__init__(app, name, config)
34 self.fs_endpoint = config.get('fs_endpoint', name) 22 self.auto_formats = app.config.get('site/auto_formats')
35 self.fs_endpoint_path = os.path.join(self.root_dir, self.fs_endpoint)
36 self.supported_extensions = list(
37 app.config.get('site/auto_formats').keys())
38 self.default_auto_format = app.config.get('site/default_auto_format') 23 self.default_auto_format = app.config.get('site/default_auto_format')
24 self.supported_extensions = list(self.auto_formats)
39 25
40 def getSupportedRouteParameters(self): 26 def _createItemMetadata(self, path):
41 return [ 27 return self._doCreateItemMetadata(path)
42 RouteParameter('slug', RouteParameter.TYPE_PATH)]
43 28
44 def buildPageFactories(self): 29 def _finalizeContent(self, parent_group, items, groups):
45 logger.debug("Scanning for pages in: %s" % self.fs_endpoint_path) 30 SimpleAssetsSubDirMixin._onFinalizeContent(
46 if not os.path.isdir(self.fs_endpoint_path): 31 self, parent_group, items, groups)
47 if self.ignore_missing_dir:
48 return
49 raise InvalidFileSystemEndpointError(self.name,
50 self.fs_endpoint_path)
51 32
52 for dirpath, dirnames, filenames in osutil.walk(self.fs_endpoint_path): 33 def _doCreateItemMetadata(self, path):
53 rel_dirpath = os.path.relpath(dirpath, self.fs_endpoint_path) 34 slug = self._makeSlug(path)
54 dirnames[:] = list(filter(filter_page_dirname, dirnames)) 35 metadata = {
55 for f in sorted(filter(filter_page_filename, filenames)): 36 'slug': slug
56 fac_path = f 37 }
57 if rel_dirpath != '.': 38 _, ext = os.path.splitext(path)
58 fac_path = os.path.join(rel_dirpath, f) 39 if ext:
40 fmt = self.auto_formats.get(ext.lstrip('.'))
41 if fmt:
42 metadata['config'] = {'format': fmt}
43 return metadata
59 44
60 slug = self._makeSlug(fac_path) 45 def _makeSlug(self, path):
61 metadata = {'slug': slug} 46 rel_path = os.path.relpath(path, self.fs_endpoint_path)
62 fac_path = fac_path.replace('\\', '/')
63 self._populateMetadata(fac_path, metadata)
64 yield PageFactory(self, fac_path, metadata)
65
66 def buildPageFactory(self, path):
67 if not path.startswith(self.fs_endpoint_path):
68 raise Exception("Page path '%s' isn't inside '%s'." % (
69 path, self.fs_enpoint_path))
70 rel_path = path[len(self.fs_endpoint_path):].lstrip('\\/')
71 slug = self._makeSlug(rel_path)
72 metadata = {'slug': slug}
73 fac_path = rel_path.replace('\\', '/')
74 self._populateMetadata(fac_path, metadata)
75 return PageFactory(self, fac_path, metadata)
76
77 def resolveRef(self, ref_path):
78 path = os.path.normpath(
79 os.path.join(self.fs_endpoint_path, ref_path.lstrip("\\/")))
80 slug = self._makeSlug(ref_path)
81 metadata = {'slug': slug}
82 self._populateMetadata(ref_path, metadata)
83 return path, metadata
84
85 def findPageFactory(self, metadata, mode):
86 uri_path = metadata.get('slug', '')
87 if not uri_path:
88 uri_path = '_index'
89 path = os.path.join(self.fs_endpoint_path, uri_path)
90 _, ext = os.path.splitext(path)
91
92 if mode == MODE_CREATING:
93 if ext == '':
94 path = '%s.%s' % (path, self.default_auto_format)
95 rel_path = os.path.relpath(path, self.fs_endpoint_path)
96 rel_path = rel_path.replace('\\', '/')
97 self._populateMetadata(rel_path, metadata, mode)
98 return PageFactory(self, rel_path, metadata)
99
100 if ext == '':
101 paths_to_check = [
102 '%s.%s' % (path, e)
103 for e in self.supported_extensions]
104 else:
105 paths_to_check = [path]
106 for path in paths_to_check:
107 if os.path.isfile(path):
108 rel_path = os.path.relpath(path, self.fs_endpoint_path)
109 rel_path = rel_path.replace('\\', '/')
110 self._populateMetadata(rel_path, metadata, mode)
111 return PageFactory(self, rel_path, metadata)
112
113 return None
114
115 def listPath(self, rel_path):
116 rel_path = rel_path.lstrip('\\/')
117 path = os.path.join(self.fs_endpoint_path, rel_path)
118 names = sorted(osutil.listdir(path))
119 items = []
120 for name in names:
121 if os.path.isdir(os.path.join(path, name)):
122 if filter_page_dirname(name):
123 rel_subdir = os.path.join(rel_path, name)
124 items.append((True, name, rel_subdir))
125 else:
126 if filter_page_filename(name):
127 slug = self._makeSlug(os.path.join(rel_path, name))
128 metadata = {'slug': slug}
129
130 fac_path = name
131 if rel_path != '.':
132 fac_path = os.path.join(rel_path, name)
133 fac_path = fac_path.replace('\\', '/')
134
135 self._populateMetadata(fac_path, metadata)
136 fac = PageFactory(self, fac_path, metadata)
137
138 name, _ = os.path.splitext(name)
139 items.append((False, name, fac))
140 return items
141
142 def getDirpath(self, rel_path):
143 return os.path.dirname(rel_path)
144
145 def getBasename(self, rel_path):
146 filename = os.path.basename(rel_path)
147 name, _ = os.path.splitext(filename)
148 return name
149
150 def setupPrepareParser(self, parser, app):
151 parser.add_argument('uri', help='The URI for the new page.')
152
153 def buildMetadata(self, args):
154 return {'slug': args.uri}
155
156 def getInteractiveFields(self):
157 return [
158 InteractiveField('slug', InteractiveField.TYPE_STRING,
159 'new-page')]
160
161 def _makeSlug(self, rel_path):
162 slug, ext = os.path.splitext(rel_path) 47 slug, ext = os.path.splitext(rel_path)
163 slug = slug.replace('\\', '/') 48 slug = slug.replace('\\', '/')
164 if ext.lstrip('.') not in self.supported_extensions: 49 if ext.lstrip('.') not in self.supported_extensions:
165 slug += ext 50 slug += ext
166 if slug.startswith('./'): 51 if slug.startswith('./'):
167 slug = slug[2:] 52 slug = slug[2:]
168 if slug == '_index': 53 if slug == '_index':
169 slug = '' 54 slug = ''
170 return slug 55 return slug
171 56
172 def _populateMetadata(self, rel_path, metadata, mode=None): 57 def getRelatedContents(self, item, relationship):
173 pass 58 if relationship == REL_ASSETS:
59 SimpleAssetsSubDirMixin._getRelatedAssetsContents(self, item)
60 raise NotImplementedError()
174 61
62 def getSupportedRouteParameters(self):
63 return [
64 RouteParameter('slug', RouteParameter.TYPE_PATH)]
65
66 def findContent(self, route_params):
67 uri_path = route_params.get('slug', '')
68 if not uri_path:
69 uri_path = '_index'
70 path = os.path.join(self.fs_endpoint_path, uri_path)
71 _, ext = os.path.splitext(path)
72
73 if ext == '':
74 paths_to_check = [
75 '%s.%s' % (path, e)
76 for e in self.supported_extensions]
77 else:
78 paths_to_check = [path]
79 for path in paths_to_check:
80 if os.path.isfile(path):
81 metadata = self._doCreateItemMetadata(path)
82 return ContentItem(path, metadata)
83 return None
84
85 def setupPrepareParser(self, parser, app):
86 parser.add_argument('uri', help='The URI for the new page.')
87
88 def createContent(self, args):
89 if not hasattr(args, 'uri'):
90 uri = None
91 else:
92 uri = args.uri
93 if not uri:
94 uri = '_index'
95 path = os.path.join(self.fs_endpoint_path, uri)
96 _, ext = os.path.splitext(path)
97 if ext == '':
98 path = '%s.%s' % (path, self.default_auto_format)
99
100 metadata = self._doCreateItemMetadata(path)
101 config = metadata.setdefault('config', {})
102 config.update({'title': uri_to_title(
103 os.path.basename(metadata['slug']))})
104 return ContentItem(path, metadata)
105
106 def getInteractiveFields(self):
107 return [
108 InteractiveField('slug', InteractiveField.TYPE_STRING,
109 'new-page')]