Mercurial > piecrust2

comparison piecrust/sources/base.py @ 242:f130365568ff

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
internal: Code reorganization to put less stuff in `sources.base`. Interfaces that sources can implement are in `sources.interfaces`. The default page source is in `sources.default`. The `SimplePageSource` is gone since most subclasses only wanted to do *some* stuff the same, but *lots* of stuff slightly different. I may have to revisit the code to extract exactly the code that's in common.
author Ludovic Chabant <ludovic@chabant.com>
date Wed, 18 Feb 2015 18:35:03 -0800
parents 4dce0e61b48c
children 88bffd469b04
comparison
equal deleted inserted replaced
241:85a6c7ba5e3b 242:f130365568ff
1 import re
2 import os
3 import os.path
4 import logging 1 import logging
5 from werkzeug.utils import cached_property 2 from werkzeug.utils import cached_property
6 from piecrust.configuration import ConfigurationError 3 from piecrust.configuration import ConfigurationError
7 from piecrust.data.base import IPaginationSource, PaginationData
8 from piecrust.data.filters import PaginationFilter
9 from piecrust.page import Page 4 from piecrust.page import Page
10 5
11 6
12 REALM_USER = 0 7 REALM_USER = 0
13 REALM_THEME = 1 8 REALM_THEME = 1
21 16
22 17
23 logger = logging.getLogger(__name__) 18 logger = logging.getLogger(__name__)
24 19
25 20
26 page_ref_pattern = re.compile(r'(?P<src>[\w]+)\:(?P<path>.*?)(;|$)')
27
28
29 def build_pages(app, factories): 21 def build_pages(app, factories):
30 with app.env.page_repository.startBatchGet(): 22 with app.env.page_repository.startBatchGet():
31 for f in factories: 23 for f in factories:
32 yield f.buildPage() 24 yield f.buildPage()
33
34
35 class PageNotFoundError(Exception):
36 pass
37 25
38 26
39 class InvalidFileSystemEndpointError(Exception): 27 class InvalidFileSystemEndpointError(Exception):
40 def __init__(self, source_name, fs_endpoint): 28 def __init__(self, source_name, fs_endpoint):
41 super(InvalidFileSystemEndpointError, self).__init__( 29 super(InvalidFileSystemEndpointError, self).__init__(
71 page = Page(self.source, self.metadata, self.rel_path) 59 page = Page(self.source, self.metadata, self.rel_path)
72 # Load it right away, especially when using the page repository, 60 # Load it right away, especially when using the page repository,
73 # because we'll be inside a critical scope. 61 # because we'll be inside a critical scope.
74 page._load() 62 page._load()
75 return page 63 return page
76
77
78 class CachedPageFactory(object):
79 """ A `PageFactory` (in appearance) that already has a page built.
80 """
81 def __init__(self, page):
82 self._page = page
83
84 @property
85 def rel_path(self):
86 return self._page.rel_path
87
88 @property
89 def metadata(self):
90 return self._page.source_metadata
91
92 @property
93 def ref_spec(self):
94 return self._page.ref_spec
95
96 @property
97 def path(self):
98 return self._page.path
99
100 def buildPage(self):
101 return self._page
102
103
104 class PageRef(object):
105 """ A reference to a page, with support for looking a page in different
106 realms.
107 """
108 def __init__(self, app, page_ref):
109 self.app = app
110 self._page_ref = page_ref
111 self._paths = None
112 self._first_valid_path_index = -2
113 self._exts = list(app.config.get('site/auto_formats').keys())
114
115 @property
116 def exists(self):
117 try:
118 self._checkPaths()
119 return True
120 except PageNotFoundError:
121 return False
122
123 @property
124 def source_name(self):
125 self._checkPaths()
126 return self._paths[self._first_valid_path_index][0]
127
128 @property
129 def source(self):
130 return self.app.getSource(self.source_name)
131
132 @property
133 def rel_path(self):
134 self._checkPaths()
135 return self._paths[self._first_valid_path_index][1]
136
137 @property
138 def path(self):
139 self._checkPaths()
140 return self._paths[self._first_valid_path_index][2]
141
142 @property
143 def possible_rel_paths(self):
144 self._load()
145 return [p[1] for p in self._paths]
146
147 @property
148 def possible_paths(self):
149 self._load()
150 return [p[2] for p in self._paths]
151
152 def _load(self):
153 if self._paths is not None:
154 return
155
156 it = list(page_ref_pattern.finditer(self._page_ref))
157 if len(it) == 0:
158 raise Exception("Invalid page ref: %s" % self._page_ref)
159
160 self._paths = []
161 for m in it:
162 source_name = m.group('src')
163 source = self.app.getSource(source_name)
164 if source is None:
165 raise Exception("No such source: %s" % source_name)
166 rel_path = m.group('path')
167 path = source.resolveRef(rel_path)
168 if '%ext%' in rel_path:
169 for e in self._exts:
170 self._paths.append((source_name,
171 rel_path.replace('%ext%', e),
172 path.replace('%ext%', e)))
173 else:
174 self._paths.append((source_name, rel_path, path))
175
176 def _checkPaths(self):
177 if self._first_valid_path_index >= 0:
178 return
179 if self._first_valid_path_index == -1:
180 raise PageNotFoundError(
181 "No valid paths were found for page reference: %s" %
182 self._page_ref)
183
184 self._load()
185 self._first_valid_path_index = -1
186 for i, path_info in enumerate(self._paths):
187 if os.path.isfile(path_info[2]):
188 self._first_valid_path_index = i
189 break
190 64
191 65
192 class PageSource(object): 66 class PageSource(object):
193 """ A source for pages, e.g. a directory with one file per page. 67 """ A source for pages, e.g. a directory with one file per page.
194 """ 68 """
233 raise NotImplementedError() 107 raise NotImplementedError()
234 108
235 def buildDataProvider(self, page, user_data): 109 def buildDataProvider(self, page, user_data):
236 if self._provider_type is None: 110 if self._provider_type is None:
237 cls = next((pt for pt in self.app.plugin_loader.getDataProviders() 111 cls = next((pt for pt in self.app.plugin_loader.getDataProviders()
238 if pt.PROVIDER_NAME == self.data_type), 112 if pt.PROVIDER_NAME == self.data_type),
239 None) 113 None)
240 if cls is None: 114 if cls is None:
241 raise ConfigurationError("Unknown data provider type: %s" % 115 raise ConfigurationError(
242 self.data_type) 116 "Unknown data provider type: %s" % self.data_type)
243 self._provider_type = cls 117 self._provider_type = cls
244 118
245 return self._provider_type(self, page, user_data) 119 return self._provider_type(self, page, user_data)
246 120
247 def getTaxonomyPageRef(self, tax_name): 121 def getTaxonomyPageRef(self, tax_name):
248 tax_pages = self.config.get('taxonomy_pages') 122 tax_pages = self.config.get('taxonomy_pages')
249 if tax_pages is None: 123 if tax_pages is None:
250 return None 124 return None
251 return tax_pages.get(tax_name) 125 return tax_pages.get(tax_name)
252 126
253
254 class IPreparingSource:
255 def setupPrepareParser(self, parser, app):
256 raise NotImplementedError()
257
258 def buildMetadata(self, args):
259 raise NotImplementedError()
260
261
262 class IListableSource:
263 def listPath(self, rel_path):
264 raise NotImplementedError()
265
266 def getDirpath(self, rel_path):
267 raise NotImplementedError()
268
269 def getBasename(self, rel_path):
270 raise NotImplementedError()
271
272
273 class SimplePaginationSourceMixin(IPaginationSource):
274 def getItemsPerPage(self):
275 return self.config['items_per_page']
276
277 def getSourceIterator(self):
278 return SourceFactoryIterator(self)
279
280 def getSorterIterator(self, it):
281 return DateSortIterator(it)
282
283 def getTailIterator(self, it):
284 return PaginationDataBuilderIterator(it)
285
286 def getPaginationFilter(self, page):
287 conf = (page.config.get('items_filters') or
288 page.app.config.get('site/items_filters'))
289 if conf == 'none' or conf == 'nil' or conf == '':
290 conf = None
291 if conf is not None:
292 f = PaginationFilter()
293 f.addClausesFromConfig(conf)
294 return f
295 return None
296
297 def getSettingAccessor(self):
298 return lambda i, n: i.config.get(n)
299
300
301 class ArraySource(PageSource, SimplePaginationSourceMixin):
302 def __init__(self, app, inner_source, name='array', config=None):
303 super(ArraySource, self).__init__(app, name, config or {})
304 self.inner_source = inner_source
305
306 @property
307 def page_count(self):
308 return len(self.inner_source)
309
310 def getPageFactories(self):
311 for p in self.inner_source:
312 yield CachedPageFactory(p)
313
314
315 class SimplePageSource(PageSource, IListableSource, IPreparingSource,
316 SimplePaginationSourceMixin):
317 def __init__(self, app, name, config):
318 super(SimplePageSource, self).__init__(app, name, config)
319 self.fs_endpoint = config.get('fs_endpoint', name)
320 self.fs_endpoint_path = os.path.join(self.root_dir, self.fs_endpoint)
321 self.supported_extensions = list(app.config.get('site/auto_formats').keys())
322 self.default_auto_format = app.config.get('site/default_auto_format')
323
324 def buildPageFactories(self):
325 logger.debug("Scanning for pages in: %s" % self.fs_endpoint_path)
326 if not os.path.isdir(self.fs_endpoint_path):
327 if self.ignore_missing_dir:
328 return
329 raise InvalidFileSystemEndpointError(self.name, self.fs_endpoint_path)
330
331 for dirpath, dirnames, filenames in os.walk(self.fs_endpoint_path):
332 rel_dirpath = os.path.relpath(dirpath, self.fs_endpoint_path)
333 dirnames[:] = list(filter(self._filterPageDirname, dirnames))
334 for f in filter(self._filterPageFilename, filenames):
335 fac_path = f
336 if rel_dirpath != '.':
337 fac_path = os.path.join(rel_dirpath, f)
338 slug = self._makeSlug(fac_path)
339 metadata = {'slug': slug}
340 fac_path = fac_path.replace('\\', '/')
341 self._populateMetadata(fac_path, metadata)
342 yield PageFactory(self, fac_path, metadata)
343
344 def resolveRef(self, ref_path):
345 return os.path.normpath(
346 os.path.join(self.fs_endpoint_path, ref_path.lstrip("\\/")))
347
348 def findPagePath(self, metadata, mode):
349 uri_path = metadata.get('slug', '')
350 if not uri_path:
351 uri_path = '_index'
352 path = os.path.join(self.fs_endpoint_path, uri_path)
353 _, ext = os.path.splitext(path)
354
355 if mode == MODE_CREATING:
356 if ext == '':
357 path = '%s.%s' % (path, self.default_auto_format)
358 rel_path = os.path.relpath(path, self.fs_endpoint_path)
359 rel_path = rel_path.replace('\\', '/')
360 self._populateMetadata(rel_path, metadata, mode)
361 return rel_path, metadata
362
363 if ext == '':
364 paths_to_check = [
365 '%s.%s' % (path, e)
366 for e in self.supported_extensions]
367 else:
368 paths_to_check = [path]
369 for path in paths_to_check:
370 if os.path.isfile(path):
371 rel_path = os.path.relpath(path, self.fs_endpoint_path)
372 rel_path = rel_path.replace('\\', '/')
373 self._populateMetadata(rel_path, metadata, mode)
374 return rel_path, metadata
375
376 return None, None
377
378 def listPath(self, rel_path):
379 rel_path = rel_path.lstrip('\\/')
380 path = os.path.join(self.fs_endpoint_path, rel_path)
381 names = sorted(os.listdir(path))
382 items = []
383 for name in names:
384 if os.path.isdir(os.path.join(path, name)):
385 if self._filterPageDirname(name):
386 rel_subdir = os.path.join(rel_path, name)
387 items.append((True, name, rel_subdir))
388 else:
389 if self._filterPageFilename(name):
390 slug = self._makeSlug(os.path.join(rel_path, name))
391 metadata = {'slug': slug}
392
393 fac_path = name
394 if rel_path != '.':
395 fac_path = os.path.join(rel_path, name)
396 fac_path = fac_path.replace('\\', '/')
397
398 self._populateMetadata(fac_path, metadata)
399 fac = PageFactory(self, fac_path, metadata)
400
401 name, _ = os.path.splitext(name)
402 items.append((False, name, fac))
403 return items
404
405 def getDirpath(self, rel_path):
406 return os.path.dirname(rel_path)
407
408 def getBasename(self, rel_path):
409 filename = os.path.basename(rel_path)
410 name, _ = os.path.splitext(filename)
411 return name
412
413 def setupPrepareParser(self, parser, app):
414 parser.add_argument('uri', help='The URI for the new page.')
415
416 def buildMetadata(self, args):
417 return {'slug': args.uri}
418
419 def _makeSlug(self, rel_path):
420 slug, ext = os.path.splitext(rel_path)
421 slug = slug.replace('\\', '/')
422 if ext.lstrip('.') not in self.supported_extensions:
423 slug += ext
424 if slug.startswith('./'):
425 slug = slug[2:]
426 if slug == '_index':
427 slug = ''
428 return slug
429
430 def _filterPageDirname(self, d):
431 return not d.endswith('-assets')
432
433 def _filterPageFilename(self, f):
434 return (f[0] != '.' and # .DS_store and other crap
435 f[-1] != '~' and # Vim temp files and what-not
436 f not in ['Thumbs.db']) # Windows bullshit
437
438 def _populateMetadata(self, rel_path, metadata, mode=None):
439 pass
440
441
442 class DefaultPageSource(SimplePageSource):
443 SOURCE_NAME = 'default'
444
445 def __init__(self, app, name, config):
446 super(DefaultPageSource, self).__init__(app, name, config)
447
448
449 class SourceFactoryIterator(object):
450 def __init__(self, source):
451 self.source = source
452 self.it = None # This is to permit recursive traversal of the
453 # iterator chain. It acts as the end.
454
455 def __iter__(self):
456 return self.source.getPages()
457
458
459 class DateSortIterator(object):
460 def __init__(self, it, reverse=True):
461 self.it = it
462 self.reverse = reverse
463
464 def __iter__(self):
465 return iter(sorted(self.it,
466 key=lambda x: x.datetime, reverse=self.reverse))
467
468
469 class PaginationDataBuilderIterator(object):
470 def __init__(self, it):
471 self.it = it
472
473 def __iter__(self):
474 for page in self.it:
475 if page is None:
476 yield None
477 else:
478 yield PaginationData(page)
479