Mercurial > piecrust2

comparison piecrust/sources/mixins.py @ 852:4850f8c21b6e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
core: Start of the big refactor for PieCrust 3.0. * Everything is a `ContentSource`, including assets directories. * Most content sources are subclasses of the base file-system source. * A source is processed by a "pipeline", and there are 2 built-in pipelines, one for assets and one for pages. The asset pipeline is vaguely functional, but the page pipeline is completely broken right now. * Rewrite the baking process as just running appropriate pipelines on each content item. This should allow for better parallelization.
author Ludovic Chabant <ludovic@chabant.com>
date Wed, 17 May 2017 00:11:48 -0700
parents ab5c6a8ae90a
children f070a4fc033c
comparison
equal deleted inserted replaced
851:2c7e57d80bba 852:4850f8c21b6e
1 import os
2 import os.path 1 import os.path
3 import logging 2 import logging
4 from piecrust.data.filters import PaginationFilter, page_value_accessor 3 from piecrust import osutil
5 from piecrust.data.paginationdata import PaginationData 4 from piecrust.data.paginationdata import PaginationData
6 from piecrust.sources.base import PageFactory 5 from piecrust.sources.base import ContentItem
7 from piecrust.sources.interfaces import IPaginationSource, IListableSource 6 from piecrust.sources.interfaces import IPaginationSource
8 from piecrust.sources.pageref import PageRef
9 7
10 8
11 logger = logging.getLogger(__name__) 9 logger = logging.getLogger(__name__)
12 10
11 assets_suffix = '-assets'
13 12
14 class SourceFactoryIterator(object): 13
14 class ContentSourceIterator(object):
15 def __init__(self, source): 15 def __init__(self, source):
16 self.source = source 16 self.source = source
17 17
18 # This is to permit recursive traversal of the 18 # This is to permit recursive traversal of the
19 # iterator chain. It acts as the end. 19 # iterator chain. It acts as the end.
20 self.it = None 20 self.it = None
21 21
22 def __iter__(self): 22 def __iter__(self):
23 return self.source.getPages() 23 return self.source.getAllContentItems()
24
25
26 class SourceFactoryWithoutGeneratorsIterator(object):
27 def __init__(self, source):
28 self.source = source
29 self._generator_pages = None
30 # See comment above.
31 self.it = None
32
33 def __iter__(self):
34 self._cacheGeneratorPages()
35 for p in self.source.getPages():
36 if p.rel_path in self._generator_pages:
37 continue
38 yield p
39
40 def _cacheGeneratorPages(self):
41 if self._generator_pages is not None:
42 return
43
44 app = self.source.app
45 self._generator_pages = set()
46 for src in app.sources:
47 for gen in app.generators:
48 for sn, rp in gen.page_ref.possible_split_ref_specs:
49 if sn == self.source.name:
50 self._generator_pages.add(rp)
51 24
52 25
53 class DateSortIterator(object): 26 class DateSortIterator(object):
54 def __init__(self, it, reverse=True): 27 def __init__(self, it, reverse=True):
55 self.it = it 28 self.it = it
64 def __init__(self, it): 37 def __init__(self, it):
65 self.it = it 38 self.it = it
66 39
67 def __iter__(self): 40 def __iter__(self):
68 for page in self.it: 41 for page in self.it:
69 if page is None: 42 if page is not None:
43 yield PaginationData(page)
44 else:
70 yield None 45 yield None
71 else:
72 yield PaginationData(page)
73 46
74 47
75 class SimplePaginationSourceMixin(IPaginationSource): 48 class SimplePaginationSourceMixin(IPaginationSource):
76 """ Implements the `IPaginationSource` interface in a standard way that 49 """ Implements the `IPaginationSource` interface in a standard way that
77 should fit most page sources. 50 should fit most page sources.
78 """ 51 """
79 def getItemsPerPage(self): 52 def getItemsPerPage(self):
80 return self.config['items_per_page'] 53 return self.config['items_per_page']
81 54
82 def getSourceIterator(self): 55 def getSourceIterator(self):
83 if self.config.get('iteration_includes_generator_pages', False): 56 return ContentSourceIterator(self)
84 return SourceFactoryIterator(self)
85 return SourceFactoryWithoutGeneratorsIterator(self)
86 57
87 def getSorterIterator(self, it): 58 def getSorterIterator(self, it):
88 return DateSortIterator(it) 59 return DateSortIterator(it)
89 60
90 def getTailIterator(self, it): 61 def getTailIterator(self, it):
91 return PaginationDataBuilderIterator(it) 62 return PaginationDataBuilderIterator(it)
92 63
93 def getPaginationFilter(self, page):
94 conf = (page.config.get('items_filters') or
95 self.config.get('items_filters'))
96 if conf == 'none' or conf == 'nil' or conf == '':
97 conf = None
98 if conf is not None:
99 f = PaginationFilter(value_accessor=page_value_accessor)
100 f.addClausesFromConfig(conf)
101 return f
102 return None
103 64
104 def getSettingAccessor(self): 65 class SimpleAssetsSubDirMixin:
105 return page_value_accessor 66 def _getRelatedAssetsContents(self, item, relationship):
67 if not item.metadata.get('__has_assets', False):
68 return None
106 69
70 assets = {}
71 assets_dir = item.spec + assets_suffix
72 for f in osutil.listdir(assets_dir):
73 fpath = os.path.join(assets_dir, f)
74 name, _ = os.path.splitext(f)
75 if name in assets:
76 raise Exception("Multiple assets are named '%s'." %
77 name)
78 assets[name] = ContentItem(fpath, {'__is_asset': True})
79 return assets
107 80
108 class SimpleListableSourceMixin(IListableSource): 81 def _onFinalizeContent(self, parent_group, items, groups):
109 """ Implements the `IListableSource` interface for sources that map to 82 assetsGroups = []
110 simple file-system structures. 83 for g in groups:
111 """ 84 if not g.spec.endswith(assets_suffix):
112 def listPath(self, rel_path): 85 continue
113 rel_path = rel_path.lstrip('\\/') 86 match = g.spec[:-len(assets_suffix)]
114 path = self._getFullPath(rel_path) 87 item = next(filter(lambda i: i.spec == match), None)
115 names = self._sortFilenames(os.listdir(path)) 88 if item:
89 item.metadata['__has_assets'] = True
90 assetsGroups.append(g)
91 for g in assetsGroups:
92 groups.remove(g)
116 93
117 items = []
118 for name in names:
119 if os.path.isdir(os.path.join(path, name)):
120 if self._filterPageDirname(name):
121 rel_subdir = os.path.join(rel_path, name)
122 items.append((True, name, rel_subdir))
123 else:
124 if self._filterPageFilename(name):
125 slug = self._makeSlug(os.path.join(rel_path, name))
126 metadata = {'slug': slug}
127
128 fac_path = name
129 if rel_path != '.':
130 fac_path = os.path.join(rel_path, name)
131 fac_path = fac_path.replace('\\', '/')
132
133 self._populateMetadata(fac_path, metadata)
134 fac = PageFactory(self, fac_path, metadata)
135
136 name, _ = os.path.splitext(name)
137 items.append((False, name, fac))
138 return items
139
140 def getDirpath(self, rel_path):
141 return os.path.dirname(rel_path)
142
143 def getBasename(self, rel_path):
144 filename = os.path.basename(rel_path)
145 name, _ = os.path.splitext(filename)
146 return name
147
148 def _getFullPath(self, rel_path):
149 return os.path.join(self.fs_endpoint_path, rel_path)
150
151 def _sortFilenames(self, names):
152 return sorted(names)
153
154 def _filterPageDirname(self, name):
155 return True
156
157 def _filterPageFilename(self, name):
158 return True
159
160 def _makeSlug(self, rel_path):
161 return rel_path.replace('\\', '/')
162
163 def _populateMetadata(self, rel_path, metadata, mode=None):
164 pass
165