Mercurial > piecrust2

comparison piecrust/dataproviders/pageiterator.py @ 854:08e02c2a2a1a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
core: Keep refactoring, this time to prepare for generator sources. - Make a few APIs simpler. - Content pipelines create their own jobs, so that generator sources can keep aborting in `getContents`, but rely on their pipeline to generate pages for baking.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 04 Jun 2017 23:34:28 -0700
parents
children 9bb22bbe093c
comparison
equal deleted inserted replaced
853:f070a4fc033c 854:08e02c2a2a1a
1 import logging
2 from piecrust.data.filters import PaginationFilter
3 from piecrust.data.paginationdata import PaginationData
4 from piecrust.events import Event
5 from piecrust.dataproviders.base import DataProvider
6 from piecrust.sources.base import AbortedSourceUseError
7
8
9 logger = logging.getLogger(__name__)
10
11
12 class _ItInfo:
13 def __init__(self):
14 self.it = None
15 self.iterated = False
16
17
18 class PageIteratorDataProvider(DataProvider):
19 """ A data provider that reads a content source as a list of pages.
20
21 This class supports wrapping another `PageIteratorDataProvider`
22 instance because several sources may want to be merged under the
23 same data endpoint (e.g. `site.pages` which lists both the user
24 pages and the theme pages).
25 """
26 PROVIDER_NAME = 'page_iterator'
27
28 debug_render_doc_dynamic = ['_debugRenderDoc']
29 debug_render_not_empty = True
30
31 def __init__(self, source, page):
32 super().__init__(source, page)
33 self._its = None
34 self._app = source.app
35
36 def __len__(self):
37 self._load()
38 return sum([len(i.it) for i in self._its])
39
40 def __iter__(self):
41 self._load()
42 for i in self._its:
43 yield from i.it
44
45 def _load(self):
46 if self._its is not None:
47 return
48
49 self._its = []
50 for source in self._sources:
51 i = _ItInfo()
52 i.it = PageIterator(source, current_page=self._page)
53 i.it._iter_event += self._onIteration
54 self._its.append(i)
55
56 def _onIteration(self, it):
57 ii = next(filter(lambda i: i.it == it, self._its))
58 if not ii.iterated:
59 rcs = self._app.env.render_ctx_stack
60 rcs.current_ctx.addUsedSource(self._source.name)
61 ii.iterated = True
62
63 def _debugRenderDoc(self):
64 return 'Provides a list of %d items' % len(self)
65
66
67 class PageIterator:
68 def __init__(self, source, *, current_page=None):
69 self._source = source
70 self._cache = None
71 self._pagination_slicer = None
72 self._has_sorter = False
73 self._next_page = None
74 self._prev_page = None
75 self._locked = False
76 self._iter_event = Event()
77 self._current_page = current_page
78 self._it = PageContentSourceIterator(self._source)
79
80 @property
81 def total_count(self):
82 self._load()
83 if self._pagination_slicer is not None:
84 return self._pagination_slicer.inner_count
85 return len(self._cache)
86
87 @property
88 def next_page(self):
89 self._load()
90 return self._next_page
91
92 @property
93 def prev_page(self):
94 self._load()
95 return self._prev_page
96
97 def __len__(self):
98 self._load()
99 return len(self._cache)
100
101 def __getitem__(self, key):
102 self._load()
103 return self._cache[key]
104
105 def __iter__(self):
106 self._load()
107 return iter(self._cache)
108
109 def __getattr__(self, name):
110 if name[:3] == 'is_' or name[:3] == 'in_':
111 def is_filter(value):
112 conf = {'is_%s' % name[3:]: value}
113 return self._simpleNonSortedWrap(SettingFilterIterator, conf)
114 return is_filter
115
116 if name[:4] == 'has_':
117 def has_filter(value):
118 conf = {name: value}
119 return self._simpleNonSortedWrap(SettingFilterIterator, conf)
120 return has_filter
121
122 if name[:5] == 'with_':
123 def has_filter(value):
124 conf = {'has_%s' % name[5:]: value}
125 return self._simpleNonSortedWrap(SettingFilterIterator, conf)
126 return has_filter
127
128 return self.__getattribute__(name)
129
130 def skip(self, count):
131 return self._simpleWrap(SliceIterator, count)
132
133 def limit(self, count):
134 return self._simpleWrap(SliceIterator, 0, count)
135
136 def slice(self, skip, limit):
137 return self._simpleWrap(SliceIterator, skip, limit)
138
139 def filter(self, filter_name):
140 if self._current_page is None:
141 raise Exception("Can't use `filter()` because no parent page was "
142 "set for this page iterator.")
143 filter_conf = self._current_page.config.get(filter_name)
144 if filter_conf is None:
145 raise Exception("Couldn't find filter '%s' in the configuration "
146 "header for page: %s" %
147 (filter_name, self._current_page.path))
148 return self._simpleNonSortedWrap(SettingFilterIterator, filter_conf)
149
150 def sort(self, setting_name, reverse=False):
151 if not setting_name:
152 raise Exception("You need to specify a configuration setting "
153 "to sort by.")
154 self._ensureUnlocked()
155 self._ensureUnloaded()
156 self._pages = SettingSortIterator(self._pages, setting_name, reverse)
157 self._has_sorter = True
158 return self
159
160 def reset(self):
161 self._ensureUnlocked()
162 self._unload()
163 return self
164
165 @property
166 def _is_loaded(self):
167 return self._cache is not None
168
169 @property
170 def _has_more(self):
171 if self._cache is None:
172 return False
173 if self._pagination_slicer:
174 return self._pagination_slicer.has_more
175 return False
176
177 def _simpleWrap(self, it_class, *args, **kwargs):
178 self._ensureUnlocked()
179 self._ensureUnloaded()
180 self._ensureSorter()
181 self._it = it_class(self._it, *args, **kwargs)
182 if self._pagination_slicer is None and it_class is SliceIterator:
183 self._pagination_slicer = self._it
184 self._pagination_slicer.current_page = self._current_page
185 return self
186
187 def _simpleNonSortedWrap(self, it_class, *args, **kwargs):
188 self._ensureUnlocked()
189 self._ensureUnloaded()
190 self._it = it_class(self._it, *args, **kwargs)
191 return self
192
193 def _lockIterator(self):
194 self._ensureUnlocked()
195 self._locked = True
196
197 def _ensureUnlocked(self):
198 if self._locked:
199 raise Exception(
200 "This page iterator has been locked and can't be modified.")
201
202 def _ensureUnloaded(self):
203 if self._cache:
204 raise Exception(
205 "This page iterator has already been iterated upon and "
206 "can't be modified anymore.")
207
208 def _ensureSorter(self):
209 if self._has_sorter:
210 return
211 self._it = DateSortIterator(self._it, reverse=True)
212 self._has_sorter = True
213
214 def _unload(self):
215 self._it = PageContentSourceIterator(self._source)
216 self._cache = None
217 self._paginationSlicer = None
218 self._has_sorter = False
219 self._next_page = None
220 self._prev_page = None
221
222 def _load(self):
223 if self._cache is not None:
224 return
225
226 if self._source.app.env.abort_source_use:
227 if self._current_page is not None:
228 logger.debug("Aborting iteration of '%s' from: %s." %
229 (self._source.name,
230 self._current_page.content_spec))
231 else:
232 logger.debug("Aborting iteration of '%s'." %
233 self._source.name)
234 raise AbortedSourceUseError()
235
236 self._ensureSorter()
237
238 tail_it = PaginationDataBuilderIterator(self._it, self._source.route)
239 self._cache = list(tail_it)
240
241 if (self._current_page is not None and
242 self._pagination_slicer is not None):
243 pn = [self._pagination_slicer.prev_page,
244 self._pagination_slicer.next_page]
245 pn_it = PaginationDataBuilderIterator(iter(pn),
246 self._source.route)
247 self._prev_page, self._next_page = (list(pn_it))
248
249 self._iter_event.fire(self)
250
251 def _debugRenderDoc(self):
252 return "Contains %d items" % len(self)
253
254
255 class SettingFilterIterator:
256 def __init__(self, it, fil_conf):
257 self.it = it
258 self.fil_conf = fil_conf
259 self._fil = None
260
261 def __iter__(self):
262 if self._fil is None:
263 self._fil = PaginationFilter()
264 self._fil.addClausesFromConfig(self.fil_conf)
265
266 for i in self.it:
267 if self._fil.pageMatches(i):
268 yield i
269
270
271 class HardCodedFilterIterator:
272 def __init__(self, it, fil):
273 self.it = it
274 self._fil = fil
275
276 def __iter__(self):
277 for i in self.it:
278 if self._fil.pageMatches(i):
279 yield i
280
281
282 class SliceIterator:
283 def __init__(self, it, offset=0, limit=-1):
284 self.it = it
285 self.offset = offset
286 self.limit = limit
287 self.current_page = None
288 self.has_more = False
289 self.inner_count = -1
290 self.next_page = None
291 self.prev_page = None
292 self._cache = None
293
294 def __iter__(self):
295 if self._cache is None:
296 inner_list = list(self.it)
297 self.inner_count = len(inner_list)
298
299 if self.limit > 0:
300 self.has_more = self.inner_count > (self.offset + self.limit)
301 self._cache = inner_list[self.offset:self.offset + self.limit]
302 else:
303 self.has_more = False
304 self._cache = inner_list[self.offset:]
305
306 if self.current_page:
307 try:
308 idx = inner_list.index(self.current_page)
309 except ValueError:
310 idx = -1
311 if idx >= 0:
312 if idx < self.inner_count - 1:
313 self.next_page = inner_list[idx + 1]
314 if idx > 0:
315 self.prev_page = inner_list[idx - 1]
316
317 return iter(self._cache)
318
319
320 class SettingSortIterator:
321 def __init__(self, it, name, reverse=False):
322 self.it = it
323 self.name = name
324 self.reverse = reverse
325
326 def __iter__(self):
327 return iter(sorted(self.it, key=self._key_getter,
328 reverse=self.reverse))
329
330 def _key_getter(self, item):
331 key = item.config.get(item)
332 if key is None:
333 return 0
334 return key
335
336
337 class DateSortIterator:
338 def __init__(self, it, reverse=True):
339 self.it = it
340 self.reverse = reverse
341
342 def __iter__(self):
343 return iter(sorted(self.it,
344 key=lambda x: x.datetime, reverse=self.reverse))
345
346
347 class PageContentSourceIterator:
348 def __init__(self, source):
349 self.source = source
350
351 # This is to permit recursive traversal of the
352 # iterator chain. It acts as the end.
353 self.it = None
354
355 def __iter__(self):
356 source = self.source
357 app = source.app
358 for item in source.getAllContents():
359 yield app.getPage(source, item)
360
361
362 class PaginationDataBuilderIterator:
363 def __init__(self, it, route):
364 self.it = it
365 self.route = route
366
367 def __iter__(self):
368 for page in self.it:
369 if page is not None:
370 yield PaginationData(page)
371 else:
372 yield None
373