piecrust2: piecrust/dataproviders/pageiterator.py comparison

comparison piecrust/dataproviders/pageiterator.py @ 979:45ad976712ec

tests: Big push to get the tests to pass again. - Lots of fixes everywhere in the code. - Try to handle debug logging in the multiprocessing worker pool when running in pytest. Not perfect, but usable for now. - Replace all `.md` test files with `.html` since now a auto-format extension always sets the format. - Replace `out` with `outfiles` in most places since now blog archives are added to the bake output and I don't want to add expected outputs for blog archives everywhere.

author	Ludovic Chabant <ludovic@chabant.com>
date	Sun, 29 Oct 2017 22:51:57 -0700
parents	abc52a6262a1
children	492b66482f12

comparison

equal deleted inserted replaced

-:7e51d14097cb
+:45ad976712ec
 logger = logging.getLogger(__name__)
-class _ItInfo:
+class _CombinedSource:
-def __init__(self):
+def __init__(self, sources):
+self.sources = sources
+self.app = sources[0].app
+self.name = None
+# This is for recursive traversal of the iterator chain.
+# See later in `PageIterator`.
 self.it = None
-self.iterated = False
-self.source_name = None
+def __iter__(self):
+sources = self.sources
+if len(sources) == 1:
+source = sources[0]
+self.name = source.name
+yield from source.getAllPages()
+self.name = None
+return
+# Return the pages from all the combined sources, but skip
+# those that are "overridden" -- e.g. a theme page that gets
+# replaced by a user page of the same name.
+used_uris = set()
+for source in sources:
+self.name = source.name
+for page in source.getAllPages():
+page_uri = page.getUri()
+if page_uri not in used_uris:
+used_uris.add(page_uri)
+yield page
+self.name = None
 class PageIteratorDataProvider(DataProvider):
 """ A data provider that reads a content source as a list of pages.
 debug_render_doc_dynamic = ['_debugRenderDoc']
 debug_render_not_empty = True
 def __init__(self, source, page):
 super().__init__(source, page)
-self._its = None
 self._app = source.app
+self._it = None
+self._iterated = False
 def __len__(self):
 self._load()
-return sum([len(i.it) for i in self._its])
+return len(self._it)
 def __iter__(self):
 self._load()
-for i in self._its:
+yield from self._it
-yield from i.it
 def _load(self):
-if self._its is not None:
+if self._it is not None:
 return
-self._its = []
+combined_source = _CombinedSource(list(reversed(self._sources)))
-for source in self._sources:
+self._it = PageIterator(combined_source, current_page=self._page)
-i = _ItInfo()
+self._it._iter_event += self._onIteration
-i.it = PageIterator(source, current_page=self._page)
-i.it._iter_event += self._onIteration
-i.source_name = source.name
-self._its.append(i)
 def _onIteration(self, it):
-ii = next(filter(lambda i: i.it == it, self._its))
+if not self._iterated:
-if not ii.iterated:
 rcs = self._app.env.render_ctx_stack
-rcs.current_ctx.addUsedSource(ii.source_name)
+rcs.current_ctx.addUsedSource(it._source)
-ii.iterated = True
+self._iterated = True
+def _addSource(self, source):
+if self._it is not None:
+raise Exception("Can't add sources after the data provider "
+"has been loaded.")
+super()._addSource(source)
 def _debugRenderDoc(self):
 return 'Provides a list of %d items' % len(self)
 class PageIterator:
 def __init__(self, source, *, current_page=None):
 self._source = source
-self._is_content_source = isinstance(source, ContentSource)
+self._is_content_source = isinstance(
+source, (ContentSource, _CombinedSource))
 self._cache = None
 self._pagination_slicer = None
 self._has_sorter = False
 self._next_page = None
 self._prev_page = None
 raise Exception("Couldn't find filter '%s' in the configuration "
 "header for page: %s" %
 (filter_name, self._current_page.path))
 return self._simpleNonSortedWrap(SettingFilterIterator, filter_conf)
-def sort(self, setting_name, reverse=False):
+def sort(self, setting_name=None, reverse=False):
-if not setting_name:
+if setting_name:
-raise Exception("You need to specify a configuration setting "
+self._wrapAsSort(SettingSortIterator, setting_name, reverse)
-"to sort by.")
+else:
-self._ensureUnlocked()
+self._wrapAsSort(NaturalSortIterator, reverse)
-self._ensureUnloaded()
-self._pages = SettingSortIterator(self._pages, setting_name, reverse)
-self._has_sorter = True
 return self
 def reset(self):
 self._ensureUnlocked()
 self._unload()
 def _is_loaded(self):
 return self._cache is not None
 @property
 def _has_more(self):
-if self._cache is None:
+self._load()
-return False
 if self._pagination_slicer:
 return self._pagination_slicer.has_more
 return False
+@property
+def _is_loaded_and_has_more(self):
+return self._is_loaded and self._has_more
 def _simpleWrap(self, it_class, *args, **kwargs):
 self._ensureUnlocked()
 self._ensureUnloaded()
 self._ensureSorter()
 self._it = DateSortIterator(self._it, reverse=True)
 self._has_sorter = True
 def _initIterator(self):
 if self._is_content_source:
-self._it = PageContentSourceIterator(self._source)
+if isinstance(self._source, _CombinedSource):
+self._it = self._source
+else:
+self._it = PageContentSourceIterator(self._source)
 app = self._source.app
 if app.config.get('baker/is_baking'):
 # While baking, automatically exclude any page with
 # the `draft` setting.
 draft_setting = app.config['baker/no_bake_setting']
 self.prev_page = inner_list[idx - 1]
 return iter(self._cache)
+class NaturalSortIterator:
+def __init__(self, it, reverse=False):
+self.it = it
+self.reverse = reverse
+def __iter__(self):
+return iter(sorted(self.it, reverse=self.reverse))
 class SettingSortIterator:
 def __init__(self, it, name, reverse=False):
 self.it = it
 self.name = name
 self.reverse = reverse
 def __iter__(self):
 return iter(sorted(self.it, key=self._key_getter,
 reverse=self.reverse))
 def _key_getter(self, item):
-key = item.config.get(item)
+key = item.config.get(self.name)
 if key is None:
 return 0
 return key

Mercurial > piecrust2

comparison piecrust/dataproviders/pageiterator.py @ 979:45ad976712ec