Mercurial > piecrust2

annotate piecrust/sources/blogarchives.py @ 989:8adc27285d93

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Big pass on bake performance. - Reduce the amount of data passed between processes. - Make inter-process data simple objects to make it easier to test with alternatives to pickle. - Make sources have the basic requirement to be able to find a content item from an item spec (path). - Make Hoedown the default Markdown formatter.
author Ludovic Chabant <ludovic@chabant.com>
date Sun, 19 Nov 2017 14:29:17 -0800
parents 1d0364614665
children ba809c221a27
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
857
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
1 import time
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import logging
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import datetime
857
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
4 import collections
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 from piecrust.data.filters import PaginationFilter, IFilterClause
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
6 from piecrust.dataproviders.pageiterator import (
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
7 PageIterator, HardCodedFilterIterator, DateSortIterator)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
8 from piecrust.page import Page
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
9 from piecrust.pipelines._pagebaker import PageBaker
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
10 from piecrust.pipelines._pagerecords import (
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
11 PagePipelineRecordEntry,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
12 add_page_job_result, merge_job_result_into_record_entry)
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
13 from piecrust.pipelines.base import (
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
14 ContentPipeline,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
15 create_job, get_record_name_for_source, content_item_from_job)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 from piecrust.routing import RouteParameter
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
17 from piecrust.sources.base import ContentItem
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
18 from piecrust.sources.generator import GeneratorSourceBase
857
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
19 from piecrust.sources.list import ListSource
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22 logger = logging.getLogger(__name__)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
25 _year_index = """---
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
26 layout: %(template)s
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
27 ---
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
28 """
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
29
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
30
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
31 class BlogArchivesSource(GeneratorSourceBase):
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
32 SOURCE_NAME = 'blog_archives'
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
33 DEFAULT_PIPELINE_NAME = 'blog_archives'
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 def __init__(self, app, name, config):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 super().__init__(app, name, config)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
38 tpl_name = config.get('template', '_year.html')
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
39 self._raw_item = _year_index % {'template': tpl_name}
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
40
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
41 def getSupportedRouteParameters(self):
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
42 return [RouteParameter('year', RouteParameter.TYPE_INT4)]
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
43
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
44 def findContentFromRoute(self, route_params):
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
45 year = route_params['year']
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
46 return ContentItem(
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
47 '_index',
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
48 {'route_params': {'year': year}})
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 def prepareRenderContext(self, ctx):
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
51 ctx.pagination_source = self.inner_source
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
53 route_params = ctx.page.source_metadata['route_params']
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
54 year = route_params.get('year')
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 if year is None:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 raise Exception(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 "Can't find the archive year in the route metadata")
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 if type(year) is not int:
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 raise Exception(
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 "The route for generator '%s' should specify an integer "
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
61 "parameter for 'year'." % self.name)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 flt = PaginationFilter()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 flt.addClause(IsFromYearFilterClause(year))
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 ctx.pagination_filter = flt
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 ctx.custom_data['year'] = year
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 flt2 = PaginationFilter()
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 flt2.addClause(IsFromYearFilterClause(year))
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
71 it = PageIterator(self.inner_source)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
72 it._simpleNonSortedWrap(HardCodedFilterIterator, flt2)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
73 it._wrapAsSort(DateSortIterator, reverse=False)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 ctx.custom_data['archives'] = it
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75
857
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
76 ctx.custom_data['monthly_archives'] = _MonthlyArchiveData(
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
77 self.inner_source, year)
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 class IsFromYearFilterClause(IFilterClause):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 def __init__(self, year):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 self.year = year
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84 def pageMatches(self, fil, page):
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 return (page.datetime.year == self.year)
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87
857
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
88 class _MonthlyArchiveData(collections.abc.Mapping):
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
89 def __init__(self, inner_source, year):
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
90 self._inner_source = inner_source
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
91 self._year = year
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
92 self._months = None
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
93
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
94 def __iter__(self):
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
95 self._load()
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
96 return iter(self._months)
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
97
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
98 def __len__(self):
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
99 self._load()
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
100 return len(self._months)
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
101
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
102 def __getitem__(self, i):
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
103 self._load()
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
104 return self._months[i]
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
105
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
106 def _load(self):
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
107 if self._months is not None:
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
108 return
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
109
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
110 month_index = {}
905
1d0364614665 internal: Sources can cache their pages in addition to their items.
Ludovic Chabant <ludovic@chabant.com>
parents: 877
diff changeset
111 for page in self._inner_source.getAllPages():
857
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
112 if page.datetime.year != self._year:
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
113 continue
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
114
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
115 month = page.datetime.month
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
116
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
117 posts_this_month = month_index.get(month)
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
118 if posts_this_month is None:
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
119 posts_this_month = []
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
120 month_index[month] = posts_this_month
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
121 posts_this_month.append(page.content_item)
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
122
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
123 self._months = []
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
124 for m, ptm in month_index.items():
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
125 timestamp = time.mktime((self._year, m, 1, 0, 0, 0, 0, 0, -1))
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
126
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
127 it = PageIterator(ListSource(self._inner_source, ptm))
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
128 it._wrapAsSort(DateSortIterator, reverse=False)
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
129
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
130 self._months.append({
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
131 'timestamp': timestamp,
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
132 'posts': it
d231a10d18f9 refactor: Make the data providers and blog archives source functional.
Ludovic Chabant <ludovic@chabant.com>
parents: 856
diff changeset
133 })
852
4850f8c21b6e core: Start of the big refactor for PieCrust 3.0.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
134
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
135
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
136 class BlogArchivesPipelineRecordEntry(PagePipelineRecordEntry):
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
137 def __init__(self):
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
138 super().__init__()
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
139 self.year = None
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
140
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
141
854
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
142 class BlogArchivesPipeline(ContentPipeline):
08e02c2a2a1a core: Keep refactoring, this time to prepare for generator sources.
Ludovic Chabant <ludovic@chabant.com>
parents: 853
diff changeset
143 PIPELINE_NAME = 'blog_archives'
877
d6d35b2efd04 bake: Rename "pass" to "step" and make the page pipeline use different steps.
Ludovic Chabant <ludovic@chabant.com>
parents: 876
diff changeset
144 PASS_NUM = 10
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
145 RECORD_ENTRY_CLASS = BlogArchivesPipelineRecordEntry
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
146
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
147 def __init__(self, source, ctx):
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
148 if not isinstance(source, BlogArchivesSource):
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
149 raise Exception("The blog archives pipeline only supports blog "
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
150 "archives content sources.")
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
151
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
152 super().__init__(source, ctx)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
153 self.inner_source = source.inner_source
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
154 self._tpl_name = source.config['template']
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
155 self._all_years = None
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
156 self._dirty_years = None
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
157 self._pagebaker = None
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
158
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
159 def initialize(self):
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
160 self._pagebaker = PageBaker(self.app,
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
161 self.ctx.out_dir,
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
162 force=self.ctx.force)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
163 self._pagebaker.startWriterQueue()
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
164
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
165 def shutdown(self):
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
166 self._pagebaker.stopWriterQueue()
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
167
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
168 def createJobs(self, ctx):
876
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
169 logger.debug("Caching template page for blog archives '%s'." %
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
170 self.inner_source.name)
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
171 page = self.app.getPage(self.source, ContentItem('_index', {}))
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
172 page._load()
d1095774bfcf refactor: Fix some issues with record/cache entry collisions, add counters.
Ludovic Chabant <ludovic@chabant.com>
parents: 871
diff changeset
173
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
174 logger.debug("Building blog archives for: %s" %
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
175 self.inner_source.name)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
176 self._buildDirtyYears(ctx)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
177 logger.debug("Got %d dirty years out of %d." %
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
178 (len(self._dirty_years), len(self._all_years)))
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
179
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
180 jobs = []
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
181 rec_fac = self.createRecordEntry
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
182 current_record = ctx.current_record
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
183
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
184 for y in self._dirty_years:
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
185 record_entry_spec = '_index[%04d]' % y
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
186
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
187 jobs.append(create_job(self, '_index',
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
188 year=y,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
189 record_entry_spec=record_entry_spec))
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
190
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
191 entry = rec_fac(record_entry_spec)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
192 current_record.addEntry(entry)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
193
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
194 if len(jobs) > 0:
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
195 return jobs
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
196 return None
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
197
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
198 def run(self, job, ctx, result):
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
199 year = job['year']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
200 content_item = ContentItem('_index',
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
201 {'year': year,
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
202 'route_params': {'year': year}})
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
203 page = Page(self.source, content_item)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
204
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
205 prev_entry = ctx.previous_entry
989
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
206 rdr_subs = self._pagebaker.bake(page, prev_entry)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
207
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
208 add_page_job_result(result)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
209 result['subs'] = rdr_subs
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
210 result['year'] = page.source_metadata['year']
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
211
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
212 def handleJobResult(self, result, ctx):
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
213 existing = ctx.record_entry
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
214 merge_job_result_into_record_entry(existing, result)
8adc27285d93 bake: Big pass on bake performance.
Ludovic Chabant <ludovic@chabant.com>
parents: 905
diff changeset
215 existing.year = result['year']
856
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
216
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
217 def postJobRun(self, ctx):
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
218 # Create bake entries for the years that were *not* dirty.
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
219 # Otherwise, when checking for deleted pages, we would not find any
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
220 # outputs and would delete those files.
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
221 all_str_years = [str(y) for y in self._all_years]
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
222 for prev, cur in ctx.record_history.diffs:
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
223 if prev and not cur:
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
224 y = prev.year
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
225 if y in all_str_years:
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
226 logger.debug(
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
227 "Creating unbaked entry for year %s archive." % y)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
228 cur.year = y
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
229 cur.out_paths = list(prev.out_paths)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
230 cur.errors = list(prev.errors)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
231 else:
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
232 logger.debug(
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
233 "No page references year %s anymore." % y)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
234
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
235 def _buildDirtyYears(self, ctx):
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
236 all_years = set()
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
237 dirty_years = set()
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
238
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
239 record_name = get_record_name_for_source(self.inner_source)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
240 current_records = ctx.record_histories.current
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
241 cur_rec = current_records.getRecord(record_name)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
242 for cur_entry in cur_rec.getEntries():
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
243 dt = datetime.datetime.fromtimestamp(cur_entry.timestamp)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
244 all_years.add(dt.year)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
245 if cur_entry.was_any_sub_baked:
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
246 dirty_years.add(dt.year)
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
247
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
248 self._all_years = all_years
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
249 self._dirty_years = dirty_years
9bb22bbe093c refactor: Make the blog archives functional again.
Ludovic Chabant <ludovic@chabant.com>
parents: 854
diff changeset
250