Mercurial > piecrust2

comparison piecrust/baking/worker.py @ 451:838f3964f400

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bake: Optimize the bake by not using custom classes for passing info. See previous changeset about pickling performance between processes. Now just use plain standard structures, or the new `fastpickle` when needed.
author Ludovic Chabant <ludovic@chabant.com>
date Mon, 06 Jul 2015 21:30:49 -0700
parents aefe70229fdd
children 8351a77e13f5
comparison
equal deleted inserted replaced
450:298f8f46432a 451:838f3964f400
47 self.job_handlers = job_handlers 47 self.job_handlers = job_handlers
48 48
49 app.env.stepTimerSince("BakeWorkerInit", self.work_start_time) 49 app.env.stepTimerSince("BakeWorkerInit", self.work_start_time)
50 50
51 def process(self, job): 51 def process(self, job):
52 handler = self.job_handlers[job.job_type] 52 handler = self.job_handlers[job['type']]
53 with self.app.env.timerScope(type(handler).__name__): 53 with self.app.env.timerScope(type(handler).__name__):
54 return handler.handleJob(job) 54 return handler.handleJob(job['job'])
55 55
56 def getReport(self): 56 def getReport(self):
57 self.app.env.stepTimerSince("BakeWorker_%d_Total" % self.wid, 57 self.app.env.stepTimerSince("BakeWorker_%d_Total" % self.wid,
58 self.work_start_time) 58 self.work_start_time)
59 return { 59 return {
60 'type': 'timers', 60 'type': 'timers',
61 'data': self.app.env._timers} 61 'data': self.app.env._timers}
62 62
63 63
64 JOB_LOAD, JOB_RENDER_FIRST, JOB_BAKE = range(0, 3) 64 JOB_LOAD, JOB_RENDER_FIRST, JOB_BAKE = range(0, 3)
65
66
67 class BakeWorkerJob(object):
68 def __init__(self, job_type, payload):
69 self.job_type = job_type
70 self.payload = payload
71 65
72 66
73 class JobHandler(object): 67 class JobHandler(object):
74 def __init__(self, app, ctx): 68 def __init__(self, app, ctx):
75 self.app = app 69 self.app = app
85 errors.append(str(ex)) 79 errors.append(str(ex))
86 ex = ex.__cause__ 80 ex = ex.__cause__
87 return errors 81 return errors
88 82
89 83
90 class PageFactoryInfo(object): 84 def save_factory(fac):
91 def __init__(self, fac): 85 return {
92 self.source_name = fac.source.name 86 'source_name': fac.source.name,
93 self.rel_path = fac.rel_path 87 'rel_path': fac.rel_path,
94 self.metadata = fac.metadata 88 'metadata': fac.metadata}
95
96 def build(self, app):
97 source = app.getSource(self.source_name)
98 return PageFactory(source, self.rel_path, self.metadata)
99 89
100 90
101 class LoadJobPayload(object): 91 def load_factory(app, info):
102 def __init__(self, fac): 92 source = app.getSource(info['source_name'])
103 self.factory_info = PageFactoryInfo(fac) 93 return PageFactory(source, info['rel_path'], info['metadata'])
104
105
106 class LoadJobResult(object):
107 def __init__(self, source_name, path):
108 self.source_name = source_name
109 self.path = path
110 self.config = None
111 self.errors = None
112
113
114 class RenderFirstSubJobPayload(object):
115 def __init__(self, fac):
116 self.factory_info = PageFactoryInfo(fac)
117
118
119 class RenderFirstSubJobResult(object):
120 def __init__(self, path):
121 self.path = path
122 self.errors = None
123
124
125 class BakeJobPayload(object):
126 def __init__(self, fac, route_metadata, previous_entry,
127 dirty_source_names, tax_info=None):
128 self.factory_info = PageFactoryInfo(fac)
129 self.route_metadata = route_metadata
130 self.previous_entry = previous_entry
131 self.dirty_source_names = dirty_source_names
132 self.taxonomy_info = tax_info
133
134
135 class BakeJobResult(object):
136 def __init__(self, path, tax_info=None):
137 self.path = path
138 self.taxonomy_info = tax_info
139 self.sub_entries = None
140 self.errors = None
141 94
142 95
143 class LoadJobHandler(JobHandler): 96 class LoadJobHandler(JobHandler):
144 def handleJob(self, job): 97 def handleJob(self, job):
145 # Just make sure the page has been cached. 98 # Just make sure the page has been cached.
146 fac = job.payload.factory_info.build(self.app) 99 fac = load_factory(self.app, job)
147 logger.debug("Loading page: %s" % fac.ref_spec) 100 logger.debug("Loading page: %s" % fac.ref_spec)
148 result = LoadJobResult(fac.source.name, fac.path) 101 result = {
102 'source_name': fac.source.name,
103 'path': fac.path,
104 'config': None,
105 'errors': None}
149 try: 106 try:
150 page = fac.buildPage() 107 page = fac.buildPage()
151 page._load() 108 page._load()
152 result.config = page.config.getAll() 109 result['config'] = page.config.getAll()
153 except Exception as ex: 110 except Exception as ex:
154 logger.debug("Got loading error. Sending it to master.") 111 logger.debug("Got loading error. Sending it to master.")
155 result.errors = _get_errors(ex) 112 result['errors'] = _get_errors(ex)
156 if self.ctx.debug: 113 if self.ctx.debug:
157 logger.exception(ex) 114 logger.exception(ex)
158 return result 115 return result
159 116
160 117
161 class RenderFirstSubJobHandler(JobHandler): 118 class RenderFirstSubJobHandler(JobHandler):
162 def handleJob(self, job): 119 def handleJob(self, job):
163 # Render the segments for the first sub-page of this page. 120 # Render the segments for the first sub-page of this page.
164 fac = job.payload.factory_info.build(self.app) 121 fac = load_factory(self.app, job)
165 122
166 # These things should be OK as they're checked upstream by the baker. 123 # These things should be OK as they're checked upstream by the baker.
167 route = self.app.getRoute(fac.source.name, fac.metadata, 124 route = self.app.getRoute(fac.source.name, fac.metadata,
168 skip_taxonomies=True) 125 skip_taxonomies=True)
169 assert route is not None 126 assert route is not None
171 page = fac.buildPage() 128 page = fac.buildPage()
172 route_metadata = create_route_metadata(page) 129 route_metadata = create_route_metadata(page)
173 qp = QualifiedPage(page, route, route_metadata) 130 qp = QualifiedPage(page, route, route_metadata)
174 ctx = PageRenderingContext(qp) 131 ctx = PageRenderingContext(qp)
175 132
176 result = RenderFirstSubJobResult(fac.path) 133 result = {
134 'path': fac.path,
135 'errors': None}
177 logger.debug("Preparing page: %s" % fac.ref_spec) 136 logger.debug("Preparing page: %s" % fac.ref_spec)
178 try: 137 try:
179 render_page_segments(ctx) 138 render_page_segments(ctx)
180 except Exception as ex: 139 except Exception as ex:
181 logger.debug("Got rendering error. Sending it to master.") 140 logger.debug("Got rendering error. Sending it to master.")
182 result.errors = _get_errors(ex) 141 result['errors'] = _get_errors(ex)
183 if self.ctx.debug: 142 if self.ctx.debug:
184 logger.exception(ex) 143 logger.exception(ex)
185 return result 144 return result
186 145
187 146
190 super(BakeJobHandler, self).__init__(app, ctx) 149 super(BakeJobHandler, self).__init__(app, ctx)
191 self.page_baker = PageBaker(app, ctx.out_dir, ctx.force) 150 self.page_baker = PageBaker(app, ctx.out_dir, ctx.force)
192 151
193 def handleJob(self, job): 152 def handleJob(self, job):
194 # Actually bake the page and all its sub-pages to the output folder. 153 # Actually bake the page and all its sub-pages to the output folder.
195 fac = job.payload.factory_info.build(self.app) 154 fac = load_factory(self.app, job['factory_info'])
196 155
197 route_metadata = job.payload.route_metadata 156 route_metadata = job['route_metadata']
198 tax_info = job.payload.taxonomy_info 157 tax_info = job['taxonomy_info']
199 if tax_info is not None: 158 if tax_info is not None:
200 route = self.app.getTaxonomyRoute(tax_info.taxonomy_name, 159 route = self.app.getTaxonomyRoute(tax_info.taxonomy_name,
201 tax_info.source_name) 160 tax_info.source_name)
202 else: 161 else:
203 route = self.app.getRoute(fac.source.name, route_metadata, 162 route = self.app.getRoute(fac.source.name, route_metadata,
205 assert route is not None 164 assert route is not None
206 165
207 page = fac.buildPage() 166 page = fac.buildPage()
208 qp = QualifiedPage(page, route, route_metadata) 167 qp = QualifiedPage(page, route, route_metadata)
209 168
210 result = BakeJobResult(fac.path, tax_info) 169 result = {
211 previous_entry = job.payload.previous_entry 170 'path': fac.path,
212 dirty_source_names = job.payload.dirty_source_names 171 'taxonomy_info': tax_info,
172 'sub_entries': None,
173 'errors': None}
174 previous_entry = job['prev_entry']
175 dirty_source_names = job['dirty_source_names']
213 logger.debug("Baking page: %s" % fac.ref_spec) 176 logger.debug("Baking page: %s" % fac.ref_spec)
214 try: 177 try:
215 sub_entries = self.page_baker.bake( 178 sub_entries = self.page_baker.bake(
216 qp, previous_entry, dirty_source_names, tax_info) 179 qp, previous_entry, dirty_source_names, tax_info)
217 result.sub_entries = sub_entries 180 result['sub_entries'] = sub_entries
218 181
219 except BakingError as ex: 182 except BakingError as ex:
220 logger.debug("Got baking error. Sending it to master.") 183 logger.debug("Got baking error. Sending it to master.")
221 result.errors = _get_errors(ex) 184 result['errors'] = _get_errors(ex)
222 if self.ctx.debug: 185 if self.ctx.debug:
223 logger.exception(ex) 186 logger.exception(ex)
224 187
225 return result 188 return result
226 189