Mercurial > piecrust2
comparison piecrust/data/linker.py @ 411:e7b865f8f335
bake: Enable multiprocess baking.
Baking is now done by running a worker per CPU, and sending jobs to them.
This changes several things across the codebase:
* Ability to not cache things related to pages other than the 'main' page
(i.e. the page at the bottom of the execution stack).
* Decouple the baking process from the bake records, so only the main process
keeps track (and modifies) the bake record.
* Remove the need for 'batch page getters' and loading a page directly from
the page factories.
There are various smaller changes too included here, including support for
scope performance timers that are saved with the bake record and can be
printed out to the console. Yes I got carried away.
For testing, the in-memory 'mock' file-system doesn't work anymore, since
we're spawning processes, so this is replaced by a 'tmpfs' file-system which
is saved in temporary files on disk and deleted after tests have run.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Fri, 12 Jun 2015 17:09:19 -0700 |
| parents | fd8e39254da0 |
| children | 32c7c2d219d2 |
comparison
equal
deleted
inserted
replaced
| 410:d1a472464e57 | 411:e7b865f8f335 |
|---|---|
| 291 if not is_listable: | 291 if not is_listable: |
| 292 raise Exception("Source '%s' can't be listed." % self._source.name) | 292 raise Exception("Source '%s' can't be listed." % self._source.name) |
| 293 | 293 |
| 294 items = list(self._source.listPath(self._dir_path)) | 294 items = list(self._source.listPath(self._dir_path)) |
| 295 self._items = collections.OrderedDict() | 295 self._items = collections.OrderedDict() |
| 296 with self._source.app.env.page_repository.startBatchGet(): | 296 for is_dir, name, data in items: |
| 297 for is_dir, name, data in items: | 297 # If `is_dir` is true, `data` will be the directory's source |
| 298 # If `is_dir` is true, `data` will be the directory's source | 298 # path. If not, it will be a page factory. |
| 299 # path. If not, it will be a page factory. | 299 if is_dir: |
| 300 if is_dir: | 300 item = Linker(self._source, data, |
| 301 item = Linker(self._source, data, | 301 root_page_path=self._root_page_path) |
| 302 root_page_path=self._root_page_path) | 302 else: |
| 303 else: | 303 page = data.buildPage() |
| 304 page = data.buildPage() | 304 is_self = (page.rel_path == self._root_page_path) |
| 305 is_self = (page.rel_path == self._root_page_path) | 305 item = _LinkedPage(page) |
| 306 item = _LinkedPage(page) | 306 item._linker_info.name = name |
| 307 item._linker_info.name = name | 307 item._linker_info.is_self = is_self |
| 308 item._linker_info.is_self = is_self | 308 if is_self: |
| 309 if is_self: | 309 self._self_item = item |
| 310 self._self_item = item | 310 |
| 311 | 311 existing = self._items.get(name) |
| 312 existing = self._items.get(name) | 312 if existing is None: |
| 313 if existing is None: | 313 self._items[name] = item |
| 314 self._items[name] = item | 314 elif is_dir: |
| 315 elif is_dir: | 315 # The current item is a directory. The existing item |
| 316 # The current item is a directory. The existing item | 316 # should be a page. |
| 317 # should be a page. | 317 existing._linker_info.child_linker = item |
| 318 existing._linker_info.child_linker = item | 318 existing._linker_info.is_dir = True |
| 319 existing._linker_info.is_dir = True | 319 else: |
| 320 else: | 320 # The current item is a page. The existing item should |
| 321 # The current item is a page. The existing item should | 321 # be a directory. |
| 322 # be a directory. | 322 item._linker_info.child_linker = existing |
| 323 item._linker_info.child_linker = existing | 323 item._linker_info.is_dir = True |
| 324 item._linker_info.is_dir = True | 324 self._items[name] = item |
| 325 self._items[name] = item | |
| 326 | 325 |
| 327 | 326 |
| 328 def filter_page_items(item): | 327 def filter_page_items(item): |
| 329 return not isinstance(item, Linker) | 328 return not isinstance(item, Linker) |
| 330 | 329 |