Mercurial > piecrust2
annotate piecrust/plugins/base.py @ 411:e7b865f8f335
bake: Enable multiprocess baking.
Baking is now done by running a worker per CPU, and sending jobs to them.
This changes several things across the codebase:
* Ability to not cache things related to pages other than the 'main' page
(i.e. the page at the bottom of the execution stack).
* Decouple the baking process from the bake records, so only the main process
keeps track (and modifies) the bake record.
* Remove the need for 'batch page getters' and loading a page directly from
the page factories.
There are various smaller changes too included here, including support for
scope performance timers that are saved with the bake record and can be
printed out to the console. Yes I got carried away.
For testing, the in-memory 'mock' file-system doesn't work anymore, since
we're spawning processes, so this is replaced by a 'tmpfs' file-system which
is saved in temporary files on disk and deleted after tests have run.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Fri, 12 Jun 2015 17:09:19 -0700 |
| parents | 10bb8e8600f5 |
| children | eacf0a3afd0c |
| rev | line source |
|---|---|
|
303
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
1 import logging |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
2 import importlib |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
3 |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
4 |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
5 logger = logging.getLogger(__name__) |
| 0 | 6 |
| 7 | |
| 8 class PieCrustPlugin(object): | |
| 9 def getFormatters(self): | |
| 10 return [] | |
| 11 | |
| 12 def getTemplateEngines(self): | |
| 13 return [] | |
| 14 | |
| 15 def getDataProviders(self): | |
| 16 return [] | |
| 17 | |
| 18 def getProcessors(self): | |
| 19 return [] | |
| 20 | |
| 21 def getImporters(self): | |
| 22 return [] | |
| 23 | |
| 24 def getCommands(self): | |
| 25 return [] | |
| 26 | |
|
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
27 def getCommandExtensions(self): |
|
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
28 return [] |
|
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
29 |
| 0 | 30 def getBakerAssistants(self): |
| 31 return [] | |
| 32 | |
|
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
33 def getSources(self): |
|
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
34 return [] |
|
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
35 |
| 0 | 36 def initialize(self, app): |
| 37 pass | |
| 38 | |
| 39 | |
| 40 class PluginLoader(object): | |
| 41 def __init__(self, app): | |
| 42 self.app = app | |
| 43 self._plugins = None | |
| 44 self._componentCache = {} | |
| 45 | |
| 46 @property | |
| 47 def plugins(self): | |
| 48 self._ensureLoaded() | |
| 49 return self._plugins | |
| 50 | |
| 51 def getFormatters(self): | |
|
312
10bb8e8600f5
plugins: Remove unused API endpoints.
Ludovic Chabant <ludovic@chabant.com>
parents:
306
diff
changeset
|
52 return self._getPluginComponents( |
|
10bb8e8600f5
plugins: Remove unused API endpoints.
Ludovic Chabant <ludovic@chabant.com>
parents:
306
diff
changeset
|
53 'getFormatters', True, order_key=lambda f: f.priority) |
| 0 | 54 |
| 55 def getTemplateEngines(self): | |
| 56 return self._getPluginComponents('getTemplateEngines', True) | |
| 57 | |
| 58 def getDataProviders(self): | |
| 59 return self._getPluginComponents('getDataProviders') | |
| 60 | |
| 61 def getProcessors(self): | |
|
312
10bb8e8600f5
plugins: Remove unused API endpoints.
Ludovic Chabant <ludovic@chabant.com>
parents:
306
diff
changeset
|
62 return self._getPluginComponents( |
|
10bb8e8600f5
plugins: Remove unused API endpoints.
Ludovic Chabant <ludovic@chabant.com>
parents:
306
diff
changeset
|
63 'getProcessors', True, order_key=lambda p: p.priority) |
| 0 | 64 |
| 65 def getImporters(self): | |
| 66 return self._getPluginComponents('getImporters') | |
| 67 | |
| 68 def getCommands(self): | |
| 69 return self._getPluginComponents('getCommands') | |
| 70 | |
|
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
71 def getCommandExtensions(self): |
|
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
72 return self._getPluginComponents('getCommandExtensions') |
|
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
73 |
| 0 | 74 def getBakerAssistants(self): |
| 75 return self._getPluginComponents('getBakerAssistants') | |
| 76 | |
|
3
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
77 def getSources(self): |
|
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
78 return self._getPluginComponents('getSources') |
|
f485ba500df3
Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
1
diff
changeset
|
79 |
| 0 | 80 def _ensureLoaded(self): |
| 81 if self._plugins is not None: | |
| 82 return | |
| 83 | |
|
1
aaa8fb7c8918
Re-arranged modules to reduce dependencies to builtin stuff.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
84 from piecrust.plugins.builtin import BuiltInPlugin |
| 0 | 85 self._plugins = [BuiltInPlugin()] |
| 86 | |
|
306
7122976bc751
plugins: Fix crash for sites that don't specify a `site/plugins` setting.
Ludovic Chabant <ludovic@chabant.com>
parents:
305
diff
changeset
|
87 to_install = self.app.config.get('site/plugins') |
|
7122976bc751
plugins: Fix crash for sites that don't specify a `site/plugins` setting.
Ludovic Chabant <ludovic@chabant.com>
parents:
305
diff
changeset
|
88 if to_install: |
|
7122976bc751
plugins: Fix crash for sites that don't specify a `site/plugins` setting.
Ludovic Chabant <ludovic@chabant.com>
parents:
305
diff
changeset
|
89 for p in to_install: |
|
7122976bc751
plugins: Fix crash for sites that don't specify a `site/plugins` setting.
Ludovic Chabant <ludovic@chabant.com>
parents:
305
diff
changeset
|
90 self._loadPlugin(p) |
| 0 | 91 |
| 92 for plugin in self._plugins: | |
| 93 plugin.initialize(self.app) | |
| 94 | |
|
303
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
95 def _loadPlugin(self, plugin_name): |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
96 try: |
|
305
9ae23409d6e9
plugins: Change how plugins are loaded. Add a `plugins` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
303
diff
changeset
|
97 mod = importlib.import_module('piecrust_' + plugin_name) |
|
9ae23409d6e9
plugins: Change how plugins are loaded. Add a `plugins` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
303
diff
changeset
|
98 except ImportError as ex: |
|
9ae23409d6e9
plugins: Change how plugins are loaded. Add a `plugins` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
303
diff
changeset
|
99 logger.error("Failed to load plugin '%s'." % plugin_name) |
|
9ae23409d6e9
plugins: Change how plugins are loaded. Add a `plugins` command.
Ludovic Chabant <ludovic@chabant.com>
parents:
303
diff
changeset
|
100 logger.error(ex) |
|
303
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
101 return |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
102 |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
103 plugin_class = getattr(mod, '__piecrust_plugin__', None) |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
104 if plugin_class is None: |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
105 logger.error("Plugin '%s' doesn't specify any " |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
106 "`__piecrust_plugin__` class." % plugin_name) |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
107 return |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
108 |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
109 try: |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
110 plugin = plugin_class() |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
111 except Exception as ex: |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
112 logger.error("Failed to create plugin '%s': %s" % |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
113 (plugin_name, ex)) |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
114 return |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
115 |
|
5dbab01daaba
plugins: First pass for a working plugin loader functionality.
Ludovic Chabant <ludovic@chabant.com>
parents:
7
diff
changeset
|
116 self._plugins.append(plugin) |
| 0 | 117 |
|
7
343d08ef5668
More PieCrust 3 fixes, and a couple of miscellaneous bug fixes.
Ludovic Chabant <ludovic@chabant.com>
parents:
3
diff
changeset
|
118 def _getPluginComponents(self, name, initialize=False, order_key=None): |
| 0 | 119 if name in self._componentCache: |
| 120 return self._componentCache[name] | |
| 121 | |
| 122 all_components = [] | |
| 123 for plugin in self.plugins: | |
| 124 plugin_components = getattr(plugin, name)() | |
| 125 all_components += plugin_components | |
| 126 if initialize: | |
| 127 for comp in plugin_components: | |
| 128 comp.initialize(self.app) | |
| 129 | |
|
7
343d08ef5668
More PieCrust 3 fixes, and a couple of miscellaneous bug fixes.
Ludovic Chabant <ludovic@chabant.com>
parents:
3
diff
changeset
|
130 if order_key is not None: |
|
343d08ef5668
More PieCrust 3 fixes, and a couple of miscellaneous bug fixes.
Ludovic Chabant <ludovic@chabant.com>
parents:
3
diff
changeset
|
131 all_components.sort(key=order_key) |
| 0 | 132 |
| 133 self._componentCache[name] = all_components | |
| 134 return all_components | |
| 135 |