Mercurial > piecrust2

annotate piecrust/records.py @ 584:9ccc933ac2c7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
internal: Refactor the app configuration class. * Moved to its own module. * More extensible validation. * Allow easier setup of defaults so `showconfig` shows more useful stuff.
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 01 Jan 2016 23:18:26 -0800
parents 938be93215cb
children 61d606fbc313
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
91
e88e330eb8dc Improvements to incremental baking and cache invalidating.
Ludovic Chabant <ludovic@chabant.com>
parents: 7
diff changeset
3 import pickle
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import logging
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
5 from piecrust import APP_VERSION
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6 from piecrust.events import Event
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 logger = logging.getLogger(__name__)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 class Record(object):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 def __init__(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 self.entries = []
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 self.entry_added = Event()
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
16 self.app_version = APP_VERSION
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
17 self.record_version = self.__class__.RECORD_VERSION
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
18
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
19 def hasLatestVersion(self):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
20 return (self.app_version == APP_VERSION and
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
21 self.record_version == self.__class__.RECORD_VERSION)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 def addEntry(self, entry):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24 self.entries.append(entry)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 self.entry_added.fire(entry)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 def save(self, path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
28 path_dir = os.path.dirname(path)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
29 if not os.path.isdir(path_dir):
5
474c9882decf Upgrade to Python 3.
Ludovic Chabant <ludovic@chabant.com>
parents: 3
diff changeset
30 os.makedirs(path_dir, 0o755)
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31
7
343d08ef5668 More PieCrust 3 fixes, and a couple of miscellaneous bug fixes.
Ludovic Chabant <ludovic@chabant.com>
parents: 5
diff changeset
32 with open(path, 'wb') as fp:
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 def __getstate__(self):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 odict = self.__dict__.copy()
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
37 del odict['entry_added']
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 return odict
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39
91
e88e330eb8dc Improvements to incremental baking and cache invalidating.
Ludovic Chabant <ludovic@chabant.com>
parents: 7
diff changeset
40 def __setstate__(self, state):
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
41 state['entry_added'] = Event()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
42 self.__dict__.update(state)
91
e88e330eb8dc Improvements to incremental baking and cache invalidating.
Ludovic Chabant <ludovic@chabant.com>
parents: 7
diff changeset
43
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 @staticmethod
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 def load(path):
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 logger.debug("Loading bake record from: %s" % path)
7
343d08ef5668 More PieCrust 3 fixes, and a couple of miscellaneous bug fixes.
Ludovic Chabant <ludovic@chabant.com>
parents: 5
diff changeset
47 with open(path, 'rb') as fp:
3
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 return pickle.load(fp)
f485ba500df3 Gigantic change to basically make PieCrust 2 vaguely functional.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
50
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
51 class TransitionalRecord(object):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
52 def __init__(self, record_class, previous_path=None):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
53 self._record_class = record_class
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
54 self.transitions = {}
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
55 self.incremental_count = 0
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
56 self.current = record_class()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
57 if previous_path:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
58 self.loadPrevious(previous_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
59 else:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
60 self.previous = record_class()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
61 self.current.entry_added += self._onCurrentEntryAdded
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
62
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
63 def loadPrevious(self, previous_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
64 previous_record_valid = True
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
65 try:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
66 self.previous = self._record_class.load(previous_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
67 except Exception as ex:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
68 logger.debug("Error loading previous record: %s" % ex)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
69 logger.debug("Will reset to an empty one.")
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
70 previous_record_valid = False
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
71
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
72 if self.previous.record_version != self._record_class.RECORD_VERSION:
166
c132a2cba521 bake: Don't crash stupidly when there was no previous version.
Ludovic Chabant <ludovic@chabant.com>
parents: 133
diff changeset
73 logger.debug(
c132a2cba521 bake: Don't crash stupidly when there was no previous version.
Ludovic Chabant <ludovic@chabant.com>
parents: 133
diff changeset
74 "Previous record has old version %s." %
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
75 self.previous.record_version)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
76 logger.debug("Will reset to an empty one.")
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
77 previous_record_valid = False
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
78
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
79 if not previous_record_valid:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
80 self.previous = self._record_class()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
81 return
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
82
133
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
83 self._rebuildTransitions()
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
84
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
85 def setPrevious(self, previous_record):
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
86 self.previous = previous_record
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
87 self._rebuildTransitions()
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
88
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
89 def clearPrevious(self):
133
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
90 self.setPrevious(self._record_class())
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
91
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
92 def saveCurrent(self, current_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
93 self.current.save(current_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
94
133
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
95 def detach(self):
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
96 res = self.current
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
97 self.current.entry_added -= self._onCurrentEntryAdded
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
98 self.current = None
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
99 self.previous = None
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
100 self.transitions = {}
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
101 return res
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
102
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
103 def addEntry(self, entry):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
104 self.current.addEntry(entry)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
105
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
106 def getTransitionKey(self, entry):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
107 raise NotImplementedError()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
108
133
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
109 def _rebuildTransitions(self):
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
110 self.transitions = {}
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
111 for e in self.previous.entries:
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
112 key = self.getTransitionKey(e)
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
113 self.transitions[key] = (e, None)
9e4c2e68a129 Optimize server for files that already exist.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
114
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
115 def _onCurrentEntryAdded(self, entry):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
116 key = self.getTransitionKey(entry)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
117 te = self.transitions.get(key)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
118 if te is None:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
119 logger.debug("Adding new record entry: %s" % key)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
120 self.transitions[key] = (None, entry)
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 166
diff changeset
121 self._onNewEntryAdded(entry)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
122 return
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
123
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
124 if te[1] is not None:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
125 raise Exception("A current entry already exists for: %s" %
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
126 key)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
127 logger.debug("Setting current record entry: %s" % key)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
128 self.transitions[key] = (te[0], entry)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 91
diff changeset
129
338
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 166
diff changeset
130 def _onNewEntryAdded(self, entry):
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 166
diff changeset
131 pass
938be93215cb bake: Improve render context and bake record, fix incremental bake bugs.
Ludovic Chabant <ludovic@chabant.com>
parents: 166
diff changeset
132