Mercurial > silorider
changeset 66:4caf6720d1dd
Only process silos that did not throw an exception in pre-process.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Tue, 26 Dec 2023 16:27:28 -0800 |
| parents | 412ff72ba091 |
| children | c5bf03406a33 |
| files | silorider/commands/process.py |
| diffstat | 1 files changed, 16 insertions(+), 9 deletions(-) [+] |
line wrap: on
line diff
--- a/silorider/commands/process.py Tue Dec 26 16:26:35 2023 -0800 +++ b/silorider/commands/process.py Tue Dec 26 16:27:28 2023 -0800 @@ -31,11 +31,11 @@ return self._silos def process(self): - self.preProcess() + ok_silos = self.preProcess() # Get all silos to return a profile URL handler. profile_url_handlers = {} - for silo in self.ctx.silos: + for silo in ok_silos: handler = silo.getProfileUrlHandler() if handler: profile_url_handlers[silo.SILO_TYPE] = handler @@ -43,9 +43,9 @@ postctx = SiloPostingContext(self.ctx, profile_url_handlers) feed = parse_url(self.url, self.name, self.config) for entry in feed.entries: - self.processEntry(postctx, entry) + self.processEntry(ok_silos, postctx, entry) - self.postProcess() + self.postProcess(ok_silos) def preProcess(self): # Pre-parse the "since" and "until" dates/times. @@ -56,14 +56,21 @@ # Go over the silos needed for this command (i.e. potentially # filtered by passing `-s`) and call their `onPostStart`. + ok_silos = [] for silo in self.silos: - silo.onPostStart(self.ctx) + try: + silo.onPostStart(self.ctx) + ok_silos.append(silo) + except Exception as ex: + logger.error("Error during pre-process of silo '%s'" % silo.name) + logger.error(ex) + return ok_silos - def postProcess(self): - for silo in self.silos: + def postProcess(self, silos): + for silo in silos: silo.onPostEnd(self.ctx) - def processEntry(self, postctx, entry): + def processEntry(self, silos, postctx, entry): entry_url = entry.get('url') if not entry_url: logger.warning("Found entry without a URL: %s" % repr(entry._mf_entry)) @@ -78,7 +85,7 @@ only_until = self.ctx.args.until logger.debug("Processing entry: %s" % entry_url) - for silo in self.silos: + for silo in silos: if only_since or only_until: entry_dt = entry.get('published') if not entry_dt: