piecrust2: piecrust/processing/base.py comparison

comparison piecrust/processing/base.py @ 129:3080b6d02f40

Add ability for the processing pipeline to only process new assets.

author	Ludovic Chabant <ludovic@chabant.com>
date	Sat, 15 Nov 2014 16:03:24 +0100
parents	e5cba2622d26
children	9e4c2e68a129

comparison

equal deleted inserted replaced

-:28444014ce7d
+:3080b6d02f40
 def filterProcessors(self, authorized_names):
 self.processors = list(filter(
 lambda p: p.PROCESSOR_NAME in authorized_names,
 self.processors))
-def run(self, src_dir_or_file=None):
+def run(self, src_dir_or_file=None, new_only=False):
 # Invoke pre-processors.
 for proc in self.processors:
 proc.onPipelineStart(self)
 # Sort our processors again in case the pre-process step involved
 (src_dir_or_file, self.mounts))
 ctx = ProcessingContext(base_dir, queue, record)
 logger.debug("Initiating processing pipeline on: %s" % src_dir_or_file)
 if os.path.isdir(src_dir_or_file):
-self.processDirectory(ctx, src_dir_or_file)
+self.processDirectory(ctx, src_dir_or_file, new_only)
 elif os.path.isfile(src_dir_or_file):
-self.processFile(ctx, src_dir_or_file)
+self.processFile(ctx, src_dir_or_file, new_only)
 else:
 # Process everything.
 for path in self.mounts:
 ctx = ProcessingContext(path, queue, record)
 logger.debug("Initiating processing pipeline on: %s" % path)
-self.processDirectory(ctx, path)
+self.processDirectory(ctx, path, new_only)
 # Wait on all workers.
 for w in pool:
 w.join()
 if abort.is_set():
 raise Exception("Worker pool was aborted.")
 # Handle deletions.
-for path, reason in record.getDeletions():
+if not new_only:
-logger.debug("Removing '%s': %s" % (path, reason))
+for path, reason in record.getDeletions():
-os.remove(path)
+logger.debug("Removing '%s': %s" % (path, reason))
-logger.info('[delete] %s' % path)
+os.remove(path)
+logger.info('[delete] %s' % path)
 # Invoke post-processors.
 for proc in self.processors:
 proc.onPipelineEnd(self)
 record.saveCurrent(record_cache.getCachePath(record_name))
 logger.debug(format_timed(t, 'saved bake record', colored=False))
 return record
-def processDirectory(self, ctx, start_dir):
+def processDirectory(self, ctx, start_dir, new_only=False):
 for dirpath, dirnames, filenames in os.walk(start_dir):
 rel_dirpath = os.path.relpath(dirpath, start_dir)
 dirnames[:] = [d for d in dirnames
 if not re_matchany(d, self.skip_patterns, rel_dirpath)]
 for filename in filenames:
 if re_matchany(filename, self.skip_patterns, rel_dirpath):
 continue
-self.processFile(ctx, os.path.join(dirpath, filename))
+self.processFile(ctx, os.path.join(dirpath, filename),
+new_only)
-def processFile(self, ctx, path):
+def processFile(self, ctx, path, new_only=False):
 logger.debug("Queuing: %s" % path)
-job = ProcessingWorkerJob(ctx.base_dir, path)
+job = ProcessingWorkerJob(ctx.base_dir, path, new_only)
 ctx.job_queue.put_nowait(job)
 class ProcessingWorkerContext(object):
 def __init__(self, pipeline, record,
 self.abort_event = abort_event
 self.pipeline_lock = pipeline_lock
 class ProcessingWorkerJob(object):
-def __init__(self, base_dir, path):
+def __init__(self, base_dir, path, new_only=False):
 self.base_dir = base_dir
 self.path = path
+self.new_only = new_only
 class ProcessingWorker(threading.Thread):
 def __init__(self, wid, ctx):
 super(ProcessingWorker, self).__init__()
 pipeline = self.ctx.pipeline
 record = self.ctx.record
 rel_path = os.path.relpath(job.path, job.base_dir)
 previous_entry = record.getPreviousEntry(rel_path)
+if job.new_only and previous_entry:
+return
 record_entry = ProcessorPipelineRecordEntry(job.base_dir, rel_path)
 record.addEntry(record_entry)
 # Figure out if a previously processed file is overriding this one.
 # This can happen if a theme file (processed via a mount point)

Mercurial > piecrust2

comparison piecrust/processing/base.py @ 129:3080b6d02f40