Mercurial > piecrust2

annotate piecrust/publishing/publisher.py @ 879:58ae026b4c31

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
chef: Optimize startup time.
author Ludovic Chabant <ludovic@chabant.com>
date Thu, 15 Jun 2017 22:38:05 -0700
parents 6abb436fea5b
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
1 import os.path
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
2 import time
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import logging
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
4 from piecrust.chefutil import format_timed
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 from piecrust.publishing.base import PublishingContext
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8 logger = logging.getLogger(__name__)
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11 class InvalidPublishTargetError(Exception):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12 pass
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 class PublishingError(Exception):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 pass
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 class Publisher(object):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 def __init__(self, app):
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 self.app = app
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
22
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
23 def run(self, target,
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
24 force=False, preview=False, extra_args=None, log_file=None,
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
25 applied_config_variant=None, applied_config_values=None):
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
26 start_time = time.perf_counter()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
27
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
28 # Get publisher for this target.
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
29 pub = self.app.getPublisher(target)
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
30 if pub is None:
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
31 raise InvalidPublishTargetError(
879
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
32 "No such publish target: %s" % target)
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
34 # Will we need to bake first?
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
35 bake_first = True
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
36 if not pub.has_url_config:
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
37 bake_first = pub.getConfigValue('bake', True)
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
39 # Setup logging stuff.
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
40 hdlr = None
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
41 root_logger = logging.getLogger()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
42 if log_file and not preview:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
43 logger.debug("Adding file handler for: %s" % log_file)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
44 hdlr = logging.FileHandler(log_file, mode='w', encoding='utf8')
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
45 root_logger.addHandler(hdlr)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
46 if not preview:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
47 logger.info("Deploying to %s" % target)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
48 else:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
49 logger.info("Previewing deployment to %s" % target)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
50
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
51 # Bake first is necessary.
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
52 rec1 = None
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
53 rec2 = None
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
54 was_baked = False
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
55 bake_out_dir = os.path.join(self.app.root_dir, '_pub', target)
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
56 if bake_first:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
57 if not preview:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
58 bake_start_time = time.perf_counter()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
59 logger.debug("Baking first to: %s" % bake_out_dir)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
60
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
61 from piecrust.baking.baker import Baker
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
62 baker = Baker(
879
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
63 self.app, bake_out_dir,
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
64 applied_config_variant=applied_config_variant,
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
65 applied_config_values=applied_config_values)
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
66 rec1 = baker.bake()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
67
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
68 from piecrust.processing.pipeline import ProcessorPipeline
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
69 proc = ProcessorPipeline(
879
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
70 self.app, bake_out_dir,
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
71 applied_config_variant=applied_config_variant,
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
72 applied_config_values=applied_config_values)
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
73 rec2 = proc.run()
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
74
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
75 was_baked = True
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
76
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
77 if not rec1.success or not rec2.success:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
78 raise Exception(
879
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
79 "Error during baking, aborting publishing.")
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
80 logger.info(format_timed(bake_start_time, "Baked website."))
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
81 else:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
82 logger.info("Would bake to: %s" % bake_out_dir)
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
83
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
84 # Publish!
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
85 logger.debug(
879
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
86 "Running publish target '%s' with publisher: %s" %
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
87 (target, pub.PUBLISHER_NAME))
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
88 pub_start_time = time.perf_counter()
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
90 ctx = PublishingContext()
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
91 ctx.bake_out_dir = bake_out_dir
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
92 ctx.bake_record = rec1
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
93 ctx.processing_record = rec2
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
94 ctx.was_baked = was_baked
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
95 ctx.preview = preview
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
96 ctx.args = extra_args
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
97 try:
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
98 pub.run(ctx)
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
99 except Exception as ex:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
100 raise PublishingError(
879
58ae026b4c31 chef: Optimize startup time.
Ludovic Chabant <ludovic@chabant.com>
parents: 758
diff changeset
101 "Error publishing to target: %s" % target) from ex
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
102 finally:
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
103 if hdlr:
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
104 root_logger.removeHandler(hdlr)
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
105 hdlr.close()
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
106
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
107 logger.info(format_timed(
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
108 pub_start_time, "Ran publisher %s" % pub.PUBLISHER_NAME))
613
e2e955a3bb25 publish: Add publish command.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
110 logger.info(format_timed(start_time, 'Deployed to %s' % target))
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
111
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
112
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
113 def find_publisher_class(app, name, is_scheme=False):
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
114 attr_name = 'PUBLISHER_SCHEME' if is_scheme else 'PUBLISHER_NAME'
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
115 for pub_cls in app.plugin_loader.getPublishers():
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
116 pub_sch = getattr(pub_cls, attr_name, None)
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
117 if pub_sch == name:
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
118 return pub_cls
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
119 return None
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
120
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
121
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
122 def find_publisher_name(app, scheme):
758
6abb436fea5b publish: Make publisher more powerful and better exposed on the command line.
Ludovic Chabant <ludovic@chabant.com>
parents: 621
diff changeset
123 pub_cls = find_publisher_class(app, scheme, True)
621
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
124 if pub_cls:
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
125 return pub_cls.PUBLISHER_NAME
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
126 return None
8f9c0bdb3724 publish: Polish/refactor the publishing workflows.
Ludovic Chabant <ludovic@chabant.com>
parents: 613
diff changeset
127