annotate tests/test_processing_base.py @ 415:0e9a94b7fdfa

bake: Improve bake record information. * Store things in the bake record that require less interaction between the master process and the workers. For instance, don't store the paginator object in the render pass info -- instead, just store whether pagination was used, and whether it had more items. * Simplify information passing between workers and bake passes by saving the rendering info to the JSON cache. This means the "render first sub" job doesn't have to return anything except errors now. * Add more performance counter info.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 20 Jun 2015 19:23:16 -0700
parents c4b3a7fd2f87
children d90ccdf18156
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
1 import time
36
485682a6de50 New site layout support.
Ludovic Chabant <ludovic@chabant.com>
parents: 35
diff changeset
2 import os.path
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
3 import shutil
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import pytest
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
5 from piecrust.processing.base import SimpleFileProcessor
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
6 from piecrust.processing.pipeline import ProcessorPipeline
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
7 from piecrust.processing.records import ProcessorPipelineRecord
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
8 from piecrust.processing.worker import get_filtered_processors
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9 from .mockutil import mock_fs, mock_fs_scope
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
12 class FooProcessor(SimpleFileProcessor):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
13 def __init__(self, exts=None, open_func=None):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
14 exts = exts or {'foo', 'foo'}
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
15 super(FooProcessor, self).__init__({exts[0]: exts[1]})
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
16 self.PROCESSOR_NAME = exts[0]
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
17 self.open_func = open_func or open
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
18
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
19 def _doProcess(self, in_path, out_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
20 with self.open_func(in_path, 'r') as f:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
21 text = f.read()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
22 with self.open_func(out_path, 'w') as f:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
23 f.write("%s: %s" % (self.PROCESSOR_NAME.upper(), text))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
24 return True
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
25
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
26
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
27 class NoopProcessor(SimpleFileProcessor):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
28 def __init__(self, exts):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
29 super(NoopProcessor, self).__init__({exts[0]: exts[1]})
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
30 self.PROCESSOR_NAME = exts[0]
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
31 self.processed = []
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
32
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
33 def _doProcess(self, in_path, out_path):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
34 self.processed.append(in_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
35 shutil.copyfile(in_path, out_path)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
36 return True
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
37
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
38
205
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
39 def _get_pipeline(fs, app=None):
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
40 app = app or fs.getApp()
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
41 return ProcessorPipeline(app, fs.path('counter'))
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42
36
485682a6de50 New site layout support.
Ludovic Chabant <ludovic@chabant.com>
parents: 35
diff changeset
43
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 def test_empty():
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 fs = mock_fs()
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 with mock_fs_scope(fs):
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 pp = _get_pipeline(fs)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
48 pp.enabled_processors = ['copy']
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 expected = {}
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 assert expected == fs.getStructure('counter')
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 pp.run()
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 expected = {}
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 assert expected == fs.getStructure('counter')
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56 def test_one_file():
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 fs = (mock_fs()
36
485682a6de50 New site layout support.
Ludovic Chabant <ludovic@chabant.com>
parents: 35
diff changeset
58 .withFile('kitchen/assets/something.html', 'A test file.'))
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 with mock_fs_scope(fs):
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 pp = _get_pipeline(fs)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
61 pp.enabled_processors = ['copy']
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 expected = {}
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 assert expected == fs.getStructure('counter')
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 pp.run()
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 expected = {'something.html': 'A test file.'}
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 assert expected == fs.getStructure('counter')
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
69 def test_one_level_dirtyness():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
70 fs = (mock_fs()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
71 .withFile('kitchen/assets/blah.foo', 'A test file.'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
72 with mock_fs_scope(fs):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
73 pp = _get_pipeline(fs)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
74 pp.enabled_processors = ['copy']
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
75 pp.run()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
76 expected = {'blah.foo': 'A test file.'}
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
77 assert expected == fs.getStructure('counter')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
78 mtime = os.path.getmtime(fs.path('/counter/blah.foo'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
79 assert abs(time.time() - mtime) <= 2
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
80
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
81 time.sleep(1)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
82 pp.run()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
83 assert expected == fs.getStructure('counter')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
84 assert mtime == os.path.getmtime(fs.path('/counter/blah.foo'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
85
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
86 time.sleep(1)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
87 fs.withFile('kitchen/assets/blah.foo', 'A new test file.')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
88 pp.run()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
89 expected = {'blah.foo': 'A new test file.'}
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
90 assert expected == fs.getStructure('counter')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
91 assert mtime < os.path.getmtime(fs.path('/counter/blah.foo'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
92
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
93
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
94 def test_two_levels_dirtyness():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
95 fs = (mock_fs()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
96 .withFile('kitchen/assets/blah.foo', 'A test file.'))
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
97 with mock_fs_scope(fs):
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
98 pp = _get_pipeline(fs)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
99 pp.enabled_processors = ['copy']
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
100 pp.additional_processors = [FooProcessor(('foo', 'bar'))]
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
101 pp.run()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
102 expected = {'blah.bar': 'FOO: A test file.'}
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
103 assert expected == fs.getStructure('counter')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
104 mtime = os.path.getmtime(fs.path('/counter/blah.bar'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
105 assert abs(time.time() - mtime) <= 2
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
106
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
107 time.sleep(1)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
108 pp.run()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
109 assert expected == fs.getStructure('counter')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
110 assert mtime == os.path.getmtime(fs.path('/counter/blah.bar'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
111
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
112 time.sleep(1)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
113 fs.withFile('kitchen/assets/blah.foo', 'A new test file.')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
114 pp.run()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
115 expected = {'blah.bar': 'FOO: A new test file.'}
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
116 assert expected == fs.getStructure('counter')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
117 assert mtime < os.path.getmtime(fs.path('/counter/blah.bar'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
118
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
119
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
120 def test_removed():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
121 fs = (mock_fs()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
122 .withFile('kitchen/assets/blah1.foo', 'A test file.')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
123 .withFile('kitchen/assets/blah2.foo', 'Ooops'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
124 with mock_fs_scope(fs):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
125 expected = {
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
126 'blah1.foo': 'A test file.',
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
127 'blah2.foo': 'Ooops'}
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
128 assert expected == fs.getStructure('kitchen/assets')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
129 pp = _get_pipeline(fs)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
130 pp.enabled_processors = ['copy']
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
131 pp.run()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
132 assert expected == fs.getStructure('counter')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
133
411
e7b865f8f335 bake: Enable multiprocess baking.
Ludovic Chabant <ludovic@chabant.com>
parents: 205
diff changeset
134 time.sleep(1)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
135 os.remove(fs.path('/kitchen/assets/blah2.foo'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
136 expected = {
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
137 'blah1.foo': 'A test file.'}
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
138 assert expected == fs.getStructure('kitchen/assets')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
139 pp.run()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
140 assert expected == fs.getStructure('counter')
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
141
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
142
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
143 def test_record_version_change():
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
144 fs = (mock_fs()
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
145 .withFile('kitchen/assets/blah.foo', 'A test file.'))
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
146 with mock_fs_scope(fs):
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
147 pp = _get_pipeline(fs)
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
148 noop = NoopProcessor(('foo', 'foo'))
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
149 pp.enabled_processors = ['copy']
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
150 pp.additional_processors = [noop]
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
151 pp.run()
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
152 assert os.path.exists(fs.path('/counter/blah.foo')) is True
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
153 mtime = os.path.getmtime(fs.path('/counter/blah.foo'))
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
154
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
155 time.sleep(1)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
156 pp.run()
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
157 assert mtime == os.path.getmtime(fs.path('/counter/blah.foo'))
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
158
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
159 time.sleep(1)
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
160 ProcessorPipelineRecord.RECORD_VERSION += 1
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
161 try:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
162 pp.run()
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
163 assert mtime < os.path.getmtime(fs.path('/counter/blah.foo'))
120
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
164 finally:
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
165 ProcessorPipelineRecord.RECORD_VERSION -= 1
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
166
133845647083 Better error management and removal support in baking/processing.
Ludovic Chabant <ludovic@chabant.com>
parents: 85
diff changeset
167
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
168 @pytest.mark.parametrize('patterns, expected', [
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
169 (['_'],
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
170 {'something.html': 'A test file.'}),
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
171 (['html'],
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
172 {}),
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
173 (['/^_/'],
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
174 {'something.html': 'A test file.',
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
175 'foo': {'_important.html': 'Important!'}})
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
176 ])
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
177 def test_ignore_pattern(patterns, expected):
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
178 fs = (mock_fs()
36
485682a6de50 New site layout support.
Ludovic Chabant <ludovic@chabant.com>
parents: 35
diff changeset
179 .withFile('kitchen/assets/something.html', 'A test file.')
485682a6de50 New site layout support.
Ludovic Chabant <ludovic@chabant.com>
parents: 35
diff changeset
180 .withFile('kitchen/assets/_hidden.html', 'Shhh')
485682a6de50 New site layout support.
Ludovic Chabant <ludovic@chabant.com>
parents: 35
diff changeset
181 .withFile('kitchen/assets/foo/_important.html', 'Important!'))
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
182 with mock_fs_scope(fs):
205
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
183 pp = _get_pipeline(fs)
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
184 pp.addIgnorePatterns(patterns)
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
185 pp.enabled_processors = ['copy']
205
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
186 assert {} == fs.getStructure('counter')
35
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
187 pp.run()
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
188 assert expected == fs.getStructure('counter')
e4c345dcf33c More unit tests, fix a bug with the skip patterns.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
189
205
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
190
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
191 @pytest.mark.parametrize('names, expected', [
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
192 ('all', ['cleancss', 'compass', 'copy', 'concat', 'less', 'requirejs',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
193 'sass', 'sitemap', 'uglifyjs']),
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
194 ('all -sitemap', ['cleancss', 'copy', 'compass', 'concat', 'less',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
195 'requirejs', 'sass', 'uglifyjs']),
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
196 ('-sitemap -less -sass all', ['cleancss', 'copy', 'compass', 'concat',
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
197 'requirejs', 'uglifyjs']),
205
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
198 ('copy', ['copy']),
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
199 ('less sass', ['less', 'sass'])
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
200 ])
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
201 def test_filter_processor(names, expected):
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
202 fs = mock_fs()
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
203 with mock_fs_scope(fs):
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
204 app = fs.getApp()
414
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
205 processors = app.plugin_loader.getProcessors()
c4b3a7fd2f87 bake: Make pipeline processing multi-process.
Ludovic Chabant <ludovic@chabant.com>
parents: 411
diff changeset
206 procs = get_filtered_processors(processors, names)
205
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
207 actual = [p.PROCESSOR_NAME for p in procs]
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
208 assert sorted(actual) == sorted(expected)
e725af1d48fb bake: Changes in how assets directories are configured.
Ludovic Chabant <ludovic@chabant.com>
parents: 120
diff changeset
209