annotate garcon/documentation.py @ 1188:a7c43131d871

bake: Fix file write flushing problem with Python 3.8+ Writing the cache files fails in Python 3.8 because it looks like flushing behaviour has changed. We need to explicitly flush. And even then, in very rare occurrences, it looks like it can still run into racing conditions, so we do a very hacky and ugly "retry" loop when fetching cached data :(
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 15 Jun 2021 22:36:23 -0700
parents bd6cc78666b7
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
642
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
670
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
3 import re
642
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 from invoke import task, run
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
6
670
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
7 pyver_re = re.compile('^Python (?P<maj>\d)\.(?P<min>\d)\.(?P<pat>\d)$')
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
8
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
9
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
10 @task(help={
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
11 'tmp_dir': "The directory in which to bake the docs temporarily.",
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
12 'out_dir': "If the bake is successful, the directory in which to deploy "
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
13 "the files at the end.",
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
14 'root_url': "Set the docs site root URL to this if needed.",
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
15 'venv_dir': "The directory of the virtual environment to use to run "
1066
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
16 "PieCrust. If none, will create a new one under `venv`."
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
17 })
990
22cf13b86cc3 cm: Upgrade Garcon tasks to the latest PyInvoke version.
Ludovic Chabant <ludovic@chabant.com>
parents: 800
diff changeset
18 def gendocs(ctx, tmp_dir=None, out_dir=None, root_url=None, venv_dir=None):
670
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
19 base_dir = os.path.abspath(
1066
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
20 os.path.join(os.path.dirname(__file__), '..'))
670
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
21 os.chdir(base_dir)
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
22
642
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 if not tmp_dir:
670
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
24 tmp_dir = os.path.join(base_dir, '_docs-counter')
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
25
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
26 if not venv_dir:
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
27 venv_dir = os.path.join(base_dir, 'venv')
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
28
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
29 if not os.path.isdir(venv_dir):
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
30 print("Creating virtual environment in: %s" % venv_dir)
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
31 run('virtualenv -p python3 "%s"' % venv_dir)
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
32
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
33 pyexe = os.path.join(venv_dir, 'bin', 'python')
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
34 pyver_out = run('%s --version' % pyexe, hide=True)
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
35 if pyver_out.failed or not pyver_out.stdout.startswith('Python 3.'):
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
36 raise Exception("Can't run Python3 from: %s" % pyexe)
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
37 print("Using: %s" % pyver_out.stdout.strip())
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
38
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
39 pipexe = os.path.join(venv_dir, 'bin', 'pip')
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
40 pipver_out = run('%s --version' % pipexe, hide=True)
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
41 if pipver_out.failed or '(python 3.' not in pipver_out.stdout:
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
42 raise Exception("Can't run pip3 from: %s" % pipexe)
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
43 print("Using: %s" % pipver_out.stdout.strip())
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
44
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
45 npmver_out = run('npm --version', hide=True)
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
46 print("Using: npm %s" % npmver_out.stdout.strip())
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
47
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
48 print("Updating virtual environment.")
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
49 run("%s install pip -U" % pipexe)
a409a7bb3948 cm: Improve documentation generation script.
Ludovic Chabant <ludovic@chabant.com>
parents: 647
diff changeset
50 run("%s install -r requirements.txt" % pipexe)
642
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51
645
3060a6f26330 cm: Update the node modules before building the documentation.
Ludovic Chabant <ludovic@chabant.com>
parents: 642
diff changeset
52 print("Update node modules")
3060a6f26330 cm: Update the node modules before building the documentation.
Ludovic Chabant <ludovic@chabant.com>
parents: 642
diff changeset
53 run("npm install")
3060a6f26330 cm: Update the node modules before building the documentation.
Ludovic Chabant <ludovic@chabant.com>
parents: 642
diff changeset
54
1066
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
55 this_pwd = os.path.dirname(os.path.dirname(__file__))
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
56 node_bin = os.path.join(this_pwd, 'node_modules', '.bin')
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
57 print("Adding '%s' to the PATH" % node_bin)
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
58 os.environ['PATH'] = (node_bin + os.pathsep + os.environ['PATH'])
642
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 print("Generate PieCrust version")
647
be67fb6add5f cm: Fixes and tweaks to the documentation generation task.
Ludovic Chabant <ludovic@chabant.com>
parents: 645
diff changeset
61 run(pyexe + ' setup.py version')
642
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
62 from piecrust.__version__ import APP_VERSION
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 version = APP_VERSION
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 print("Baking documentation for version: %s" % version)
647
be67fb6add5f cm: Fixes and tweaks to the documentation generation task.
Ludovic Chabant <ludovic@chabant.com>
parents: 645
diff changeset
66 if root_url:
be67fb6add5f cm: Fixes and tweaks to the documentation generation task.
Ludovic Chabant <ludovic@chabant.com>
parents: 645
diff changeset
67 print("Using root URL: %s" % root_url)
642
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 args = [
1066
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
69 pyexe, 'chef.py',
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
70 '--root', 'docs',
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
71 '--config', 'dist']
647
be67fb6add5f cm: Fixes and tweaks to the documentation generation task.
Ludovic Chabant <ludovic@chabant.com>
parents: 645
diff changeset
72 if root_url:
be67fb6add5f cm: Fixes and tweaks to the documentation generation task.
Ludovic Chabant <ludovic@chabant.com>
parents: 645
diff changeset
73 args += ['--config-set', 'site/root', root_url]
be67fb6add5f cm: Fixes and tweaks to the documentation generation task.
Ludovic Chabant <ludovic@chabant.com>
parents: 645
diff changeset
74 args += [
1066
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
75 'bake',
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
76 '-o', tmp_dir
bd6cc78666b7 cm: Update `garcon`'s documentation script to work with latest toolchain.
Ludovic Chabant <ludovic@chabant.com>
parents: 990
diff changeset
77 ]
642
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
78 run(' '.join(args))
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
79
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
80 if out_dir:
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
81 print("Synchronizing %s" % out_dir)
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
82 if not os.path.isdir(out_dir):
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
83 os.makedirs(out_dir)
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
84
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
85 tmp_dir = tmp_dir.rstrip('/') + '/'
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
86 out_dir = out_dir.rstrip('/') + '/'
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
87 run('rsync -av --delete %s %s' % (tmp_dir, out_dir))
79aefe82c6b6 cm: Move all scripts into a `garcon` package with `invoke` support.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
88