Mercurial > piecrust2

annotate piecrust/importing/piecrust.py @ 469:f14796cbab0f 2.0.0a12

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cm: Fix wrong directory for utilities.
author Ludovic Chabant <ludovic@chabant.com>
date Tue, 14 Jul 2015 23:02:38 -0700
parents 2823ea40cfac
children 233234e02816
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import re
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import shutil
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import logging
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
6 import yaml
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 from piecrust.importing.base import FileWalkingImporter
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 logger = logging.getLogger(__name__)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 class PieCrust1Importer(FileWalkingImporter):
297
2823ea40cfac import: Put importer metadata on the class, and allow return values.
Ludovic Chabant <ludovic@chabant.com>
parents: 291
diff changeset
14 name = 'piecrust1'
2823ea40cfac import: Put importer metadata on the class, and allow return values.
Ludovic Chabant <ludovic@chabant.com>
parents: 291
diff changeset
15 description = "Imports content from a PieCrust 1 website."
2823ea40cfac import: Put importer metadata on the class, and allow return values.
Ludovic Chabant <ludovic@chabant.com>
parents: 291
diff changeset
16 requires_website = False
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 def setupParser(self, parser, app):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 super(PieCrust1Importer, self).setupParser(parser, app)
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
20 parser.add_argument('root_dir', nargs='?',
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 help="The root directory of the PieCrust 1 website.")
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
22 parser.add_argument('--upgrade', action='store_true',
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
23 help="Upgrade the current website in place.")
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 def importWebsite(self, app, args):
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
26 if app.root_dir and args.upgrade:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
27 raise Exception("Can't specifiy both a root directory and `--upgrade`.")
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
28 if app.root_dir is None and not args.upgrade:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
29 raise Exception("Need to specify either a root directory or `--upgrade`.")
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
30
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
31 root_dir = os.getcwd() if args.upgrade else app.root_dir
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
32 logger.debug("Importing PieCrust 1 site from: %s" % root_dir)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 exclude = args.exclude or []
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 exclude += ['_cache', '_counter']
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
35 self._startWalk(root_dir, exclude, root_dir, args.upgrade)
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
36 if args.upgrade:
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
37 self._cleanEmptyDirectories(root_dir)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 logger.info("The PieCrust website was successfully imported.")
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
40 def _importFile(self, full_fn, rel_fn, out_root_dir, is_move):
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 logger.debug("- %s" % rel_fn)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 dest_path = rel_fn
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 convert_func = None
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 if rel_fn.replace('\\', '/') == '_content/config.yml':
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 dest_path = 'config.yml'
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 convert_func = self.convertConfig
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 elif rel_fn.startswith('_content'):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 dest_path = rel_fn[len('_content/'):]
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 fn_dirname = os.path.dirname(rel_fn)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 if not fn_dirname.endswith('-assets'):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 convert_func = self.convertPage
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 else:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 dest_path = 'assets/' + rel_fn
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 logger.debug(" %s -> %s" % (rel_fn, dest_path))
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
56 full_dest_path = os.path.join(out_root_dir, dest_path)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 os.makedirs(os.path.dirname(full_dest_path), 0o755, True)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 if convert_func is None:
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
59 if is_move:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
60 shutil.move(full_fn, full_dest_path)
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
61 else:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
62 shutil.copy2(full_fn, full_dest_path)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 else:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 with open(full_fn, 'r', encoding='utf8') as fp:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 content = fp.read()
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 converted_content = convert_func(content)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 with open(full_dest_path, 'w', encoding='utf8') as fp:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 fp.write(converted_content)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 if converted_content != content:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 logger.warning("'%s' has been modified. The original version "
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 "has been kept for reference." % rel_fn)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 shutil.copy2(full_fn, full_dest_path + '.orig')
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
73 if is_move:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
74 os.remove(full_fn)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
76 def _cleanEmptyDirectories(self, root_dir):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
77 for item in os.listdir(root_dir):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
78 if not os.path.isdir(item):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
79 continue
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
80
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
81 file_count = 0
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
82 item_path = os.path.join(root_dir, item)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
83 for _, __, filenames in os.walk(item_path):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
84 file_count += len(filenames)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
85 if file_count == 0:
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
86 logger.debug("Deleting empty directory: %s" % item)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
87 shutil.rmtree(item_path)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
88
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 def convertConfig(self, content):
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
90 config = yaml.load(content)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
91 sitec = config.setdefault('site', {})
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
92 if 'templates_dirs' in sitec:
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
93 tdc = sitec['templates_dirs']
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
94 cl = len('_content/')
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
95 if isinstance(tdc, str) and re.match(r'^_content[/\\]', tdc):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
96 sitec['templates_dirs'] = tdc[cl:]
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
97 elif isinstance(tdc, list):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
98 sitec['templates_dirs'] = list(map(
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
99 lambda d: d[cl:] if re.match(r'^_content[/\\]', d) else d,
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
100 tdc))
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
101
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
102 jinjac = config.setdefault('jinja', {})
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
103 jinjac['twig_compatibility'] = True
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
104
291
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
105 if 'baker' in config:
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
106 if 'skip_patterns' in config['baker']:
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
107 config['baker']['ignore'] = config['baker']['skip_patterns']
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
108 del config['baker']['skip_patterns']
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
109 if 'force_patterns' in config['baker']:
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
110 config['baker']['force'] = config['baker']['force_patterns']
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
111 del config['baker']['force_patterns']
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
112
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
113 content = yaml.dump(config, default_flow_style=False)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 return content
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116 def convertPage(self, content):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 return content
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118