Mercurial > piecrust2

annotate piecrust/importing/piecrust.py @ 160:de09d41bae23

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Moved all installation instructions to a new `INSTALL` file.
author Ludovic Chabant <ludovic@chabant.com>
date Thu, 01 Jan 2015 19:35:18 -0800
parents 9ae3237365eb
children 46842f71f31f
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import re
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import shutil
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import logging
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
6 import yaml
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 from piecrust.importing.base import FileWalkingImporter
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 logger = logging.getLogger(__name__)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 class PieCrust1Importer(FileWalkingImporter):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
14 def __init__(self):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
15 super(PieCrust1Importer, self).__init__()
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
16 self.name = 'piecrust1'
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17 self.description = "Imports content from a PieCrust 1 website."
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
18 self.requires_website = False
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
20 def setupParser(self, parser, app):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 super(PieCrust1Importer, self).setupParser(parser, app)
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
22 parser.add_argument('root_dir', nargs='?',
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
23 help="The root directory of the PieCrust 1 website.")
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
24 parser.add_argument('--upgrade', action='store_true',
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
25 help="Upgrade the current website in place.")
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
26
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
27 def importWebsite(self, app, args):
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
28 if app.root_dir and args.upgrade:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
29 raise Exception("Can't specifiy both a root directory and `--upgrade`.")
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
30 if app.root_dir is None and not args.upgrade:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
31 raise Exception("Need to specify either a root directory or `--upgrade`.")
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
32
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
33 root_dir = os.getcwd() if args.upgrade else app.root_dir
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
34 logger.debug("Importing PieCrust 1 site from: %s" % root_dir)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
35 exclude = args.exclude or []
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
36 exclude += ['_cache', '_counter']
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
37 self._startWalk(root_dir, exclude, root_dir, args.upgrade)
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
38 if args.upgrade:
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
39 self._cleanEmptyDirectories(root_dir)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
40 logger.info("The PieCrust website was successfully imported.")
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
42 def _importFile(self, full_fn, rel_fn, out_root_dir, is_move):
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 logger.debug("- %s" % rel_fn)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 dest_path = rel_fn
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 convert_func = None
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 if rel_fn.replace('\\', '/') == '_content/config.yml':
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 dest_path = 'config.yml'
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 convert_func = self.convertConfig
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 elif rel_fn.startswith('_content'):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 dest_path = rel_fn[len('_content/'):]
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 fn_dirname = os.path.dirname(rel_fn)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 if not fn_dirname.endswith('-assets'):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 convert_func = self.convertPage
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54 else:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 dest_path = 'assets/' + rel_fn
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
56
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 logger.debug(" %s -> %s" % (rel_fn, dest_path))
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
58 full_dest_path = os.path.join(out_root_dir, dest_path)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
59 os.makedirs(os.path.dirname(full_dest_path), 0o755, True)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
60 if convert_func is None:
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
61 if is_move:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
62 shutil.move(full_fn, full_dest_path)
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
63 else:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
64 shutil.copy2(full_fn, full_dest_path)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 else:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 with open(full_fn, 'r', encoding='utf8') as fp:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 content = fp.read()
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 converted_content = convert_func(content)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 with open(full_dest_path, 'w', encoding='utf8') as fp:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 fp.write(converted_content)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 if converted_content != content:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 logger.warning("'%s' has been modified. The original version "
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
73 "has been kept for reference." % rel_fn)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
74 shutil.copy2(full_fn, full_dest_path + '.orig')
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
75 if is_move:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
76 os.remove(full_fn)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
77
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
78 def _cleanEmptyDirectories(self, root_dir):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
79 for item in os.listdir(root_dir):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
80 if not os.path.isdir(item):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
81 continue
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
82
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
83 file_count = 0
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
84 item_path = os.path.join(root_dir, item)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
85 for _, __, filenames in os.walk(item_path):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
86 file_count += len(filenames)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
87 if file_count == 0:
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
88 logger.debug("Deleting empty directory: %s" % item)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
89 shutil.rmtree(item_path)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
90
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
91 def convertConfig(self, content):
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
92 config = yaml.load(content)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
93 sitec = config.setdefault('site', {})
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
94 if 'templates_dirs' in sitec:
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
95 tdc = sitec['templates_dirs']
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
96 cl = len('_content/')
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
97 if isinstance(tdc, str) and re.match(r'^_content[/\\]', tdc):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
98 sitec['templates_dirs'] = tdc[cl:]
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
99 elif isinstance(tdc, list):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
100 sitec['templates_dirs'] = list(map(
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
101 lambda d: d[cl:] if re.match(r'^_content[/\\]', d) else d,
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
102 tdc))
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
103
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
104 jinjac = config.setdefault('jinja', {})
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
105 jinjac['twig_compatibility'] = True
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
106
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
107 content = yaml.dump(config, default_flow_style=False)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
108 return content
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
109
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
110 def convertPage(self, content):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
111 return content
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
112