Mercurial > piecrust2

annotate piecrust/importing/piecrust.py @ 298:b7ab1b503510

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
data: Fix incorrect next/previous page URLs in pagination data. Consolidate splitting an URL between its first URL and its sub page number. Be careful about the index page's URL not losing its slash.
author Ludovic Chabant <ludovic@chabant.com>
date Wed, 11 Mar 2015 23:46:42 -0700
parents 2823ea40cfac
children 233234e02816
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
1 import os
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
2 import os.path
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
3 import re
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
4 import shutil
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
5 import logging
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
6 import yaml
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
7 from piecrust.importing.base import FileWalkingImporter
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
8
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
9
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
10 logger = logging.getLogger(__name__)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
11
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
12
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
13 class PieCrust1Importer(FileWalkingImporter):
297
2823ea40cfac import: Put importer metadata on the class, and allow return values.
Ludovic Chabant <ludovic@chabant.com>
parents: 291
diff changeset
14 name = 'piecrust1'
2823ea40cfac import: Put importer metadata on the class, and allow return values.
Ludovic Chabant <ludovic@chabant.com>
parents: 291
diff changeset
15 description = "Imports content from a PieCrust 1 website."
2823ea40cfac import: Put importer metadata on the class, and allow return values.
Ludovic Chabant <ludovic@chabant.com>
parents: 291
diff changeset
16 requires_website = False
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
17
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
18 def setupParser(self, parser, app):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
19 super(PieCrust1Importer, self).setupParser(parser, app)
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
20 parser.add_argument('root_dir', nargs='?',
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
21 help="The root directory of the PieCrust 1 website.")
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
22 parser.add_argument('--upgrade', action='store_true',
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
23 help="Upgrade the current website in place.")
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
24
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
25 def importWebsite(self, app, args):
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
26 if app.root_dir and args.upgrade:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
27 raise Exception("Can't specifiy both a root directory and `--upgrade`.")
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
28 if app.root_dir is None and not args.upgrade:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
29 raise Exception("Need to specify either a root directory or `--upgrade`.")
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
30
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
31 root_dir = os.getcwd() if args.upgrade else app.root_dir
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
32 logger.debug("Importing PieCrust 1 site from: %s" % root_dir)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
33 exclude = args.exclude or []
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
34 exclude += ['_cache', '_counter']
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
35 self._startWalk(root_dir, exclude, root_dir, args.upgrade)
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
36 if args.upgrade:
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
37 self._cleanEmptyDirectories(root_dir)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
38 logger.info("The PieCrust website was successfully imported.")
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
39
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
40 def _importFile(self, full_fn, rel_fn, out_root_dir, is_move):
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
41 logger.debug("- %s" % rel_fn)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
42 dest_path = rel_fn
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
43 convert_func = None
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
44 if rel_fn.replace('\\', '/') == '_content/config.yml':
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
45 dest_path = 'config.yml'
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
46 convert_func = self.convertConfig
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
47 elif rel_fn.startswith('_content'):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
48 dest_path = rel_fn[len('_content/'):]
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
49 fn_dirname = os.path.dirname(rel_fn)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
50 if not fn_dirname.endswith('-assets'):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
51 convert_func = self.convertPage
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
52 else:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
53 dest_path = 'assets/' + rel_fn
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
54
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
55 logger.debug(" %s -> %s" % (rel_fn, dest_path))
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
56 full_dest_path = os.path.join(out_root_dir, dest_path)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
57 os.makedirs(os.path.dirname(full_dest_path), 0o755, True)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
58 if convert_func is None:
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
59 if is_move:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
60 shutil.move(full_fn, full_dest_path)
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
61 else:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
62 shutil.copy2(full_fn, full_dest_path)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
63 else:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
64 with open(full_fn, 'r', encoding='utf8') as fp:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
65 content = fp.read()
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
66 converted_content = convert_func(content)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
67 with open(full_dest_path, 'w', encoding='utf8') as fp:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
68 fp.write(converted_content)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
69 if converted_content != content:
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
70 logger.warning("'%s' has been modified. The original version "
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
71 "has been kept for reference." % rel_fn)
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
72 shutil.copy2(full_fn, full_dest_path + '.orig')
63
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
73 if is_move:
28958565a17b In-place upgrade for PieCrust 1 sites.
Ludovic Chabant <ludovic@chabant.com>
parents: 62
diff changeset
74 os.remove(full_fn)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
75
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
76 def _cleanEmptyDirectories(self, root_dir):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
77 for item in os.listdir(root_dir):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
78 if not os.path.isdir(item):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
79 continue
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
80
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
81 file_count = 0
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
82 item_path = os.path.join(root_dir, item)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
83 for _, __, filenames in os.walk(item_path):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
84 file_count += len(filenames)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
85 if file_count == 0:
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
86 logger.debug("Deleting empty directory: %s" % item)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
87 shutil.rmtree(item_path)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
88
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
89 def convertConfig(self, content):
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
90 config = yaml.load(content)
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
91 sitec = config.setdefault('site', {})
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
92 if 'templates_dirs' in sitec:
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
93 tdc = sitec['templates_dirs']
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
94 cl = len('_content/')
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
95 if isinstance(tdc, str) and re.match(r'^_content[/\\]', tdc):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
96 sitec['templates_dirs'] = tdc[cl:]
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
97 elif isinstance(tdc, list):
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
98 sitec['templates_dirs'] = list(map(
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
99 lambda d: d[cl:] if re.match(r'^_content[/\\]', d) else d,
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
100 tdc))
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
101
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
102 jinjac = config.setdefault('jinja', {})
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
103 jinjac['twig_compatibility'] = True
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
104
291
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
105 if 'baker' in config:
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
106 if 'skip_patterns' in config['baker']:
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
107 config['baker']['ignore'] = config['baker']['skip_patterns']
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
108 del config['baker']['skip_patterns']
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
109 if 'force_patterns' in config['baker']:
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
110 config['baker']['force'] = config['baker']['force_patterns']
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
111 del config['baker']['force_patterns']
46842f71f31f import: Upgrade more settings for the PieCrust 1 importer.
Ludovic Chabant <ludovic@chabant.com>
parents: 64
diff changeset
112
64
9ae3237365eb PieCrust 1 import: clean empty directories and convert some config values.
Ludovic Chabant <ludovic@chabant.com>
parents: 63
diff changeset
113 content = yaml.dump(config, default_flow_style=False)
62
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
114 return content
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
115
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
116 def convertPage(self, content):
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
117 return content
52e4d9a1f917 Simple importer for PieCrust 1 websites.
Ludovic Chabant <ludovic@chabant.com>
parents:
diff changeset
118