Mercurial > wikked

changeset 47:86ee1b696070

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Big refactoring: - Trying to simplify APIs - Use SQLite to cache information
author Ludovic Chabant <ludovic@chabant.com>
date Fri, 25 Jan 2013 22:35:56 -0800
parents 0b6ce6837d22
children 9658edea3121
files manage.py wikked/auth.py wikked/db.py wikked/formatter.py wikked/fs.py wikked/indexer.py wikked/page.py wikked/resources/defaults.cfg wikked/settings.py wikked/views.py wikked/web.py wikked/wiki.py
diffstat 12 files changed, 1039 insertions(+), 338 deletions(-) [+]
line wrap: on
line diff
--- a/manage.py	Fri Jan 25 22:20:58 2013 -0800
+++ b/manage.py	Fri Jan 25 22:35:56 2013 -0800
@@ -1,9 +1,14 @@
-import os.path
-import logging
-from flask.ext.script import Manager, Command, prompt, prompt_pass
+
+# Configure a simpler log format.
+from wikked import settings
+settings.LOG_FORMAT = "[%(levelname)s]: %(message)s"
+
+# Create the app and the wiki.
 from wikked.web import app, wiki
+from wikked.page import Page, DatabasePage
 
-
+# Create the manager.
+from flask.ext.script import Manager, prompt, prompt_pass
 manager = Manager(app)
 
 
@@ -15,6 +20,7 @@
         print " - " + user.username
     print ""
 
+
 @manager.command
 def new_user():
     """Generates the entry for a new user so you can
@@ -28,21 +34,37 @@
 
 
 @manager.command
-def reset_index():
-    """ Re-generates the index, if search is broken
-        somehow in your wiki.
+def reset():
+    """ Re-generates the database and the full-text-search index.
     """
+    wiki.db.reset(wiki.getPages(from_db=False, factory=Page.factory))
     wiki.index.reset(wiki.getPages())
 
 
 @manager.command
+def update():
+    """ Updates the database and the full-text-search index with any
+        changed/new files.
+    """
+    wiki.db.update(wiki.getPages(from_db=False, factory=Page.factory))
+    wiki.index.update(wiki.getPages())
+
+
+@manager.command
 def list():
     """ Lists page names in the wiki.
     """
-    for url in wiki.getPageUrls():
+    for url in wiki.db.getPageUrls():
         print url
 
 
+@manager.command
+def get(url):
+    """ Gets a page that matches the given URL.
+    """
+    page = wiki.getPage(url)
+    print page.text
+
+
 if __name__ == "__main__":
     manager.run()
-
--- a/wikked/auth.py	Fri Jan 25 22:20:58 2013 -0800
+++ b/wikked/auth.py	Fri Jan 25 22:35:56 2013 -0800
@@ -3,6 +3,8 @@
 
 
 class User(object):
+    """ A user with an account on the wiki.
+    """
     def __init__(self, username, password):
         self.username = username
         self.password = password
@@ -25,6 +27,8 @@
 
 
 class UserManager(object):
+    """ A class that keeps track of users and their permissions.
+    """
     def __init__(self, config, logger=None):
         if logger is None:
             logger = logging.getLogger('wikked.auth')
@@ -49,11 +53,11 @@
         return self._isAllowedForMeta(page, 'writers', username)
 
     def _isAllowedForMeta(self, page, meta_name, username):
-        if (self._permissions[meta_name] is not None and 
+        if (self._permissions[meta_name] is not None and
                 username not in self._permissions[meta_name]):
             return False
-        if meta_name in page.all_meta['user']:
-            allowed = [r.strip() for r in re.split(r'[ ,;]', page.all_meta['user'][meta_name])]
+        if meta_name in page.all_meta:
+            allowed = [r.strip() for r in re.split(r'[ ,;]', page.all_meta[meta_name])]
             if username is None:
                 return 'anonymous' in allowed
             else:
@@ -78,7 +82,7 @@
                 groups = config.items('groups')
 
             for user in config.items('users'):
-                user_info = { 'username': user[0], 'password': user[1], 'groups': [] }
+                user_info = {'username': user[0], 'password': user[1], 'groups': []}
                 for group in groups:
                     users_in_group = [u.strip() for u in re.split(r'[ ,;]', group[1])]
                     if user[0] in users_in_group:
@@ -89,4 +93,3 @@
         user = User(user_info['username'], user_info['password'])
         user.groups = list(user_info['groups'])
         return user
-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/wikked/db.py	Fri Jan 25 22:35:56 2013 -0800
@@ -0,0 +1,417 @@
+import os
+import os.path
+import types
+import string
+import logging
+import datetime
+import sqlite3
+
+
+class conn_scope(object):
+    """ Helper class, disguised as a function, to ensure the database
+        has been opened before doing something. If the database wasn't
+        open, it will be closed after the operation.
+    """
+    def __init__(self, db):
+        self.db = db
+        self.do_close = False
+
+    def __enter__(self):
+        self.do_close = (self.db.conn is None)
+        self.db.open()
+
+    def __exit__(self, type, value, traceback):
+        if self.do_close:
+            self.db.close()
+
+
+class Database(object):
+    """ The base class for a database cache.
+    """
+    def __init__(self, wiki, logger=None):
+        self.wiki = wiki
+
+        if logger is None:
+            logger = logging.getLogger('wikked.db')
+        self.logger = logger
+
+    def initDb(self):
+        raise NotImplementedError()
+
+    def open(self):
+        raise NotImplementedError()
+
+    def close(self):
+        raise NotImplementedError()
+
+    def reset(self, pages):
+        raise NotImplementedError()
+
+    def update(self, pages):
+        raise NotImplementedError()
+
+    def getPageUrls(self, subdir=None):
+        raise NotImplementedError()
+
+    def getPages(self, subdir=None):
+        raise NotImplementedError()
+
+    def getPage(self, url):
+        raise NotImplementedError()
+
+    def pageExists(self, url):
+        raise NotImplementedError()
+
+    def getLinksTo(self, url):
+        raise NotImplementedError()
+
+    def getConfigValues(self, section):
+        raise NotImplementedError()
+
+    def getConfigValue(self, section, name):
+        raise NotImplementedError()
+
+
+class SQLiteDatabase(Database):
+    """ A database cache based on SQLite.
+    """
+    schema_version = 1
+
+    def __init__(self, wiki, logger=None):
+        Database.__init__(self, wiki, logger)
+        self.db_path = os.path.join(wiki.root, '.wiki', 'wiki.db')
+        self.conn = None
+
+    def initDb(self):
+        create_schema = False
+        if not os.path.isdir(os.path.dirname(self.db_path)):
+            # No database on disk... create one.
+            self.logger.debug("Creating SQL database.")
+            os.makedirs(os.path.dirname(self.db_path))
+            create_schema = True
+        else:
+            # The existing schema is outdated, re-create it.
+            schema_version = self._getSchemaVersion()
+            if schema_version < self.schema_version:
+                create_schema = True
+        if create_schema:
+            with conn_scope(self):
+                self._createSchema()
+
+        # Cache the configuration.
+        cache_config = False
+        config_time = self._getInfoTime('config_time')
+        if os.path.isfile(self.wiki.config_path):
+            if (config_time is None or
+                config_time <= datetime.datetime.fromtimestamp(
+                    os.path.getmtime(self.wiki.config_path))):
+                cache_config = True
+        elif config_time is not None:
+            cache_config = True
+        if cache_config:
+            self._cacheConfig(self.wiki.config)
+
+    def open(self):
+        if self.conn is None:
+            self.conn = sqlite3.connect(self.db_path,
+                detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
+            self.conn.row_factory = sqlite3.Row
+
+    def close(self):
+        if self.conn is not None:
+            self.conn.close()
+            self.conn = None
+
+    def reset(self, pages):
+        self.logger.debug("Re-creating SQL database.")
+        with conn_scope(self):
+            self._createSchema()
+            c = self.conn.cursor()
+            for page in pages:
+                self._addPage(page, c)
+            self.conn.commit()
+
+    def update(self, pages):
+        self.logger.debug("Updating SQL database...")
+        to_update = set()
+        already_added = set()
+
+        with conn_scope(self):
+            c = self.conn.cursor()
+            c.execute('''SELECT id, time, path FROM pages''')
+            for r in c.fetchall():
+                if not os.path.isfile(r['path']):
+                    # File was deleted.
+                    self._removePage(r['id'], c)
+                else:
+                    already_added.add(r['path'])
+                    path_time = datetime.datetime.fromtimestamp(
+                        os.path.getmtime(r['path']))
+                    if path_time > r['time']:
+                        # File has changed since last index.
+                        self._removePage(r['id'], c)
+                        to_update.add(r['path'])
+            self.conn.commit()
+
+            for page in pages:
+                # We want the page's path, but getting it may load all kinds
+                # of metadata that is time-consuming, so we shortcut the
+                # system by querying the file-system directly.
+                fs_meta = self.wiki.fs.getPage(page.url)
+                if (fs_meta['path'] in to_update or
+                    fs_meta['path'] not in already_added):
+                    self._addPage(page, c)
+
+            # TODO: update any page with a query in it.
+
+            self.conn.commit()
+            self.logger.debug("...done updating SQL database.")
+
+    def getPageUrls(self, subdir=None):
+        with conn_scope(self):
+            c = self.conn.cursor()
+            if subdir:
+                subdir = string.rstrip(subdir, '/') + '/%'
+                c.execute('''SELECT url FROM pages WHERE url LIKE ?''',
+                    (subdir,))
+            else:
+                c.execute('''SELECT url FROM pages''')
+            urls = []
+            for row in c.fetchall():
+                urls.append(row['url'])
+            return urls
+
+    def getPages(self, subdir=None):
+        with conn_scope(self):
+            c = self.conn.cursor()
+            if subdir:
+                subdir = string.rstrip(subdir, '/') + '/%'
+                c.execute('''SELECT id, url, path, time, title, raw_text,
+                    formatted_text
+                    FROM pages WHERE url LIKE ?''',
+                    (subdir,))
+            else:
+                c.execute('''SELECT id, url, path, time, title, raw_text,
+                    formatted_text
+                    FROM pages''')
+            pages = []
+            for row in c.fetchall():
+                pages.append(self._getPage(row, c))
+            return pages
+
+    def getPage(self, url):
+        with conn_scope(self):
+            c = self.conn.cursor()
+            c.execute('''SELECT id, url, path, time, title, raw_text,
+                formatted_text FROM pages WHERE url=?''', (url,))
+            row = c.fetchone()
+            if row is None:
+                return None
+            return self._getPage(row, c)
+
+    def pageExists(self, url):
+        with conn_scope(self):
+            c = self.conn.cursor()
+            c.execute('''SELECT id FROM pages WHERE url=?''', (url,))
+            return c.fetchone() is not None
+
+    def getLinksTo(self, url):
+        with conn_scope(self):
+            c = self.conn.cursor()
+            c.execute('''SELECT source FROM links WHERE target=?''', (url,))
+            sources = []
+            for r in c.fetchall():
+                sources.append(r['source'])
+            return sources
+
+    def getConfigValues(self, section):
+        with conn_scope(self):
+            pass
+
+    def getConfigValue(self, section, name):
+        with conn_scope(self):
+            pass
+
+    def _createSchema(self):
+        self.logger.debug("Creating SQL schema...")
+        c = self.conn.cursor()
+        c.execute('''DROP TABLE IF EXISTS pages''')
+        c.execute('''CREATE TABLE pages
+            (id INTEGER PRIMARY KEY AUTOINCREMENT,
+             time TIMESTAMP,
+             url TEXT,
+             path TEXT,
+             title TEXT,
+             raw_text TEXT,
+             formatted_text TEXT)''')
+        c.execute('''DROP TABLE IF EXISTS includes''')
+        c.execute('''CREATE TABLE includes
+            (id INTEGER PRIMARY KEY AUTOINCREMENT,
+             source TEXT,
+             target TEXT)''')
+        c.execute('''DROP TABLE IF EXISTS links''')
+        c.execute('''CREATE TABLE links
+            (id INTEGER PRIMARY KEY AUTOINCREMENT,
+             source TEXT,
+             target TEXT)''')
+        c.execute('''DROP TABLE IF EXISTS meta''')
+        c.execute('''CREATE TABLE meta
+            (id INTEGER PRIMARY KEY AUTOINCREMENT,
+             page_id INTEGER,
+             name TEXT,
+             value TEXT)''')
+        c.execute('''DROP TABLE IF EXISTS config''')
+        c.execute('''CREATE TABLE config
+            (section TEXT,
+             name TEXT,
+             value TEXT)''')
+        c.execute('''DROP TABLE IF EXISTS info''')
+        c.execute('''CREATE TABLE info
+            (name TEXT UNIQUE NOT NULL,
+             str_value TEXT,
+             int_value INTEGER,
+             time_value TIMESTAMP)''')
+        c.execute('''INSERT INTO info (name, int_value) VALUES (?, ?)''',
+            ('schema_version', self.schema_version))
+        self.conn.commit()
+
+    def _getInfo(self, name, default=None):
+        with conn_scope(self):
+            c = self.conn.cursor()
+            c.execute('''SELECT name, str_value FROM info
+                WHERE name=?''', (name,))
+            row = c.fetchone()
+            if row is None:
+                return default
+            return row['str_value']
+
+    def _getInfoInt(self, name, default=None):
+        with conn_scope(self):
+            c = self.conn.cursor()
+            c.execute('''SELECT name, int_value FROM info
+                WHERE name=?''', (name,))
+            row = c.fetchone()
+            if row is None:
+                return default
+            return row['int_value']
+
+    def _getInfoTime(self, name, default=None):
+        with conn_scope(self):
+            c = self.conn.cursor()
+            c.execute('''SELECT name, time_value FROM info
+                WHERE name=?''', (name,))
+            row = c.fetchone()
+            if row is None:
+                return default
+            return row['time_value']
+
+    def _getSchemaVersion(self):
+        with conn_scope(self):
+            c = self.conn.cursor()
+            c.execute('''SELECT name FROM sqlite_master
+                WHERE type="table" AND name="info"''')
+            if c.fetchone() is None:
+                return 0
+            c.execute('''SELECT int_value FROM info
+                WHERE name="schema_version"''')
+            row = c.fetchone()
+            if row is None:
+                return 0
+            return row[0]
+
+    def _cacheConfig(self, config):
+        self.logger.debug("Re-caching configuration into SQL database.")
+        with conn_scope(self):
+            c = self.conn.cursor()
+            c.execute('''DELETE FROM config''')
+            for section in config.sections():
+                items = config.items(section)
+                for item in items:
+                    c.execute('''INSERT INTO config
+                        (section, name, value) VALUES (?, ?, ?)''',
+                        (section, item[0], item[1]))
+            c.execute('''INSERT OR REPLACE INTO info (name, time_value)
+                VALUES ("config_time", ?)''', (datetime.datetime.now(),))
+            self.conn.commit()
+
+    def _addPage(self, page, c):
+        self.logger.debug("Adding page '%s' to SQL database." % page.url)
+        now = datetime.datetime.now()
+        c.execute('''INSERT INTO pages
+            (time, url, path, title, raw_text, formatted_text)
+            VALUES (?, ?, ?, ?, ?, ?)''',
+            (now, page.url, page.path, page.title,
+                page.raw_text, page.formatted_text))
+        page_id = c.lastrowid
+
+        for name, value in page.local_meta.iteritems():
+            if isinstance(value, bool):
+                value = ""
+            if isinstance(value, types.StringTypes):
+                c.execute('''INSERT INTO meta
+                    (page_id, name, value) VALUES (?, ?, ?)''',
+                    (page_id, name, value))
+            else:
+                for v in value:
+                    c.execute('''INSERT INTO meta
+                        (page_id, name, value) VALUES (?, ?, ?)''',
+                        (page_id, name, v))
+
+        for link_url in page.local_links:
+            c.execute('''INSERT INTO links
+                (source, target) VALUES (?, ?)''',
+                (page.url, link_url))
+
+        for inc_url in page.local_includes:
+            c.execute('''INSERT INTO includes
+                (source, target) VALUES (?, ?)''',
+                (page.url, inc_url))
+
+    def _removePage(self, page_id, c):
+        c.execute('''SELECT url FROM pages WHERE id=?''', (page_id,))
+        row = c.fetchone()
+        self.logger.debug("Removing page '%s' [%d] from SQL database." %
+            (row['url'], page_id))
+        c.execute('''DELETE FROM pages WHERE id=?''', (page_id,))
+        c.execute('''DELETE FROM meta WHERE page_id=?''', (page_id,))
+        c.execute('''DELETE FROM links WHERE source=?''', (row['url'],))
+        c.execute('''DELETE FROM includes WHERE source=?''', (row['url'],))
+
+    def _getPage(self, row, c):
+        db_page = {
+            'url': row['url'],
+            'path': row['path'],
+            'time': row['time'],
+            'title': row['title'],
+            'content': row['raw_text'],
+            'formatted': row['formatted_text'],
+            'links': [],
+            'includes': [],
+            'meta': {}
+            }
+
+        c.execute('''SELECT target FROM links
+            WHERE source=?''', (row['url'],))
+        for r in c.fetchall():
+            db_page['links'].append(r['target'])
+
+        c.execute('''SELECT target FROM includes
+            WHERE source=?''', (row['url'],))
+        for r in c.fetchall():
+            db_page['includes'].append(r['target'])
+
+        c.execute('''SELECT page_id, name, value
+            FROM meta WHERE page_id=?''', (row['id'],))
+        for r in c.fetchall():
+            value = r['value']
+            if value == '':
+                value = True
+            name = r['name']
+            if name not in db_page['meta']:
+                db_page['meta'][name] = value
+            elif db_page['meta'][name] is list:
+                db_page['meta'][name].append(value)
+            else:
+                db_page['meta'][name] = [db_page['meta'][name], value]
+
+        return db_page
--- a/wikked/formatter.py	Fri Jan 25 22:20:58 2013 -0800
+++ b/wikked/formatter.py	Fri Jan 25 22:35:56 2013 -0800
@@ -7,14 +7,16 @@
     pass
 
 
-class PageFormattingContext(object):
-    def __init__(self, url, ext, slugify=None):
+class CircularIncludeError(Exception):
+    def __init__(self, message, url_trail):
+        Exception.__init__(self, message)
+        self.url_trail = url_trail
+
+
+class BaseContext(object):
+    def __init__(self, url, slugify=None):
         self.url = url
-        self.ext = ext
         self.slugify = slugify
-        self.out_links = []
-        self.included_pages = []
-        self.meta = {}
 
     @property
     def urldir(self):
@@ -31,10 +33,19 @@
             raw_abs_url = os.path.join(self.urldir, url)
             abs_url = os.path.normpath(raw_abs_url).replace('\\', '/')
         if self.slugify is not None:
-            abs_url = self.slugify(url)
+            abs_url = self.slugify(abs_url)
         return abs_url
 
 
+class FormattingContext(BaseContext):
+    def __init__(self, url, ext, slugify):
+        BaseContext.__init__(self, url, slugify)
+        self.ext = ext
+        self.out_links = []
+        self.included_pages = []
+        self.meta = {}
+
+
 class PageFormatter(object):
     def __init__(self, wiki):
         self.wiki = wiki
@@ -47,7 +58,6 @@
         return formatter(text)
 
     def _getFormatter(self, extension):
-        formatter = None
         for k, v in self.wiki.formatters.iteritems():
             if extension in v:
                 return k
@@ -63,7 +73,7 @@
 
     def _processWikiMeta(self, ctx, text):
         def repl(m):
-            meta_name = str(m.group(1))
+            meta_name = str(m.group(1)).lower()
             meta_value = str(m.group(3))
             if meta_value is not None and len(meta_value) > 0:
                 if meta_name not in ctx.meta:
@@ -71,7 +81,7 @@
                 elif ctx.meta[meta_name] is list:
                     ctx.meta[meta_name].append(meta_value)
                 else:
-                    ctx.meta[meta_name] = [ ctx.meta[meta_name], meta_value ]
+                    ctx.meta[meta_name] = [ctx.meta[meta_name], meta_value]
             else:
                 ctx.meta[meta_name] = True
             if meta_name == 'include':
@@ -80,7 +90,7 @@
                 return self._processQuery(ctx, meta_value)
             return ''
 
-        text = re.sub(r'^\[\[((__|\+)?[a-zA-Z][a-zA-Z0-9_\-]+):\s*(.*)\]\]\s*$', repl, text, flags=re.MULTILINE)
+        text = re.sub(r'^\{\{((__|\+)?[a-zA-Z][a-zA-Z0-9_\-]+):\s*(.*)\}\}\s*$', repl, text, flags=re.MULTILINE)
         return text
 
     def _processWikiLinks(self, ctx, text):
@@ -101,38 +111,127 @@
         return text
 
     def _processInclude(self, ctx, value):
-        # TODO: handle self-includes or cyclic includes.
-        abs_included_url = ctx.getAbsoluteUrl(value)
-        included_page = self.wiki.getPage(abs_included_url)
-        ctx.included_pages.append(abs_included_url)
-        return included_page.formatted_text
+        included_url = ctx.getAbsoluteUrl(value)
+        ctx.included_pages.append(included_url)
+        # Includes are run on the fly.
+        return '<div class="wiki-include">%s</div>\n' % included_url
 
     def _processQuery(self, ctx, query):
-        parameters = {
-                'header': "<ul>",
-                'footer': "</ul>",
-                'item': "<li><a class=\"wiki-link\" data-wiki-url=\"{{url}}\">{{title}}</a></li>",
-                'empty': "<p>No page matches the query.</p>"
-                }
+        # Queries are run on the fly.
+        return '<div class="wiki-query">%s</div>\n' % query
+
+    def _formatWikiLink(self, ctx, display, url):
+        abs_url = ctx.getAbsoluteUrl(url)
+        ctx.out_links.append(abs_url)
+
+        css_class = 'wiki-link'
+        if not self.wiki.pageExists(abs_url, from_db=False):
+            css_class += ' missing'
+        return '<a class="%s" data-wiki-url="%s">%s</a>' % (css_class, abs_url, display)
+
+
+class ResolvingContext(object):
+    def __init__(self):
+        self.url_trail = set()
+        self.meta = {}
+        self.out_links = []
+        self.included_pages = []
+
+    def add(self, ctx):
+        self.url_trail += ctx.url_trail
+        self.out_links += ctx.out_links
+        self.included_pages += ctx.included_pages
+        for original_key, val in ctx.meta.iteritems():
+            # Ignore internal properties. Strip include-only properties
+            # from their prefix.
+            key = original_key
+            if key[0:2] == '__':
+                continue
+            if key[0] == '+':
+                key = key[1:]
+
+            if key not in self.meta:
+                self.meta[key] = val
+            elif self.meta[key] is list:
+                self.meta[key].append(val)
+            else:
+                self.meta[key] = [self.meta[key], val]
+
+
+class PageResolver(object):
+    default_parameters = {
+        'header': "<ul>",
+        'footer': "</ul>",
+        'item': "<li><a class=\"wiki-link\" data-wiki-url=\"{{url}}\">" +
+            "{{title}}</a></li>",
+        'empty': "<p>No page matches the query.</p>"
+        }
+
+    def __init__(self, page, ctx):
+        self.page = page
+        self.ctx = ctx
+
+    @property
+    def wiki(self):
+        return self.page.wiki
+
+    def run(self):
+        def repl(m):
+            meta_name = str(m.group(1))
+            meta_value = str(m.group(2))
+            if meta_name == 'query':
+                return self._runQuery(meta_value)
+            elif meta_name == 'include':
+                return self._runInclude(meta_value)
+            return ''
+
+        self.ctx.url_trail = [self.page.url]
+        self.ctx.out_links = self.page.local_links
+        self.ctx.included_pages = self.page.local_includes
+        self.ctx.meta = self.page.local_meta
+
+        text = self.page.formatted_text
+        return re.sub(r'^<div class="wiki-([a-z]+)">(.*)</div>$', repl, text,
+            flags=re.MULTILINE)
+
+    def _runInclude(self, include_url):
+        if include_url in self.ctx.url_trail:
+            raise CircularIncludeError("Circular include detected at: %s" % include_url, self.ctx.url_trail)
+        page = self.wiki.getPage(include_url)
+        child_ctx = ResolvingContext()
+        child = PageResolver(page, child_ctx)
+        text = child.run()
+        self.ctx.add(child_ctx)
+        return text
+
+    def _runQuery(self, query):
+        # Parse the query.
+        parameters = dict(self.default_parameters)
         meta_query = {}
-        arg_pattern = r"(^|\|)(?P<name>[a-zA-Z][a-zA-Z0-9_\-]+)=(?P<value>[^\|]+)"
+        arg_pattern = r"(^|\|)(?P<name>[a-zA-Z][a-zA-Z0-9_\-]+)="\
+            r"(?P<value>[^\|]+)"
         for m in re.findall(arg_pattern, query):
-            if m[1] not in parameters:
-                meta_query[m[1]] = m[2]
+            key = m[1].lower()
+            if key not in parameters:
+                meta_query[key] = m[2]
             else:
-                parameters[m[1]] = m[2]
+                parameters[key] = m[2]
 
+        # Find pages that match the query, excluding any page
+        # that is in the URL trail.
         matched_pages = []
         for p in self.wiki.getPages():
-            if p.url == ctx.url:
+            if p.url in self.ctx.url_trail:
                 continue
             for key, value in meta_query.iteritems():
-                actual = p.getUserMeta(key)
-                if (type(actual) is list and value in actual) or (actual == value):
+                if self._isPageMatch(p, key, value):
                     matched_pages.append(p)
+
+        # No match: return the 'empty' template.
         if len(matched_pages) == 0:
             return parameters['empty']
 
+        # Combine normal templates to build the output.
         text = parameters['header']
         for p in matched_pages:
             item_str = parameters['item']
@@ -147,12 +246,23 @@
 
         return text
 
-    def _formatWikiLink(self, ctx, display, url):
-        abs_url = ctx.getAbsoluteUrl(url)
-        ctx.out_links.append(abs_url)
+    def _isPageMatch(self, page, name, value, level=0):
+        # Check the page's local meta properties.
+        actual = page.local_meta.get(name)
+        if ((type(actual) is list and value in actual) or
+            (actual == value)):
+            return True
 
-        css_class = 'wiki-link'
-        if not self.wiki.pageExists(abs_url):
-            css_class += ' missing'
-        return '<a class="%s" data-wiki-url="%s">%s</a>' % (css_class, abs_url, display)
+        # If this is an include, also look for 'include-only'
+        # meta properties.
+        if level > 0:
+            actual = page.local_meta.get('+' + name)
+            if ((type(actual) is list and value in actual) or
+                (actual == value)):
+                return True
 
+        # Recurse into included pages.
+        for url in page.local_includes:
+            p = self.wiki.getPage(url)
+            if self._isPageMatch(p, name, value, level + 1):
+                return True
--- a/wikked/fs.py	Fri Jan 25 22:20:58 2013 -0800
+++ b/wikked/fs.py	Fri Jan 25 22:35:56 2013 -0800
@@ -53,13 +53,9 @@
         path = self.getPhysicalPagePath(url)
         with codecs.open(path, 'r', encoding='utf-8') as f:
             content = f.read()
-        name = os.path.basename(path)
-        name_split = os.path.splitext(name)
         return {
                 'url': url,
                 'path': path,
-                'name': name_split[0],
-                'ext': name_split[1].lstrip('.'),
                 'content': content
                 }
 
@@ -85,9 +81,7 @@
         url = self.slugify(name)
         return {
                 'url': url,
-                'path': path,
-                'name': name,
-                'ext': ext
+                'path': path
                 }
 
     def getPhysicalPagePath(self, url):
--- a/wikked/indexer.py	Fri Jan 25 22:20:58 2013 -0800
+++ b/wikked/indexer.py	Fri Jan 25 22:35:56 2013 -0800
@@ -3,7 +3,7 @@
 import codecs
 import logging
 from whoosh.index import create_in, open_dir
-from whoosh.fields import Schema, ID, KEYWORD, TEXT, STORED
+from whoosh.fields import Schema, ID, TEXT, STORED
 from whoosh.qparser import QueryParser
 
 
@@ -14,7 +14,10 @@
         if logger is None:
             self.logger = logging.getLogger('wikked.index')
 
-    def open(self):
+    def initIndex(self):
+        raise NotImplementedError()
+
+    def reset(self, pages):
         raise NotImplementedError()
 
     def update(self, pages):
@@ -28,24 +31,16 @@
     def __init__(self, store_dir, logger=None):
         WikiIndex.__init__(self, store_dir, logger)
 
-    def open(self):
+    def initIndex(self):
         if not os.path.isdir(self.store_dir):
+            self.logger.debug("Creating new index in: " + self.store_dir)
             os.makedirs(self.store_dir)
             self.ix = create_in(self.store_dir, self._getSchema())
         else:
             self.ix = open_dir(self.store_dir)
 
-    def _getSchema(self):
-        schema = Schema(
-                url=ID(stored=True), 
-                title=TEXT(stored=True), 
-                content=TEXT,
-                path=STORED,
-                time=STORED
-                )
-        return schema
-    
     def reset(self, pages):
+        self.logger.debug("Re-creating new index in: " + self.store_dir)
         self.ix = create_in(self.store_dir, schema=self._getSchema())
         writer = self.ix.writer()
         for page in pages:
@@ -54,6 +49,7 @@
         writer.commit()
 
     def update(self, pages):
+        self.logger.debug("Updating index...")
         to_reindex = set()
         already_indexed = set()
 
@@ -67,12 +63,12 @@
 
                 if not os.path.isfile(indexed_path):
                     # File was deleted.
-                    writer.delete_by_term('url', indexed_url)
+                    self._unindexPage(writer, indexed_url)
                 else:
                     already_indexed.add(indexed_path)
-                    if os.path.getmtime(indexed_path) > fields['time']:
-                        # File as changed since last index.
-                        writer.delete_by_term('url', indexed_url)
+                    if os.path.getmtime(indexed_path) > indexed_time:
+                        # File has changed since last index.
+                        self._unindexPage(writer, indexed_url)
                         to_reindex.add(indexed_path)
 
             for page in pages:
@@ -82,6 +78,7 @@
                     self._indexPage(writer, page)
 
             writer.commit()
+        self.logger.debug("...done updating index.")
 
     def search(self, query):
         with self.ix.searcher() as searcher:
@@ -103,8 +100,18 @@
                 page_infos.append(page_info)
             return page_infos
 
+    def _getSchema(self):
+        schema = Schema(
+                url=ID(stored=True),
+                title=TEXT(stored=True),
+                content=TEXT,
+                path=STORED,
+                time=STORED
+                )
+        return schema
+
     def _indexPage(self, writer, page):
-        self.logger.debug("Indexing: %s" % page.url)
+        self.logger.debug("Indexing '%s'." % page.url)
         writer.add_document(
             url=unicode(page.url),
             title=unicode(page.title),
@@ -113,3 +120,6 @@
             time=os.path.getmtime(page._meta['path'])
             )
 
+    def _unindexPage(self, writer, url):
+        self.logger.debug("Removing '%s' from index." % url)
+        writer.delete_by_term('url', url)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/wikked/page.py	Fri Jan 25 22:35:56 2013 -0800
@@ -0,0 +1,205 @@
+import os
+import os.path
+import re
+import datetime
+import unicodedata
+from formatter import (
+    PageFormatter, FormattingContext,
+    PageResolver, ResolvingContext
+    )
+
+
+class Page(object):
+    """ A wiki page.
+    """
+    def __init__(self, wiki, url):
+        self.wiki = wiki
+        self.url = url
+        self._meta = None
+        self._ext_meta = None
+
+    @property
+    def path(self):
+        self._ensureMeta()
+        return self._meta['path']
+
+    @property
+    def title(self):
+        self._ensureMeta()
+        return self._meta['title']
+
+    @property
+    def raw_text(self):
+        self._ensureMeta()
+        return self._meta['content']
+
+    @property
+    def formatted_text(self):
+        self._ensureMeta()
+        return self._meta['formatted']
+
+    @property
+    def text(self):
+        self._ensureExtendedMeta()
+        return self._ext_meta['text']
+
+    @property
+    def local_meta(self):
+        self._ensureMeta()
+        return self._meta['meta']
+
+    @property
+    def local_links(self):
+        self._ensureMeta()
+        return self._meta['links']
+
+    @property
+    def local_includes(self):
+        self._ensureMeta()
+        return self._meta['includes']
+
+    @property
+    def all_meta(self):
+        self._ensureExtendedMeta()
+        return self._ext_meta['meta']
+
+    @property
+    def all_links(self):
+        self._ensureExtendedMeta()
+        return self._ext_meta['links']
+
+    @property
+    def all_includes(self):
+        self._ensureExtendedMeta()
+        return self._ext_meta['includes']
+
+    @property
+    def in_links(self):
+        return self.wiki.db.getLinksTo(self.url)
+
+    def getHistory(self):
+        return self.wiki.scm.getHistory(self.path)
+
+    def getState(self):
+        return self.wiki.scm.getState(self.path)
+
+    def getRevision(self, rev):
+        return self.wiki.scm.getRevision(self.path, rev)
+
+    def getDiff(self, rev1, rev2):
+        return self.wiki.scm.diff(self.path, rev1, rev2)
+
+    def _ensureMeta(self):
+        if self._meta is not None:
+            return
+
+        self._meta = self._loadCachedMeta()
+        if self._meta is not None:
+            return
+
+        self._meta = self._loadOriginalMeta()
+        self._saveCachedMeta(self._meta)
+
+    def _loadCachedMeta(self):
+        return None
+
+    def _saveCachedMeta(self, meta):
+        pass
+
+    def _loadOriginalMeta(self):
+        # Get info from the file-system.
+        meta = self.wiki.fs.getPage(self.url)
+
+        # Format the page and get the meta properties.
+        filename = os.path.basename(meta['path'])
+        filename_split = os.path.splitext(filename)
+        extension = filename_split[1].lstrip('.')
+        ctx = FormattingContext(self.url, extension, slugify=Page.title_to_url)
+        f = PageFormatter(self.wiki)
+        meta['formatted'] = f.formatText(ctx, meta['content'])
+        meta['meta'] = ctx.meta
+        meta['links'] = ctx.out_links
+        meta['includes'] = ctx.included_pages
+
+        # Add some common meta.
+        meta['title'] = re.sub(r'\-', ' ', filename_split[0])
+        if 'title' in meta['meta']:
+            meta['title'] = meta['meta']['title']
+
+        return meta
+
+    def _ensureExtendedMeta(self):
+        if self._ext_meta is not None:
+            return
+
+        self._ext_meta = {}
+        ctx = ResolvingContext()
+        r = PageResolver(self, ctx)
+        self._ext_meta['text'] = r.run()
+        self._ext_meta['meta'] = ctx.meta
+        self._ext_meta['links'] = ctx.out_links
+        self._ext_meta['includes'] = ctx.included_pages
+
+    @staticmethod
+    def title_to_url(title):
+        # Remove diacritics (accents, etc.) and replace them with ASCII
+        # equivelent.
+        ansi_title = ''.join((c for c in
+            unicodedata.normalize('NFD', unicode(title))
+            if unicodedata.category(c) != 'Mn'))
+        # Now replace spaces and punctuation with a hyphen.
+        return re.sub(r'[^A-Za-z0-9_\.\-\(\)/]+', '-', ansi_title.lower())
+
+    @staticmethod
+    def url_to_title(url):
+        def upperChar(m):
+            return m.group(0).upper()
+        return re.sub(r'^.|\s\S', upperChar, url.lower().replace('-', ' '))
+
+    @staticmethod
+    def factory(wiki, url):
+        return Page(wiki, url)
+
+
+class DatabasePage(Page):
+    """ A page that can load its properties from a
+        database.
+    """
+    def __init__(self, wiki, url):
+        Page.__init__(self, wiki, url)
+        if getattr(wiki, 'db', None) is None:
+            raise Exception("The wiki doesn't have a database.")
+        self.auto_update = wiki.config.get('wiki', 'auto_update')
+
+    def _loadCachedMeta(self):
+        if self.wiki.db is None:
+            return None
+        db_page = self.wiki.db.getPage(self.url)
+        if db_page is None:
+            return None
+        if self.auto_update:
+            path_time = datetime.datetime.fromtimestamp(
+                os.path.getmtime(db_page['path']))
+            if path_time >= db_page['time']:
+                return None
+        meta = {
+                'url': self.url,
+                'path': db_page['path'],
+                'content': db_page['content'],
+                'formatted': db_page['formatted'],
+                'meta': db_page['meta'],
+                'title': db_page['title'],
+                'links': db_page['links'],
+                'includes': db_page['includes']
+                }
+        return meta
+
+    def _saveCachedMeta(self, meta):
+        if self.wiki.db is not None:
+            self.wiki.logger.debug(
+                "Updated database cache for page '%s'." % self.url)
+            self.wiki.db.update([self])
+
+    @staticmethod
+    def factory(wiki, url):
+        return DatabasePage(wiki, url)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/wikked/resources/defaults.cfg	Fri Jan 25 22:35:56 2013 -0800
@@ -0,0 +1,3 @@
+[wiki]
+scm=hg
+auto_update=False
--- a/wikked/settings.py	Fri Jan 25 22:20:58 2013 -0800
+++ b/wikked/settings.py	Fri Jan 25 22:35:56 2013 -0800
@@ -1,4 +1,3 @@
 
 SECRET_KEY = '\xef)*\xbc\xd7\xa9t\x7f\xbc3pH1o\xc1\xe2\xb0\x19\\L\xeb\xe3\x00\xa3'
 DEBUG = True
-
--- a/wikked/views.py	Fri Jan 25 22:20:58 2013 -0800
+++ b/wikked/views.py	Fri Jan 25 22:35:56 2013 -0800
@@ -1,18 +1,12 @@
 import time
 import os.path
-from flask import (
-        Response, 
-        render_template, url_for, redirect, abort, request, flash,
-        jsonify
-        )
-from flask.ext.login import login_required, login_user, logout_user, current_user
+from flask import render_template, abort, request, g, jsonify
+from flask.ext.login import login_user, logout_user, current_user
 from pygments import highlight
 from pygments.lexers import get_lexer_by_name
 from pygments.formatters import get_formatter_by_name
-from web import app, wiki
+from web import app, login_manager
 from wiki import Page
-from auth import User
-from forms import RegistrationForm, EditPageForm
 from fs import PageNotFoundError
 import scm
 
@@ -21,15 +15,20 @@
 CHECK_FOR_READ = 1
 CHECK_FOR_WRITE = 2
 
+COERCE_META = {
+    'redirect': Page.title_to_url
+    }
+
 
 def get_page_or_none(url):
     try:
-        page = wiki.getPage(url)
+        page = g.wiki.getPage(url)
         page._ensureMeta()
         return page
     except PageNotFoundError:
         return None
 
+
 def get_page_or_404(url, check_perms=DONT_CHECK):
     page = get_page_or_none(url)
     if page is not None:
@@ -49,6 +48,19 @@
     return page.wiki.auth.isPageWritable(page, user.get_id())
 
 
+def get_page_meta(page, local_only=False):
+    if local_only:
+        meta = dict(page.local_meta)
+    else:
+        meta = dict(page.all_meta)
+    meta['title'] = page.title
+    meta['url'] = page.url
+    for name in COERCE_META:
+        if name in meta:
+            meta[name] = COERCE_META(meta[name])
+    return meta
+
+
 def get_history_data(history):
     hist_data = []
     for i, rev in enumerate(reversed(history)):
@@ -62,10 +74,10 @@
             'pages': []
             }
         for f in rev.files:
-            f_info = wiki.fs.getPageInfo(f['path'])
+            f_info = g.wiki.fs.getPageInfo(f['path'])
             if f_info is None:
                 continue
-            page = wiki.getPage(f_info['url'])
+            page = g.wiki.getPage(f_info['url'])
             if not is_page_readable(page):
                 continue
             rev_data['pages'].append({
@@ -79,7 +91,7 @@
 
 def make_auth_response(data):
     if current_user.is_authenticated():
-        data['auth'] = { 
+        data['auth'] = {
                 'username': current_user.username,
                 'is_admin': current_user.is_admin()
                 }
@@ -103,28 +115,28 @@
 
 @app.route('/api/list')
 def api_list_all_pages():
-    return list_pages(None)
+    return api_list_pages(None)
 
 
 @app.route('/api/list/<path:url>')
 def api_list_pages(url):
-    pages = filter(is_page_readable, wiki.getPages(url))
-    page_metas = [page.all_meta for page in pages]
-    result = { 'path': url, 'pages': list(page_metas) }
+    pages = filter(is_page_readable, g.wiki.getPages(url))
+    page_metas = [get_page_meta(page) for page in pages]
+    result = {'path': url, 'pages': list(page_metas)}
     return make_auth_response(result)
 
 
 @app.route('/api/read/<path:url>')
 def api_read_page(url):
     page = get_page_or_404(url, CHECK_FOR_READ)
-    result = { 'path': url, 'meta': page.all_meta, 'text': page.formatted_text }
+    result = {'meta': get_page_meta(page), 'text': page.text}
     return make_auth_response(result)
 
 
 @app.route('/api/raw/<path:url>')
 def api_read_page_raw(url):
     page = get_page_or_404(url, CHECK_FOR_READ)
-    result = { 'path': url, 'meta': page.all_meta, 'text': page.raw_text }
+    result = {'meta': get_page_meta(page), 'text': page.raw_text}
     return make_auth_response(result)
 
 
@@ -135,8 +147,8 @@
         abort(400)
     page = get_page_or_404(url, CHECK_FOR_READ)
     page_rev = page.getRevision(rev)
-    meta = dict(page.all_meta, rev=rev)
-    result = { 'path': url, 'meta': meta, 'text': page_rev }
+    meta = dict(get_page_meta(page, True), rev=rev)
+    result = {'meta': meta, 'text': page_rev}
     return make_auth_response(result)
 
 
@@ -153,10 +165,10 @@
         formatter = get_formatter_by_name('html')
         diff = highlight(diff, lexer, formatter)
     if rev2 is None:
-        meta = dict(page.all_meta, change=rev1)
+        meta = dict(get_page_meta(page, True), change=rev1)
     else:
-        meta = dict(page.all_meta, rev1=rev1, rev2=rev2)
-    result = { 'path': url, 'meta': meta, 'diff': diff }
+        meta = dict(get_page_meta(page, True), rev1=rev1, rev2=rev2)
+    result = {'meta': meta, 'diff': diff}
     return make_auth_response(result)
 
 
@@ -164,10 +176,9 @@
 def api_get_state(url):
     page = get_page_or_404(url, CHECK_FOR_READ)
     state = page.getState()
-    return make_auth_response({ 
-        'path': url, 
-        'meta': page.all_meta, 
-        'state': scm.STATE_NAMES[state] 
+    return make_auth_response({
+        'meta': get_page_meta(page, True),
+        'state': scm.STATE_NAMES[state]
         })
 
 
@@ -183,9 +194,9 @@
                 'title': other.title
                 })
         else:
-            links.append({ 'url': link, 'missing': True })
+            links.append({'url': link, 'missing': True})
 
-    result = { 'path': url, 'meta': page.all_meta, 'out_links': links }
+    result = {'meta': get_page_meta(page, True), 'out_links': links}
     return make_auth_response(result)
 
 
@@ -198,12 +209,12 @@
         if other is not None and is_page_readable(other):
             links.append({
                 'url': link,
-                'meta': other.all_meta
+                'title': other.title
                 })
         else:
-            links.append({ 'url': link, 'missing': True })
+            links.append({'url': link, 'missing': True})
 
-    result = { 'path': url, 'meta': page.all_meta, 'in_links': links }
+    result = {'meta': get_page_meta(page, True), 'in_links': links}
     return make_auth_response(result)
 
 
@@ -213,21 +224,18 @@
         page = get_page_or_none(url)
         if page is None:
             result = {
-                    'path': url,
                     'meta': {
                         'url': url,
                         'name': os.path.basename(url),
-                        'title': Page.url_to_title(url),
-                        'user': {}
+                        'title': Page.url_to_title(url)
                         },
                     'text': ''
                     }
         else:
             if not is_page_writable(page):
                 abort(401)
-            result = { 
-                    'path': url, 
-                    'meta': page.all_meta, 
+            result = {
+                    'meta': get_page_meta(page, True),
                     'text': page.raw_text
                     }
         result['commit_meta'] = {
@@ -253,8 +261,8 @@
             'author': author,
             'message': message
             }
-    wiki.setPage(url, page_fields)
-    result = { 'path': url, 'saved': 1 }
+    g.wiki.setPage(url, page_fields)
+    result = {'saved': 1}
     return make_auth_response(result)
 
 
@@ -271,18 +279,18 @@
 @app.route('/api/orphans')
 def api_special_orphans():
     orphans = []
-    for page in filter(is_page_readable, wiki.getPages()):
+    for page in filter(is_page_readable, g.wiki.getPages()):
         if len(page.in_links) == 0:
-            orphans.append({ 'path': page.url, 'meta': page.all_meta })
-    result = { 'orphans': orphans }
+            orphans.append({'path': page.url, 'meta': get_page_meta(page, True)})
+    result = {'orphans': orphans}
     return make_auth_response(result)
 
 
 @app.route('/api/history')
 def api_site_history():
-    history = wiki.getHistory()
+    history = g.wiki.getHistory()
     hist_data = get_history_data(history)
-    result = { 'history': hist_data }
+    result = {'history': hist_data}
     return make_auth_response(result)
 
 
@@ -291,18 +299,19 @@
     page = get_page_or_404(url, CHECK_FOR_READ)
     history = page.getHistory()
     hist_data = get_history_data(history)
-    result = { 'url': url, 'meta': page.all_meta, 'history': hist_data }
+    result = {'url': url, 'meta': get_page_meta(page, True), 'history': hist_data}
     return make_auth_response(result)
 
 
 @app.route('/api/search')
 def api_search():
     query = request.args.get('q')
+
     def is_hit_readable(hit):
         page = get_page_or_none(hit['url'])
         return page is None or is_page_readable(page)
-    hits = filter(is_hit_readable, wiki.index.search(query))
-    result = { 'query': query, 'hits': hits }
+    hits = filter(is_hit_readable, g.wiki.index.search(query))
+    result = {'query': query, 'hits': hits}
     return make_auth_response(result)
 
 
@@ -310,8 +319,8 @@
 def api_admin_reindex():
     if not current_user.is_authenticated() or not current_user.is_admin():
         return login_manager.unauthorized()
-    wiki.index.reset(wiki.getPages())
-    result = { 'ok': 1 }
+    g.wiki.index.reset(g.wiki.getPages())
+    result = {'ok': 1}
     return make_auth_response(result)
 
 
@@ -321,11 +330,11 @@
     password = request.form.get('password')
     remember = request.form.get('remember')
 
-    user = wiki.auth.getUser(username)
+    user = g.wiki.auth.getUser(username)
     if user is not None:
         if app.bcrypt.check_password_hash(user.password, password):
             login_user(user, remember=bool(remember))
-            result = { 'username': username, 'logged_in': 1 }
+            result = {'username': username, 'logged_in': 1}
             return make_auth_response(result)
     abort(401)
 
@@ -333,7 +342,7 @@
 @app.route('/api/user/is_logged_in')
 def api_user_is_logged_in():
     if current_user.is_authenticated():
-        result = { 'logged_in': True }
+        result = {'logged_in': True}
         return make_auth_response(result)
     abort(401)
 
@@ -341,15 +350,14 @@
 @app.route('/api/user/logout', methods=['POST'])
 def api_user_logout():
     logout_user()
-    result = { 'ok': 1 }
+    result = {'ok': 1}
     return make_auth_response(result)
 
 
 @app.route('/api/user/info/<name>')
 def api_user_info(name):
-    user = wiki.auth.getUser(name)
+    user = g.wiki.auth.getUser(name)
     if user is not None:
-        result = { 'username': user.username, 'groups': user.groups }
+        result = {'username': user.username, 'groups': user.groups}
         return make_auth_response(result)
     abort(404)
-
--- a/wikked/web.py	Fri Jan 25 22:20:58 2013 -0800
+++ b/wikked/web.py	Fri Jan 25 22:35:56 2013 -0800
@@ -1,10 +1,36 @@
-from flask import Flask, abort
+from flask import Flask, abort, g
+from wiki import Wiki
 
 # Create the main app.
-app = Flask(__name__)
+app = Flask("wikked")
 app.config.from_object('wikked.settings')
 app.config.from_envvar('WIKKED_SETTINGS', silent=True)
 
+
+def create_wiki():
+    wiki = Wiki(root=app.config.get('WIKI_ROOT'), logger=app.logger)
+    wiki.start()
+    return wiki
+
+wiki = create_wiki()
+
+
+# Set the wiki as a request global, and open/close the database.
+@app.before_request
+def before_request():
+    if getattr(wiki, 'db', None):
+        wiki.db.open()
+    g.wiki = wiki
+
+
+@app.teardown_request
+def teardown_request(exception):
+    if wiki is not None:
+        if getattr(wiki, 'db', None):
+            wiki.db.close()
+
+
+# Make is serve static content in DEBUG mode.
 if app.config['DEBUG']:
     from werkzeug import SharedDataMiddleware
     import os
@@ -12,22 +38,32 @@
       '/': os.path.join(os.path.dirname(__file__), 'static')
     })
 
-# The main Wiki instance.
-from wiki import Wiki
-wiki = Wiki(root=app.config.get('WIKI_ROOT'), logger=app.logger)
-wiki.start()
 
-# Import views and user loader.
-import views
+# Customize logging.
+if app.config.get('LOG_FORMAT'):
+    import logging
+    handler = logging.StreamHandler()
+    handler.setLevel(logging.DEBUG)
+    handler.setFormatter(logging.Formatter(app.config['LOG_FORMAT']))
+    app.logger.handlers = []
+    app.logger.addHandler(handler)
+
 
 # Login extension.
+def user_loader(username):
+    return g.wiki.auth.getUser(username)
+
 from flask.ext.login import LoginManager
 login_manager = LoginManager()
 login_manager.init_app(app)
-login_manager.user_loader(wiki.auth.getUser)
+login_manager.user_loader(user_loader)
 login_manager.unauthorized_handler(lambda: abort(401))
 
+
 # Bcrypt extension.
 from flaskext.bcrypt import Bcrypt
 app.bcrypt = Bcrypt(app)
 
+
+# Import the views.
+import views
--- a/wikked/wiki.py	Fri Jan 25 22:20:58 2013 -0800
+++ b/wikked/wiki.py	Fri Jan 25 22:35:56 2013 -0800
@@ -1,178 +1,25 @@
 import os
 import os.path
-import re
 import time
 import logging
 import itertools
-import unicodedata
 from ConfigParser import SafeConfigParser
 import markdown
 import textile
 import creole
+from page import Page, DatabasePage
+from cache import Cache
 from fs import FileSystem
-from cache import Cache
+from db import SQLiteDatabase
 from scm import MercurialSourceControl
 from indexer import WhooshWikiIndex
 from auth import UserManager
-from formatter import PageFormatter, PageFormattingContext
 
 
 class InitializationError(Exception):
     pass
 
 
-class Page(object):
-    def __init__(self, wiki, url):
-        self.wiki = wiki
-        self.url = url
-        self._meta = None
-
-        self._promoted_meta = [
-                'title',
-                'redirect',
-                'notitle'
-                ]
-        self._coerce_promoted_meta = {
-                'redirect': Page.title_to_url
-                }
-
-    @property
-    def title(self):
-        self._ensureMeta()
-        return self._meta['title']
-
-    @property
-    def raw_text(self):
-        if self._meta is not None:
-            return self._meta['content']
-        page = self.wiki.fs.getPage(self.url)
-        return page['content']
-
-    @property
-    def formatted_text(self):
-        self._ensureMeta()
-        return self._meta['formatted']
-
-    @property
-    def out_links(self):
-        self._ensureMeta()
-        return self._meta['out_links']
-
-    @property
-    def in_links(self):
-        links = []
-        for other_url in self.wiki.getPageUrls():
-            if other_url == self.url:
-                continue
-            other_page = Page(self.wiki, other_url)
-            for l in other_page.out_links:
-                if l == self.url:
-                    links.append(other_url)
-        return links
-
-    @property
-    def all_meta(self):
-        self._ensureMeta()
-        meta = {
-                'url': self._meta['url'],
-                'name': self._meta['name'],
-                'title': self._meta['title'],
-                'user': self._meta['user']
-                }
-        for name in self._promoted_meta:
-            if name in self._meta['user']:
-                meta[name] = self._meta['user'][name]
-                if name in self._coerce_promoted_meta:
-                    meta[name] = self._coerce_promoted_meta[name](meta[name])
-        return meta
-
-    def getUserMeta(self, key):
-        self._ensureMeta()
-        return self._meta['user'].get(key)
-
-    def getHistory(self):
-        self._ensureMeta()
-        return self.wiki.scm.getHistory(self._meta['path'])
-
-    def getState(self):
-        self._ensureMeta()
-        return self.wiki.scm.getState(self._meta['path'])
-
-    def getRevision(self, rev):
-        self._ensureMeta()
-        return self.wiki.scm.getRevision(self._meta['path'], rev)
-
-    def getDiff(self, rev1, rev2):
-        self._ensureMeta()
-        return self.wiki.scm.diff(self._meta['path'], rev1, rev2)
-
-    def _ensureMeta(self):
-        if self._meta is not None:
-            return
-
-        cache_key = self.url + '.info.cache'
-        cached_meta = self._getCached(cache_key)
-        if cached_meta is not None:
-            # We have a valid cache for our content, but if we are including
-            # other pages, we need to check if they have changed since last
-            # time.
-            base_url = os.path.dirname(self.url)
-            for included_url in cached_meta['included_pages']:
-                included_path = self.wiki.fs.getPhysicalPagePath(included_url)
-                included_time = os.path.getmtime(included_path)
-                included_cache_key = included_url + '.info.cache'
-                if not self.wiki.cache.isValid(included_cache_key, included_time):
-                    break
-            else:
-                self._meta = cached_meta
-                return
-
-        self._meta = self.wiki.fs.getPage(self.url)
-
-        ext = self._meta['ext']
-        if ext[0] == '.':
-            ext = ext[1:]
-        ctx = PageFormattingContext(self.url, ext, slugify=Page.title_to_url)
-        f = PageFormatter(self.wiki)
-        self._meta['formatted'] = f.formatText(ctx, self._meta['content'])
-        self._meta['user'] = ctx.meta
-
-        self._meta['title'] = re.sub(r'\-', ' ', self._meta['name'])
-        for name in self._promoted_meta:
-            if name in ctx.meta:
-                self._meta[name] = ctx.meta[name]
-
-        self._meta['out_links'] = ctx.out_links
-        self._meta['included_pages'] = ctx.included_pages
-
-        self._putCached(cache_key, self._meta)
-
-    def _getCached(self, cache_key):
-        if self.wiki.cache is not None:
-            page_path = self.wiki.fs.getPhysicalPagePath(self.url)
-            page_time = os.path.getmtime(page_path)
-            return self.wiki.cache.read(cache_key, page_time)
-        return None
-
-    def _putCached(self, cache_key, data):
-        if self.wiki.cache is not None:
-            self.wiki.logger.debug("Updated cached %s for page '%s'." % (cache_key, self.url))
-            self.wiki.cache.write(cache_key, data)
-
-    @staticmethod
-    def title_to_url(title):
-        # Remove diacritics (accents, etc.) and replace them with ASCII equivelent.
-        ansi_title = ''.join((c for c in unicodedata.normalize('NFD', title) if unicodedata.category(c) != 'Mn'))
-        # Now replace spaces and punctuation with a hyphen.
-        return re.sub(r'[^A-Za-z0-9_\.\-\(\)/]+', '-', ansi_title.lower())
-
-    @staticmethod
-    def url_to_title(url):
-        def upperChar(m):
-            return m.group(0).upper()
-        return re.sub(r'^.|\s\S', upperChar, url.lower().replace('-', ' '))
-
-
 class Wiki(object):
     def __init__(self, root=None, logger=None):
         if root is None:
@@ -183,71 +30,96 @@
             self.logger = logging.getLogger('wikked.wiki')
         self.logger.debug("Initializing wiki at: " + root)
 
+        self.page_factory = DatabasePage.factory
+        self.use_db = True
         self.formatters = {
-                markdown.markdown: [ 'md', 'mdown', 'markdown' ],
-                textile.textile: [ 'tl', 'text', 'textile' ],
-                creole.creole2html: [ 'cr', 'creole' ],
-                self._passthrough: [ 'txt', 'html' ]
+                markdown.markdown: ['md', 'mdown', 'markdown'],
+                textile.textile: ['tl', 'text', 'textile'],
+                creole.creole2html: ['cr', 'creole'],
+                self._passthrough: ['txt', 'html']
                 }
 
-        self.config = SafeConfigParser()
-        config_path = os.path.join(root, '.wikirc')
-        if os.path.isfile(config_path):
-            self.config.read(config_path)
+        self.default_config_path = os.path.join(
+            os.path.dirname(__file__), 'resources', 'defaults.cfg')
+        self.config_path = os.path.join(root, '.wikirc')
+        self.config = self._loadConfig()
 
         self.fs = FileSystem(root, slugify=Page.title_to_url)
         self.auth = UserManager(self.config, logger=self.logger)
-        self.index = WhooshWikiIndex(os.path.join(root, '.index'), logger=self.logger)
+        self.index = WhooshWikiIndex(os.path.join(root, '.wiki', 'index'),
+            logger=self.logger)
+        self.db = SQLiteDatabase(self, logger=self.logger)
+        self.scm = self._createScm()
+        self.cache = self._createJsonCache()
 
-        scm_type = 'hg'
-        if self.config.has_option('wiki', 'scm'):
-            scm_type = self.config.get('wiki', 'scm')
+        self.fs.page_extensions = list(set(
+            itertools.chain(*self.formatters.itervalues())))
+        self.fs.excluded.append(self.config_path)
+        self.fs.excluded.append(os.path.join(root, '.wiki'))
+        if self.scm is not None:
+            self.fs.excluded += self.scm.getSpecialDirs()
+
+    def _createScm(self):
+        scm_type = self.config.get('wiki', 'scm')
         if scm_type == 'hg':
-            self.scm = MercurialSourceControl(root, self.logger)
+            return MercurialSourceControl(self.fs.root, self.logger)
         else:
             raise InitializationError("No such source control: " + scm_type)
 
+    def _createJsonCache(self):
         if (not self.config.has_option('wiki', 'cache') or
                 self.config.getboolean('wiki', 'cache')):
-            self.cache = Cache(os.path.join(root, '.cache'))
+            return Cache(os.path.join(self.fs.root, '.wiki', 'cache'))
         else:
-            self.cache = None
+            return None
 
-        self.fs.page_extensions = list(set(itertools.chain(*self.formatters.itervalues())))
-        self.fs.excluded.append(config_path)
-        if self.scm is not None:
-            self.fs.excluded += self.scm.getSpecialDirs()
-        if self.cache is not None:
-            self.fs.excluded.append(self.cache.cache_dir)
-        if self.index is not None:
-            self.fs.excluded.append(self.index.store_dir)
+    def _loadConfig(self):
+        config = SafeConfigParser()
+        config.readfp(open(self.default_config_path))
+        config.read(self.config_path)
+        return config
 
-    def start(self):
+    def start(self, update=True):
         if self.scm is not None:
             self.scm.initRepo()
         if self.index is not None:
-            self.index.open()
+            self.index.initIndex()
+        if self.db is not None:
+            self.db.initDb()
+
+        if update:
+            pass
 
     @property
     def root(self):
         return self.fs.root
 
-    def getPageUrls(self, subdir=None):
-        for info in self.fs.getPageInfos(subdir):
-            yield info['url']
+    def getPageUrls(self, subdir=None, from_db=True):
+        if from_db and self.db:
+            for url in self.db.getPageUrls(subdir):
+                yield url
+        else:
+            for info in self.fs.getPageInfos(subdir):
+                yield info['url']
 
-    def getPages(self, subdir=None):
-        for url in self.getPageUrls(subdir):
-            yield Page(self, url)
+    def getPages(self, subdir=None, from_db=True, factory=None):
+        if factory is None:
+            factory = self.page_factory
+        for url in self.getPageUrls(subdir, from_db):
+            yield factory(self, url)
 
-    def getPage(self, url):
-        return Page(self, url)
+    def getPage(self, url, factory=None):
+        if factory is None:
+            factory = self.page_factory
+        return factory(self, url)
 
     def setPage(self, url, page_fields):
         if 'author' not in page_fields:
-            raise ValueError("No author specified for editing page '%s'." % url)
+            raise ValueError(
+                "No author specified for editing page '%s'." % url)
         if 'message' not in page_fields:
-            raise ValueError("No commit message specified for editing page '%s'." % url)
+            raise ValueError(
+                "No commit message specified for editing page '%s'." % url)
 
         do_commit = False
         path = self.fs.getPhysicalPagePath(url)
@@ -262,17 +134,39 @@
                     'author': page_fields['author'],
                     'message': page_fields['message']
                     }
-            self.scm.commit([ path ], commit_meta)
+            self.scm.commit([path], commit_meta)
 
+        if self.db is not None:
+            self.db.update([self.getPage(url)])
         if self.index is not None:
-            self.index.update([ self.getPage(url) ])
+            self.index.update([self.getPage(url)])
 
-    def pageExists(self, url):
+    def pageExists(self, url, from_db=True):
+        if from_db:
+            return self.db.pageExists(url)
         return self.fs.pageExists(url)
 
     def getHistory(self):
-        return self.scm.getHistory();
+        return self.scm.getHistory()
 
     def _passthrough(self, content):
         return content
 
+
+def reloader_stat_loop(wiki, interval=1):
+    mtimes = {}
+    while 1:
+        for page_info in wiki.fs.getPageInfos():
+            path = page_info['path']
+            try:
+                mtime = os.stat(path).st_mtime
+            except OSError:
+                continue
+
+            old_time = mtimes.get(path)
+            if old_time is None:
+                mtimes[path] = mtime
+                continue
+            elif mtime > old_time:
+                print "Change detected in '%s'." % path
+        time.sleep(interval)