# HG changeset patch # User Ludovic Chabant # Date 1359182156 28800 # Node ID 86ee1b6960702ed777d3203756e42056c9b3b007 # Parent 0b6ce6837d22a570e75f538785a2e28952734fa5 Big refactoring: - Trying to simplify APIs - Use SQLite to cache information diff -r 0b6ce6837d22 -r 86ee1b696070 manage.py --- a/manage.py Fri Jan 25 22:20:58 2013 -0800 +++ b/manage.py Fri Jan 25 22:35:56 2013 -0800 @@ -1,9 +1,14 @@ -import os.path -import logging -from flask.ext.script import Manager, Command, prompt, prompt_pass + +# Configure a simpler log format. +from wikked import settings +settings.LOG_FORMAT = "[%(levelname)s]: %(message)s" + +# Create the app and the wiki. from wikked.web import app, wiki +from wikked.page import Page, DatabasePage - +# Create the manager. +from flask.ext.script import Manager, prompt, prompt_pass manager = Manager(app) @@ -15,6 +20,7 @@ print " - " + user.username print "" + @manager.command def new_user(): """Generates the entry for a new user so you can @@ -28,21 +34,37 @@ @manager.command -def reset_index(): - """ Re-generates the index, if search is broken - somehow in your wiki. +def reset(): + """ Re-generates the database and the full-text-search index. """ + wiki.db.reset(wiki.getPages(from_db=False, factory=Page.factory)) wiki.index.reset(wiki.getPages()) @manager.command +def update(): + """ Updates the database and the full-text-search index with any + changed/new files. + """ + wiki.db.update(wiki.getPages(from_db=False, factory=Page.factory)) + wiki.index.update(wiki.getPages()) + + +@manager.command def list(): """ Lists page names in the wiki. """ - for url in wiki.getPageUrls(): + for url in wiki.db.getPageUrls(): print url +@manager.command +def get(url): + """ Gets a page that matches the given URL. + """ + page = wiki.getPage(url) + print page.text + + if __name__ == "__main__": manager.run() - diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/auth.py --- a/wikked/auth.py Fri Jan 25 22:20:58 2013 -0800 +++ b/wikked/auth.py Fri Jan 25 22:35:56 2013 -0800 @@ -3,6 +3,8 @@ class User(object): + """ A user with an account on the wiki. + """ def __init__(self, username, password): self.username = username self.password = password @@ -25,6 +27,8 @@ class UserManager(object): + """ A class that keeps track of users and their permissions. + """ def __init__(self, config, logger=None): if logger is None: logger = logging.getLogger('wikked.auth') @@ -49,11 +53,11 @@ return self._isAllowedForMeta(page, 'writers', username) def _isAllowedForMeta(self, page, meta_name, username): - if (self._permissions[meta_name] is not None and + if (self._permissions[meta_name] is not None and username not in self._permissions[meta_name]): return False - if meta_name in page.all_meta['user']: - allowed = [r.strip() for r in re.split(r'[ ,;]', page.all_meta['user'][meta_name])] + if meta_name in page.all_meta: + allowed = [r.strip() for r in re.split(r'[ ,;]', page.all_meta[meta_name])] if username is None: return 'anonymous' in allowed else: @@ -78,7 +82,7 @@ groups = config.items('groups') for user in config.items('users'): - user_info = { 'username': user[0], 'password': user[1], 'groups': [] } + user_info = {'username': user[0], 'password': user[1], 'groups': []} for group in groups: users_in_group = [u.strip() for u in re.split(r'[ ,;]', group[1])] if user[0] in users_in_group: @@ -89,4 +93,3 @@ user = User(user_info['username'], user_info['password']) user.groups = list(user_info['groups']) return user - diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/db.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/wikked/db.py Fri Jan 25 22:35:56 2013 -0800 @@ -0,0 +1,417 @@ +import os +import os.path +import types +import string +import logging +import datetime +import sqlite3 + + +class conn_scope(object): + """ Helper class, disguised as a function, to ensure the database + has been opened before doing something. If the database wasn't + open, it will be closed after the operation. + """ + def __init__(self, db): + self.db = db + self.do_close = False + + def __enter__(self): + self.do_close = (self.db.conn is None) + self.db.open() + + def __exit__(self, type, value, traceback): + if self.do_close: + self.db.close() + + +class Database(object): + """ The base class for a database cache. + """ + def __init__(self, wiki, logger=None): + self.wiki = wiki + + if logger is None: + logger = logging.getLogger('wikked.db') + self.logger = logger + + def initDb(self): + raise NotImplementedError() + + def open(self): + raise NotImplementedError() + + def close(self): + raise NotImplementedError() + + def reset(self, pages): + raise NotImplementedError() + + def update(self, pages): + raise NotImplementedError() + + def getPageUrls(self, subdir=None): + raise NotImplementedError() + + def getPages(self, subdir=None): + raise NotImplementedError() + + def getPage(self, url): + raise NotImplementedError() + + def pageExists(self, url): + raise NotImplementedError() + + def getLinksTo(self, url): + raise NotImplementedError() + + def getConfigValues(self, section): + raise NotImplementedError() + + def getConfigValue(self, section, name): + raise NotImplementedError() + + +class SQLiteDatabase(Database): + """ A database cache based on SQLite. + """ + schema_version = 1 + + def __init__(self, wiki, logger=None): + Database.__init__(self, wiki, logger) + self.db_path = os.path.join(wiki.root, '.wiki', 'wiki.db') + self.conn = None + + def initDb(self): + create_schema = False + if not os.path.isdir(os.path.dirname(self.db_path)): + # No database on disk... create one. + self.logger.debug("Creating SQL database.") + os.makedirs(os.path.dirname(self.db_path)) + create_schema = True + else: + # The existing schema is outdated, re-create it. + schema_version = self._getSchemaVersion() + if schema_version < self.schema_version: + create_schema = True + if create_schema: + with conn_scope(self): + self._createSchema() + + # Cache the configuration. + cache_config = False + config_time = self._getInfoTime('config_time') + if os.path.isfile(self.wiki.config_path): + if (config_time is None or + config_time <= datetime.datetime.fromtimestamp( + os.path.getmtime(self.wiki.config_path))): + cache_config = True + elif config_time is not None: + cache_config = True + if cache_config: + self._cacheConfig(self.wiki.config) + + def open(self): + if self.conn is None: + self.conn = sqlite3.connect(self.db_path, + detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) + self.conn.row_factory = sqlite3.Row + + def close(self): + if self.conn is not None: + self.conn.close() + self.conn = None + + def reset(self, pages): + self.logger.debug("Re-creating SQL database.") + with conn_scope(self): + self._createSchema() + c = self.conn.cursor() + for page in pages: + self._addPage(page, c) + self.conn.commit() + + def update(self, pages): + self.logger.debug("Updating SQL database...") + to_update = set() + already_added = set() + + with conn_scope(self): + c = self.conn.cursor() + c.execute('''SELECT id, time, path FROM pages''') + for r in c.fetchall(): + if not os.path.isfile(r['path']): + # File was deleted. + self._removePage(r['id'], c) + else: + already_added.add(r['path']) + path_time = datetime.datetime.fromtimestamp( + os.path.getmtime(r['path'])) + if path_time > r['time']: + # File has changed since last index. + self._removePage(r['id'], c) + to_update.add(r['path']) + self.conn.commit() + + for page in pages: + # We want the page's path, but getting it may load all kinds + # of metadata that is time-consuming, so we shortcut the + # system by querying the file-system directly. + fs_meta = self.wiki.fs.getPage(page.url) + if (fs_meta['path'] in to_update or + fs_meta['path'] not in already_added): + self._addPage(page, c) + + # TODO: update any page with a query in it. + + self.conn.commit() + self.logger.debug("...done updating SQL database.") + + def getPageUrls(self, subdir=None): + with conn_scope(self): + c = self.conn.cursor() + if subdir: + subdir = string.rstrip(subdir, '/') + '/%' + c.execute('''SELECT url FROM pages WHERE url LIKE ?''', + (subdir,)) + else: + c.execute('''SELECT url FROM pages''') + urls = [] + for row in c.fetchall(): + urls.append(row['url']) + return urls + + def getPages(self, subdir=None): + with conn_scope(self): + c = self.conn.cursor() + if subdir: + subdir = string.rstrip(subdir, '/') + '/%' + c.execute('''SELECT id, url, path, time, title, raw_text, + formatted_text + FROM pages WHERE url LIKE ?''', + (subdir,)) + else: + c.execute('''SELECT id, url, path, time, title, raw_text, + formatted_text + FROM pages''') + pages = [] + for row in c.fetchall(): + pages.append(self._getPage(row, c)) + return pages + + def getPage(self, url): + with conn_scope(self): + c = self.conn.cursor() + c.execute('''SELECT id, url, path, time, title, raw_text, + formatted_text FROM pages WHERE url=?''', (url,)) + row = c.fetchone() + if row is None: + return None + return self._getPage(row, c) + + def pageExists(self, url): + with conn_scope(self): + c = self.conn.cursor() + c.execute('''SELECT id FROM pages WHERE url=?''', (url,)) + return c.fetchone() is not None + + def getLinksTo(self, url): + with conn_scope(self): + c = self.conn.cursor() + c.execute('''SELECT source FROM links WHERE target=?''', (url,)) + sources = [] + for r in c.fetchall(): + sources.append(r['source']) + return sources + + def getConfigValues(self, section): + with conn_scope(self): + pass + + def getConfigValue(self, section, name): + with conn_scope(self): + pass + + def _createSchema(self): + self.logger.debug("Creating SQL schema...") + c = self.conn.cursor() + c.execute('''DROP TABLE IF EXISTS pages''') + c.execute('''CREATE TABLE pages + (id INTEGER PRIMARY KEY AUTOINCREMENT, + time TIMESTAMP, + url TEXT, + path TEXT, + title TEXT, + raw_text TEXT, + formatted_text TEXT)''') + c.execute('''DROP TABLE IF EXISTS includes''') + c.execute('''CREATE TABLE includes + (id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT, + target TEXT)''') + c.execute('''DROP TABLE IF EXISTS links''') + c.execute('''CREATE TABLE links + (id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT, + target TEXT)''') + c.execute('''DROP TABLE IF EXISTS meta''') + c.execute('''CREATE TABLE meta + (id INTEGER PRIMARY KEY AUTOINCREMENT, + page_id INTEGER, + name TEXT, + value TEXT)''') + c.execute('''DROP TABLE IF EXISTS config''') + c.execute('''CREATE TABLE config + (section TEXT, + name TEXT, + value TEXT)''') + c.execute('''DROP TABLE IF EXISTS info''') + c.execute('''CREATE TABLE info + (name TEXT UNIQUE NOT NULL, + str_value TEXT, + int_value INTEGER, + time_value TIMESTAMP)''') + c.execute('''INSERT INTO info (name, int_value) VALUES (?, ?)''', + ('schema_version', self.schema_version)) + self.conn.commit() + + def _getInfo(self, name, default=None): + with conn_scope(self): + c = self.conn.cursor() + c.execute('''SELECT name, str_value FROM info + WHERE name=?''', (name,)) + row = c.fetchone() + if row is None: + return default + return row['str_value'] + + def _getInfoInt(self, name, default=None): + with conn_scope(self): + c = self.conn.cursor() + c.execute('''SELECT name, int_value FROM info + WHERE name=?''', (name,)) + row = c.fetchone() + if row is None: + return default + return row['int_value'] + + def _getInfoTime(self, name, default=None): + with conn_scope(self): + c = self.conn.cursor() + c.execute('''SELECT name, time_value FROM info + WHERE name=?''', (name,)) + row = c.fetchone() + if row is None: + return default + return row['time_value'] + + def _getSchemaVersion(self): + with conn_scope(self): + c = self.conn.cursor() + c.execute('''SELECT name FROM sqlite_master + WHERE type="table" AND name="info"''') + if c.fetchone() is None: + return 0 + c.execute('''SELECT int_value FROM info + WHERE name="schema_version"''') + row = c.fetchone() + if row is None: + return 0 + return row[0] + + def _cacheConfig(self, config): + self.logger.debug("Re-caching configuration into SQL database.") + with conn_scope(self): + c = self.conn.cursor() + c.execute('''DELETE FROM config''') + for section in config.sections(): + items = config.items(section) + for item in items: + c.execute('''INSERT INTO config + (section, name, value) VALUES (?, ?, ?)''', + (section, item[0], item[1])) + c.execute('''INSERT OR REPLACE INTO info (name, time_value) + VALUES ("config_time", ?)''', (datetime.datetime.now(),)) + self.conn.commit() + + def _addPage(self, page, c): + self.logger.debug("Adding page '%s' to SQL database." % page.url) + now = datetime.datetime.now() + c.execute('''INSERT INTO pages + (time, url, path, title, raw_text, formatted_text) + VALUES (?, ?, ?, ?, ?, ?)''', + (now, page.url, page.path, page.title, + page.raw_text, page.formatted_text)) + page_id = c.lastrowid + + for name, value in page.local_meta.iteritems(): + if isinstance(value, bool): + value = "" + if isinstance(value, types.StringTypes): + c.execute('''INSERT INTO meta + (page_id, name, value) VALUES (?, ?, ?)''', + (page_id, name, value)) + else: + for v in value: + c.execute('''INSERT INTO meta + (page_id, name, value) VALUES (?, ?, ?)''', + (page_id, name, v)) + + for link_url in page.local_links: + c.execute('''INSERT INTO links + (source, target) VALUES (?, ?)''', + (page.url, link_url)) + + for inc_url in page.local_includes: + c.execute('''INSERT INTO includes + (source, target) VALUES (?, ?)''', + (page.url, inc_url)) + + def _removePage(self, page_id, c): + c.execute('''SELECT url FROM pages WHERE id=?''', (page_id,)) + row = c.fetchone() + self.logger.debug("Removing page '%s' [%d] from SQL database." % + (row['url'], page_id)) + c.execute('''DELETE FROM pages WHERE id=?''', (page_id,)) + c.execute('''DELETE FROM meta WHERE page_id=?''', (page_id,)) + c.execute('''DELETE FROM links WHERE source=?''', (row['url'],)) + c.execute('''DELETE FROM includes WHERE source=?''', (row['url'],)) + + def _getPage(self, row, c): + db_page = { + 'url': row['url'], + 'path': row['path'], + 'time': row['time'], + 'title': row['title'], + 'content': row['raw_text'], + 'formatted': row['formatted_text'], + 'links': [], + 'includes': [], + 'meta': {} + } + + c.execute('''SELECT target FROM links + WHERE source=?''', (row['url'],)) + for r in c.fetchall(): + db_page['links'].append(r['target']) + + c.execute('''SELECT target FROM includes + WHERE source=?''', (row['url'],)) + for r in c.fetchall(): + db_page['includes'].append(r['target']) + + c.execute('''SELECT page_id, name, value + FROM meta WHERE page_id=?''', (row['id'],)) + for r in c.fetchall(): + value = r['value'] + if value == '': + value = True + name = r['name'] + if name not in db_page['meta']: + db_page['meta'][name] = value + elif db_page['meta'][name] is list: + db_page['meta'][name].append(value) + else: + db_page['meta'][name] = [db_page['meta'][name], value] + + return db_page diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/formatter.py --- a/wikked/formatter.py Fri Jan 25 22:20:58 2013 -0800 +++ b/wikked/formatter.py Fri Jan 25 22:35:56 2013 -0800 @@ -7,14 +7,16 @@ pass -class PageFormattingContext(object): - def __init__(self, url, ext, slugify=None): +class CircularIncludeError(Exception): + def __init__(self, message, url_trail): + Exception.__init__(self, message) + self.url_trail = url_trail + + +class BaseContext(object): + def __init__(self, url, slugify=None): self.url = url - self.ext = ext self.slugify = slugify - self.out_links = [] - self.included_pages = [] - self.meta = {} @property def urldir(self): @@ -31,10 +33,19 @@ raw_abs_url = os.path.join(self.urldir, url) abs_url = os.path.normpath(raw_abs_url).replace('\\', '/') if self.slugify is not None: - abs_url = self.slugify(url) + abs_url = self.slugify(abs_url) return abs_url +class FormattingContext(BaseContext): + def __init__(self, url, ext, slugify): + BaseContext.__init__(self, url, slugify) + self.ext = ext + self.out_links = [] + self.included_pages = [] + self.meta = {} + + class PageFormatter(object): def __init__(self, wiki): self.wiki = wiki @@ -47,7 +58,6 @@ return formatter(text) def _getFormatter(self, extension): - formatter = None for k, v in self.wiki.formatters.iteritems(): if extension in v: return k @@ -63,7 +73,7 @@ def _processWikiMeta(self, ctx, text): def repl(m): - meta_name = str(m.group(1)) + meta_name = str(m.group(1)).lower() meta_value = str(m.group(3)) if meta_value is not None and len(meta_value) > 0: if meta_name not in ctx.meta: @@ -71,7 +81,7 @@ elif ctx.meta[meta_name] is list: ctx.meta[meta_name].append(meta_value) else: - ctx.meta[meta_name] = [ ctx.meta[meta_name], meta_value ] + ctx.meta[meta_name] = [ctx.meta[meta_name], meta_value] else: ctx.meta[meta_name] = True if meta_name == 'include': @@ -80,7 +90,7 @@ return self._processQuery(ctx, meta_value) return '' - text = re.sub(r'^\[\[((__|\+)?[a-zA-Z][a-zA-Z0-9_\-]+):\s*(.*)\]\]\s*$', repl, text, flags=re.MULTILINE) + text = re.sub(r'^\{\{((__|\+)?[a-zA-Z][a-zA-Z0-9_\-]+):\s*(.*)\}\}\s*$', repl, text, flags=re.MULTILINE) return text def _processWikiLinks(self, ctx, text): @@ -101,38 +111,127 @@ return text def _processInclude(self, ctx, value): - # TODO: handle self-includes or cyclic includes. - abs_included_url = ctx.getAbsoluteUrl(value) - included_page = self.wiki.getPage(abs_included_url) - ctx.included_pages.append(abs_included_url) - return included_page.formatted_text + included_url = ctx.getAbsoluteUrl(value) + ctx.included_pages.append(included_url) + # Includes are run on the fly. + return '
%s
\n' % included_url def _processQuery(self, ctx, query): - parameters = { - 'header': "", - 'item': "
  • {{title}}
    • ", - 'empty': "

    No page matches the query.

    " - } + # Queries are run on the fly. + return '
    %s
    \n' % query + + def _formatWikiLink(self, ctx, display, url): + abs_url = ctx.getAbsoluteUrl(url) + ctx.out_links.append(abs_url) + + css_class = 'wiki-link' + if not self.wiki.pageExists(abs_url, from_db=False): + css_class += ' missing' + return '%s' % (css_class, abs_url, display) + + +class ResolvingContext(object): + def __init__(self): + self.url_trail = set() + self.meta = {} + self.out_links = [] + self.included_pages = [] + + def add(self, ctx): + self.url_trail += ctx.url_trail + self.out_links += ctx.out_links + self.included_pages += ctx.included_pages + for original_key, val in ctx.meta.iteritems(): + # Ignore internal properties. Strip include-only properties + # from their prefix. + key = original_key + if key[0:2] == '__': + continue + if key[0] == '+': + key = key[1:] + + if key not in self.meta: + self.meta[key] = val + elif self.meta[key] is list: + self.meta[key].append(val) + else: + self.meta[key] = [self.meta[key], val] + + +class PageResolver(object): + default_parameters = { + 'header': "", + 'item': "
  • " + + "{{title}}
    • ", + 'empty': "

    No page matches the query.

    " + } + + def __init__(self, page, ctx): + self.page = page + self.ctx = ctx + + @property + def wiki(self): + return self.page.wiki + + def run(self): + def repl(m): + meta_name = str(m.group(1)) + meta_value = str(m.group(2)) + if meta_name == 'query': + return self._runQuery(meta_value) + elif meta_name == 'include': + return self._runInclude(meta_value) + return '' + + self.ctx.url_trail = [self.page.url] + self.ctx.out_links = self.page.local_links + self.ctx.included_pages = self.page.local_includes + self.ctx.meta = self.page.local_meta + + text = self.page.formatted_text + return re.sub(r'^
    (.*)
    $', repl, text, + flags=re.MULTILINE) + + def _runInclude(self, include_url): + if include_url in self.ctx.url_trail: + raise CircularIncludeError("Circular include detected at: %s" % include_url, self.ctx.url_trail) + page = self.wiki.getPage(include_url) + child_ctx = ResolvingContext() + child = PageResolver(page, child_ctx) + text = child.run() + self.ctx.add(child_ctx) + return text + + def _runQuery(self, query): + # Parse the query. + parameters = dict(self.default_parameters) meta_query = {} - arg_pattern = r"(^|\|)(?P[a-zA-Z][a-zA-Z0-9_\-]+)=(?P[^\|]+)" + arg_pattern = r"(^|\|)(?P[a-zA-Z][a-zA-Z0-9_\-]+)="\ + r"(?P[^\|]+)" for m in re.findall(arg_pattern, query): - if m[1] not in parameters: - meta_query[m[1]] = m[2] + key = m[1].lower() + if key not in parameters: + meta_query[key] = m[2] else: - parameters[m[1]] = m[2] + parameters[key] = m[2] + # Find pages that match the query, excluding any page + # that is in the URL trail. matched_pages = [] for p in self.wiki.getPages(): - if p.url == ctx.url: + if p.url in self.ctx.url_trail: continue for key, value in meta_query.iteritems(): - actual = p.getUserMeta(key) - if (type(actual) is list and value in actual) or (actual == value): + if self._isPageMatch(p, key, value): matched_pages.append(p) + + # No match: return the 'empty' template. if len(matched_pages) == 0: return parameters['empty'] + # Combine normal templates to build the output. text = parameters['header'] for p in matched_pages: item_str = parameters['item'] @@ -147,12 +246,23 @@ return text - def _formatWikiLink(self, ctx, display, url): - abs_url = ctx.getAbsoluteUrl(url) - ctx.out_links.append(abs_url) + def _isPageMatch(self, page, name, value, level=0): + # Check the page's local meta properties. + actual = page.local_meta.get(name) + if ((type(actual) is list and value in actual) or + (actual == value)): + return True - css_class = 'wiki-link' - if not self.wiki.pageExists(abs_url): - css_class += ' missing' - return '%s' % (css_class, abs_url, display) + # If this is an include, also look for 'include-only' + # meta properties. + if level > 0: + actual = page.local_meta.get('+' + name) + if ((type(actual) is list and value in actual) or + (actual == value)): + return True + # Recurse into included pages. + for url in page.local_includes: + p = self.wiki.getPage(url) + if self._isPageMatch(p, name, value, level + 1): + return True diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/fs.py --- a/wikked/fs.py Fri Jan 25 22:20:58 2013 -0800 +++ b/wikked/fs.py Fri Jan 25 22:35:56 2013 -0800 @@ -53,13 +53,9 @@ path = self.getPhysicalPagePath(url) with codecs.open(path, 'r', encoding='utf-8') as f: content = f.read() - name = os.path.basename(path) - name_split = os.path.splitext(name) return { 'url': url, 'path': path, - 'name': name_split[0], - 'ext': name_split[1].lstrip('.'), 'content': content } @@ -85,9 +81,7 @@ url = self.slugify(name) return { 'url': url, - 'path': path, - 'name': name, - 'ext': ext + 'path': path } def getPhysicalPagePath(self, url): diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/indexer.py --- a/wikked/indexer.py Fri Jan 25 22:20:58 2013 -0800 +++ b/wikked/indexer.py Fri Jan 25 22:35:56 2013 -0800 @@ -3,7 +3,7 @@ import codecs import logging from whoosh.index import create_in, open_dir -from whoosh.fields import Schema, ID, KEYWORD, TEXT, STORED +from whoosh.fields import Schema, ID, TEXT, STORED from whoosh.qparser import QueryParser @@ -14,7 +14,10 @@ if logger is None: self.logger = logging.getLogger('wikked.index') - def open(self): + def initIndex(self): + raise NotImplementedError() + + def reset(self, pages): raise NotImplementedError() def update(self, pages): @@ -28,24 +31,16 @@ def __init__(self, store_dir, logger=None): WikiIndex.__init__(self, store_dir, logger) - def open(self): + def initIndex(self): if not os.path.isdir(self.store_dir): + self.logger.debug("Creating new index in: " + self.store_dir) os.makedirs(self.store_dir) self.ix = create_in(self.store_dir, self._getSchema()) else: self.ix = open_dir(self.store_dir) - def _getSchema(self): - schema = Schema( - url=ID(stored=True), - title=TEXT(stored=True), - content=TEXT, - path=STORED, - time=STORED - ) - return schema - def reset(self, pages): + self.logger.debug("Re-creating new index in: " + self.store_dir) self.ix = create_in(self.store_dir, schema=self._getSchema()) writer = self.ix.writer() for page in pages: @@ -54,6 +49,7 @@ writer.commit() def update(self, pages): + self.logger.debug("Updating index...") to_reindex = set() already_indexed = set() @@ -67,12 +63,12 @@ if not os.path.isfile(indexed_path): # File was deleted. - writer.delete_by_term('url', indexed_url) + self._unindexPage(writer, indexed_url) else: already_indexed.add(indexed_path) - if os.path.getmtime(indexed_path) > fields['time']: - # File as changed since last index. - writer.delete_by_term('url', indexed_url) + if os.path.getmtime(indexed_path) > indexed_time: + # File has changed since last index. + self._unindexPage(writer, indexed_url) to_reindex.add(indexed_path) for page in pages: @@ -82,6 +78,7 @@ self._indexPage(writer, page) writer.commit() + self.logger.debug("...done updating index.") def search(self, query): with self.ix.searcher() as searcher: @@ -103,8 +100,18 @@ page_infos.append(page_info) return page_infos + def _getSchema(self): + schema = Schema( + url=ID(stored=True), + title=TEXT(stored=True), + content=TEXT, + path=STORED, + time=STORED + ) + return schema + def _indexPage(self, writer, page): - self.logger.debug("Indexing: %s" % page.url) + self.logger.debug("Indexing '%s'." % page.url) writer.add_document( url=unicode(page.url), title=unicode(page.title), @@ -113,3 +120,6 @@ time=os.path.getmtime(page._meta['path']) ) + def _unindexPage(self, writer, url): + self.logger.debug("Removing '%s' from index." % url) + writer.delete_by_term('url', url) diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/page.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/wikked/page.py Fri Jan 25 22:35:56 2013 -0800 @@ -0,0 +1,205 @@ +import os +import os.path +import re +import datetime +import unicodedata +from formatter import ( + PageFormatter, FormattingContext, + PageResolver, ResolvingContext + ) + + +class Page(object): + """ A wiki page. + """ + def __init__(self, wiki, url): + self.wiki = wiki + self.url = url + self._meta = None + self._ext_meta = None + + @property + def path(self): + self._ensureMeta() + return self._meta['path'] + + @property + def title(self): + self._ensureMeta() + return self._meta['title'] + + @property + def raw_text(self): + self._ensureMeta() + return self._meta['content'] + + @property + def formatted_text(self): + self._ensureMeta() + return self._meta['formatted'] + + @property + def text(self): + self._ensureExtendedMeta() + return self._ext_meta['text'] + + @property + def local_meta(self): + self._ensureMeta() + return self._meta['meta'] + + @property + def local_links(self): + self._ensureMeta() + return self._meta['links'] + + @property + def local_includes(self): + self._ensureMeta() + return self._meta['includes'] + + @property + def all_meta(self): + self._ensureExtendedMeta() + return self._ext_meta['meta'] + + @property + def all_links(self): + self._ensureExtendedMeta() + return self._ext_meta['links'] + + @property + def all_includes(self): + self._ensureExtendedMeta() + return self._ext_meta['includes'] + + @property + def in_links(self): + return self.wiki.db.getLinksTo(self.url) + + def getHistory(self): + return self.wiki.scm.getHistory(self.path) + + def getState(self): + return self.wiki.scm.getState(self.path) + + def getRevision(self, rev): + return self.wiki.scm.getRevision(self.path, rev) + + def getDiff(self, rev1, rev2): + return self.wiki.scm.diff(self.path, rev1, rev2) + + def _ensureMeta(self): + if self._meta is not None: + return + + self._meta = self._loadCachedMeta() + if self._meta is not None: + return + + self._meta = self._loadOriginalMeta() + self._saveCachedMeta(self._meta) + + def _loadCachedMeta(self): + return None + + def _saveCachedMeta(self, meta): + pass + + def _loadOriginalMeta(self): + # Get info from the file-system. + meta = self.wiki.fs.getPage(self.url) + + # Format the page and get the meta properties. + filename = os.path.basename(meta['path']) + filename_split = os.path.splitext(filename) + extension = filename_split[1].lstrip('.') + ctx = FormattingContext(self.url, extension, slugify=Page.title_to_url) + f = PageFormatter(self.wiki) + meta['formatted'] = f.formatText(ctx, meta['content']) + meta['meta'] = ctx.meta + meta['links'] = ctx.out_links + meta['includes'] = ctx.included_pages + + # Add some common meta. + meta['title'] = re.sub(r'\-', ' ', filename_split[0]) + if 'title' in meta['meta']: + meta['title'] = meta['meta']['title'] + + return meta + + def _ensureExtendedMeta(self): + if self._ext_meta is not None: + return + + self._ext_meta = {} + ctx = ResolvingContext() + r = PageResolver(self, ctx) + self._ext_meta['text'] = r.run() + self._ext_meta['meta'] = ctx.meta + self._ext_meta['links'] = ctx.out_links + self._ext_meta['includes'] = ctx.included_pages + + @staticmethod + def title_to_url(title): + # Remove diacritics (accents, etc.) and replace them with ASCII + # equivelent. + ansi_title = ''.join((c for c in + unicodedata.normalize('NFD', unicode(title)) + if unicodedata.category(c) != 'Mn')) + # Now replace spaces and punctuation with a hyphen. + return re.sub(r'[^A-Za-z0-9_\.\-\(\)/]+', '-', ansi_title.lower()) + + @staticmethod + def url_to_title(url): + def upperChar(m): + return m.group(0).upper() + return re.sub(r'^.|\s\S', upperChar, url.lower().replace('-', ' ')) + + @staticmethod + def factory(wiki, url): + return Page(wiki, url) + + +class DatabasePage(Page): + """ A page that can load its properties from a + database. + """ + def __init__(self, wiki, url): + Page.__init__(self, wiki, url) + if getattr(wiki, 'db', None) is None: + raise Exception("The wiki doesn't have a database.") + self.auto_update = wiki.config.get('wiki', 'auto_update') + + def _loadCachedMeta(self): + if self.wiki.db is None: + return None + db_page = self.wiki.db.getPage(self.url) + if db_page is None: + return None + if self.auto_update: + path_time = datetime.datetime.fromtimestamp( + os.path.getmtime(db_page['path'])) + if path_time >= db_page['time']: + return None + meta = { + 'url': self.url, + 'path': db_page['path'], + 'content': db_page['content'], + 'formatted': db_page['formatted'], + 'meta': db_page['meta'], + 'title': db_page['title'], + 'links': db_page['links'], + 'includes': db_page['includes'] + } + return meta + + def _saveCachedMeta(self, meta): + if self.wiki.db is not None: + self.wiki.logger.debug( + "Updated database cache for page '%s'." % self.url) + self.wiki.db.update([self]) + + @staticmethod + def factory(wiki, url): + return DatabasePage(wiki, url) diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/resources/defaults.cfg --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/wikked/resources/defaults.cfg Fri Jan 25 22:35:56 2013 -0800 @@ -0,0 +1,3 @@ +[wiki] +scm=hg +auto_update=False diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/settings.py --- a/wikked/settings.py Fri Jan 25 22:20:58 2013 -0800 +++ b/wikked/settings.py Fri Jan 25 22:35:56 2013 -0800 @@ -1,4 +1,3 @@ SECRET_KEY = '\xef)*\xbc\xd7\xa9t\x7f\xbc3pH1o\xc1\xe2\xb0\x19\\L\xeb\xe3\x00\xa3' DEBUG = True - diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/views.py --- a/wikked/views.py Fri Jan 25 22:20:58 2013 -0800 +++ b/wikked/views.py Fri Jan 25 22:35:56 2013 -0800 @@ -1,18 +1,12 @@ import time import os.path -from flask import ( - Response, - render_template, url_for, redirect, abort, request, flash, - jsonify - ) -from flask.ext.login import login_required, login_user, logout_user, current_user +from flask import render_template, abort, request, g, jsonify +from flask.ext.login import login_user, logout_user, current_user from pygments import highlight from pygments.lexers import get_lexer_by_name from pygments.formatters import get_formatter_by_name -from web import app, wiki +from web import app, login_manager from wiki import Page -from auth import User -from forms import RegistrationForm, EditPageForm from fs import PageNotFoundError import scm @@ -21,15 +15,20 @@ CHECK_FOR_READ = 1 CHECK_FOR_WRITE = 2 +COERCE_META = { + 'redirect': Page.title_to_url + } + def get_page_or_none(url): try: - page = wiki.getPage(url) + page = g.wiki.getPage(url) page._ensureMeta() return page except PageNotFoundError: return None + def get_page_or_404(url, check_perms=DONT_CHECK): page = get_page_or_none(url) if page is not None: @@ -49,6 +48,19 @@ return page.wiki.auth.isPageWritable(page, user.get_id()) +def get_page_meta(page, local_only=False): + if local_only: + meta = dict(page.local_meta) + else: + meta = dict(page.all_meta) + meta['title'] = page.title + meta['url'] = page.url + for name in COERCE_META: + if name in meta: + meta[name] = COERCE_META(meta[name]) + return meta + + def get_history_data(history): hist_data = [] for i, rev in enumerate(reversed(history)): @@ -62,10 +74,10 @@ 'pages': [] } for f in rev.files: - f_info = wiki.fs.getPageInfo(f['path']) + f_info = g.wiki.fs.getPageInfo(f['path']) if f_info is None: continue - page = wiki.getPage(f_info['url']) + page = g.wiki.getPage(f_info['url']) if not is_page_readable(page): continue rev_data['pages'].append({ @@ -79,7 +91,7 @@ def make_auth_response(data): if current_user.is_authenticated(): - data['auth'] = { + data['auth'] = { 'username': current_user.username, 'is_admin': current_user.is_admin() } @@ -103,28 +115,28 @@ @app.route('/api/list') def api_list_all_pages(): - return list_pages(None) + return api_list_pages(None) @app.route('/api/list/') def api_list_pages(url): - pages = filter(is_page_readable, wiki.getPages(url)) - page_metas = [page.all_meta for page in pages] - result = { 'path': url, 'pages': list(page_metas) } + pages = filter(is_page_readable, g.wiki.getPages(url)) + page_metas = [get_page_meta(page) for page in pages] + result = {'path': url, 'pages': list(page_metas)} return make_auth_response(result) @app.route('/api/read/') def api_read_page(url): page = get_page_or_404(url, CHECK_FOR_READ) - result = { 'path': url, 'meta': page.all_meta, 'text': page.formatted_text } + result = {'meta': get_page_meta(page), 'text': page.text} return make_auth_response(result) @app.route('/api/raw/') def api_read_page_raw(url): page = get_page_or_404(url, CHECK_FOR_READ) - result = { 'path': url, 'meta': page.all_meta, 'text': page.raw_text } + result = {'meta': get_page_meta(page), 'text': page.raw_text} return make_auth_response(result) @@ -135,8 +147,8 @@ abort(400) page = get_page_or_404(url, CHECK_FOR_READ) page_rev = page.getRevision(rev) - meta = dict(page.all_meta, rev=rev) - result = { 'path': url, 'meta': meta, 'text': page_rev } + meta = dict(get_page_meta(page, True), rev=rev) + result = {'meta': meta, 'text': page_rev} return make_auth_response(result) @@ -153,10 +165,10 @@ formatter = get_formatter_by_name('html') diff = highlight(diff, lexer, formatter) if rev2 is None: - meta = dict(page.all_meta, change=rev1) + meta = dict(get_page_meta(page, True), change=rev1) else: - meta = dict(page.all_meta, rev1=rev1, rev2=rev2) - result = { 'path': url, 'meta': meta, 'diff': diff } + meta = dict(get_page_meta(page, True), rev1=rev1, rev2=rev2) + result = {'meta': meta, 'diff': diff} return make_auth_response(result) @@ -164,10 +176,9 @@ def api_get_state(url): page = get_page_or_404(url, CHECK_FOR_READ) state = page.getState() - return make_auth_response({ - 'path': url, - 'meta': page.all_meta, - 'state': scm.STATE_NAMES[state] + return make_auth_response({ + 'meta': get_page_meta(page, True), + 'state': scm.STATE_NAMES[state] }) @@ -183,9 +194,9 @@ 'title': other.title }) else: - links.append({ 'url': link, 'missing': True }) + links.append({'url': link, 'missing': True}) - result = { 'path': url, 'meta': page.all_meta, 'out_links': links } + result = {'meta': get_page_meta(page, True), 'out_links': links} return make_auth_response(result) @@ -198,12 +209,12 @@ if other is not None and is_page_readable(other): links.append({ 'url': link, - 'meta': other.all_meta + 'title': other.title }) else: - links.append({ 'url': link, 'missing': True }) + links.append({'url': link, 'missing': True}) - result = { 'path': url, 'meta': page.all_meta, 'in_links': links } + result = {'meta': get_page_meta(page, True), 'in_links': links} return make_auth_response(result) @@ -213,21 +224,18 @@ page = get_page_or_none(url) if page is None: result = { - 'path': url, 'meta': { 'url': url, 'name': os.path.basename(url), - 'title': Page.url_to_title(url), - 'user': {} + 'title': Page.url_to_title(url) }, 'text': '' } else: if not is_page_writable(page): abort(401) - result = { - 'path': url, - 'meta': page.all_meta, + result = { + 'meta': get_page_meta(page, True), 'text': page.raw_text } result['commit_meta'] = { @@ -253,8 +261,8 @@ 'author': author, 'message': message } - wiki.setPage(url, page_fields) - result = { 'path': url, 'saved': 1 } + g.wiki.setPage(url, page_fields) + result = {'saved': 1} return make_auth_response(result) @@ -271,18 +279,18 @@ @app.route('/api/orphans') def api_special_orphans(): orphans = [] - for page in filter(is_page_readable, wiki.getPages()): + for page in filter(is_page_readable, g.wiki.getPages()): if len(page.in_links) == 0: - orphans.append({ 'path': page.url, 'meta': page.all_meta }) - result = { 'orphans': orphans } + orphans.append({'path': page.url, 'meta': get_page_meta(page, True)}) + result = {'orphans': orphans} return make_auth_response(result) @app.route('/api/history') def api_site_history(): - history = wiki.getHistory() + history = g.wiki.getHistory() hist_data = get_history_data(history) - result = { 'history': hist_data } + result = {'history': hist_data} return make_auth_response(result) @@ -291,18 +299,19 @@ page = get_page_or_404(url, CHECK_FOR_READ) history = page.getHistory() hist_data = get_history_data(history) - result = { 'url': url, 'meta': page.all_meta, 'history': hist_data } + result = {'url': url, 'meta': get_page_meta(page, True), 'history': hist_data} return make_auth_response(result) @app.route('/api/search') def api_search(): query = request.args.get('q') + def is_hit_readable(hit): page = get_page_or_none(hit['url']) return page is None or is_page_readable(page) - hits = filter(is_hit_readable, wiki.index.search(query)) - result = { 'query': query, 'hits': hits } + hits = filter(is_hit_readable, g.wiki.index.search(query)) + result = {'query': query, 'hits': hits} return make_auth_response(result) @@ -310,8 +319,8 @@ def api_admin_reindex(): if not current_user.is_authenticated() or not current_user.is_admin(): return login_manager.unauthorized() - wiki.index.reset(wiki.getPages()) - result = { 'ok': 1 } + g.wiki.index.reset(g.wiki.getPages()) + result = {'ok': 1} return make_auth_response(result) @@ -321,11 +330,11 @@ password = request.form.get('password') remember = request.form.get('remember') - user = wiki.auth.getUser(username) + user = g.wiki.auth.getUser(username) if user is not None: if app.bcrypt.check_password_hash(user.password, password): login_user(user, remember=bool(remember)) - result = { 'username': username, 'logged_in': 1 } + result = {'username': username, 'logged_in': 1} return make_auth_response(result) abort(401) @@ -333,7 +342,7 @@ @app.route('/api/user/is_logged_in') def api_user_is_logged_in(): if current_user.is_authenticated(): - result = { 'logged_in': True } + result = {'logged_in': True} return make_auth_response(result) abort(401) @@ -341,15 +350,14 @@ @app.route('/api/user/logout', methods=['POST']) def api_user_logout(): logout_user() - result = { 'ok': 1 } + result = {'ok': 1} return make_auth_response(result) @app.route('/api/user/info/') def api_user_info(name): - user = wiki.auth.getUser(name) + user = g.wiki.auth.getUser(name) if user is not None: - result = { 'username': user.username, 'groups': user.groups } + result = {'username': user.username, 'groups': user.groups} return make_auth_response(result) abort(404) - diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/web.py --- a/wikked/web.py Fri Jan 25 22:20:58 2013 -0800 +++ b/wikked/web.py Fri Jan 25 22:35:56 2013 -0800 @@ -1,10 +1,36 @@ -from flask import Flask, abort +from flask import Flask, abort, g +from wiki import Wiki # Create the main app. -app = Flask(__name__) +app = Flask("wikked") app.config.from_object('wikked.settings') app.config.from_envvar('WIKKED_SETTINGS', silent=True) + +def create_wiki(): + wiki = Wiki(root=app.config.get('WIKI_ROOT'), logger=app.logger) + wiki.start() + return wiki + +wiki = create_wiki() + + +# Set the wiki as a request global, and open/close the database. +@app.before_request +def before_request(): + if getattr(wiki, 'db', None): + wiki.db.open() + g.wiki = wiki + + +@app.teardown_request +def teardown_request(exception): + if wiki is not None: + if getattr(wiki, 'db', None): + wiki.db.close() + + +# Make is serve static content in DEBUG mode. if app.config['DEBUG']: from werkzeug import SharedDataMiddleware import os @@ -12,22 +38,32 @@ '/': os.path.join(os.path.dirname(__file__), 'static') }) -# The main Wiki instance. -from wiki import Wiki -wiki = Wiki(root=app.config.get('WIKI_ROOT'), logger=app.logger) -wiki.start() -# Import views and user loader. -import views +# Customize logging. +if app.config.get('LOG_FORMAT'): + import logging + handler = logging.StreamHandler() + handler.setLevel(logging.DEBUG) + handler.setFormatter(logging.Formatter(app.config['LOG_FORMAT'])) + app.logger.handlers = [] + app.logger.addHandler(handler) + # Login extension. +def user_loader(username): + return g.wiki.auth.getUser(username) + from flask.ext.login import LoginManager login_manager = LoginManager() login_manager.init_app(app) -login_manager.user_loader(wiki.auth.getUser) +login_manager.user_loader(user_loader) login_manager.unauthorized_handler(lambda: abort(401)) + # Bcrypt extension. from flaskext.bcrypt import Bcrypt app.bcrypt = Bcrypt(app) + +# Import the views. +import views diff -r 0b6ce6837d22 -r 86ee1b696070 wikked/wiki.py --- a/wikked/wiki.py Fri Jan 25 22:20:58 2013 -0800 +++ b/wikked/wiki.py Fri Jan 25 22:35:56 2013 -0800 @@ -1,178 +1,25 @@ import os import os.path -import re import time import logging import itertools -import unicodedata from ConfigParser import SafeConfigParser import markdown import textile import creole +from page import Page, DatabasePage +from cache import Cache from fs import FileSystem -from cache import Cache +from db import SQLiteDatabase from scm import MercurialSourceControl from indexer import WhooshWikiIndex from auth import UserManager -from formatter import PageFormatter, PageFormattingContext class InitializationError(Exception): pass -class Page(object): - def __init__(self, wiki, url): - self.wiki = wiki - self.url = url - self._meta = None - - self._promoted_meta = [ - 'title', - 'redirect', - 'notitle' - ] - self._coerce_promoted_meta = { - 'redirect': Page.title_to_url - } - - @property - def title(self): - self._ensureMeta() - return self._meta['title'] - - @property - def raw_text(self): - if self._meta is not None: - return self._meta['content'] - page = self.wiki.fs.getPage(self.url) - return page['content'] - - @property - def formatted_text(self): - self._ensureMeta() - return self._meta['formatted'] - - @property - def out_links(self): - self._ensureMeta() - return self._meta['out_links'] - - @property - def in_links(self): - links = [] - for other_url in self.wiki.getPageUrls(): - if other_url == self.url: - continue - other_page = Page(self.wiki, other_url) - for l in other_page.out_links: - if l == self.url: - links.append(other_url) - return links - - @property - def all_meta(self): - self._ensureMeta() - meta = { - 'url': self._meta['url'], - 'name': self._meta['name'], - 'title': self._meta['title'], - 'user': self._meta['user'] - } - for name in self._promoted_meta: - if name in self._meta['user']: - meta[name] = self._meta['user'][name] - if name in self._coerce_promoted_meta: - meta[name] = self._coerce_promoted_meta[name](meta[name]) - return meta - - def getUserMeta(self, key): - self._ensureMeta() - return self._meta['user'].get(key) - - def getHistory(self): - self._ensureMeta() - return self.wiki.scm.getHistory(self._meta['path']) - - def getState(self): - self._ensureMeta() - return self.wiki.scm.getState(self._meta['path']) - - def getRevision(self, rev): - self._ensureMeta() - return self.wiki.scm.getRevision(self._meta['path'], rev) - - def getDiff(self, rev1, rev2): - self._ensureMeta() - return self.wiki.scm.diff(self._meta['path'], rev1, rev2) - - def _ensureMeta(self): - if self._meta is not None: - return - - cache_key = self.url + '.info.cache' - cached_meta = self._getCached(cache_key) - if cached_meta is not None: - # We have a valid cache for our content, but if we are including - # other pages, we need to check if they have changed since last - # time. - base_url = os.path.dirname(self.url) - for included_url in cached_meta['included_pages']: - included_path = self.wiki.fs.getPhysicalPagePath(included_url) - included_time = os.path.getmtime(included_path) - included_cache_key = included_url + '.info.cache' - if not self.wiki.cache.isValid(included_cache_key, included_time): - break - else: - self._meta = cached_meta - return - - self._meta = self.wiki.fs.getPage(self.url) - - ext = self._meta['ext'] - if ext[0] == '.': - ext = ext[1:] - ctx = PageFormattingContext(self.url, ext, slugify=Page.title_to_url) - f = PageFormatter(self.wiki) - self._meta['formatted'] = f.formatText(ctx, self._meta['content']) - self._meta['user'] = ctx.meta - - self._meta['title'] = re.sub(r'\-', ' ', self._meta['name']) - for name in self._promoted_meta: - if name in ctx.meta: - self._meta[name] = ctx.meta[name] - - self._meta['out_links'] = ctx.out_links - self._meta['included_pages'] = ctx.included_pages - - self._putCached(cache_key, self._meta) - - def _getCached(self, cache_key): - if self.wiki.cache is not None: - page_path = self.wiki.fs.getPhysicalPagePath(self.url) - page_time = os.path.getmtime(page_path) - return self.wiki.cache.read(cache_key, page_time) - return None - - def _putCached(self, cache_key, data): - if self.wiki.cache is not None: - self.wiki.logger.debug("Updated cached %s for page '%s'." % (cache_key, self.url)) - self.wiki.cache.write(cache_key, data) - - @staticmethod - def title_to_url(title): - # Remove diacritics (accents, etc.) and replace them with ASCII equivelent. - ansi_title = ''.join((c for c in unicodedata.normalize('NFD', title) if unicodedata.category(c) != 'Mn')) - # Now replace spaces and punctuation with a hyphen. - return re.sub(r'[^A-Za-z0-9_\.\-\(\)/]+', '-', ansi_title.lower()) - - @staticmethod - def url_to_title(url): - def upperChar(m): - return m.group(0).upper() - return re.sub(r'^.|\s\S', upperChar, url.lower().replace('-', ' ')) - - class Wiki(object): def __init__(self, root=None, logger=None): if root is None: @@ -183,71 +30,96 @@ self.logger = logging.getLogger('wikked.wiki') self.logger.debug("Initializing wiki at: " + root) + self.page_factory = DatabasePage.factory + self.use_db = True self.formatters = { - markdown.markdown: [ 'md', 'mdown', 'markdown' ], - textile.textile: [ 'tl', 'text', 'textile' ], - creole.creole2html: [ 'cr', 'creole' ], - self._passthrough: [ 'txt', 'html' ] + markdown.markdown: ['md', 'mdown', 'markdown'], + textile.textile: ['tl', 'text', 'textile'], + creole.creole2html: ['cr', 'creole'], + self._passthrough: ['txt', 'html'] } - self.config = SafeConfigParser() - config_path = os.path.join(root, '.wikirc') - if os.path.isfile(config_path): - self.config.read(config_path) + self.default_config_path = os.path.join( + os.path.dirname(__file__), 'resources', 'defaults.cfg') + self.config_path = os.path.join(root, '.wikirc') + self.config = self._loadConfig() self.fs = FileSystem(root, slugify=Page.title_to_url) self.auth = UserManager(self.config, logger=self.logger) - self.index = WhooshWikiIndex(os.path.join(root, '.index'), logger=self.logger) + self.index = WhooshWikiIndex(os.path.join(root, '.wiki', 'index'), + logger=self.logger) + self.db = SQLiteDatabase(self, logger=self.logger) + self.scm = self._createScm() + self.cache = self._createJsonCache() - scm_type = 'hg' - if self.config.has_option('wiki', 'scm'): - scm_type = self.config.get('wiki', 'scm') + self.fs.page_extensions = list(set( + itertools.chain(*self.formatters.itervalues()))) + self.fs.excluded.append(self.config_path) + self.fs.excluded.append(os.path.join(root, '.wiki')) + if self.scm is not None: + self.fs.excluded += self.scm.getSpecialDirs() + + def _createScm(self): + scm_type = self.config.get('wiki', 'scm') if scm_type == 'hg': - self.scm = MercurialSourceControl(root, self.logger) + return MercurialSourceControl(self.fs.root, self.logger) else: raise InitializationError("No such source control: " + scm_type) + def _createJsonCache(self): if (not self.config.has_option('wiki', 'cache') or self.config.getboolean('wiki', 'cache')): - self.cache = Cache(os.path.join(root, '.cache')) + return Cache(os.path.join(self.fs.root, '.wiki', 'cache')) else: - self.cache = None + return None - self.fs.page_extensions = list(set(itertools.chain(*self.formatters.itervalues()))) - self.fs.excluded.append(config_path) - if self.scm is not None: - self.fs.excluded += self.scm.getSpecialDirs() - if self.cache is not None: - self.fs.excluded.append(self.cache.cache_dir) - if self.index is not None: - self.fs.excluded.append(self.index.store_dir) + def _loadConfig(self): + config = SafeConfigParser() + config.readfp(open(self.default_config_path)) + config.read(self.config_path) + return config - def start(self): + def start(self, update=True): if self.scm is not None: self.scm.initRepo() if self.index is not None: - self.index.open() + self.index.initIndex() + if self.db is not None: + self.db.initDb() + + if update: + pass @property def root(self): return self.fs.root - def getPageUrls(self, subdir=None): - for info in self.fs.getPageInfos(subdir): - yield info['url'] + def getPageUrls(self, subdir=None, from_db=True): + if from_db and self.db: + for url in self.db.getPageUrls(subdir): + yield url + else: + for info in self.fs.getPageInfos(subdir): + yield info['url'] - def getPages(self, subdir=None): - for url in self.getPageUrls(subdir): - yield Page(self, url) + def getPages(self, subdir=None, from_db=True, factory=None): + if factory is None: + factory = self.page_factory + for url in self.getPageUrls(subdir, from_db): + yield factory(self, url) - def getPage(self, url): - return Page(self, url) + def getPage(self, url, factory=None): + if factory is None: + factory = self.page_factory + return factory(self, url) def setPage(self, url, page_fields): if 'author' not in page_fields: - raise ValueError("No author specified for editing page '%s'." % url) + raise ValueError( + "No author specified for editing page '%s'." % url) if 'message' not in page_fields: - raise ValueError("No commit message specified for editing page '%s'." % url) + raise ValueError( + "No commit message specified for editing page '%s'." % url) do_commit = False path = self.fs.getPhysicalPagePath(url) @@ -262,17 +134,39 @@ 'author': page_fields['author'], 'message': page_fields['message'] } - self.scm.commit([ path ], commit_meta) + self.scm.commit([path], commit_meta) + if self.db is not None: + self.db.update([self.getPage(url)]) if self.index is not None: - self.index.update([ self.getPage(url) ]) + self.index.update([self.getPage(url)]) - def pageExists(self, url): + def pageExists(self, url, from_db=True): + if from_db: + return self.db.pageExists(url) return self.fs.pageExists(url) def getHistory(self): - return self.scm.getHistory(); + return self.scm.getHistory() def _passthrough(self, content): return content + +def reloader_stat_loop(wiki, interval=1): + mtimes = {} + while 1: + for page_info in wiki.fs.getPageInfos(): + path = page_info['path'] + try: + mtime = os.stat(path).st_mtime + except OSError: + continue + + old_time = mtimes.get(path) + if old_time is None: + mtimes[path] = mtime + continue + elif mtime > old_time: + print "Change detected in '%s'." % path + time.sleep(interval)