changeset 307:e4e13e1138b2

Frontend refactor and backend support for cached page lists. * Page lists like orphans or broken redirects are now cached in the DB, and invalidated when something changes. * Refactor the frontend code to make it easier to add lots of other such page lists. Reskin the admin dashboard a bit. * Add new "broken redirects" list for testing new code.
author Ludovic Chabant <ludovic@chabant.com>
date Sat, 04 Oct 2014 21:05:22 -0700
parents a62188fc0ed2
children 31667edfb072
files wikked/assets/js/wikked/app.js wikked/assets/js/wikked/models.js wikked/assets/js/wikked/views.js wikked/assets/tpl/special-orphans.html wikked/assets/tpl/special-pagelist.html wikked/assets/tpl/special-pages.html wikked/db/base.py wikked/db/sql.py wikked/views/__init__.py wikked/views/read.py wikked/views/special.py wikked/web.py wikked/wiki.py
diffstat 13 files changed, 448 insertions(+), 91 deletions(-) [+]
line wrap: on
line diff
--- a/wikked/assets/js/wikked/app.js	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/assets/js/wikked/app.js	Sat Oct 04 21:05:22 2014 -0700
@@ -77,7 +77,7 @@
             'special':               "showSpecialPages",
             'special/changes':       "showSiteChanges",
             'special/changes/:rev':  "showSiteChangesAfterRev",
-            'special/orphans':       "showOrphans"
+            'special/list/:name':    "showPageList"
         },
         readPage: function(path) {
             var path_clean = this.stripQuery(path);
@@ -196,12 +196,12 @@
             this.viewManager.switchView(view);
             this.navigate('/special/changes/' + rev);
         },
-        showOrphans: function() {
-            var view = new Views.SpecialOrphansView({
-                model: new Models.SpecialOrphansModel()
+        showPageList: function(name) {
+            var view = new Views.SpecialPageListView({
+                model: new Models.SpecialPageListModel({ name: name })
             });
             this.viewManager.switchView(view);
-            this.navigate('/special/orphans');
+            this.navigate('/special/list/' + name);
         },
         stripQuery: function(url) {
             q = url.indexOf("?");
--- a/wikked/assets/js/wikked/models.js	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/assets/js/wikked/models.js	Sat Oct 04 21:05:22 2014 -0700
@@ -454,6 +454,45 @@
         },
         initialize: function() {
             SpecialPagesModel.__super__.initialize.apply(this, arguments);
+            this.set('sections', [
+                {
+                    title: "Wiki",
+                    pages: [
+                        {
+                            title: "Recent Changes",
+                            url: '/#/special/changes',
+                            description: "See all changes in the wiki."
+                        }
+                    ]
+                },
+                {
+                    title: "Page Lists",
+                    pages: [
+                        {
+                            title: "Orphaned Pages",
+                            url: '/#/special/list/orphans',
+                            description: ("Lists pages in the wiki that have " +
+                                          "no links to them.")
+                        },
+                        {
+                            title: "Broken Redirects",
+                            url: '/#/special/list/broken-redirects',
+                            description: ("Lists pages that redirect to a " +
+                                          "missing page.")
+                        }
+                    ]
+                },
+                {
+                    title: "Users",
+                    pages: [
+                        {
+                            title: "All Users",
+                            url: '/#/special/users',
+                            description: "A list of all registered users."
+                        }
+                    ]
+                }
+            ]);
         },
         _addFooterExtraUrls: function() {
         }
@@ -504,9 +543,44 @@
         }
     });
 
-    var SpecialOrphansModel = exports.SpecialOrphansModel = SpecialPageModel.extend({
-        title: "Orphaned Pages",
-        url: '/api/orphans'
+    var SpecialPageListModel = exports.SpecialPageListModel = SpecialPageModel.extend({
+        title: function() { return this.titleMap[this.get('name')]; },
+        url: function() { return '/api/' + this.get('name'); },
+        initialize: function() {
+            SpecialPageListModel.__super__.initialize.apply(this, arguments);
+            var name = this.get('name');
+            this.set({
+                'title': this.titleMap[name],
+                'message': this.messageMap[name],
+                'aside': this.asideMap[name],
+                'empty': this.emptyMap[name],
+                'url_suffix': this.urlSuffix[name]
+            });
+        },
+        titleMap: {
+            'orphans': "Orphaned Pages",
+            'broken-redirects': "Broken Redirects"
+        },
+        messageMap: {
+            'orphans': ("Here is a list of pages that don't have any pages " +
+                        "linking to them. This means user will only be able " +
+                        "to find them by searching for them, or by getting " +
+                        "a direct link."),
+            'broken-redirects':
+                        ("Here is a list of pages that redirect to a non-" +
+                         "existing page.")
+        },
+        asideMap: {
+            'orphans': ("The main page usually shows up here but that's " +
+                        "OK since it's the page everyone sees first.")
+        },
+        emptyMap: {
+            'orphans': "No orphaned pages!",
+            'broken-redirects': "No broken redirects!"
+        },
+        urlSuffix: {
+            'broken-redirects': '?no_redirect'
+        }
     });
 
     return exports;
--- a/wikked/assets/js/wikked/views.js	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/assets/js/wikked/views.js	Sat Oct 04 21:05:22 2014 -0700
@@ -34,7 +34,7 @@
         'text!tpl/special-nav.html',
         'text!tpl/special-pages.html',
         'text!tpl/special-changes.html',
-        'text!tpl/special-orphans.html'
+        'text!tpl/special-pagelist.html'
         ],
     function($, JQueryValidate, _, Backbone, Handlebars,
         BootstrapTooltip, BootstrapAlert, BootstrapCollapse,
@@ -43,7 +43,7 @@
         tplReadPage, tplMetaPage, tplEditPage, tplHistoryPage, tplRevisionPage, tplDiffPage, tplInLinksPage,
         tplNav, tplFooter, tplSearchResults, tplLogin,
         tplErrorNotAuthorized, tplErrorNotFound, tplErrorUnauthorizedEdit, tplStateWarning,
-        tplSpecialNav, tplSpecialPages, tplSpecialChanges, tplSpecialOrphans) {
+        tplSpecialNav, tplSpecialPages, tplSpecialChanges, tplSpecialPageList) {
 
     var exports = {};
 
@@ -191,7 +191,7 @@
 
             // Cache some stuff for handling the menu.
             this.wikiMenu = this.$('#wiki-menu');
-            this.wrapperAndWikiMenu = this.$('.wrapper, #wiki-menu');
+            this.wrapperAndWikiMenu = $([this.parentEl, this.$('#wiki-menu')]);
             this.isMenuActive = (this.wikiMenu.css('left') == '0px');
             this.isMenuActiveLocked = false;
         },
@@ -356,6 +356,7 @@
         initialize: function() {
             MasterPageView.__super__.initialize.apply(this, arguments);
             this.nav = this._createNavigation(this.model.nav);
+            this.nav.parentEl = this.el;
             this.footer = this._createFooter(this.model.footer);
             return this;
         },
@@ -613,7 +614,13 @@
     });
 
     var SpecialMasterPageView = exports.SpecialMasterPageView = MasterPageView.extend({
-        className: 'wrapper special'
+        className: function() {
+            var cls = 'wrapper special';
+            // See comment for `MasterPageView`.
+            var ima = localStorage.getItem('wikked.nav.isMenuActive');
+            if (ima == 'true') cls += ' wiki-menu-active';
+            return cls;
+        }
     });
 
     var SpecialPagesView = exports.SpecialPagesView = SpecialMasterPageView.extend({
@@ -648,8 +655,8 @@
         }
     });
 
-    var SpecialOrphansView = exports.SpecialOrphansView = SpecialMasterPageView.extend({
-        defaultTemplateSource: tplSpecialOrphans
+    var SpecialPageListView = exports.SpecialPageListView = SpecialMasterPageView.extend({
+        defaultTemplateSource: tplSpecialPageList
     });
 
     return exports;
--- a/wikked/assets/tpl/special-orphans.html	Sat Oct 04 21:02:05 2014 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,18 +0,0 @@
-<article>
-    <header>
-        <h1>Orphaned Pages</h1>
-    </header>
-    <section>
-        <p>The following pages are not linked to by any other page in the wiki.</p>
-        <aside><p>The main page can occasionally show up here but that's OK since it's the page visitors will see first anyway.</p></aside>
-        {{#if orphans}}
-        <ul>
-        {{#each orphans}}
-            <li><a href="{{get_read_url meta.url}}">{{meta.title}}</a></li>
-        {{/each}}
-        </ul>
-        {{else}}
-        <p>No orphaned pages!</p>
-        {{/if}}
-    </section>
-</article>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/wikked/assets/tpl/special-pagelist.html	Sat Oct 04 21:05:22 2014 -0700
@@ -0,0 +1,20 @@
+<article>
+    <header>
+        <h1>{{title}}</h1>
+    </header>
+    <section>
+        <p>{{message}}</p>
+        {{#if aside}}
+        <aside><p>{{aside}}</p></aside>
+        {{/if}}
+        {{#if pages}}
+        <ul>
+        {{#each pages}}
+            <li><a href="{{get_read_url url}}{{../url_suffix}}">{{title}}</a></li>
+        {{/each}}
+        </ul>
+        {{else}}
+        <p>{{empty}}</p>
+        {{/if}}
+    </section>
+</article>
--- a/wikked/assets/tpl/special-pages.html	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/assets/tpl/special-pages.html	Sat Oct 04 21:05:22 2014 -0700
@@ -3,9 +3,18 @@
         <h1>Special Pages</h1>
     </header>
     <section>
-        <h3><a href="/#/special/changes">Recent Changes</a></h3>
-        <p>See all changes in the wiki.</p>
-        <h3><a href="/#/special/orphans">Orphaned Pages</a></h3>
-        <p>Lists pages in the wiki that have no links to them.</p>
+        <div class="pure-g">
+            {{#each sections}}
+            <div class="pure-u-1-3">
+                <h2>{{title}}</h2>
+                {{#each pages}}
+                <h3><a href="{{url}}">{{title}}</a></h3>
+                {{#if description}}
+                <p>{{description}}</p>
+                {{/if}}
+                {{/each}}
+            </div>
+            {{/each}}
+        </div>
     </section>
 </article>
--- a/wikked/db/base.py	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/db/base.py	Sat Oct 04 21:05:22 2014 -0700
@@ -1,38 +1,56 @@
 from wikked.utils import PageNotFoundError
 
 
+class PageListNotFound(Exception):
+    def __init__(self, list_name):
+        super(PageListNotFound, self).__init__("No such page list: %s" % list_name)
+
+
 class Database(object):
     """ The base class for a database cache.
     """
     def start(self, wiki):
+        """ Called when the wiki is started. """
         pass
 
     def init(self, wiki):
+        """ Called when a new wiki is created. """
         pass
 
     def postInit(self):
+        """ Called after a new wiki has been created. """
         pass
 
     def close(self, commit, exception):
+        """ Called when the wiki is disposed of. """
         pass
 
     def reset(self, page_infos):
+        """ Called when the DB cache should be re-build from scratch
+            based on the given page infos. """
         pass
 
     def updatePage(self, page_info):
+        """ Update the given page's cache info based on the given page
+            info. """
         pass
 
     def updateAll(self, page_infos, force=False):
+        """ Update all the pages in the wiki based on the given pages
+            infos. """
         pass
 
     def getPageUrls(self, subdir=None, uncached_only=False):
+        """ Return page URLs. """
         raise NotImplementedError()
 
     def getPages(self, subdir=None, meta_query=None, uncached_only=False,
                  endpoint_only=None, no_endpoint_only=False, fields=None):
+        """ Return pages from the DB cache. """
         raise NotImplementedError()
 
     def getPage(self, url=None, path=None, fields=None, raise_if_none=True):
+        """ Gets a page from the DB cache. """
         if not url and not path:
             raise ValueError("Either URL or path need to be specified.")
         if url and path:
@@ -48,15 +66,19 @@
         return page
 
     def cachePage(self, page):
+        """ Cache resolved information from the given page. """
         pass
 
     def uncachePages(self, except_url=None, only_required=False):
+        """ Invalidates resolved information for pages in the wiki. """
         pass
 
     def pageExists(self, url=None, path=None):
+        """ Returns whether a given page exists. """
         raise NotImplementedError()
 
     def getLinksTo(self, url):
+        """ Gets the list of links to a given page. """
         raise NotImplementedError()
 
     def _getPageByUrl(self, url, fields):
@@ -64,3 +86,22 @@
 
     def _getPageByPath(self, path, fields):
         raise NotImplementedError()
+
+    def addPageList(self, list_name, pages):
+        pass
+
+    def getPageList(self, list_name, fields=None, valid_only=True):
+        raise PageListNotFound(list_name)
+
+    def getPageListOrNone(self, list_name, fields=None, valid_only=True):
+        try:
+            return list(self.getPageList(list_name, fields, valid_only))
+        except PageListNotFound:
+            return None
+
+    def removePageList(self, list_name):
+        pass
+
+    def removeAllPageLists(self):
+        pass
+
--- a/wikked/db/sql.py	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/db/sql.py	Sat Oct 04 21:05:22 2014 -0700
@@ -12,9 +12,10 @@
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import (
     scoped_session, sessionmaker,
-    relationship, backref, load_only, subqueryload, joinedload)
+    relationship, backref, load_only, subqueryload, joinedload,
+    Load)
 from sqlalchemy.orm.exc import NoResultFound
-from wikked.db.base import Database
+from wikked.db.base import Database, PageListNotFound
 from wikked.page import Page, PageData, FileSystemPage
 from wikked.utils import split_page_url
 
@@ -125,10 +126,34 @@
     time_value = Column(DateTime)
 
 
+class SQLPageList(Base):
+    __tablename__ = 'page_lists'
+
+    id = Column(Integer, primary_key=True)
+    list_name = Column(String(64), unique=True)
+    is_valid = Column(Boolean)
+
+    page_refs = relationship(
+        'SQLPageListItem',
+        order_by='SQLPageListItem.id',
+        cascade='all, delete, delete-orphan')
+
+
+class SQLPageListItem(Base):
+    __tablename__ = 'page_list_items'
+
+    id = Column(Integer, primary_key=True)
+    list_id = Column(Integer, ForeignKey('page_lists.id'))
+    page_id = Column(Integer, ForeignKey('pages.id'))
+
+    page = relationship(
+            'SQLPage')
+
+
 class SQLDatabase(Database):
     """ A database cache based on SQL.
     """
-    schema_version = 6
+    schema_version = 7
 
     def __init__(self, config):
         Database.__init__(self)
@@ -403,34 +428,45 @@
             return None
         return SQLDatabasePage(self, page, fields)
 
-    def _addFieldOptions(self, query, fields, use_joined=True):
+    def _addFieldOptions(self, query, fields, use_joined=True,
+            use_load_obj=False):
         if fields is None:
             return query
 
+        if use_load_obj:
+            obj = Load(SQLPage)
+            l_load_only = obj.load_only
+            l_joinedload = obj.joinedload
+            l_subqueryload = obj.subqueryload
+        else:
+            l_load_only = load_only
+            l_joinedload = joinedload
+            l_subqueryload = subqueryload
+
         fieldnames = {
-                'local_meta': 'meta',
-                'local_links': 'links',
-                'meta': 'ready_meta',
-                'links': 'ready_links',
-                'text': 'ready_text',
-                'is_resolved': 'is_ready'}
+                'local_meta': SQLPage.meta,
+                'local_links': SQLPage.links,
+                'meta': SQLPage.ready_meta,
+                'links': SQLPage.ready_links,
+                'text': SQLPage.ready_text,
+                'is_resolved': SQLPage.is_ready}
         subqueryfields = {
                 'local_meta': SQLPage.meta,
                 'local_links': SQLPage.links,
                 'meta': SQLPage.ready_meta,
                 'links': SQLPage.ready_links}
         # Always load the ID.
-        query = query.options(load_only('id'))
+        query = query.options(l_load_only(SQLPage.id))
         # Load requested fields... some need subqueries.
         for f in fields:
             col = fieldnames.get(f) or f
-            query = query.options(load_only(col))
+            query = query.options(l_load_only(col))
             sqf = subqueryfields.get(f)
             if sqf:
                 if use_joined:
-                    query = query.options(joinedload(sqf))
+                    query = query.options(l_joinedload(sqf))
                 else:
-                    query = query.options(subqueryload(sqf))
+                    query = query.options(l_subqueryload(sqf))
         return query
 
     def _addPage(self, page):
@@ -463,6 +499,65 @@
 
         return po
 
+    def addPageList(self, list_name, pages):
+        page_list = self.session.query(SQLPageList)\
+                .filter(SQLPageList.list_name == list_name)\
+                .first()
+        if page_list is not None:
+            # We may have a previous list marked as non-valid. Let's
+            # revive it.
+            if page_list.is_valid:
+                raise Exception("Page list already exists and is valid: %s" % list_name)
+            logger.debug("Reviving page list '%s'." % list_name)
+            self.session.query(SQLPageListItem)\
+                    .filter(SQLPageListItem.list_id == page_list.id)\
+                    .delete()
+            page_list.is_valid = True
+        else:
+            logger.debug("Creating page list '%s'." % list_name)
+            page_list = SQLPageList()
+            page_list.list_name = list_name
+            page_list.is_valid = True
+            self.session.add(page_list)
+
+        for p in pages:
+            item = SQLPageListItem()
+            item.page_id = p._id
+            page_list.page_refs.append(item)
+
+        self.session.commit()
+
+    def getPageList(self, list_name, fields=None, valid_only=True):
+        page_list = self.session.query(SQLPageList)\
+                .filter(SQLPageList.list_name == list_name)\
+                .first()
+        if page_list is None or (
+                valid_only and not page_list.is_valid):
+            raise PageListNotFound(list_name)
+
+        q = self.session.query(SQLPageListItem)\
+                .filter(SQLPageListItem.list_id == page_list.id)\
+                .join(SQLPageListItem.page)
+        q = self._addFieldOptions(q, fields, use_load_obj=True)
+        for po in q.all():
+            yield SQLDatabasePage(self, po.page, fields)
+
+    def removePageList(self, list_name):
+        # Just mark the list as not valid anymore.
+        page_list = self.session.query(SQLPageList)\
+                .filter(SQLPageList.list_name == list_name)\
+                .first()
+        if page_list is None:
+            raise Exception("No such list: %s" % list_name)
+        page_list.is_valid = False
+        self.session.commit()
+
+    def removeAllPageLists(self):
+        q = self.session.query(SQLPageList)
+        for pl in q.all():
+            pl.is_valid = False
+        self.session.commit()
+
 
 class SQLDatabasePage(Page):
     """ A page that can load its properties from a database.
@@ -529,3 +624,4 @@
                 data.ext_links = [l.target_url for l in db_obj.ready_links]
 
         return data
+
--- a/wikked/views/__init__.py	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/views/__init__.py	Sat Oct 04 21:05:22 2014 -0700
@@ -5,7 +5,7 @@
 from flask import g, abort, jsonify
 from flask.ext.login import current_user
 from wikked.fs import PageNotFoundError
-from wikked.utils import split_page_url
+from wikked.utils import split_page_url, get_absolute_url
 from wikked.web import app
 
 
@@ -114,6 +114,51 @@
     return result
 
 
+class CircularRedirectError(Exception):
+    def __init__(self, url, visited):
+        super(CircularRedirectError, self).__init__(
+                "Circular redirect detected at '%s' "
+                "after visiting: %s" % (url, visited))
+
+
+class RedirectNotFound(Exception):
+    def __init__(self, url, not_found):
+        super(RedirectNotFound, self).__init__(
+                "Target redirect page '%s' not found from '%s'." %
+                (url, not_found))
+
+
+def get_redirect_target(path, fields=None, convert_url=False,
+                        check_perms=DONT_CHECK, first_only=False):
+    page = None
+    orig_path = path
+    visited_paths = []
+
+    while True:
+        page = get_page_or_none(
+                path,
+                fields=fields,
+                convert_url=convert_url,
+                check_perms=check_perms)
+        if page is None:
+            raise RedirectNotFound(orig_path, path)
+
+        visited_paths.append(path)
+        redirect_meta = page.getMeta('redirect')
+        if redirect_meta is None:
+            break
+
+        path = get_absolute_url(path, redirect_meta)
+        if first_only:
+            visited_paths.append(path)
+            break
+
+        if path in visited_paths:
+            raise CircularRedirectError(path, visited_paths)
+
+    return page, visited_paths
+
+
 COERCE_META = {
     'category': get_category_meta
     }
@@ -127,3 +172,38 @@
                 }
     return jsonify(data)
 
+
+def get_or_build_pagelist(list_name, builder, fields=None):
+    # If the wiki is using background jobs, we can accept invalidated
+    # lists... it just means that a background job is hopefully still
+    # just catching up.
+    # Otherwise, everything is synchronous and we need to build the
+    # list if needed.
+    build_inline = not app.config['WIKI_ASYNC_UPDATE']
+    page_list = g.wiki.db.getPageListOrNone(list_name, fields=fields,
+                                            valid_only=build_inline)
+    if page_list is None and build_inline:
+        app.logger.info("Regenerating list: %s" % list_name)
+        page_list = builder()
+        g.wiki.db.addPageList(list_name, page_list)
+
+    return page_list
+
+
+def get_generic_pagelist_builder(filter_func):
+    def builder():
+        # Make sure all pages have been resolved.
+        g.wiki.resolve()
+
+        pages = []
+        for page in g.wiki.getPages(no_endpoint_only=True,
+                                    fields=['url', 'title', 'meta']):
+            try:
+                if filter_func(page):
+                    pages.append(page)
+            except Exception as e:
+                app.logger.error("Error while inspecting page: %s" % page.url)
+                app.logger.error("   %s" % e)
+        return pages
+    return builder
+
--- a/wikked/views/read.py	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/views/read.py	Sat Oct 04 21:05:22 2014 -0700
@@ -1,12 +1,13 @@
 import time
 import urllib
-from flask import render_template, request, g, jsonify, make_response
+from flask import (render_template, request, g, jsonify, make_response,
+                   abort)
 from flask.ext.login import current_user
 from wikked.views import (get_page_meta, get_page_or_404, get_page_or_none,
-        is_page_readable,
+        is_page_readable, get_redirect_target,
         url_from_viewarg, split_url_from_viewarg,
+        RedirectNotFound, CircularRedirectError,
         CHECK_FOR_READ)
-from wikked.utils import get_absolute_url
 from wikked.web import app
 from wikked.scm.base import STATE_NAMES
 
@@ -73,26 +74,25 @@
     endpoint, path = split_url_from_viewarg(url)
     if endpoint is None:
         # Normal page.
-        visited_paths = []
-        while True:
-            page = get_page_or_404(
+        try:
+            page, visited_paths = get_redirect_target(
                     path,
                     fields=['url', 'title', 'text', 'meta'],
                     convert_url=False,
-                    check_perms=CHECK_FOR_READ)
-            visited_paths.append(path)
-            redirect_meta = page.getMeta('redirect')
-            if redirect_meta is None:
-                break
-            path = get_absolute_url(path, redirect_meta)
-            if no_redirect:
-                additional_info['redirects_to'] = path
-                break
-            if path in visited_paths:
-                app.logger.error("Circular redirect detected: " + url)
-                abort(409)
+                    check_perms=CHECK_FOR_READ,
+                    first_only=no_redirect)
+        except RedirectNotFound as e:
+            app.logger.exception(e)
+            abort(404)
+        except CircularRedirectError as e:
+            app.logger.exception(e)
+            abort(409)
+        if page is None:
+            abort(404)
 
-        if len(visited_paths) > 1:
+        if no_redirect:
+            additional_info['redirects_to'] = visited_paths[-1]
+        elif len(visited_paths) > 1:
             additional_info['redirected_from'] = visited_paths[:-1]
 
         result = {'meta': get_page_meta(page), 'text': page.text}
--- a/wikked/views/special.py	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/views/special.py	Sat Oct 04 21:05:22 2014 -0700
@@ -1,28 +1,54 @@
 from flask import g, jsonify, request, abort
-from wikked.views import is_page_readable, get_page_meta, get_page_or_none
+from wikked.views import (
+    is_page_readable, get_page_meta, get_page_or_none,
+    get_or_build_pagelist, get_generic_pagelist_builder,
+    get_redirect_target, CircularRedirectError, RedirectNotFound)
+from wikked.utils import get_absolute_url
 from wikked.web import app
 
 
+def orphans_filter_func(page):
+    for link in page.getIncomingLinks():
+        return False
+    return True
+
+
+def broken_redirects_filter_func(page):
+    redirect_meta = page.getMeta('redirect')
+    if redirect_meta is None:
+        return False
+
+    path = get_absolute_url(page.url, redirect_meta)
+    try:
+        target, visited = get_redirect_target(
+                path,
+                fields=['url', 'meta'])
+    except CircularRedirectError:
+        return True
+    except RedirectNotFound:
+        return True
+    return False
+
+
+def generic_pagelist_view(list_name, filter_func):
+    pages = get_or_build_pagelist(
+            list_name,
+            get_generic_pagelist_builder(filter_func),
+            fields=['url', 'title', 'meta'])
+    data = [get_page_meta(p) for p in pages if is_page_readable(p)]
+    result = {'pages': data}
+    return jsonify(result)
+
+
 @app.route('/api/orphans')
 def api_special_orphans():
-    orphans = []
-    for page in g.wiki.getPages(no_endpoint_only=True):
-        try:
-            if not is_page_readable(page):
-                continue
-            is_orphan = True
-            for link in page.getIncomingLinks():
-                is_orphan = False
-                break
-            if is_orphan:
-                orphans.append({'path': page.url, 'meta': get_page_meta(page)})
-        except Exception as e:
-            app.logger.error("Error while inspecting page: %s" % page.url)
-            app.logger.error("   %s" % e)
+    return generic_pagelist_view('orphans', orphans_filter_func)
+
 
-    result = {'orphans': orphans}
-    return jsonify(result)
-
+@app.route('/api/broken-redirects')
+def api_special_broken_redirects():
+    return generic_pagelist_view('broken_redirects',
+                                 broken_redirects_filter_func)
 
 
 @app.route('/api/search')
@@ -36,9 +62,15 @@
     for h in hits:
         page = get_page_or_none(h.url, convert_url=False)
         if page is not None and is_page_readable(page):
-            readable_hits.append({'url': h.url, 'title': h.title, 'text': h.hl_text})
+            readable_hits.append({
+                    'url': h.url,
+                    'title': h.title,
+                    'text': h.hl_text})
 
-    result = {'query': query, 'hit_count': len(readable_hits), 'hits': readable_hits}
+    result = {
+            'query': query,
+            'hit_count': len(readable_hits),
+            'hits': readable_hits}
     return jsonify(result)
 
 
@@ -55,6 +87,9 @@
         if page is not None and is_page_readable(page):
             readable_hits.append({'url': h.url, 'title': h.title})
 
-    result = {'query': query, 'hit_count': len(readable_hits), 'hits': readable_hits}
+    result = {
+            'query': query,
+            'hit_count': len(readable_hits),
+            'hits': readable_hits}
     return jsonify(result)
 
--- a/wikked/web.py	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/web.py	Sat Oct 04 21:05:22 2014 -0700
@@ -69,6 +69,10 @@
 app.logger.debug("Creating Flask application...")
 
 
+def remove_page_lists(wiki, url):
+    wiki.db.removeAllPageLists()
+
+
 # Set the default wiki parameters.
 app.wiki_params = WikiParameters(wiki_root)
 
@@ -81,6 +85,7 @@
 @app.before_request
 def before_request():
     wiki = Wiki(app.wiki_params)
+    wiki.post_update_hooks.append(remove_page_lists)
     wiki.start()
     g.wiki = wiki
 
--- a/wikked/wiki.py	Sat Oct 04 21:02:05 2014 -0700
+++ b/wikked/wiki.py	Sat Oct 04 21:05:22 2014 -0700
@@ -211,6 +211,7 @@
         self.auth = parameters.auth_factory()
 
         self._wiki_updater = parameters.wiki_updater
+        self.post_update_hooks = []
 
     @property
     def root(self):
@@ -341,8 +342,13 @@
         # Update the DB and index with the new/modified page.
         self.updatePage(path=page_info.path)
 
+        # Invalidate all page lists.
+        self.db.removeAllPageLists()
+
         # Update all the other pages.
         self._wiki_updater(self, url)
+        for hook in self.post_update_hooks:
+            hook(self, url)
 
     def revertPage(self, url, page_fields):
         """ Reverts the page with the given URL to an older revision.
@@ -376,6 +382,8 @@
 
         # Update all the other pages.
         self._wiki_updater(self, url)
+        for hook in self.post_update_hooks:
+            hook(self, url)
 
     def pageExists(self, url):
         """ Returns whether a page exists at the given URL.