class Database(object):
    """ The base class for a database cache.
    """
    def start(self, wiki):
        """ Called when the wiki is started. """
        pass

    def init(self, wiki):
        """ Called when a new wiki is created. """
        pass

    def postInit(self):
        """ Called after a new wiki has been created. """
        pass

    def close(self, exception):
        """ Called when the wiki is disposed of. """
        pass

    def reset(self, page_infos):
        """ Called when the DB cache should be re-build from scratch
            based on the given page infos. """
        pass

    def updatePage(self, page_info):
        """ Update the given page's cache info based on the given page
            info. """
        pass

    def updateAll(self, page_infos, force=False):
        """ Update all the pages in the wiki based on the given pages
            infos. """
        pass

    def getPageUrls(self, subdir=None, uncached_only=False):
        """ Return page URLs. """
        raise NotImplementedError()

    def getPages(self, subdir=None, meta_query=None, uncached_only=False,
                 endpoint_only=None, no_endpoint_only=False, fields=None):
        """ Return pages from the DB cache. """
        raise NotImplementedError()

    def getPage(self, url=None, path=None, fields=None, raise_if_none=True):
        """ Gets a page from the DB cache. """
        if not url and not path:
            raise ValueError("Either URL or path need to be specified.")
        if url and path:
            raise ValueError("Can't specify both URL and path.")
        if url:
            page = self._getPageByUrl(url, fields)
        elif path:
            page = self._getPageByPath(path, fields)
        else:
            raise NotImplementedError()
        if page is None and raise_if_none:
            raise PageNotFoundError(url or path)
        return page

    def cachePage(self, page):
        """ Cache resolved information from the given page. """
        pass

    def uncachePages(self, except_url=None, only_required=False):
        """ Invalidates resolved information for pages in the wiki. """
        pass

    def pageExists(self, url):
        """ Returns whether a given page exists. """
        raise NotImplementedError()

    def validateUrl(self, url):
        """ Returns the proper URL, given a URL with potentially different
            casing, or `None` if that URL doesn't exist. """
        raise NotImplementedError()

    def getLinksTo(self, url):
        """ Gets the list of links to a given page. """
        raise NotImplementedError()

    def _getPageByUrl(self, url, fields):
        raise NotImplementedError()

    def _getPageByPath(self, path, fields):
        raise NotImplementedError()

    def saveWantedPages(self, wanted_pages):
        pass

    def getWantedPages(self):
        raise NotImplementedError()

    def addPageList(self, list_name, pages):
        pass

    def getPageList(self, list_name, fields=None, valid_only=True):
        raise PageListNotFound(list_name)

    def getPageListOrNone(self, list_name, fields=None, valid_only=True):
        try:
            return list(self.getPageList(list_name, fields, valid_only))
        except PageListNotFound:
            return None

    def removePageList(self, list_name):
        pass

    def removeAllPageLists(self):
        pass class DatabaseUpgradeRequired(Exception):
    def __init__(self, *args):
        super().__init__(
            "This wiki needs a database update. Please run `wk reset`.",
            *args) class PageListNotFound(Exception):
    def __init__(self, list_name):
        super(PageListNotFound, self).__init__(
                "No such page list: %s" % list_name) class NoWantedPages(Exception):
    pass class Page(object):
    """ A wiki page. This is a non-functional class, as it doesn't know where
        to load things from. Use `FileSystemPage` or `DatabasePage` instead.
    """
    def __init__(self, wiki, data):
        self.wiki = wiki
        self._data = data

    @property
    def url(self):
        return self._data.url

    @property
    def path(self):
        return self._data.path

    @property
    def cache_time(self):
        return self._data.cache_time

    @property
    def is_resolved(self):
        return self._data.is_resolved

    @property
    def extension(self):
        if self._data.path is None:
            raise Exception("The 'path' field was not loaded.")
        return os.path.splitext(self._data.path)[1].lstrip('.')

    @property
    def filename(self):
        if self._data.path is None:
            raise Exception("The 'path' field was not loaded.")
        basename = os.path.basename(self._data.path)
        return os.path.splitext(basename)[0]

    @property
    def title(self):
        return self._data.title

    @property
    def raw_text(self):
        return self._data.raw_text

    @property
    def text(self):
        return self._data.text

    @property
    def links(self):
        return self._data.ext_links

    def getIncomingLinks(self):
        return self.wiki.db.getLinksTo(self.url)

    def getHistory(self):
        return self.wiki.scm.getHistory(self.path)

    def getState(self):
        return self.wiki.scm.getState(self.path)

    def getRevision(self, rev):
        return self.wiki.scm.getRevision(self.path, rev)

    def getDiff(self, rev1, rev2):
        return self.wiki.scm.diff(self.path, rev1, rev2)

    def getFormattedText(self):
        return self._data.formatted_text

    def getMeta(self, name=None, is_single=False):
        if name is None:
            return self._data.ext_meta
        return get_meta_value(self._data.ext_meta, name, is_single=is_single)

    def getLocalMeta(self, name=None, is_single=False):
        if name is None:
            return self._data.local_meta
        return get_meta_value(self._data.local_meta, name, is_single=is_single)

    def getLocalLinks(self):
        return self._data.local_links

    def _setExtendedData(self, result):
        self._data.text = result.text
        self._data.ext_meta = result.meta
        self._data.ext_links = result.out_links class PageData(object):
    def __init__(self):
        self.url = None
        self.path = None
        self.cache_time = None
        self.title = None
        self.raw_text = None
        self.formatted_text = None
        self.local_meta = None
        self.local_links = None
        self.text = None
        self.ext_meta = None
        self.ext_links = None class FileSystemPage(Page):
    """ A page that can load its properties directly from the file-system.
    """
    def __init__(self, wiki, page_info):
        data = self._loadFromPageInfo(wiki, page_info)
        super(FileSystemPage, self).__init__(wiki, data)

    def _loadFromPageInfo(self, wiki, page_info):
        data = PageData()
        data.url = page_info.url
        data.path = page_info.path
        data.cache_time = None
        data.raw_text = page_info.content

        # Format the page and get the meta properties.
        ctx = FormattingContext(page_info.url)
        f = PageFormatter()
        data.formatted_text = f.formatText(ctx, data.raw_text)
        data.local_meta = ctx.meta
        data.local_links = ctx.out_links

        # Add some common meta.
        data.title = data.local_meta.get('title')
        if data.title is None:
            filename = os.path.basename(data.path)
            filename_split = os.path.splitext(filename)
            data.title = re.sub(r'\-', ' ', filename_split[0])
        elif isinstance(data.title, list):
            data.title = data.title[0]

        return data class WantedPage:
    def __init__(self, url, wanted_by):
        self.url = url
        self.wanted_by = wanted_by def split_page_url(url):
    m = endpoint_regex.match(url)
    if m is None:
        return (None, url)
    endpoint = m.group(1)
    path = m.group(2)
    return (endpoint, path) def lower_url(url):
    return unicodedata.normalize("NFKD", url.casefold())
import os
import os.path
import string
import logging
import datetime
import threading
from sqlalchemy import (
    create_engine,
    and_,
    Column, Boolean, Integer, DateTime, ForeignKey,
    String, Text, UnicodeText)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import (
    scoped_session,
    relationship, backref, load_only, subqueryload, joinedload,
    Load)
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.orm.session import Session
from wikked.db.base import (
    Database,
    DatabaseUpgradeRequired, PageListNotFound, NoWantedPages)
from wikked.page import Page, PageData, FileSystemPage, WantedPage
from wikked.utils import split_page_url, lower_url
        from flask import g, _app_ctx_stack
                db_obj.ready_meta.append(SQLReadyMeta(name, value))
            else:
                for v in value:
                    db_obj.ready_meta.append(SQLReadyMeta(name, v))
            if name in ['include', 'query']:
                db_obj.needs_invalidate = True

        del db_obj.ready_links[:]
        for link_url in page._data.ext_links:
            db_obj.ready_links.append(SQLReadyLink(link_url))

        db_obj.is_ready = True

        self.session.commit()

    def uncachePages(self, except_url=None, only_required=False):
        q = self.session.query(SQLPage)\
                .options(load_only('id', 'url', 'needs_invalidate',
                                   'is_ready'))
        if except_url:
            q = q.filter(SQLPage.url != except_url)
        if only_required:
            q = q.filter(SQLPage.needs_invalidate is True)

        uncached_urls = []
        for p in q.all():
            uncached_urls.append(p.url)
            p.is_ready = False
        logger.debug("Uncaching: %s" % ', '.join(uncached_urls))
        self.session.commit()

    def pageExists(self, url):
        l = lower_url(url)
        q = self.session.query(SQLPage.id, SQLPage.url_ci).filter_by(url_ci=l)
        res = self.session.query(q.exists())
        return res.scalar()

    def validateUrl(self, url):
        l = lower_url(url)
        try:
            q = self.session.query(SQLPage.id, SQLPage.url, SQLPage.url_ci).\
                filter_by(url_ci=l).one()
            return q.url
        except NoResultFound:
            return None

    def getLinksTo(self, url):
        q = self.session.query(SQLReadyLink).\
                filter(SQLReadyLink.target_url == url).\
                join(SQLReadyLink.source).\
                all()
        for l in q:
            yield l.source.url

    def _getPageByUrl(self, url, fields):
        q = self.session.query(SQLPage).\
            filter(SQLPage.url == url)
        q = self._addFieldOptions(q, fields)
        page = q.first()
        if page is None:
            return None
        return SQLDatabasePage(self, page, fields)

    def _getPageByPath(self, path, fields):
        q = self.session.query(SQLPage).\
            filter(SQLPage.path == path)
        q = self._addFieldOptions(q, fields)
        page = q.first()
        if page is None:
            return None
        return SQLDatabasePage(self, page, fields)

    def _addFieldOptions(self, query, fields, use_joined=True,
                         use_load_obj=False):
        if fields is None:
            return query

        if use_load_obj:
            obj = Load(SQLPage)
            l_load_only = obj.load_only
            l_joinedload = obj.joinedload
            l_subqueryload = obj.subqueryload
        else:
            l_load_only = load_only
            l_joinedload = joinedload
            l_subqueryload = subqueryload

        fieldnames = {
                'local_meta': SQLPage.meta,
                'local_links': SQLPage.links,
                'meta': SQLPage.ready_meta,
                'links': SQLPage.ready_links,
                'text': SQLPage.ready_text,
                'is_resolved': SQLPage.is_ready}
        subqueryfields = {
                'local_meta': SQLPage.meta,
                'local_links': SQLPage.links,
                'meta': SQLPage.ready_meta,
                'links': SQLPage.ready_links}
        # Always load the ID.
        query = query.options(l_load_only(SQLPage.id))
        # Load requested fields... some need subqueries.
        for f in fields:
            col = fieldnames.get(f) or f
            query = query.options(l_load_only(col))
            sqf = subqueryfields.get(f)
            if sqf:
                if use_joined:
                    query = query.options(l_joinedload(sqf))
                else:
                    query = query.options(l_subqueryload(sqf))
        return query

    def _addPage(self, page):
        logger.debug("Adding page '%s' to SQL database." % page.url)

        po = SQLPage()
        po.cache_time = datetime.datetime.now()
        po.url = page.url
        po.url_ci = lower_url(page.url)

        po.endpoint, _ = split_page_url(page.url)