class IAxiomaticCommand(Interface):
    """
    Subcommand for 'axiomatic' and 'tell-axiom' command line programs.

    Should subclass twisted.python.usage.Options and provide a command to run.

    '.parent' attribute will be set to an object with a getStore method.
    """

    name = Attribute("""
    """)

    description = Attribute("""
    """) IN_MEMORY_DATABASE = ':memory:'
STORE_SELF_ID = -1
        _NEEDS_DEFAULT = object() # token for lookup failure
                T = mostRecent
                T = self.getOldVersionOf(typename, version)
class NoEmptyItems(Exception):
class AtomicFile(io.FileIO, object):
class SchedulingService(Service):
class BaseQuery:
class _FakeItemForFilter:
class ItemQuery(BaseQuery):
class MultipleItemQuery(BaseQuery):
class _DistinctQuery(object):
class _MultipleItemDistinctQuery(_DistinctQuery):
class AttributeQuery(BaseQuery):
class Store(Empowered):
        class FakeItem:
def _mkdirIfNotExists(dirname):
    def __init__(self, tempname, destpath):
    def close(self):
    def abort(self):
    def __init__(self):
    def startService(self):
    def stopService(self):
def storeServiceSpecialCase(st, pups):
def _typeIsTotallyUnknown(typename, version):
    def __init__(self, store, tableClass,
                 comparison=None, limit=None,
                 offset=None, sort=None):
    def cloneQuery(self, limit=_noItem, sort=_noItem):
    def __repr__(self):
    def explain(self):
    def _involvedTables(self):
    def _computeFromClause(self, tables):
    def _sqlAndArgs(self, verb, subject):
    def _runQuery(self, verb, subject):
    def locateCallSite(self):
    def _selectStuff(self, verb='SELECT'):
    def _massageData(self, row):
    def distinct(self):
    def __iter__(self):
    def __init__(self, store):
def _isColumnUnique(col):
    def __init__(self, *a, **k):
    def paginate(self, pagesize=20):
        def _AND(a, b):
    def _massageData(self, row):
    def getColumn(self, attributeName, raw=False):
    def count(self):
    def deleteFromStore(self):
            def itemsToDelete(attr):
    def __init__(self, *a, **k):
    def _involvedTables(self):
    def _massageData(self, row):
    def count(self):
    def distinct(self):
    def __init__(self, query):
    def cloneQuery(self, limit=_noItem, sort=_noItem):
    def __iter__(self):
    def count(self):
    def count(self):
    def __init__(self,
                 store,
                 tableClass,
                 comparison=None, limit=None,
                 offset=None, sort=None,
                 attribute=None,
                 raw=False):
    def _massageData(self, row):
    def count(self):
    def sum(self):
    def average(self):
    def max(self, default=_noDefault):
    def min(self, default=_noDefault):
    def _functionOnTarget(self, which, default):
def _storeBatchServiceSpecialCase(*args, **kwargs):
def _schedulerServiceSpecialCase(empowered, pups):
def _diffSchema(diskSchema, memorySchema):
    def _currentlyValidAsReferentFor(self, store):
    def __init__(self, dbdir=None, filesdir=None, debug=False, parent=None,
                 idInParent=None, journalMode=None):
            def logUpgradeFailure(aFailure):
    def _attachChild(self, child):
    def attachToParent(self):
    def _initSchema(self):
    def _startup(self):
    def _loadExistingIndexes(self):
    def _initdb(self, dbfname):
    def __repr__(self):
    def findOrCreate(self, userItemClass, __ifnew=None, **attrs):
    def newFilePath(self, *path):
    def newTemporaryFilePath(self, *path):
    def newFile(self, *path):
    def newDirectory(self, *path):
    def _loadTypeSchema(self):
    def _checkTypeSchemaConsistency(self, actualType, onDiskSchema):
    def _prepareOldVersionOf(self, typename, version, persistedSchema):
    def whenFullyUpgraded(self):
    def getOldVersionOf(self, typename, version):
    def findUnique(self, tableClass, comparison=None, default=_noItem):
    def findFirst(self, tableClass, comparison=None,
                  offset=None, sort=None, default=None):
    def query(self, tableClass, comparison=None,
              limit=None, offset=None, sort=None):
    def sum(self, summableAttribute, *a, **k):
    def count(self, *a, **k):
    def batchInsert(self, itemType, itemAttributes, dataRows):
    def _loadedItem(self, itemClass, storeID, attrs):
    def changed(self, item):
    def checkpoint(self):
    def transact(self, f, *a, **k):
    def _begin(self):
    def _setupTxnState(self):
    def _commit(self):
    def _postCommitHook(self):
    def _rollback(self):
    def revert(self):
    def _inMemoryRollback(self):
    def _cleanupTxnState(self):
    def close(self, _report=True):
    def avgms(self, l):
    def _indexNameOf(self, tableClass, attrname):
    def _tableNameOnlyFor(self, typename, version):
    def _tableNameFor(self, typename, version):
    def getTableName(self, tableClass):
    def getShortColumnName(self, attribute):
    def getColumnName(self, attribute):
    def getTypeID(self, tableClass):
    def _justCreateTable(self, tableClass):
    def _maybeCreateTable(self, tableClass, key):
    def _createIndexesFor(self, tableClass, extantIndexes):
    def getItemByID(self, storeID, default=_noItem, autoUpgrade=True):
    def querySchemaSQL(self, sql, args=()):
    def querySQL(self, sql, args=()):
    def _queryandfetch(self, sql, args):
    def createSQL(self, sql, args=()):
    def _execSQL(self, sql, args):
    def executeSchemaSQL(self, sql, args=()):
    def executeSQL(self, sql, args=()):
def timeinto(l, f, *a, **k): class _StoreUpgrade(object):
    def __init__(self, store):
    def upgradesPending(self):
    def checkUpgradePaths(self):
    def queueTypeUpgrade(self, oldtype):
    def upgradeItem(self, thisItem):
    def upgradeEverything(self):
    def upgradeBatch(self, n):
        def _doBatch(itemType):
def registerUpgrader(upgrader, typeName, oldVersion, newVersion):
def registerAttributeCopyingUpgrader(itemType, fromVersion, toVersion, postCopy=None):
    def upgrader(old):
def registerDeletionUpgrader(itemType, fromVersion, toVersion):
    def upgrader(old):
def upgradeAllTheWay(o):
def _hasExplicitOid(store, table):
def _upgradeTableOid(store, table, createTable, postCreate=lambda: None):
def upgradeSystemOid(store):
def upgradeExplicitOid(store): def normalize(qualName):
    def __new__(meta, name, bases, dictionary):
    def __cmp__(self, other):
    def _makeComparator(opfunc):
        def __op__(self, other):
    def __hash__(self):
def noop():
    def __init__(self, type):
    def __repr__(self):
    def fullyQualifiedName(self):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def getShortColumnName(self, store):
    def getColumnName(self, store):
    def __get__(self, item, type=None):
    def __get__(self, oself, type=None):
def serviceSpecialCase(item, pups):
    def inMemoryPowerUp(self, powerup, interface):
    def powerUp(self, powerup, interface=None, priority=0):
    def powerDown(self, powerup, interface=None):
    def __conform__(self, interface):
    def powerupsFor(self, interface):
    def interfacesFor(self, powerup):
    def _getPowerupInterfaces(self):
def transacted(func):
    def transactionified(item, *a, **kw):
def dependentItems(store, tableClass, comparisonFactory):
def allowDeletion(store, tableClass, comparisonFactory):
    def _currentlyValidAsReferentFor(self, store):
    def _schemaPrepareInsert(self, store):
    def store():
        def get(self):
        def set(self, store):
    def __repr__(self):
    def __subinit__(self, **kw):
    def __init__(self, **kw):
    def __finalizer__(self):
    def existingInStore(cls, store, storeID, attrs):
    def activate(self):
    def getSchema(cls):
    def persistentValues(self):
    def touch(self):
    def revert(self):
    def deleted(self):
    def stored(self):
    def committed(self):
    def checkpoint(self):
    def upgradeVersion(self, typename, oldversion, newversion, **kw):
    def deleteFromStore(self, deleteObject=True):
    def _baseSelectSQL(cls, st):
    def _baseInsertSQL(cls, st):
    def _baseDeleteSQL(cls, st):
    def _updateSQL(self):
    def getTableName(cls, store):
    def getTableAlias(cls, store, currentAliases):
    def __init__(self, placeholder, column):
    def __repr__(self):
    def __get__(self, inst):
    def fullyQualifiedName(self):
    def compare(self, other, op):
    def getShortColumnName(self, store):
    def getColumnName(self, store):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def __init__(self, itemClass):
    def __lt__(self, other):
    def __eq__(self, other):
    def __cmp__(self, other):
    def __getattr__(self, name):
    def getSchema(self):
    def getTableName(self, store):
    def getTableAlias(self, store, currentAliases):
def declareLegacyItem(typeName, schemaVersion, attributes, dummyBases=()):
def empowerment(iface, priority=0):
    def _deco(cls):
class NoInheritance(RuntimeError):
class NotInStore(RuntimeError):
class CantInstantiateItem(RuntimeError):
class MetaItem(slotmachine.SchemaMetaMachine):
class _StoreIDComparer(Comparable):
class _SpecialStoreIDAttribute(slotmachine.SetOnce):
class Empowered(object):
class Item(six.with_metaclass(MetaItem, Empowered, slotmachine._Strict)):
class _PlaceholderColumn(_ContainableMixin, _ComparisonOperatorMuxer,
                         _MatchingOperationMuxer, _OrderingMixin):
class Placeholder(object):
class _PowerupConnector(Item):
        T = slotmachine.SchemaMetaMachine.__new__(meta, name, bases, dictionary)
        T = None
            T = _legacyTypes[key]
                T = mostRecent
POWERUP_BEFORE = 1              # Priority for 'high' priority powerups.
POWERUP_AFTER = -1              # Priority for 'low' priority powerups. class TimeoutError(Exception):
class BadCredentials(UnauthorizedLogin):
class NoSuchUser(UnauthorizedLogin):
class MissingDomainPart(NoSuchUser):
class DuplicateUser(Exception):
class CannotOpenStore(RuntimeError):
class NoUpgradePathAvailable(CannotOpenStore):
class NoCrossStoreReferences(AttributeError):
class SQLError(RuntimeError):
class TableAlreadyExists(SQLError):
class UnknownItemType(Exception):
class SQLWarning(Warning):
class TableCreationConcurrencyError(RuntimeError):
class DuplicateUniqueItem(KeyError):
class ItemNotFound(KeyError):
class ItemClassesOnly(TypeError):
class ChangeRejected(Exception):
class DependencyError(Exception):
class DeletionDisallowed(ValueError):
class DataIntegrityError(RuntimeError):
class BrokenReference(DataIntegrityError):
class UpgraderRecursion(RuntimeError):
class ItemUpgradeError(RuntimeError):
class UnsatisfiedRequirement(AttributeError):
    def __init__(self, statement, timeout, underlying):
    def __init__(self, sql, args, underlying):
    def __str__(self):
    def __init__(self, originalFailure, storeID, oldType, newType): _NEEDS_FETCH = object()         # token indicating that a value was not found
_ASC = 'ASC'
_DESC = 'DESC'
LARGEST_POSITIVE = (2 ** 63)-1
LARGEST_NEGATIVE = -(2 ** 63)
MICRO = 1000000.
    NULLIFY = object()
    DISALLOW = object()
    CASCADE = object()
class _ComparisonOperatorMuxer:
class _MatchingOperationMuxer:
class _OrderingMixin:
class _ContainableMixin:
class Comparable(_ContainableMixin, _ComparisonOperatorMuxer,
                 _MatchingOperationMuxer, _OrderingMixin):
class SimpleOrdering:
class CompoundOrdering:
class UnspecifiedOrdering:
class SQLAttribute(inmemory, Comparable):
class TwoAttributeComparison:
class AttributeValueComparison:
class NullComparison:
class LikeFragment:
class LikeNull(LikeFragment):
class LikeValue(LikeFragment):
class LikeColumn(LikeFragment):
class LikeComparison:
class AggregateComparison:
class SequenceComparison:
class AND(AggregateComparison):
class OR(AggregateComparison):
class TableOrderComparisonWrapper(object):
class boolean(SQLAttribute):
class ConstraintError(TypeError):
class integer(SQLAttribute):
class bytes(SQLAttribute):
class InvalidPathError(ValueError):
class text(SQLAttribute):
class textlist(text):
class path(text):
class timestamp(integer):
class reference(integer):
class ieee754_double(SQLAttribute):
class AbstractFixedPointDecimal(integer):
class point1decimal(AbstractFixedPointDecimal):
class point2decimal(AbstractFixedPointDecimal):
class point3decimal(AbstractFixedPointDecimal):
class point4decimal(AbstractFixedPointDecimal):
class point5decimal(AbstractFixedPointDecimal):
class point6decimal(AbstractFixedPointDecimal):
class point7decimal(AbstractFixedPointDecimal):
class point8decimal(AbstractFixedPointDecimal):
class point9decimal(AbstractFixedPointDecimal):
class point10decimal(AbstractFixedPointDecimal):
class money(point4decimal):
    def compare(self, other, op):
    def __eq__(self, other):
    def __ne__(self, other):
    def __gt__(self, other):
    def __lt__(self, other):
    def __ge__(self, other):
    def __le__(self, other):
def compare(left, right, op):
    def _like(self, negate, firstOther, *others):
    def like(self, *others):
    def notLike(self, *others):
    def startswith(self, other):
    def endswith(self, other):
    def _asc(self):
    def _desc(self):
    def oneOf(self, seq, negate=False):
    def notOneOf(self, seq):
    def compare(self, other, sqlop):
    def __init__(self, attribute, direction=''):
    def orderColumns(self):
    def __repr__(self):
    def __add__(self, other):
    def __eq__(self, other):
    def __radd__(self, other):
    def __init__(self, seq):
    def __repr__(self):
    def __add__(self, other):
    def __radd__(self, other):
    def orderColumns(self):
    def __init__(self, null):
    def __add__(self, other):
    def orderColumns(self):
def compoundIndex(*columns):
    def __init__(self, doc='', indexed=False, default=None, allowNone=True, defaultFactory=None):
    def computeDefault(self):
    def reprFor(self, oself):
    def getShortColumnName(self, store):
    def getColumnName(self, store):
    def prepareInsert(self, oself, store):
    def coercer(self, value):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def requiredSlots(self, modname, classname, attrname):
    def fullyQualifiedName(self):
    def __repr__(self):
    def type():
        def get(self):
    def __get__(self, oself, cls=None):
    def loaded(self, oself, dbval):
    def _convertPyval(self, oself, pyval):
    def __set__(self, oself, pyval):
    def __init__(self, leftAttribute, operationString, rightAttribute):
    def getQuery(self, store):
    def getInvolvedTables(self):
    def getArgs(self, store):
    def __repr__(self):
    def __init__(self, attribute, operationString, value):
    def getQuery(self, store):
    def getArgs(self, store):
    def getInvolvedTables(self):
    def __repr__(self):
    def __init__(self, attribute, negate=False):
    def getQuery(self, store):
    def getArgs(self, store):
    def getInvolvedTables(self):
    def getLikeArgs(self):
    def getLikeQuery(self, st):
    def getLikeTables(self):
    def getLikeQuery(self, st):
    def __init__(self, value):
    def getLikeQuery(self, st):
    def getLikeArgs(self):
    def __init__(self, attribute):
    def getLikeQuery(self, st):
    def getLikeTables(self):
    def __init__(self, attribute, negate, likeParts):
    def getInvolvedTables(self):
    def getQuery(self, store):
    def getArgs(self, store):
    def __init__(self, *conditions):
    def getQuery(self, store):
    def getArgs(self, store):
    def getInvolvedTables(self):
    def __repr__(self):
    def __init__(self, attribute, container, negate):
    def _columnContainer(self, store):
    def _columnArgs(self, store):
    def _queryContainer(self, store):
    def _queryArgs(self, store):
    def _sequenceContainer(self, store):
    def _sequenceArgs(self, store):
    def getQuery(self, store):
    def getInvolvedTables(self):
    def __init__(self, tables, comparison):
    def getInvolvedTables(self):
    def getQuery(self, store):
    def getArgs(self, store):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def __init__(self,
                 attributeObj,
                 requiredTypes,
                 providedValue):
def requireType(attributeObj, value, typerepr, *types):
    def infilter(self, pyval, oself, store):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def like(self, *others):
    def notLike(self, *others):
    def startswith(self, other):
    def endswith(self, other):
    def __init__(self, caseSensitive=False, **kw):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def outfilter(self, dbval, oself):
    def infilter(self, pyval, oself, store):
    def __init__(self, relative=True, **kw):
    def prepareInsert(self, oself, store):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def __init__(self, doc='', indexed=True, allowNone=True, reftype=None,
                 whenDeleted=NULLIFY):
    def reprFor(self, oself):
    def __get__(self, oself, cls=None):
    def prepareInsert(self, oself, store):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def __init__(self, **kw):
    def infilter(self, pyval, oself, store):
    def outfilter(self, dbval, oself):
    def compare(self, other, sqlop): class _StoreUpgrade(object):
    """
    Manage Item upgrades and upgrade batching for a store.

    @type _currentlyUpgrading: C{dict}
    @ivar _currentlyUpgrading: A map of storeIDs to Items currently in the
        middle of an upgrader.  Used to make sure that the same item isn't
        upgraded reentrantly.

    @type _oldTypesRemaining: C{list}
    @ivar _oldTypesRemaining: All the old types which have not been fully
        upgraded in this database.
    """

    def __init__(self, store):
        self.store = store
        self._currentlyUpgrading = {}
        self._oldTypesRemaining = []


    def upgradesPending(self):
        return bool(self._oldTypesRemaining)

    upgradesPending = property(
        upgradesPending,
        doc="""
        Flag indicating whether there any types that still need to be upgraded
        or not.
        """)


    def checkUpgradePaths(self):
        """
        Check that all of the accumulated old Item types have a way to get
        from their current version to the latest version.

        @raise axiom.errors.NoUpgradePathAvailable: for any, and all, Items
            that do not have a valid upgrade path
        """
        cantUpgradeErrors = []

        for oldVersion in self._oldTypesRemaining:
            # We have to be able to get from oldVersion.schemaVersion to
            # the most recent type.

            currentType = _typeNameToMostRecentClass.get(
                oldVersion.typeName, None)

            if currentType is None:
                # There isn't a current version of this type; it's entirely
                # legacy, will be upgraded by deleting and replacing with
                # something else.
                continue

            typeInQuestion = oldVersion.typeName
            upgver = oldVersion.schemaVersion

            while upgver < currentType.schemaVersion:
                # Do we have enough of the schema present to upgrade?
                if ((typeInQuestion, upgver)
                    not in _upgradeRegistry):
                    cantUpgradeErrors.append(
                        "No upgrader present for %s (%s) from %d to %d" % (
                            typeInQuestion, qual(currentType), upgver,
                            upgver + 1))

                # Is there a type available for each upgrader version?
                if upgver+1 != currentType.schemaVersion:
                    if (typeInQuestion, upgver+1) not in _legacyTypes:
                        cantUpgradeErrors.append(
                            "Type schema required for upgrade missing:"
                            " %s version %d" % (
                                typeInQuestion, upgver+1))
                upgver += 1

            if cantUpgradeErrors:
                raise NoUpgradePathAvailable('\n    '.join(cantUpgradeErrors))


    def queueTypeUpgrade(self, oldtype):
        """
        Queue a type upgrade for C{oldtype}.
        """
        if oldtype not in self._oldTypesRemaining:
            self._oldTypesRemaining.append(oldtype)


    def upgradeItem(self, thisItem):
        """
        Upgrade a legacy item.

        @raise axiom.errors.UpgraderRecursion: If the given item is already in
            the process of being upgraded.
        """
        sid = thisItem.storeID
        if sid in self._currentlyUpgrading:
            raise UpgraderRecursion()
        self._currentlyUpgrading[sid] = thisItem
        try:
            return upgradeAllTheWay(thisItem)
        finally:
            self._currentlyUpgrading.pop(sid)


    def upgradeEverything(self):
        """
        Upgrade every item in the store, one at a time.

        @raise axiom.errors.ItemUpgradeError: if an item upgrade failed

        @return: A generator that yields for each item upgrade.
        """
        return self.upgradeBatch(1)


    def upgradeBatch(self, n):
        """
        Upgrade the entire store in batches, yielding after each batch.

        @param n: Number of upgrades to perform per transaction
        @type n: C{int}

        @raise axiom.errors.ItemUpgradeError: if an item upgrade failed

        @return: A generator that yields after each batch upgrade. This needs
            to be consumed for upgrading to actually take place.
        """
        store = self.store

        def _doBatch(itemType):
            upgradedAnything = False

            for theItem in store.query(itemType, limit=n):
                upgradedAnything = True
                try:
                    self.upgradeItem(theItem)
                except:
                    f = Failure()
                    raise ItemUpgradeError(
                        f, theItem.storeID, itemType,
                        _typeNameToMostRecentClass[itemType.typeName])

            return upgradedAnything

        if self.upgradesPending:
            didAny = False

            while self._oldTypesRemaining:
                t0 = self._oldTypesRemaining[0]

                upgradedAnything = store.transact(_doBatch, t0)
                if not upgradedAnything:
                    self._oldTypesRemaining.pop(0)
                    if didAny:
                        msg("{} finished upgrading {}".format(store.dbdir.path, qual(t0)))
                    continue
                elif not didAny:
                    didAny = True
                    msg("{} beginning upgrade...".format(store.dbdir.path))

                yield None

            if didAny:
                msg("{} completely upgraded.".format(store.dbdir.path)) def declareLegacyItem(typeName, schemaVersion, attributes, dummyBases=()):
    """
    Generate a dummy subclass of Item that will have the given attributes,
    and the base Item methods, but no methods of its own.  This is for use
    with upgrading.

    @param typeName: a string, the Axiom TypeName to have attributes for.
    @param schemaVersion: an int, the (old) version of the schema this is a proxy
    for.
    @param attributes: a dict mapping {columnName: attr instance} describing
    the schema of C{typeName} at C{schemaVersion}.

    @param dummyBases: a sequence of 4-tuples of (baseTypeName,
    baseSchemaVersion, baseAttributes, baseBases) representing the dummy bases
    of this legacy class.
    """
    if (typeName, schemaVersion) in _legacyTypes:
        return _legacyTypes[typeName, schemaVersion]
    if dummyBases:
        realBases = [declareLegacyItem(*A) for A in dummyBases]
    else:
        realBases = (Item,)
    attributes = attributes.copy()
    attributes['__module__'] = 'item_dummy'
    attributes['__legacy__'] = True
    attributes['typeName'] = typeName
    attributes['schemaVersion'] = schemaVersion
    result = type(str('DummyItem<%s,%d>' % (typeName, schemaVersion)),
                  realBases,
                  attributes)
    assert result is not None, 'wtf, {!r}'.format(type)
    _legacyTypes[(typeName, schemaVersion)] = result
    return result class AxiomaticSubCommandMixin(object):
class _AxiomaticCommandMeta(type):
class AxiomaticSubCommand(usage.Options, AxiomaticSubCommandMixin):
class AxiomaticCommand(six.with_metaclass(_AxiomaticCommandMeta, usage.Options, AxiomaticSubCommandMixin)):
class PIDMixin:
class Stop(usage.Options, PIDMixin):
class Status(usage.Options, PIDMixin):
class Start(twistd.ServerOptions):
class Options(usage.Options):
    def decodeCommandLine(self, cmdline):
    def __new__(cls, name, bases, attrs):
    def _sendSignal(self, signal):
    def signalServer(self, signal):
    def postOptions(self):
    def postOptions(self):
    def subCommands(self):
    def getArguments(self, store, args):
    def parseOptions(self, args):
    def subCommands():
        def get(self):
    def usedb(self, potentialdb):
    def getStoreDirectory(self):
    def getStore(self):
    def postOptions(self):
def main(argv=None): class Store(Empowered):
    """
    I am a database that Axiom Items can be stored in.

    Store an item in me by setting its 'store' attribute to be me.

    I can be created one of two ways::

        Store()                      # Create an in-memory database

        Store("/path/to/file.axiom") # create an on-disk database in the
                                     # directory /path/to/file.axiom

    @ivar typeToTableNameCache: a dictionary mapping Item subclass type objects
    to the fully-qualified sqlite table name where items of that type are
    stored.  This cache is generated from the saved schema metadata when this
    store is opened and updated when schema changes from other store objects
    (such as in other processes) are detected.

    @cvar __legacy__: an L{Item} may refer to a L{Store} via a L{reference},
    and this attribute tells the item reference system that the store itself is
    not an old version of an item; i.e. it does not need to have its upgraders
    invoked.

    @cvar storeID: an L{Item} may refer to a L{Store} via a L{reference}, and
    this attribute tells the item reference system that the L{Store} has a
    special ID that to use (which is never allocated to any item).
    """

    aggregateInterfaces = {
        IService: storeServiceSpecialCase,
        IServiceCollection: storeServiceSpecialCase,
        iaxiom.IBatchService: _storeBatchServiceSpecialCase,
        iaxiom.IScheduler: _schedulerServiceSpecialCase}

    transaction = None          # set of objects changed in the current transaction
    touched = None              # set of objects changed since the last checkpoint

    databaseName = 'main'       # can differ if database is attached to another
                                # database.

    dbdir = None # FilePath to the Axiom database directory, or None for
                 # in-memory Stores.
    filesdir = None # FilePath to the filesystem-storage subdirectory of the
                    # database directory, or None for in-memory Stores.

    store = property(lambda self: self) # I have a 'store' attribute because I
                                        # am 'stored' within myself; this is
                                        # also for references to use.


    # Counter indicating things are going on which disallows changes to the
    # database.  Callbacks dispatched to application code while this is
    # non-zero will reject database changes with a ChangeRejected exception.
    _rejectChanges = 0

    # The following method and attributes are the ad-hoc interface required as
    # targets of attributes.reference attributes.  (In other words, the store
    # is a little bit like a fake item.)  These should probably eventually be
    # on an interface somewhere, and be better named.

    def _currentlyValidAsReferentFor(self, store):
        """
        Check to see if this store is currently valid as a target of a
        reference from an item in the given L{Store}.  This is true iff the
        given L{Store} is this L{Store}.

        @param store: the store that the referring item is present in.

        @type store: L{Store}
        """
        if store is self:
            return True
        else:
            return False

    __legacy__ = False

    storeID = STORE_SELF_ID


    def __init__(self, dbdir=None, filesdir=None, debug=False, parent=None,
                 idInParent=None, journalMode=None):
        """
        Create a store.

        @param dbdir: A L{FilePath} to (or name of) an existing Axiom directory, or
        directory that does not exist yet which will be created as this Store
        is instantiated.  If unspecified, this database will be kept in memory.

        @param filesdir: A L{FilePath} to (or name of) a directory to keep files in for in-memory
        stores. An exception will be raised if both this attribute and C{dbdir}
        are specified.

        @param debug: set to True if this Store should print out every SQL
        statement it sends to SQLite.

        @param parent: (internal) If this is opened using an
        L{axiom.substore.Substore}, a reference to its parent.

        @param idInParent: (internal) If this is opened using an
        L{axiom.substore.Substore}, the storeID of the item within its parent
        which opened it.

        @raises: C{ValueError} if both C{dbdir} and C{filesdir} are specified.
        """
        if parent is not None or idInParent is not None:
            assert parent is not None
            assert idInParent is not None
        self.parent = parent
        self.idInParent = idInParent
        self.debug = debug
        self.autocommit = True
        self.journalMode = journalMode
        self.queryTimes = []
        self.execTimes = []

        self._inMemoryPowerups = {}

        self._attachedChildren = {} # database name => child store object

        self.statementCache = {} # non-normalized => normalized qmark SQL
                                 # statements

        self.activeTables = {}  # tables which have had items added/removed
                                # this run

        self.objectCache = _fincache.FinalizingCache()

        self.typenameAndVersionToID = {} # map database-persistent typename and
                                         # version to an oid in the types table

        self.typeToInsertSQLCache = {}
        self.typeToSelectSQLCache = {}
        self.typeToDeleteSQLCache = {}

        self.typeToTableNameCache = {}
        self.attrNameToColumnNameCache = {}  # map fully-qualified attribute
                                             # names (native str?) to database
                                             # column names

        self._upgradeManager = upgrade._StoreUpgrade(self)

        self._axiom_service = None


        if self.parent is None:
            self._upgradeService = SchedulingService()
        else:
            # Substores should hook into their parent, since they shouldn't
            # expect to have their own substore service started.
            self._upgradeService = self.parent._upgradeService


        # OK!  Everything that can be set up without touching the filesystem
        # has been done.  Let's get ready to open the actual database...

        _initialOpenFailure = None
        if dbdir is None:
            self._initdb(IN_MEMORY_DATABASE)
            self._initSchema()
            self._memorySubstores = []
            if filesdir is not None:
                if not isinstance(filesdir, filepath.FilePath):
                    filesdir = filepath.FilePath(filesdir)
                self.filesdir = filesdir
                if not self.filesdir.isdir():
                    self.filesdir.makedirs()
                    self.filesdir.child("temp").createDirectory()
        else:
            if filesdir is not None:
                raise ValueError("Only one of dbdir and filesdir"
                                 " may be specified")
            if not isinstance(dbdir, filepath.FilePath):
                dbdir = filepath.FilePath(dbdir)
                # required subdirs: files, temp, run
                # datafile: db.sqlite
            self.dbdir = dbdir
            self.filesdir = self.dbdir.child('files')

            if not dbdir.isdir():
                tempdbdir = dbdir.temporarySibling()
                tempdbdir.makedirs() # maaaaaaaybe this is a bad idea, we
                                     # probably shouldn't be doing this
                                     # automatically.
                for child in ('files', 'temp', 'run'):
                    tempdbdir.child(child).createDirectory()
                self._initdb(tempdbdir.child('db.sqlite').path)
                self._initSchema()
                self.close(_report=False)
                try:
                    tempdbdir.moveTo(dbdir)
                except:
                    _initialOpenFailure = Failure()

            try:
                self._initdb(dbdir.child('db.sqlite').path)
            except:
                if _initialOpenFailure is not None:
                    log.msg("Failed to initialize axiom database."
                            "  Possible cause of error: ")
                    log.err(_initialOpenFailure)
                raise

        self.transact(self._startup)

        # _startup may have found some things which we must now upgrade.
        if self._upgradeManager.upgradesPending:
            # Automatically upgrade when possible.
            self._upgradeComplete = self._upgradeService.coop.cooperate(
                self._upgradeManager.upgradeEverything())
            d = self._upgradeComplete.whenDone()
            def logUpgradeFailure(aFailure):
                if aFailure.check(errors.ItemUpgradeError):
                    log.err(aFailure.value.originalFailure, 'Item upgrade error')
                log.err(aFailure, "upgrading %r failed" % (self,))
            d.addErrback(logUpgradeFailure)
        else:
            self._upgradeComplete = None

        log.msg(
            interface=iaxiom.IStatEvent,
            store_opened=self.dbdir is not None and self.dbdir.path or '')

    _childCounter = 0

    def _attachChild(self, child):
        "attach a child database, returning an identifier for it"
        self._childCounter += 1
        databaseName = 'child_db_%d' % (self._childCounter,)
        self._attachedChildren[databaseName] = child
        # ATTACH DATABASE statements can't use bind paramaters, blech.
        self.executeSQL("ATTACH DATABASE '%s' AS %s" % (
                child.dbdir.child('db.sqlite').path,
                databaseName,))
        return databaseName

    attachedToParent = False

    def attachToParent(self):
        assert self.parent is not None, 'must have a parent to attach'
        assert self.transaction is None, "can't attach within a transaction"

        self.close()

        self.attachedToParent = True
        self.databaseName = self.parent._attachChild(self)
        self.connection = self.parent.connection
        self.cursor = self.parent.cursor

#     def detachFromParent(self):
#         pass


    def _initSchema(self):
        # No point in even attempting to transactionalize this:
        # every single statement is a CREATE TABLE or a CREATE
        # INDEX and those commit transactions silently anyway.
        for stmt in _schema.BASE_SCHEMA:
            self.executeSchemaSQL(stmt)


    def _startup(self):
        """
        Called during __init__.  Check consistency of schema in database with
        classes in memory.  Load all Python modules for stored items, and load
        version information for upgrader service to run later.
        """
        typesToCheck = []

        for oid, module, typename, version in self.querySchemaSQL(_schema.ALL_TYPES):
            if self.debug:
                print()
                print('SCHEMA:', oid, module, typename, version)
            if typename not in _typeNameToMostRecentClass:
                try:
                    namedAny(module)
                except ValueError as err:
                    raise ImportError('cannot find module ' + module, str(err))
            self.typenameAndVersionToID[typename, version] = oid

        # Can't call this until typenameAndVersionToID is populated, since this
        # depends on building a reverse map of that.
        persistedSchema = self._loadTypeSchema()

        # Now that we have persistedSchema, loop over everything again and
        # prepare old types.
        for (typename, version), typeID in six.iteritems(self.typenameAndVersionToID):
            cls = _typeNameToMostRecentClass.get(typename)

            if cls is not None:
                if version != cls.schemaVersion:
                    typesToCheck.append(
                        self._prepareOldVersionOf(
                            typename, version, persistedSchema))
                else:
                    typesToCheck.append(cls)

        for cls in typesToCheck:
            self._checkTypeSchemaConsistency(cls, persistedSchema)

        # Schema is consistent!  Now, if I forgot to create any indexes last
        # time I saw this table, do it now...
        extantIndexes = self._loadExistingIndexes()
        for cls in typesToCheck:
            self._createIndexesFor(cls, extantIndexes)

        self._upgradeManager.checkUpgradePaths()


    def _loadExistingIndexes(self):
        """
        Return a C{set} of the SQL indexes which already exist in the
        underlying database.  It is important to load all of this information
        at once (as opposed to using many CREATE INDEX IF NOT EXISTS statements
        or many CREATE INDEX statements and handling the errors) to minimize
        the cost of opening a store.  Loading all the indexes at once is much
        faster than doing pretty much anything that involves doing something
        once per required index.
        """
        # Totally SQLite-specific: look up what indexes exist already in
        # sqlite_master so we can skip trying to create them (which can be
        # really slow).
        return set(
            name
            for (name,) in self.querySchemaSQL(
                "SELECT name FROM *DATABASE*.sqlite_master "
                "WHERE type = 'index'"))


    def _initdb(self, dbfname):
        self.connection = Connection.fromDatabaseName(dbfname)
        self.cursor = self.connection.cursor()
        if self.journalMode is not None:
            self.querySchemaSQL(
                'PRAGMA *DATABASE*.journal_mode = {}'.format(self.journalMode))


    def __repr__(self):
        dbdir = self.dbdir
        if self.dbdir is None:
            dbdir = '(in memory)'

        return "<Store {dbdir}@{id:#x}".format(dbdir=dbdir, id=id(self))


    def findOrCreate(self, userItemClass, __ifnew=None, **attrs):
        """
        Usage::

            s.findOrCreate(userItemClass [, function] [, x=1, y=2, ...])

        Example::

            class YourItemType(Item):
                a = integer()
                b = text()
                c = integer()

            def f(x):
                print x, \"-- it's new!\"
            s.findOrCreate(YourItemType, f, a=1, b=u'2')

        Search for an item with columns in the database that match the passed
        set of keyword arguments, returning the first match if one is found,
        creating one with the given attributes if not.  Takes an optional
        positional argument function to call on the new item if it is new.
        """
        andargs = []
        for k, v in six.iteritems(attrs):
            col = getattr(userItemClass, k)
            andargs.append(col == v)

        if len(andargs) == 0:
            cond = []
        elif len(andargs) == 1:
            cond = [andargs[0]]
        else:
            cond = [attributes.AND(*andargs)]

        for result in self.query(userItemClass, *cond):
            return result
        newItem = userItemClass(store=self, **attrs)
        if __ifnew is not None:
            __ifnew(newItem)
        return newItem

    def newFilePath(self, *path):
        p = self.filesdir
        for subdir in path:
            p = p.child(subdir)
        return p

    def newTemporaryFilePath(self, *path):
        p = self.dbdir.child('temp')
        for subdir in path:
            p = p.child(subdir)
        return p

    def newFile(self, *path):
        """
        Open a new file somewhere in this Store's file area.

        @param path: a sequence of path segments.

        @return: an L{AtomicFile}.
        """
        assert len(path) > 0, "newFile requires a nonzero number of segments"
        if self.dbdir is None:
            if self.filesdir is None:
                raise RuntimeError("This in-memory store has no file directory")
            else:
                tmpbase = self.filesdir
        else:
            tmpbase = self.dbdir
        tmpname = tmpbase.child('temp').child(str(next(tempCounter)) + ".tmp")
        return AtomicFile(tmpname.path, self.newFilePath(*path))

    def newDirectory(self, *path):
        p = self.filesdir
        for subdir in path:
            p = p.child(subdir)
        return p


    def _loadTypeSchema(self):
        """
        Load all of the stored schema information for all types known by this
        store.  It's important to load everything all at once (rather than
        loading the schema for each type separately as it is needed) to keep
        store opening fast.  A single query with many results is much faster
        than many queries with a few results each.

        @return: A dict with two-tuples of item type name and schema version as
            keys and lists of five-tuples of attribute schema information for
            that type.  The elements of the five-tuple are::

              - a string giving the name of the Python attribute
              - a string giving the SQL type
              - a boolean indicating whether the attribute is indexed
              - the Python attribute type object (eg, axiom.attributes.integer)
              - a string giving documentation for the attribute
        """

        # Oops, need an index going the other way.  This only happens once per
        # store open, and it's based on data queried from the store, so there
        # doesn't seem to be any broader way to cache and re-use the result.
        # However, if we keyed the resulting dict on the database typeID rather
        # than (typeName, schemaVersion), we wouldn't need the information this
        # dict gives us.  That would mean changing the callers of this function
        # to use typeID instead of that tuple, which may be possible.  Probably
        # only represents a very tiny possible speedup.
        typeIDToNameAndVersion = {}
        for key, value in six.iteritems(self.typenameAndVersionToID):
            typeIDToNameAndVersion[value] = key

        # Indexing attribute, ordering by it, and getting rid of row_offset
        # from the schema and the sorted() here doesn't seem to be any faster
        # than doing this.
        persistedSchema = sorted(self.querySchemaSQL(_schema.PERSISTED_SCHEMA))

        # This is trivially (but measurably!) faster than getattr(attributes,
        # pythontype).
        getAttribute = attributes.__dict__.__getitem__

        result = {}
        for (attribute, typeID, sqltype, indexed, pythontype,
             docstring) in persistedSchema:
            key = typeIDToNameAndVersion[typeID]
            if key not in result:
                result[key] = []
            result[key].append((
                    attribute, sqltype, indexed,
                    getAttribute(pythontype), docstring))
        return result


    def _checkTypeSchemaConsistency(self, actualType, onDiskSchema):
        """
        Called for all known types at database startup: make sure that what we
        know (in memory) about this type agrees with what is stored about this
        type in the database.

        @param actualType: A L{MetaItem} instance which is associated with a
            table in this store.  The schema it defines in memory will be
            checked against the schema known in the database to ensure they
            agree.

        @param onDiskSchema: A mapping from L{MetaItem} instances (such as
            C{actualType}) to the schema known in the database and associated
            with C{actualType}.

        @raise RuntimeError: if the schema defined by C{actualType} does not
            match the database-present schema given in C{onDiskSchema} or if
            C{onDiskSchema} contains a newer version of the schema associated
            with C{actualType} than C{actualType} represents.
        """
        # make sure that both the runtime and the database both know about this
        # type; if they don't both know, we can't check that their views are
        # consistent
        try:
            inMemorySchema = _inMemorySchemaCache[actualType]
        except KeyError:
            inMemorySchema = _inMemorySchemaCache[actualType] = [
                (storedAttribute.attrname, storedAttribute.sqltype)
                for (name, storedAttribute) in actualType.getSchema()]

        key = (actualType.typeName, actualType.schemaVersion)
        persistedSchema = [(storedAttribute[0], storedAttribute[1])
                           for storedAttribute in onDiskSchema[key]]
        if inMemorySchema != persistedSchema:
            raise RuntimeError(
                "Schema mismatch on already-loaded %r <%r> object version %d:\n%s" %
                (actualType, actualType.typeName, actualType.schemaVersion,
                 _diffSchema(persistedSchema, inMemorySchema)))

        if actualType.__legacy__:
            return

        if (key[0], key[1] + 1) in onDiskSchema:
            raise RuntimeError(
                "Memory version of %r is %d; database has newer" % (
                    actualType.typeName, key[1]))


    # finally find old versions of the data and prepare to upgrade it.
    def _prepareOldVersionOf(self, typename, version, persistedSchema):
        """
        Note that this database contains old versions of a particular type.
        Create the appropriate dummy item subclass and queue the type to be
        upgraded.

        @param typename: The I{typeName} associated with the schema for which
            to create a dummy item class.

        @param version: The I{schemaVersion} of the old version of the schema
            for which to create a dummy item class.

        @param persistedSchema: A mapping giving information about all schemas
            stored in the database, used to create the attributes of the dummy
            item class.
        """
        appropriateSchema = persistedSchema[typename, version]
        # create actual attribute objects
        dummyAttributes = {}
        for (attribute, sqlType, indexed, pythontype,
             docstring) in appropriateSchema:
            atr = pythontype(indexed=indexed, doc=docstring)
            dummyAttributes[attribute] = atr
        dummyBases = []
        oldType = declareLegacyItem(
            typename, version, dummyAttributes, dummyBases)
        self._upgradeManager.queueTypeUpgrade(oldType)
        return oldType


    def whenFullyUpgraded(self):
        """
        Return a Deferred which fires when this Store has been fully upgraded.
        """
        if self._upgradeComplete is not None:
            return self._upgradeComplete.whenDone()
        else:
            return defer.succeed(None)

    def getOldVersionOf(self, typename, version):
        return _legacyTypes[typename, version]



        # grab the schema for that version
        # look up upgraders which push it forward

    def findUnique(self, tableClass, comparison=None, default=_noItem):
        """
        Find an Item in the database which should be unique.  If it is found,
        return it.  If it is not found, return 'default' if it was passed,
        otherwise raise L{errors.ItemNotFound}.  If more than one item is
        found, raise L{errors.DuplicateUniqueItem}.

        @param comparison: implementor of L{iaxiom.IComparison}.

        @param default: value to use if the item is not found.
        """
        results = list(self.query(tableClass, comparison, limit=2))
        lr = len(results)

        if lr == 0:
            if default is _noItem:
                raise errors.ItemNotFound(comparison)
            else:
                return default
        elif lr == 2:
            raise errors.DuplicateUniqueItem(comparison, results)
        elif lr == 1:
            return results[0]
        else:
            raise AssertionError("limit=2 database query returned 3+ results: ",
                                 comparison, results)


    def findFirst(self, tableClass, comparison=None,
                  offset=None, sort=None, default=None):
        """
        Usage::

            s.findFirst(tableClass [, query arguments except 'limit'])

        Example::

            class YourItemType(Item):
                a = integer()
                b = text()
                c = integer()
            ...
            it = s.findFirst(YourItemType,
                             AND(YourItemType.a == 1,
                                 YourItemType.b == u'2'),
                                 sort=YourItemType.c.descending)

        Search for an item with columns in the database that match the passed
        comparison, offset and sort, returning the first match if one is found,
        or the passed default (None if none is passed) if one is not found.
        """

        limit = 1
        for item in self.query(tableClass, comparison, limit, offset, sort):
            return item
        return default

    def query(self, tableClass, comparison=None,
              limit=None, offset=None, sort=None):
        """
        Return a generator of instances of C{tableClass},
        or tuples of instances if C{tableClass} is a
        tuple of classes.

        Examples::

            fastCars = s.query(Vehicle,
                axiom.attributes.AND(
                    Vehicle.wheels == 4,
                    Vehicle.maxKPH > 200),
                limit=100,
                sort=Vehicle.maxKPH.descending)

            quotesByClient = s.query( (Client, Quote),
                axiom.attributes.AND(
                    Client.active == True,
                    Quote.client == Client.storeID,
                    Quote.created >= someDate),
                limit=10,
                sort=(Client.name.ascending,
                      Quote.created.descending))

        @param tableClass: a subclass of Item to look for instances of,
        or a tuple of subclasses.

        @param comparison: a provider of L{IComparison}, or None, to match
        all items available in the store. If tableClass is a tuple, then
        the comparison must refer to all Item subclasses in that tuple,
        and specify the relationships between them.

        @param limit: an int to limit the total length of the results, or None
        for all available results.

        @param offset: an int to specify a starting point within the available
        results, or None to start at 0.

        @param sort: an L{ISort}, something that comes from an SQLAttribute's
        'ascending' or 'descending' attribute.

        @return: an L{ItemQuery} object, which is an iterable of Items or
        tuples of Items, according to tableClass.
        """
        if isinstance(tableClass, tuple):
            queryClass = MultipleItemQuery
        else:
            queryClass = ItemQuery

        return queryClass(self, tableClass, comparison, limit, offset, sort)

    def sum(self, summableAttribute, *a, **k):
        args = (self, summableAttribute.type) + a
        return AttributeQuery(attribute=summableAttribute,
                              *args, **k).sum()
    def count(self, *a, **k):
        return self.query(*a, **k).count()

    def batchInsert(self, itemType, itemAttributes, dataRows):
        """
        Create multiple items in the store without loading
        corresponding Python objects into memory.

        the items' C{stored} callback will not be called.

        Example::

            myData = [(37, u"Fred",  u"Wichita"),
                      (28, u"Jim",   u"Fresno"),
                      (43, u"Betty", u"Dubuque")]
            myStore.batchInsert(FooItem,
                                [FooItem.age, FooItem.name, FooItem.city],
                                myData)

        @param itemType: an Item subclass to create instances of.

        @param itemAttributes: an iterable of attributes on the Item subclass.

        @param dataRows: an iterable of iterables, each the same
        length as C{itemAttributes} and containing data corresponding
        to each attribute in it.

        @return: None.
        """
        class FakeItem:
            pass
        _NEEDS_DEFAULT = object() # token for lookup failure
        fakeOSelf = FakeItem()
        fakeOSelf.store = self
        sql = itemType._baseInsertSQL(self)
        indices = {}
        schema = [attr for (name, attr) in itemType.getSchema()]
        for i, attr in enumerate(itemAttributes):
            indices[attr.attrname] = i
        for row in dataRows:
            oid = self.store.executeSchemaSQL(
                _schema.CREATE_OBJECT, [self.store.getTypeID(itemType)])
            insertArgs = [oid]
            for attr in schema:
                i = indices.get(attr.attrname, _NEEDS_DEFAULT)
                if i is _NEEDS_DEFAULT:
                    pyval = attr.default
                else:
                    pyval = row[i]
                dbval = attr._convertPyval(fakeOSelf, pyval)
                insertArgs.append(dbval)
            self.executeSQL(sql, insertArgs)

    def _loadedItem(self, itemClass, storeID, attrs):
        try:
            result = self.objectCache.get(storeID)
            # XXX do checks on consistency between attrs and DB object, maybe?
        except KeyError:
            result = itemClass.existingInStore(self, storeID, attrs)
            if not result.__legacy__:
                self.objectCache.cache(storeID, result)
        return result


    def changed(self, item):
        """
        An item in this store was changed.  Add it to the current transaction's
        list of changed items, if a transaction is currently underway, or raise
        an exception if this L{Store} is currently in a state which does not
        allow changes.
        """
        if self._rejectChanges:
            raise errors.ChangeRejected()
        if self.transaction is not None:
            self.transaction.add(item)
            self.touched.add(item)


    def checkpoint(self):
        self._rejectChanges += 1
        try:
            for item in self.touched:
                # XXX: it should be possible here, using various clever hacks, to
                # automatically optimize functionally identical statements into
                # executemany.
                item.checkpoint()
            self.touched.clear()
        finally:
            self._rejectChanges -= 1

    executedThisTransaction = None
    tablesCreatedThisTransaction = None

    def transact(self, f, *a, **k):
        """
        Execute C{f(*a, **k)} in the context of a database transaction.

        Any changes made to this L{Store} by C{f} will be committed when C{f}
        returns.  If C{f} raises an exception, those changes will be reverted
        instead.

        If a transaction is already in progress (in this thread - ie, if a
        frame executing L{Store.transact} is already on the call stack), this
        will B{not} start a nested transaction.  Changes will not be committed
        until the existing transaction completes, and an exception raised by
        C{f} will not revert changes made by C{f}.  You probably don't want to
        ever call this if another transaction is in progress.

        @return: Whatever C{f(*a, **kw)} returns.
        @raise: Whatever C{f(*a, **kw)} raises, or a database exception.
        """
        if self.transaction is not None:
            return f(*a, **k)
        if self.attachedToParent:
            return self.parent.transact(f, *a, **k)
        try:
            self._begin()
            try:
                result = f(*a, **k)
                self.checkpoint()
            except:
                exc = Failure()
                try:
                    self.revert()
                except:
                    log.err(exc)
                    raise
                raise
            else:
                self._commit()
            return result
        finally:
            self._cleanupTxnState()

    # The following three methods are necessary...
    # - in PySQLite: because PySQLite has some buggy transaction handling which
    #   makes it impossible to issue explicit BEGIN statements - which we
    #   _need_ to do to provide guarantees for read/write transactions.

    def _begin(self):
        if self.debug:
            print('<'*10, 'BEGIN', '>'*10)
        self.cursor.execute("BEGIN IMMEDIATE TRANSACTION")
        self._setupTxnState()

    def _setupTxnState(self):
        self.executedThisTransaction = []
        self.tablesCreatedThisTransaction = []
        if self.attachedToParent:
            self.transaction = self.parent.transaction
            self.touched = self.parent.touched
        else:
            self.transaction = set()
            self.touched = set()
        self.autocommit = False
        for sub in self._attachedChildren.values():
            sub._setupTxnState()

    def _commit(self):
        if self.debug:
            print('*'*10, 'COMMIT', '*'*10)
        # self.connection.commit()
        self.cursor.execute("COMMIT")
        log.msg(interface=iaxiom.IStatEvent, stat_commits=1)
        self._postCommitHook()


    def _postCommitHook(self):
        self._rejectChanges += 1
        try:
            for committed in self.transaction:
                committed.committed()
        finally:
            self._rejectChanges -= 1


    def _rollback(self):
        if self.debug:
            print('>'*10, 'ROLLBACK', '<'*10)
        # self.connection.rollback()
        self.cursor.execute("ROLLBACK")
        log.msg(interface=iaxiom.IStatEvent, stat_rollbacks=1)


    def revert(self):
        self._rollback()
        self._inMemoryRollback()


    def _inMemoryRollback(self):
        self._rejectChanges += 1
        try:
            for item in self.transaction:
                item.revert()
        finally:
            self._rejectChanges -= 1
        self.transaction.clear()
        for tableClass in self.tablesCreatedThisTransaction:
            del self.typenameAndVersionToID[tableClass.typeName,
                                            tableClass.schemaVersion]
            # Clear all cache related to this table
            for cache in (self.typeToInsertSQLCache,
                          self.typeToDeleteSQLCache,
                          self.typeToSelectSQLCache,
                          self.typeToTableNameCache) :
                if tableClass in cache:
                    del cache[tableClass]
            qualifiedStoreIDName = tableClass.storeID.fullyQualifiedName()
            if qualifiedStoreIDName in self.attrNameToColumnNameCache:
                del self.attrNameToColumnNameCache[qualifiedStoreIDName]
            for name, attr in tableClass.getSchema():
                attrFQN = attr.fullyQualifiedName()
                if attrFQN in self.attrNameToColumnNameCache:
                    del self.attrNameToColumnNameCache[attrFQN]

        for sub in self._attachedChildren.values():
            sub._inMemoryRollback()


    def _cleanupTxnState(self):
        self.autocommit = True
        self.transaction = None
        self.touched = None
        self.executedThisTransaction = None
        self.tablesCreatedThisTransaction = []
        for sub in list(self._attachedChildren.values()):
            sub._cleanupTxnState()

    def close(self, _report=True):
        self.cursor.close()
        self.connection.close()
        self.cursor = self.connection = None
        if self.debug and _report:
            if not self.queryTimes:
                print('no queries')
            else:
                print('query:', self.avgms(self.queryTimes))
            if not self.execTimes:
                print('no execs')
            else:
                print('exec:', self.avgms(self.execTimes))

    def avgms(self, l):
        return 'count: %d avg: %dus' % (len(l),
                                        int( (sum(l)/len(l)) * 1000000.),)

    def _indexNameOf(self, tableClass, attrname):
        """
        Return the unqualified (ie, no database name) name of the given
        attribute of the given table.

        @type tableClass: L{MetaItem}
        @param tableClass: The Python class associated with a table in the
            database.

        @param attrname: A sequence of the names of the columns of the
            indicated table which will be included in the named index.

        @return: A C{str} giving the name of the index which will index the
            given attributes of the given table.
        """
        return "axiomidx_%s_v%d_%s" % (tableClass.typeName,
                                       tableClass.schemaVersion,
                                       '_'.join(attrname))


    def _tableNameOnlyFor(self, typename, version):
        return 'item_{}_v{:d}'.format(typename, version)


    def _tableNameFor(self, typename, version):
        return '.'.join([self.databaseName,
                         self._tableNameOnlyFor(typename, version)])


    def getTableName(self, tableClass):
        """
        Retrieve the fully qualified name of the table holding items
        of a particular class in this store.  If the table does not
        exist in the database, it will be created as a side-effect.

        @param tableClass: an Item subclass

        @raises axiom.errors.ItemClassesOnly: if an object other than a
            subclass of Item is passed.

        @return: a string
        """
        if not (isinstance(tableClass, type) and issubclass(tableClass, item.Item)):
            raise errors.ItemClassesOnly("Only subclasses of Item have table names.")

        if tableClass not in self.typeToTableNameCache:
            self.typeToTableNameCache[tableClass] = self._tableNameFor(tableClass.typeName, tableClass.schemaVersion)
            # make sure the table exists
            self.getTypeID(tableClass)
        return self.typeToTableNameCache[tableClass]


    def getShortColumnName(self, attribute):
        """
        Retreive the column name for a particular attribute in this
        store.  The attribute must be bound to an Item subclass (its
        type must be valid). If the underlying table does not exist in
        the database, it will be created as a side-effect.

        @param tableClass: an Item subclass

        @return: a string

        XXX: The current implementation does not really match the
        description, which is actually more restrictive. But it will
        be true soon, so I guess it is ok for now.  The reason is
        that this method is used during table creation.
        """
        if isinstance(attribute, _StoreIDComparer):
            # The column is named "oid" instead of "storeID" for backwards
            # compatibility with the implicit oid/rowid column in old Stores.
            return 'oid'
        return '[' + attribute.attrname + ']'


    def getColumnName(self, attribute):
        """
        Retreive the fully qualified column name for a particular
        attribute in this store.  The attribute must be bound to an
        Item subclass (its type must be valid). If the underlying
        table does not exist in the database, it will be created as a
        side-effect.

        @param tableClass: an Item subclass

        @return: a string
        """
        name = attribute.fullyQualifiedName()
        if name not in self.attrNameToColumnNameCache:
            self.attrNameToColumnNameCache[name] = '.'.join(
                (self.getTableName(attribute.type),
                 self.getShortColumnName(attribute)))
        return self.attrNameToColumnNameCache[name]


    def getTypeID(self, tableClass):
        """
        Retrieve the typeID associated with a particular table in the
        in-database schema for this Store.  A typeID is an opaque integer
        representing the Item subclass, and the associated table in this
        Store's SQLite database.

        @param tableClass: a subclass of Item

        @return: an integer
        """
        key = (tableClass.typeName,
               tableClass.schemaVersion)
        if key in self.typenameAndVersionToID:
            return self.typenameAndVersionToID[key]
        return self.transact(self._maybeCreateTable, tableClass, key)


    def _justCreateTable(self, tableClass):
        """
        Execute the table creation DDL for an Item subclass.

        Indexes are *not* created.

        @type tableClass: type
        @param tableClass: an Item subclass
        """
        sqlstr = []
        sqlarg = []

        # needs to be calculated including version
        tableName = self._tableNameFor(tableClass.typeName,
                                       tableClass.schemaVersion)

        sqlstr.append("CREATE TABLE %s (" % tableName)

        # The column is named "oid" instead of "storeID" for backwards
        # compatibility with the implicit oid/rowid column in old Stores.
        sqlarg.append("oid INTEGER PRIMARY KEY")
        for nam, atr in tableClass.getSchema():
            sqlarg.append("\n%s %s" %
                          (atr.getShortColumnName(self), atr.sqltype))

        sqlstr.append(', '.join(sqlarg))
        sqlstr.append(')')
        self.createSQL(''.join(sqlstr))


    def _maybeCreateTable(self, tableClass, key):
        """
        A type ID has been requested for an Item subclass whose table was not
        present when this Store was opened.  Attempt to create the table, and
        if that fails because another Store object (perhaps in another process)
        has created the table, re-read the schema.  When that's done, return
        the typeID.

        This method is internal to the implementation of getTypeID.  It must be
        run in a transaction.

        @param tableClass: an Item subclass
        @param key: a 2-tuple of the tableClass's typeName and schemaVersion

        @return: a typeID for the table; a new one if no table exists, or the
        existing one if the table was created by another Store object
        referencing this database.
        """
        try:
            self._justCreateTable(tableClass)
        except errors.TableAlreadyExists:
            # Although we don't have a memory of this table from the last time
            # we called "_startup()", another process has updated the schema
            # since then.
            self._startup()
            return self.typenameAndVersionToID[key]


        typeID = self.executeSchemaSQL(_schema.CREATE_TYPE,
                                       [tableClass.typeName,
                                        tableClass.__module__,
                                        tableClass.schemaVersion])

        self.typenameAndVersionToID[key] = typeID

        if self.tablesCreatedThisTransaction is not None:
            self.tablesCreatedThisTransaction.append(tableClass)

        # If the new type is a legacy type (not the current version), we need
        # to queue it for upgrade to ensure that if we are in the middle of an
        # upgrade, legacy items of this version get upgraded.
        cls = _typeNameToMostRecentClass.get(tableClass.typeName)
        if cls is not None and tableClass.schemaVersion != cls.schemaVersion:
            self._upgradeManager.queueTypeUpgrade(tableClass)

        # We can pass () for extantIndexes here because since the table didn't
        # exist for tableClass, none of its indexes could have either.
        # Whatever checks _createIndexesFor will make would give the same
        # result against the actual set of existing indexes as they will
        # against ().
        self._createIndexesFor(tableClass, ())

        for n, (name, storedAttribute) in enumerate(tableClass.getSchema()):
            self.executeSchemaSQL(
                _schema.ADD_SCHEMA_ATTRIBUTE,
                [typeID, n, storedAttribute.indexed, storedAttribute.sqltype,
                 storedAttribute.allowNone, storedAttribute.attrname,
                 storedAttribute.doc, storedAttribute.__class__.__name__])
            # XXX probably need something better for pythontype eventually,
            # when we figure out a good way to do user-defined attributes or we
            # start parameterizing references.

        return typeID


    def _createIndexesFor(self, tableClass, extantIndexes):
        """
        Create any indexes which don't exist and are required by the schema
        defined by C{tableClass}.

        @param tableClass: A L{MetaItem} instance which may define a schema
            which includes indexes.

        @param extantIndexes: A container (anything which can be the right-hand
            argument to the C{in} operator) which contains the unqualified
            names of all indexes which already exist in the underlying database
            and do not need to be created.
        """
        try:
            indexes = _requiredTableIndexes[tableClass]
        except KeyError:
            indexes = set()
            for nam, atr in tableClass.getSchema():
                if atr.indexed:
                    indexes.add(((atr.getShortColumnName(self),), (atr.attrname,)))
                for compound in atr.compoundIndexes:
                    indexes.add((tuple(inatr.getShortColumnName(self) for inatr in compound),
                                 tuple(inatr.attrname for inatr in compound)))
            _requiredTableIndexes[tableClass] = indexes

        # _ZOMFG_ SQL is such a piece of _shit_: you can't fully qualify the
        # table name in CREATE INDEX statements because the _INDEX_ is fully
        # qualified!

        indexColumnPrefix = '.'.join(self.getTableName(tableClass).split(".")[1:])

        for (indexColumns, indexAttrs) in indexes:
            nameOfIndex = self._indexNameOf(tableClass, indexAttrs)
            if nameOfIndex in extantIndexes:
                continue
            csql = 'CREATE INDEX %s.%s ON %s(%s)' % (
                self.databaseName, nameOfIndex, indexColumnPrefix,
                ', '.join(indexColumns))
            self.createSQL(csql)


    def getItemByID(self, storeID, default=_noItem, autoUpgrade=True):
        """
        Retrieve an item by its storeID, and return it.

        Note: most of the failure modes of this method are catastrophic and
        should not be handled by application code.  The only one that
        application programmers should be concerned with is KeyError.  They are
        listed for educational purposes.

        @param storeID: an L{int} which refers to the store.

        @param default: if passed, return this value rather than raising in the
        case where no Item is found.

        @raise TypeError: if storeID is not an integer.

        @raise UnknownItemType: if the storeID refers to an item row in the
        database, but the corresponding type information is not available to
        Python.

        @raise RuntimeError: if the found item's class version is higher than
        the current application is aware of.  (In other words, if you have
        upgraded a database to a new schema and then attempt to open it with a
        previous version of the code.)

        @raise errors.ItemNotFound: if no item existed with the given storeID.

        @return: an Item, or the given default, if it was passed and no row
        corresponding to the given storeID can be located in the database.
        """

        if not isinstance(storeID, six.integer_types):
            raise TypeError("storeID *must* be an int or long, not %r" % (
                    type(storeID).__name__,))
        if storeID == STORE_SELF_ID:
            return self
        try:
            return self.objectCache.get(storeID)
        except KeyError:
            pass
        log.msg(interface=iaxiom.IStatEvent, stat_cache_misses=1, key=storeID)
        results = self.querySchemaSQL(_schema.TYPEOF_QUERY, [storeID])
        assert (len(results) in [1, 0]),\
            "Database panic: more than one result for TYPEOF!"
        if results:
            typename, module, version = results[0]
            useMostRecent = False
            moreRecentAvailable = False

            # The schema may have changed since the last time I saw the
            # database.  Let's look to see if this is suspiciously broken...

            if _typeIsTotallyUnknown(typename, version):
                # Another process may have created it - let's re-up the schema
                # and see what we get.
                self._startup()

                # OK, all the modules have been loaded now, everything
                # verified.
                if _typeIsTotallyUnknown(typename, version):

                    # If there is STILL no inkling of it anywhere, we are
                    # almost certainly boned.  Let's tell the user in a
                    # structured way, at least.
                    raise errors.UnknownItemType(
                        "cannot load unknown schema/version pair: %r %r - id: %r" %
                        (typename, version, storeID))

            if typename in _typeNameToMostRecentClass:
                moreRecentAvailable = True
                mostRecent = _typeNameToMostRecentClass[typename]

                if mostRecent.schemaVersion < version:
                    raise RuntimeError("%s:%d - was found in the database and most recent %s is %d" %
                                       (typename, version, typename, mostRecent.schemaVersion))
                if mostRecent.schemaVersion == version:
                    useMostRecent = True
            if useMostRecent:
                T = mostRecent
            else:
                T = self.getOldVersionOf(typename, version)

            # for the moment we're going to assume no inheritance
            attrs = self.querySQL(T._baseSelectSQL(self), [storeID])
            if len(attrs) == 0:
                if default is _noItem:
                    raise errors.ItemNotFound(
                        'No results for known-to-be-good object')
                return default
            elif len(attrs) > 1:
                raise errors.DataIntegrityError(
                    'Too many results for {:d}'.format(storeID))
            attrs = attrs[0]
            x = T.existingInStore(self, storeID, attrs)
            if moreRecentAvailable and (not useMostRecent) and autoUpgrade:
                # upgradeVersion will do caching as necessary, we don't have to
                # cache here.  (It must, so that app code can safely call
                # upgradeVersion and get a consistent object out of it.)
                x = self.transact(self._upgradeManager.upgradeItem, x)
            elif not x.__legacy__:
                # We loaded the most recent version of an object
                self.objectCache.cache(storeID, x)
            return x
        if default is _noItem:
            raise errors.ItemNotFound(storeID)
        return default


    def querySchemaSQL(self, sql, args=()):
        sql = sql.replace("*DATABASE*", self.databaseName)
        return self.querySQL(sql, args)


    def querySQL(self, sql, args=()):
        """For use with SELECT (or SELECT-like PRAGMA) statements.
        """
        if self.debug:
            result = timeinto(self.queryTimes, self._queryandfetch, sql, args)
        else:
            result = self._queryandfetch(sql, args)
        return result


    def _queryandfetch(self, sql, args):
        if self.debug:
            print('**', sql, '--', ', '.join(map(str, args)))
        self.cursor.execute(sql, args)
        before = time.time()
        result = list(self.cursor)
        after = time.time()
        if after - before > 2.0:
            log.msg('Extremely long list(cursor): %s' % (after - before,))
            log.msg(sql)
            # import traceback; traceback.print_stack()
        if self.debug:
            print('  lastrow:', self.cursor.lastRowID())
            print('  result:', result)
        return result


    def createSQL(self, sql, args=()):
        """
        For use with auto-committing statements such as CREATE TABLE or CREATE
        INDEX.
        """
        before = time.time()
        self._execSQL(sql, args)
        after = time.time()
        if after - before > 2.0:
            log.msg('Extremely long CREATE: %s' % (after - before,))
            log.msg(sql)
            # import traceback; traceback.print_stack()


    def _execSQL(self, sql, args):
        if self.debug:
            rows = timeinto(self.execTimes, self._queryandfetch, sql, args)
        else:
            rows = self._queryandfetch(sql, args)
        assert not rows
        return sql


    def executeSchemaSQL(self, sql, args=()):
        sql = sql.replace("*DATABASE*", self.databaseName)
        return self.executeSQL(sql, args)


    def executeSQL(self, sql, args=()):
        """
        For use with UPDATE or INSERT statements.
        """
        sql = self._execSQL(sql, args)
        result = self.cursor.lastRowID()
        if self.executedThisTransaction is not None:
            self.executedThisTransaction.append((result, sql, args))
        return result class SubStore(Item):

    schemaVersion = 1
    typeName = 'substore'

    storepath = path()
    substore = inmemory()

    @classmethod
    def createNew(cls, store, pathSegments):
        """
        Create a new SubStore, allocating a new file space for it.
        """
        if isinstance(pathSegments, six.string_types):
            raise ValueError(
                'Received {!r} instead of a sequence'.format(pathSegments))
        if store.dbdir is None:
            self = cls(store=store, storepath=None)
        else:
            storepath = store.newDirectory(*pathSegments)
            self = cls(store=store, storepath=storepath)
        self.open()
        self.close()
        return self


    def close(self):
        self.substore.close()
        del self.substore._openSubStore
        del self.substore


    def open(self, debug=None):
        if hasattr(self, 'substore'):
            return self.substore
        else:
            if debug is None:
                debug = self.store.debug
            s = self.substore = self.createStore(debug, self.store.journalMode)
            s._openSubStore = self # don't fall out of cache as long as the
                                   # store is alive!
            return s


    def createStore(self, debug, journalMode=None):
        """
        Create the actual Store this Substore represents.
        """
        if self.storepath is None:
            self.store._memorySubstores.append(self) # don't fall out of cache
            if self.store.filesdir is None:
                filesdir = None
            else:
                filesdir = (self.store.filesdir.child("_substore_files")
                                               .child(str(self.storeID))
                                               .path)
            return Store(parent=self.store,
                         filesdir=filesdir,
                         idInParent=self.storeID,
                         debug=debug,
                         journalMode=journalMode)
        else:
            return Store(self.storepath.path,
                         parent=self.store,
                         idInParent=self.storeID,
                         debug=debug,
                         journalMode=journalMode)


    def __conform__(self, interface):
        """
        I adapt my store object to whatever interface I am adapted to.  This
        allows for avatar adaptation in L{axiom.userbase} to work properly
        without having to know explicitly that all 'avatars' objects are
        SubStore instances, since it is valid to have non-SubStore avatars,
        which are simply adaptable to the cred interfaces they represent.
        """
        ifa = interface(self.open(debug=self.store.debug), None)
        return ifa


    def indirect(self, interface):
        """
        Like __conform__, I adapt my store to whatever interface I am asked to
        produce a powerup for.  This allows for app stores to be installed as
        powerups for their site stores directly, rather than having an
        additional item type for each interface that we might wish to adapt to.
        """
        return interface(self) class Upgrade(axiomatic.AxiomaticCommand):
    name = 'upgrade'
    description = 'Synchronously upgrade an Axiom store and substores'

    optParameters = [
        ('count', 'n', '100', 'Number of upgrades to perform per transaction')]

    errorMessageFormat = 'Error upgrading item (with typeName=%s and storeID=%d) from version %d to %d.'

    def upgradeEverything(self, store):
        """
        Upgrade all the items in C{store}.
        """
        for dummy in store._upgradeManager.upgradeBatch(self.count):
            pass


    def upgradeStore(self, store):
        """
        Recursively upgrade C{store}.
        """
        self.upgradeEverything(store)
        upgradeExplicitOid(store)

        for substore in store.query(SubStore):
            print(u'Upgrading: {!r}'.format(substore))
            self.upgradeStore(substore.open())

    def perform(self, store, count):
        """
        Upgrade C{store} performing C{count} upgrades per transaction.

        Also, catch any exceptions and print out something useful.
        """
        self.count = count

        try:
            self.upgradeStore(store)
            print(u'Upgrade complete')
        except errors.ItemUpgradeError as e:
            print(u'Upgrader error:')
            e.originalFailure.printTraceback(file=sys.stdout)
            print(self.errorMessageFormat % (
                e.oldType.typeName, e.storeID, e.oldType.schemaVersion,
                e.newType.schemaVersion))


    def postOptions(self):
        try:
            count = int(self['count'])
        except ValueError:
            raise usage.UsageError('count must be an integer')

        siteStore = self.parent.getStore()
        self.perform(siteStore, count) class CommandStub(object):
    """
    Mock for L{axiom.scripts.axiomatic.Options} which is always set as the
    C{parent} attribute of an I{axiomatic} subcommand.

    @ivar _store: The L{Store} associated which will be supplied to the
        subcommand.
    """
    def __init__(self, store, subCommand):
        self._store = store
        self.subCommand = subCommand


    def getSynopsis(self):
        return "Usage: axiomatic [options]"


    def getStore(self):
        return self._store def callWithStdoutRedirect(f, *a, **kw):
    """
    Redirect stdout and invoke C{f}.

    @returns: C{(returnValue, stdout})
    """
    if six.PY3:
        output = io.StringIO()
    else:
        # We need to emulate stdio implicit encoding goofiness on python 2,
        # because some things write text to stdout and some things write bytes
        import StringIO
        output = StringIO.StringIO()
    sys.stdout, stdout = output, sys.stdout
    try:
        result = f(*a, **kw)
    finally:
        sys.stdout = stdout
    return result, output
import sys, io
import six
from zope.interface import Interface
from zope.interface.verify import verifyObject
from twisted.trial import unittest
from twisted.python import filepath
from twisted.application.service import IService
from twisted.internet.defer import maybeDeferred, succeed
from twisted.python.reflect import namedModule
from twisted.python import log
from axiom.iaxiom import IAxiomaticCommand
from axiom import store, upgrade, item, errors, attributes
from axiom.upgrade import _StoreUpgrade
from axiom.item import declareLegacyItem
from axiom.scripts import axiomatic
from axiom.store import Store
from axiom.substore import SubStore
from axiom.plugins.axiom_plugins import Upgrade
from axiom.test.util import CommandStub, callWithStdoutRedirect
# Copyright (c) 2008 Divmod.  See LICENSE for details.

"""
Tests for the Axiom upgrade system.
"""







def axiomInvalidate(itemClass):
    """
    Remove the registered item class from the Axiom module system's memory,
    including: the item's current schema, legacy schema declarations, and
    upgraders.

    This makes it possible, for example, to reload a module without Axiom
    complaining about it.

    This API is still in a test module because it is _NOT YET SAFE_ for using
    while databases are open; it does not interact with open databases' caches,
    for example.

    @param itemClass: an Item subclass that you no longer wish to use.
    """
    # Note, be very careful not to use comparison on attributes here.  For
    # example, do not use list.remove(), since it is equality based. -exarkun
    for cascades in six.itervalues(attributes._cascadingDeletes):
        for i in six.moves.builtins.range(len(cascades) - 1, -1, -1):
            if cascades[i].type is itemClass:
                del cascades[i]

    store._typeNameToMostRecentClass.pop(itemClass.typeName, None)


    for (tnam, schever) in list(item._legacyTypes.keys()):