'''
Rom - the Redis object mapper for Python

Copyright 2013-2016 Josiah Carlson

Released under the LGPL license version 2.1 and version 3 (you can choose
which you'd like to be bound under).
'''

from datetime import datetime, date, time as dtime
from decimal import Decimal as _Decimal
from functools import wraps
from itertools import product
import json

import six

from .exceptions import (ORMError, InvalidOperation, ColumnError,
    MissingColumn, InvalidColumnValue, RestrictError)
from .util import (_numeric_keygen, _string_keygen, _many_to_one_keygen,
    _boolean_keygen, dt2ts, ts2dt, t2ts, ts2t, session, _connect,
    STRING_INDEX_KEYGENS_STR)

NULL = object()
MODELS = {}
MODELS_REFERENCED = {}
_NUMERIC = (0, 0.0, _Decimal('0'), datetime(1970, 1, 1), date(1970, 1, 1), dtime(0, 0, 0))
NO_ACTION_DEFAULT = object()
SKIP_ON_DELETE = object()
ON_DELETE = ('no action', 'restrict', 'cascade', 'set null', 'set default')
six.string_types_ex = six.string_types
if six.PY3:
    six.string_types_ex += (bytes,)

def is_numeric(allowed):
    return any(isinstance(i, allowed) for i in _NUMERIC)

def is_string(allowed):
    allowed = (allowed,) if isinstance(allowed, type) else allowed
    return any(issubclass(a, i) for a,i in product(allowed, six.string_types_ex))

def _restrict(entity, attr, refs):
    name = entity._namespace
    name2 = refs[0]._namespace
    return RestrictError(
        "Cannot delete entity %s with pk %s, %i foreign references from %s.%s exist"%(
            name, getattr(entity, entity._pkey), len(refs), name2, attr))

def _on_delete(ent):
    '''
    This function handles all on_delete semantics defined on OneToMany columns.

    This function only exists because 'cascade' is *very* hard to get right.
    '''
    seen_d = set([ent._pk])
    to_delete = [ent]
    seen_s = set()
    to_save = []

    def _set_default(ent, attr, de=NULL):
        pk = ent._pk
        if pk in seen_d:
            # going to be deleted, don't need to modify
            return

        col = ent.__class__._columns[attr]
        de = de if de is not NULL else col._default
        if de in (None, NULL):
            setattr(ent, attr, None)
        elif callable(col._default):
            setattr(ent, attr, col._default())
        else:
            setattr(ent, attr, col._default)

        if pk not in seen_s:
            seen_s.add(pk)
            to_save.append(ent)

    for self in to_delete:
        for tbl, attr, action in MODELS_REFERENCED.get(self._namespace, ()):
            if action == 'no action':
                continue

            refs = MODELS[tbl].get_by(**{attr: self.id})
            if not refs:
                continue

            if action == 'restrict':
                # raise the exception here for a better traceback
                raise _restrict(self, attr, refs)
            elif action == 'set null':
                for ref in refs:
                    _set_default(ref, attr, None)
                continue
            elif action == 'set default':
                for ref in refs:
                    _set_default(ref, attr)
                continue

            # otherwise col._on_delete == 'cascade'
            for ent in (refs if isinstance(refs, list) else [refs]):
                if ent._pk not in seen_d:
                    seen_d.add(ent._pk)
                    to_delete.append(ent)

    # If we got here, then to_delete includes all items to delete. Let's delete
    # them!
    for self in to_delete:
        self.delete(skip_on_delete_i_really_mean_it=SKIP_ON_DELETE)
    for self in to_save:
        # Careful not to resurrect deleted entities
        if self._pk not in seen_d:
            self.save()

def _check_on_delete(on_delete, required, default):
    if on_delete is NO_ACTION_DEFAULT:
        return ColumnError("No on_delete action specified")
    elif on_delete not in ON_DELETE:
        return ColumnError("on_delete argument must be one of %r, you provided %r"%(
            list(ON_DELETE), on_delete))
    elif required:
        if on_delete == 'set null':
            return ColumnError("on_delete action is 'set null', but this column is required")
        elif on_delete == 'set default' and default in (NULL, None):
            return ColumnError("on_delete action is 'set default', but this required column has no default")

def _keygen_wrapper(keygen):
    @wraps(keygen)
    def _wrapper(attr, dct):
        return keygen(dct.get(attr))
    return _wrapper

_missing_keygen_warning = '''You have not specified a keygen for generating keys
to index your String() or Text() column. By default, rom has been using its
FULL_TEXT index on these columns in the past, but now requests/requires the
specification of a keygen function. Provide an explicit keygen argument of
%s or some other keygen that
matches the rom index API to remove this warning. This warning will become an
exception in rom >= 0.31.0.'''.replace('\n', ' ')%STRING_INDEX_KEYGENS_STR

class Column(object):
    '''
    Column objects handle data conversion to/from strings, store metadata
    about indices, etc. Note that these are "heavy" columns, in that whenever
    data is read/written, it must go through descriptor processing. This is
    primarily so that (for example) if you try to write a Decimal to a Float
    column, you get an error the moment you try to do it, not some time later
    when you try to save the object (though saving can still cause an error
    during the conversion process).

    Standard Arguments:

        * *required* - determines whether this column is required on
          creation
        * *default* - a default value (either a callable or a simple value)
          when this column is not provided
        * *unique* - can be enabled on string, unicode, and integer columns, and
          allows for required distinct column values (like an email address on
          a User model)
        * *index* - can be enabled on numeric, string, and unicode columns.
          Will create a ZSET-based numeric index for numeric columns and a
          "full word"-based search for string/unicode columns. If enabled
          for other (or custom) columns, remember to provide the
          ``keygen`` argument
        * *keygen* - pass a function that takes your column's value and
          returns the data that you want to index (see the keygen docs for
          what kinds of data to return)
        * *keygen2* - pass a function that takes your column name and the dict
          representing the current entity's complete data - can be used for
          creating multi-column indexes

    String/Text arguments:

        * *prefix* - can be enabled on any column that generates a list of
          strings as a result of the default or passed *keygen* function, and
          will allow the searching of prefix matches (autocomplete) over your
          data. See ``Query.startswith()`` for details.
        * *suffix* - can be enabled in the same contexts as *prefix* and
          enables suffix matching over your data. Any individual string in the
          returned data will be reversed (you need to make sure this makes
          conceptual sense with your data) before being stored or used. See
          ``Query.endswith()`` for details.

    .. warning:: Enabling prefix or suffix matching on a column only makes
       sense for columns defining a non-numeric *keygen* function.

    Notes:

        * If you have disabled Lua support, you can only have at most one
          unique column on each model
        * *Unique* and *index* are not mutually exclusive
        * The *keygen* argument determines how index values are generated
          from column data (with a reasonably sensible default for numeric
          columns, and 2 convenient options for string/text columns)
        * If you set *required* to True, then you must have the column set
          during object construction: ``MyModel(col=val)``
        * If *index* and *prefix*, or *index* and *suffix* are set, the same
          keygen will be used for both the regular *index* as well as the
          *prefix* and/or *suffix* searches
        * If *prefix* is set, you can perform pattern matches over your data.
          See documention on ``Query.like()`` for details.
        * Pattern matching over data is only guaranteed to be valid or correct
          for ANSI strings that do not include nulls, though we make an effort
          to support unicode strings and strings with embedded nulls
        * Prefix, suffix, and pattern matching are performed within a Lua
          script, so may have substantial execution time if there are a large
          number of matching prefix or suffix entries
        * Whenever possible, pattern matching will attempt to use any
          non-wildcard prefixes on the pattern to limit the items to be
          scanned. A pattern starting with ``?``, ``*``, ``+``, or ``!`` will
          not be able to use any prefix, so will scan the entire index for
          matches (aka: expensive)

    There are 3 types of string indexes that rom currently supports:

        * *SIMPLE*/*SIMPLE_CI* - sorting only with ``query.order_by('x')`` -
          https://josiahcarlson.github.io/rom/util.html#rom.util.SIMPLE
        * *IDENTITY*/*IDENTITY_CI* - equality only with ``query.filter(x=...)`` -
          https://josiahcarlson.github.io/rom/util.html#rom.util.IDENTITY
        * *FULL_TEXT* - bag of words inverted index for ``query.filter(x=...)`` -
          https://josiahcarlson.github.io/rom/util.html#rom.util.FULL_TEXT

    To each of those 3 index types, you can further add a prefix/suffix index,
    whose semantics are as follows:

        * *prefix* - ``query.startswith(column=pfix)`` and ``query.like(column='stuff?*')``
        * *suffix* - ``query.endswith(column=sfix)``
        * *SIMPLE*/*SIMPLE_CI*/*IDENTITY*/*IDENTITY_CI* - *prefix*/*suffix* the
          whole string case sensitive or insensitive
        * *FULL_TEXT* - *prefix*/*suffix* on individual words parsed out of the
          full text

    '''
    _allowed = ()

    __slots__ = '_required _default _init _unique _index _model _attr _keygen _prefix _suffix'.split()

    def __init__(self, required=False, default=NULL, unique=False, index=False, keygen=None, prefix=False, suffix=False, keygen2=None):
        self._required = required
        self._default = default
        self._unique = unique
        self._index = index
        self._prefix = prefix
        self._suffix = suffix
        self._init = False
        self._model = None
        self._attr = None
        self._keygen = None

        if (keygen or keygen2) and not (index or prefix or suffix):
            raise ColumnError("Explicit keygen provided, but no index type spcified (index, prefix, and suffix all False)")

        if not self._allowed and not hasattr(self, '_fmodel') and not hasattr(self, '_ftable') and not isinstance(self, IndexOnly):
            raise ColumnError("Missing valid class-level _allowed attribute on %r"%(type(self),))

        allowed = (self._allowed,) if isinstance(self._allowed, type) else self._allowed
        is_integer = all(issubclass(x, six.integer_types) for x in allowed)
        if unique:
            if not (is_string or is_integer):
                raise ColumnError("Unique columns can only be strings or integers")

        if keygen and keygen2:
            raise ColumnError("Can only specify one of 'keygen' and 'keygen2' arguments at a time, you provided both")

        if keygen2:
            # new-style keygen is ready :D
            self._keygen = keygen2
            return

        numeric = True
        if index and not isinstance(self, (ManyToOne, OneToOne)):
            if not is_numeric(allowed):
                numeric = False
                if issubclass(bool, allowed):
                    keygen = keygen or _boolean_keygen
                if not is_string(allowed) and not keygen:
                    raise ColumnError("Non-numeric/string indexed columns must provide keygen argument on creation")

        if (index or prefix or suffix) and is_string(allowed) and keygen is None:
            raise ColumnError("Indexed string column missing explicit keygen argument, try one of: %s"%STRING_INDEX_KEYGENS_STR)

        if index:
            keygen = keygen if keygen else (
                _numeric_keygen if numeric else _string_keygen)
        elif prefix or suffix:
            keygen = keygen if keygen else _string_keygen

        if keygen:
            # old-style keygen needs a wrapper
            self._keygen = _keygen_wrapper(keygen)

    def _from_redis(self, value):
        convert = self._allowed if callable(self._allowed) else self._allowed[0]
        return convert(value)

    def _to_redis(self, value):
        return repr(value)

    def _validate(self, value):
        if value is not None:
            if isinstance(value, self._allowed):
                return
        elif not self._required:
            return
        raise InvalidColumnValue("%s.%s has type %r but must be of type %r"%(
            self._model, self._attr, type(value), self._allowed))

    def _init_(self, obj, model, attr, value, loading):
        # You shouldn't be calling this directly, but this is what sets up all
        # of the necessary pieces when creating an entity from scratch, or
        # loading the entity from Redis
        self._model = model
        self._attr = attr

        if value is None:
            if self._default in (NULL, None):
                if self._required and not loading:
                    raise MissingColumn("%s.%s cannot be missing"%(self._model, self._attr))
            elif callable(self._default):
                value = self._default()
            else:
                value = self._default
        elif not isinstance(value, self._allowed) and self._allowed:
            try:
                value = self._from_redis(value)
            except (ValueError, TypeError) as e:
                raise InvalidColumnValue(*e.args)

        if not loading:
            self._validate(value)
        obj._data[attr] = value

    def __set__(self, obj, value):
        if not obj._init:
            self._init_(obj, *value)
            return
        try:
            if value is None:
                try:
                    return self.__delete__(obj)
                except AttributeError:
                    # We can safely suppress this, the column was already set
                    # to None or deleted
                    return
            if not isinstance(value, self._allowed) and self._allowed:
                value = self._from_redis(value)
        except (ValueError, TypeError):
            raise InvalidColumnValue("Cannot convert %r into type %s"%(value, self._allowed))
        self._validate(value)
        obj._data[self._attr] = value
        obj._modified = True
        session.add(obj)

    def __get__(self, obj, objtype):
        try:
            return obj._data[self._attr]
        except KeyError:
            AttributeError("%s.%s does not exist"%(self._model, self._attr))

    def __delete__(self, obj):
        if self._required:
            raise InvalidOperation("%s.%s cannot be null"%(self._model, self._attr))
        try:
            obj._data.pop(self._attr)
        except KeyError:
            raise AttributeError("%s.%s does not exist"%(self._model, self._attr))
        obj._modified = True
        session.add(obj)

class Integer(Column):
    '''
    Used for integer numeric columns.

    All standard arguments supported. See ``Column`` for details on supported
    arguments.

    Used via::

        class MyModel(Model):
            col = Integer()
    '''
    _allowed = six.integer_types
    def _to_redis(self, value):
        return str(value)

class Boolean(Column):
    '''
    Used for boolean columns.

    All standard arguments supported. See ``Column`` for details on supported
    arguments.

    All values passed in on creation are casted via bool(), with the exception
    of None (which behaves as though the value was missing), and any existing
    data in Redis is considered ``False`` if empty, and ``True`` otherwise.

    Used via::

        class MyModel(Model):
            col = Boolean()

    Queries via ``MyModel.get_by(...)`` and ``MyModel.query.filter(...)`` work
    as expected when passed ``True`` or ``False``.

    .. note:: these columns are not sortable by default.
    '''
    _allowed = bool
    def _to_redis(self, obj):
        return '1' if obj else ''
    def _from_redis(self, obj):
        return bool(obj)

class Float(Column):
    '''
    Numeric column that supports integers and floats (values are turned into
    floats on load from Redis).

    All standard arguments supported. See ``Column`` for details on supported
    arguments.

    Used via::

        class MyModel(Model):
            col = Float()
    '''
    _allowed = (float,) + six.integer_types

class Decimal(Column):
    '''
    A Decimal-only numeric column (converts ints/longs into Decimals
    automatically). Attempts to assign Python float will fail.

    All standard arguments supported. See ``Column`` for details on supported
    arguments.

    Used via::

        class MyModel(Model):
            col = Decimal()
    '''
    _allowed = _Decimal
    def _from_redis(self, value):
        return _Decimal(value)
    def _to_redis(self, value):
        return str(value)

class DateTime(Column):
    '''
    A datetime column.

    All standard arguments supported. See ``Column`` for details on supported
    arguments.

    Used via::

        class MyModel(Model):
            col = DateTime()

    .. note:: tzinfo objects are not stored
    '''
    _allowed = datetime
    def _from_redis(self, value):
        return ts2dt(float(value))
    def _to_redis(self, value):
        return repr(dt2ts(value))

class SaferDateTime(DateTime):
    '''
    A (safer) datetime column (see Issue #109 or the below note for more
    information)

    All standard arguments supported. See ``Column`` for details on supported
    arguments.

    Used via::

        class MyModel(Model):
            col = SaferDateTime()

    .. note:: tzinfo objects are not stored
    .. note:: what makes this "safer" than other datetime objects is that there
      are exactly two types of values that can be set here: a ``datetime``
      object, or a ``str`` that can represent a float, which is the number of
      seconds since the unix epoch.
    '''

    def _from_redis(self, value):
        if isinstance(value, datetime):
            return value
        elif isinstance(value, str):
            try:
                return ts2dt(float(value))
            except ValueError:
                raise InvalidColumnValue("Cannot convert %r into type %s"%(value, self._allowed))
        raise InvalidColumnValue("Cannot convert %r into type %s"%(value, self._allowed))

    def __set__(self, obj, value):
        if obj._init and not isinstance(value, self._allowed):
            raise InvalidColumnValue("Value %r is not the right type: %s"%(value, self._allowed))
        elif (not obj._init) and obj._new and not isinstance(value[2], self._allowed):
            raise InvalidColumnValue("Value %r is not the right type: %s"%(value, self._allowed))
        return super(SaferDateTime, self).__set__(obj, value)

class Date(Column):
    '''
    A date column.

    All standard arguments supported. See ``Column`` for details on supported
    arguments.

    Used via::

        class MyModel(Model):
            col = Date()
    '''
    _allowed = date
    def _from_redis(self, value):
        return ts2dt(float(value)).date()
    def _to_redis(self, value):
        return repr(dt2ts(value))

class Time(Column):
    '''
    A time column. Timezones are ignored.

    All standard arguments supported. See ``Column`` for details on supported
    arguments.

    Used via::

        class MyModel(Model):
            col = Time()

    .. note:: tzinfo objects are not stored
    '''
    _allowed = dtime
    def _from_redis(self, value):
        return ts2t(float(value))
    def _to_redis(self, value):
        return repr(t2ts(value))

class String(Column):
    '''
    A plain string column (str in 2.x, bytes in 3.x). Trying to save unicode
    strings will probably result in an error, if not corrupted data.

    All standard arguments and String/Text arguments supported. See ``Column``
    for details on supported arguments.

    This column can be indexed in one of five ways - a sorted index on a 7
    byte prefix of the value (``keygen=rom.SIMPLE``), a sorted index on a
    lowercased 7 byte prefix of the value (``keygen=rom.SIMPLE_CI``),
    a case-insensitive full-text index (``keygen=rom.FULL_TEXT``),
    a case-sensitive identity index (``keygen=rom.IDENTITY``), and a
    case-insensitive identity index (``keygen=rom.IDENTITY_CI``).

    Used via::

        class MyModel(Model):
            col = String()
    '''
    _allowed = str if six.PY2 else bytes
    def _to_redis(self, value):
        return value.decode('latin-1')

    def _from_redis(self, value):
        return value

    def _init_(self, obj, model, attr, value, loading):
        if value != None:
            if not isinstance(value, self._allowed):
                value = value.encode('latin-1')
            elif loading:
                value = value.decode('utf-8').encode('latin-1')
        return Column._init_(self, obj, model, attr, value, loading)

class Text(Column):
    '''
    A unicode string column. Behavior is more or less identical to the String
    column type, except that unicode is supported (unicode in 2.x, str in 3.x).
    UTF-8 is used by default as the encoding to bytes on the wire, which *will*
    affect ``rom.SIMPLE`` and ``rom.SIMPLE_CI`` indexes.

    All standard arguments supported. See ``Column`` for details on supported
    arguments.

    This column can be indexed in one of five ways - a sorted index on a 7
    byte prefix of the value (``keygen=rom.SIMPLE``), a sorted index on a
    lowercased 7 byte prefix of the value (``keygen=rom.SIMPLE_CI``),
    a case-insensitive full-text index (``keygen=rom.FULL_TEXT``),
    a case-sensitive identity index (``keygen=rom.IDENTITY``), and a
    case-insensitive identity index (``keygen=rom.IDENTITY_CI``).

    For the 7 byte prefix/suffixes on indexes using the ``rom.SIMPLE`` and
    ``rom.SIMPLE_CI`` keygen, because we use UTF-8 to encode text, a single
    character can turn into 1-3 bytes, so may not be useful in practice.

    Used via::

        class MyModel(Model):
            col = Text()
    '''
    _allowed = six.text_type
    def _to_redis(self, value):
        return value.encode('utf-8') if six.PY2 else value
    def _from_redis(self, value):
        if isinstance(value, six.binary_type):
            return value.decode('utf-8')
        return value

class Json(Column):
    '''
    Allows for more complicated nested structures as attributes.

    All standard arguments supported. The ``keygen`` argument must be provided
    if ``index`` is ``True``.

    Used via::

        class MyModel(Model):
            col = Json()
    '''
    _allowed = (dict, list, tuple)
    def _to_redis(self, value):
        return json.dumps(value, sort_keys=True)
    def _from_redis(self, value):
        if isinstance(value, self._allowed):
            return value
        return json.loads(value)

class PrimaryKey(Column):
    '''
    This is a primary key column, used when you want the primary key to be
    named something other than 'id'. If you omit a PrimaryKey column on your
    Model classes, one will be automatically created for you.

    Only the ``index`` argument will be used. You may want to enable indexing
    on this column if you want to be able to perform queries and sort the
    results by primary key.

    Used via::

        class MyModel(Model):
            id = PrimaryKey()
    '''
    _allowed = six.integer_types

    def __init__(self, index=False):
        Column.__init__(self, required=False, default=None, unique=False, index=index)

    def _init_(self, obj, model, attr, value, loading):
        self._model = model
        self._attr = attr
        if value is None:
            if loading:
                raise InvalidColumnValue("Cannot set none primary key on object loading")
            value = _connect(obj).incr('%s:%s:'%(model, attr))
            obj._modified = True
        else:
            value = int(value)
        obj._data[attr] = value
        if obj._init:
            session.add(obj)

    def __set__(self, obj, value):
        if not obj._init:
            self._init_(obj, *value)
            return
        raise InvalidOperation("Cannot update primary key value")

class IndexOnly(Column):
    '''
    This column doesn't actually store data, except in indexes for other
    columns. Say, for example, you have an email text field that you want to
    be unique, look-up by identity, but also be parsed out for a sort of
    address-book index. Normally that would suck. But not with ``IndexOnly()``
    columns! Data set on this attribute will raise an exception, and will be
    silently ignored (and potentially deleted) if already stored in Redis.

    Used via::

        import re
        def split_email(val):
            if val:
                return set(filter(None, re.split('[^\w]', val.lower())))

        class MyModel(Model):
            email = Text(unique=True)
            elookup = IndexOnly('email', keygen=split_email)

        MyModel(email='user@host.com').save()
        MyModel.get_by(elookup='user')
        MyModel.query.filter(elookup=['user', 'host']).all()

    .. note:: I've been using a variation of this internally for some of my
        own work, and I thought I'd release it as a convenience column.

    '''

    def __init__(self, column=None, keygen=None, prefix=False, suffix=False, keygen2=None):
        '''
        Two ways to use:

            * With ``column`` AND ``keygen`` - automatically index an external column
            a second way using standard keygens
            * With ``keygen2`` - generate an arbitrary index on arbitrary model data

        Standard Arguments:

            * *column* - column with the data you want to index
            * *keygen* - passed through, see ``Column`` docs for more information
            * *prefix* - passed through, see ``Column`` docs for more information
            * *suffix* - passed through, see ``Column`` docs for more information
            * *keygen2* - passed through, see ``Column`` docs for more information

        .. note:: `unique` is not available or passed through, as it relies on
            data in the column (IndexOnly columns only store data in index(es)).
        '''
        if column and keygen:
            keygen2 = lambda _,dct: keygen(dct.get(column))
        elif keygen2:
            pass
        else:
            raise ValueError("Need both of `column` and `keygen` or just `keygen2` argument")
        Column.__init__(self, index=True, keygen2=keygen2, prefix=prefix, suffix=suffix)

    def _validate(self, value):
        if value is not None:
            raise InvalidColumnValue("%s.%s has type %r but column cannot hold data"%(
                self._model, self._attr, type(value)))

    def _from_redis(self, value):
        return

    def _to_redis(self, value):
        return

twrap = (lambda x: (x,))

@twrap
class DoesntMatterInThisContext:
    pass

class ManyToOne(Column):
    '''
    This ManyToOne column allows for one model to reference another model.
    While a ManyToOne column does not require a reverse OneToMany column
    (which will return a list of models that reference it via a ManyToOne), it
    is generally seen as being useful to have both sides of the relationship
    defined.

    Aside from the name of the other model, only the ``required`` and
    ``default`` arguments are accepted.

    Four arguments are supported:

        * *ftable* - the name of the other model (required argument)
        * *on_delete* - how to handle foreign key references on delete,
            supported options include: 'no action', 'restrict', 'cascade'
            'set default', and 'set null' (required argument)
        * *required* - determines whether this column is required on
          creation
        * *default* - a default value (either a callable or a simple value)
          when this column is not provided

    Used via::

        class MyModel(Model):
            col = ManyToOne('OtherModelName')

    .. note:: All ``ManyToOne`` columns are indexed numerically, which means
      that you can find entities referencing specific id ranges or even sort by
      referenced ids.

    '''
    __slots__ = Column.__slots__ + ['_ftable', '_on_delete']
    _allowed = DoesntMatterInThisContext
    def __init__(self, ftable, on_delete=NO_ACTION_DEFAULT, required=False, default=NULL):
        exc = _check_on_delete(on_delete, required, default)
        if exc:
            raise exc

        self._ftable = ftable
        self._on_delete = on_delete
        Column.__init__(self, required, default, index=True, keygen=_many_to_one_keygen)

    def _from_redis(self, value):
        try:
            model = MODELS[self._ftable]
        except KeyError:
            raise ORMError("Missing foreign table %r referenced by %s.%s"%(self._ftable, self._model, self._attr))
        if isinstance(value, model):
            return value
        return model.get(value)

    def _validate(self, value):
        try:
            model = MODELS[self._ftable]
        except KeyError:
            raise ORMError("Missing foreign table %r referenced by %s.%s"%(self._ftable, self._model, self._attr))
        if not self._required and value is None:
            return
        if not isinstance(value, model):
            raise InvalidColumnValue("%s.%s has type %r but must be of type %r"%(
                self._model, self._attr, type(value), model))

    def _to_redis(self, value):
        if not value:
            return None
        if isinstance(value, six.integer_types):
            return str(value)
        if value._new:
            # should spew a warning here
            value.save()
        v = str(getattr(value, value._pkey))
        return v

class OneToOne(ManyToOne):
    '''
    This OneToOne column allows for one model to reference another model.
    A OneToOne column does not require a reverse OneToOne column, and provides
    ``on_delete`` behavior.

    Five arguments are supported:

        * *ftable* - the name of the other model (required argument)
        * *on_delete* - how to handle foreign key references on delete,
            supported options include: 'no action', 'restrict', 'cascade'
            'set default', and 'set null' (required argument)
        * *required* - determines whether this column is required on
          creation
        * *default* - a default value (either a callable or a simple value)
          when this column is not provided
        * *unique* - whether or not the referenced entity must be a unique
            reference

    Used via::

        class MyModel(Model):
            col = OneToOne('OtherModelName', 'no action')

    .. note:: All ``OneToOne`` columns are indexed numerically, which means
      that you can find entities referencing specific id ranges or even sort by
      referenced ids.

    '''
    __slots__ = Column.__slots__ + ['_ftable', '_on_delete']
    _allowed = DoesntMatterInThisContext
    def __init__(self, ftable, on_delete=NO_ACTION_DEFAULT, required=False, default=NULL, unique=False):
        exc = _check_on_delete(on_delete, required, default)
        if exc:
            raise exc

        self._on_delete = on_delete
        self._ftable = ftable
        Column.__init__(self, required, default, unique, index=True, keygen=_many_to_one_keygen)

class ForeignModel(Column):
    '''
    This column allows for ``rom`` models to reference an instance of another
    model from an unrelated ORM or otherwise.

    .. note:: In order for this mechanism to work, the foreign model *must*
      have an ``id`` attribute or property represents its primary key, and
      *must* have a classmethod or staticmethod named ``get()`` that returns
      the proper database entity.

    Used via::

        class MyModel(Model):
            col = ForeignModel(DjangoModel)

        dm = DjangoModel(col1='foo')
        django.db.transaction.commit()

        x = MyModel(col=dm)
        x.save()
    '''
    __slots__ = Column.__slots__ + ['_fmodel']
    _allowed = DoesntMatterInThisContext
    def __init__(self, fmodel, required=False, default=NULL):
        self._fmodel = fmodel
        Column.__init__(self, required, default, index=True, keygen=_many_to_one_keygen)

    def _from_redis(self, value):
        if isinstance(value, self._fmodel):
            return value
        if isinstance(value, six.string_types_ex) and value.isdigit():
            value = int(value, 10)
        return self._fmodel.get(value)

    def _validate(self, value):
        if not self._required and value is None:
            return
        if not isinstance(value, self._fmodel):
            raise InvalidColumnValue("%s.%s has type %r but must be of type %r"%(
                self._model, self._attr, type(value), self._fmodel))

    def _to_redis(self, value):
        if not value:
            return None
        if isinstance(value, six.integer_types):
            return str(value)
        return str(value.id)

class OneToMany(Column):
    '''
    OneToMany columns do not actually store any information. They rely on a
    properly defined reverse ManyToOne column on the referenced model in order
    to be able to fetch a list of referring entities.

    Two arguments are supported:

        * *ftable* - the name of the other model
        * *column* - the attribute on the other model with the reference to
            this column, required if the foreign model has multiple columns
            referencing this model with OneToOne or ManyToOne columns

    Used via::

        class MyModel(Model):
            col = OneToMany('OtherModelName')
            ocol = OneToMany('ModelName')
    '''
    __slots__ = '_model _attr _ftable _required _unique _index _prefix _suffix _keygen _column'.split()
    _allowed = DoesntMatterInThisContext
    def __init__(self, ftable, column=None):
        if column in ON_DELETE or column is NO_ACTION_DEFAULT:
            raise ColumnError("OneToMany lost its on_delete argument - pass it to the ManyToOne instead")
        self._ftable = ftable
        self._required = self._unique = self._index = self._prefix = self._suffix = False
        self._model = self._attr = self._keygen = None
        self._column = column

    def _to_redis(self, value):
        return ''

    def __set__(self, obj, value):
        if not obj._init:
            self._model, self._attr = value[:2]
            try:
                MODELS[self._ftable]
            except KeyError:
                raise ORMError("Missing foreign table %r referenced by %s.%s"%(self._ftable, self._model, self._attr))
            return
        raise InvalidOperation("Cannot assign to OneToMany relationships")

    def __get__(self, obj, objtype):
        try:
            model = MODELS[self._ftable]
        except KeyError:
            raise ORMError("Missing foreign table %r referenced by %s.%s"%(self._ftable, self._model, self._attr))

        if self._column:
            return model.get_by(**{self._column: getattr(obj, obj._pkey)})

        for attr, col in model._columns.items():
            if isinstance(col, (ManyToOne, OneToOne)) and col._ftable == self._model:
                return model.get_by(**{attr: getattr(obj, obj._pkey)})

        raise ORMError("Reverse ManyToOne or OneToOne relationship not found for %s.%s -> %s"%(self._model, self._attr, self._ftable))

    def __delete__(self, obj):
        raise InvalidOperation("Cannot delete OneToMany relationships")

COLUMN_TYPES = [v for v in globals().values() if isinstance(v, type) and issubclass(v, Column)]
__all__ = [v.__name__ for v in COLUMN_TYPES] + 'MODELS MODELS_REFERENCED ON_DELETE'.split()
__all__.sort()


'''
Rom - the Redis object mapper for Python

Copyright 2013-2016 Josiah Carlson

Released under the LGPL license version 2.1 and version 3 (you can choose
which you'd like to be bound under).
'''

from collections import namedtuple
import json
import re
import uuid

import six

from .exceptions import QueryError
from .util import _prefix_score, _script_load, _to_score

_skip = None
_skip = set(globals()) - set(['__doc__'])

Prefix = namedtuple('Prefix', 'attr prefix')
Suffix = namedtuple('Suffix', 'attr suffix')
Pattern = namedtuple('Pattern', 'attr pattern')
Geofilter = namedtuple('Geo', 'name lon lat radius measure count')

GeoIndex = namedtuple('GeoIndex', 'name callback')

SPECIAL = re.compile('([-().%[^$])')
def _pattern_to_lua_pattern(pat):
    if isinstance(pat, six.string_types) and not isinstance(pat, str):
        # XXX: Decode only py2k unicode. Why can't we run the unicode
        # pattern through the re? -JM
        pat = pat.encode('utf-8')
    # We use '-' instead of '*' to get the shortest matches possible, which is
    # usually the desired case for pattern matching.
    return SPECIAL.sub('%\1', pat) \
        .replace('?', '.?') \
        .replace('*', '.-') \
        .replace('+', '.+') \
        .replace('!', '.') \
        .replace('\0', '%z') + '.*%z%d+'

def _find_prefix(pat):
    pat = SPECIAL.sub('%\1', pat)
    if isinstance(pat, six.string_types) and not isinstance(pat, str):
        # XXX: Decode only py2k unicode. Why can't we run the unicode
        # pattern through the re? -JM
        pat = pat.encode('utf-8')
    x = []
    for i in pat:
        if i in '?*+!':
            break
        x.append(i)
    return ''.join(x[:7])

MAX_PREFIX_SCORE = _prefix_score(7*'\xff', True)
def _start_end(prefix):
    return _prefix_score(prefix), (_prefix_score(prefix, True) if prefix else MAX_PREFIX_SCORE)

if six.PY3:
    def _ts(v):
        if not isinstance(v, str):
            return v.decode('utf-8')
        return v
else:
    def _ts(v):
        if not isinstance(v, unicode):
            return v.decode('utf-8')
        return v

class GeneralIndex(object):
    '''
    This class implements general indexing and search for the ``rom`` package.

    .. warning:: You probably don't want to be calling this directly. Instead,
      you should rely on the ``Query`` object returned from ``Model.query`` to
      handle all of your query pre-processing.

    Generally speaking, numeric indices use ZSETs, and text indices use SETs
    built using an 'inverted index'.

    Say that we have words ``hello world`` in a column ``c`` on a model with
    primary key ``MyModel:1``. The member ``1`` will be added to SETs with
    keys::

        MyModel:c:hello
        MyModel:c:world

    Text searching performs a sequence of intersections of SETs for the words
    to be searched for.

    Numeric range searching performs a sequence of intersections of ZSETs,
    removing items outside the requested range after each intersection.

    Searches will pre-sort intersections from smallest to largest SET/ZSET
    prior to performing the search to improve performance.

    Prefix, suffix, and pattern matching change this operation. Given a key
    generated of ``hello`` on a column ``c`` on a model with primary key
    ``MyModel:1``, the member ``hello\\01`` with score 0 will be added to a
    ZSET with the key name ``MyModel:c:pre`` for the prefix/pattern index.
    On a suffix index, the member ``olleh\\01`` with score 0 will be added to
    a ZSET with the key name ``MyModel:c:suf``.

    Prefix and suffix matches are excuted in Lua with a variant of the
    autocomplete method described in Redis in Action. These methods ensure a
    runtime proportional to the number of matched entries.

    Pattern matching also uses a Lua script to scan over data in the prefix
    index, exploiting prefixes in patterns if they exist.

    '''
    def __init__(self, namespace):
        self.namespace = namespace

    def _prepare(self, conn, filters):
        temp_id = "%s:%s"%(self.namespace, uuid.uuid4())
        pipe = conn.pipeline(True)
        sfilters = filters
        sizes = [(None, 0)]
        if filters:
            # reorder filters based on the size of the underlying set/zset
            for fltr in filters:
                if isinstance(fltr, six.string_types):
                    estimate_work_lua(pipe, '%s:%s:idx'%(self.namespace, fltr), None)
                elif isinstance(fltr, Prefix):
                    estimate_work_lua(pipe, '%s:%s:pre'%(self.namespace, fltr.attr), fltr.prefix)
                elif isinstance(fltr, Suffix):
                    estimate_work_lua(pipe, '%s:%s:suf'%(self.namespace, fltr.attr), fltr.suffix)
                elif isinstance(fltr, Pattern):
                    estimate_work_lua(pipe, '%s:%s:pre'%(self.namespace, fltr.attr), _find_prefix(fltr.pattern))
                elif isinstance(fltr, list):
                    estimate_work_lua(pipe, '%s:%s:idx'%(self.namespace, _ts(fltr[0])), None)
                elif isinstance(fltr, Geofilter):
                    estimate_work_lua(pipe, '%s:%s:geo'%(self.namespace, fltr.name), fltr.count)
                elif isinstance(fltr, tuple):
                    estimate_work_lua(pipe, '%s:%s:idx'%(self.namespace, fltr[0]), fltr[1:3])
                else:
                    raise QueryError("Don't know how to handle a filter of: %r"%(fltr,))
            sizes = list(enumerate(pipe.execute()))
            sizes.sort(key=lambda x:abs(x[1]))
            sfilters = [filters[x[0]] for x in sizes]

        # the first "intersection" is actually a union to get us started, unless
        # we can explicitly create a sub-range in Lua for a fast start to
        # intersection
        intersect = pipe.zunionstore
        first = True
        for ii, fltr in enumerate(sfilters):
            if isinstance(fltr, list):
                # or string string/tag search
                if len(fltr) == 1:
                    # only 1? Use the simple version.
                    fltr = _ts(fltr[0])
                elif not fltr:
                    continue
                else:
                    temp_id2 = str(uuid.uuid4())
                    pipe.zunionstore(temp_id2, dict(
                        ('%s:%s:idx'%(self.namespace, fi), 0) for fi in fltr))
                    intersect(temp_id, {temp_id: 0, temp_id2: 0})
                    pipe.delete(temp_id2)
            if isinstance(fltr, six.string_types):
                # simple string/tag search
                intersect(temp_id, {temp_id:0, '%s:%s:idx'%(self.namespace, fltr):0})
            elif isinstance(fltr, Prefix):
                redis_prefix_lua(pipe, temp_id, '%s:%s:pre'%(self.namespace, fltr.attr), fltr.prefix, first)
            elif isinstance(fltr, Suffix):
                redis_prefix_lua(pipe, temp_id, '%s:%s:suf'%(self.namespace, fltr.attr), fltr.suffix, first)
            elif isinstance(fltr, Pattern):
                redis_prefix_lua(pipe, temp_id,
                    '%s:%s:pre'%(self.namespace, fltr.attr),
                    _find_prefix(fltr.pattern),
                    first, '^' + _pattern_to_lua_pattern(fltr.pattern),
                )
            elif isinstance(fltr, Geofilter):
                # Prep the georadius command
                args = [
                    'georadius', '%s:%s:geo'%(self.namespace, fltr.name),
                    repr(fltr.lon), repr(fltr.lat), fltr.radius, fltr.measure
                ]
                if fltr.count and fltr.count >= 0:
                    args.append('COUNT')
                    args.append(fltr.count)
                args.append('STOREDIST')
                first = intersect == pipe.zunionstore
                args.append(temp_id if first else str(uuid.uuid4()))

                pipe.pipeline_execute_command(*args)
                if not first:
                    intersect(temp_id, {temp_id: 0, args[-1]: 1})
                    pipe.delete(args[-1])

            elif isinstance(fltr, tuple):
                # zset range search
                if len(fltr) != 3:
                    raise QueryError("Cannot filter range of data without 2 endpoints (%s given)"%(len(fltr)-1,))
                fltr, mi, ma = fltr
                if not ii and sizes[0][1] < 0:
                    # We've got a special case where we want to explicitly extract
                    # a subrange instead of starting from a larger index, because
                    # it turns out that this is going to be faster :P
                    lua_subrange(pipe, [temp_id, '%s:%s:idx'%(self.namespace, fltr)],
                        ['-inf' if mi is None else _to_score(mi), 'inf' if ma is None else _to_score(ma)]
                    )

                else:
                    intersect(temp_id, {temp_id:0, '%s:%s:idx'%(self.namespace, fltr):1})
                    if mi is not None:
                        pipe.zremrangebyscore(temp_id, '-inf', _to_score(mi, True))
                    if ma is not None:
                        pipe.zremrangebyscore(temp_id, _to_score(ma, True), 'inf')
            first = False
            intersect = pipe.zinterstore
        return pipe, intersect, temp_id

    def search(self, conn, filters, order_by, offset=None, count=None, timeout=None):
        '''
        Search for model ids that match the provided filters.

        Arguments:

            * *filters* - A list of filters that apply to the search of one of
              the following two forms:

                1. ``'column:string'`` - a plain string will match a word in a
                   text search on the column

                .. note:: Read the documentation about the ``Query`` object
                  for what is actually passed during text search

                2. ``('column', min, max)`` - a numeric column range search,
                   between min and max (inclusive by default)

                .. note:: Read the documentation about the ``Query`` object
                  for information about open-ended ranges

                3. ``['column:string1', 'column:string2']`` - will match any
                   of the provided words in a text search on the column

                4. ``Prefix('column', 'prefix')`` - will match prefixes of
                   words in a text search on the column

                5. ``Suffix('column', 'suffix')`` - will match suffixes of
                   words in a text search on the column

                6. ``Pattern('column', 'pattern')`` - will match patterns over
                   words in a text search on the column

            * *order_by* - A string that names the numeric column by which to
              sort the results by. Prefixing with '-' will return results in
              descending order

            .. note:: While you can technically pass a non-numeric index as an
              *order_by* clause, the results will basically be to order the
              results by string comparison of the ids (10 will come before 2).

            .. note:: If you omit the ``order_by`` argument, results will be
              ordered by the last filter. If the last filter was a text
              filter, see the previous note. If the last filter was numeric,
              then results will be ordered by that result.

            * *offset* - A numeric starting offset for results
            * *count* - The maximum number of results to return from the query
        '''
        # prepare the filters
        pipe, intersect, temp_id = self._prepare(conn, filters)

        # handle ordering
        if order_by:
            reverse = order_by and order_by.startswith('-')
            order_clause = '%s:%s:idx'%(self.namespace, order_by.lstrip('-'))
            intersect(temp_id, {temp_id:0, order_clause: -1 if reverse else 1})

        # handle returning the temporary result key
        if timeout is not None:
            pipe.expire(temp_id, timeout)
            pipe.execute()
            return temp_id

        offset = offset if offset is not None else 0
        end = (offset + count - 1) if count and count > 0 else -1
        pipe.zrange(temp_id, offset, end)
        pipe.delete(temp_id)
        return pipe.execute()[-2]

    def count(self, conn, filters):
        '''
        Returns the count of the items that match the provided filters.

        For the meaning of what the ``filters`` argument means, see the
        ``.search()`` method docs.
        '''
        pipe, intersect, temp_id = self._prepare(conn, filters)
        pipe.zcard(temp_id)
        pipe.delete(temp_id)
        return pipe.execute()[-2]

_redis_prefix_lua = _script_load('''
-- first unpack most of our passed variables
local dest = KEYS[1]
local tkey = KEYS[2]
local idx = KEYS[3]

local start_score = ARGV[1]
local end_score = ARGV[2]
local prefix = ARGV[3]
local psize = #prefix
local is_first = tonumber(ARGV[5])

-- find the start offset of our matching
local start_index = 0
if psize > 0 then
    -- All entries end with a null and the id, so we can go before all of them
    -- by just adding a null.
    local pfix = prefix .. '\\0'
    redis.call('ZADD', idx, start_score, pfix)
    start_index = tonumber(redis.call('ZRANK', idx, pfix))
    redis.call('ZREM', idx, pfix)
else
    local start_member = redis.call('ZRANGEBYSCORE', idx, start_score, 'inf', 'limit', 0, 1)
    if #start_member == 1 then
        start_index = tonumber(redis.call('ZRANK', idx, start_member[1]))
    end
end

-- Find the end offset of our matching. We don't bother with the prefix-based
-- ZADD/ZREM pair here because we do an endpoint check every 100 items or so,
--
local end_member = redis.call('ZREVRANGEBYSCORE', idx, '('..end_score, '-inf', 'limit', 0, 1)
local end_index = 0
if #end_member == 1 then
    end_index = tonumber(redis.call('ZRANK', idx, end_member[1]))
end

-- use functions to check instead of embedding an if inside the core loop
local check_match
if tonumber(ARGV[4]) > 0 then
    check_match = function(v, pattern) return string.match(v, pattern) end
else
    check_match = function(v, prefix) return string.sub(v, 1, psize) == prefix end
end

local matched = 0
local found_match = function(v)
    local endv = #v
    while string.sub(v, endv, endv) ~= '\\0' do
        endv = endv - 1
    end
    return redis.call('ZADD', tkey, 0, string.sub(v, endv+1, #v))
end

-- core matching loop
local has_prefix = psize > 0 and tonumber(ARGV[4]) == 0
for i=start_index,end_index,100 do
    local data = redis.call('ZRANGE', idx, i, i+99)
    local last
    for j, v in ipairs(data) do
        if check_match(v, prefix) then
            matched = matched + tonumber(found_match(v))
        end
        last = v
    end
    -- bail early if we've passed all of the shared prefixes
    if not last then
    elseif has_prefix and string.sub(last, 1, psize) > prefix then
        break
    end
end

if is_first > 0 then
    if matched > 0 then
        redis.call('RENAME', tkey, dest)
    end
else
    matched = redis.call('ZINTERSTORE', dest, 2, tkey, dest, 'WEIGHTS', 1, 0)
    redis.call('DEL', tkey)
end

return matched
''')

def redis_prefix_lua(conn, dest, index, prefix, is_first, pattern=None):
    '''
    Performs the actual prefix, suffix, and pattern match operations.
    '''
    tkey = '%s:%s'%(index.partition(':')[0], uuid.uuid4())
    start, end = _start_end(prefix)
    return _redis_prefix_lua(conn,
        [dest, tkey, index],
        [start, end, pattern or prefix, int(pattern is not None), int(bool(is_first))]
    )

lua_subrange = _script_load('''
-- KEYS - {dest_key, source_key}
-- ARGV - {start_value, end_value}

local idx = KEYS[2]

local start_member = redis.call('ZRANGEBYSCORE', idx, ARGV[1], 'inf', 'limit', 0, 1)
local start_index = 0
if #start_member == 1 then
    start_index = tonumber(redis.call('ZRANK', idx, start_member[1]))
end

local end_member = redis.call('ZREVRANGEBYSCORE', idx, ARGV[2], '-inf', 'limit', 0, 1)
local end_index
if #end_member == 1 then
    end_index = tonumber(redis.call('ZRANK', idx, end_member[1]))
else
    end_index = tonumber(redis.call('ZCARD', idx))
end

for i=start_index, end_index, 100 do
    local members = redis.call('ZRANGE', idx, i, math.min(i+99, end_index), 'withscores')
    for j=1, #members, 2 do
        members[j], members[j+1] = members[j+1], members[j]
    end
    redis.call('ZADD', KEYS[1], unpack(members))
end
return math.max(0, end_index - start_index + 1)
''')

_estimate_work_lua = _script_load('''
-- These indexes will be on numbers, strings, and/or prefixes. We'll check set
-- and sorted set cardinality, as well as range size.
local idx = KEYS[1]

-- redis.call('type') returns a table {"ok":<type>} ... looks like a bug, so use
-- redis.pcall() instead.
local typ = redis.pcall('TYPE', idx).ok
if typ == 'set' then
    return tonumber(redis.call('scard', idx))
elseif typ == 'zset' then
    local size = tonumber(redis.call('zcard', idx))

    if string.sub(idx, -4) == ':geo' then
        if #ARGV > 0 then
            return math.min(size, tonumber(ARGV[1]))
        end
        return size

    elseif #ARGV == 2 then
        local start_member = redis.call('ZRANGEBYSCORE', idx, ARGV[1], 'inf', 'limit', 0, 1)
        local start_index = 0
        if #start_member == 1 then
            start_index = tonumber(redis.call('ZRANK', idx, start_member[1]))
        end

        local end_member = redis.call('ZREVRANGEBYSCORE', idx, ARGV[2], '-inf', 'limit', 0, 1)
        local end_index = -1
        if #end_member == 1 then
            end_index = tonumber(redis.call('ZRANK', idx, end_member[1]))
        end

        local range_size = math.max(0, end_index - start_index + 1)
        -- take into consideration deletions from the full copied zset
        if string.sub(idx, -4) == ':idx' then
            size = size + size - range_size
        end

        -- pulling ranges isn't free, call it 2x as expensive per operation as
        -- the union/delete
        range_size = range_size * 2

        -- If you do the (simple) algebra, this will pick the range variant if
        -- range <= 2/3 * size. We keep the long form here to explain the *why*,
        -- even if a couple multiplies and a compare might be faster.

        if range_size < size then
            return -range_size
        end
    end
    return size
end
return 0
''')

def estimate_work_lua(conn, index, prefix):
    '''
    Estimates the total work necessary to calculate the prefix match over the
    given index with the provided prefix.
    '''
    if index.endswith(':idx'):
        args = [] if not prefix else list(prefix)
        if args:
            args[0] = '-inf' if args[0] is None else repr(float(args[0]))
            args[1] = 'inf' if args[1] is None else repr(float(args[1]))
        return _estimate_work_lua(conn, [index], args, force_eval=True)
    elif index.endswith(':geo'):
        return _estimate_work_lua(conn, [index], filter(None, [prefix]), force_eval=True)

    start, end = _start_end(prefix)
    return _estimate_work_lua(conn, [index], [start, '(' + end], force_eval=True)

__all__ = [k for k, v in globals().items() if getattr(v, '__doc__', None) and k not in _skip]
__all__.sort()


'''
Rom - the Redis object mapper for Python

Copyright 2013-2016 Josiah Carlson

Released under the LGPL license version 2.1 and version 3 (you can choose
which you'd like to be bound under).

Changing connection settings
============================

There are four ways to change the way that ``rom`` connects to Redis.

1. Set the global default connection settings by calling
   ``rom.util.set_connection_settings()`` with the same arguments you would
   pass to the redis.Redis() constructor::

    import rom.util

    rom.util.set_connection_settings(host='myhost', db=7)

2. Give each model its own Redis connection on creation, called _conn, which
   will be used whenever any Redis-related calls are made on instances of that
   model::

    import rom

    class MyModel(rom.Model):
        _conn = redis.Redis(host='myhost', db=7)

        attribute = rom.String()
        other_attribute = rom.String()

3. Replace the ``CONNECTION`` object in ``rom.util``::

    import rom.util

    rom.util.CONNECTION = redis.Redis(host='myhost', db=7)

4. Monkey-patch ``rom.util.get_connection`` with a function that takes no
   arguments and returns a Redis connection::

    import rom.util

    def my_connection():
        # Note that this function doesn't use connection pooling,
        # you would want to instantiate and cache a real connection
        # as necessary.
        return redis.Redis(host='myhost', db=7)

    rom.util.get_connection = my_connection

Using a non-caching session object
==================================

If you would like to stop ``rom`` from caching your data for later
``session.commit()`` or faster data fetching (and there are several reasons for
doing this), ``rom`` offers two methods to enable or disable caching on either a
global or per-thread basis.

1. To set the global default behavior as not caching anything, you can::

    import rom.util
    rom.util.use_null_session()

   From the point that ``rom.util.use_null_session()`` is called, no additional
   caching will be performed. You must explicitly ``.save()`` any newly created
   entities, and ``session.commit()`` will only save those objects that had been
   cached prior to the ``rom.util.use_null_session()`` call.

   You can switch back to the standard ``rom`` behavior by calling
   ``rom.util.use_rom_session()``.

2. To override behavior on a per-thread basis, you can set the attribute
   ``null_session`` on the ``session`` object (which is available as
   ``rom.session``, ``rom.columns.session``, or ``rom.util.session``), which
   will set the thread's behavior to be uncached (``session.null_session =
   True``), cached (``session.null_session = False``), or the global default
   (``del session.null_session``).

Using a geo index
=================

Your model must have an attribute called ``geo_index``, and that attribute must
be a list of ``GeoIndex()`` objects. Names for ``GeoIndex`` items must be unique
per model.::

    import rom

    def cb(data):
        # ``data`` can act as an object or dictionary. This function could have
        # actually just read::
        #     return data
        # ... but we will be explicit in what we return... yes this works!
        return {'lon': data['lon'], 'lat': data.lat}

    class PointOfInterest(rom.Model):
        ...
        tags = rom.String(index=True, keygen=rom.FULL_TEXT)
        avg_rating = rom.Float(index=True)
        lon = rom.Float()
        lat = rom.Float()
        geo_index = [
            # callback function passed to GeoIndex as the 2nd argument *must*
            # return a dictionary containing 'lon' and 'lat' values, as degrees
            rom.GeoIndex('geo_index', cb),
            ...
        ]

    # Gets the 50 *best* restaurants, up to 25 miles away, from a given point,
    # ordered by highest to lowest rating
    points = PointOfInterest.query \\
        .filter(tags='restaurant') \\
        .near('geo_index', <lon>, <lat>, 25, 'mi') \\
        .order_by('-avg_rating') \\
        .limit(0, 50) \\
        .all()

    # geo queries can also independently have their own counts, which can offer
    # performance improvements for some queries (filtering earlier rather than
    # later), for slightly different semantics...

    # Gets the 50 *closest* restaurants, up to 25 miles away, from a given point,
    # ordered by highest to lowest rating.
    points = PointOfInterest.query \\
        .filter(tags='restaurant') \\
        .near('geo_index', <lon>, <lat>, 25, 'mi', 50) \\
        .order_by('-avg_rating') \\
        .all()

'''

from __future__ import print_function
from collections import deque
from datetime import datetime, date, time as dtime
from hashlib import sha1
from itertools import chain
import math
import os
import string
import threading
import time
import weakref
import warnings

import redis
import six

from .exceptions import DataRaceError, ORMError

if six.PY3:
    import binascii

_skip = None
_skip = set(globals()) - set(['__doc__'])

if redis.VERSION >= (2, 8):
    REDIS_URI = os.environ.get('ROM_REDIS_URI', 'redis://localhost:6379/0')
    CONNECTION = redis.Redis.from_url(REDIS_URI)
else:
    CONNECTION = redis.Redis()

def set_connection_settings(*args, **kwargs):
    '''
    Update the global connection settings for models that don't have
    model-specific connections.
    '''
    global CONNECTION
    CONNECTION = redis.Redis(*args, **kwargs)

def get_connection():
    '''
    Override me for one of the ways to change the way I connect to Redis.
    '''
    return CONNECTION

def _connect(obj):
    '''
    Tries to get the _conn attribute from a model. Barring that, gets the
    global default connection using other methods.
    '''
    from .columns import MODELS
    if isinstance(obj, MODELS['Model']):
        obj = obj.__class__
    if hasattr(obj, '_conn'):
        return obj._conn
    if hasattr(obj, 'CONN'):
        return obj.CONN
    return get_connection()

class ClassProperty(object):
    '''
    Borrowed from: https://gist.github.com/josiahcarlson/1561563
    '''
    def __init__(self, get, set=None, delete=None):
        self.get = get
        self.set = set
        self.delete = delete

    def __get__(self, obj, cls=None):
        if cls is None:
            cls = type(obj)
        return self.get(cls)

    def __set__(self, obj, value):
        cls = type(obj)
        self.set(cls, value)

    def __delete__(self, obj):
        cls = type(obj)
        self.delete(cls)

    def getter(self, get):
        return ClassProperty(get, self.set, self.delete)

    def setter(self, set):
        return ClassProperty(self.get, set, self.delete)

    def deleter(self, delete):
        return ClassProperty(self.get, self.set, delete)

# These are just the default index key generators for numeric and string
# types. Numeric keys are used as values in a ZSET, for range searching and
# sorting. Strings are tokenized trivially, and are used as keys to map into
# standard SETs for fairly simple intersection/union searches (like for tags).
# With a bit of work on the string side of things, you could build a simple
# autocomplete, or on the numeric side of things, you could take things like
# 15.23 and turn it into a "$$" key (representing that the value is between 10
# and 20).
# If you return a dictionary with {'':...}, then you can search by just that
# attribute. But if you return a dictionary with {'x':...}, when searching,
# you must use <attr>:x as the attribute for the column.
# For an example of fetching a range based on a numeric index...
# results = Model.query.filter(col=(start, end)).execute()
# vs.
# results = Model.query.filter(**{'col:ex':(start, end)}).execute()

def _numeric_keygen(val):
    if val is None:
        return None
    if isinstance(val, (datetime, date)):
        val = dt2ts(val)
    elif isinstance(val, dtime):
        val = t2ts(val)
    return {'': repr(val) if isinstance(val, float) else str(val)}

def _boolean_keygen(val):
    return [str(bool(val))]

def FULL_TEXT(val):
    '''
    This is a basic full-text index keygen function. Words are lowercased, split
    by whitespace, and stripped of punctuation from both ends before an inverted
    index is created for term searching.
    '''
    if isinstance(val, float):
        val = repr(val)
    elif val in (None, ''):
        return None
    elif not isinstance(val, six.string_types):
        if six.PY3 and isinstance(val, bytes):
            val = val.decode('latin-1')
        else:
            val = str(val)
    r = sorted(set([x for x in [s.lower().strip(string.punctuation) for s in val.split()] if x]))
    if not isinstance(val, str):  # unicode on py2k
        return [s.encode('utf-8') for s in r]
    return r

# For compatability with the rest of the package, as well as those who are
# explicitly using this keygen as part of query calculation.
_string_keygen = FULL_TEXT

def SIMPLE(val):
    '''
    This is a basic case-sensitive "sorted order" index keygen function for
    strings. This will return a value that is suitable to be used for ordering
    by a 7-byte prefix of a string (that is 7 characters from a byte-string, and
    1.75-7 characters from a unicode string, depending on character -> encoding
    length).

    .. warning:: Case sensitivity is based on the (encoded) byte prefixes of the
        strings/text being indexed, so ordering *may be different* than a native
        comparison ordering (especially if an order is different based on
        characters past the 7th encoded byte).
    '''
    if not val:
        return None
    if not isinstance(val, six.string_types):
        if six.PY3 and isinstance(val, bytes):
            val = val.decode('latin-1')
        else:
            val = str(val)
    return {'': _prefix_score(val)}

def SIMPLE_CI(val):
    '''
    The same as SIMPLE, only case-insensitive.
    '''
    return SIMPLE(val.lower())

def CASE_INSENSITIVE(val):
    '''
    Old alias for SIMPLE_CI
    '''
    return SIMPLE_CI(val)

def IDENTITY(val):
    '''
    This is a basic "equality" index keygen, primarily meant to be used for
    things like::

        Model.query.filter(col='value')

    Where ``FULL_TEXT`` would transform a sentence like "A Simple Sentence" into
    an inverted index searchable by the words "a", "simple", and/or "sentence",
    ``IDENTITY`` will only be searchable by the orginal full sentence with the
    same capitalization - "A Simple Sentence". See ``IDENTITY_CI`` for the
    same function, only case-insensitive.
    '''
    if not val:
        return None
    if not isinstance(val, six.string_types_ex):
        val = str(val)
    return [val]

def IDENTITY_CI(val):
    '''
    Case-insensitive version of IDENTITY
    '''
    return IDENTITY(val.lower())

STRING_INDEX_KEYGENS = (FULL_TEXT, SIMPLE, SIMPLE_CI, IDENTITY, IDENTITY_CI, CASE_INSENSITIVE)
STRING_INDEX_KEYGENS_STR = ', '.join(x.__name__ for x in STRING_INDEX_KEYGENS)
STRING_SORT_KEYGENS = (SIMPLE, SIMPLE_CI, CASE_INSENSITIVE)
STRING_SORT_KEYGENS_STR = ', '.join(x.__name__ for x in STRING_SORT_KEYGENS)

def _many_to_one_keygen(val):
    if val is None:
        return []
    if hasattr(val, '_data') and hasattr(val, '_pkey'):
        return {'': val._data[val._pkey]}
    return {'': val.id}

def _to_score(v, s=False):
    v = repr(v) if isinstance(v, float) else str(v)
    if s:
        if v[:1] != '(':
            return '(' + v
    return v.lstrip('(')

# borrowed and modified from:
# https://gist.github.com/josiahcarlson/8459874
def _bigint_to_float(v):
    assert isinstance(v, six.integer_types)
    sign = -1 if v < 0 else 1
    v *= sign
    assert v < 0x7fe0000000000000
    exponent, mantissa = divmod(v, 2**52)
    return sign * (2**52 + mantissa) * 2.0**(exponent-52-1022)

def _prefix_score(v, next=False):
    if isinstance(v, six.text_type):
        v = v.encode('utf-8')
    # We only get 7 characters of score-based prefix.
    score = 0
    for ch in six.iterbytes(v[:7]):
        score *= 258
        score += ch + 1
    if next:
        score += 1
    score *= 258 ** max(0, 7-len(v))
    return repr(_bigint_to_float(score))

_epoch = datetime(1970, 1, 1)
_epochd = _epoch.date()
def dt2ts(value):
    if isinstance(value, datetime):
        delta = value - _epoch
    else:
        delta = value - _epochd
    return delta.days * 86400 + delta.seconds + delta.microseconds / 1000000.

def ts2dt(value):
    return datetime.utcfromtimestamp(value)

def t2ts(value):
    return value.hour*3600 + value.minute * 60 + value.second + value.microsecond / 1000000.

def ts2t(value):
    hour, value = divmod(value, 3600)
    minute, value = divmod(value, 60)
    second, value = divmod(value, 1)
    return dtime(*map(int, [hour, minute, second, value*1000000]))

def _encode_unique_constraint(data):
    cleaned = []
    for col in data:
        col = (col or b'')
        if not isinstance(col, (six.text_type, six.binary_type)):
            col = six.text_type(col)
        if isinstance(col, six.text_type):
            col = col.encode('utf-8')
        cleaned.append(b'\0\0' + col)

    ret = b'\0'.join(cleaned)
    return ret if six.PY2 else ret.decode('latin-1')

NULL_SESSION = False

class Session(threading.local):
    '''
    This is a very dumb session. All it tries to do is to keep a cache of
    loaded entities, offering the ability to call ``.save()`` on modified (or
    all) entities with ``.flush()`` or ``.commit()``.

    This is exposed via the ``session`` global variable, which is available
    when you ``import rom`` as ``rom.session``.

    .. note:: calling ``.flush()`` or ``.commit()`` doesn't cause all objects
        to be written simultanously. They are written one-by-one, with any
        error causing the call to fail.
    '''
    def __init__(self, *args, **kwargs):
        threading.local.__init__(self, *args, **kwargs)
        self._init()

    def _init(self):
        try:
            self.known
        except AttributeError:
            self.known = {}
            self.wknown = weakref.WeakValueDictionary()

    @property
    def null_session(self):
        return getattr(self, '_null_session', NULL_SESSION)

    @null_session.setter
    def null_session(self, value):
        self._null_session = value

    @null_session.deleter
    def null_session(self):
        self._null_session = None
        del self._null_session

    def add(self, obj):
        '''
        Adds an entity to the session.
        '''
        if self.null_session:
            return
        self._init()
        pk = obj._pk
        if not pk.endswith(':None'):
            self.known[pk] = obj
            self.wknown[pk] = obj

    def forget(self, obj):
        '''
        Forgets about an entity (automatically called when an entity is
        deleted). Call this to ensure that an entity that you've modified is
        not automatically saved on ``session.commit()`` .
        '''
        self._init()
        self.known.pop(obj._pk, None)
        self.wknown.pop(obj._pk, None)

    def get(self, pk):
        '''
        Fetches an entity from the session based on primary key.
        '''
        self._init()
        return self.known.get(pk) or self.wknown.get(pk)

    def rollback(self):
        '''
        Forget about all entities in the session (``.commit()`` will do
        nothing).
        '''
        self.wknown = weakref.WeakValueDictionary()
        self.known = {}

    def flush(self, full=False, all=False, force=False):
        '''
        Call ``.save()`` on all modified entities in the session. Use when you
        want to flush changes to Redis, but don't want to lose your local
        session cache.

        See the ``.commit()`` method for arguments and their meanings.
        '''
        self._init()

        return self.save(*self.known.values(), full=full, all=all, force=force)

    def commit(self, full=False, all=False, force=False):
        '''
        Call ``.save()`` on all modified entities in the session. Also forgets
        all known entities in the session, so this should only be called at
        the end of a request.

        Arguments:

            * *full* - pass ``True`` to force save full entities, not only
              changes
            * *all* - pass ``True`` to save all entities known, not only those
              entities that have been modified.
            * *full* - pass ``True`` to force-save all entities known, ignoring
              DataRaceError and EntityDeletedError exceptions
        '''
        changes = self.flush(full, all, force)
        self.known = {}
        return changes

    def save(self, *objects, **kwargs):
        '''
        This method is an alternate API for saving many entities (possibly not
        tracked by the session). You can call::

            session.save(obj)
            session.save(obj1, obj2, ...)
            session.save([obj1, obj2, ...])

        And the entities will be flushed to Redis.

        You can pass the keyword arguments ``full``, ``all``, and ``force`` with
        the same meaning and semantics as the ``.commit()`` method.
        '''
        from rom import Model
        full = kwargs.get('full')
        all = kwargs.get('all')
        force = kwargs.get('force')
        changes = 0
        items = deque()
        items.extend(objects)
        while items:
            o = items.popleft()
            if isinstance(o, (list, tuple)):
                items.extendleft(reversed(o))
            elif isinstance(o, Model):
                if not o._deleted and (all or o._modified):
                    changes += o.save(full, force)

            else:
                raise ORMError(
                    "Cannot save an object that is not an instance of a Model (you provided %r)"%(
                        o,))

        return changes

    def delete(self, *objects, **kwargs):
        '''
        This method offers the ability to delete multiple entities in a single
        round trip to Redis (assuming your models are all stored on the same
        server). You can call::

            session.delete(obj)
            session.delete(obj1, obj2, ...)
            session.delete([obj1, obj2, ...])

        The keyword argument ``force=True`` can be provided, which can force
        the deletion of an entitiy again, even if we believe it to already be
        deleted.

        If ``force=True``, we won't re-call the object's ``_before_delete()``
        method, but we will re-call ``_after_delete()``.

        .. note:: Objects are automatically dropped from the session after
            delete for the sake of cache coherency.
        '''
        force = kwargs.get('force')
        from .model import Model, SKIP_ON_DELETE

        flat = []
        items = deque()
        items.extend(objects)
        types = set()
        # flatten what was passed in, more or less arbitrarily deep
        while items:
            o = items.popleft()
            if isinstance(o, (list, tuple)):
                items.extendleft(reversed(o))
            elif isinstance(o, Model):
                if force or not o._deleted:
                    flat.append(o)
                    types.add(type(o))

        # make sure we can bulk delete everything we've been requested to
        from .columns import MODELS_REFERENCED
        for t in types:
            if not t._no_fk or t._namespace in MODELS_REFERENCED:
                raise ORMError("Can't bulk delete entities of models with foreign key relationships")

        c2p = {}
        for o in flat:
            # prepare delete
            if not o._deleted:
                o._before_delete()

            # make sure we've got connections
            c = o._connection
            if c not in c2p:
                c2p[c] = c.pipeline()

            # use our existing delete, and pass through a pipeline :P
            o.delete(_conn=c2p[c],
                skip_on_delete_i_really_mean_it=SKIP_ON_DELETE)

        # actually delete the data in Redis
        for p in c2p.values():
            p.execute()

        # remove the objects from the session
        forget = self.forget
        for o in flat:
            if o._deleted == 1:
                o._after_delete()
                o._deleted = 2
            forget(o)

    def refresh(self, *objects, **kwargs):
        '''
        This method is an alternate API for refreshing many entities (possibly
        not tracked by the session). You can call::

            session.refresh(obj)
            session.refresh(obj1, obj2, ...)
            session.refresh([obj1, obj2, ...])

        And all provided entities will be reloaded from Redis.

        To force reloading for modified entities, you can pass ``force=True``.
        '''
        self._init()
        from rom import Model
        force = kwargs.get('force')
        for o in objects:
            if isinstance(o, (list, tuple)):
                self.refresh(*o, force=force)
            elif isinstance(o, Model):
                if not o._new:
                    o.refresh(force=force)
                else:
                    # all objects are re-added to the session after refresh,
                    # except for deleted entities...
                    self.add(o)
            else:
                raise ORMError(
                    "Cannot refresh an object that is not an instance of a Model (you provided %r)"%(
                        o,))

    def refresh_all(self, *objects, **kwargs):
        '''
        This method is an alternate API for refreshing all entities tracked
        by the session. You can call::

            session.refresh_all()
            session.refresh_all(force=True)

        And all entities known by the session will be reloaded from Redis.

        To force reloading for modified entities, you can pass ``force=True``.
        '''
        self.refresh(*self.known.values(), force=kwargs.get('force'))

def use_null_session():
    '''
    If you call ``use_null_session()``, you will change the default session for
    all threads to *not cache*. You can override the default on a per-thread
    basis by manipulating ``session.null_session`` (set to ``True``, ``False``,
    or delete the attribute to not cache, cache, or use the global default,
    respectively).
    '''
    global NULL_SESSION
    NULL_SESSION = True

def use_rom_session():
    '''
    If you call ``use_rom_session()``, you will change the default session for
    all threads to *cache*. You can override the default on a per-thread basis
    by manipulating ``session.null_session`` (set to ``True``, ``False``, or
    delete the attribute to not cache, cache, or use the global default,
    respectively).
    '''
    global NULL_SESSION
    NULL_SESSION = False

session = Session()

def refresh_indices(model, block_size=100):
    '''
    This utility function will iterate over all entities of a provided model,
    refreshing their indices. This is primarily useful after adding an index
    on a column.

    Arguments:

        * *model* - the model whose entities you want to reindex
        * *block_size* - the maximum number of entities you want to fetch from
          Redis at a time, defaulting to 100

    This function will yield its progression through re-indexing all of your
    entities.

    Example use::

        for progress, total in refresh_indices(MyModel, block_size=200):
            print "%s of %s"%(progress, total)

    .. note:: This uses the session object to handle index refresh via calls to
      ``.commit()``. If you have any outstanding entities known in the
      session, they will be committed.
    '''
    conn = _connect(model)
    max_id = int(conn.get('%s:%s:'%(model._namespace, model._pkey)) or '0')
    block_size = max(block_size, 10)
    for i in range(1, max_id+1, block_size):
        # fetches entities, keeping a record in the session
        models = model.get(list(range(i, i+block_size)))
        models # for pyflakes
        # re-save un-modified data, resulting in index-only updates
        session.commit(all=True)
        yield min(i+block_size, max_id), max_id

def clean_old_index(model, block_size=100, **kwargs):
    '''
    This utility function will clean out old index data that was accidentally
    left during item deletion in rom versions <= 0.27.0 . You should run this
    after you have upgraded all of your clients to version 0.28.0 or later.

    Arguments:

        * *model* - the model whose entities you want to reindex
        * *block_size* - the maximum number of items to check at a time
          defaulting to 100

    This function will yield its progression through re-checking all of the
    data that could be left over.

    Example use::

        for progress, total in clean_old_index(MyModel, block_size=200):
            print "%s of %s"%(progress, total)
    '''

    conn = _connect(model)
    version = list(map(int, conn.info()['redis_version'].split('.')[:2]))
    has_hscan = version >= [2, 8]
    pipe = conn.pipeline(True)
    prefix = '%s:'%model._namespace
    index = prefix + ':'
    block_size = max(block_size, 10)

    force_hscan = kwargs.get('force_hscan', False)
    if (has_hscan or force_hscan) and force_hscan is not None:
        max_id = conn.hlen(index)
        cursor = None
        scanned = 0
        while cursor != b'0':
            cursor, remove = _scan_index_lua(conn, [index, prefix], [cursor or '0', block_size, 0, 0])
            if remove:
                _clean_index_lua(conn, [model._namespace], remove)

            scanned += block_size
            if scanned > max_id:
                max_id = scanned + 1
            yield scanned, max_id

        # need to scan over unique indexes :/
        for uniq in chain(model._unique, model._cunique):
            name = uniq if isinstance(uniq, six.string_types) else ':'.join(uniq)
            idx = prefix + name + ':uidx'

            cursor = None
            while cursor != b'0':
                cursor, remove = _scan_index_lua(conn, [idx, prefix], [cursor or '0', block_size, 1, 0])
                if remove:
                    conn.hdel(idx, *remove)

                scanned += block_size
                if scanned > max_id:
                    max_id = scanned + 1
                yield scanned, max_id
    else:
        if model._unique or model._cunique:
            if has_hscan:
                warnings.warn("You have disabled the use of HSCAN to clean up indexes, this will prevent unique index cleanup", stacklevel=2)
            else:
                warnings.warn("Unique indexes cannot be cleaned up in Redis versions prior to 2.8", stacklevel=2)

        max_id = int(conn.get('%s%s:'%(prefix, model._pkey)) or '0')
        for i in range(1, max_id+1, block_size):
            ids = list(range(i, min(i+block_size, max_id+1)))
            for id in ids:
                pipe.exists(prefix + str(id))
                pipe.hexists(index, id)

            result = iter(pipe.execute())
            remove = [id for id, ent, ind in zip(ids, result, result) if ind and not ent]
            if remove:
                _clean_index_lua(conn, [model._namespace], remove)

            yield min(i+block_size, max_id-1), max_id

    yield max_id, max_id

def show_progress(job):
    '''
    This utility function will print the progress of a passed iterator job as
    started by ``refresh_indices()`` and ``clean_old_index()``.

    Usage example::

        class RomTest(Model):
            pass

        for i in xrange(1000):
            RomTest().save()

        util.show_progress(util.clean_old_index(RomTest))
    '''
    start = time.time()
    last_print = 0
    last_line = 0
    for prog, total in chain(job, [(1, 1)]):
        # Only print a line when we start, finish, or every .1 seconds
        if (time.time() - last_print) > .1 or prog >= total:
            delta = (time.time() - start) or .0001
            line = "%.1f%% complete, %.1f seconds elapsed, %.1f seconds remaining"%(
                100. * prog / (total or 1), delta, total * delta / (prog or 1) - delta)
            length = len(line)
            # pad the line out with spaces just in case our line got shorter
            line += max(last_line - length, 0) * ' '
            print(line, end="\r")
            last_line = length
            last_print = time.time()
    print()

NO_SCRIPT_MESSAGES = ['NOSCRIPT', 'No matching script.']
def _script_load(script):
    '''
    Borrowed/modified from my book, Redis in Action:
    https://github.com/josiahcarlson/redis-in-action/blob/master/python/ch11_listing_source.py

    Used for Lua scripting support when writing against Redis 2.6+ to allow
    for multiple unique columns per model.
    '''
    script = script.encode('utf-8') if isinstance(script, six.text_type) else script
    sha = [None, sha1(script).hexdigest()]
    def call(conn, keys=[], args=[], force_eval=False):
        keys = tuple(keys)
        args = tuple(args)
        if not force_eval:
            if not sha[0]:
                try:
                    # executing the script implicitly loads it
                    return conn.execute_command(
                        'EVAL', script, len(keys), *(keys + args))
                finally:
                    # thread safe by re-using the GIL ;)
                    del sha[:-1]

            try:
                return conn.execute_command(
                    "EVALSHA", sha[0], len(keys), *(keys+args))

            except redis.exceptions.ResponseError as msg:
                if not any(msg.args[0].startswith(nsm) for nsm in NO_SCRIPT_MESSAGES):
                    raise

        return conn.execute_command(
            "EVAL", script, len(keys), *(keys+args))

    return call

_scan_index_lua = _script_load('''
local page = redis.call('HSCAN', KEYS[1], ARGV[1], 'COUNT', ARGV[2] or 100)
local clear = {}
local skip = tonumber(ARGV[3])
local exists = tonumber(ARGV[4])
for i=(1+skip), #page[2], 2 do
    if redis.call('EXISTS', KEYS[2] .. page[2][i]) == exists then
        table.insert(clear, page[2][i - skip])
    end
end

return {page[1], clear}
''')

_clean_index_lua = _script_load('''
-- remove old index data
-- [1] string.format("%s", d) will truncate d to the first null value, so we
--     can't rely on string.format() where we can reasonably expect nulls.

local namespace = KEYS[1]
local cleaned = 0
for _, id in ipairs(ARGV) do
    local idata = redis.call('HGET', namespace .. '::', id)
    if idata then
        cleaned = cleaned + 1
        idata = cjson.decode(idata)
        while #idata < 4 do
            idata[#idata + 1] = {}
        end
        for i, key in ipairs(idata[1]) do
            redis.call('SREM', string.format('%s:%s:idx', namespace, key), id)
            -- see note [1]
            redis.call('SREM', namespace .. ':' .. key .. ':idx', id)
        end
        for i, key in ipairs(idata[2]) do
            redis.call('ZREM', string.format('%s:%s:idx', namespace, key), id)
            -- see note [1]
            redis.call('ZREM', namespace .. ':' .. key .. ':idx', id)
        end
        for i, data in ipairs(idata[3]) do
            local key = string.format('%s:%s:pre', namespace, data[1])
            local mem = string.format('%s\0%s', data[2], id)
            redis.call('ZREM', key, mem)
            -- see note [1]
            local key = namespace .. ':' .. data[1] .. ':pre'
            local mem = data[2] .. '\0' .. id
            redis.call('ZREM', key, mem)
        end
        for i, data in ipairs(idata[4]) do
            local key = string.format('%s:%s:suf', namespace, data[1])
            local mem = string.format('%s\0%s', data[2], id)
            redis.call('ZREM', key, mem)
            -- see note [1]
            local key = namespace .. ':' .. data[1] .. ':suf'
            local mem = data[2] .. '\0' .. id
            redis.call('ZREM', key, mem)
        end
        redis.call('HDEL', namespace .. '::', id)
    end
end
return cleaned
''')

def _random_hex(bytes):
    if six.PY2:
        return os.urandom(bytes).encode('hex')
    return binascii.hexlify(os.urandom(bytes))

class Lock(object):
    '''
    Borrowed/modified from my book, Redis in Action:
    https://github.com/josiahcarlson/redis-in-action/blob/master/python/ch11_listing_source.py

    Useful for locking over a string key in Redis. Minimally correct for the
    required semantics. Mostly intended as a general building block for use by
    EntityLock.
    '''
    __slots__ = 'identifier', 'conn', 'lockname', 'lock_timeout', 'acquire_timeout'
    def __init__(self, conn, lockname, acquire_timeout, lock_timeout):
        self.identifier = None
        self.conn = conn
        self.lockname = 'lock:' + lockname
        self.lock_timeout = int(math.ceil(lock_timeout))
        self.acquire_timeout = int(math.ceil(acquire_timeout))

    def _acquire(self):
        self.identifier = self.identifier or str(_random_hex(16))
        return _acquire_refresh_lock_with_timeout_lua(
            self.conn, [self.lockname], [self.lock_timeout, self.identifier]) in ('OK', 1)

    def acquire(self):
        acquired = False
        end = time.time() + self.acquire_timeout
        while time.time() < end and not acquired:
            acquired = self._acquire()

            time.sleep(.001 * (not acquired))

        return acquired

    def refresh(self):
        refreshed = self._acquire()
        if not refreshed:
            self.identifier = None
        return refreshed

    def release(self):
        return bool(_release_lock_lua(self.conn, [self.lockname], [self.identifier]))

    def __enter__(self):
        if not self.acquire():
            raise DataRaceError("Lock is already held")
        return self

    def __exit__(self, *args, **kwargs):
        self.release()

def EntityLock(entity, acquire_timeout, lock_timeout):
    '''
    Useful when you want exclusive access to an entity across all writers.::

        # example
        import rom

        class Document(rom.Model):
            owner = rom.ManyToOne('User', on_delete='restrict')
            ...

        def change_owner(document, new_owner):
            with rom.util.EntityLock(document, 5, 90):
                document.owner = new_owner
                document.save()

    '''
    return Lock(entity._connection, entity._pk, acquire_timeout, lock_timeout)

_acquire_refresh_lock_with_timeout_lua = _script_load('''
if redis.call('exists', KEYS[1]) == 0 then
    return redis.call('setex', KEYS[1], unpack(ARGV))
elseif redis.call('get', KEYS[1]) == ARGV[2] then
    return redis.call('expire', KEYS[1], ARGV[1])
end
''')

_release_lock_lua = _script_load('''
if redis.call('get', KEYS[1]) == ARGV[1] then
    return redis.call('del', KEYS[1])
end
''')

__all__ = [k for k, v in globals().items() if getattr(v, '__doc__', None) and k not in _skip]
__all__.sort()