# -*- coding: utf-8 -*-

from marshmallow import fields
from marshmallow.utils import is_iterable_but_not_string

from sqlalchemy.orm.exc import NoResultFound

def get_primary_keys(model):
    """Get primary key properties for a SQLAlchemy model.

    :param model: SQLAlchemy model class
    """
    mapper = model.__mapper__
    return [mapper.get_property_by_column(column) for column in mapper.primary_key]

def get_schema_for_field(field):
    if hasattr(field, "root"):  # marshmallow>=2.1
        return field.root
    else:
        return field.parent

def ensure_list(value):
    return value if is_iterable_but_not_string(value) else [value]

class RelatedList(fields.List):
    def get_value(self, obj, attr, accessor=None):
        # Do not call `fields.List`'s get_value as it calls the container's
        # `get_value` if the container has `attribute`.
        # Instead call the `get_value` from the parent of `fields.List`
        # so the special handling is avoided.
        return super(fields.List, self).get_value(obj, attr, accessor=accessor)

class Related(fields.Field):
    """Related data represented by a SQLAlchemy `relationship`. Must be attached
    to a :class:`Schema` class whose options includes a SQLAlchemy `model`, such
    as :class:`ModelSchema`.

    :param list columns: Optional column names on related model. If not provided,
        the primary key(s) of the related model will be used.
    """

    default_error_messages = {
        "invalid": "Could not deserialize related value {value!r}; "
        "expected a dictionary with keys {keys!r}"
    }

    def __init__(self, column=None, **kwargs):
        super(Related, self).__init__(**kwargs)
        self.columns = ensure_list(column or [])

    @property
    def model(self):
        schema = get_schema_for_field(self)
        return schema.opts.model

    @property
    def related_model(self):
        model_attr = getattr(self.model, self.attribute or self.name)
        if hasattr(model_attr, "remote_attr"):  # handle association proxies
            model_attr = model_attr.remote_attr
        return model_attr.property.mapper.class_

    @property
    def related_keys(self):
        if self.columns:
            return [
                self.related_model.__mapper__.columns[column] for column in self.columns
            ]
        return get_primary_keys(self.related_model)

    @property
    def session(self):
        schema = get_schema_for_field(self)
        return schema.session

    @property
    def transient(self):
        schema = get_schema_for_field(self)
        return schema.transient

    def _serialize(self, value, attr, obj):
        ret = {prop.key: getattr(value, prop.key, None) for prop in self.related_keys}
        return ret if len(ret) > 1 else list(ret.values())[0]

    def _deserialize(self, value, *args, **kwargs):
        """Deserialize a serialized value to a model instance.

        If the parent schema is transient, create a new (transient) instance.
        Otherwise, attempt to find an existing instance in the database.
        :param value: The value to deserialize.
        """
        if not isinstance(value, dict):
            if len(self.related_keys) != 1:
                self.fail(
                    "invalid",
                    value=value,
                    keys=[prop.key for prop in self.related_keys],
                )
            value = {self.related_keys[0].key: value}
        if self.transient:
            return self.related_model(**value)
        try:
            result = self._get_existing_instance(
                self.session.query(self.related_model), value
            )
        except NoResultFound:
            # The related-object DNE in the DB, but we still want to deserialize it
            # ...perhaps we want to add it to the DB later
            return self.related_model(**value)
        return result

    def _get_existing_instance(self, query, value):
        """Retrieve the related object from an existing instance in the DB.

        :param query: A SQLAlchemy `Query <sqlalchemy.orm.query.Query>` object.
        :param value: The serialized value to mapto an existing instance.
        :raises NoResultFound: if there is no matching record.
        """
        if self.columns:
            result = query.filter_by(
                **{prop.key: value.get(prop.key) for prop in self.related_keys}
            ).one()
        else:
            # Use a faster path if the related key is the primary key.
            result = query.get([value.get(prop.key) for prop in self.related_keys])
            if result is None:
                raise NoResultFound
        return result


# -*- coding: utf-8 -*-
import inspect
import functools

import uuid
import marshmallow as ma
from marshmallow import validate, fields
from sqlalchemy.dialects import postgresql, mysql, mssql
import sqlalchemy as sa

from .exceptions import ModelConversionError
from .fields import Related, RelatedList

def _is_field(value):
    return isinstance(value, type) and issubclass(value, fields.Field)

def _has_default(column):
    return (
        column.default is not None
        or column.server_default is not None
        or _is_auto_increment(column)
    )

def _is_auto_increment(column):
    return column.table is not None and column is column.table._autoincrement_column

def _postgres_array_factory(converter, data_type):
    return functools.partial(
        fields.List, converter._get_field_class_for_data_type(data_type.item_type)
    )

class ModelConverter(object):
    """Class that converts a SQLAlchemy model into a dictionary of corresponding
    marshmallow `Fields <marshmallow.fields.Field>`.
    """

    SQLA_TYPE_MAPPING = {
        sa.Enum: fields.Field,
        postgresql.BIT: fields.Integer,
        postgresql.UUID: fields.UUID,
        postgresql.MACADDR: fields.String,
        postgresql.INET: fields.String,
        postgresql.JSON: fields.Raw,
        postgresql.JSONB: fields.Raw,
        postgresql.HSTORE: fields.Raw,
        postgresql.ARRAY: _postgres_array_factory,
        mysql.BIT: fields.Integer,
        mysql.YEAR: fields.Integer,
        mysql.SET: fields.List,
        mysql.ENUM: fields.Field,
        mssql.BIT: fields.Integer,
    }
    if hasattr(sa, "JSON"):
        SQLA_TYPE_MAPPING[sa.JSON] = fields.Raw

    DIRECTION_MAPPING = {"MANYTOONE": False, "MANYTOMANY": True, "ONETOMANY": True}

    def __init__(self, schema_cls=None):
        self.schema_cls = schema_cls

    @property
    def type_mapping(self):
        if self.schema_cls:
            return self.schema_cls.TYPE_MAPPING
        else:
            return ma.Schema.TYPE_MAPPING

    def fields_for_model(
        self,
        model,
        include_fk=False,
        fields=None,
        exclude=None,
        base_fields=None,
        dict_cls=dict,
    ):
        result = dict_cls()
        base_fields = base_fields or {}
        for prop in model.__mapper__.iterate_properties:
            if self._should_exclude_field(prop, fields=fields, exclude=exclude):
                # Allow marshmallow to validate and exclude the field key.
                result[prop.key] = None
                continue
            if hasattr(prop, "columns"):
                if not include_fk:
                    # Only skip a column if there is no overriden column
                    # which does not have a Foreign Key.
                    for column in prop.columns:
                        if not column.foreign_keys:
                            break
                    else:
                        continue
            field = base_fields.get(prop.key) or self.property2field(prop)
            if field:
                result[prop.key] = field
        return result

    def fields_for_table(
        self,
        table,
        include_fk=False,
        fields=None,
        exclude=None,
        base_fields=None,
        dict_cls=dict,
    ):
        result = dict_cls()
        base_fields = base_fields or {}
        for column in table.columns:
            if self._should_exclude_field(column, fields=fields, exclude=exclude):
                # Allow marshmallow to validate and exclude the field key.
                result[column.key] = None
                continue
            if not include_fk and column.foreign_keys:
                continue
            field = base_fields.get(column.key) or self.column2field(column)
            if field:
                result[column.key] = field
        return result

    def property2field(self, prop, instance=True, field_class=None, **kwargs):
        field_class = field_class or self._get_field_class_for_property(prop)
        if not instance:
            return field_class
        field_kwargs = self._get_field_kwargs_for_property(prop)
        field_kwargs.update(kwargs)
        ret = field_class(**field_kwargs)
        if (
            hasattr(prop, "direction")
            and self.DIRECTION_MAPPING[prop.direction.name]
            and prop.uselist is True
        ):
            ret = RelatedList(ret, **kwargs)
        return ret

    def column2field(self, column, instance=True, **kwargs):
        field_class = self._get_field_class_for_column(column)
        if not instance:
            return field_class
        field_kwargs = self.get_base_kwargs()
        self._add_column_kwargs(field_kwargs, column)
        field_kwargs.update(kwargs)
        return field_class(**field_kwargs)

    def field_for(self, model, property_name, **kwargs):
        prop = model.__mapper__.get_property(property_name)
        return self.property2field(prop, **kwargs)

    def _get_field_class_for_column(self, column):
        return self._get_field_class_for_data_type(column.type)

    def _get_field_class_for_data_type(self, data_type):
        field_cls = None
        types = inspect.getmro(type(data_type))
        # First search for a field class from self.SQLA_TYPE_MAPPING
        for col_type in types:
            if col_type in self.SQLA_TYPE_MAPPING:
                field_cls = self.SQLA_TYPE_MAPPING[col_type]
                if callable(field_cls) and not _is_field(field_cls):
                    field_cls = field_cls(self, data_type)
                break
        else:
            # Try to find a field class based on the column's python_type
            try:
                python_type = data_type.python_type
            except NotImplementedError:
                python_type = None

            if python_type in self.type_mapping:
                field_cls = self.type_mapping[python_type]
            else:
                if hasattr(data_type, "impl"):
                    return self._get_field_class_for_data_type(data_type.impl)
                raise ModelConversionError(
                    "Could not find field column of type {}.".format(types[0])
                )
        return field_cls

    def _get_field_class_for_property(self, prop):
        if hasattr(prop, "direction"):
            field_cls = Related
        else:
            column = prop.columns[0]
            field_cls = self._get_field_class_for_column(column)
        return field_cls

    def _merge_validators(self, defaults, new):
        new_classes = [validator.__class__ for validator in new]
        return [
            validator
            for validator in defaults
            if validator.__class__ not in new_classes
        ] + new

    def _get_field_kwargs_for_property(self, prop):
        kwargs = self.get_base_kwargs()
        if hasattr(prop, "columns"):
            column = prop.columns[0]
            self._add_column_kwargs(kwargs, column)
            prop = column
        if hasattr(prop, "direction"):  # Relationship property
            self._add_relationship_kwargs(kwargs, prop)
        if getattr(prop, "doc", None):  # Useful for documentation generation
            kwargs["description"] = prop.doc
        info = getattr(prop, "info", dict())
        overrides = info.get("marshmallow")
        if overrides is not None:
            validate = overrides.pop("validate", [])
            kwargs["validate"] = self._merge_validators(
                kwargs["validate"], validate
            )  # Ensure we do not override the generated validators.
            kwargs.update(overrides)  # Override other kwargs.
        return kwargs

    def _add_column_kwargs(self, kwargs, column):
        """Add keyword arguments to kwargs (in-place) based on the passed in
        `Column <sqlalchemy.schema.Column>`.
        """
        if column.nullable:
            kwargs["allow_none"] = True
        kwargs["required"] = not column.nullable and not _has_default(column)

        if hasattr(column.type, "enums"):
            kwargs["validate"].append(validate.OneOf(choices=column.type.enums))

        # Add a length validator if a max length is set on the column
        # Skip UUID columns
        # (see https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/54)
        if hasattr(column.type, "length"):
            try:
                python_type = column.type.python_type
            except (AttributeError, NotImplementedError):
                python_type = None
            if not python_type or not issubclass(python_type, uuid.UUID):
                kwargs["validate"].append(validate.Length(max=column.type.length))

        if hasattr(column.type, "scale"):
            kwargs["places"] = getattr(column.type, "scale", None)

    def _add_relationship_kwargs(self, kwargs, prop):
        """Add keyword arguments to kwargs (in-place) based on the passed in
        relationship `Property`.
        """
        nullable = True
        for pair in prop.local_remote_pairs:
            if not pair[0].nullable:
                if prop.uselist is True:
                    nullable = False
                break
        kwargs.update({"allow_none": nullable, "required": not nullable})

    def _should_exclude_field(self, column, fields=None, exclude=None):
        if fields and column.key not in fields:
            return True
        if exclude and column.key in exclude:
            return True
        return False

    def get_base_kwargs(self):
        return {"validate": []}

default_converter = ModelConverter()

fields_for_model = default_converter.fields_for_model
"""Generate a dict of field_name: `marshmallow.fields.Field` pairs for the
given model.

:param model: The SQLAlchemy model
:param bool include_fk: Whether to include foreign key fields in the output.
:return: dict of field_name: Field instance pairs
"""

property2field = default_converter.property2field
"""Convert a SQLAlchemy `Property` to a field instance or class.

:param Property prop: SQLAlchemy Property.
:param bool instance: If `True`, return  `Field` instance, computing relevant kwargs
    from the given property. If `False`, return the `Field` class.
:param kwargs: Additional keyword arguments to pass to the field constructor.
:return: A `marshmallow.fields.Field` class or instance.
"""

column2field = default_converter.column2field
"""Convert a SQLAlchemy `Column <sqlalchemy.schema.Column>` to a field instance or class.

:param sqlalchemy.schema.Column column: SQLAlchemy Column.
:param bool instance: If `True`, return  `Field` instance, computing relevant kwargs
    from the given property. If `False`, return the `Field` class.
:return: A `marshmallow.fields.Field` class or instance.
"""

field_for = default_converter.field_for
"""Convert a property for a mapped SQLAlchemy class to a marshmallow `Field`.
Example: ::

    date_created = field_for(Author, 'date_created', dump_only=True)
    author = field_for(Book, 'author')

:param type model: A SQLAlchemy mapped class.
:param str property_name: The name of the property to convert.
:param kwargs: Extra keyword arguments to pass to `property2field`
:return: A `marshmallow.fields.Field` class or instance.
"""


# -*- coding: utf-8 -*-
import marshmallow as ma
from marshmallow.compat import with_metaclass, iteritems

from .convert import ModelConverter
from .fields import get_primary_keys

class TableSchemaOpts(ma.SchemaOpts):
    """Options class for `TableSchema`.
    Adds the following options:

    - ``table``: The SQLAlchemy table to generate the `Schema` from (required).
    - ``model_converter``: `ModelConverter` class to use for converting the SQLAlchemy table to
        marshmallow fields.
    - ``include_fk``: Whether to include foreign fields; defaults to `False`.
    """

    def __init__(self, meta, *args, **kwargs):
        super(TableSchemaOpts, self).__init__(meta, *args, **kwargs)
        self.table = getattr(meta, "table", None)
        self.model_converter = getattr(meta, "model_converter", ModelConverter)
        self.include_fk = getattr(meta, "include_fk", False)

class ModelSchemaOpts(ma.SchemaOpts):
    """Options class for `ModelSchema`.
    Adds the following options:

    - ``model``: The SQLAlchemy model to generate the `Schema` from (required).
    - ``sqla_session``: SQLAlchemy session to be used for deserialization. This is optional; you
        can also pass a session to the Schema's `load` method.
    - ``model_converter``: `ModelConverter` class to use for converting the SQLAlchemy model to
        marshmallow fields.
    - ``include_fk``: Whether to include foreign fields; defaults to `False`.
    - ``transient``: Whether load model instances in a transient state (effectively ignoring
        the session).
    """

    def __init__(self, meta, *args, **kwargs):
        super(ModelSchemaOpts, self).__init__(meta, *args, **kwargs)
        self.model = getattr(meta, "model", None)
        self.sqla_session = getattr(meta, "sqla_session", None)
        self.model_converter = getattr(meta, "model_converter", ModelConverter)
        self.include_fk = getattr(meta, "include_fk", False)
        self.transient = getattr(meta, "transient", False)

class SchemaMeta(ma.schema.SchemaMeta):
    """Metaclass for `ModelSchema`."""

    # override SchemaMeta
    @classmethod
    def get_declared_fields(mcs, klass, cls_fields, inherited_fields, dict_cls):
        """Updates declared fields with fields converted from the SQLAlchemy model
        passed as the `model` class Meta option.
        """
        opts = klass.opts
        Converter = opts.model_converter
        converter = Converter(schema_cls=klass)
        declared_fields = super(SchemaMeta, mcs).get_declared_fields(
            klass, cls_fields, inherited_fields, dict_cls
        )
        fields = mcs.get_fields(converter, opts, declared_fields, dict_cls)
        fields.update(declared_fields)
        return fields

    @classmethod
    def get_fields(mcs, converter, base_fields, opts):
        pass

class TableSchemaMeta(SchemaMeta):
    @classmethod
    def get_fields(mcs, converter, opts, base_fields, dict_cls):
        if opts.table is not None:
            return converter.fields_for_table(
                opts.table,
                fields=opts.fields,
                exclude=opts.exclude,
                include_fk=opts.include_fk,
                base_fields=base_fields,
                dict_cls=dict_cls,
            )
        return dict_cls()

class ModelSchemaMeta(SchemaMeta):
    @classmethod
    def get_fields(mcs, converter, opts, base_fields, dict_cls):
        if opts.model is not None:
            return converter.fields_for_model(
                opts.model,
                fields=opts.fields,
                exclude=opts.exclude,
                include_fk=opts.include_fk,
                base_fields=base_fields,
                dict_cls=dict_cls,
            )
        return dict_cls()

class TableSchema(with_metaclass(TableSchemaMeta, ma.Schema)):
    """Base class for SQLAlchemy model-based Schemas.

    Example: ::

        from marshmallow_sqlalchemy import TableSchema
        from mymodels import engine, users

        class UserSchema(TableSchema):
            class Meta:
                table = users

        schema = UserSchema()

        select = users.select().limit(1)
        user = engine.execute(select).fetchone()
        serialized = schema.dump(user).data
    """

    OPTIONS_CLASS = TableSchemaOpts

class ModelSchema(with_metaclass(ModelSchemaMeta, ma.Schema)):
    """Base class for SQLAlchemy model-based Schemas.

    Example: ::

        from marshmallow_sqlalchemy import ModelSchema
        from mymodels import User, session

        class UserSchema(ModelSchema):
            class Meta:
                model = User

        schema = UserSchema()

        user = schema.load({'name': 'Bill'}, session=session)
        existing_user = schema.load({'name': 'Bill'}, instance=User.query.first())

    :param session: Optional SQLAlchemy session; may be overridden in `load.`
    :param instance: Optional existing instance to modify; may be overridden in `load`.
    """

    OPTIONS_CLASS = ModelSchemaOpts

    @property
    def session(self):
        return self._session or self.opts.sqla_session

    @session.setter
    def session(self, session):
        self._session = session

    @property
    def transient(self):
        return self._transient or self.opts.transient

    @transient.setter
    def transient(self, transient):
        self._transient = transient

    def __init__(self, *args, **kwargs):
        self._session = kwargs.pop("session", None)
        self.instance = kwargs.pop("instance", None)
        self._transient = kwargs.pop("transient", None)
        super(ModelSchema, self).__init__(*args, **kwargs)

    def get_instance(self, data):
        """Retrieve an existing record by primary key(s). If the schema instance
        is transient, return None.

        :param data: Serialized data to inform lookup.
        """
        if self.transient:
            return None
        props = get_primary_keys(self.opts.model)
        filters = {prop.key: data.get(prop.key) for prop in props}
        if None not in filters.values():
            return self.session.query(self.opts.model).filter_by(**filters).first()
        return None

    @ma.post_load
    def make_instance(self, data):
        """Deserialize data to an instance of the model. Update an existing row
        if specified in `self.instance` or loaded by primary key(s) in the data;
        else create a new row.

        :param data: Data to deserialize.
        """
        instance = self.instance or self.get_instance(data)
        if instance is not None:
            for key, value in iteritems(data):
                setattr(instance, key, value)
            return instance
        kwargs, association_attrs = self._split_model_kwargs_association(data)
        instance = self.opts.model(**kwargs)
        for attr, value in iteritems(association_attrs):
            setattr(instance, attr, value)
        return instance

    def load(self, data, session=None, instance=None, transient=False, *args, **kwargs):
        """Deserialize data to internal representation.

        :param session: Optional SQLAlchemy session.
        :param instance: Optional existing instance to modify.
        :param transient: Optional switch to allow transient instantiation.
        """
        self._session = session or self._session
        self._transient = transient or self._transient
        if not (self.transient or self.session):
            raise ValueError("Deserialization requires a session")
        self.instance = instance or self.instance
        try:
            return super(ModelSchema, self).load(data, *args, **kwargs)
        finally:
            self.instance = None

    def validate(self, data, session=None, *args, **kwargs):
        self._session = session or self._session
        if not (self.transient or self.session):
            raise ValueError("Validation requires a session")
        return super(ModelSchema, self).validate(data, *args, **kwargs)

    def _split_model_kwargs_association(self, data):
        """Split serialized attrs to ensure association proxies are passed separately.

        This is necessary for Python < 3.6.0, as the order in which kwargs are passed
        is non-deterministic, and associations must be parsed by sqlalchemy after their
        intermediate relationship, unless their `creator` has been set.

        Ignore invalid keys at this point - behaviour for unknowns should be
        handled elsewhere.

        :param data: serialized dictionary of attrs to split on association_proxy.
        """
        association_attrs = {
            key: value
            for key, value in iteritems(data)
            # association proxy
            if hasattr(getattr(self.opts.model, key, None), "remote_attr")
        }
        kwargs = {
            key: value
            for key, value in iteritems(data)
            if (hasattr(self.opts.model, key) and key not in association_attrs)
        }
        return kwargs, association_attrs


import sys
from datetime import datetime
from time import sleep

import humanize
import click
import logging
from sqlalchemy import create_engine
from sqlalchemy.exc import OperationalError

from .app import Stellar, __version__
from .config import InvalidConfig, MissingConfig, load_config, save_config
from .operations import database_exists, list_of_databases, SUPPORTED_DIALECTS

def upgrade_from_old_version(app):
    if app.config['migrate_from_0_3_2']:
        if app.is_old_database():
            click.echo('Upgrading from old Stellar version...')
            def after_rename(old_name, new_name):
                click.echo('* Renamed %s to %s' % (old_name, new_name))
            app.update_database_names_to_new_version(after_rename=after_rename)

        app.config['migrate_from_0_3_2'] = False
        save_config(app.config)

def get_app():
    app = Stellar()
    upgrade_from_old_version(app)
    return app

@click.group()
def stellar():
    """Fast database snapshots for development. It's like Git for databases."""
    pass

@stellar.command()
def version():
    """Shows version number"""
    click.echo("Stellar %s" % __version__)

@stellar.command()
def gc():
    """Deletes old stellar tables that are not used anymore"""
    def after_delete(database):
        click.echo("Deleted table %s" % database)

    app = get_app()
    upgrade_from_old_version(app)
    app.delete_orphan_snapshots(after_delete)

@stellar.command()
@click.argument('name', required=False)
def snapshot(name):
    """Takes a snapshot of the database"""
    app = get_app()
    upgrade_from_old_version(app)
    name = name or app.default_snapshot_name

    if app.get_snapshot(name):
        click.echo("Snapshot with name %s already exists" % name)
        sys.exit(1)
    else:
        def before_copy(table_name):
            click.echo("Snapshotting database %s" % table_name)
        app.create_snapshot(name, before_copy=before_copy)

@stellar.command()
def list():
    """Returns a list of snapshots"""
    snapshots = get_app().get_snapshots()

    click.echo('\n'.join(
        '%s: %s' % (
            s.snapshot_name,
            humanize.naturaltime(datetime.utcnow() - s.created_at)
        )
        for s in snapshots
    ))

@stellar.command()
@click.argument('name', required=False)
def restore(name):
    """Restores the database from a snapshot"""
    app = get_app()

    if not name:
        snapshot = app.get_latest_snapshot()
        if not snapshot:
            click.echo(
                "Couldn't find any snapshots for project %s" %
                load_config()['project_name']
            )
            sys.exit(1)
    else:
        snapshot = app.get_snapshot(name)
        if not snapshot:
            click.echo(
                "Couldn't find snapshot with name %s.\n"
                "You can list snapshots with 'stellar list'" % name
            )
            sys.exit(1)

    # Check if slaves are ready
    if not snapshot.slaves_ready:
        if app.is_copy_process_running(snapshot):
            sys.stdout.write(
                'Waiting for background process(%s) to finish' %
                snapshot.worker_pid
            )
            sys.stdout.flush()
            while not snapshot.slaves_ready:
                sys.stdout.write('.')
                sys.stdout.flush()
                sleep(1)
                app.db.session.refresh(snapshot)
            click.echo('')
        else:
            click.echo('Background process missing, doing slow restore.')
            app.inline_slave_copy(snapshot)

    app.restore(snapshot)
    click.echo('Restore complete.')

@stellar.command()
@click.argument('name')
def remove(name):
    """Removes a snapshot"""
    app = get_app()

    snapshot = app.get_snapshot(name)
    if not snapshot:
        click.echo("Couldn't find snapshot %s" % name)
        sys.exit(1)

    click.echo("Deleting snapshot %s" % name)
    app.remove_snapshot(snapshot)
    click.echo("Deleted")

@stellar.command()
@click.argument('old_name')
@click.argument('new_name')
def rename(old_name, new_name):
    """Renames a snapshot"""
    app = get_app()

    snapshot = app.get_snapshot(old_name)
    if not snapshot:
        click.echo("Couldn't find snapshot %s" % old_name)
        sys.exit(1)

    new_snapshot = app.get_snapshot(new_name)
    if new_snapshot:
        click.echo("Snapshot with name %s already exists" % new_name)
        sys.exit(1)

    app.rename_snapshot(snapshot, new_name)
    click.echo("Renamed snapshot %s to %s" % (old_name, new_name))

@stellar.command()
@click.argument('name')
def replace(name):
    """Replaces a snapshot"""
    app = get_app()

    snapshot = app.get_snapshot(name)
    if not snapshot:
        click.echo("Couldn't find snapshot %s" % name)
        sys.exit(1)

    app.remove_snapshot(snapshot)
    app.create_snapshot(name)
    click.echo("Replaced snapshot %s" % name)

@stellar.command()
def init():
    """Initializes Stellar configuration."""
    while True:
        url = click.prompt(
            "Please enter the url for your database.\n\n"
            "For example:\n"
            "PostgreSQL: postgresql://localhost:5432/\n"
            "MySQL: mysql+pymysql://root@localhost/"
        )
        if url.count('/') == 2 and not url.endswith('/'):
            url = url + '/'

        if (
            url.count('/') == 3 and
            url.endswith('/') and
            url.startswith('postgresql://')
        ):
            connection_url = url + 'template1'
        else:
            connection_url = url

        engine = create_engine(connection_url, echo=False)
        try:
            conn = engine.connect()
        except OperationalError as err:
            click.echo("Could not connect to database: %s" % url)
            click.echo("Error message: %s" % err.message)
            click.echo('')
        else:
            break

    if engine.dialect.name not in SUPPORTED_DIALECTS:
        click.echo("Your engine dialect %s is not supported." % (
            engine.dialect.name
        ))
        click.echo("Supported dialects: %s" % (
            ', '.join(SUPPORTED_DIALECTS)
        ))

    if url.count('/') == 3 and url.endswith('/'):
        while True:
            click.echo("You have the following databases: %s" % ', '.join([
                db for db in list_of_databases(conn)
                if not db.startswith('stellar_')
            ]))

            db_name = click.prompt(
                "Please enter the name of the database (eg. projectdb)"
            )
            if database_exists(conn, db_name):
                break
            else:
                click.echo("Could not find database %s" % db_name)
                click.echo('')
    else:
        db_name = url.rsplit('/', 1)[-1]
        url = url.rsplit('/', 1)[0] + '/'

    name = click.prompt(
        'Please enter your project name (used internally, eg. %s)' % db_name,
        default=db_name
    )

    raw_url = url

    if engine.dialect.name == 'postgresql':
        raw_url = raw_url + 'template1'

    with open('stellar.yaml', 'w') as project_file:
        project_file.write(
            """
project_name: '%(name)s'
tracked_databases: ['%(db_name)s']
url: '%(raw_url)s'
stellar_url: '%(url)sstellar_data'
            """.strip() %
            {
                'name': name,
                'raw_url': raw_url,
                'url': url,
                'db_name': db_name
            }
        )

    click.echo("Wrote stellar.yaml")
    click.echo('')
    if engine.dialect.name == 'mysql':
        click.echo("Warning: MySQL support is still in beta.")
    click.echo("Tip: You probably want to take a snapshot: stellar snapshot")

def main():
    try:
        stellar()
    except MissingConfig:
        click.echo("You don't have stellar.yaml configuration yet.")
        click.echo("Initialize it by running: stellar init")
        sys.exit(1)
    except InvalidConfig as e:
        click.echo("Your stellar.yaml configuration is wrong: %s" % e.message)
        sys.exit(1)
    except ImportError as e:
        libraries = {
            'psycopg2': 'PostreSQL',
            'pymysql': 'MySQL',
        }
        for library, name in libraries.items():
            if 'No module named' in str(e) and library in str(e):
                click.echo(
                    "Python library %s is required for %s support." %
                    (library, name)
                )
                click.echo("You can install it with pip:")
                click.echo("pip install %s" % library)
                sys.exit(1)
            elif 'No module named' in str(e) and 'MySQLdb' in str(e):
                click.echo(
                    "MySQLdb binary drivers are required for MySQL support. "
                    "You can try installing it with these instructions: "
                    "http://stackoverflow.com/questions/454854/no-module-named"
                    "-mysqldb"
                )
                click.echo('')
                click.echo("Alternatively you can use pymysql instead:")
                click.echo("1. Install it first: pip install pymysql")
                click.echo(
                    "2. Specify database url as "
                    "mysql+pymysql://root@localhost/ and not as "
                    "mysql://root@localhost/"
                )
                sys.exit(1)
        raise

if __name__ == '__main__':
    main()


import numpy as np
from typing import ClassVar, Tuple, List
from keras.utils import Sequence
import logging

class Neg_Sampling_Data_Gen(Sequence):
    """
    Keras custom data generator that allows for negative sampling per batch.

    This object creates a data generator for Keras that is useful when you have
    parallel corpuses of text and want to build a model to predict which pairs of
    text are matching pairs.

    Inspiration taken from: https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly.html
    """
    def __init__(self,
                 vectorized_text1: np.ndarray,
                 vectorized_text2: np.ndarray,
                 bs: int,
                 neg_sample_per_pair: int,
                 shuffle: bool = True) -> ClassVar:
        """
        vectorized_text1 and vectorized_text2 are complimentary pieces of text that
        you want to train a neural network to match.

        vectorized_text1 : np.ndarray[int]
            This is the vectorized version of 1 body of text.
        vectorized_text2 : np.ndarray[int]
            This is the vectorized version of a complimentary body of text that has the
            same number of rows of `vectorized_text1`
        bs : int
            batch size.
        neg_sample_per_pair : int
            The number of negative samples you want to generate per pair of matching texts. This
            will cause your effective batch size to increase to bs * neg_sample_per_pair
        shuffle : bool
            Whether or not you want to shuffle the data after each epoch.  True by default.
        """
        # checks
        max_neg_sample = bs - 1
        self.eff_bs = (bs * neg_sample_per_pair) + bs
        assert vectorized_text1.shape[0] == vectorized_text2.shape[0], 'Number of rows in vectorized_text{1,2} should be equivalent.'
        assert bs <= vectorized_text1.shape[0], f'Batch size cannot exceed number of rows in data: {vectorized_text1.shape[0]:,}'
        assert vectorized_text1.ndim == 2, 'Numpy array `vectorized_text1` should only have 2 dimensions.'
        assert vectorized_text2.ndim == 2, 'Numpy array `vectorized_text2` should only have 2 dimensions.'
        assert neg_sample_per_pair <= max_neg_sample, f'`negative_samples_per_pair` cannot exceed {max_neg_sample:,}. Based upon batch size of {bs:,}'
        logging.warning(f'Effective batch size is {bs} + ({bs}*{neg_sample_per_pair}) = {self.eff_bs:,}')

        self.nrows = vectorized_text1.shape[0]
        self.text1 = vectorized_text1
        self.text2 = vectorized_text2
        self.bs = bs
        self.shuffle = shuffle
        self.on_epoch_end()
        self.k_neg = neg_sample_per_pair * bs

    def __len__(self):
        'Denotes the number of batches per epoch'
        return self.nrows // self.bs

    def __getitem__(self, index: int) -> Tuple:
        'Generate one batch of data'
        # Generate indexes of the batch
        indexes = self.indexes[index*self.bs:(index+1)*self.bs]

        # Generate data
        X, y = self.__negative_sampling(indexes)

        return X, y

    def on_epoch_end(self) -> None:
        'Updates indexes after each epoch for shuffling'
        self.indexes = np.arange(self.nrows)
        if self.shuffle:
            np.random.shuffle(self.indexes)

    def __negative_sampling(self,
                            indexes: np.ndarray) -> Tuple[List, np.ndarray]:

        # cartesian product of indexes paired with itself
        idxs = np.array(np.meshgrid(indexes, indexes))

        # get the indexes for each part of the two pairs (t1, t2)
        t1_idxs = idxs[0].ravel()
        t2_idxs = idxs[1].ravel()
        match_flag = np.equal(t1_idxs, t2_idxs)

        # get negative indices
        neg_exs = np.stack([t1_idxs[~match_flag], t2_idxs[~match_flag]], axis=1)
        neg_idxs = neg_exs[np.random.choice(neg_exs.shape[0], self.k_neg, replace=False)]

        # get positive indices
        pos_idxs = np.stack([t1_idxs[match_flag], t2_idxs[match_flag]], axis=1)
        assert pos_idxs.shape[0] == self.bs, 'Number of positive examples: {pos_idxs.shape[0]:,} must equal batch size: {self.bs:,}.'

        # combine negative and positive indices
        final_idxs = np.vstack([pos_idxs, neg_idxs])
        np.random.shuffle(final_idxs)

        # use the indexes to retrieve the data
        text1 = self.text1[final_idxs[:, 0]]
        text2 = self.text2[final_idxs[:, 1]]

        # label = 1 when indices are equal, otherwise 0
        labels = np.equal(final_idxs[:, 0], final_idxs[:, 1]) * 1  # convert to int

        # checks
        assert text1.shape[0] == self.eff_bs, f'Num of rows returned from text1 {text1.shape[0]:,} does not match expected value of {self.eff_bs:,}.'
        assert text2.shape[0] == self.eff_bs, f'Num of rows returned from text2 {text2.shape[0]:,} does not match expected value of {self.eff_bs:,}.'
        assert text1.shape[0] == text2.shape[0], f'Num of rows returned by text1: {text1.shape[0]:,} is not equal to text2: {text2.shape[0]:,}.'
        assert labels.shape[0] == text1.shape[0], f'Num of rows in label: {labels.shape[0]:,} must be equal to rows in text: {text1.shape[0]:,}.'

        return [text1, text2], labels


"""
Utilities for cleaning, tokenizing and vectorizing text.
Author: Hamel Husain
"""
from pathos.multiprocessing import Pool, cpu_count
from more_itertools import chunked
from typing import List, Callable, Union, Any
from math import ceil
from textacy.preprocess import preprocess_text
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import text_to_word_sequence, Tokenizer
from collections import OrderedDict
import pandas as pd
import logging
from itertools import chain
from collections import Counter
import timeit

def flattenlist(listoflists):
    return list(chain.from_iterable(listoflists))

def get_time():
    return timeit.default_timer()

def time_diff(start):
    return int(get_time() - start)

def myround(x, base=5):
    """Custom rounding for histogram."""
    return int(base * round(float(x) / base))

def count_len(tkn_doc: List[str]) -> List[int]:
    """Build histogram of length of documents (# of tokens) in increments of 5."""
    return myround(len(tkn_doc))

def textacy_cleaner(text: str) -> str:
    """
    Defines the default function for cleaning text.

    This function operates over a list.
    """
    return preprocess_text(text,
                           fix_unicode=True,
                           lowercase=True,
                           transliterate=True,
                           no_urls=True,
                           no_emails=True,
                           no_phone_numbers=True,
                           no_numbers=True,
                           no_currency_symbols=True,
                           no_punct=True,
                           no_contractions=False,
                           no_accents=True)

def apply_parallel(func: Callable,
                   data: List[Any],
                   cpu_cores: int = None) -> List[Any]:
    """
    Apply function to list of elements.

    Automatically determines the chunk size.
    """
    if not cpu_cores:
        cpu_cores = cpu_count()

    try:
        chunk_size = ceil(len(data) / cpu_cores)
        pool = Pool(cpu_cores)
        transformed_data = pool.map(func, chunked(data, chunk_size), chunksize=1)
    finally:
        pool.close()
        pool.join()
        return transformed_data

def process_text_constructor(cleaner: Callable,
                             tokenizer: Callable,
                             append_indicators: bool,
                             start_tok: str,
                             end_tok: str):
    """Generate a function that will clean and tokenize text."""
    def process_text(text):
        if append_indicators:
            return [[start_tok] + tokenizer(cleaner(doc)) + [end_tok] for doc in text]
        return [tokenizer(cleaner(doc)) for doc in text]

    return process_text

class processor_base(object):
    """
    Utility for preprocessing of text for deep learning.
    """

    def __init__(self,
                 cleaner: Callable = None,
                 tokenizer: Callable = None) -> None:
        """
        cleaner: function that takes as input string and outputs a string
        tokenizer: function that takes as input a list of strings and outputs
                   a list of lists of tokens.
        """
        if cleaner is None:
            self.cleaner = textacy_cleaner

        if tokenizer is None:
            self.tokenizer = text_to_word_sequence

    def set_tokenizer(self, func: Callable) -> None:
        """
        Set the tokenizer you wish to use.

        This is a function that f(str) -> List(str)
        """
        self.tokenizer = func

    def set_cleaner(self, func: Callable) -> None:
        """
        Set the cleaner you wish to use.

        This is a function that f(str) -> str
        """
        self.cleaner = func

class processor(processor_base):
    """
    Pre-process text in memory.

    Includes utilities for cleaning, tokenization, and vectorization in parallel.
    """
    def __init__(self,
                 heuristic_pct_padding: float = .90,
                 append_indicators: bool = False,
                 keep_n: int = 150000,
                 padding: str = 'pre',
                 padding_maxlen: Union[int, None] = None,
                 truncating: str = 'post'
                 ):
        """
        Parameters:
        ----------
        heuristic_pct_padding: float
            This parameter is only used if `padding_maxlen` = None.  A histogram
            of documents is calculated, and the maxlen is set heuristic_pct_padding.
        append_indicators: bool
            If True, will append the tokens '_start_' and '_end_' to the beginning
            and end of your tokenized documents.  This can be useful when training
            seq2seq models.
        keep_n: int = 150000
            This is the maximum size of your vocabulary (unique number of words
            allowed).  Consider limiting this to a reasonable size based upon
            your corpus.
        padding : str
            'pre' or 'post', pad either before or after each sequence.
        padding_maxlen : int or None
            Maximum sequence length, longer sequences are truncated and shorter
            sequences are padded with zeros at the end.  Note if this is specified,
            the `heuristic_pct_padding` is ignored.
        truncating : str
            'pre' or 'post', remove values from sequences larger than padding_maxlen
            either in the beginning or in the end of the sequence.

        See https://keras.io/preprocessing/sequence/

        Attributes:
        -----------
        n_tokens : int
            The total number of tokens in the corpus.  Will be less than or
            equal to keep_n
        id2token : dict
            dict with { int : str} ex: {'the': 2, 'cat': 3}
            this is used for converting tokens to integers.
        token2id : dict
            dict with {str: int} ex: {2: 'the', 3: 'cat'}
            this is used for decoding predictions back to tokens
        document_length_stats : pandas.DataFrame
            histogram of document lengths.  Can be used to decide padding_maxlen.
        """
        super().__init__()
        self.heuristic_pct = heuristic_pct_padding
        self.append_indicators = append_indicators
        self.keep_n = keep_n
        self.padding = padding
        self.padding_maxlen = padding_maxlen
        self.truncating = truncating

        # These are placeholders for data that will be collected or calculated
        self.indexer = None
        self.n_tokens = None
        self.id2token = None
        self.token2id = None
        self.document_length_histogram = Counter()
        self.document_length_stats = None
        self.doc_length_huerestic = None
        self.num_cores = cpu_count()

        # These values are 'hardcoded' for now
        self.padding_value = 0.0
        self.padding_dtype = 'int32'
        self.start_tok = '_start_'
        self.end_tok = '_end_'

    def __clear_data(self):
        self.indexer = None
        self.document_length_histogram = None
        self.document_length_stats = None

    def set_num_processes(self, n):
        """Set the number of processes for process bases threading."""
        self.num_cores = min(int(n), cpu_count())

    def process_text(self, text: List[str]) -> List[List[str]]:
        """Combine the cleaner and tokenizer."""
        process_text = process_text_constructor(cleaner=self.cleaner,
                                                tokenizer=self.tokenizer,
                                                append_indicators=self.append_indicators,
                                                start_tok=self.start_tok,
                                                end_tok=self.end_tok)
        return process_text(text)

    def parallel_process_text(self, data: List[str]) -> List[List[str]]:
        """Apply cleaner -> tokenizer."""
        process_text = process_text_constructor(cleaner=self.cleaner,
                                                tokenizer=self.tokenizer,
                                                append_indicators=self.append_indicators,
                                                start_tok=self.start_tok,
                                                end_tok=self.end_tok)
        n_cores = self.num_cores
        return flattenlist(apply_parallel(process_text, data, n_cores))

    def generate_doc_length_stats(self):
        """Analyze document length statistics for padding strategy"""
        heuristic = self.heuristic_pct
        histdf = (pd.DataFrame([(a, b) for a, b in self.document_length_histogram.items()],
                               columns=['bin', 'doc_count'])
                  .sort_values(by='bin'))
        histdf['cumsum_pct'] = histdf.doc_count.cumsum() / histdf.doc_count.sum()

        self.document_length_stats = histdf
        self.doc_length_huerestic = histdf.query(f'cumsum_pct >= {heuristic}').bin.head(1).values[0]
        logging.warning(' '.join(["Setting maximum document length to",
                                  f'{self.doc_length_huerestic} based upon',
                                  f'heuristic of {heuristic} percentile.\n',
                                  'See full histogram by insepecting the',
                                  "`document_length_stats` attribute."]))
        self.padding_maxlen = self.doc_length_huerestic

    def fit(self,
            data: List[str],
            return_tokenized_data: bool = False) -> Union[None, List[List[str]]]:
        """
        TODO: update docs

        Apply cleaner and tokenzier to raw data and build vocabulary.

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]
        return_tokenized_data : bool
            Return the tokenized strings.  This is primarly used for debugging
            purposes.

        Returns
        -------
        None or List[List[str]]
            if return_tokenized_data=True then will return tokenized documents,
            otherwise will not return anything.
        """
        self.__clear_data()
        now = get_time()
        logging.warning(f'....tokenizing data')
        tokenized_data = self.parallel_process_text(data)

        if not self.padding_maxlen:
            # its not worth the overhead to parallelize document length counts
            length_counts = map(count_len, tokenized_data)
            self.document_length_histogram = Counter(length_counts)
            self.generate_doc_length_stats()

        # Learn corpus on single thread
        logging.warning(f'(1/2) done. {time_diff(now)} sec')
        logging.warning(f'....building corpus')
        now = get_time()
        self.indexer = custom_Indexer(num_words=self.keep_n)
        self.indexer.fit_on_tokenized_texts(tokenized_data)

        # Build Dictionary accounting For 0 padding, and reserve 1 for unknown and rare Words
        self.token2id = self.indexer.word_index
        self.id2token = {v: k for k, v in self.token2id.items()}
        self.n_tokens = max(self.indexer.word_index.values())

        # logging
        logging.warning(f'(2/2) done. {time_diff(now)} sec')
        logging.warning(f'Finished parsing {self.indexer.document_count:,} documents.')

        if return_tokenized_data:
            return tokenized_data

    def token_count_pandas(self):
        """ See token counts as pandas dataframe"""
        freq_df = pd.DataFrame.from_dict(self.indexer.word_counts, orient='index')
        freq_df.columns = ['count']
        return freq_df.sort_values('count', ascending=False)

    def fit_transform(self,
                      data: List[str]) -> List[List[int]]:
        """
        Apply cleaner and tokenzier to raw data, build vocabulary and return
        transfomred dataset that is a List[List[int]].  This will use
        process-based-threading on all available cores.

        ex:
        >>> data = [["The quick brown fox"], ["jumps over the lazy dog"]]
        >>> pp = preprocess(maxlen=5, no_below=0)
        >>> pp.fit_transform(data)
        # 0 padding is applied
        [[0, 2, 3, 4, 5], [6, 7, 2, 8, 9]]

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]

        Returns
        -------
        numpy.array with shape (number of documents, max_len)

        """
        tokenized_data = self.fit(data, return_tokenized_data=True)

        logging.warning(f'...fit is finished, beginning transform')
        now = get_time()
        indexed_data = self.indexer.tokenized_texts_to_sequences(tokenized_data)
        logging.warning(f'...padding data')
        final_data = self.pad(indexed_data)
        logging.warning(f'done. {time_diff(now)} sec')
        return final_data

    def transform(self, data: List[str]) -> List[List[int]]:
        """
        Transform List of documents into List[List[int]]
        If transforming a large number of documents consider using the method
        `transform_parallel` instead.

        ex:
        >> pp = processor()
        >> pp.fit(docs)
        >> new_docs = [["The quick brown fox"], ["jumps over the lazy dog"]]
        >> pp.transform(new_docs)
        [[1, 2, 3, 4], [5, 6, 1, 7, 8]]
        """
        tokenized_data = self.process_text(data)
        indexed_data = self.indexer.tokenized_texts_to_sequences(tokenized_data)
        return self.pad(indexed_data)

    def transform_parallel(self, data: List[str]) -> List[List[int]]:
        """
        Transform List of documents into List[List[int]].  Uses process based
        threading on all available cores.  If only processing a small number of
        documents ( < 10k ) then consider using the method `transform` instead.

        ex:
        >> pp = processor()
        >> pp.fit(docs)
        >> new_docs = [["The quick brown fox"], ["jumps over the lazy dog"]]
        >> pp.transform_parallel(new_docs)
        [[1, 2, 3, 4], [5, 6, 1, 7, 8]]
        """
        logging.warning(f'...tokenizing data')
        tokenized_data = self.parallel_process_text(data)
        logging.warning(f'...indexing data')
        indexed_data = self.indexer.tokenized_texts_to_sequences(tokenized_data)
        logging.warning(f'...padding data')
        return self.pad(indexed_data)

    def pad(self, docs: List[List[int]]) -> List[List[int]]:
        """
        Vectorize and apply padding on a set of tokenized doucments
        ex: [['hello, 'world'], ['goodbye', 'now']]
        """
        # First apply indexing on all the rows then pad_sequnces (i found this
        # faster than trying to do these steps on each row
        return pad_sequences(docs,
                             maxlen=self.padding_maxlen,
                             dtype=self.padding_dtype,
                             padding=self.padding,
                             truncating=self.truncating,
                             value=self.padding_value)

class custom_Indexer(Tokenizer):
    """
    Text vectorization utility class.
    This class inherits keras.preprocess.text.Tokenizer but adds methods
    to fit and transform on already tokenized text.

    Parameters
    ----------
    num_words : int
        the maximum number of words to keep, based
        on word frequency. Only the most common `num_words` words will
        be kept.
    """

    def __init__(self, num_words):
        self.num_words = num_words
        self.word_counts = OrderedDict()
        self.word_docs = {}
        self.document_count = 0

    def fit_on_tokenized_texts(self, tokenized_texts):
            self.document_count = 0
            for seq in tokenized_texts:
                self.document_count += 1

                for w in seq:
                    if w in self.word_counts:
                        self.word_counts[w] += 1
                    else:
                        self.word_counts[w] = 1
                for w in set(seq):
                    if w in self.word_docs:
                        self.word_docs[w] += 1
                    else:
                        self.word_docs[w] = 1

            wcounts = list(self.word_counts.items())
            wcounts.sort(key=lambda x: x[1], reverse=True)
            sorted_voc = [wc[0] for wc in wcounts][:self.num_words]
            # note that index 0 and 1 are reserved, never assigned to an existing word
            self.word_index = dict(list(zip(sorted_voc, list(range(2, len(sorted_voc) + 2)))))

    def tokenized_texts_to_sequences(self, tok_texts):
        """Transforms tokenized text to a sequence of integers.
        Only top "num_words" most frequent words will be taken into account.
        Only words known by the tokenizer will be taken into account.
        # Arguments
            tokenized texts:  List[List[str]]
        # Returns
            A list of integers.
        """
        res = []
        for vect in self.tokenized_texts_to_sequences_generator(tok_texts):
            res.append(vect)
        return res

    def tokenized_texts_to_sequences_generator(self, tok_texts):
        """Transforms tokenized text to a sequence of integers.
        Only top "num_words" most frequent words will be taken into account.
        Only words known by the tokenizer will be taken into account.
        # Arguments
            tokenized texts:  List[List[str]]
        # Yields
            Yields individual sequences.
        """
        for seq in tok_texts:
            vect = []
            for w in seq:
                # if the word is missing you get oov_index
                i = self.word_index.get(w, 1)
                vect.append(i)
            yield vect


"""
python-phabricator
------------------
>>> api = phabricator.Phabricator()
>>> api.user.whoami().userName
'example'

For more endpoints, see https://secure.phabricator.com/conduit/

"""
try:
    __version__ = __import__('pkg_resources') \
        .get_distribution('phabricator').version
except:
    __version__ = 'unknown'

import collections
import copy
import hashlib
import json
import os.path
import re
import socket
import time

from ._compat import (
    MutableMapping, iteritems, string_types, httplib, urlparse, urlencode,
)

__all__ = ['Phabricator']

ON_WINDOWS = os.name == 'nt'
CURRENT_DIR = os.getcwd()

# Default Phabricator interfaces
INTERFACES = {}
with open(os.path.join(os.path.dirname(__file__), 'interfaces.json')) as fobj:
    INTERFACES = json.load(fobj)

# Load arc config
ARC_CONFIGS = (
    # System config
    os.path.join(
        os.environ['ProgramData'],
        'Phabricator',
        'Arcanist',
        'config'
    ) if ON_WINDOWS else os.path.join('/etc', 'arcconfig'),

    # User config
    os.path.join(
        os.environ['AppData'] if ON_WINDOWS else os.path.expanduser('~'),
        '.arcrc'
    ),

    # Project config
    os.path.join(CURRENT_DIR, '.arcconfig'),

    # Local project config
    os.path.join(CURRENT_DIR, '.git', 'arc', 'config'),
)

ARCRC = {}
for conf in ARC_CONFIGS:
    if os.path.exists(conf):
        with open(conf, 'r') as fobj:
            ARCRC.update(json.load(fobj))

# Map Phabricator types to Python types
PARAM_TYPE_MAP = {
    # int types
    'int': int,
    'uint': int,
    'revisionid': int,
    'revision_id': int,
    'diffid': int,
    'diff_id': int,
    'id': int,

    # bool types
    'bool': bool,

    # dict types
    'map': dict,
    'dict': dict,

    # list types
    'list': list,

    # tuple types
    'pair': tuple,

    # str types
    'str': string_types,
    'string': string_types,
    'phid': string_types,
    'guids': string_types,
    'type': string_types,
}

TYPE_INFO_COMMENT_RE = re.compile(r'\s*\([^)]+\)\s*$')
TYPE_INFO_SPLITTER_RE = re.compile(r'(\w+(?:<.+>)?)(?:\s+|$)')
TYPE_INFO_RE = re.compile(r'<?(\w+)(<[^>]+>>?)?(?:.+|$)')

def map_param_type(param_type):
    """
    Perform param type mapping
    This requires a bit of logic since this isn't standardized.
    If a type doesn't map, assume str
    """
    main_type, sub_type = TYPE_INFO_RE.match(param_type).groups()

    if main_type in ('list', 'array'):
        # Handle no sub-type: "required list"
        if sub_type is not None:
            sub_type = sub_type.strip()

        if not sub_type:
            sub_type = 'str'

        # Handle list of pairs: "optional list<pair<callsign, path>>"
        sub_match = TYPE_INFO_RE.match(sub_type)
        if sub_match:
            sub_type = sub_match.group(1).lower()

        return [PARAM_TYPE_MAP.setdefault(sub_type, string_types)]

    return PARAM_TYPE_MAP.setdefault(main_type, string_types)

def parse_interfaces(interfaces):
    """
    Parse the conduit.query json dict response
    This performs the logic of parsing the non-standard params dict
        and then returning a dict Resource can understand
    """
    parsed_interfaces = collections.defaultdict(dict)

    for m, d in iteritems(interfaces):
        app, func = m.split('.', 1)

        method = parsed_interfaces[app][func] = {}

        # Make default assumptions since these aren't provided by Phab
        method['formats'] = ['json', 'human']
        method['method'] = 'POST'

        method['optional'] = {}
        method['required'] = {}

        for name, type_info in iteritems(dict(d['params'])):
            # Set the defaults
            optionality = 'required'
            param_type = 'string'

            # Usually in the format: <optionality> <param_type>
            type_info = TYPE_INFO_COMMENT_RE.sub('', type_info)
            info_pieces = TYPE_INFO_SPLITTER_RE.findall(type_info)
            for info_piece in info_pieces:
                if info_piece in ('optional', 'required'):
                    optionality = info_piece
                elif info_piece == 'ignored':
                    optionality = 'optional'
                    param_type = 'string'
                elif info_piece == 'nonempty':
                    optionality = 'required'
                elif info_piece == 'deprecated':
                    optionality = 'optional'
                else:
                    param_type = info_piece

            method[optionality][name] = map_param_type(param_type)

    return dict(parsed_interfaces)

class ConfigurationError(Exception):
    pass

class APIError(Exception):
    def __init__(self, code, message):
        self.code = code
        self.message = message

    def __str__(self):
        return '%s: %s' % (self.code, self.message)

class Result(MutableMapping):
    def __init__(self, response):
        self.response = response

    def __getitem__(self, key):
        return self.response[key]

    __getattr__ = __getitem__

    def __setitem__(self, key, value):
        self.response[key] = value

    def __delitem__(self, key):
        del self.response[key]

    def __iter__(self):
        return iter(self.response)

    def __len__(self):
        return len(self.response)

    def __repr__(self):
        return '<%s: %s>' % (type(self).__name__, repr(self.response))

class Resource(object):
    def __init__(self, api, interface=None, endpoint=None, method=None, nested=False):
        self.api = api
        self.interface = interface or copy.deepcopy(parse_interfaces(INTERFACES))
        self.endpoint = endpoint
        self.method = method
        self.nested = nested

    def __getattr__(self, attr):
        if attr in getattr(self, '__dict__'):
            return getattr(self, attr)
        interface = self.interface
        if self.nested:
            attr = "%s.%s" % (self.endpoint, attr)
        submethod_exists = False
        submethod_match = attr + '.'
        for key in interface.keys():
            if key.startswith(submethod_match):
                submethod_exists = True
                break
        if attr not in interface and submethod_exists:
            return Resource(self.api, interface, attr, self.endpoint, nested=True)
        elif attr not in interface:
            interface[attr] = {}
        if self.nested:
            return Resource(self.api, interface[attr], attr, self.method)
        return Resource(self.api, interface[attr], attr, self.endpoint)

    def __call__(self, **kwargs):
        return self._request(**kwargs)

    def _request(self, **kwargs):
        # Check for missing variables
        resource = self.interface

        def validate_kwarg(key, target):
            # Always allow list
            if isinstance(target, list):
                return (
                    isinstance(key, (list, tuple, set)) and
                    all(validate_kwarg(x, target[0]) for x in key)
                )

            return isinstance(key, tuple(target) if isinstance(target, list) else target)

        for key, val in resource.get('required', {}).items():
            if key not in [x.split(':')[0] for x in kwargs.keys()]:
                raise ValueError('Missing required argument: %s' % key)
            if isinstance(kwargs.get(key), list) and not isinstance(val, list):
                raise ValueError('Wrong argument type: %s is not a list' % key)
            elif not validate_kwarg(kwargs.get(key), val):
                if isinstance(val, list):
                    raise ValueError('Wrong argument type: %s is not a list of %ss' % (key, val[0]))
                raise ValueError('Wrong argument type: %s is not a %s' % (key, val))

        conduit = self.api._conduit

        if conduit:
            # Already authenticated, add session key to json data
            kwargs['__conduit__'] = conduit
        elif self.method == 'conduit' and self.endpoint == 'connect':
            # Not authenticated, requesting new session key
            token = str(int(time.time()))
            kwargs['authToken'] = token
            kwargs['authSignature'] = self.api.generate_hash(token)
        else:
            # Authorization is required, silently auth the user
            self.api.connect()
            kwargs['__conduit__'] = self.api._conduit

        url = urlparse.urlparse(self.api.host)
        if url.scheme == 'https':
            conn = httplib.HTTPSConnection(url.netloc, timeout=self.api.timeout)
        else:
            conn = httplib.HTTPConnection(url.netloc, timeout=self.api.timeout)

        path = url.path + '%s.%s' % (self.method, self.endpoint)

        headers = {
            'User-Agent': 'python-phabricator/%s' % str(self.api.clientVersion),
            'Content-Type': 'application/x-www-form-urlencoded'
        }

        body = urlencode({
            "params": json.dumps(kwargs),
            "output": self.api.response_format
        })

        # TODO: Use HTTP "method" from interfaces.json
        conn.request('POST', path, body, headers)
        response = conn.getresponse()

        # Make sure we got a 2xx response indicating success
        if not response.status >= 200 or not response.status < 300:
            raise httplib.HTTPException(
                'Bad response status: {0}'.format(response.status)
            )

        response_data = response.read()
        if isinstance(response_data, str):
            response = response_data
        else:
            response = response_data.decode("utf-8")

        data = self._parse_response(response)

        return Result(data['result'])

    def _parse_response(self, data):
        # Process the response back to python
        parsed = self.api.formats[self.api.response_format](data)

        # Errors return 200, so check response content for exception
        if parsed['error_code']:
            raise APIError(parsed['error_code'], parsed['error_info'])

        return parsed

class Phabricator(Resource):
    formats = {
        'json': lambda x: json.loads(x),
    }

    def __init__(self, username=None, certificate=None, host=None,
            timeout=5, response_format='json', token=None, **kwargs):

        defined_hosts = ARCRC.get('hosts', {})

        try:
            self.host = host if host else list(defined_hosts.keys())[0]
        except IndexError:
            raise ConfigurationError("No host found or provided.")

        current_host_config = defined_hosts.get(self.host, {})
        self.token = token if token else current_host_config.get('token')

        if self.token is None:
            self.username = username if username else current_host_config.get('user')
            self.certificate = certificate if certificate else current_host_config.get('cert')

        self.timeout = timeout
        self.response_format = response_format
        self.client = 'python-phabricator'
        self.clientVersion = 1
        self.clientDescription = socket.gethostname() + ':python-phabricator'
        self._conduit = None

        super(Phabricator, self).__init__(self)

    def _request(self, **kwargs):
        raise SyntaxError('You cannot call the Conduit API without a resource.')

    def connect(self):
        if self.token:
            self._conduit = {
                'token': self.token
            }
            return

        auth = Resource(api=self, method='conduit', endpoint='connect')

        response = auth(
            user=self.username,
            host=self.host,
            client=self.client,
            clientVersion=self.clientVersion
        )

        self._conduit = {
            'sessionKey': response.sessionKey,
            'connectionID': response.connectionID
        }

    def generate_hash(self, token):
        source_string = (token + self.api.certificate).encode('utf-8')
        return hashlib.sha1(source_string).hexdigest()

    def update_interfaces(self):
        query = Resource(api=self, method='conduit', endpoint='query')

        interfaces = query()

        self.interface = parse_interfaces(interfaces)


# -*- coding: utf-8 -*-
# Copyright 2009-2019 Joshua Bronson. All Rights Reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

#==============================================================================
#                    * Welcome to the bidict source code *
#==============================================================================

# Doing a code review? You'll find a "Code review nav" comment like the one
# below at the top and bottom of the most important source files. This provides
# a suggested initial path through the source when reviewing.
#
# Note: If you aren't reading this on https://github.com/jab/bidict, you may be
# viewing an outdated version of the code. Please head to GitHub to review the
# latest version, which contains important improvements over older versions.
#
# Thank you for reading and for any feedback you provide.

#                             * Code review nav *
#==============================================================================
# ← Prev: _abc.py             Current: _base.py   Next: _delegating_mixins.py →
#==============================================================================

"""Provides :class:`BidictBase`."""

from collections import namedtuple
from weakref import ref

from ._abc import BidirectionalMapping
from ._dup import RAISE, OVERWRITE, IGNORE, _OnDup
from ._exc import (
    DuplicationError, KeyDuplicationError, ValueDuplicationError, KeyAndValueDuplicationError)
from ._miss import _MISS
from ._noop import _NOOP
from ._util import _iteritems_args_kw
from .compat import PY2, KeysView, ItemsView, Mapping, iteritems

# Since BidirectionalMapping implements __subclasshook__, and BidictBase
# provides all the required attributes that the __subclasshook__ checks for,
# BidictBase would be a (virtual) subclass of BidirectionalMapping even if
# it didn't subclass it explicitly. But subclassing BidirectionalMapping
# explicitly allows BidictBase to inherit any useful implementations that
# BidirectionalMapping provides that aren't part of the required interface,
# such as its `__inverted__` implementation and `inverse` alias.

class BidictBase(BidirectionalMapping):
    """Base class implementing :class:`BidirectionalMapping`."""

    __slots__ = ('_fwdm', '_invm', '_inv', '_invweak', '_hash') + (() if PY2 else ('__weakref__',))

    #: The default :class:`DuplicationPolicy`
    #: (in effect during e.g. :meth:`~bidict.bidict.__init__` calls)
    #: that governs behavior when a provided item
    #: duplicates only the key of another item.
    #:
    #: Defaults to :attr:`~bidict.OVERWRITE`
    #: to match :class:`dict`'s behavior.
    #:
    #: *See also* :ref:`basic-usage:Values Must Be Unique`, :doc:`extending`
    on_dup_key = OVERWRITE

    #: The default :class:`DuplicationPolicy`
    #: (in effect during e.g. :meth:`~bidict.bidict.__init__` calls)
    #: that governs behavior when a provided item
    #: duplicates only the value of another item.
    #:
    #: Defaults to :attr:`~bidict.RAISE`
    #: to prevent unintended overwrite of another item.
    #:
    #: *See also* :ref:`basic-usage:Values Must Be Unique`, :doc:`extending`
    on_dup_val = RAISE

    #: The default :class:`DuplicationPolicy`
    #: (in effect during e.g. :meth:`~bidict.bidict.__init__` calls)
    #: that governs behavior when a provided item
    #: duplicates the key of another item and the value of a third item.
    #:
    #: Defaults to ``None``, which causes the *on_dup_kv* policy to match
    #: whatever *on_dup_val* policy is in effect.
    #:
    #: *See also* :ref:`basic-usage:Values Must Be Unique`, :doc:`extending`
    on_dup_kv = None

    _fwdm_cls = dict
    _invm_cls = dict

    #: The object used by :meth:`__repr__` for printing the contained items.
    _repr_delegate = dict

    def __init__(self, *args, **kw):  # pylint: disable=super-init-not-called
        """Make a new bidirectional dictionary.
        The signature is the same as that of regular dictionaries.
        Items passed in are added in the order they are passed,
        respecting the current duplication policies in the process.

        *See also* :attr:`on_dup_key`, :attr:`on_dup_val`, :attr:`on_dup_kv`
        """
        #: The backing :class:`~collections.abc.Mapping`
        #: storing the forward mapping data (*key* → *value*).
        self._fwdm = self._fwdm_cls()
        #: The backing :class:`~collections.abc.Mapping`
        #: storing the inverse mapping data (*value* → *key*).
        self._invm = self._invm_cls()
        self._init_inv()  # lgtm [py/init-calls-subclass]
        if args or kw:
            self._update(True, None, *args, **kw)

    def _init_inv(self):
        # Compute the type for this bidict's inverse bidict (will be different from this
        # bidict's type if _fwdm_cls and _invm_cls are different).
        inv_cls = self._inv_cls()
        # Create the inverse bidict instance via __new__, bypassing its __init__ so that its
        # _fwdm and _invm can be assigned to this bidict's _invm and _fwdm. Store it in self._inv,
        # which holds a strong reference to a bidict's inverse, if one is available.
        self._inv = inv = inv_cls.__new__(inv_cls)
        inv._fwdm = self._invm  # pylint: disable=protected-access
        inv._invm = self._fwdm  # pylint: disable=protected-access
        # Only give the inverse a weak reference to this bidict to avoid creating a reference cycle,
        # stored in the _invweak attribute. See also the docs in
        # :ref:`addendum:Bidict Avoids Reference Cycles`
        inv._inv = None  # pylint: disable=protected-access
        inv._invweak = ref(self)  # pylint: disable=protected-access
        # Since this bidict has a strong reference to its inverse already, set its _invweak to None.
        self._invweak = None

    @classmethod
    def _inv_cls(cls):
        """The inverse of this bidict type, i.e. one with *_fwdm_cls* and *_invm_cls* swapped."""
        if cls._fwdm_cls is cls._invm_cls:
            return cls
        if not getattr(cls, '_inv_cls_', None):
            class _Inv(cls):
                _fwdm_cls = cls._invm_cls
                _invm_cls = cls._fwdm_cls
                _inv_cls_ = cls
            _Inv.__name__ = cls.__name__ + 'Inv'
            cls._inv_cls_ = _Inv
        return cls._inv_cls_

    @property
    def _isinv(self):
        return self._inv is None

    @property
    def inverse(self):
        """The inverse of this bidict.

        *See also* :attr:`inv`
        """
        # Resolve and return a strong reference to the inverse bidict.
        # One may be stored in self._inv already.
        if self._inv is not None:
            return self._inv
        # Otherwise a weakref is stored in self._invweak. Try to get a strong ref from it.
        inv = self._invweak()
        if inv is not None:
            return inv
        # Refcount of referent must have dropped to zero, as in `bidict().inv.inv`. Init a new one.
        self._init_inv()  # Now this bidict will retain a strong ref to its inverse.
        return self._inv

    @property
    def inv(self):
        """Alias for :attr:`inverse`."""
        return self.inverse

    def __getstate__(self):
        """Needed to enable pickling due to use of :attr:`__slots__` and weakrefs.

        *See also* :meth:`object.__getstate__`
        """
        state = {}
        for cls in self.__class__.__mro__:
            slots = getattr(cls, '__slots__', ())
            for slot in slots:
                if hasattr(self, slot):
                    state[slot] = getattr(self, slot)
        # These weakrefs can't be pickled, and don't need to be anyway.
        state.pop('__weakref__', None)
        state.pop('_invweak', None)
        return state

    def __setstate__(self, state):
        """Implemented because use of :attr:`__slots__` would prevent unpickling otherwise.

        *See also* :meth:`object.__setstate__`
        """
        for slot, value in iteritems(state):
            setattr(self, slot, value)

    def __repr__(self):
        """See :func:`repr`."""
        clsname = self.__class__.__name__
        if not self:
            return '%s()' % clsname
        return '%s(%r)' % (clsname, self._repr_delegate(iteritems(self)))

    # The inherited Mapping.__eq__ implementation would work, but it's implemented in terms of an
    # inefficient ``dict(self.items()) == dict(other.items())`` comparison, so override it with a
    # more efficient implementation.
    def __eq__(self, other):
        u"""*x.__eq__(other)　⟺　x == other*

        Equivalent to *dict(x.items()) == dict(other.items())*
        but more efficient.

        Note that :meth:`bidict's __eq__() <bidict.bidict.__eq__>` implementation
        is inherited by subclasses,
        in particular by the ordered bidict subclasses,
        so even with ordered bidicts,
        :ref:`== comparison is order-insensitive <eq-order-insensitive>`.

        *See also* :meth:`bidict.FrozenOrderedBidict.equals_order_sensitive`
        """
        if not isinstance(other, Mapping) or len(self) != len(other):
            return False
        selfget = self.get
        return all(selfget(k, _MISS) == v for (k, v) in iteritems(other))

    # The following methods are mutating and so are not public. But they are implemented in this
    # non-mutable base class (rather than the mutable `bidict` subclass) because they are used here
    # during initialization (starting with the `_update` method). (Why is this? Because `__init__`
    # and `update` share a lot of the same behavior (inserting the provided items while respecting
    # the active duplication policies), so it makes sense for them to share implementation too.)
    def _pop(self, key):
        val = self._fwdm.pop(key)
        del self._invm[val]
        return val

    def _put(self, key, val, on_dup):
        dedup_result = self._dedup_item(key, val, on_dup)
        if dedup_result is not _NOOP:
            self._write_item(key, val, dedup_result)

    def _dedup_item(self, key, val, on_dup):
        """
        Check *key* and *val* for any duplication in self.

        Handle any duplication as per the duplication policies given in *on_dup*.

        (key, val) already present is construed as a no-op, not a duplication.

        If duplication is found and the corresponding duplication policy is
        :attr:`~bidict.RAISE`, raise the appropriate error.

        If duplication is found and the corresponding duplication policy is
        :attr:`~bidict.IGNORE`, return *None*.

        If duplication is found and the corresponding duplication policy is
        :attr:`~bidict.OVERWRITE`,
        or if no duplication is found,
        return the _DedupResult *(isdupkey, isdupval, oldkey, oldval)*.
        """
        fwdm = self._fwdm
        invm = self._invm
        oldval = fwdm.get(key, _MISS)
        oldkey = invm.get(val, _MISS)
        isdupkey = oldval is not _MISS
        isdupval = oldkey is not _MISS
        dedup_result = _DedupResult(isdupkey, isdupval, oldkey, oldval)
        if isdupkey and isdupval:
            if self._isdupitem(key, val, dedup_result):
                # (key, val) duplicates an existing item -> no-op.
                return _NOOP
            # key and val each duplicate a different existing item.
            if on_dup.kv is RAISE:
                raise KeyAndValueDuplicationError(key, val)
            elif on_dup.kv is IGNORE:
                return _NOOP
            assert on_dup.kv is OVERWRITE, 'invalid on_dup_kv: %r' % on_dup.kv
            # Fall through to the return statement on the last line.
        elif isdupkey:
            if on_dup.key is RAISE:
                raise KeyDuplicationError(key)
            elif on_dup.key is IGNORE:
                return _NOOP
            assert on_dup.key is OVERWRITE, 'invalid on_dup.key: %r' % on_dup.key
            # Fall through to the return statement on the last line.
        elif isdupval:
            if on_dup.val is RAISE:
                raise ValueDuplicationError(val)
            elif on_dup.val is IGNORE:
                return _NOOP
            assert on_dup.val is OVERWRITE, 'invalid on_dup.val: %r' % on_dup.val
            # Fall through to the return statement on the last line.
        # else neither isdupkey nor isdupval.
        return dedup_result

    @staticmethod
    def _isdupitem(key, val, dedup_result):
        isdupkey, isdupval, oldkey, oldval = dedup_result
        isdupitem = oldkey == key
        assert isdupitem == (oldval == val), '%r %r %r' % (key, val, dedup_result)
        if isdupitem:
            assert isdupkey
            assert isdupval
        return isdupitem

    @classmethod
    def _get_on_dup(cls, on_dup=None):
        if on_dup is None:
            on_dup = _OnDup(cls.on_dup_key, cls.on_dup_val, cls.on_dup_kv)
        elif not isinstance(on_dup, _OnDup):
            on_dup = _OnDup(*on_dup)
        if on_dup.kv is None:
            on_dup = on_dup._replace(kv=on_dup.val)
        return on_dup

    def _write_item(self, key, val, dedup_result):
        isdupkey, isdupval, oldkey, oldval = dedup_result
        fwdm = self._fwdm
        invm = self._invm
        fwdm[key] = val
        invm[val] = key
        if isdupkey:
            del invm[oldval]
        if isdupval:
            del fwdm[oldkey]
        return _WriteResult(key, val, oldkey, oldval)

    def _update(self, init, on_dup, *args, **kw):
        if not args and not kw:
            return
        on_dup = self._get_on_dup(on_dup)
        empty = not self
        only_copy_from_bimap = empty and not kw and isinstance(args[0], BidirectionalMapping)
        if only_copy_from_bimap:  # no need to check for duplication
            write_item = self._write_item
            for (key, val) in iteritems(args[0]):
                write_item(key, val, _NODUP)
            return
        raise_on_dup = RAISE in on_dup
        rollback = raise_on_dup and not init
        if rollback:
            self._update_with_rollback(on_dup, *args, **kw)
            return
        _put = self._put
        for (key, val) in _iteritems_args_kw(*args, **kw):
            _put(key, val, on_dup)

    def _update_with_rollback(self, on_dup, *args, **kw):
        """Update, rolling back on failure."""
        writelog = []
        appendlog = writelog.append
        dedup_item = self._dedup_item
        write_item = self._write_item
        for (key, val) in _iteritems_args_kw(*args, **kw):
            try:
                dedup_result = dedup_item(key, val, on_dup)
            except DuplicationError:
                undo_write = self._undo_write
                for dedup_result, write_result in reversed(writelog):
                    undo_write(dedup_result, write_result)
                raise
            if dedup_result is not _NOOP:
                write_result = write_item(key, val, dedup_result)
                appendlog((dedup_result, write_result))

    def _undo_write(self, dedup_result, write_result):
        isdupkey, isdupval, _, _ = dedup_result
        key, val, oldkey, oldval = write_result
        if not isdupkey and not isdupval:
            self._pop(key)
            return
        fwdm = self._fwdm
        invm = self._invm
        if isdupkey:
            fwdm[key] = oldval
            invm[oldval] = key
            if not isdupval:
                del invm[val]
        if isdupval:
            invm[val] = oldkey
            fwdm[oldkey] = val
            if not isdupkey:
                del fwdm[key]

    def copy(self):
        """A shallow copy."""
        # Could just ``return self.__class__(self)`` here instead, but the below is faster. It uses
        # __new__ to create a copy instance while bypassing its __init__, which would result
        # in copying this bidict's items into the copy instance one at a time. Instead, make whole
        # copies of each of the backing mappings, and make them the backing mappings of the copy,
        # avoiding copying items one at a time.
        copy = self.__class__.__new__(self.__class__)
        copy._fwdm = self._fwdm.copy()  # pylint: disable=protected-access
        copy._invm = self._invm.copy()  # pylint: disable=protected-access
        copy._init_inv()  # pylint: disable=protected-access
        return copy

    def __copy__(self):
        """Used for the copy protocol.

        *See also* the :mod:`copy` module
        """
        return self.copy()

    def __len__(self):
        """The number of contained items."""
        return len(self._fwdm)

    def __iter__(self):  # lgtm [py/inheritance/incorrect-overridden-signature]
        """Iterator over the contained items."""
        # No default implementation for __iter__ inherited from Mapping ->
        # always delegate to _fwdm.
        return iter(self._fwdm)

    def __getitem__(self, key):
        u"""*x.__getitem__(key)　⟺　x[key]*"""
        return self._fwdm[key]

    def values(self):
        """A set-like object providing a view on the contained values.

        Note that because the values of a :class:`~bidict.BidirectionalMapping`
        are the keys of its inverse,
        this returns a :class:`~collections.abc.KeysView`
        rather than a :class:`~collections.abc.ValuesView`,
        which has the advantages of constant-time containment checks
        and supporting set operations.
        """
        return self.inverse.keys()

    if PY2:
        # For iterkeys and iteritems, inheriting from Mapping already provides
        # the best default implementations so no need to define here.

        def itervalues(self):
            """An iterator over the contained values."""
            return self.inverse.iterkeys()

        def viewkeys(self):  # noqa: D102; pylint: disable=missing-docstring
            return KeysView(self)

        def viewvalues(self):  # noqa: D102; pylint: disable=missing-docstring
            return self.inverse.viewkeys()

        viewvalues.__doc__ = values.__doc__
        values.__doc__ = 'A list of the contained values.'

        def viewitems(self):  # noqa: D102; pylint: disable=missing-docstring
            return ItemsView(self)

        # __ne__ added automatically in Python 3 when you implement __eq__, but not in Python 2.
        def __ne__(self, other):  # noqa: N802
            u"""*x.__ne__(other)　⟺　x != other*"""
            return not self == other  # Implement __ne__ in terms of __eq__.

_DedupResult = namedtuple('_DedupResult', 'isdupkey isdupval invbyval fwdbykey')
_WriteResult = namedtuple('_WriteResult', 'key val oldkey oldval')
_NODUP = _DedupResult(False, False, _MISS, _MISS)

#                             * Code review nav *
#==============================================================================
# ← Prev: _abc.py             Current: _base.py   Next: _delegating_mixins.py →
#==============================================================================