class AttributedField(Expression, FieldOperators):
    __visit_name__ = 'attributed_field'

    def __init__(self, parent, attr_name, field):
        self._parent = parent
        self._attr_name = attr_name
        self._field = field

        self._dynamic_fields = OrderedAttributes()

        if self._field._type.doc_cls:
            doc_cls = self._field._type.doc_cls
            dynamic_defaults = {}
            for dyn_field_name, dyn_attr_field in \
                    doc_cls.dynamic_fields.items():
                dyn_field = dyn_attr_field._field.clone()
                dyn_field._name = self._make_field_name(dyn_field._name)
                self._dynamic_fields[dyn_field_name] = AttributedField(
                    self._parent, dyn_field_name, dyn_field
                )
                default = _attributed_field_factory(
                    AttributedField, self._parent, dyn_field,
                    self._make_field_name
                )
                dynamic_defaults[dyn_field_name] = default

            self._sub_fields = OrderedAttributes(defaults=dynamic_defaults)
            for attr_name, attr_field in doc_cls.user_fields.items():
                field = attr_field._field.clone()
                field._name = self._make_field_name(field._name)
                self._sub_fields[attr_name] = AttributedField(
                    self._parent, attr_name, field
                )
        elif self._field._fields:
            self._sub_fields = OrderedAttributes()
            for field_attr, field in self._field._fields.items():
                field = field.clone()
                field._name = self._make_field_name(field_attr)
                self._sub_fields[field_attr] = AttributedField(
                    self, field_attr, field
                )
        else:
            self._sub_fields = OrderedAttributes()

    def _make_field_name(self, name):
        return '{}.{}'.format(self._field._name, name)

    @property
    def fields(self):
        return self._sub_fields

    @property
    def dynamic_fields(self):
        return self._dynamic_fields

    def __getattr__(self, name):
        return getattr(self.fields, name)

    def wildcard(self, name):
        return DynamicAttributedField(
            self._parent, name, Field(self._make_field_name(name))
        )

    def get_parent(self):
        return self._parent

    def get_attr_name(self):
        return self._attr_name

    def get_field(self):
        return self._field

    def get_field_name(self):
        return self._field._name

    def get_type(self):
        return self._field.get_type()

    def __get__(self, obj, type=None):
        if obj is None:
            return self

        obj.__dict__[self._attr_name] = None
        return None

    def _collect_doc_classes(self):
        if isinstance(self._parent, AttributedField):
            return self._parent._collect_doc_classes()
        return {self._parent}

    def _to_python(self, value):
        return self._field._to_python(value)

    def _from_python(self, value):
        return self._field._from_python(value) PY2 = sys.version_info[0] == 2
def force_unicode(value):
def with_metaclass(meta, *bases):
        def __new__(cls, name, this_bases, d):
    class metaclass(meta): PY2 = sys.version_info[0] == 2
def force_unicode(value):
def with_metaclass(meta, *bases):
        def __new__(cls, name, this_bases, d):
    class metaclass(meta): PY2 = sys.version_info[0] == 2
def force_unicode(value):
def with_metaclass(meta, *bases):
        def __new__(cls, name, this_bases, d):
    class metaclass(meta): DOC_TYPE_JOIN_FIELD = '_doc_type_join' DOC_TYPE_FIELD = '_doc_type' DOC_TYPE_NAME_FIELD = '{}.name'.format(DOC_TYPE_FIELD) DOC_TYPE_PARENT_FIELD = '{}.parent'.format(DOC_TYPE_FIELD) class Document(with_metaclass(DocumentMeta)):
    __visit_name__ = 'document'

    _uid = MappingField(String)
    _id = MappingField(String)
    _type = MappingField(String)
    _source = MappingField(String)
    _all = MappingField(String)
    _analyzer = MappingField(String)
    _boost = MappingField(String)
    _parent = MappingField(String)
    _field_names = MappingField(String)
    _routing = MappingField(String)
    _index = MappingField(String)
    _size = MappingField(Integer)
    _timestamp = MappingField(Date)
    _ttl = MappingField(String)
    _version = MappingField(Integer)
    _score = MappingField(Float)

    __dynamic_fields__ = []

    __mapping_options__ = {}

    def __init__(self, _hit=None, _result=None, **kwargs):
        self.__hit_fields = None
        self.__highlight = None
        self.__matched_queries = None
        self.__explanation = None
        self._index = self._type = self._id = self._score = None
        if _hit:
            self._score = _hit.get('_score')
            source = _hit.get('_source')
            fields = _hit.get('fields')
            custom_doc_type = fields.get(DOC_TYPE_NAME_FIELD) \
                if fields else None

            for attr_field in self._mapping_fields:
                setattr(
                    self,
                    attr_field._attr_name,
                    _hit.get(attr_field._field._name)
                )

            if custom_doc_type:
                doc_type = custom_doc_type[0]
                _, _, self._id = _hit['_id'].rpartition(DOC_TYPE_ID_DELIMITER)
                self._type = doc_type

                custom_parent_id = fields.get(DOC_TYPE_PARENT_FIELD)
                if custom_parent_id:
                    parent_id = custom_parent_id[0]
                    _, _, self._parent = parent_id.rpartition(
                        DOC_TYPE_ID_DELIMITER
                    )

            if source:
                for hit_key, hit_value in source.items():
                    setattr(
                        self,
                        *self._process_source_key_value(hit_key, hit_value)
                    )

            if fields:
                # we cannot construct document from fields
                # in next example we cannot decide
                # which tag has name and which has not:
                # {"tags.id": [1, 2], "tags.name": ["Test"]}
                self.__hit_fields = self._process_fields(fields)

            if _hit.get('highlight'):
                self.__highlight = _hit['highlight']

            if _hit.get('matched_queries'):
                self.__matched_queries = _hit['matched_queries']

            if _hit.get('_explanation'):
                self.__explanation = _hit['_explanation']

        for fkey, fvalue in kwargs.items():
            setattr(self, fkey, fvalue)

        self.__result = _result

    def _process_source_key_value(self, key, value):
        if key in self._field_name_map:
            attr_field = self._field_name_map[key]
            return (
                attr_field._attr_name,
                attr_field.get_type().to_python(value)
            )
        return key, value

    def _process_fields(self, hit_fields):
        processed_fields = {}
        for field_name, field_values in hit_fields.items():
            field_path = field_name.split('.')
            doc_cls = self.__class__
            field_type = None
            for fname in field_path:
                attr_field = doc_cls._field_name_map.get(fname)
                if not attr_field:
                    break
                field_type = attr_field.get_field().get_type()
                doc_cls = field_type.doc_cls
            if field_type:
                processed_values = list(
                    map(field_type.to_python, field_values)
                )
            else:
                processed_values = field_values
            processed_fields[field_name] = processed_values
        return processed_fields

    @classmethod
    def get_doc_type(cls):
        return getattr(cls, '__doc_type__', None)

    @classmethod
    def has_parent_doc_cls(cls):
        return hasattr(cls, '__parent__')

    @classmethod
    def get_parent_doc_cls(cls):
        return getattr(cls, '__parent__', None)

    def to_meta(self, compiler):
        meta_compiler = compiler.compiled_bulk.compiled_meta
        return meta_compiler(self).body

    def to_source(self, compiler, validate=False):
        source_compiler = compiler.compiled_bulk.compiled_source
        return source_compiler(self, validate=validate).body

    def get_highlight(self):
        return self.__highlight or {}

    def get_matched_queries(self):
        return self.__matched_queries or []

    def get_fields(self):
        return self.__hit_fields or {}

    def get_hit_fields(self):
        return self.get_fields()

    def get_explanation(self):
        return self.__explanation or {}

    @classmethod
    def to_mapping(cls, compiler, ordered=False):
        return compiler.compiled_put_mapping(cls, ordered=ordered).body

    @cached_property
    def instance(self):
        if self.__result:
            self.__result._populate_instances(self.__class__)
            return self.__dict__['instance'] class DynamicDocument(with_metaclass(DynamicDocumentMeta, Document)):
    def _process_source_key_value(self, key, value):
        key, value = (
            super(DynamicDocument, self)._process_source_key_value(key, value)
        )
        if isinstance(value, dict):
            return key, DynamicDocument(**value)
        return key, value def get_doc_type_for_hit(hit):
    fields = hit.get('fields', {})
    custom_doc_type = fields.get(DOC_TYPE_NAME_FIELD)
    if custom_doc_type:
        return custom_doc_type[0]
    return hit['_type'] def mk_uid(doc_type, doc_id):
    return '{}{}{}'.format(doc_type, DOC_TYPE_ID_DELIMITER, doc_id) class Bool(QueryExpression):
    __query_name__ = 'bool'

    def __init__(
            self, must=None, filter=None, must_not=None, should=None,
            minimum_should_match=None, boost=None, disable_coord=None,
            **kwargs
    ):
        super(Bool, self).__init__(
            must=must, filter=filter, must_not=must_not, should=should,
            minimum_should_match=minimum_should_match,
            boost=boost, disable_coord=disable_coord, **kwargs)

    @classmethod
    def must(cls, *expressions):
        if len(expressions) == 1:
            return expressions[0]
        return cls(must=expressions)

    @classmethod
    def must_not(cls, *expressions):
        return cls(must_not=expressions)

    @classmethod
    def should(cls, *expressions):
        if len(expressions) == 1:
            return expressions[0]
        return cls(should=expressions) class Exists(QueryExpression):
    __query_name__ = 'exists'

    def __init__(self, field, **kwargs):
        super(Exists, self).__init__(field=field, **kwargs) class Filtered(QueryExpression):
    __query_name__ = 'filtered'

    def __init__(self, filter=None, query=None, strategy=None, **kwargs):
        super(Filtered, self).__init__(
            filter=filter, query=query, strategy=strategy, **kwargs) class FunctionScore(QueryExpression):
    __query_name__ = 'function_score'

    def __init__(
            self, query=None, filter=None, boost=None,
            script_score=None, boost_factor=None, random_score=None,
            field_value_factor=None, linear=None, exp=None, gauss=None,
            functions=None, max_boost=None, score_mode=None, boost_mode=None,
            **kwargs
    ):
        super(FunctionScore, self).__init__(
            query=query, filter=filter, boost=boost,
            script_score=script_score, boost_factor=boost_factor,
            random_score=random_score, field_value_factor=field_value_factor,
            linear=linear, exp=exp, gauss=gauss, functions=functions,
            max_boost=max_boost, score_mode=score_mode, boost_mode=boost_mode,
            **kwargs
        ) class HighlightedField(Expression):
    __visit_name__ = 'highlighted_field'

    def __init__(self, field, **kwargs):
        self.field = field
        self.params = Params(kwargs)

    def _collect_doc_classes(self):
        return set().union(
            collect_doc_classes(self.field),
            collect_doc_classes(self.params),
        ) class Ids(QueryExpression):
    __visit_name__ = 'ids'

    def __init__(self, values, type=None, **kwargs):
        self.values = values
        self.type = type
        super(Ids, self).__init__(**kwargs) class MatchPhrase(FieldQueryExpression):
    __query_name__ = 'match_phrase'

    def __init__(
            self, field, query, slop=None, analyzer=None, boost=None,
            **kwargs
    ):
        super(MatchPhrase, self).__init__(
            field, query,
            slop=slop, analyzer=analyzer, boost=boost,
            **kwargs
        ) class MatchPhrasePrefix(FieldQueryExpression):
    __query_name__ = 'match_phrase_prefix'

    def __init__(
            self, field, query, slop=None, analyzer=None, boost=None,
            max_expansions=None,
            **kwargs
    ):
        super(MatchPhrasePrefix, self).__init__(
            field, query,
            slop=slop, analyzer=analyzer, boost=boost,
            max_expansions=max_expansions,
            **kwargs
        ) class Params(Expression, Mapping):
    __visit_name__ = 'params'

    def __init__(self, *args, **kwargs):
        params = {}
        for d in args:
            params.update(d)
        params.update(kwargs)
        self._params = {}
        for k, v in clean_params(params).items():
            if k.endswith('_') and not k.startswith('_'):
                k = k.rstrip('_')
            self._params[k] = v

    def _collect_doc_classes(self):
        return collect_doc_classes(self._params)

    def __len__(self):
        return len(self._params)

    def __iter__(self):
        return iter(self._params)

    def __getitem__(self, key):
        return self._params[key]

    def __contains__(self, key):
        return key in self._params class Terms(FieldExpression):
    __visit_name__ = 'terms'

    def __init__(
            self, field, terms, minimum_should_match=None, boost=None,
            **kwargs
    ):
        super(Terms, self).__init__(
            field, minimum_should_match=minimum_should_match, boost=boost,
            **kwargs
        )
        self.terms = list(terms) class BulkResult(Result):
    def __init__(self, raw_result):
        super(BulkResult, self).__init__(raw_result)
        self.took = raw_result['took']
        self.errors = raw_result['errors']
        self.items = list(map(ActionResult, raw_result['items']))

    def __iter__(self):
        return iter(self.items) class CountResult(Result):
    def __init__(self, raw_result):
        super(CountResult, self).__init__(raw_result)
        self.count = raw_result['count'] class DeleteByQueryResult(Result):
    def __init__(self, raw_result):
        super(DeleteByQueryResult, self).__init__(raw_result)
        self.took = raw_result.get('took')
        self.timed_out = raw_result.get('timed_out')
        self.deleted = raw_result.get('deleted')
        self.batches = raw_result.get('batches')
        self.version_conflicts = raw_result.get('version_conflicts')
        self.noops = raw_result.get('noops')
        self.retries = self.Retries(raw_result.get('retries') or {})
        self.throttled_millis = raw_result.get('throttled_millis')
        self.requests_per_second = raw_result.get('requests_per_second')
        self.throttled_until_millis = raw_result.get('throttled_until_millis')
        self.total = raw_result.get('total')
        self.failures = raw_result.get('failures')

    class Retries(object):
        def __init__(self, raw_result):
            self.bulk = raw_result.get('bulk')
            self.search = raw_result.get('search') class DeleteResult(Result):
    def __init__(self, raw_result):
        super(DeleteResult, self).__init__(raw_result)
        self.found = raw_result.get('found')
        self.result = raw_result.get('result')
        self._index = raw_result['_index']
        self._type = raw_result['_type']
        self._id = raw_result['_id']
        self._version = raw_result['_version'] class ExistsResult(Result):
    def __init__(self, raw_result):
        super(ExistsResult, self).__init__(raw_result)
        self.exists = raw_result['exists'] class ExplainResult(Result):
    def __init__(self, raw_result, doc_cls_map=None, _store_hit=False):
        super(ExplainResult, self).__init__(raw_result)

        doc_cls_map = doc_cls_map or {}

        self.matched = raw_result['matched']
        self.explanation = raw_result['explanation']

        raw_hit = raw_result.get('get', {}).copy()
        self._id = raw_result['_id']
        self._type = raw_result['_type']
        self._index = raw_result['_index']
        self.hit = None
        if raw_hit:
            raw_hit['_id'] = raw_result['_id']
            raw_hit['_type'] = raw_result['_type']
            raw_hit['_index'] = raw_result['_index']
            doc_type = get_doc_type_for_hit(raw_hit)
            doc_cls = doc_cls_map.get(doc_type, DynamicDocument)
            hit = doc_cls(raw_hit)
            self._id = hit._id
            self._type = hit._type
            if _store_hit:
                self.hit = hit class PutMappingResult(Result):
    pass class SearchResult(Result):
    def __init__(
            self, raw_result, aggregations=None, doc_cls_map=None,
            instance_mapper=None,
    ):
        super(SearchResult, self).__init__(raw_result)

        self._query_aggs = aggregations or {}

        self._doc_cls_map = doc_cls_map or {}
        doc_classes = list(self._doc_cls_map.values())

        self._mapper_registry = {}
        if isinstance(instance_mapper, dict):
            self._instance_mappers = instance_mapper
        else:
            self._instance_mappers = {
                doc_cls: instance_mapper for doc_cls in doc_classes
            }

        self.error = raw_result.get('error')
        self.took = raw_result.get('took')
        self.timed_out = raw_result.get('timed_out')

        hits = raw_result.get('hits') or {}
        total = hits.get('total')
        if isinstance(total, dict):
            self.total = total['value']
        else:
            self.total = total
        self.max_score = hits.get('max_score')
        self.hits = []
        for hit in hits.get('hits', []):
            doc_type = get_doc_type_for_hit(hit)
            doc_cls = self._doc_cls_map.get(doc_type, DynamicDocument)
            self.hits.append(doc_cls(_hit=hit, _result=self))

        self.aggregations = {}
        for agg_name, agg_expr in self._query_aggs.items():
            raw_agg_data = raw_result.get('aggregations', {}).get(agg_name, {})
            agg_result = agg_expr.build_agg_result(
                raw_agg_data, self._doc_cls_map,
                mapper_registry=self._mapper_registry
            )
            self.aggregations[agg_name] = agg_result

        self.scroll_id = raw_result.get('_scroll_id')

    def __iter__(self):
        return iter(self.hits)

    def __len__(self):
        return len(self.hits)

    def get_aggregation(self, name):
        return self.aggregations.get(name)

    def _populate_instances(self, doc_cls):
        docs = [doc for doc in self.hits if isinstance(doc, doc_cls)]
        instances = self._instance_mappers.get(doc_cls)(
            [doc._id for doc in docs]
        )
        for doc in docs:
            doc.__dict__['instance'] = instances.get(doc._id) class BaseSearchQuery(with_metaclass(ABCMeta)):
    _q = None
    _source = None
    _fields = None
    _filters = ()
    _filters_meta = ()
    _post_filters = ()
    _post_filters_meta = ()
    _order_by = ()
    _aggregations = Params()
    _ext = Params()
    _function_scores = OrderedDict([
        (
            BOOST_FUNCTION_SCORE.name,
            _FunctionScore((), BOOST_FUNCTION_SCORE.settings)
        ),
        (
            GENERAL_FUNCTION_SCORE.name,
            _FunctionScore((), GENERAL_FUNCTION_SCORE.settings)
        ),
    ])
    _limit = None
    _offset = None
    _min_score = None
    _rescores = ()
    _suggest = Params()
    _highlight = Params()
    _script_fields = Params()

    _cluster = None
    _index = None
    _doc_cls = None
    _doc_type = None

    _search_params = Params()

    _instance_mapper = None
    _iter_instances = False

    _cached_result = None

    def __init__(
            self, q=None,
            cluster=None, index=None, doc_cls=None, doc_type=None,
            routing=None, preference=None, timeout=None, search_type=None,
            query_cache=None, terminate_after=None, scroll=None, stats=None,
            **kwargs
    ):
        if q is not None:
            self._q = q
        if cluster:
            self._cluster = cluster
        if index:
            self._index = index
        if doc_cls:
            self._doc_cls = doc_cls
        if doc_type:
            self._doc_type = doc_type

        search_params = Params(
            routing=routing,
            preference=preference,
            timeout=timeout,
            search_type=search_type,
            query_cache=query_cache,
            terminate_after=terminate_after,
            scroll=scroll,
            stats=stats,
            **kwargs
        )
        if search_params:
            self._search_params = search_params

    def clone(self):
        """Clones this query so you can modify both queries independently.
        """
        cls = self.__class__
        q = cls.__new__(cls)
        q.__dict__ = {
            k: v for k, v in self.__dict__.items()
            if not k.startswith('_cached_')
        }
        return q

    @_with_clone
    def source(self, *fields, **kwargs):
        """Controls which fields of the document's ``_source`` field
        to retrieve.

        .. _fields_arg:

        :param \\*fields: list of fields which should be returned by
           elasticsearch. Can be one of the following types:

           - field expression, for example: ``PostDocument.title``
           - ``str`` means field name or glob pattern. For example:
             ``"title"``, ``"user.*"``
           - ``False`` disables retrieving source
           - ``True`` enables retrieving all source document
           - ``None`` cancels source filtering applied before

        :param include: list of fields to include

        :param exclude: list of fields to exclude

        Example:

        .. testcode:: source

           search_query = SearchQuery().source(PostDocument.title, 'user.*')

        .. testcode:: source

           assert search_query.to_dict(Compiler_5_0) == {'_source': ['title', 'user.*']}

        See `source filtering <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html>`_
        for more information.
        """  # noqa:E501
        if len(fields) == 1 and fields[0] is None:
            if '_source' in self.__dict__:
                del self._source
        elif len(fields) == 1 and isinstance(fields[0], bool):
            self._source = Source(fields[0], **kwargs)
        else:
            self._source = Source(fields, **kwargs)

    @_with_clone
    def stored_fields(self, *fields):
        """Allows to load fields that marked as ``store: true``.

        Example:

        .. testcode:: stored_fields

           search_query = SearchQuery().stored_fields(PostDocument.rank)

        .. testcode:: stored_fields

           assert search_query.to_dict(Compiler_5_0) == {'stored_fields': ['rank']}

        See `stored fields <https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-store.html>`_ and
        `stored fields filtering <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-stored-fields.html>`_
        for more information.
        """  # noqa:E501
        if len(fields) == 1 and fields[0] is None:
            if '_fields' in self.__dict__:
                del self._fields
        elif len(fields) == 1 and isinstance(fields[0], bool):
            self._fields = fields[0]
        else:
            self._fields = fields

    def fields(self, *fields):
        return self.stored_fields(*fields)

    @_with_clone
    def script_fields(self, *args, **kwargs):
        """Allows to evaluate fields based on scripts.

        .. testcode:: script_fields

           from elasticmagic import Script

           search_query = SearchQuery().script_fields(
               rating=Script(
                   inline='doc[params.positive_opinions_field].value / '
                          'doc[params.total_opinions_field].value * 5',
                   params={
                       'total_opinions_field': PostDocument.total_opinions,
                       'positive_opinions_field': PostDocument.positive_opinions,
                   }
               )
           )

        .. testcode:: script_fields

           expected = {
               'script_fields': {
                   'rating': {
                       'script': {
                           'inline': 'doc[params.positive_opinions_field].value / '
                                     'doc[params.total_opinions_field].value * 5',
                           'params': {
                               'total_opinions_field': 'total_opinions',
                               'positive_opinions_field': 'positive_opinions'
                           }
                       }
                   }
               }
           }

           assert search_query.to_dict(Compiler_5_0) == {
               'script_fields': {
                   'rating': {
                       'script': {
                           'inline': 'doc[params.positive_opinions_field].value / '
                                     'doc[params.total_opinions_field].value * 5',
                           'params': {
                               'total_opinions_field': 'total_opinions',
                               'positive_opinions_field': 'positive_opinions'
                           }
                       }
                   }
               }
           }

        See `script fields <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-script-fields.html>`_
        """  # noqa:E501
        if len(args) == 1 and args[0] is None:
            if '_script_fields' in self.__dict__:
                del self._script_fields
        for key in kwargs:
            if isinstance(kwargs[key], Script):
                kwargs[key] = dict(script=kwargs[key])
        script_fields = dict()
        for key in self._script_fields:
            script_fields[key] = self._script_fields[key]
        script_fields.update(kwargs)
        self._script_fields = Params(script_fields)

    @_with_clone
    def query(self, q):
        """Replaces query clause. Elasticsearch's query clause will calculate
        ``_score`` for every matching document.

        :param q: query expression. Existing query can be cancelled by passing
           ``None``.

        .. testcode:: query

           search_query = SearchQuery().query(
               PostDocument.title.match('test', minimum_should_match='100%'))

        .. testcode:: query

           assert search_query.to_dict(Compiler_5_0) == {
               'query': {'match': {'title': {
                   'query': 'test',
                   'minimum_should_match': '100%'}}}}
        """
        if q is None:
            if '_q' in self.__dict__:
                del self._q
        else:
            self._q = q

    @_with_clone
    def filter(self, *filters, **kwargs):
        """Adds a filters into elasticsearch
        `filter context <https://www.elastic.co/guide/en/elasticsearch/reference/current/query-filter-context.html>`_.

        Returns new :class:`.SearchQuery` object with applied filters.

        .. testcode:: filter

           search_query = SearchQuery().filter(
               PostDocument.status == 'published',
               PostDocument.publish_date >= datetime.date(2015, 1, 1),
           )

        .. testcode:: filter

           assert search_query.to_dict(Compiler_5_0) == {
               'query': {'bool': {'filter': [
                   {'term': {'status': 'published'}},
                   {'range': {
                       'publish_date': {
                           'gte': datetime.date(2015, 1, 1)}}}]}}}

        Filter expression can be a python dictionary object:

        .. testcode:: filter

           search_query = SearchQuery().filter({'term': {'status': 'published'}})

        """  # noqa:E501
        meta = kwargs.pop('meta', None)
        self._filters = self._filters + filters
        self._filters_meta = self._filters_meta + (meta,) * len(filters)

    @_with_clone
    def post_filter(self, *filters, **kwargs):
        """Adds a filters into elasticsearch
        `post filter context <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-post-filter.html>`_.

        All parameters have the same meaning as for
        :meth:`.filter` method.
        """  # noqa:E501
        if len(filters) == 1 and filters[0] is None:
            if '_post_filters' in self.__dict__:
                del self._post_filters
        else:
            meta = kwargs.pop('meta', None)
            self._post_filters = self._post_filters + filters
            self._post_filters_meta = \
                self._post_filters_meta + (meta,) * len(filters)

    @_with_clone
    def order_by(self, *orders):
        """Apply sorting criterion to the search query.
        Corresponds elasticsearch's
        `sort <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html>`_.

        .. testcode:: order_by

           search_query = SearchQuery().order_by(
               PostDocument.publish_date.desc(),
               PostDocument._score,
           )
           assert search_query.to_dict(Compiler_5_0) == {
               'sort': [
                   {'publish_date': 'desc'},
                   '_score'
               ]
           }

        When called with single ``None`` argument clears any sorting criterion
        applied before:

        .. testcode:: order_by

           search_query = SearchQuery().order_by(None)
           assert search_query.to_dict(Compiler_5_0) == {}
        """  # noqa:E501
        if len(orders) == 1 and orders[0] is None:
            if '_order_by' in self.__dict__:
                del self._order_by
        else:
            self._order_by = self._order_by + orders

    @_with_clone
    def aggregations(self, *args, **kwargs):
        """Adds `aggregations <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html>`_
        to the search query.

        :param \\*aggs: dictionaries with aggregations. Can be ``None`` that
           cleans up previous aggregations.

        After executing the query you can get aggregation result by its name
        calling :meth:`SearchResult.get_aggregation` method.

        .. testcode:: aggs

           from elasticmagic import agg

           search_query = SearchQuery().aggregations({
               'stars': agg.Terms(PostDocument.stars, size=50, aggs={
                   'profit': agg.Sum(PostDocument.profit)})})

        .. testcode:: aggs

           assert search_query.to_dict(Compiler_5_0) == {
               'aggregations': {
                   'stars': {'terms': {'field': 'stars', 'size': 50},
                       'aggregations': {
                           'profit': {'sum': {'field': 'profit'}}}}}}
        """  # noqa:E501
        if len(args) == 1 and args[0] is None:
            if '_aggregations' in self.__dict__:
                del self._aggregations
        else:
            self._aggregations = merge_params(self._aggregations, args, kwargs)

    def aggs(self, *args, **kwargs):
        """A shortcut for the :meth:`.aggregations` method
        """
        return self.aggregations(*args, **kwargs)

    @_with_clone
    def ext(self, *args, **kwargs):
        if len(args) == 1 and args[0] is None:
            if '_ext' in self.__dict__:
                del self._ext
        else:
            self._ext = merge_params(self._ext, args, kwargs)

    @_with_clone
    def function_score_settings(self, *function_score_settings):
        """Adds or updates function score level.
        Levels will be inserted before existing.
        """
        function_scores = OrderedDict()
        for fs_settings in function_score_settings:
            if fs_settings.name in self._function_scores:
                fs = self._function_score[fs_settings.name]
                self._function_score[fs_settings.name] = _FunctionScore(
                    fs.functions, fs_settings.settings
                )
            else:
                function_scores[fs_settings.name] = _FunctionScore(
                    (), fs_settings.settings
                )
        function_scores.update(self._function_scores)
        self._function_scores = function_scores

    @_with_clone
    def function_score(self, *args, **kwargs):
        """Adds `function scores <https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html>`_
        to the search query.

        :param \\*functions: list of function scores.

        .. testcode:: function_score

           from elasticmagic import Weight, FieldValueFactor

           search_query = (
               SearchQuery(PostDocument.title.match('test'))
               .function_score(
                   Weight(2, filter=PostDocument.created_date == 'now/d'),
                   FieldValueFactor(
                       PostDocument.popularity, factor=1.2, modifier='sqrt'
                   )
               )
           )

        .. testcode:: function_score

           assert search_query.to_dict(Compiler_5_0) == {
               'query': {
                   'function_score': {
                       'query': {'match': {'title': 'test'}},
                       'functions': [
                           {'weight': 2,
                            'filter': {'term': {'created_date': 'now/d'}}},
                           {'field_value_factor': {
                                'field': 'popularity',
                                'factor': 1.2,
                                 'modifier': 'sqrt'}}]}}}
        """  # noqa:E501
        if len(args) >= 1 and isinstance(args[0], FunctionScoreSettings):
            self._function_score(args[0].name, args[1:], kwargs)
        else:
            self._function_score(GENERAL_FUNCTION_SCORE.name, args, kwargs)

    @_with_clone
    def boost_score(self, *args, **kwargs):
        """Adds one more level of the function_score query with default
        sum modes. It is especially useful for complex ordering scenarios.

        :param \\*functions: See :meth:`.function_score`

        .. testcode:: boost_score

           from elasticmagic import Factor, ScriptScore, Script

           search_query = (
               SearchQuery(PostDocument.title.match('test'))
               .function_score(
                   # Slightly boost hits on post popularity
                   Factor(PostDocument.popularity, modifier='sqrt'))
               .boost_score(
                   # Display advertized posts higher than any others
                   ScriptScore(
                       Script(inline='log10(10.0 + doc[cpc_field].value)',
                              params={'cpc_field': PostDocument.adv_cpc}),
                       weight=1000,
                       filter=PostDocument.adv_cpc > 0))
           )

        .. testcode:: boost_score

           assert search_query.to_dict(Compiler_5_0) == {
               'query': {
                   'function_score': {
                       'query': {
                           'function_score': {
                               'query': {'match': {'title': 'test'}},
                               'functions': [
                                   {'field_value_factor': {
                                       'field': 'popularity',
                                       'modifier': 'sqrt'}}]}},
                       'functions': [
                           {'script_score': {'script': {
                                'inline': 'log10(10.0 + doc[cpc_field].value)',
                                'params': {'cpc_field': 'adv_cpc'}}},
                            'filter': {'range': {'adv_cpc': {'gt': 0}}},
                            'weight': 1000}],
                       'boost_mode': 'sum',
                       'score_mode': 'sum'}}}
        """
        self._function_score(BOOST_FUNCTION_SCORE.name, args, kwargs)

    def _function_score(self, key, functions, kwargs):
        if key not in self._function_scores:
            raise ValueError(
                'Cannot find function score: [{}]'.format(key)
            )
        self._function_scores = OrderedDict(self._function_scores)
        if functions == (None,):
            fs = self._function_scores[key]
            self._function_scores[key] = _FunctionScore((), fs.params)
        else:
            self._function_scores[key] = _FunctionScore(
                self._function_scores[key].functions + functions,
                Params(dict(self._function_scores[key].params), **kwargs)
            )

    @_with_clone
    def limit(self, limit):
        """Sets size of the maximum amount of hits. Used for pagination.

        See `from / size <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-from-size.html>`_
        """  # noqa:E501
        self._limit = limit

    size = limit

    @_with_clone
    def offset(self, offset):
        """Sets the offset - the number of hits to skip. Used for pagination.

        See `from / size <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-from-size.html>`_
        """  # noqa:E501
        self._offset = offset

    from_ = offset

    @_with_clone
    def min_score(self, min_score):
        """Excludes hits with a ``_score`` less then ``min_score``. See
        `min score <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-min-score.html>`_
        """  # noqa:E501
        self._min_score = min_score

    @_with_clone
    def rescore(self, rescorer, window_size=None):
        """Adds a rescorer for the query. See
        `rescoring <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-rescore.html>`_
        """  # noqa:E501
        if rescorer is None:
            if '_rescores' in self.__dict__:
                del self._rescores
            return
        rescore = Rescore(rescorer, window_size=window_size)
        self._rescores = self._rescores + (rescore,)

    @_with_clone
    def suggest(self, *args, **kwargs):
        """Adds `suggesters <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-suggesters.html>`_
        to the query"""  # noqa:E501
        if args == (None,):
            if'_suggest' in self.__dict__:
                del self._suggest
        else:
            self._suggest = merge_params(self._suggest, args, kwargs)

    @_with_clone
    def highlight(
            self, fields=None, type=None, pre_tags=None, post_tags=None,
            fragment_size=None, number_of_fragments=None, order=None,
            encoder=None, require_field_match=None, boundary_max_scan=None,
            highlight_query=None, matched_fields=None, fragment_offset=None,
            no_match_size=None, phrase_limit=None,
            **kwargs
    ):
        """Highlights search results.

        .. testcode:: highlight

           from elasticmagic import MultiMatch

           search_query = (
               SearchQuery(
                   MultiMatch('The quick brown fox',
                              [PostDocument.title, PostDocument.content])
               )
               .highlight([PostDocument.title, PostDocument.content])
           )

        When processing search result you can get hit highlight by calling
        :meth:`.Document.get_highlight`.

        See `highlighting <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-highlighting.html>`_
        for details.
        """  # noqa:E501
        self._highlight = Highlight(
            fields=fields, type=type, pre_tags=pre_tags, post_tags=post_tags,
            fragment_size=fragment_size,
            number_of_fragments=number_of_fragments,
            order=order, encoder=encoder,
            require_field_match=require_field_match,
            boundary_max_scan=boundary_max_scan,
            highlight_query=highlight_query, matched_fields=matched_fields,
            fragment_offset=fragment_offset, no_match_size=no_match_size,
            phrase_limit=phrase_limit,
            **kwargs
        )

    @_with_clone
    def instances(self):
        self._iter_instances = True

    @_with_clone
    def with_cluster(self, cluster):
        self._cluster = cluster

    @_with_clone
    def with_index(self, index):
        self._index = index

    @_with_clone
    def with_document(self, doc_cls):
        self._doc_cls = doc_cls

    @_with_clone
    def with_doc_type(self, doc_type):
        self._doc_type = doc_type

    @_with_clone
    def with_instance_mapper(self, instance_mapper):
        self._instance_mapper = instance_mapper

    def with_routing(self, routing):
        return self.with_search_params(routing=routing)

    def with_preference(self, preference):
        return self.with_search_params(preference=preference)

    def with_timeout(self, timeout):
        return self.with_search_params(timeout=timeout)

    def with_search_type(self, search_type):
        return self.with_search_params(search_type=search_type)

    def with_query_cache(self, query_cache):
        return self.with_search_params(query_cache=query_cache)

    def with_terminate_after(self, terminate_after):
        return self.with_search_params(terminate_after=terminate_after)

    def with_scroll(self, scroll):
        return self.with_search_params(scroll=scroll)

    def with_stats(self, stats):
        return self.with_search_params(stats=stats)

    @_with_clone
    def with_search_params(self, *args, **kwargs):
        if len(args) == 1 and args[0] is None:
            if '_search_params' in self.__dict__:
                del self._search_params
        elif args or kwargs:
            search_params = Params(self._search_params, *args, **kwargs)
            if not search_params and '_search_params' in self.__dict__:
                del self._search_params
            else:
                self._search_params = search_params

    def _collect_doc_classes(self):
        return set().union(
            *map(
                collect_doc_classes,
                [
                    self._q,
                    self._source,
                    self._fields,
                    self._filters,
                    self._post_filters,
                    tuple(
                        fs.functions for fs in self._function_scores.values()
                    ),
                    tuple(self._aggregations.values()),
                    self._order_by,
                    self._rescores,
                    self._highlight,
                    self._ext,
                ]
            )
        )

    @property
    def _index_or_cluster(self):
        if not self._index and not self._cluster:
            raise ValueError('Search query is not bound to index or cluster')
        return self._index or self._cluster

    def get_compiler_context(self):
        return SearchQueryContext(self)

    get_context = get_compiler_context

    def slice(self, offset, limit):
        """Applies offset and limit to the query."""
        sliced_query, _ = self._prepare_slice(slice(offset, limit))
        return sliced_query

    def _prepare_slice(self, k):
        if not isinstance(k, (slice, int)):
            raise TypeError('Index must be of type int or slice')

        if isinstance(k, slice):
            start, stop, step = k.start, k.stop, k.step or 1
            if (
                    start is not None and start < 0 or
                    stop is not None and stop < 0
            ):
                raise ValueError(
                    'Negative slices are not supported'
                )
            if step is not None and step != 1:
                raise ValueError(
                    'Slices with step different from 1 are not supported'
                )

            clone = self.clone()
            if start is not None:
                clone._offset = start
            if stop is not None:
                if start is None:
                    clone._limit = stop
                else:
                    clone._limit = stop - start
            return clone, True
        else:
            if k < 0:
                raise ValueError('Negative indexes are not supported')
            clone = self.clone()
            clone._offset = k
            clone._limit = 1
            return clone, False

    def _iter_result(self, res):
        if self._iter_instances:
            return iter(
                doc.instance for doc in res.hits
                if doc.instance
            )
        return iter(res) class SearchQueryContext(object):
    __visit_name__ = 'search_query_context'

    def __init__(self, search_query):
        self.q = search_query._q
        self.source = search_query._source
        self.fields = search_query._fields
        self.filters = search_query._filters
        self.filters_meta = search_query._filters_meta
        self.post_filters = search_query._post_filters
        self.post_filters_meta = search_query._post_filters_meta
        self.order_by = search_query._order_by
        self.aggregations = search_query._aggregations
        self.ext = search_query._ext
        self.function_scores = search_query._function_scores
        self.limit = search_query._limit
        self.offset = search_query._offset
        self.min_score = search_query._min_score
        self.rescores = search_query._rescores
        self.suggest = search_query._suggest
        self.highlight = search_query._highlight

        self.cluster = search_query._cluster
        self.index = search_query._index
        doc_cls = search_query._doc_cls
        if not doc_cls:
            doc_classes = search_query._collect_doc_classes()
        elif not isinstance(doc_cls, Iterable):
            doc_classes = [doc_cls]
        else:
            doc_classes = doc_cls
        self.doc_classes = tuple(doc_classes)

        if not search_query._doc_type:
            doc_types = []
        elif isinstance(search_query._doc_type, string_types):
            doc_types = [
                t.strip() for t in search_query._doc_type.split(',')
            ]
        else:
            doc_types = list(search_query._doc_type)
        self.doc_types = self._get_unique_doc_types(
            doc_types, self.doc_classes
        )

        self.script_fields = search_query._script_fields

        self.search_params = search_query._search_params

        self.instance_mapper = search_query._instance_mapper
        self.iter_instances = search_query._iter_instances

    @staticmethod
    def _get_unique_doc_types(doc_types=None, doc_classes=None):
        doc_types = list(doc_types) if doc_types else []
        unique_doc_types = set(doc_types)
        if isinstance(doc_classes, (list, tuple, set)):
            for doc_cls in doc_classes:
                doc_type = doc_cls.get_doc_type()
                if doc_type and doc_type not in unique_doc_types:
                    doc_types.append(doc_type)
        return tuple(doc_types)

    def iter_filters_with_meta(self):
        return zip(self.filters, self.filters_meta)

    def iter_filters(self):
        return iter(self.filters)

    def iter_post_filters_with_meta(self):
        return zip(self.post_filters, self.post_filters_meta)

    def iter_post_filters(self):
        return iter(self.post_filters) class ValidationError(ValueError):
    pass def collect_doc_classes(expr):
    if hasattr(expr, '_collect_doc_classes'):
        return expr._collect_doc_classes()

    if isinstance(expr, dict):
        return set().union(
            *[collect_doc_classes(e)
              for e in chain(expr.keys(), expr.values())]
        )

    if isinstance(expr, (list, tuple)):
        return set().union(*[collect_doc_classes(e) for e in expr])

    return set()
import operator
from collections import OrderedDict
from collections import namedtuple
from functools import partial
from elasticsearch import ElasticsearchException
from elasticmagic.attribute import AttributedField
from .compat import Iterable
from .compat import Mapping
from .compat import string_types
from .document import DOC_TYPE_JOIN_FIELD
from .document import DOC_TYPE_FIELD
from .document import DOC_TYPE_NAME_FIELD
from .document import DOC_TYPE_PARENT_FIELD
from .document import Document
from .document import DynamicDocument
from .document import get_doc_type_for_hit
from .document import mk_uid
from .expression import Bool
from .expression import Exists
from .expression import Filtered
from .expression import FunctionScore
from .expression import HighlightedField
from .expression import Ids
from .expression import MatchPhrase
from .expression import MatchPhrasePrefix
from .expression import Params
from .expression import Terms
from .result import BulkResult
from .result import CountResult
from .result import DeleteByQueryResult
from .result import DeleteResult
from .result import ExistsResult
from .result import ExplainResult
from .result import PutMappingResult
from .result import SearchResult
from .search import BaseSearchQuery
from .search import SearchQueryContext
from .types import ValidationError
from .util import collect_doc_classes
        doc_cls.__doc_type__: doc_cls for doc_cls in doc_classes
    }
    if not supports_doc_type and len(doc_classes) == 1:
        doc_cls_map['_doc'] = doc_classes[0]

    return doc_cls_map


class Compiled(object):
    compiler = None
    features = None

    def __init__(self, expression, params=None):
        self.expression = expression
        self.body = self.visit(expression)
        self.params = self.prepare_params(params or {})

    def prepare_params(self, params):
        return params

    def visit(self, expr, **kwargs):
        visit_name = None
        if hasattr(expr, '__visit_name__'):
            visit_name = expr.__visit_name__

        if visit_name:
            visit_func = getattr(self, 'visit_{}'.format(visit_name))
            return visit_func(expr, **kwargs)

        if isinstance(expr, dict):
            return self.visit_dict(expr)

        if isinstance(expr, (list, tuple)):
            return self.visit_list(expr)

        return expr

    def visit_params(self, params):
        res = {}
        for k, v in params.items():
            res[self.visit(k)] = self.visit(v)
        return res

    def visit_dict(self, dct):
        return {self.visit(k): self.visit(v) for k, v in dct.items()}

    def visit_list(self, lst):
        return [self.visit(v) for v in lst]

    def visit_script(self, script):
        if not self.features.supports_nested_script:
            raise CompilationError(
                'Elasticsearch v0.x and v1.x does not support Script')
        res = dict()
        if script.lang:
            res['lang'] = script.lang
        if script.script_params:
            res['params'] = script.script_params
        if script.inline:
            res[self.features.script_source_field_name] = script.inline
        elif script.id:
            res[self.features.script_id_field_name] = script.id
        elif self.features.supports_script_file and script.file:
            res['file'] = script.file
        else:
            raise CompilationError('Invalid arguments for Script')
        return self.visit_dict(res)


class CompiledEndpoint(Compiled):
    def process_result(self, raw_result):
        raise NotImplementedError


class CompiledExpression(Compiled):
    def __init__(self, expr, params=None, doc_classes=None):
        self.doc_classes = doc_classes
        super(CompiledExpression, self).__init__(expr, params)

    def visit_literal(self, expr):
        return expr.obj

    def visit_field(self, field):
        return field._name

    def visit_mapping_field(self, field):
        return field._name

    def visit_attributed_field(self, field):
        return field._field._name

    def visit_boost_expression(self, expr):
        return '{}^{}'.format(self.visit(expr.expr), self.visit(expr.weight))

    def visit_query_expression(self, expr):
        return {
            expr.__query_name__: self.visit(expr.params)
        }

    def visit_field_query(self, expr, **kwargs):
        expr_params = Params(expr.params, **kwargs)
        if expr_params:
            params = {expr.__query_key__: self.visit(expr.query)}
            params.update(expr_params)
            return {
                expr.__query_name__: {
                    self.visit(expr.field): params
                }
            }
        else:
            return {
                expr.__query_name__: {
                    self.visit(expr.field): self.visit(expr.query)
                }
            }

    def visit_match(self, expr):
        if not self.features.supports_match_type and expr.type:
            if expr.type == 'phrase':
                return self.visit(

                    MatchPhrase(expr.field, expr.query, **expr.params)