def _aggregate(self, scroll_field, scroll_size=SEARCH_LIMIT):
        """Perform an advanced query, and return *all* matching results.
        Will automatically perform multiple queries in order to retrieve all results.

        Note: All ``aggregate`` queries run in advanced mode.

        Arguments:
            scroll_field (str): The field on which to scroll. This should be a field
                    that counts/indexes the entries.
            scroll_size (int): Maximum number of records returned per query. Must be
                    between one and the ``SEARCH_LIMIT`` (inclusive).
                    **Default:** ``SEARCH_LIMIT``.

        Returns:
            list of dict: All matching entries.
        """
        # Make sure scroll_field is valid
        if not scroll_field:
            raise AttributeError("scroll_field is required.")

        # Make sure the query is set
        if not self.initialized:
            raise AttributeError('No query has been set.')

        # Warn the user if we are changing the setting of advanced
        if not self._SearchHelper__query["advanced"]:
            warnings.warn('This query will be run in advanced mode.', RuntimeWarning)
            self._SearchHelper__query["advanced"] = True

        # Inform the user if they set an invalid value for the query size
        if scroll_size <= 0:
            raise AttributeError('Scroll size must greater than zero')

        # Get the total number of records
        total = self.search(limit=0, info=True, reset_query=False)[1]["total_query_matches"]

        # If aggregate is unnecessary, use Search automatically instead
        if total <= SEARCH_LIMIT:
            return self.search(limit=SEARCH_LIMIT, reset_query=False)

        # Scroll until all results are found
        output = []

        scroll_pos = 0
        while len(output) < total:

            # Scroll until the width is small enough to get all records
            #   `scroll_id`s are unique to each dataset. If multiple datasets
            #   match a certain query, the total number of matching records
            #   may exceed the maximum that search will return - even if the
            #   scroll width is much smaller than that maximum
            scroll_width = scroll_size
            while True:
                query = "({q}) AND ({field}:>={start} AND {field}:<{end})".format(
                        q=self._SearchHelper__query["q"], field=scroll_field, start=scroll_pos,
                        end=scroll_pos+scroll_width)

                results, info = self.search(q=query, advanced=True, info=True)

                # Check to make sure that all the matching records were returned
                if info["total_query_matches"] <= len(results):
                    break

                # If not, reduce the scroll width
                # new_width is proportional with the proportion of results returned
                new_width = scroll_width * (len(results) // info["total_query_matches"])

                # scroll_width should never be 0, and should only be 1 in rare circumstances
                scroll_width = new_width if new_width > 1 else max(scroll_width//2, 1)

            # Append the results to the output
            output.extend(results)
            scroll_pos += scroll_width

        return output
def _aggregate ( self , scroll_field , scroll_size = SEARCH_LIMIT ) : if not scroll_field : raise AttributeError ( "scroll_field is required." ) if not self . initialized : raise AttributeError ( 'No query has been set.' ) if not self . _SearchHelper__query [ "advanced" ] : warnings . warn ( 'This query will be run in advanced mode.' , RuntimeWarning ) self . _SearchHelper__query [ "advanced" ] = True if scroll_size <= 0 : raise AttributeError ( 'Scroll size must greater than zero' ) total = self . search ( limit = 0 , info = True , reset_query = False ) [ 1 ] [ "total_query_matches" ] if total <= SEARCH_LIMIT : return self . search ( limit = SEARCH_LIMIT , reset_query = False ) output = [ ] scroll_pos = 0 while len ( output ) < total : scroll_width = scroll_size while True : query = "({q}) AND ({field}:>={start} AND {field}:<{end})" . format ( q = self . _SearchHelper__query [ "q" ] , field = scroll_field , start = scroll_pos , end = scroll_pos + scroll_width ) results , info = self . search ( q = query , advanced = True , info = True ) if info [ "total_query_matches" ] <= len ( results ) : break new_width = scroll_width * ( len ( results ) // info [ "total_query_matches" ] ) scroll_width = new_width if new_width > 1 else max ( scroll_width // 2 , 1 ) output . extend ( results ) scroll_pos += scroll_width return output
Perform an advanced query, and return *all* matching results.
        Will automatically perform multiple queries in order to retrieve all results.

        Note: All ``aggregate`` queries run in advanced mode.

        Arguments:
            scroll_field (str): The field on which to scroll. This should be a field
                    that counts/indexes the entries.
            scroll_size (int): Maximum number of records returned per query. Must be
                    between one and the ``SEARCH_LIMIT`` (inclusive).
                    **Default:** ``SEARCH_LIMIT``.

        Returns:
            list of dict: All matching entries.