def tanimoto_coeff(self, src, tar, qval=2):
        """Return the Tanimoto distance between two strings.

        Tanimoto distance :cite:`Tanimoto:1958` is
        :math:`-log_{2} sim_{Tanimoto}(X, Y)`.

        Parameters
        ----------
        src : str
            Source string (or QGrams/Counter objects) for comparison
        tar : str
            Target string (or QGrams/Counter objects) for comparison
        qval : int
            The length of each q-gram; 0 for non-q-gram version

        Returns
        -------
        float
            Tanimoto distance

        Examples
        --------
        >>> cmp = Jaccard()
        >>> cmp.tanimoto_coeff('cat', 'hat')
        -1.5849625007211563
        >>> cmp.tanimoto_coeff('Niall', 'Neil')
        -2.1699250014423126
        >>> cmp.tanimoto_coeff('aluminum', 'Catalan')
        -4.0
        >>> cmp.tanimoto_coeff('ATCG', 'TAGC')
        -inf

        """
        coeff = self.sim(src, tar, qval)
        if coeff != 0:
            return log(coeff, 2)

        return float('-inf')

def tanimoto_coeff ( self , src , tar , qval = 2 ) : coeff = self . sim ( src , tar , qval ) if coeff != 0 : return log ( coeff , 2 ) return float ( '-inf' )

Return the Tanimoto distance between two strings .


def sim_sift4(src, tar, max_offset=5, max_distance=0):
    """Return the normalized "common" Sift4 similarity of two terms.

    This is a wrapper for :py:meth:`Sift4.sim`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    max_offset : int
        The number of characters to search for matching letters
    max_distance : int
        The distance at which to stop and exit

    Returns
    -------
    float
        The normalized Sift4 similarity

    Examples
    --------
    >>> round(sim_sift4('cat', 'hat'), 12)
    0.666666666667
    >>> sim_sift4('Niall', 'Neil')
    0.6
    >>> sim_sift4('Colin', 'Cuilen')
    0.5
    >>> sim_sift4('ATCG', 'TAGC')
    0.5

    """
    return Sift4().sim(src, tar, max_offset, max_distance)

def sim_sift4 ( src , tar , max_offset = 5 , max_distance = 0 ) : return Sift4 ( ) . sim ( src , tar , max_offset , max_distance )

Return the normalized common Sift4 similarity of two terms .


def dist(self, src, tar):
        """Return the NCD between two strings using bzip2 compression.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        float
            Compression distance

        Examples
        --------
        >>> cmp = NCDbz2()
        >>> cmp.dist('cat', 'hat')
        0.06666666666666667
        >>> cmp.dist('Niall', 'Neil')
        0.03125
        >>> cmp.dist('aluminum', 'Catalan')
        0.17647058823529413
        >>> cmp.dist('ATCG', 'TAGC')
        0.03125

        """
        if src == tar:
            return 0.0

        src = src.encode('utf-8')
        tar = tar.encode('utf-8')

        src_comp = bz2.compress(src, self._level)[10:]
        tar_comp = bz2.compress(tar, self._level)[10:]
        concat_comp = bz2.compress(src + tar, self._level)[10:]
        concat_comp2 = bz2.compress(tar + src, self._level)[10:]

        return (
            min(len(concat_comp), len(concat_comp2))
            - min(len(src_comp), len(tar_comp))
        ) / max(len(src_comp), len(tar_comp))

def dist ( self , src , tar ) : if src == tar : return 0.0 src = src . encode ( 'utf-8' ) tar = tar . encode ( 'utf-8' ) src_comp = bz2 . compress ( src , self . _level ) [ 10 : ] tar_comp = bz2 . compress ( tar , self . _level ) [ 10 : ] concat_comp = bz2 . compress ( src + tar , self . _level ) [ 10 : ] concat_comp2 = bz2 . compress ( tar + src , self . _level ) [ 10 : ] return ( min ( len ( concat_comp ) , len ( concat_comp2 ) ) - min ( len ( src_comp ) , len ( tar_comp ) ) ) / max ( len ( src_comp ) , len ( tar_comp ) )

Return the NCD between two strings using bzip2 compression .


def encode(self, word, lang='en'):
        """Return the MetaSoundex code for a word.

        Parameters
        ----------
        word : str
            The word to transform
        lang : str
            Either ``en`` for English or ``es`` for Spanish

        Returns
        -------
        str
            The MetaSoundex code

        Examples
        --------
        >>> pe = MetaSoundex()
        >>> pe.encode('Smith')
        '4500'
        >>> pe.encode('Waters')
        '7362'
        >>> pe.encode('James')
        '1520'
        >>> pe.encode('Schmidt')
        '4530'
        >>> pe.encode('Ashcroft')
        '0261'
        >>> pe.encode('Perez', lang='es')
        '094'
        >>> pe.encode('Martinez', lang='es')
        '69364'
        >>> pe.encode('Gutierrez', lang='es')
        '83994'
        >>> pe.encode('Santiago', lang='es')
        '4638'
        >>> pe.encode('Nicolás', lang='es')
        '6754'

        """
        if lang == 'es':
            return self._phonetic_spanish.encode(
                self._spanish_metaphone.encode(word)
            )

        word = self._soundex.encode(self._metaphone.encode(word))
        word = word[0].translate(self._trans) + word[1:]
        return word

def encode ( self , word , lang = 'en' ) : if lang == 'es' : return self . _phonetic_spanish . encode ( self . _spanish_metaphone . encode ( word ) ) word = self . _soundex . encode ( self . _metaphone . encode ( word ) ) word = word [ 0 ] . translate ( self . _trans ) + word [ 1 : ] return word

Return the MetaSoundex code for a word .


def stem(self, word):
        """Return the S-stemmed form of a word.

        Parameters
        ----------
        word : str
            The word to stem

        Returns
        -------
        str
            Word stem

        Examples
        --------
        >>> stmr = SStemmer()
        >>> stmr.stem('summaries')
        'summary'
        >>> stmr.stem('summary')
        'summary'
        >>> stmr.stem('towers')
        'tower'
        >>> stmr.stem('reading')
        'reading'
        >>> stmr.stem('census')
        'census'

        """
        lowered = word.lower()
        if lowered[-3:] == 'ies' and lowered[-4:-3] not in {'e', 'a'}:
            return word[:-3] + ('Y' if word[-1:].isupper() else 'y')
        if lowered[-2:] == 'es' and lowered[-3:-2] not in {'a', 'e', 'o'}:
            return word[:-1]
        if lowered[-1:] == 's' and lowered[-2:-1] not in {'u', 's'}:
            return word[:-1]
        return word

def stem ( self , word ) : lowered = word . lower ( ) if lowered [ - 3 : ] == 'ies' and lowered [ - 4 : - 3 ] not in { 'e' , 'a' } : return word [ : - 3 ] + ( 'Y' if word [ - 1 : ] . isupper ( ) else 'y' ) if lowered [ - 2 : ] == 'es' and lowered [ - 3 : - 2 ] not in { 'a' , 'e' , 'o' } : return word [ : - 1 ] if lowered [ - 1 : ] == 's' and lowered [ - 2 : - 1 ] not in { 'u' , 's' } : return word [ : - 1 ] return word


def sim(self, src, tar):
        """Return the Ratcliff-Obershelp similarity of two strings.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        float
            Ratcliff-Obershelp similarity

        Examples
        --------
        >>> cmp = RatcliffObershelp()
        >>> round(cmp.sim('cat', 'hat'), 12)
        0.666666666667
        >>> round(cmp.sim('Niall', 'Neil'), 12)
        0.666666666667
        >>> round(cmp.sim('aluminum', 'Catalan'), 12)
        0.4
        >>> cmp.sim('ATCG', 'TAGC')
        0.5

        """

        def _lcsstr_stl(src, tar):
            """Return start positions & length for Ratcliff-Obershelp.

            Parameters
            ----------
            src : str
                Source string for comparison
            tar : str
            Target string for comparison

            Returns
            -------
            tuple
                The start position in the source string, start position in the
                target string, and length of the longest common substring of
                strings src and tar.

            """
            lengths = np_zeros((len(src) + 1, len(tar) + 1), dtype=np_int)
            longest, src_longest, tar_longest = 0, 0, 0
            for i in range(1, len(src) + 1):
                for j in range(1, len(tar) + 1):
                    if src[i - 1] == tar[j - 1]:
                        lengths[i, j] = lengths[i - 1, j - 1] + 1
                        if lengths[i, j] > longest:
                            longest = lengths[i, j]
                            src_longest = i
                            tar_longest = j
                    else:
                        lengths[i, j] = 0
            return src_longest - longest, tar_longest - longest, longest

        def _sstr_matches(src, tar):
            """Return the sum of substring match lengths.

            This follows the Ratcliff-Obershelp algorithm
            :cite:`Ratcliff:1988`:
                 1. Find the length of the longest common substring in src &
                     tar.
                 2. Recurse on the strings to the left & right of each this
                     substring in src & tar.
                 3. Base case is a 0 length common substring, in which case,
                     return 0.
                 4. Return the sum.

            Parameters
            ----------
            src : str
                Source string for comparison
            tar : str
                Target string for comparison

            Returns
            -------
            int
                Sum of substring match lengths

            """
            src_start, tar_start, length = _lcsstr_stl(src, tar)
            if length == 0:
                return 0
            return (
                _sstr_matches(src[:src_start], tar[:tar_start])
                + length
                + _sstr_matches(
                    src[src_start + length :], tar[tar_start + length :]
                )
            )

        if src == tar:
            return 1.0
        elif not src or not tar:
            return 0.0
        return 2 * _sstr_matches(src, tar) / (len(src) + len(tar))

Return the Ratcliff - Obershelp similarity of two strings .


def dist_abs(self, src, tar):
        """Return the MRA comparison rating of two strings.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        int
            MRA comparison rating

        Examples
        --------
        >>> cmp = MRA()
        >>> cmp.dist_abs('cat', 'hat')
        5
        >>> cmp.dist_abs('Niall', 'Neil')
        6
        >>> cmp.dist_abs('aluminum', 'Catalan')
        0
        >>> cmp.dist_abs('ATCG', 'TAGC')
        5

        """
        if src == tar:
            return 6
        if src == '' or tar == '':
            return 0
        src = list(mra(src))
        tar = list(mra(tar))

        if abs(len(src) - len(tar)) > 2:
            return 0

        length_sum = len(src) + len(tar)
        if length_sum < 5:
            min_rating = 5
        elif length_sum < 8:
            min_rating = 4
        elif length_sum < 12:
            min_rating = 3
        else:
            min_rating = 2

        for _ in range(2):
            new_src = []
            new_tar = []
            minlen = min(len(src), len(tar))
            for i in range(minlen):
                if src[i] != tar[i]:
                    new_src.append(src[i])
                    new_tar.append(tar[i])
            src = new_src + src[minlen:]
            tar = new_tar + tar[minlen:]
            src.reverse()
            tar.reverse()

        similarity = 6 - max(len(src), len(tar))

        if similarity >= min_rating:
            return similarity
        return 0

def dist_abs ( self , src , tar ) : if src == tar : return 6 if src == '' or tar == '' : return 0 src = list ( mra ( src ) ) tar = list ( mra ( tar ) ) if abs ( len ( src ) - len ( tar ) ) > 2 : return 0 length_sum = len ( src ) + len ( tar ) if length_sum < 5 : min_rating = 5 elif length_sum < 8 : min_rating = 4 elif length_sum < 12 : min_rating = 3 else : min_rating = 2 for _ in range ( 2 ) : new_src = [ ] new_tar = [ ] minlen = min ( len ( src ) , len ( tar ) ) for i in range ( minlen ) : if src [ i ] != tar [ i ] : new_src . append ( src [ i ] ) new_tar . append ( tar [ i ] ) src = new_src + src [ minlen : ] tar = new_tar + tar [ minlen : ] src . reverse ( ) tar . reverse ( ) similarity = 6 - max ( len ( src ) , len ( tar ) ) if similarity >= min_rating : return similarity return 0

Return the MRA comparison rating of two strings .


def encode(self, word):
        """Return the Parmar-Kumbharana encoding of a word.

        Parameters
        ----------
        word : str
            The word to transform

        Returns
        -------
        str
            The Parmar-Kumbharana encoding

        Examples
        --------
        >>> pe = ParmarKumbharana()
        >>> pe.encode('Gough')
        'GF'
        >>> pe.encode('pneuma')
        'NM'
        >>> pe.encode('knight')
        'NT'
        >>> pe.encode('trice')
        'TRS'
        >>> pe.encode('judge')
        'JJ'

        """
        word = word.upper()  # Rule 3
        word = self._delete_consecutive_repeats(word)  # Rule 4

        # Rule 5
        i = 0
        while i < len(word):
            for match_len in range(4, 1, -1):
                if word[i : i + match_len] in self._rules[match_len]:
                    repl = self._rules[match_len][word[i : i + match_len]]
                    word = word[:i] + repl + word[i + match_len :]
                    i += len(repl)
                    break
            else:
                i += 1

        word = word[:1] + word[1:].translate(self._del_trans)  # Rule 6
        return word

def
encode
(
self
,
word
)
:
word
=
word
.
upper
(
)
# Rule 3
word
=
self
.
_delete_consecutive_repeats
(
word
)
# Rule 4
# Rule 5
i
=
0
while
i
<
len
(
word
)
:
for
match_len
in
range
(
4
,
1
,
1
)
:
if
word
[
i
:
i
+
match_len
]
in
self
.
_rules
[
match_len
]
:
repl
=
self
.
_rules
[
match_len
]
[
word
[
i
:
i
+
match_len
]
]
word
=
word
[
:
i
]
+
repl
+
word
[
i
+
match_len
:
]
i
+=
len
(
repl
)
break
else
:
i
+=
1
word
=
word
[
:
1
]
+
word
[
1
:
]
.
translate
(
self
.
_del_trans
)
# Rule 6
return
word

Return the Parmar - Kumbharana encoding of a word .


def eudex_hamming(
    src, tar, weights='exponential', max_length=8, normalized=False
):
    """Calculate the Hamming distance between the Eudex hashes of two terms.

    This is a wrapper for :py:meth:`Eudex.eudex_hamming`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    weights : str, iterable, or generator function
        The weights or weights generator function
    max_length : int
        The number of characters to encode as a eudex hash
    normalized : bool
        Normalizes to [0, 1] if True

    Returns
    -------
    int
        The Eudex Hamming distance

    Examples
    --------
    >>> eudex_hamming('cat', 'hat')
    128
    >>> eudex_hamming('Niall', 'Neil')
    2
    >>> eudex_hamming('Colin', 'Cuilen')
    10
    >>> eudex_hamming('ATCG', 'TAGC')
    403

    >>> eudex_hamming('cat', 'hat', weights='fibonacci')
    34
    >>> eudex_hamming('Niall', 'Neil', weights='fibonacci')
    2
    >>> eudex_hamming('Colin', 'Cuilen', weights='fibonacci')
    7
    >>> eudex_hamming('ATCG', 'TAGC', weights='fibonacci')
    117

    >>> eudex_hamming('cat', 'hat', weights=None)
    1
    >>> eudex_hamming('Niall', 'Neil', weights=None)
    1
    >>> eudex_hamming('Colin', 'Cuilen', weights=None)
    2
    >>> eudex_hamming('ATCG', 'TAGC', weights=None)
    9

    >>> # Using the OEIS A000142:
    >>> eudex_hamming('cat', 'hat', [1, 1, 2, 6, 24, 120, 720, 5040])
    1
    >>> eudex_hamming('Niall', 'Neil', [1, 1, 2, 6, 24, 120, 720, 5040])
    720
    >>> eudex_hamming('Colin', 'Cuilen', [1, 1, 2, 6, 24, 120, 720, 5040])
    744
    >>> eudex_hamming('ATCG', 'TAGC', [1, 1, 2, 6, 24, 120, 720, 5040])
    6243

    """
    return Eudex().dist_abs(src, tar, weights, max_length, normalized)

def eudex_hamming ( src , tar , weights = 'exponential' , max_length = 8 , normalized = False ) : return Eudex ( ) . dist_abs ( src , tar , weights , max_length , normalized )

Calculate the Hamming distance between the Eudex hashes of two terms .


def dist_eudex(src, tar, weights='exponential', max_length=8):
    """Return normalized Hamming distance between Eudex hashes of two terms.

    This is a wrapper for :py:meth:`Eudex.dist`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    weights : str, iterable, or generator function
        The weights or weights generator function
    max_length : int
        The number of characters to encode as a eudex hash

    Returns
    -------
    int
        The normalized Eudex Hamming distance

    Examples
    --------
    >>> round(dist_eudex('cat', 'hat'), 12)
    0.062745098039
    >>> round(dist_eudex('Niall', 'Neil'), 12)
    0.000980392157
    >>> round(dist_eudex('Colin', 'Cuilen'), 12)
    0.004901960784
    >>> round(dist_eudex('ATCG', 'TAGC'), 12)
    0.197549019608

    """
    return Eudex().dist(src, tar, weights, max_length)

def dist_eudex ( src , tar , weights = 'exponential' , max_length = 8 ) : return Eudex ( ) . dist ( src , tar , weights , max_length )

Return normalized Hamming distance between Eudex hashes of two terms .


def sim_eudex(src, tar, weights='exponential', max_length=8):
    """Return normalized Hamming similarity between Eudex hashes of two terms.

    This is a wrapper for :py:meth:`Eudex.sim`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    weights : str, iterable, or generator function
        The weights or weights generator function
    max_length : int
        The number of characters to encode as a eudex hash

    Returns
    -------
    int
        The normalized Eudex Hamming similarity

    Examples
    --------
    >>> round(sim_eudex('cat', 'hat'), 12)
    0.937254901961
    >>> round(sim_eudex('Niall', 'Neil'), 12)
    0.999019607843
    >>> round(sim_eudex('Colin', 'Cuilen'), 12)
    0.995098039216
    >>> round(sim_eudex('ATCG', 'TAGC'), 12)
    0.802450980392

    """
    return Eudex().sim(src, tar, weights, max_length)

def sim_eudex ( src , tar , weights = 'exponential' , max_length = 8 ) : return Eudex ( ) . sim ( src , tar , weights , max_length )

Return normalized Hamming similarity between Eudex hashes of two terms .


def gen_fibonacci():
        """Yield the next Fibonacci number.

        Based on https://www.python-course.eu/generators.php
        Starts at Fibonacci number 3 (the second 1)

        Yields
        ------
        int
            The next Fibonacci number

        """
        num_a, num_b = 1, 2
        while True:
            yield num_a
            num_a, num_b = num_b, num_a + num_b


def dist_abs(
        self, src, tar, weights='exponential', max_length=8, normalized=False
    ):
        """Calculate the distance between the Eudex hashes of two terms.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison
        weights : str, iterable, or generator function
            The weights or weights generator function

                - If set to ``None``, a simple Hamming distance is calculated.
                - If set to ``exponential``, weight decays by powers of 2, as
                  proposed in the eudex specification:
                  https://github.com/ticki/eudex.
                - If set to ``fibonacci``, weight decays through the Fibonacci
                  series, as in the eudex reference implementation.
                - If set to a callable function, this assumes it creates a
                  generator and the generator is used to populate a series of
                  weights.
                - If set to an iterable, the iterable's values should be
                  integers and will be used as the weights.

        max_length : int
            The number of characters to encode as a eudex hash
        normalized : bool
            Normalizes to [0, 1] if True

        Returns
        -------
        int
            The Eudex Hamming distance

        Examples
        --------
        >>> cmp = Eudex()
        >>> cmp.dist_abs('cat', 'hat')
        128
        >>> cmp.dist_abs('Niall', 'Neil')
        2
        >>> cmp.dist_abs('Colin', 'Cuilen')
        10
        >>> cmp.dist_abs('ATCG', 'TAGC')
        403

        >>> cmp.dist_abs('cat', 'hat', weights='fibonacci')
        34
        >>> cmp.dist_abs('Niall', 'Neil', weights='fibonacci')
        2
        >>> cmp.dist_abs('Colin', 'Cuilen', weights='fibonacci')
        7
        >>> cmp.dist_abs('ATCG', 'TAGC', weights='fibonacci')
        117

        >>> cmp.dist_abs('cat', 'hat', weights=None)
        1
        >>> cmp.dist_abs('Niall', 'Neil', weights=None)
        1
        >>> cmp.dist_abs('Colin', 'Cuilen', weights=None)
        2
        >>> cmp.dist_abs('ATCG', 'TAGC', weights=None)
        9

        >>> # Using the OEIS A000142:
        >>> cmp.dist_abs('cat', 'hat', [1, 1, 2, 6, 24, 120, 720, 5040])
        1
        >>> cmp.dist_abs('Niall', 'Neil', [1, 1, 2, 6, 24, 120, 720, 5040])
        720
        >>> cmp.dist_abs('Colin', 'Cuilen',
        ... [1, 1, 2, 6, 24, 120, 720, 5040])
        744
        >>> cmp.dist_abs('ATCG', 'TAGC', [1, 1, 2, 6, 24, 120, 720, 5040])
        6243

        """
        # Calculate the eudex hashes and XOR them
        xored = eudex(src, max_length=max_length) ^ eudex(
            tar, max_length=max_length
        )

        # Simple hamming distance (all bits are equal)
        if not weights:
            binary = bin(xored)
            distance = binary.count('1')
            if normalized:
                return distance / (len(binary) - 2)
            return distance

        # If weights is a function, it should create a generator,
        # which we now use to populate a list
        if callable(weights):
            weights = weights()
        elif weights == 'exponential':
            weights = Eudex.gen_exponential()
        elif weights == 'fibonacci':
            weights = Eudex.gen_fibonacci()
        if isinstance(weights, GeneratorType):
            weights = [next(weights) for _ in range(max_length)][::-1]

        # Sum the weighted hamming distance
        distance = 0
        max_distance = 0
        while (xored or normalized) and weights:
            max_distance += 8 * weights[-1]
            distance += bin(xored & 0xFF).count('1') * weights.pop()
            xored >>= 8

        if normalized:
            distance /= max_distance

        return distance

Calculate the distance between the Eudex hashes of two terms .


def dist(self, src, tar, weights='exponential', max_length=8):
        """Return normalized distance between the Eudex hashes of two terms.

        This is Eudex distance normalized to [0, 1].

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison
        weights : str, iterable, or generator function
            The weights or weights generator function
        max_length : int
            The number of characters to encode as a eudex hash

        Returns
        -------
        int
            The normalized Eudex Hamming distance

        Examples
        --------
        >>> cmp = Eudex()
        >>> round(cmp.dist('cat', 'hat'), 12)
        0.062745098039
        >>> round(cmp.dist('Niall', 'Neil'), 12)
        0.000980392157
        >>> round(cmp.dist('Colin', 'Cuilen'), 12)
        0.004901960784
        >>> round(cmp.dist('ATCG', 'TAGC'), 12)
        0.197549019608

        """
        return self.dist_abs(src, tar, weights, max_length, True)

def dist ( self , src , tar , weights = 'exponential' , max_length = 8 ) : return self . dist_abs ( src , tar , weights , max_length , True )

Return normalized distance between the Eudex hashes of two terms .


def euclidean(src, tar, qval=2, normalized=False, alphabet=None):
    """Return the Euclidean distance between two strings.

    This is a wrapper for :py:meth:`Euclidean.dist_abs`.

    Parameters
    ----------
    src : str
        Source string (or QGrams/Counter objects) for comparison
    tar : str
        Target string (or QGrams/Counter objects) for comparison
    qval : int
        The length of each q-gram; 0 for non-q-gram version
    normalized : bool
        Normalizes to [0, 1] if True
    alphabet : collection or int
        The values or size of the alphabet

    Returns
    -------
    float: The Euclidean distance

    Examples
    --------
    >>> euclidean('cat', 'hat')
    2.0
    >>> round(euclidean('Niall', 'Neil'), 12)
    2.645751311065
    >>> euclidean('Colin', 'Cuilen')
    3.0
    >>> round(euclidean('ATCG', 'TAGC'), 12)
    3.162277660168

    """
    return Euclidean().dist_abs(src, tar, qval, normalized, alphabet)

def euclidean ( src , tar , qval = 2 , normalized = False , alphabet = None ) : return Euclidean ( ) . dist_abs ( src , tar , qval , normalized , alphabet )

Return the Euclidean distance between two strings .


def dist_euclidean(src, tar, qval=2, alphabet=None):
    """Return the normalized Euclidean distance between two strings.

    This is a wrapper for :py:meth:`Euclidean.dist`.

    Parameters
    ----------
    src : str
        Source string (or QGrams/Counter objects) for comparison
    tar : str
        Target string (or QGrams/Counter objects) for comparison
    qval : int
        The length of each q-gram; 0 for non-q-gram version
    alphabet : collection or int
        The values or size of the alphabet

    Returns
    -------
    float
        The normalized Euclidean distance

    Examples
    --------
    >>> round(dist_euclidean('cat', 'hat'), 12)
    0.57735026919
    >>> round(dist_euclidean('Niall', 'Neil'), 12)
    0.683130051064
    >>> round(dist_euclidean('Colin', 'Cuilen'), 12)
    0.727606875109
    >>> dist_euclidean('ATCG', 'TAGC')
    1.0

    """
    return Euclidean().dist(src, tar, qval, alphabet)

def dist_euclidean ( src , tar , qval = 2 , alphabet = None ) : return Euclidean ( ) . dist ( src , tar , qval , alphabet )

Return the normalized Euclidean distance between two strings .


def sim_euclidean(src, tar, qval=2, alphabet=None):
    """Return the normalized Euclidean similarity of two strings.

    This is a wrapper for :py:meth:`Euclidean.sim`.

    Parameters
    ----------
    src : str
        Source string (or QGrams/Counter objects) for comparison
    tar : str
        Target string (or QGrams/Counter objects) for comparison
    qval : int
        The length of each q-gram; 0 for non-q-gram version
    alphabet : collection or int
        The values or size of the alphabet

    Returns
    -------
    float
        The normalized Euclidean similarity

    Examples
    --------
    >>> round(sim_euclidean('cat', 'hat'), 12)
    0.42264973081
    >>> round(sim_euclidean('Niall', 'Neil'), 12)
    0.316869948936
    >>> round(sim_euclidean('Colin', 'Cuilen'), 12)
    0.272393124891
    >>> sim_euclidean('ATCG', 'TAGC')
    0.0

    """
    return Euclidean().sim(src, tar, qval, alphabet)

def sim_euclidean ( src , tar , qval = 2 , alphabet = None ) : return Euclidean ( ) . sim ( src , tar , qval , alphabet )

Return the normalized Euclidean similarity of two strings .


def _cond_k(self, word, suffix_len):
        """Return Lovins' condition K.

        Parameters
        ----------
        word : str
            Word to check
        suffix_len : int
            Suffix length

        Returns
        -------
        bool
            True if condition is met

        """
        return (len(word) - suffix_len >= 3) and (
            word[-suffix_len - 1] in {'i', 'l'}
            or (word[-suffix_len - 3] == 'u' and word[-suffix_len - 1] == 'e')
        )

def _cond_k ( self , word , suffix_len ) : return ( len ( word ) - suffix_len >= 3 ) and ( word [ - suffix_len - 1 ] in { 'i' , 'l' } or ( word [ - suffix_len - 3 ] == 'u' and word [ - suffix_len - 1 ] == 'e' ) )


def _cond_n(self, word, suffix_len):
        """Return Lovins' condition N.

        Parameters
        ----------
        word : str
            Word to check
        suffix_len : int
            Suffix length

        Returns
        -------
        bool
            True if condition is met

        """
        if len(word) - suffix_len >= 3:
            if word[-suffix_len - 3] == 's':
                if len(word) - suffix_len >= 4:
                    return True
            else:
                return True
        return False

def _cond_n ( self , word , suffix_len ) : if len ( word ) - suffix_len >= 3 : if word [ - suffix_len - 3 ] == 's' : if len ( word ) - suffix_len >= 4 : return True else : return True return False


def _cond_s(self, word, suffix_len):
        """Return Lovins' condition S.

        Parameters
        ----------
        word : str
            Word to check
        suffix_len : int
            Suffix length

        Returns
        -------
        bool
            True if condition is met

        """
        return word[-suffix_len - 2 : -suffix_len] == 'dr' or (
            word[-suffix_len - 1] == 't'
            and word[-suffix_len - 2 : -suffix_len] != 'tt'
        )

def _cond_s ( self , word , suffix_len ) : return word [ - suffix_len - 2 : - suffix_len ] == 'dr' or ( word [ - suffix_len - 1 ] == 't' and word [ - suffix_len - 2 : - suffix_len ] != 'tt' )


def _cond_x(self, word, suffix_len):
        """Return Lovins' condition X.

        Parameters
        ----------
        word : str
            Word to check
        suffix_len : int
            Suffix length

        Returns
        -------
        bool
            True if condition is met

        """
        return word[-suffix_len - 1] in {'i', 'l'} or (
            word[-suffix_len - 3 : -suffix_len] == 'u'
            and word[-suffix_len - 1] == 'e'
        )

def _cond_x ( self , word , suffix_len ) : return word [ - suffix_len - 1 ] in { 'i' , 'l' } or ( word [ - suffix_len - 3 : - suffix_len ] == 'u' and word [ - suffix_len - 1 ] == 'e' )


def _cond_bb(self, word, suffix_len):
        """Return Lovins' condition BB.

        Parameters
        ----------
        word : str
            Word to check
        suffix_len : int
            Suffix length

        Returns
        -------
        bool
            True if condition is met

        """
        return (
            len(word) - suffix_len >= 3
            and word[-suffix_len - 3 : -suffix_len] != 'met'
            and word[-suffix_len - 4 : -suffix_len] != 'ryst'
        )

def _cond_bb ( self , word , suffix_len ) : return ( len ( word ) - suffix_len >= 3 and word [ - suffix_len - 3 : - suffix_len ] != 'met' and word [ - suffix_len - 4 : - suffix_len ] != 'ryst' )


def stem(self, word):
        """Return Lovins stem.

        Parameters
        ----------
        word : str
            The word to stem

        Returns
        -------
        str
            Word stem

        Examples
        --------
        >>> stmr = Lovins()
        >>> stmr.stem('reading')
        'read'
        >>> stmr.stem('suspension')
        'suspens'
        >>> stmr.stem('elusiveness')
        'elus'

        """
        # lowercase, normalize, and compose
        word = normalize('NFC', text_type(word.lower()))

        for suffix_len in range(11, 0, -1):
            ending = word[-suffix_len:]
            if (
                ending in self._suffix
                and len(word) - suffix_len >= 2
                and (
                    self._suffix[ending] is None
                    or self._suffix[ending](word, suffix_len)
                )
            ):
                word = word[:-suffix_len]
                break

        if word[-2:] in {
            'bb',
            'dd',
            'gg',
            'll',
            'mm',
            'nn',
            'pp',
            'rr',
            'ss',
            'tt',
        }:
            word = word[:-1]

        for ending, replacement in self._recode:
            if word.endswith(ending):
                if callable(replacement):
                    word = replacement(word)
                else:
                    word = word[: -len(ending)] + replacement

        return word

def
stem
(
self
,
word
)
:
# lowercase, normalize, and compose
word
=
normalize
(
'NFC'
,
text_type
(
word
.
lower
(
)
)
)
for
suffix_len
in
range
(
11
,
0
,
1
)
:
ending
=
word
[
suffix_len
:
]
if
(
ending
in
self
.
_suffix
and
len
(
word
)
suffix_len
>=
2
and
(
self
.
_suffix
[
ending
]
is
or
self
.
_suffix
[
ending
]
(
word
,
suffix_len
)
)
)
:
word
=
word
[
:
suffix_len
]
break
if
word
[
2
:
]
in
{
'bb'
,
'dd'
,
'gg'
,
'll'
,
'mm'
,
'nn'
,
'pp'
,
'rr'
,
'ss'
,
'tt'
,
}
:
word
=
word
[
:
1
]
for
ending
,
replacement
in
self
.
_recode
:
if
word
.
endswith
(
ending
)
:
if
callable
(
replacement
)
:
word
=
replacement
(
word
)
else
:
word
=
word
[
:
len
(
ending
)
]
+
replacement
return
word


def dist(self, src, tar):
        """Return the NCD between two strings using zlib compression.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        float
            Compression distance

        Examples
        --------
        >>> cmp = NCDzlib()
        >>> cmp.dist('cat', 'hat')
        0.3333333333333333
        >>> cmp.dist('Niall', 'Neil')
        0.45454545454545453
        >>> cmp.dist('aluminum', 'Catalan')
        0.5714285714285714
        >>> cmp.dist('ATCG', 'TAGC')
        0.4

        """
        if src == tar:
            return 0.0

        src = src.encode('utf-8')
        tar = tar.encode('utf-8')

        self._compressor.compress(src)
        src_comp = self._compressor.flush(zlib.Z_FULL_FLUSH)
        self._compressor.compress(tar)
        tar_comp = self._compressor.flush(zlib.Z_FULL_FLUSH)
        self._compressor.compress(src + tar)
        concat_comp = self._compressor.flush(zlib.Z_FULL_FLUSH)
        self._compressor.compress(tar + src)
        concat_comp2 = self._compressor.flush(zlib.Z_FULL_FLUSH)

        return (
            min(len(concat_comp), len(concat_comp2))
            - min(len(src_comp), len(tar_comp))
        ) / max(len(src_comp), len(tar_comp))

def dist ( self , src , tar ) : if src == tar : return 0.0 src = src . encode ( 'utf-8' ) tar = tar . encode ( 'utf-8' ) self . _compressor . compress ( src ) src_comp = self . _compressor . flush ( zlib . Z_FULL_FLUSH ) self . _compressor . compress ( tar ) tar_comp = self . _compressor . flush ( zlib . Z_FULL_FLUSH ) self . _compressor . compress ( src + tar ) concat_comp = self . _compressor . flush ( zlib . Z_FULL_FLUSH ) self . _compressor . compress ( tar + src ) concat_comp2 = self . _compressor . flush ( zlib . Z_FULL_FLUSH ) return ( min ( len ( concat_comp ) , len ( concat_comp2 ) ) - min ( len ( src_comp ) , len ( tar_comp ) ) ) / max ( len ( src_comp ) , len ( tar_comp ) )

Return the NCD between two strings using zlib compression .


def pylint_color(score):
    """Return Pylint badge color.

    Parameters
    ----------
    score : float
        A Pylint score

    Returns
    -------
    str
        Badge color

    """
    # These are the score cutoffs for each color above.
    # I.e. score==10 -> brightgreen, down to 7.5 > score >= 5 -> orange
    score_cutoffs = (10, 9.5, 8.5, 7.5, 5)
    for i in range(len(score_cutoffs)):
        if score >= score_cutoffs[i]:
            return BADGE_COLORS[i]
    # and score < 5 -> red
    return BADGE_COLORS[-1]


def pydocstyle_color(score):
    """Return pydocstyle badge color.

    Parameters
    ----------
    score : float
        A pydocstyle score

    Returns
    -------
    str
        Badge color

    """
    # These are the score cutoffs for each color above.
    # I.e. score==0 -> brightgreen, down to 100 < score <= 200 -> orange
    score_cutoffs = (0, 10, 25, 50, 100)
    for i in range(len(score_cutoffs)):
        if score <= score_cutoffs[i]:
            return BADGE_COLORS[i]
    # and score > 200 -> red
    return BADGE_COLORS[-1]


def flake8_color(score):
    """Return flake8 badge color.

    Parameters
    ----------
    score : float
        A flake8 score

    Returns
    -------
    str
        Badge color

    """
    # These are the score cutoffs for each color above.
    # I.e. score==0 -> brightgreen, down to 100 < score <= 200 -> orange
    score_cutoffs = (0, 20, 50, 100, 200)
    for i in range(len(score_cutoffs)):
        if score <= score_cutoffs[i]:
            return BADGE_COLORS[i]
    # and score > 200 -> red
    return BADGE_COLORS[-1]


def dist_abs(self, src, tar):
        """Return the bag distance between two strings.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        int
            Bag distance

        Examples
        --------
        >>> cmp = Bag()
        >>> cmp.dist_abs('cat', 'hat')
        1
        >>> cmp.dist_abs('Niall', 'Neil')
        2
        >>> cmp.dist_abs('aluminum', 'Catalan')
        5
        >>> cmp.dist_abs('ATCG', 'TAGC')
        0
        >>> cmp.dist_abs('abcdefg', 'hijklm')
        7
        >>> cmp.dist_abs('abcdefg', 'hijklmno')
        8

        """
        if tar == src:
            return 0
        elif not src:
            return len(tar)
        elif not tar:
            return len(src)

        src_bag = Counter(src)
        tar_bag = Counter(tar)
        return max(
            sum((src_bag - tar_bag).values()),
            sum((tar_bag - src_bag).values()),
        )

def dist_abs ( self , src , tar ) : if tar == src : return 0 elif not src : return len ( tar ) elif not tar : return len ( src ) src_bag = Counter ( src ) tar_bag = Counter ( tar ) return max ( sum ( ( src_bag - tar_bag ) . values ( ) ) , sum ( ( tar_bag - src_bag ) . values ( ) ) , )

Return the bag distance between two strings .


def dist(self, src, tar):
        """Return the normalized bag distance between two strings.

        Bag distance is normalized by dividing by :math:`max( |src|, |tar| )`.

        Parameters
        ----------
        src : str
            Source string for comparison
        tar : str
            Target string for comparison

        Returns
        -------
        float
            Normalized bag distance

        Examples
        --------
        >>> cmp = Bag()
        >>> cmp.dist('cat', 'hat')
        0.3333333333333333
        >>> cmp.dist('Niall', 'Neil')
        0.4
        >>> cmp.dist('aluminum', 'Catalan')
        0.625
        >>> cmp.dist('ATCG', 'TAGC')
        0.0

        """
        if tar == src:
            return 0.0
        if not src or not tar:
            return 1.0

        max_length = max(len(src), len(tar))

        return self.dist_abs(src, tar) / max_length

def dist ( self , src , tar ) : if tar == src : return 0.0 if not src or not tar : return 1.0 max_length = max ( len ( src ) , len ( tar ) ) return self . dist_abs ( src , tar ) / max_length

Return the normalized bag distance between two strings .


def stem(self, word):
        """Return 'CLEF German stemmer plus' stem.

        Parameters
        ----------
        word : str
            The word to stem

        Returns
        -------
        str
            Word stem

        Examples
        --------
        >>> stmr = CLEFGermanPlus()
        >>> clef_german_plus('lesen')
        'les'
        >>> clef_german_plus('graues')
        'grau'
        >>> clef_german_plus('buchstabieren')
        'buchstabi'

        """
        # lowercase, normalize, and compose
        word = normalize('NFC', text_type(word.lower()))

        # remove umlauts
        word = word.translate(self._accents)

        # Step 1
        wlen = len(word) - 1
        if wlen > 4 and word[-3:] == 'ern':
            word = word[:-3]
        elif wlen > 3 and word[-2:] in {'em', 'en', 'er', 'es'}:
            word = word[:-2]
        elif wlen > 2 and (
            word[-1] == 'e'
            or (word[-1] == 's' and word[-2] in self._st_ending)
        ):
            word = word[:-1]

        # Step 2
        wlen = len(word) - 1
        if wlen > 4 and word[-3:] == 'est':
            word = word[:-3]
        elif wlen > 3 and (
            word[-2:] in {'er', 'en'}
            or (word[-2:] == 'st' and word[-3] in self._st_ending)
        ):
            word = word[:-2]

        return word

def
stem
(
self
,
word
)
:
# lowercase, normalize, and compose
word
=
normalize
(
'NFC'
,
text_type
(
word
.
lower
(
)
)
)
# remove umlauts
word
=
word
.
translate
(
self
.
_accents
)
# Step 1
wlen
=
len
(
word
)
1
if
wlen
>
4
and
word
[
3
:
]
==
'ern'
:
word
=
word
[
:
3
]
elif
wlen
>
3
and
word
[
2
:
]
in
{
'em'
,
'en'
,
'er'
,
'es'
}
:
word
=
word
[
:
2
]
elif
wlen
>
2
and
(
word
[
1
]
==
'e'
or
(
word
[
1
]
==
's'
and
word
[
2
]
in
self
.
_st_ending
)
)
:
word
=
word
[
:
1
]
# Step 2
wlen
=
len
(
word
)
1
if
wlen
>
4
and
word
[
3
:
]
==
'est'
:
word
=
word
[
:
3
]
elif
wlen
>
3
and
(
word
[
2
:
]
in
{
'er'
,
'en'
}
or
(
word
[
2
:
]
==
'st'
and
word
[
3
]
in
self
.
_st_ending
)
)
:
word
=
word
[
:
2
]
return
word


def dist_mlipns(src, tar, threshold=0.25, max_mismatches=2):
    """Return the MLIPNS distance between two strings.

    This is a wrapper for :py:meth:`MLIPNS.dist`.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    threshold : float
        A number [0, 1] indicating the maximum similarity score, below which
        the strings are considered 'similar' (0.25 by default)
    max_mismatches : int
        A number indicating the allowable number of mismatches to remove before
        declaring two strings not similar (2 by default)

    Returns
    -------
    float
        MLIPNS distance

    Examples
    --------
    >>> dist_mlipns('cat', 'hat')
    0.0
    >>> dist_mlipns('Niall', 'Neil')
    1.0
    >>> dist_mlipns('aluminum', 'Catalan')
    1.0
    >>> dist_mlipns('ATCG', 'TAGC')
    1.0

    """
    return MLIPNS().dist(src, tar, threshold, max_mismatches)

def dist_mlipns ( src , tar , threshold = 0.25 , max_mismatches = 2 ) : return MLIPNS ( ) . dist ( src , tar , threshold , max_mismatches )

Return the MLIPNS distance between two strings .


def sim(src, tar, method=sim_levenshtein):
    """Return a similarity of two strings.

    This is a generalized function for calling other similarity functions.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    method : function
        Specifies the similarity metric (:py:func:`sim_levenshtein` by default)

    Returns
    -------
    float
        Similarity according to the specified function

    Raises
    ------
    AttributeError
        Unknown distance function

    Examples
    --------
    >>> round(sim('cat', 'hat'), 12)
    0.666666666667
    >>> round(sim('Niall', 'Neil'), 12)
    0.4
    >>> sim('aluminum', 'Catalan')
    0.125
    >>> sim('ATCG', 'TAGC')
    0.25

    """
    if callable(method):
        return method(src, tar)
    else:
        raise AttributeError('Unknown similarity function: ' + str(method))

def
sim
(
src
,
tar
,
method
=
sim_levenshtein
)
:
if
callable
(
method
)
:
return
method
(
src
,
tar
)
else
:
raise
AttributeError
(
'Unknown similarity function: '
+
str
(
method
)
)


def dist(src, tar, method=sim_levenshtein):
    """Return a distance between two strings.

    This is a generalized function for calling other distance functions.

    Parameters
    ----------
    src : str
        Source string for comparison
    tar : str
        Target string for comparison
    method : function
        Specifies the similarity metric (:py:func:`sim_levenshtein` by default)
        -- Note that this takes a similarity metric function, not a distance
        metric function.

    Returns
    -------
    float
        Distance according to the specified function

    Raises
    ------
    AttributeError
        Unknown distance function

    Examples
    --------
    >>> round(dist('cat', 'hat'), 12)
    0.333333333333
    >>> round(dist('Niall', 'Neil'), 12)
    0.6
    >>> dist('aluminum', 'Catalan')
    0.875
    >>> dist('ATCG', 'TAGC')
    0.75

    """
    if callable(method):
        return 1 - method(src, tar)
    else:
        raise AttributeError('Unknown distance function: ' + str(method))

def
dist
(
src
,
tar
,
method
=
sim_levenshtein
)
:
if
callable
(
method
)
:
return
1
method
(
src
,
tar
)
else
:
raise
AttributeError
(
'Unknown distance function: '
+
str
(
method
)
)


def _m_degree(self, term):
        """Return Porter helper function _m_degree value.

        m-degree is equal to the number of V to C transitions

        Parameters
        ----------
        term : str
            The word for which to calculate the m-degree

        Returns
        -------
        int
            The m-degree as defined in the Porter stemmer definition

        """
        mdeg = 0
        last_was_vowel = False
        for letter in term:
            if letter in self._vowels:
                last_was_vowel = True
            else:
                if last_was_vowel:
                    mdeg += 1
                last_was_vowel = False
        return mdeg

def _m_degree ( self , term ) : mdeg = 0 last_was_vowel = False for letter in term : if letter in self . _vowels : last_was_vowel = True else : if last_was_vowel : mdeg += 1 last_was_vowel = False return mdeg

Return Porter helper function _m_degree value .


def _has_vowel(self, term):
        """Return Porter helper function _has_vowel value.

        Parameters
        ----------
        term : str
            The word to scan for vowels

        Returns
        -------
        bool
            True iff a vowel exists in the term (as defined in the Porter
            stemmer definition)

        """
        for letter in term:
            if letter in self._vowels:
                return True
        return False

def _has_vowel ( self , term ) : for letter in term : if letter in self . _vowels : return True return False

Return Porter helper function _has_vowel value .


def _ends_in_doubled_cons(self, term):
        """Return Porter helper function _ends_in_doubled_cons value.

        Parameters
        ----------
        term : str
            The word to check for a final doubled consonant

        Returns
        -------
        bool
            True iff the stem ends in a doubled consonant (as defined in the
            Porter stemmer definition)

        """
        return (
            len(term) > 1
            and term[-1] not in self._vowels
            and term[-2] == term[-1]
        )

def _ends_in_doubled_cons ( self , term ) : return ( len ( term ) > 1 and term [ - 1 ] not in self . _vowels and term [ - 2 ] == term [ - 1 ] )

Return Porter helper function _ends_in_doubled_cons value .


def _ends_in_cvc(self, term):
        """Return Porter helper function _ends_in_cvc value.

        Parameters
        ----------
        term : str
            The word to scan for cvc

        Returns
        -------
        bool
            True iff the stem ends in cvc (as defined in the Porter stemmer
            definition)

        """
        return len(term) > 2 and (
            term[-1] not in self._vowels
            and term[-2] in self._vowels
            and term[-3] not in self._vowels
            and term[-1] not in tuple('wxY')
        )

def _ends_in_cvc ( self , term ) : return len ( term ) > 2 and ( term [ - 1 ] not in self . _vowels and term [ - 2 ] in self . _vowels and term [ - 3 ] not in self . _vowels and term [ - 1 ] not in tuple ( 'wxY' ) )

Return Porter helper function _ends_in_cvc value .


def filter_symlog(y, base=10.0):
    """Symmetrical logarithmic scale.

    Optional arguments:

    *base*:
        The base of the logarithm.
    """
    log_base = np.log(base)
    sign = np.sign(y)
    logs = np.log(np.abs(y) / log_base)
    return sign * logs

def filter_symlog ( y , base = 10.0 ) : log_base = np . log ( base ) sign = np . sign ( y ) logs = np . log ( np . abs ( y ) / log_base ) return sign * logs


def usage_function(parser):
    """Show usage and available curve functions."""
    parser.print_usage()
    print('')
    print('available functions:')
    for function in sorted(FUNCTION):
        doc = FUNCTION[function].__doc__.strip().splitlines()[0]
        print('    %-12s %s' % (function + ':', doc))

    return 0

def
usage_function
(
parser
)
:
parser
.
print_usage
(
)
print
(
''
)
print
(
'available functions:'
)
for
function
in
sorted
(
FUNCTION
)
:
doc
=
FUNCTION
[
function
]
.
__doc__
.
strip
(
)
.
splitlines
(
)
[
0
]
print
(
'    %-12s %s'
%
(
function
+
':'
,
doc
)
)
return
0

Show usage and available curve functions .


def usage_palette(parser):
    """Show usage and available palettes."""
    parser.print_usage()
    print('')
    print('available palettes:')
    for palette in sorted(PALETTE):
        print('    %-12s' % (palette,))

    return 0

def
usage_palette
(
parser
)
:
parser
.
print_usage
(
)
print
(
''
)
print
(
'available palettes:'
)
for
palette
in
sorted
(
PALETTE
)
:
print
(
'    %-12s'
%
(
palette
,
)
)
return
0


def size(self):
        """Get the current terminal size."""
        for fd in range(3):
            cr = self._ioctl_GWINSZ(fd)
            if cr:
                break
        if not cr:
            try:
                fd = os.open(os.ctermid(), os.O_RDONLY)
                cr = self._ioctl_GWINSZ(fd)
                os.close(fd)
            except Exception:
                pass

        if not cr:
            env = os.environ
            cr = (env.get('LINES', 25), env.get('COLUMNS', 80))

        return int(cr[1]), int(cr[0])

def size ( self ) : for fd in range ( 3 ) : cr = self . _ioctl_GWINSZ ( fd ) if cr : break if not cr : try : fd = os . open ( os . ctermid ( ) , os . O_RDONLY ) cr = self . _ioctl_GWINSZ ( fd ) os . close ( fd ) except Exception : pass if not cr : env = os . environ cr = ( env . get ( 'LINES' , 25 ) , env . get ( 'COLUMNS' , 80 ) ) return int ( cr [ 1 ] ) , int ( cr [ 0 ] )


def color(self, index):
        """Get the escape sequence for indexed color ``index``.

        The ``index`` is a color index in the 256 color space. The color space
        consists of:

        * 0x00-0x0f: default EGA colors
        * 0x10-0xe7: 6x6x6 RGB cubes
        * 0xe8-0xff: gray scale ramp
        """
        if self.colors == 16:
            if index >= 8:
                return self.csi('bold') + self.csi('setaf', index - 8)
            else:
                return self.csi('sgr0') + self.csi('setaf', index)
        else:
            return self.csi('setaf', index)

def color ( self , index ) : if self . colors == 16 : if index >= 8 : return self . csi ( 'bold' ) + self . csi ( 'setaf' , index - 8 ) else : return self . csi ( 'sgr0' ) + self . csi ( 'setaf' , index ) else : return self . csi ( 'setaf' , index )

Get the escape sequence for indexed color index .


def csi(self, capname, *args):
        """Return the escape sequence for the selected Control Sequence."""
        value = curses.tigetstr(capname)
        if value is None:
            return b''
        else:
            return curses.tparm(value, *args)

def csi ( self , capname , * args ) : value = curses . tigetstr ( capname ) if value is None : return b'' else : return curses . tparm ( value , * args )

Return the escape sequence for the selected Control Sequence .


def csi_wrap(self, value, capname, *args):
        """Return a value wrapped in the selected CSI and does a reset."""
        if isinstance(value, str):
            value = value.encode('utf-8')
        return b''.join([
            self.csi(capname, *args),
            value,
            self.csi('sgr0'),
        ])

def csi_wrap ( self , value , capname , * args ) : if isinstance ( value , str ) : value = value . encode ( 'utf-8' ) return b'' . join ( [ self . csi ( capname , * args ) , value , self . csi ( 'sgr0' ) , ] )

Return a value wrapped in the selected CSI and does a reset .


def consume(self, istream, ostream, batch=False):
        """Read points from istream and output to ostream."""
        datapoints = []  # List of 2-tuples

        if batch:
            sleep = max(0.01, self.option.sleep)
            fd = istream.fileno()
            while True:
                try:
                    if select.select([fd], [], [], sleep):
                        try:
                            line = istream.readline()
                            if line == '':
                                break
                            datapoints.append(self.consume_line(line))
                        except ValueError:
                            continue

                        if self.option.sort_by_column:
                            datapoints = sorted(datapoints, key=itemgetter(self.option.sort_by_column - 1))

                        if len(datapoints) > 1:
                            datapoints = datapoints[-self.maximum_points:]
                            self.update([dp[0] for dp in datapoints], [dp[1] for dp in datapoints])
                            self.render(ostream)

                        time.sleep(sleep)

                except KeyboardInterrupt:
                    break

        else:
            for line in istream:
                try:
                    datapoints.append(self.consume_line(line))
                except ValueError:
                    pass

            if self.option.sort_by_column:
                datapoints = sorted(datapoints, key=itemgetter(self.option.sort_by_column - 1))

            self.update([dp[0] for dp in datapoints], [dp[1] for dp in datapoints])
            self.render(ostream)

def
consume
(
self
,
istream
,
ostream
,
batch
=
False
)
:
datapoints
=
[
]
# List of 2-tuples
if
batch
:
sleep
=
max
(
0.01
,
self
.
option
.
sleep
)
fd
=
istream
.
fileno
(
)
while
True
:
try
:
if
select
.
select
(
[
fd
]
,
[
]
,
[
]
,
sleep
)
:
try
:
line
=
istream
.
readline
(
)
if
line
==
''
:
break
datapoints
.
append
(
self
.
consume_line
(
line
)
)
except
ValueError
:
continue
if
self
.
option
.
sort_by_column
:
datapoints
=
sorted
(
datapoints
,
key
=
itemgetter
(
self
.
option
.
sort_by_column
1
)
)
if
len
(
datapoints
)
>
1
:
datapoints
=
datapoints
[
self
.
maximum_points
:
]
self
.
update
(
[
dp
[
0
]
for
dp
in
datapoints
]
,
[
dp
[
1
]
for
dp
in
datapoints
]
)
self
.
render
(
ostream
)
time
.
sleep
(
sleep
)
except
KeyboardInterrupt
:
break
else
:
for
line
in
istream
:
try
:
datapoints
.
append
(
self
.
consume_line
(
line
)
)
except
ValueError
:
pass
if
self
.
option
.
sort_by_column
:
datapoints
=
sorted
(
datapoints
,
key
=
itemgetter
(
self
.
option
.
sort_by_column
1
)
)
self
.
update
(
[
dp
[
0
]
for
dp
in
datapoints
]
,
[
dp
[
1
]
for
dp
in
datapoints
]
)
self
.
render
(
ostream
)

Read points from istream and output to ostream .


def consume_line(self, line):
        """Consume data from a line."""
        data = RE_VALUE_KEY.split(line.strip(), 1)
        if len(data) == 1:
            return float(data[0]), None
        else:
            return float(data[0]), data[1].strip()

def consume_line ( self , line ) : data = RE_VALUE_KEY . split ( line . strip ( ) , 1 ) if len ( data ) == 1 : return float ( data [ 0 ] ) , None else : return float ( data [ 0 ] ) , data [ 1 ] . strip ( )


def update(self, points, values=None):
        """Add a set of data points."""
        self.values = values or [None] * len(points)

        if np is None:
            if self.option.function:
                warnings.warn('numpy not available, function ignored')
            self.points = points
            self.minimum = min(self.points)
            self.maximum = max(self.points)
            self.current = self.points[-1]

        else:
            self.points = self.apply_function(points)
            self.minimum = np.min(self.points)
            self.maximum = np.max(self.points)
            self.current = self.points[-1]

        if self.maximum == self.minimum:
            self.extents = 1
        else:
            self.extents = (self.maximum - self.minimum)
            self.extents = (self.maximum - self.minimum)

def
update
(
self
,
points
,
values
=
)
:
self
.
values
=
values
or
[
]
*
len
(
points
)
if
np
is
:
if
self
.
option
.
function
:
warnings
.
warn
(
'numpy not available, function ignored'
)
self
.
points
=
points
self
.
minimum
=
min
(
self
.
points
)
self
.
maximum
=
max
(
self
.
points
)
self
.
current
=
self
.
points
[
1
]
else
:
self
.
points
=
self
.
apply_function
(
points
)
self
.
minimum
=
np
.
min
(
self
.
points
)
self
.
maximum
=
np
.
max
(
self
.
points
)
self
.
current
=
self
.
points
[
1
]
if
self
.
maximum
==
self
.
minimum
:
self
.
extents
=
1
else
:
self
.
extents
=
(
self
.
maximum
self
.
minimum
)
self
.
extents
=
(
self
.
maximum
self
.
minimum
)


def color_ramp(self, size):
        """Generate a color ramp for the current screen height."""
        color = PALETTE.get(self.option.palette, {})
        color = color.get(self.term.colors, None)
        color_ramp = []
        if color is not None:
            ratio = len(color) / float(size)
            for i in range(int(size)):
                color_ramp.append(self.term.color(color[int(ratio * i)]))

        return color_ramp

def color_ramp ( self , size ) : color = PALETTE . get ( self . option . palette , { } ) color = color . get ( self . term . colors , None ) color_ramp = [ ] if color is not None : ratio = len ( color ) / float ( size ) for i in range ( int ( size ) ) : color_ramp . append ( self . term . color ( color [ int ( ratio * i ) ] ) ) return color_ramp

Generate a color ramp for the current screen height .


def human(self, size, base=1000, units=' kMGTZ'):
        """Convert the input ``size`` to human readable, short form."""
        sign = '+' if size >= 0 else '-'
        size = abs(size)
        if size < 1000:
            return '%s%d' % (sign, size)
        for i, suffix in enumerate(units):
            unit = 1000 ** (i + 1)
            if size < unit:
                return ('%s%.01f%s' % (
                    sign,
                    size / float(unit) * base,
                    suffix,
                )).strip()
        raise OverflowError

def
human
(
self
,
size
,
base
=
1000
,
units
=
' kMGTZ'
)
:
sign
=
'+'
if
size
>=
0
else
'-'
size
=
abs
(
size
)
if
size
<
1000
:
return
'%s%d'
%
(
sign
,
size
)
for
i
,
suffix
in
enumerate
(
units
)
:
unit
=
1000
**
(
i
+
1
)
if
size
<
unit
:
return
(
'%s%.01f%s'
%
(
sign
,
size
/
float
(
unit
)
*
base
,
suffix
,
)
)
.
strip
(
)
raise
OverflowError

Convert the input size to human readable short form .


def apply_function(self, points):
        """Run the filter function on the provided points."""
        if not self.option.function:
            return points

        if np is None:
            raise ImportError('numpy is not available')

        if ':' in self.option.function:
            function, arguments = self.option.function.split(':', 1)
            arguments = arguments.split(',')
        else:
            function = self.option.function
            arguments = []

        # Resolve arguments
        arguments = list(map(self._function_argument, arguments))

        # Resolve function
        filter_function = FUNCTION.get(function)

        if filter_function is None:
            raise TypeError('Invalid function "%s"' % (function,))

        else:
            # We wrap in ``list()`` to consume generators and iterators, as
            # ``np.array`` doesn't do this for us.
            return filter_function(np.array(list(points)), *arguments)

Run the filter function on the provided points .


def line(self, p1, p2, resolution=1):
        """Resolve the points to make a line between two points."""
        xdiff = max(p1.x, p2.x) - min(p1.x, p2.x)
        ydiff = max(p1.y, p2.y) - min(p1.y, p2.y)
        xdir = [-1, 1][int(p1.x <= p2.x)]
        ydir = [-1, 1][int(p1.y <= p2.y)]
        r = int(round(max(xdiff, ydiff)))
        if r == 0:
            return

        for i in range((r + 1) * resolution):
            x = p1.x
            y = p1.y

            if xdiff:
                x += (float(i) * xdiff) / r * xdir / resolution
            if ydiff:
                y += (float(i) * ydiff) / r * ydir / resolution

            yield Point((x, y))

def line ( self , p1 , p2 , resolution = 1 ) : xdiff = max ( p1 . x , p2 . x ) - min ( p1 . x , p2 . x ) ydiff = max ( p1 . y , p2 . y ) - min ( p1 . y , p2 . y ) xdir = [ - 1 , 1 ] [ int ( p1 . x <= p2 . x ) ] ydir = [ - 1 , 1 ] [ int ( p1 . y <= p2 . y ) ] r = int ( round ( max ( xdiff , ydiff ) ) ) if r == 0 : return for i in range ( ( r + 1 ) * resolution ) : x = p1 . x y = p1 . y if xdiff : x += ( float ( i ) * xdiff ) / r * xdir / resolution if ydiff : y += ( float ( i ) * ydiff ) / r * ydir / resolution yield Point ( ( x , y ) )

Resolve the points to make a line between two points .


def set_text(self, point, text):
        """Set a text value in the screen canvas."""
        if not self.option.legend:
            return

        if not isinstance(point, Point):
            point = Point(point)

        for offset, char in enumerate(str(text)):
            self.screen.canvas[point.y][point.x + offset] = char

def set_text ( self , point , text ) : if not self . option . legend : return if not isinstance ( point , Point ) : point = Point ( point ) for offset , char in enumerate ( str ( text ) ) : self . screen . canvas [ point . y ] [ point . x + offset ] = char


def render(self, stream):
        """Render graph to stream."""
        encoding = self.option.encoding or self.term.encoding or "utf8"

        if self.option.color:
            ramp = self.color_ramp(self.size.y)[::-1]
        else:
            ramp = None

        if self.cycle >= 1 and self.lines:
            stream.write(self.term.csi('cuu', self.lines))

        zero = int(self.null / 4)  # Zero crossing
        lines = 0
        for y in range(self.screen.size.y):
            if y == zero and self.size.y > 1:
                stream.write(self.term.csi('smul'))
            if ramp:
                stream.write(ramp[y])

            for x in range(self.screen.size.x):
                point = Point((x, y))
                if point in self.screen:
                    value = self.screen[point]
                    if isinstance(value, int):
                        stream.write(chr(self.base + value).encode(encoding))
                    else:
                        stream.write(self.term.csi('sgr0'))
                        stream.write(self.term.csi_wrap(
                            value.encode(encoding),
                            'bold'
                        ))
                        if y == zero and self.size.y > 1:
                            stream.write(self.term.csi('smul'))
                        if ramp:
                            stream.write(ramp[y])
                else:
                    stream.write(b' ')

            if y == zero and self.size.y > 1:
                stream.write(self.term.csi('rmul'))
            if ramp:
                stream.write(self.term.csi('sgr0'))

            stream.write(b'\n')
            lines += 1
        stream.flush()

        self.cycle = self.cycle + 1
        self.lines = lines

def
render
(
self
,
stream
)
:
encoding
=
self
.
option
.
encoding
or
self
.
term
.
encoding
or
"utf8"
if
self
.
option
.
color
:
ramp
=
self
.
color_ramp
(
self
.
size
.
y
)
[
:
:
1
]
else
:
ramp
=
if
self
.
cycle
>=
1
and
self
.
lines
:
stream
.
write
(
self
.
term
.
csi
(
'cuu'
,
self
.
lines
)
)
zero
=
int
(
self
.
/
4
)
# Zero crossing
lines
=
0
for
y
in
range
(
self
.
screen
.
size
.
y
)
:
if
y
==
zero
and
self
.
size
.
y
>
1
:
stream
.
write
(
self
.
term
.
csi
(
'smul'
)
)
if
ramp
:
stream
.
write
(
ramp
[
y
]
)
for
x
in
range
(
self
.
screen
.
size
.
x
)
:
point
=
Point
(
(
x
,
y
)
)
if
point
in
self
.
screen
:
value
=
self
.
screen
[
point
]
if
isinstance
(
value
,
int
)
:
stream
.
write
(
chr
(
self
.
base
+
value
)
.
encode
(
encoding
)
)
else
:
stream
.
write
(
self
.
term
.
csi
(
'sgr0'
)
)
stream
.
write
(
self
.
term
.
csi_wrap
(
value
.
encode
(
encoding
)
,
'bold'
)
)
if
y
==
zero
and
self
.
size
.
y
>
1
:
stream
.
write
(
self
.
term
.
csi
(
'smul'
)
)
if
ramp
:
stream
.
write
(
ramp
[
y
]
)
else
:
stream
.
write
(
b' '
)
if
y
==
zero
and
self
.
size
.
y
>
1
:
stream
.
write
(
self
.
term
.
csi
(
'rmul'
)
)
if
ramp
:
stream
.
write
(
self
.
term
.
csi
(
'sgr0'
)
)
stream
.
write
(
b'\n'
)
lines
+=
1
stream
.
flush
(
)
self
.
cycle
=
self
.
cycle
+
1
self
.
lines
=
lines


def _normalised_numpy(self):
        """Normalised data points using numpy."""
        dx = (self.screen.width / float(len(self.points)))
        oy = (self.screen.height)
        points = np.array(self.points) - self.minimum
        points = points * 4.0 / self.extents * self.size.y
        for x, y in enumerate(points):
            yield Point((
                dx * x,
                min(oy, oy - y),
            ))

def _normalised_numpy ( self ) : dx = ( self . screen . width / float ( len ( self . points ) ) ) oy = ( self . screen . height ) points = np . array ( self . points ) - self . minimum points = points * 4.0 / self . extents * self . size . y for x , y in enumerate ( points ) : yield Point ( ( dx * x , min ( oy , oy - y ) , ) )


def _normalised_python(self):
        """Normalised data points using pure Python."""
        dx = (self.screen.width / float(len(self.points)))
        oy = (self.screen.height)
        for x, point in enumerate(self.points):
            y = (point - self.minimum) * 4.0 / self.extents * self.size.y
            yield Point((
                dx * x,
                min(oy, oy - y),
            ))

def _normalised_python ( self ) : dx = ( self . screen . width / float ( len ( self . points ) ) ) oy = ( self . screen . height ) for x , point in enumerate ( self . points ) : y = ( point - self . minimum ) * 4.0 / self . extents * self . size . y yield Point ( ( dx * x , min ( oy , oy - y ) , ) )

Normalised data points using pure Python .


def null(self):
        """Zero crossing value."""
        if not self.option.axis:
            return -1
        else:
            return self.screen.height - (
                -self.minimum * 4.0 / self.extents * self.size.y
            )

def null ( self ) : if not self . option . axis : return - 1 else : return self . screen . height - ( - self . minimum * 4.0 / self . extents * self . size . y )


def mem_size(self):
        '''used when allocating memory ingame'''

        data_len = self._data_mem_size
        node_count = len(list(self.xml_doc.iter(tag=etree.Element)))

        if self.compressed:
            size = 52 * node_count + data_len + 630
        else:
            tags_len = 0
            for e in self.xml_doc.iter(tag=etree.Element):
                e_len = max(len(e.tag), 8)
                e_len = (e_len + 3) & ~3
                tags_len += e_len

            size = 56 * node_count + data_len + 630 + tags_len

        # debugging
        #print('nodes:{} ({}) data:{} ({})'.format(node_count,hex(node_count), data_len, hex(data_len)))

        return (size + 8) & ~7


def _load_class(class_path, default):
    """ Loads the class from the class_path string """
    if class_path is None:
        return default

    component = class_path.rsplit('.', 1)
    result_processor = getattr(
        importlib.import_module(component[0]),
        component[1],
        default
    ) if len(component) > 1 else default

    return result_processor

def _load_class ( class_path , default ) : if class_path is None : return default component = class_path . rsplit ( '.' , 1 ) result_processor = getattr ( importlib . import_module ( component [ 0 ] ) , component [ 1 ] , default ) if len ( component ) > 1 else default return result_processor


def _process_pagination_values(request):
    """ process pagination requests from request parameter """
    size = 20
    page = 0
    from_ = 0
    if "page_size" in request.POST:
        size = int(request.POST["page_size"])
        max_page_size = getattr(settings, "SEARCH_MAX_PAGE_SIZE", 100)
        # The parens below are superfluous, but make it much clearer to the reader what is going on
        if not (0 < size <= max_page_size):  # pylint: disable=superfluous-parens
            raise ValueError(_('Invalid page size of {page_size}').format(page_size=size))

        if "page_index" in request.POST:
            page = int(request.POST["page_index"])
            from_ = page * size
    return size, from_, page


def _process_field_values(request):
    """ Create separate dictionary of supported filter values provided """
    return {
        field_key: request.POST[field_key]
        for field_key in request.POST
        if field_key in course_discovery_filter_fields()
    }

def _process_field_values ( request ) : return { field_key : request . POST [ field_key ] for field_key in request . POST if field_key in course_discovery_filter_fields ( ) }


def do_search(request, course_id=None):
    """
    Search view for http requests

    Args:
        request (required) - django request object
        course_id (optional) - course_id within which to restrict search

    Returns:
        http json response with the following fields
            "took" - how many seconds the operation took
            "total" - how many results were found
            "max_score" - maximum score from these results
            "results" - json array of result documents

            or

            "error" - displayable information about an error that occured on the server

    POST Params:
        "search_string" (required) - text upon which to search
        "page_size" (optional)- how many results to return per page (defaults to 20, with maximum cutoff at 100)
        "page_index" (optional) - for which page (zero-indexed) to include results (defaults to 0)
    """

    # Setup search environment
    SearchInitializer.set_search_enviroment(request=request, course_id=course_id)

    results = {
        "error": _("Nothing to search")
    }
    status_code = 500

    search_term = request.POST.get("search_string", None)

    try:
        if not search_term:
            raise ValueError(_('No search term provided for search'))

        size, from_, page = _process_pagination_values(request)

        # Analytics - log search request
        track.emit(
            'edx.course.search.initiated',
            {
                "search_term": search_term,
                "page_size": size,
                "page_number": page,
            }
        )

        results = perform_search(
            search_term,
            user=request.user,
            size=size,
            from_=from_,
            course_id=course_id
        )

        status_code = 200

        # Analytics - log search results before sending to browser
        track.emit(
            'edx.course.search.results_displayed',
            {
                "search_term": search_term,
                "page_size": size,
                "page_number": page,
                "results_count": results["total"],
            }
        )

    except ValueError as invalid_err:
        results = {
            "error": six.text_type(invalid_err)
        }
        log.debug(six.text_type(invalid_err))

    except QueryParseError:
        results = {
            "error": _('Your query seems malformed. Check for unmatched quotes.')
        }

    # Allow for broad exceptions here - this is an entry point from external reference
    except Exception as err:  # pylint: disable=broad-except
        results = {
            "error": _('An error occurred when searching for "{search_string}"').format(search_string=search_term)
        }
        log.exception(
            'Search view exception when searching for %s for user %s: %r',
            search_term,
            request.user.id,
            err
        )

    return JsonResponse(results, status=status_code)


def course_discovery(request):
    """
    Search for courses

    Args:
        request (required) - django request object

    Returns:
        http json response with the following fields
            "took" - how many seconds the operation took
            "total" - how many results were found
            "max_score" - maximum score from these resutls
            "results" - json array of result documents

            or

            "error" - displayable information about an error that occured on the server

    POST Params:
        "search_string" (optional) - text with which to search for courses
        "page_size" (optional)- how many results to return per page (defaults to 20, with maximum cutoff at 100)
        "page_index" (optional) - for which page (zero-indexed) to include results (defaults to 0)
    """
    results = {
        "error": _("Nothing to search")
    }
    status_code = 500

    search_term = request.POST.get("search_string", None)

    try:
        size, from_, page = _process_pagination_values(request)
        field_dictionary = _process_field_values(request)

        # Analytics - log search request
        track.emit(
            'edx.course_discovery.search.initiated',
            {
                "search_term": search_term,
                "page_size": size,
                "page_number": page,
            }
        )

        results = course_discovery_search(
            search_term=search_term,
            size=size,
            from_=from_,
            field_dictionary=field_dictionary,
        )

        # Analytics - log search results before sending to browser
        track.emit(
            'edx.course_discovery.search.results_displayed',
            {
                "search_term": search_term,
                "page_size": size,
                "page_number": page,
                "results_count": results["total"],
            }
        )

        status_code = 200

    except ValueError as invalid_err:
        results = {
            "error": six.text_type(invalid_err)
        }
        log.debug(six.text_type(invalid_err))

    except QueryParseError:
        results = {
            "error": _('Your query seems malformed. Check for unmatched quotes.')
        }

    # Allow for broad exceptions here - this is an entry point from external reference
    except Exception as err:  # pylint: disable=broad-except
        results = {
            "error": _('An error occurred when searching for "{search_string}"').format(search_string=search_term)
        }
        log.exception(
            'Search view exception when searching for %s for user %s: %r',
            search_term,
            request.user.id,
            err
        )

    return JsonResponse(results, status=status_code)


def _translate_hits(es_response):
    """ Provide resultset in our desired format from elasticsearch results """

    def translate_result(result):
        """ Any conversion from ES result syntax into our search engine syntax """
        translated_result = copy.copy(result)
        data = translated_result.pop("_source")

        translated_result.update({
            "data": data,
            "score": translated_result["_score"]
        })

        return translated_result

    def translate_facet(result):
        """ Any conversion from ES facet syntax into our search engine sytax """
        terms = {term["term"]: term["count"] for term in result["terms"]}
        return {
            "terms": terms,
            "total": result["total"],
            "other": result["other"],
        }

    results = [translate_result(hit) for hit in es_response["hits"]["hits"]]
    response = {
        "took": es_response["took"],
        "total": es_response["hits"]["total"],
        "max_score": es_response["hits"]["max_score"],
        "results": results,
    }

    if "facets" in es_response:
        response["facets"] = {facet: translate_facet(es_response["facets"][facet]) for facet in es_response["facets"]}

    return response


def _get_filter_field(field_name, field_value):
    """ Return field to apply into filter, if an array then use a range, otherwise look for a term match """
    filter_field = None
    if isinstance(field_value, ValueRange):
        range_values = {}
        if field_value.lower:
            range_values.update({"gte": field_value.lower_string})
        if field_value.upper:
            range_values.update({"lte": field_value.upper_string})
        filter_field = {
            "range": {
                field_name: range_values
            }
        }
    elif _is_iterable(field_value):
        filter_field = {
            "terms": {
                field_name: field_value
            }
        }
    else:
        filter_field = {
            "term": {
                field_name: field_value
            }
        }
    return filter_field

def _get_filter_field ( field_name , field_value ) : filter_field = None if isinstance ( field_value , ValueRange ) : range_values = { } if field_value . lower : range_values . update ( { "gte" : field_value . lower_string } ) if field_value . upper : range_values . update ( { "lte" : field_value . upper_string } ) filter_field = { "range" : { field_name : range_values } } elif _is_iterable ( field_value ) : filter_field = { "terms" : { field_name : field_value } } else : filter_field = { "term" : { field_name : field_value } } return filter_field

Return field to apply into filter if an array then use a range otherwise look for a term match


def _process_field_queries(field_dictionary):
    """
    We have a field_dictionary - we want to match the values for an elasticsearch "match" query
    This is only potentially useful when trying to tune certain search operations
    """
    def field_item(field):
        """ format field match as "match" item for elasticsearch query """
        return {
            "match": {
                field: field_dictionary[field]
            }
        }

    return [field_item(field) for field in field_dictionary]

We have a field_dictionary - we want to match the values for an elasticsearch match query This is only potentially useful when trying to tune certain search operations


def _process_filters(filter_dictionary):
    """
    We have a filter_dictionary - this means that if the field is included
    and matches, then we can include, OR if the field is undefined, then we
    assume it is safe to include
    """
    def filter_item(field):
        """ format elasticsearch filter to pass if value matches OR field is not included """
        if filter_dictionary[field] is not None:
            return {
                "or": [
                    _get_filter_field(field, filter_dictionary[field]),
                    {
                        "missing": {
                            "field": field
                        }
                    }
                ]
            }

        return {
            "missing": {
                "field": field
            }
        }

    return [filter_item(field) for field in filter_dictionary]

We have a filter_dictionary - this means that if the field is included and matches then we can include OR if the field is undefined then we assume it is safe to include


def _process_exclude_dictionary(exclude_dictionary):
    """
    Based on values in the exclude_dictionary generate a list of term queries that
    will filter out unwanted results.
    """
    # not_properties will hold the generated term queries.
    not_properties = []
    for exclude_property in exclude_dictionary:
        exclude_values = exclude_dictionary[exclude_property]
        if not isinstance(exclude_values, list):
            exclude_values = [exclude_values]
        not_properties.extend([{"term": {exclude_property: exclude_value}} for exclude_value in exclude_values])

    # Returning a query segment with an empty list freaks out ElasticSearch,
    #   so just return an empty segment.
    if not not_properties:
        return {}

    return {
        "not": {
            "filter": {
                "or": not_properties
            }
        }
    }

Based on values in the exclude_dictionary generate a list of term queries that will filter out unwanted results .


def _process_facet_terms(facet_terms):
    """ We have a list of terms with which we return facets """
    elastic_facets = {}
    for facet in facet_terms:
        facet_term = {"field": facet}
        if facet_terms[facet]:
            for facet_option in facet_terms[facet]:
                facet_term[facet_option] = facet_terms[facet][facet_option]

        elastic_facets[facet] = {
            "terms": facet_term
        }

    return elastic_facets

def _process_facet_terms ( facet_terms ) : elastic_facets = { } for facet in facet_terms : facet_term = { "field" : facet } if facet_terms [ facet ] : for facet_option in facet_terms [ facet ] : facet_term [ facet_option ] = facet_terms [ facet ] [ facet_option ] elastic_facets [ facet ] = { "terms" : facet_term } return elastic_facets


def get_mappings(cls, index_name, doc_type):
        """ fetch mapped-items structure from cache """
        return cache.get(cls.get_cache_item_name(index_name, doc_type), {})

def get_mappings ( cls , index_name , doc_type ) : return cache . get ( cls . get_cache_item_name ( index_name , doc_type ) , { } )

fetch mapped - items structure from cache


def set_mappings(cls, index_name, doc_type, mappings):
        """ set new mapped-items structure into cache """
        cache.set(cls.get_cache_item_name(index_name, doc_type), mappings)

def set_mappings ( cls , index_name , doc_type , mappings ) : cache . set ( cls . get_cache_item_name ( index_name , doc_type ) , mappings )

set new mapped - items structure into cache


def log_indexing_error(cls, indexing_errors):
        """ Logs indexing errors and raises a general ElasticSearch Exception"""
        indexing_errors_log = []
        for indexing_error in indexing_errors:
            indexing_errors_log.append(str(indexing_error))
        raise exceptions.ElasticsearchException(', '.join(indexing_errors_log))

def
log_indexing_error
(
cls
,
indexing_errors
)
:
indexing_errors_log
=
[
]
for
indexing_error
in
indexing_errors
:
indexing_errors_log
.
append
(
str
(
indexing_error
)
)
raise
exceptions
.
ElasticsearchException
(
', '
.
join
(
indexing_errors_log
)
)


def _get_mappings(self, doc_type):
        """
        Interfaces with the elasticsearch mappings for the index
        prevents multiple loading of the same mappings from ES when called more than once

        Mappings format in elasticsearch is as follows:
        {
           "doc_type": {
              "properties": {
                 "nested_property": {
                    "properties": {
                       "an_analysed_property": {
                          "type": "string"
                       },
                       "another_analysed_property": {
                          "type": "string"
                       }
                    }
                 },
                 "a_not_analysed_property": {
                    "type": "string",
                    "index": "not_analyzed"
                 },
                 "a_date_property": {
                    "type": "date"
                 }
              }
           }
        }

        We cache the properties of each doc_type, if they are not available, we'll load them again from Elasticsearch
        """
        # Try loading the mapping from the cache.
        mapping = ElasticSearchEngine.get_mappings(self.index_name, doc_type)

        # Fall back to Elasticsearch
        if not mapping:
            mapping = self._es.indices.get_mapping(
                index=self.index_name,
                doc_type=doc_type,
            ).get(self.index_name, {}).get('mappings', {}).get(doc_type, {})

            # Cache the mapping, if one was retrieved
            if mapping:
                ElasticSearchEngine.set_mappings(
                    self.index_name,
                    doc_type,
                    mapping
                )

        return mapping


def index(self, doc_type, sources, **kwargs):
        """
        Implements call to add documents to the ES index
        Note the call to _check_mappings which will setup fields with the desired mappings
        """

        try:
            actions = []
            for source in sources:
                self._check_mappings(doc_type, source)
                id_ = source['id'] if 'id' in source else None
                log.debug("indexing %s object with id %s", doc_type, id_)
                action = {
                    "_index": self.index_name,
                    "_type": doc_type,
                    "_id": id_,
                    "_source": source
                }
                actions.append(action)
            # bulk() returns a tuple with summary information
            # number of successfully executed actions and number of errors if stats_only is set to True.
            _, indexing_errors = bulk(
                self._es,
                actions,
                **kwargs
            )
            if indexing_errors:
                ElasticSearchEngine.log_indexing_error(indexing_errors)
        # Broad exception handler to protect around bulk call
        except Exception as ex:
            # log information and re-raise
            log.exception("error while indexing - %s", str(ex))
            raise


def remove(self, doc_type, doc_ids, **kwargs):
        """ Implements call to remove the documents from the index """

        try:
            # ignore is flagged as an unexpected-keyword-arg; ES python client documents that it can be used
            # pylint: disable=unexpected-keyword-arg
            actions = []
            for doc_id in doc_ids:
                log.debug("Removing document of type %s and index %s", doc_type, doc_id)
                action = {
                    '_op_type': 'delete',
                    "_index": self.index_name,
                    "_type": doc_type,
                    "_id": doc_id
                }
                actions.append(action)
            bulk(self._es, actions, **kwargs)
        except BulkIndexError as ex:
            valid_errors = [error for error in ex.errors if error['delete']['status'] != 404]

            if valid_errors:
                log.exception("An error occurred while removing documents from the index.")
                raise


def search(self,
               query_string=None,
               field_dictionary=None,
               filter_dictionary=None,
               exclude_dictionary=None,
               facet_terms=None,
               exclude_ids=None,
               use_field_match=False,
               **kwargs):  # pylint: disable=too-many-arguments, too-many-locals, too-many-branches, arguments-differ
        """
        Implements call to search the index for the desired content.

        Args:
            query_string (str): the string of values upon which to search within the
            content of the objects within the index

            field_dictionary (dict): dictionary of values which _must_ exist and
            _must_ match in order for the documents to be included in the results

            filter_dictionary (dict): dictionary of values which _must_ match if the
            field exists in order for the documents to be included in the results;
            documents for which the field does not exist may be included in the
            results if they are not otherwise filtered out

            exclude_dictionary(dict): dictionary of values all of which which must
            not match in order for the documents to be included in the results;
            documents which have any of these fields and for which the value matches
            one of the specified values shall be filtered out of the result set

            facet_terms (dict): dictionary of terms to include within search
            facets list - key is the term desired to facet upon, and the value is a
            dictionary of extended information to include. Supported right now is a
            size specification for a cap upon how many facet results to return (can
            be an empty dictionary to use default size for underlying engine):

            e.g.
            {
                "org": {"size": 10},  # only show top 10 organizations
                "modes": {}
            }

            use_field_match (bool): flag to indicate whether to use elastic
            filtering or elastic matching for field matches - this is nothing but a
            potential performance tune for certain queries

            (deprecated) exclude_ids (list): list of id values to exclude from the results -
            useful for finding maches that aren't "one of these"

        Returns:
            dict object with results in the desired format
            {
                "took": 3,
                "total": 4,
                "max_score": 2.0123,
                "results": [
                    {
                        "score": 2.0123,
                        "data": {
                            ...
                        }
                    },
                    {
                        "score": 0.0983,
                        "data": {
                            ...
                        }
                    }
                ],
                "facets": {
                    "org": {
                        "total": total_count,
                        "other": 1,
                        "terms": {
                            "MITx": 25,
                            "HarvardX": 18
                        }
                    },
                    "modes": {
                        "total": modes_count,
                        "other": 15,
                        "terms": {
                            "honor": 58,
                            "verified": 44,
                        }
                    }
                }
            }

        Raises:
            ElasticsearchException when there is a problem with the response from elasticsearch

        Example usage:
            .search(
                "find the words within this string",
                {
                    "must_have_field": "mast_have_value for must_have_field"
                },
                {

                }
            )
        """

        log.debug("searching index with %s", query_string)

        elastic_queries = []
        elastic_filters = []

        # We have a query string, search all fields for matching text within the "content" node
        if query_string:
            if six.PY2:
                query_string = query_string.encode('utf-8').translate(None, RESERVED_CHARACTERS)
            else:
                query_string = query_string.translate(query_string.maketrans('', '', RESERVED_CHARACTERS))
            elastic_queries.append({
                "query_string": {
                    "fields": ["content.*"],
                    "query": query_string
                }
            })

        if field_dictionary:
            if use_field_match:
                elastic_queries.extend(_process_field_queries(field_dictionary))
            else:
                elastic_filters.extend(_process_field_filters(field_dictionary))

        if filter_dictionary:
            elastic_filters.extend(_process_filters(filter_dictionary))

        # Support deprecated argument of exclude_ids
        if exclude_ids:
            if not exclude_dictionary:
                exclude_dictionary = {}
            if "_id" not in exclude_dictionary:
                exclude_dictionary["_id"] = []
            exclude_dictionary["_id"].extend(exclude_ids)

        if exclude_dictionary:
            elastic_filters.append(_process_exclude_dictionary(exclude_dictionary))

        query_segment = {
            "match_all": {}
        }
        if elastic_queries:
            query_segment = {
                "bool": {
                    "must": elastic_queries
                }
            }

        query = query_segment
        if elastic_filters:
            filter_segment = {
                "bool": {
                    "must": elastic_filters
                }
            }
            query = {
                "filtered": {
                    "query": query_segment,
                    "filter": filter_segment,
                }
            }

        body = {"query": query}
        if facet_terms:
            facet_query = _process_facet_terms(facet_terms)
            if facet_query:
                body["facets"] = facet_query

        try:
            es_response = self._es.search(
                index=self.index_name,
                body=body,
                **kwargs
            )
        except exceptions.ElasticsearchException as ex:
            message = six.text_type(ex)
            if 'QueryParsingException' in message:
                log.exception("Malformed search query: %s", message)
                raise QueryParseError('Malformed search query.')
            else:
                # log information and re-raise
                log.exception("error while searching index - %s", str(message))
                raise

        return _translate_hits(es_response)

Implements call to search the index for the desired content .


def perform_search(
        search_term,
        user=None,
        size=10,
        from_=0,
        course_id=None):
    """ Call the search engine with the appropriate parameters """
    # field_, filter_ and exclude_dictionary(s) can be overridden by calling application
    # field_dictionary includes course if course_id provided
    (field_dictionary, filter_dictionary, exclude_dictionary) = SearchFilterGenerator.generate_field_filters(
        user=user,
        course_id=course_id
    )

    searcher = SearchEngine.get_search_engine(getattr(settings, "COURSEWARE_INDEX_NAME", "courseware_index"))
    if not searcher:
        raise NoSearchEngineError("No search engine specified in settings.SEARCH_ENGINE")

    results = searcher.search_string(
        search_term,
        field_dictionary=field_dictionary,
        filter_dictionary=filter_dictionary,
        exclude_dictionary=exclude_dictionary,
        size=size,
        from_=from_,
        doc_type="courseware_content",
    )

    # post-process the result
    for result in results["results"]:
        result["data"] = SearchResultProcessor.process_result(result["data"], search_term, user)

    results["access_denied_count"] = len([r for r in results["results"] if r["data"] is None])
    results["results"] = [r for r in results["results"] if r["data"] is not None]

    return results


def course_discovery_search(search_term=None, size=20, from_=0, field_dictionary=None):
    """
    Course Discovery activities against the search engine index of course details
    """
    # We'll ignore the course-enrollemnt informaiton in field and filter
    # dictionary, and use our own logic upon enrollment dates for these
    use_search_fields = ["org"]
    (search_fields, _, exclude_dictionary) = SearchFilterGenerator.generate_field_filters()
    use_field_dictionary = {}
    use_field_dictionary.update({field: search_fields[field] for field in search_fields if field in use_search_fields})
    if field_dictionary:
        use_field_dictionary.update(field_dictionary)
    if not getattr(settings, "SEARCH_SKIP_ENROLLMENT_START_DATE_FILTERING", False):
        use_field_dictionary["enrollment_start"] = DateRange(None, datetime.utcnow())

    searcher = SearchEngine.get_search_engine(getattr(settings, "COURSEWARE_INDEX_NAME", "courseware_index"))
    if not searcher:
        raise NoSearchEngineError("No search engine specified in settings.SEARCH_ENGINE")

    results = searcher.search(
        query_string=search_term,
        doc_type="course_info",
        size=size,
        from_=from_,
        # only show when enrollment start IS provided and is before now
        field_dictionary=use_field_dictionary,
        # show if no enrollment end is provided and has not yet been reached
        filter_dictionary={"enrollment_end": DateRange(datetime.utcnow(), None)},
        exclude_dictionary=exclude_dictionary,
        facet_terms=course_discovery_facets(),
    )

    return results


def strings_in_dictionary(dictionary):
        """ Used by default implementation for finding excerpt """
        strings = [value for value in six.itervalues(dictionary) if not isinstance(value, dict)]
        for child_dict in [dv for dv in six.itervalues(dictionary) if isinstance(dv, dict)]:
            strings.extend(SearchResultProcessor.strings_in_dictionary(child_dict))
        return strings

def strings_in_dictionary ( dictionary ) : strings = [ value for value in six . itervalues ( dictionary ) if not isinstance ( value , dict ) ] for child_dict in [ dv for dv in six . itervalues ( dictionary ) if isinstance ( dv , dict ) ] : strings . extend ( SearchResultProcessor . strings_in_dictionary ( child_dict ) ) return strings


def find_matches(strings, words, length_hoped):
        """ Used by default property excerpt """
        lower_words = [w.lower() for w in words]

        def has_match(string):
            """ Do any of the words match within the string """
            lower_string = string.lower()
            for test_word in lower_words:
                if test_word in lower_string:
                    return True
            return False

        shortened_strings = [textwrap.wrap(s) for s in strings]
        short_string_list = list(chain.from_iterable(shortened_strings))
        matches = [ms for ms in short_string_list if has_match(ms)]

        cumulative_len = 0
        break_at = None
        for idx, match in enumerate(matches):
            cumulative_len += len(match)
            if cumulative_len >= length_hoped:
                break_at = idx
                break

        return matches[0:break_at]


def decorate_matches(match_in, match_word):
        """ decorate the matches within the excerpt """
        matches = re.finditer(match_word, match_in, re.IGNORECASE)
        for matched_string in set([match.group() for match in matches]):
            match_in = match_in.replace(
                matched_string,
                getattr(settings, "SEARCH_MATCH_DECORATION", u"<b>{}</b>").format(matched_string)
            )
        return match_in

def decorate_matches ( match_in , match_word ) : matches = re . finditer ( match_word , match_in , re . IGNORECASE ) for matched_string in set ( [ match . group ( ) for match in matches ] ) : match_in = match_in . replace ( matched_string , getattr ( settings , "SEARCH_MATCH_DECORATION" , u"<b>{}</b>" ) . format ( matched_string ) ) return match_in