#!/usr/bin/python
#
# Copyright (C) Christian Thurau, 2010.
# Licensed under the GNU General Public License (GPL).
# http://www.gnu.org/licenses/gpl.txt
"""
PyMF Singular Value Decomposition.

    SVD : Class for Singular Value Decomposition
    pinv() : Compute the pseudoinverse of a Matrix

"""

from numpy.linalg import eigh
import scipy.sparse

try:
    import scipy.sparse.linalg.eigen.arpack as linalg
except (ImportError, AttributeError):
    import scipy.sparse.linalg as linalg

import numpy as np

def pinv(A, k=-1, eps=10**-8):
    # Compute Pseudoinverse of a matrix
    # calculate SVD
    svd_mdl =  SVD(A, k=k)
    svd_mdl.factorize()

    S = svd_mdl.S
    Sdiag = S.diagonal()
    Sdiag = np.where(Sdiag >eps, 1.0/Sdiag, 0.0)

    for i in range(S.shape[0]):
        S[i,i] = Sdiag[i]

    if scipy.sparse.issparse(A):
        A_p = svd_mdl.V.T * (S *  svd_mdl.U.T)
    else:
        A_p = np.dot(svd_mdl.V.T, np.core.multiply(np.diag(S)[:,np.newaxis], svd_mdl.U.T))

    return A_p

class SVD():
    """
    SVD(data, show_progress=False)

    Singular Value Decomposition. Factorize a data matrix into three matrices s.t.
    F = | data - USV| is minimal. U and V correspond to eigenvectors of the matrices
    data*data.T and data.T*data.

    Parameters
    ----------
    data : array_like [data_dimension x num_samples]
        the input data

    Attributes
    ----------
        U,S,V : submatrices s.t. data = USV

    Example
    -------
    >>> import numpy as np
    >>> data = np.array([[1.0, 0.0, 2.0], [0.0, 1.0, 1.0]])
    >>> svd_mdl = SVD(data, show_progress=False)
    >>> svd_mdl.factorize()
    """

    _EPS=10**-8

    def __init__(self, data, k=-1, rrank=0, crank=0):
        self.data = data
        (self._rows, self._cols) = self.data.shape
        if rrank > 0:
            self._rrank = rrank
        else:
            self._rrank = self._rows

        if crank > 0:
            self._crank = crank
        else:
            self._crank = self._cols

        # set the rank to either rrank or crank
        self._k = k

    def frobenius_norm(self):
        """ Frobenius norm (||data - USV||) for a data matrix and a low rank
        approximation given by SVH using rank k for U and V

        Returns:
            frobenius norm: F = ||data - USV||
        """
        if scipy.sparse.issparse(self.data):
            err = self.data - self.U*self.S*self.V
            err = err.multiply(err)
            err = np.sqrt(err.sum())
        else:
            err = self.data[:,:] - np.dot(np.dot(self.U, self.S), self.V)
            err = np.sqrt(np.sum(err**2))

        return err

    def factorize(self):
        def _right_svd():
            AA = np.dot(self.data[:,:], self.data[:,:].T)
            values, u_vectors = eigh(AA)

            # get rid of too low eigenvalues
            u_vectors = u_vectors[:, values > self._EPS]
            values = values[values > self._EPS]

            # sort eigenvectors according to largest value
            idx = np.argsort(values)
            values = values[idx[::-1]]

            # argsort sorts in ascending order -> access is backwards
            self.U = u_vectors[:,idx[::-1]]

            # compute S
            self.S = np.diag(np.sqrt(values))

            # and the inverse of it
            S_inv = np.diag(np.sqrt(values)**-1)

            # compute V from it
            self.V = np.dot(S_inv, np.dot(self.U[:,:].T, self.data[:,:]))

        def _left_svd():
            AA = np.dot(self.data[:,:].T, self.data[:,:])
            values, v_vectors = eigh(AA)

            # get rid of too low eigenvalues
            v_vectors = v_vectors[:, values > self._EPS]
            values = values[values > self._EPS]

            # sort eigenvectors according to largest value
            # argsort sorts in ascending order -> access is backwards
            idx = np.argsort(values)[::-1]
            values = values[idx]

            # compute S
            self.S= np.diag(np.sqrt(values))

            # and the inverse of it
            S_inv = np.diag(1.0/np.sqrt(values))

            Vtmp = v_vectors[:,idx]

            self.U = np.dot(np.dot(self.data[:,:], Vtmp), S_inv)
            self.V = Vtmp.T

        def _sparse_right_svd():
            ## for some reasons arpack does not allow computation of rank(A) eigenvectors (??)    #
            AA = self.data*self.data.transpose()
            if self.data.shape[0] > 1:
                # do not compute full rank if desired
                if self._k > 0 and self._k < self.data.shape[0]-1:
                    k = self._k
                else:
                    k = self.data.shape[0]-1

                values, u_vectors = linalg.eigen_symmetric(AA,k=k)
            else:
                values, u_vectors = eigh(AA.todense())

            # get rid of too low eigenvalues
            u_vectors = u_vectors[:, values > self._EPS]
            values = values[values > self._EPS]

            # sort eigenvectors according to largest value
            idx = np.argsort(values)
            values = values[idx[::-1]]

            # argsort sorts in ascending order -> access is backwards
            self.U = scipy.sparse.csc_matrix(u_vectors[:,idx[::-1]])

            # compute S
            self.S = scipy.sparse.csc_matrix(np.diag(np.sqrt(values)))

            # and the inverse of it
            S_inv = scipy.sparse.csc_matrix(np.diag(1.0/np.sqrt(values)))

            # compute V from it
            self.V = self.U.transpose() * self.data
            self.V = S_inv * self.V

        def _sparse_left_svd():
            # for some reasons arpack does not allow computation of rank(A) eigenvectors (??)
            AA = self.data.transpose()*self.data

            if self.data.shape[1] > 1:
                # do not compute full rank if desired
                if self._k > 0 and self._k < self.data.shape[1]-1:
                    k = self._k
                else:
                    k = self.data.shape[1]-1
                values, v_vectors = linalg.eigen_symmetric(AA,k=k)
            else:
                values, v_vectors = eigh(AA.todense())
            # get rid of too low eigenvalues
            v_vectors = v_vectors[:, values > self._EPS]
            values = values[values > self._EPS]

            # sort eigenvectors according to largest value
            idx = np.argsort(values)
            values = values[idx[::-1]]

            # argsort sorts in ascending order -> access is backwards
            self.V = scipy.sparse.csc_matrix(v_vectors[:,idx[::-1]])

            # compute S
            self.S = scipy.sparse.csc_matrix(np.diag(np.sqrt(values)))

            # and the inverse of it
            S_inv = scipy.sparse.csc_matrix(np.diag(1.0/np.sqrt(values)))

            self.U = self.data * self.V * S_inv
            self.V = self.V.transpose()

        if self._rows > self._cols:
            if scipy.sparse.issparse(self.data):
                _sparse_left_svd()
            else:
                _left_svd()
        else:
            if scipy.sparse.issparse(self.data):
                _sparse_right_svd()
            else:
                _right_svd()

if __name__ == "__main__":
    import doctest
    doctest.testmod()


#!/usr/bin/python
#
# Copyright (C) Christian Thurau, 2010.
# Licensed under the GNU General Public License (GPL).
# http://www.gnu.org/licenses/gpl.txt
"""
PyMF Convex Matrix Factorization [1]

    CNMF(NMF) : Class for convex matrix factorization

[1] Ding, C., Li, T. and Jordan, M.. Convex and Semi-Nonnegative Matrix Factorizations.
IEEE Trans. on Pattern Analysis and Machine Intelligence 32(1), 45-55.
"""

import numpy as np
import logging
from .nmf import NMF
from .kmeans import Kmeans

__all__ = ["CNMF"]

class CNMF(NMF):
    """
    CNMF(data, num_bases=4)

    Convex NMF. Factorize a data matrix into two matrices s.t.
    F = | data - W*H | = | data - data*beta*H| is minimal. H and beta
    are restricted to convexity (beta >=0, sum(beta, axis=1) = [1 .. 1]).

    Parameters
    ----------
    data : array_like, shape (_data_dimension, _num_samples)
        the input data
    num_bases: int, optional
        Number of bases to compute (column rank of W and row rank of H).
        4 (default)

    Attributes
    ----------
    W : "data_dimension x num_bases" matrix of basis vectors
    H : "num bases x num_samples" matrix of coefficients
    ferr : frobenius norm (after calling .factorize())

    Example
    -------
    Applying CNMF to some rather stupid data set:

    >>> import numpy as np
    >>> from cnmf import CNMF
    >>> data = np.array([[1.0, 0.0, 2.0], [0.0, 1.0, 1.0]])
    >>> cnmf_mdl = CNMF(data, num_bases=2)
    >>> cnmf_mdl.factorize(niter=10)

    The basis vectors are now stored in cnmf_mdl.W, the coefficients in cnmf_mdl.H.
    To compute coefficients for an existing set of basis vectors simply    copy W
    to cnmf_mdl.W, and set compute_w to False:

    >>> data = np.array([[1.5, 1.3], [1.2, 0.3]])
    >>> W = [[1.0, 0.0], [0.0, 1.0]]
    >>> cnmf_mdl = CNMF(data, num_bases=2)
    >>> cnmf_mdl.W = W
    >>> cnmf_mdl.factorize(compute_w=False, niter=1)

    The result is a set of coefficients acnmf_mdl.H, s.t. data = W * cnmf_mdl.H.
    """

    # see .factorize() for the update of W and H
    # -> proper decoupling of W/H not possible ...
    def update_w(self):
        pass

    def update_h(self):
        pass

    def init_h(self):
        if not hasattr(self, 'H'):
            # init basic matrices
            self.H = np.zeros((self._num_bases, self._num_samples))

            # initialize using k-means
            km = Kmeans(self.data[:,:], num_bases=self._num_bases)
            km.factorize(niter=10)
            assign = km.assigned

            num_i = np.zeros(self._num_bases)
            for i in range(self._num_bases):
                num_i[i] = len(np.where(assign == i)[0])

            self.H.T[range(len(assign)), assign] = 1.0
            self.H += 0.2*np.ones((self._num_bases, self._num_samples))

        if not hasattr(self, 'G'):
            self.G = np.zeros((self._num_samples, self._num_bases))

            self.G[range(len(assign)), assign] = 1.0
            self.G += 0.01
            self.G /= np.tile(np.reshape(num_i[assign],(-1,1)), self.G.shape[1])

        if not hasattr(self,'W'):
            self.W = np.dot(self.data[:,:], self.G)

    def init_w(self):
        pass

    def factorize(self, niter=10, compute_w=True, compute_h=True,
                  compute_err=True, show_progress=False):
        """ Factorize s.t. WH = data

            Parameters
            ----------
            niter : int
                    number of iterations.
            show_progress : bool
                    print some extra information to stdout.
            compute_h : bool
                    iteratively update values for H.
            compute_w : bool
                    iteratively update values for W.
            compute_err : bool
                    compute Frobenius norm |data-WH| after each update and store
                    it to .ferr[k].

            Updated Values
            --------------
            .W : updated values for W.
            .H : updated values for H.
            .ferr : Frobenius norm |data-WH| for each iteration.
        """

        if not hasattr(self,'W'):
               self.init_w()

        if not hasattr(self,'H'):
                self.init_h()

        def separate_positive(m):
            return (np.abs(m) + m)/2.0

        def separate_negative(m):
            return (np.abs(m) - m)/2.0

        if show_progress:
            self._logger.setLevel(logging.INFO)
        else:
            self._logger.setLevel(logging.ERROR)

        XtX = np.dot(self.data[:,:].T, self.data[:,:])
        XtX_pos = separate_positive(XtX)
        XtX_neg = separate_negative(XtX)

        self.ferr = np.zeros(niter)
        # iterate over W and H

        for i in range(niter):
            # update H
            XtX_neg_x_W = np.dot(XtX_neg, self.G)
            XtX_pos_x_W = np.dot(XtX_pos, self.G)

            if compute_h:
                H_x_WT = np.dot(self.H.T, self.G.T)
                ha = XtX_pos_x_W + np.dot(H_x_WT, XtX_neg_x_W)
                hb = XtX_neg_x_W + np.dot(H_x_WT, XtX_pos_x_W) + 10**-9
                self.H = (self.H.T*np.sqrt(ha/hb)).T

            # update W
            if compute_w:
                HT_x_H = np.dot(self.H, self.H.T)
                wa = np.dot(XtX_pos, self.H.T) + np.dot(XtX_neg_x_W, HT_x_H)
                wb = np.dot(XtX_neg, self.H.T) + np.dot(XtX_pos_x_W, HT_x_H) + 10**-9

                self.G *= np.sqrt(wa/wb)
                self.W = np.dot(self.data[:,:], self.G)

            if compute_err:
                self.ferr[i] = self.frobenius_norm()
                self._logger.info('Iteration ' + str(i+1) + '/' + str(niter) +
                ' FN:' + str(self.ferr[i]))
            else:
                self._logger.info('Iteration ' + str(i+1) + '/' + str(niter))

            if i > 1 and compute_err:
                if self.converged(i):
                    self.ferr = self.ferr[:i]
                    break

if __name__ == "__main__":
    import doctest
    doctest.testmod()


# coding: utf-8
import numpy as np
from scipy.ndimage import filters

import msaf.utils as U
from msaf.algorithms.interface import SegmenterInterface
from msaf import pymf

def median_filter(X, M=8):
    """Median filter along the first axis of the feature matrix X."""
    for i in range(X.shape[1]):
        X[:, i] = filters.median_filter(X[:, i], size=M)
    return X

def cnmf(S, rank, niter=500, hull=False):
    """(Convex) Non-Negative Matrix Factorization.

    Parameters
    ----------
    S: np.array(p, N)
        Features matrix. p row features and N column observations.
    rank: int
        Rank of decomposition
    niter: int
        Number of iterations to be used

    Returns
    -------
    F: np.array
        Cluster matrix (decomposed matrix)
    G: np.array
        Activation matrix (decomposed matrix)
        (s.t. S ~= F * G)
    """
    if hull:
        nmf_mdl = pymf.CHNMF(S, num_bases=rank)
    else:
        nmf_mdl = pymf.CNMF(S, num_bases=rank)
    nmf_mdl.factorize(niter=niter)
    F = np.asarray(nmf_mdl.W)
    G = np.asarray(nmf_mdl.H)
    return F, G

def most_frequent(x):
    """Returns the most frequent value in x."""
    return np.argmax(np.bincount(x))

def compute_labels(X, rank, R, bound_idxs, niter=300):
    """Computes the labels using the bounds."""

    try:
        F, G = cnmf(X, rank, niter=niter, hull=False)
    except:
        return [1]

    label_frames = filter_activation_matrix(G.T, R)
    label_frames = np.asarray(label_frames, dtype=int)

    #labels = [label_frames[0]]
    labels = []
    bound_inters = zip(bound_idxs[:-1], bound_idxs[1:])
    for bound_inter in bound_inters:
        if bound_inter[1] - bound_inter[0] <= 0:
            labels.append(np.max(label_frames) + 1)
        else:
            labels.append(most_frequent(
                label_frames[bound_inter[0]: bound_inter[1]]))
        #print bound_inter, labels[-1]
    #labels.append(label_frames[-1])

    return labels

def filter_activation_matrix(G, R):
    """Filters the activation matrix G, and returns a flattened copy."""

    #import pylab as plt
    #plt.imshow(G, interpolation="nearest", aspect="auto")
    #plt.show()

    idx = np.argmax(G, axis=1)
    max_idx = np.arange(G.shape[0])
    max_idx = (max_idx, idx.flatten())
    G[:, :] = 0
    G[max_idx] = idx + 1

    # TODO: Order matters?
    G = np.sum(G, axis=1)
    G = median_filter(G[:, np.newaxis], R)

    return G.flatten()

def get_segmentation(X, rank, R, rank_labels, R_labels, niter=300,
                     bound_idxs=None, in_labels=None):
    """
    Gets the segmentation (boundaries and labels) from the factorization
    matrices.

    Parameters
    ----------
    X: np.array()
        Features matrix (e.g. chromagram)
    rank: int
        Rank of decomposition
    R: int
        Size of the median filter for activation matrix
    niter: int
        Number of iterations for k-means
    bound_idxs : list
        Use previously found boundaries (None to detect them)
    in_labels : np.array()
        List of input labels (None to compute them)

    Returns
    -------
    bounds_idx: np.array
        Bound indeces found
    labels: np.array
        Indeces of the labels representing the similarity between segments.
    """

    #import pylab as plt
    #plt.imshow(X, interpolation="nearest", aspect="auto")
    #plt.show()

    # Find non filtered boundaries
    compute_bounds = True if bound_idxs is None else False
    while True:
        if bound_idxs is None:
            try:
                F, G = cnmf(X, rank, niter=niter, hull=False)
            except:
                return np.empty(0), [1]

            # Filter G
            G = filter_activation_matrix(G.T, R)
            if bound_idxs is None:
                bound_idxs = np.where(np.diff(G) != 0)[0] + 1

        # Increase rank if we found too few boundaries
        if compute_bounds and len(np.unique(bound_idxs)) <= 2:
            rank += 1
            bound_idxs = None
        else:
            break

    # Add first and last boundary
    bound_idxs = np.concatenate(([0], bound_idxs, [X.shape[1] - 1]))
    bound_idxs = np.asarray(bound_idxs, dtype=int)
    if in_labels is None:
        labels = compute_labels(X, rank_labels, R_labels, bound_idxs,
                                niter=niter)
    else:
        labels = np.ones(len(bound_idxs) - 1)

    #plt.imshow(G[:, np.newaxis], interpolation="nearest", aspect="auto")
    #for b in bound_idxs:
        #plt.axvline(b, linewidth=2.0, color="k")
    #plt.show()

    return bound_idxs, labels

class Segmenter(SegmenterInterface):
    """
    This script identifies the structure of a given track using a modified version
    of the C-NMF method described here:

    Nieto, O., Jehan, T., Convex Non-negative Matrix Factorization For Automatic
    Music Structure Identification. Proc. of the 38th IEEE International
    Conference on Acoustics, Speech, and Signal Processing (ICASSP).
    Vancouver, Canada, 2013 (`PDF`_).

    The modified version can be found in Oriol Nieto's `PhD Thesis`_.

    .. _PDF: http://marl.smusic.nyu.edu/nieto/publications/Nieto-ICASSP13.pdf
    .. _PhD Thesis: http://marl.smusic.nyu.edu/nieto/publications/Nieto-Dissertation.pdf
    """
    def processFlat(self):
        """Main process.
        Returns
        -------
        est_idxs : np.array(N)
            Estimated indeces for the segment boundaries in frames.
        est_labels : np.array(N-1)
            Estimated labels for the segments.
        """
        # C-NMF params
        niter = self.config["niters"]  # Iterations for the MF and clustering

        # Preprocess to obtain features, times, and input boundary indeces
        F = self._preprocess()

        # Normalize
        F = U.normalize(F, norm_type=self.config["norm_feats"])

        if F.shape[0] >= self.config["h"]:
            # Median filter
            F = median_filter(F, M=self.config["h"])
            #plt.imshow(F.T, interpolation="nearest", aspect="auto"); plt.show()

            # Find the boundary indices and labels using matrix factorization
            est_idxs, est_labels = get_segmentation(
                F.T, self.config["rank"], self.config["R"],
                self.config["rank_labels"], self.config["R_labels"],
                niter=niter, bound_idxs=self.in_bound_idxs, in_labels=None)

            # Remove empty segments if needed
            est_idxs, est_labels = U.remove_empty_segments(est_idxs, est_labels)
        else:
            # The track is too short. We will only output the first and last
            # time stamps
            if self.in_bound_idxs is None:
                est_idxs = np.array([0, F.shape[0] - 1])
                est_labels = [1]
            else:
                est_idxs = self.in_bound_idxs
                est_labels = [1] * (len(est_idxs) + 1)

        # Make sure that the first and last boundaries are included
        assert est_idxs[0] == 0 and est_idxs[-1] == F.shape[0] - 1

        # Post process estimations
        est_idxs, est_labels = self._postprocess(est_idxs, est_labels)

        return est_idxs, est_labels


"""
This module contains multiple functions in order to run MSAF algorithms.
"""
from copy import deepcopy
from joblib import Parallel, delayed
import librosa
import logging
import numpy as np
import os

import msaf
from msaf import input_output as io
from msaf import utils
from msaf import plotting
from msaf.features import Features
from msaf.exceptions import NoHierBoundaryError, NoAudioFileError
import msaf.algorithms as algorithms

def get_boundaries_module(boundaries_id):
    """Obtains the boundaries module given a boundary algorithm identificator.

    Parameters
    ----------
    boundaries_id: str
        Boundary algorithm identificator (e.g., foote, sf).

    Returns
    -------
    module: object
        Object containing the selected boundary module.
        None for "ground truth".
    """
    if boundaries_id == "gt":
        return None
    try:
        module = eval(algorithms.__name__ + "." + boundaries_id)
    except AttributeError:
        raise RuntimeError("Algorithm %s can not be found in msaf!" %
                           boundaries_id)
    if not module.is_boundary_type:
        raise RuntimeError("Algorithm %s can not identify boundaries!" %
                           boundaries_id)
    return module

def get_labels_module(labels_id):
    """Obtains the label module given a label algorithm identificator.

    Parameters
    ----------
    labels_id: str
        Label algorithm identificator (e.g., fmc2d, cnmf).

    Returns
    -------
    module: object
        Object containing the selected label module.
        None for not computing the labeling part of music segmentation.
    """
    if labels_id is None:
        return None
    try:
        module = eval(algorithms.__name__ + "." + labels_id)
    except AttributeError:
        raise RuntimeError("Algorithm %s can not be found in msaf!" %
                           labels_id)
    if not module.is_label_type:
        raise RuntimeError("Algorithm %s can not label segments!" %
                           labels_id)
    return module

def run_hierarchical(audio_file, bounds_module, labels_module, frame_times,
                     config, annotator_id=0):
    """Runs hierarchical algorithms with the specified identifiers on the
    audio_file. See run_algorithm for more information.
    """
    # Sanity check
    if bounds_module is None:
        raise NoHierBoundaryError("A boundary algorithm is needed when using "
                                  "hierarchical segmentation.")

    # Get features to make code nicer
    features = config["features"].features

    # Compute boundaries
    S = bounds_module.Segmenter(audio_file, **config)
    est_idxs, est_labels = S.processHierarchical()

    # Compute labels if needed
    if labels_module is not None and \
            bounds_module.__name__ != labels_module.__name__:
        # Compute labels for each level in the hierarchy
        flat_config = deepcopy(config)
        flat_config["hier"] = False
        for i, level_idxs in enumerate(est_idxs):
            S = labels_module.Segmenter(audio_file,
                                        in_bound_idxs=level_idxs,
                                        **flat_config)
            est_labels[i] = S.processFlat()[1]

    # Make sure the first and last boundaries are included for each
    # level in the hierarchy
    est_times = []
    cleaned_est_labels = []
    for level in range(len(est_idxs)):
        est_level_times, est_level_labels = \
            utils.process_segmentation_level(
                est_idxs[level], est_labels[level], features.shape[0],
                frame_times, config["features"].dur)
        est_times.append(est_level_times)
        cleaned_est_labels.append(est_level_labels)
    est_labels = cleaned_est_labels

    return est_times, est_labels

def run_flat(file_struct, bounds_module, labels_module, frame_times, config,
             annotator_id):
    """Runs the flat algorithms with the specified identifiers on the
    audio_file. See run_algorithm for more information.
    """
    # Get features to make code nicer
    features = config["features"].features

    # Segment using the specified boundaries and labels
    # Case when boundaries and labels algorithms are the same
    if bounds_module is not None and labels_module is not None and \
            bounds_module.__name__ == labels_module.__name__:
        S = bounds_module.Segmenter(file_struct, **config)
        est_idxs, est_labels = S.processFlat()
    # Different boundary and label algorithms
    else:
        # Identify segment boundaries
        if bounds_module is not None:
            S = bounds_module.Segmenter(file_struct, in_labels=[], **config)
            est_idxs, est_labels = S.processFlat()
        else:
            try:
                # Ground-truth boundaries
                est_times, est_labels = io.read_references(
                    file_struct.audio_file, annotator_id=annotator_id)
                est_idxs = io.align_times(est_times, frame_times)
                if est_idxs[0] != 0:
                    est_idxs = np.concatenate(([0], est_idxs))
            except IOError:
                logging.warning("No references found for file: %s" %
                                file_struct.audio_file)
                return [], []

        # Label segments
        if labels_module is not None:
            if len(est_idxs) == 2:
                est_labels = np.array([0])
            else:
                S = labels_module.Segmenter(file_struct,
                                            in_bound_idxs=est_idxs,
                                            **config)
                est_labels = S.processFlat()[1]

    # Make sure the first and last boundaries are included
    est_times, est_labels = utils.process_segmentation_level(
        est_idxs, est_labels, features.shape[0], frame_times,
        config["features"].dur)

    return est_times, est_labels

def run_algorithms(file_struct, boundaries_id, labels_id, config,
                   annotator_id=0):
    """Runs the algorithms with the specified identifiers on the audio_file.

    Parameters
    ----------
    file_struct: `msaf.io.FileStruct`
        Object with the file paths.
    boundaries_id: str
        Identifier of the boundaries algorithm to use ("gt" for ground truth).
    labels_id: str
        Identifier of the labels algorithm to use (None for not labeling).
    config: dict
        Dictionary containing the custom parameters of the algorithms to use.
    annotator_id: int
        Annotator identificator in the ground truth.

    Returns
    -------
    est_times: np.array or list
        List of estimated times for the segment boundaries.
        If `list`, it will be a list of np.arrays, sorted by segmentation
        layer.
    est_labels: np.array or list
        List of all the labels associated segments.
        If `list`, it will be a list of np.arrays, sorted by segmentation
        layer.
    """
    # Check that there are enough audio frames
    if config["features"].features.shape[0] <= msaf.config.minimum_frames:
        logging.warning("Audio file too short, or too many few beats "
                        "estimated. Returning empty estimations.")
        return np.asarray([0, config["features"].dur]), \
            np.asarray([0], dtype=int)

    # Get the corresponding modules
    bounds_module = get_boundaries_module(boundaries_id)
    labels_module = get_labels_module(labels_id)

    # Get the correct frame times
    frame_times = config["features"].frame_times

    # Segment audio based on type of segmentation
    run_fun = run_hierarchical if config["hier"] else run_flat
    est_times, est_labels = run_fun(file_struct, bounds_module, labels_module,
                                    frame_times, config, annotator_id)

    return est_times, est_labels

def process_track(file_struct, boundaries_id, labels_id, config,
                  annotator_id=0):
    """Prepares the parameters, runs the algorithms, and saves results.

    Parameters
    ----------
    file_struct: `msaf.io.FileStruct`
        FileStruct containing the paths of the input files (audio file,
        features file, reference file, output estimation file).
    boundaries_id: str
        Identifier of the boundaries algorithm to use ("gt" for ground truth).
    labels_id: str
        Identifier of the labels algorithm to use (None for not labeling).
    config: dict
        Dictionary containing the custom parameters of the algorithms to use.
    annotator_id: int
        Annotator identificator in the ground truth.

    Returns
    -------
    est_times: np.array
        List of estimated times for the segment boundaries.
    est_labels: np.array
        List of all the labels associated segments.
    """
    logging.info("Segmenting %s" % file_struct.audio_file)

    # Get features
    config["features"] = Features.select_features(
        config["feature"], file_struct, config["annot_beats"],
        config["framesync"])

    # Get estimations
    est_times, est_labels = run_algorithms(file_struct,
                                           boundaries_id, labels_id, config,
                                           annotator_id=annotator_id)

    # Save
    logging.info("Writing results in: %s" % file_struct.est_file)
    io.save_estimations(file_struct, est_times, est_labels,
                        boundaries_id, labels_id, **config)

    return est_times, est_labels

def process(in_path, annot_beats=False, feature="pcp", framesync=False,
            boundaries_id=msaf.config.default_bound_id,
            labels_id=msaf.config.default_label_id, hier=False,
            sonify_bounds=False, plot=False, n_jobs=4, annotator_id=0,
            config=None, out_bounds="out_bounds.wav", out_sr=22050):
    """Main process to segment a file or a collection of files.

    Parameters
    ----------
    in_path: str
        Input path. If a directory, MSAF will function in collection mode.
        If audio file, MSAF will be in single file mode.
    annot_beats: bool
        Whether to use annotated beats or not.
    feature: str
        String representing the feature to be used (e.g. pcp, mfcc, tonnetz)
    framesync: str
        Whether to use framesync features or not (default: False -> beatsync)
    boundaries_id: str
        Identifier of the boundaries algorithm (use "gt" for groundtruth)
    labels_id: str
        Identifier of the labels algorithm (use None to not compute labels)
    hier : bool
        Whether to compute a hierarchical or flat segmentation.
    sonify_bounds: bool
        Whether to write an output audio file with the annotated boundaries
        or not (only available in Single File Mode).
    plot: bool
        Whether to plot the boundaries and labels against the ground truth.
    n_jobs: int
        Number of processes to run in parallel. Only available in collection
        mode.
    annotator_id: int
        Annotator identificator in the ground truth.
    config: dict
        Dictionary containing custom configuration parameters for the
        algorithms.  If None, the default parameters are used.
    out_bounds: str
        Path to the output for the sonified boundaries (only in single file
        mode, when sonify_bounds is True.
    out_sr : int
        Sampling rate for the sonified bounds.

    Returns
    -------
    results : list
        List containing tuples of (est_times, est_labels) of estimated
        boundary times and estimated labels.
        If labels_id is None, est_labels will be a list of -1.
    """
    # Seed random to reproduce results
    np.random.seed(123)

    # Set up configuration based on algorithms parameters
    if config is None:
        config = io.get_configuration(feature, annot_beats, framesync,
                                      boundaries_id, labels_id)
        config["features"] = None

    # Save multi-segment (hierarchical) configuration
    config["hier"] = hier
    if not os.path.exists(in_path):
        raise NoAudioFileError("File or directory does not exists, %s" %
                               in_path)
    if os.path.isfile(in_path):
        # Single file mode
        # Get (if they exitst) or compute features
        file_struct = msaf.io.FileStruct(in_path)

        # Use temporary file in single mode
        file_struct.features_file = msaf.config.features_tmp_file

        # Get features
        config["features"] = Features.select_features(
            feature, file_struct, annot_beats, framesync)

        # And run the algorithms
        est_times, est_labels = run_algorithms(file_struct, boundaries_id,
                                               labels_id, config,
                                               annotator_id=annotator_id)

        if sonify_bounds:
            logging.info("Sonifying boundaries in %s..." % out_bounds)
            audio_hq, sr = librosa.load(in_path, sr=out_sr)
            utils.sonify_clicks(audio_hq, est_times, out_bounds, out_sr)

        if plot:
            plotting.plot_one_track(file_struct, est_times, est_labels,
                                    boundaries_id, labels_id)

        # TODO: Only save if needed
        # Save estimations
        msaf.utils.ensure_dir(os.path.dirname(file_struct.est_file))
        io.save_estimations(file_struct, est_times, est_labels,
                            boundaries_id, labels_id, **config)

        return est_times, est_labels
    else:
        # Collection mode
        file_structs = io.get_dataset_files(in_path)

        return Parallel(n_jobs=n_jobs)(delayed(process_track)(
            file_struct, boundaries_id, labels_id, config,
            annotator_id=annotator_id) for file_struct in file_structs[:])


#!/usr/bin/python
#
# Copyright (C) Christian Thurau, 2010.
# Licensed under the GNU General Public License (GPL).
# http://www.gnu.org/licenses/gpl.txt
"""
PyMF Archetypal Analysis [1]

    AA: class for Archetypal Analysis

[1] Cutler, A. Breiman, L. (1994), "Archetypal Analysis", Technometrics 36(4),
338-347.
"""

import numpy as np
from .dist import vq
from cvxopt import solvers, base

from .svd import pinv
from .nmf import NMF

__all__ = ["AA"]

class AA(NMF):
    """
    AA(data, num_bases=4)

    Archetypal Analysis. Factorize a data matrix into two matrices s.t.
    F = | data - W*H | = | data - data*beta*H| is minimal. H and beta
    are restricted to convexity (beta >=0, sum(beta, axis=1) = [1 .. 1]).
    Factorization is solved via an alternating least squares optimization
    using the quadratic programming solver from cvxopt.

    Parameters
    ----------
    data : array_like, shape (_data_dimension, _num_samples)
        the input data
    num_bases: int, optional
        Number of bases to compute (column rank of W and row rank of H).
        4 (default)

    Attributes
    ----------
    W : "data_dimension x num_bases" matrix of basis vectors
    H : "num bases x num_samples" matrix of coefficients
    beta : "num_bases x num_samples" matrix of basis vector coefficients
        (for constructing W s.t. W = beta * data.T )
    ferr : frobenius norm (after calling .factorize())

    Example
    -------
    Applying AA to some rather stupid data set:

    >>> import numpy as np
    >>> from aa import AA
    >>> data = np.array([[1.0, 0.0, 2.0], [0.0, 1.0, 1.0]])

    Use 2 basis vectors -> W shape(data_dimension, 2).

    >>> aa_mdl = AA(data, num_bases=2)

    Set number of iterations to 5 and start computing the factorization.

    >>> aa_mdl.factorize(niter=5)

    The basis vectors are now stored in aa_mdl.W, the coefficients in aa_mdl.H.
    To compute coefficients for an existing set of basis vectors simply copy W
    to aa_mdl.W, and set compute_w to False:

    >>> data = np.array([[1.5], [1.2]])
    >>> W = np.array([[1.0, 0.0], [0.0, 1.0]])
    >>> aa_mdl = AA(data, num_bases=2)
    >>> aa_mdl.W = W
    >>> aa_mdl.factorize(niter=5, compute_w=False)

    The result is a set of coefficients aa_mdl.H, s.t. data = W * aa_mdl.H.
    """
    # set cvxopt options
    solvers.options['show_progress'] = False

    def init_h(self):
        self.H = np.random.random((self._num_bases, self._num_samples))
        self.H /= self.H.sum(axis=0)

    def init_w(self):
        self.beta = np.random.random((self._num_bases, self._num_samples))
        self.beta /= self.beta.sum(axis=0)
        self.W = np.dot(self.beta, self.data.T).T
        self.W = np.random.random((self._data_dimension, self._num_bases))

    def update_h(self):
        """ alternating least squares step, update H under the convexity
        constraint """
        def update_single_h(i):
            """ compute single H[:,i] """
            # optimize alpha using qp solver from cvxopt
            FA = base.matrix(np.float64(np.dot(-self.W.T, self.data[:,i])))
            al = solvers.qp(HA, FA, INQa, INQb, EQa, EQb)
            self.H[:,i] = np.array(al['x']).reshape((1, self._num_bases))

        EQb = base.matrix(1.0, (1,1))
        # float64 required for cvxopt
        HA = base.matrix(np.float64(np.dot(self.W.T, self.W)))
        INQa = base.matrix(-np.eye(self._num_bases))
        INQb = base.matrix(0.0, (self._num_bases,1))
        EQa = base.matrix(1.0, (1, self._num_bases))

        for i in range(self._num_samples):
            update_single_h(i)

    def update_w(self):
        """ alternating least squares step, update W under the convexity
        constraint """
        def update_single_w(i):
            """ compute single W[:,i] """
            # optimize beta     using qp solver from cvxopt
            FB = base.matrix(np.float64(np.dot(-self.data.T, W_hat[:,i])))
            be = solvers.qp(HB, FB, INQa, INQb, EQa, EQb)
            self.beta[i,:] = np.array(be['x']).reshape((1, self._num_samples))

        # float64 required for cvxopt
        HB = base.matrix(np.float64(np.dot(self.data[:,:].T, self.data[:,:])))
        EQb = base.matrix(1.0, (1, 1))
        W_hat = np.dot(self.data, pinv(self.H))
        INQa = base.matrix(-np.eye(self._num_samples))
        INQb = base.matrix(0.0, (self._num_samples, 1))
        EQa = base.matrix(1.0, (1, self._num_samples))

        for i in range(self._num_bases):
            update_single_w(i)

        self.W = np.dot(self.beta, self.data.T).T

if __name__ == "__main__":
    import doctest
    doctest.testmod()


# -*- coding: utf-8 -*-
"""
Copyright: 2014 Sang Han

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys

from argparse import ArgumentParser, RawDescriptionHelpFormatter, FileType, REMAINDER
from functools import partial

from .__version__ import __version__, __build__
from .translator import translator
from .coroutines import spool, source, set_task
from .languages import print_table, translation_table

__all__ = []

class TermWidthHelpFormatter(RawDescriptionHelpFormatter):

    def __init__(self,  *args, **kwargs):
        super(TermWidthHelpFormatter, self).__init__(
            width=120, max_help_position=35, *args, **kwargs
        )

def command_line():

    description = 'A Translation Tool for Humans'
    version     = ' '.join([__version__, __build__])
    table       = sorted(translation_table('en').keys())

    codes = ArgumentParser(add_help=False)
    codes.add_argument(
        '-l', '--list',
        nargs='?',
        default=False,
        const='en',
        metavar='code',
        dest='code',
        help=' '.join(
            [
                'Enumerate the name of country and language code pair.',
                'Optionally specify output language',
            ])
        )

    # Preparse Language Code Flag
    language,_ = codes.parse_known_args()
    if language.code:
        print_table(language.code)
        exit(0)

    # Main Parser
    parser = ArgumentParser(
        parents=[codes],
        prog='translate',
        description=description,
        formatter_class=TermWidthHelpFormatter,
    )
    parser.add_argument(
        '--translit',
        action='store_true',
        help='Print out the transliteration of the text',
    )
    parser.add_argument(
        '-v', '--version',
        action='version',
        version="%s v%s" % ('translate', version)
    )
    parser.add_argument(
        'source',
        nargs='?',
        default=None,
        help='Source language code',
        metavar='source',
    )
    parser.add_argument(
        'dest',
        type=str,
        choices=table,
        help='Destination language code',
        metavar='target',
    )
    parser.add_argument(
        'text',
        nargs='?',
        default=sys.stdin,
        type=FileType('r'),
        help='Text file',
        metavar='file',
    )

    return parser.parse_args()

def main():
    '''
    Main Entry point for translator and argument parser
    '''
    args      = command_line()
    translate = partial(translator, args.source, args.dest,
                        version=' '.join([__version__, __build__]))

    return source(spool(set_task(translate, translit=args.translit)), args.text)

if __name__ == '__main__':
    sys.exit(main())


# -*- coding: utf-8 -*-
"""
coroutines
~~~~~~~~~~

All functions definied within this file are essentially coroutines or
helper functions for working with coroutines.

Therefore members follow the coroutine pattern either as consumers, producers or
consumer/producers
"""

from __future__ import print_function

import sys
import operator

from functools import wraps, partial, reduce
from concurrent.futures import ThreadPoolExecutor

__all__ = ['coroutine', 'spool', 'source', 'set_task', 'write_stream', 'accumulator']

def coroutine(func):
    """
    Initializes coroutine essentially priming it to the yield statement.
    Used as a decorator over functions that generate coroutines.

    .. code-block:: python

        # Basic coroutine producer/consumer pattern
        from translate import coroutine

        @coroutine
        def coroutine_foo(bar):
            try:
                while True:
                    baz = (yield)
                    bar.send(baz)

            except GeneratorExit:
                bar.close()

    :param func: Unprimed Generator
    :type func: Function

    :return: Initialized Coroutine
    :rtype: Function
    """

    @wraps(func)
    def initialization(*args, **kwargs):

        start = func(*args, **kwargs)
        next(start)

        return start

    return initialization

def accumulator(init, update):
    """
    Generic accumulator function.

    .. code-block:: python

        # Simplest Form
        >>> a = 'this' + ' '
        >>> b = 'that'
        >>> c = functools.reduce(accumulator, a, b)
        >>> c
        'this that'

        # The type of the initial value determines output type.
        >>> a = 5
        >>> b = Hello
        >>> c = functools.reduce(accumulator, a, b)
        >>> c
        10

    :param init:  Initial Value
    :param update: Value to accumulate

    :return: Combined Values
    """
    return (
        init + len(update)
            if isinstance(init, int) else
        init + update
    )

def write_stream(script, output='trans'):
    """
    :param script: Translated Text
    :type script: Iterable

    :param output: Output Type (either 'trans' or 'translit')
    :type output: String
    """
    first    = operator.itemgetter(0)
    sentence, _ = script
    printer  = partial(print, file=sys.stdout, end='')

    for line in sentence:
        if isinstance(first(line), str):
            printer(first(line))
        else:
            printer(first(line).encode('UTF-8'))

    printer('\n')

    return sys.stdout.flush()

# TODO: Get rid of all this context crap
@coroutine
def set_task(translator, translit=False):
    """
    Task Setter Coroutine

    End point destination coroutine of a purely consumer type.
    Delegates Text IO to the `write_stream` function.

    :param translation_function: Translator
    :type translation_function: Function

    :param translit: Transliteration Switch
    :type translit: Boolean
    """
    # Initialize Task Queue
    task    = str()
    queue   = list()

    # Function Partial
    output  = ('translit' if translit else 'trans')
    stream  = partial(write_stream, output=output)
    workers = ThreadPoolExecutor(max_workers=8)

    try:
        while True:

            task = yield
            queue.append(task)

    except GeneratorExit:
        list(map(stream, workers.map(translator, queue)))

@coroutine
def spool(iterable, maxlen=1250):
    """
    Consumes text streams and spools them together for more io
    efficient processes.

    :param iterable: Sends text stream for further processing
    :type iterable: Coroutine

    :param maxlen: Maximum query string size
    :type maxlen: Integer
    """
    words = int()
    text  = str()

    try:
        while True:

            while words < maxlen:
                stream = yield
                text   = reduce(accumulator, stream, text)
                words  = reduce(accumulator, stream, words)

            iterable.send(text)
            words = int()
            text  = str()

    except GeneratorExit:
        iterable.send(text)
        iterable.close()

def source(target, inputstream=sys.stdin):
    """
    Coroutine starting point. Produces text stream and forwards to consumers

    :param target: Target coroutine consumer
    :type target: Coroutine

    :param inputstream: Input Source
    :type inputstream: BufferedTextIO Object
    """
    for line in inputstream:

        while len(line) > 600:
            init, sep, line = line.partition(' ')
            assert len(init) <= 600
            target.send(''.join([init, sep]))

        target.send(line)

    inputstream.close()

    return target.close()


# -*- coding: utf-8 -*-
"""
translator
~~~~~~~~~~

Defines the interaction with the translation service.
Since the program interfaces with the google web service, this
module deals with the client side logic of pushing the translation request
to the the server.
"""

import functools
import requests
import re
import json

from requests import Request, Session
from requests.adapters import HTTPAdapter

__all__ = ['push_url', 'translator']

def push_url(interface):
    '''
    Decorates a function returning the url of translation API.
    Creates and maintains HTTP connection state

    Returns a dict response object from the server containing the translated
    text and metadata of the request body

    :param interface: Callable Request Interface
    :type interface: Function
    '''

    @functools.wraps(interface)
    def connection(*args, **kwargs):
        """
        Extends and wraps a HTTP interface.

        :return: Response Content
        :rtype: Dictionary
        """
        session = Session()
        session.mount('http://',  HTTPAdapter(max_retries=2))
        session.mount('https://', HTTPAdapter(max_retries=2))

        request  = Request(**interface(*args, **kwargs))
        prepare  = session.prepare_request(request)
        response = session.send(prepare, verify=True)

        if response.status_code != requests.codes.ok:
            response.raise_for_status()

        cleanup = re.subn(r',(?=,)', '', response.content.decode('utf-8'))[0]

        return json.loads(cleanup.replace(r'\xA0', r' ').replace('[,', '[1,'), encoding='UTF-8')

    return connection

@push_url
def translator(source, target, phrase, version='0.0 test', charset='utf-8'):
    """
    Returns the url encoded string that will be pushed to the translation
    server for parsing.

    List of acceptable language codes for source and target languages
    can be found as a JSON file in the etc directory.

    Some source languages are limited in scope of the possible target languages
    that are available.

    .. code-block:: python

        >>> from translate import translator
        >>> translator('en', 'zh-TW', 'Hello World!')
            '你好世界！'

    :param source: Language code for translation source
    :type source: String

    :param target: Language code that source will be translate into
    :type target: String

    :param phrase: Text body string that will be url encoded and translated
    :type phrase: String

    :return: Request Interface
    :rtype: Dictionary
    """

    url     = 'https://translate.google.com/translate_a/single'
    agent   = 'User-Agent',   'py-translate v{}'.format(version)
    content = 'Content-Type', 'application/json; charset={}'.format(charset)

    params  = {'client': 'a', 'ie': charset, 'oe': charset,
                   'dt': 't', 'sl':  source, 'tl':  target,  'q': phrase}

    request = {'method': 'GET',
                  'url': url,
               'params': params,
              'headers': dict([agent, content])}

    return request


# -*- coding: utf-8 -*-
from __future__ import print_function

import json
import operator

from os.path import dirname, abspath, join, isfile

__all__ = ['translation_table', 'print_table']

def translation_table(language, filepath='supported_translations.json'):
    '''
    Opens up file located under the etc directory containing language
    codes and prints them out.

    :param file: Path to location of json file
    :type file: str

    :return: language codes
    :rtype: dict
    '''
    fullpath = abspath(join(dirname(__file__), 'etc', filepath))

    if not isfile(fullpath):
        raise IOError('File does not exist at {0}'.format(fullpath))

    with open(fullpath, 'rt') as fp:
        raw_data = json.load(fp).get(language, None)
        assert(raw_data is not None)

    return dict((code['language'], code['name']) for code in raw_data)

def print_table(language):
    '''
    Generates a formatted table of language codes
    '''
    table = translation_table(language)

    for code, name in sorted(table.items(), key=operator.itemgetter(0)):
        print(u'{language:<8} {name:\u3000<20}'.format(
            name=name, language=code
        ))

    return None


import pandas as pd

def remove_nodes(network, rm_nodes):
    """
    Create DataFrames of nodes and edges that do not include specified nodes.

    Parameters
    ----------
    network : pandana.Network
    rm_nodes : array_like
        A list, array, Index, or Series of node IDs that should *not*
        be saved as part of the Network.

    Returns
    -------
    nodes, edges : pandas.DataFrame

    """
    rm_nodes = set(rm_nodes)
    ndf = network.nodes_df
    edf = network.edges_df

    nodes_to_keep = ~ndf.index.isin(rm_nodes)
    edges_to_keep = ~(edf['from'].isin(rm_nodes) | edf['to'].isin(rm_nodes))

    return ndf.loc[nodes_to_keep], edf.loc[edges_to_keep]

def network_to_pandas_hdf5(network, filename, rm_nodes=None):
    """
    Save a Network's data to a Pandas HDFStore.

    Parameters
    ----------
    network : pandana.Network
    filename : str
    rm_nodes : array_like
        A list, array, Index, or Series of node IDs that should *not*
        be saved as part of the Network.

    """
    if rm_nodes is not None:
        nodes, edges = remove_nodes(network, rm_nodes)
    else:
        nodes, edges = network.nodes_df, network.edges_df

    with pd.HDFStore(filename, mode='w') as store:
        store['nodes'] = nodes
        store['edges'] = edges

        store['two_way'] = pd.Series([network._twoway])
        store['impedance_names'] = pd.Series(network.impedance_names)

def network_from_pandas_hdf5(cls, filename):
    """
    Build a Network from data in a Pandas HDFStore.

    Parameters
    ----------
    cls : class
        Class to instantiate, usually pandana.Network.
    filename : str

    Returns
    -------
    network : pandana.Network

    """
    with pd.HDFStore(filename) as store:
        nodes = store['nodes']
        edges = store['edges']
        two_way = store['two_way'][0]
        imp_names = store['impedance_names'].tolist()

    return cls(
        nodes['x'], nodes['y'], edges['from'], edges['to'], edges[imp_names],
        twoway=two_way)


from __future__ import division, print_function

import matplotlib
# this might fix the travis build
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.neighbors import KDTree

from .cyaccess import cyaccess
from .loaders import pandash5 as ph5

def reserve_num_graphs(num):
    raise Exception("reserve_num_graphs is no longer required - remove from your code")

class Network:
    """
    Create the transportation network in the city.  Typical data would be
    distance based from OpenStreetMap or travel time from GTFS transit data.

    Parameters
    ----------
    node_x : Pandas Series, float
        Defines the x attribute for nodes in the network (e.g. longitude)
    node_y : Pandas Series, float
        Defines the y attribute for nodes in the network (e.g. latitude)
        This param and the one above should have the *same* index which
        should be the node_ids that are referred to in the edges below.
    edge_from : Pandas Series, int
        Defines the node id that begins an edge - should refer to the index
        of the two series objects above
    edge_to : Pandas Series, int
        Defines the node id that ends an edge - should refer to the index
        of the two series objects above
    edge_weights : Pandas DataFrame, all numerics
        Specifies one or more *impedances* on the network which define the
        distances between nodes.  Multiple impedances can be used to
        capture travel times at different times of day, for instance
    twoway : boolean, optional
        Whether the edges in this network are two way edges or one way (
        where the one direction is directed from the from node to the to
        node). If twoway = True, it is assumed that the from and to id in the
        edge table occurs once and that travel can occur in both directions
        on the single edge record. Pandana will internally flip and append
        the from and to ids to the original edges to create a two direction
        network. If twoway = False, it is assumed that travel can only occur
        in the explicit direction indicated by the from and to id in the edge
        table.

    """

    def __init__(self, node_x, node_y, edge_from, edge_to, edge_weights,
                 twoway=True):

        nodes_df = pd.DataFrame({'x': node_x, 'y': node_y})
        edges_df = pd.DataFrame({'from': edge_from, 'to': edge_to}).\
            join(edge_weights)

        self.nodes_df = nodes_df
        self.edges_df = edges_df

        self.impedance_names = list(edge_weights.columns)
        self.variable_names = set()
        self.poi_category_names = []
        self.poi_category_indexes = {}

        # this maps ids to indexes which are used internally
        # this is a constant source of headaches, but all node identifiers
        # in the c extension are actually indexes ordered from 0 to numnodes-1
        # node ids are thus translated back and forth in the python layer, which
        # allows non-integer node ids as well
        self.node_idx = pd.Series(np.arange(len(nodes_df), dtype="int"),
                                  index=nodes_df.index)

        edges = pd.concat([self._node_indexes(edges_df["from"]),
                          self._node_indexes(edges_df["to"])], axis=1)

        self.net = cyaccess(self.node_idx.values,
                            nodes_df.astype('double').as_matrix(),
                            edges.as_matrix(),
                            edges_df[edge_weights.columns].transpose()
                                                          .astype('double')
                                                          .as_matrix(),
                            twoway)

        self._twoway = twoway

        self.kdtree = KDTree(nodes_df.as_matrix())

    @classmethod
    def from_hdf5(cls, filename):
        """
        Load a previously saved Network from a Pandas HDF5 file.

        Parameters
        ----------
        filename : str

        Returns
        -------
        network : pandana.Network

        """
        return ph5.network_from_pandas_hdf5(cls, filename)

    def save_hdf5(self, filename, rm_nodes=None):
        """
        Save network data to a Pandas HDF5 file.

        Only the nodes and edges of the actual network are saved,
        points-of-interest and data attached to nodes are not saved.

        Parameters
        ----------
        filename : str
        rm_nodes : array_like
            A list, array, Index, or Series of node IDs that should *not*
            be saved as part of the Network.

        """
        ph5.network_to_pandas_hdf5(self, filename, rm_nodes)

    def _node_indexes(self, node_ids):
        # for some reason, merge is must faster than .loc
        df = pd.merge(pd.DataFrame({"node_ids": node_ids}),
                      pd.DataFrame({"node_idx": self.node_idx}),
                      left_on="node_ids",
                      right_index=True,
                      how="left")
        return df.node_idx

    @property
    def aggregations(self):
        return self.net.get_available_aggregations()

    @property
    def decays(self):
        return self.net.get_available_decays()

    @property
    def node_ids(self):
        """
        The node ids which will be used as the index of many return series
        """
        return self.node_idx.index

    @property
    def bbox(self):
        """
        The bounding box for nodes in this network [xmin, ymin, xmax, ymax]
        """
        return [self.nodes_df.x.min(), self.nodes_df.y.min(),
                self.nodes_df.x.max(), self.nodes_df.y.max()]

    def shortest_path(self, node_a, node_b, imp_name=None):
        """
        Return the shortest path between two node ids in the network.

        Parameters
        ----------
        node_a : int
             The source node label
        node_b : int
             The destination node label
        imp_name : string, optional
            The impedance name to use for the shortest path

        Returns
        -------
        A numpy array of the nodes that are traversed in the shortest
        path between the two nodes
        """
        # map to internal node indexes
        node_idx = self._node_indexes(pd.Series([node_a, node_b]))
        node_a = node_idx.iloc[0]
        node_b = node_idx.iloc[1]

        imp_num = self._imp_name_to_num(imp_name)

        path = self.net.shortest_path(node_a, node_b, imp_num)

        # map back to external node ids
        return self.node_ids.values[path]

    def set(self, node_ids, variable=None, name="tmp"):
        """
        Characterize urban space with a variable that is related to nodes in
        the network.

        Parameters
        ----------
        node_ids : Pandas Series, int
            A series of node_ids which are usually computed using
            get_node_ids on this object.
        variable : Pandas Series, numeric, optional
            A series which represents some variable defined in urban space.
            It could be the location of buildings, or the income of all
            households - just about anything can be aggregated using the
            network queries provided here and this provides the api to set
            the variable at its disaggregate locations.  Note that node_id
            and variable should have the same index (although the index is
            not actually used).  If variable is not set, then it is assumed
            that the variable is all "ones" at the location specified by
            node_ids.  This could be, for instance, the location of all
            coffee shops which don't really have a variable to aggregate. The
            variable is connected to the closest node in the Pandana network
            which assumes no impedance between the location of the variable
            and the location of the closest network node.
        name : string, optional
            Name the variable.  This is optional in the sense that if you don't
            specify it, the default name will be used.  Since the same
            default name is used by aggregate on this object, you can
            alternate between characterize and aggregate calls without
            setting names.

        Returns
        -------
        Nothing
        """

        if variable is None:
            variable = pd.Series(np.ones(len(node_ids)), index=node_ids.index)

        df = pd.DataFrame({name: variable,
                           "node_idx": self._node_indexes(node_ids)})

        length = len(df)
        df = df.dropna(how="any")
        newl = len(df)
        if length-newl > 0:
            print(
                "Removed %d rows because they contain missing values" %
                (length-newl))

        self.variable_names.add(name)

        self.net.initialize_access_var(name.encode('utf-8'),
                                       df.node_idx.values.astype('int'),
                                       df[name].values.astype('double'))

    def precompute(self, distance):
        """
        Precomputes the range queries (the reachable nodes within this
        maximum distance.  So as long as you use a smaller distance, cached
        results will be used.)

        Parameters
        ----------
        distance : float
            The maximum distance to use. This will usually be a distance unit
            in meters however if you have customized the impedance this could
            be in other units such as utility or time etc.

        Returns
        -------
        Nothing
        """
        self.net.precompute_range(distance)

    def _imp_name_to_num(self, imp_name):
        if imp_name is None:
            assert len(self.impedance_names) == 1,\
                "must pass impedance name if there are multiple impedances set"
            imp_name = self.impedance_names[0]

        assert imp_name in self.impedance_names, "An impedance with that name" \
                                                 "was not found"

        return self.impedance_names.index(imp_name)

    def aggregate(self, distance, type="sum", decay="linear", imp_name=None,
                  name="tmp"):
        """
        Aggregate information for every source node in the network - this is
        really the main purpose of this library.  This allows you to touch
        the data specified by calling set and perform some aggregation on it
        within the specified distance.  For instance, summing the population
        within 1000 meters.

        Parameters
        ----------
        distance : float
            The maximum distance to aggregate data within. 'distance' can
            represent any impedance unit that you have set as your edge
            weight. This will usually be a distance unit in meters however
            if you have customized the impedance this could be in other
            units such as utility or time etc.
        type : string
            The type of aggregation, can be one of "ave", "sum", "std",
            "count", and now "min", "25pct", "median", "75pct", and "max" will
            compute the associated quantiles.  (Quantiles are computed by
            sorting so might be slower than the others.)
        decay : string
            The type of decay to apply, which makes things that are further
            away count less in the aggregation - must be one of "linear",
            "exponential" or "flat" (which means no decay).  Linear is the
            fastest computation to perform.  When performing an "ave",
            the decay is typically "flat"
        imp_name : string, optional
            The impedance name to use for the aggregation on this network.
            Must be one of the impedance names passed in the constructor of
            this object.  If not specified, there must be only one impedance
            passed in the constructor, which will be used.
        name : string, optional
            The variable to aggregate.  This variable will have been created
            and named by a call to set.  If not specified, the default
            variable name will be used so that the most recent call to set
            without giving a name will be the variable used.

        Returns
        -------
        agg : Pandas Series
            Returns a Pandas Series for every origin node in the network,
            with the index which is the same as the node_ids passed to the
            init method and the values are the aggregations for each source
            node in the network.
        """

        imp_num = self._imp_name_to_num(imp_name)
        type = type.lower()
        if type == "ave":
            type = "mean"  # changed generic ave to mean

        assert name in self.variable_names, "A variable with that name " \
                                            "has not yet been initialized"

        res = self.net.get_all_aggregate_accessibility_variables(distance,
                                                                 name.encode('utf-8'),
                                                                 type.encode('utf-8'),
                                                                 decay.encode('utf-8'),
                                                                 imp_num)

        return pd.Series(res, index=self.node_ids)

    def get_node_ids(self, x_col, y_col, mapping_distance=None):
        """
        Assign node_ids to data specified by x_col and y_col

        Parameters
        ----------
        x_col : Pandas series (float)
            A Pandas Series where values specify the x (e.g. longitude)
            location of dataset.
        y_col : Pandas series (float)
            A Pandas Series where values specify the y (e.g. latitude)
            location of dataset.  x_col and y_col should use the same index.
        mapping_distance : float, optional
            The maximum distance that will be considered a match between the
            x, y data and the nearest node in the network.  This will usually
            be a distance unit in meters however if you have customized the
            impedance this could be in other units such as utility or time
            etc. If not specified, every x, y coordinate will be mapped to
            the nearest node.

        Returns
        -------
        node_ids : Pandas series (int)
            Returns a Pandas Series of node_ids for each x, y in the
            input data. The index is the same as the indexes of the
            x, y input data, and the values are the mapped node_ids.
            If mapping distance is not passed and if there are no nans in the
            x, y data, this will be the same length as the x, y data.
            If the mapping is imperfect, this function returns all the
            input x, y's that were successfully mapped to node_ids.
        """
        xys = pd.DataFrame({'x': x_col, 'y': y_col})

        distances, indexes = self.kdtree.query(xys.as_matrix())
        indexes = np.transpose(indexes)[0]
        distances = np.transpose(distances)[0]

        node_ids = self.nodes_df.iloc[indexes].index

        df = pd.DataFrame({"node_id": node_ids, "distance": distances},
                          index=xys.index)

        if mapping_distance is not None:
            df = df[df.distance <= mapping_distance]

        return df.node_id

    def plot(
            self, data, bbox=None, plot_type='scatter',
            fig_kwargs=None, bmap_kwargs=None, plot_kwargs=None,
            cbar_kwargs=None):
        """
        Plot an array of data on a map using matplotlib and Basemap,
        automatically matching the data to the Pandana network node positions.

        Keyword arguments are passed to the plotting routine.

        Parameters
        ----------
        data : pandas.Series
            Numeric data with the same length and index as the nodes
            in the network.
        bbox : tuple, optional
            (lat_min, lng_min, lat_max, lng_max)
        plot_type : {'hexbin', 'scatter'}, optional
        fig_kwargs : dict, optional
            Keyword arguments that will be passed to
            matplotlib.pyplot.subplots. Use this to specify things like
            figure size or background color.
        bmap_kwargs : dict, optional
            Keyword arguments that will be passed to the Basemap constructor.
            This can be used to specify a projection or coastline resolution.
        plot_kwargs : dict, optional
            Keyword arguments that will be passed to the matplotlib plotting
            command used. Use this to control plot styles and color maps used.
        cbar_kwargs : dict, optional
            Keyword arguments passed to the Basemap.colorbar method.
            Use this to control color bar location and label.

        Returns
        -------
        bmap : Basemap
        fig : matplotlib.Figure
        ax : matplotlib.Axes

        """
        from mpl_toolkits.basemap import Basemap

        fig_kwargs = fig_kwargs or {}
        bmap_kwargs = bmap_kwargs or {}
        plot_kwargs = plot_kwargs or {}
        cbar_kwargs = cbar_kwargs or {}

        if not bbox:
            bbox = (
                self.nodes_df.y.min(),
                self.nodes_df.x.min(),
                self.nodes_df.y.max(),
                self.nodes_df.x.max())

        fig, ax = plt.subplots(**fig_kwargs)

        bmap = Basemap(
            bbox[1], bbox[0], bbox[3], bbox[2], ax=ax, **bmap_kwargs)
        bmap.drawcoastlines()
        bmap.drawmapboundary()

        x, y = bmap(self.nodes_df.x.values, self.nodes_df.y.values)

        if plot_type == 'scatter':
            plot = bmap.scatter(
                x, y, c=data.values, **plot_kwargs)
        elif plot_type == 'hexbin':
            plot = bmap.hexbin(
                x, y, C=data.values, **plot_kwargs)

        bmap.colorbar(plot, **cbar_kwargs)

        return bmap, fig, ax

    def set_pois(self, category, maxdist, maxitems, x_col, y_col):
        """
        Set the location of all the pois of this category. The pois are
        connected to the closest node in the Pandana network which assumes
        no impedance between the location of the variable and the location
        of the closest network node.

        Parameters
        ----------
        category : string
            The name of the category for this set of pois
        maxdist - the maximum distance that will later be used in
            find_all_nearest_pois
        maxitems - the maximum number of items that will later be requested
            in find_all_nearest_pois
        x_col : Pandas Series (float)
            The x location (longitude) of pois in this category
        y_col : Pandas Series (Float)
            The y location (latitude) of pois in this category

        Returns
        -------
        Nothing
        """
        if category not in self.poi_category_names:
            self.poi_category_names.append(category)

        self.max_pois = maxitems

        node_ids = self.get_node_ids(x_col, y_col)

        self.poi_category_indexes[category] = node_ids.index

        node_idx = self._node_indexes(node_ids)

        self.net.initialize_category(maxdist, maxitems, category.encode('utf-8'), node_idx.values)

    def nearest_pois(self, distance, category, num_pois=1, max_distance=None,
                     imp_name=None, include_poi_ids=False):
        """
        Find the distance to the nearest pois from each source node.  The
        bigger values in this case mean less accessibility.

        Parameters
        ----------
        distance : float
            The maximum distance to look for pois. This will usually be a
            distance unit in meters however if you have customized the
            impedance this could be in other units such as utility or time
            etc.
        category : string
            The name of the category of poi to look for
        num_pois : int
            The number of pois to look for, this also sets the number of
            columns in the DataFrame that gets returned
        max_distance : float, optional
            The value to set the distance to if there is NO poi within the
            specified distance - if not specified, gets set to distance. This
            will usually be a distance unit in meters however if you have
            customized the impedance this could be in other units such as
            utility or time etc.
        imp_name : string, optional
            The impedance name to use for the aggregation on this network.
            Must be one of the impedance names passed in the constructor of
            this object.  If not specified, there must be only one impedance
            passed in the constructor, which will be used.
        include_poi_ids : bool, optional
            If this flag is set to true, the call will add columns to the
            return DataFrame - instead of just returning the distance for
            the nth POI, it will also return the id of that POI.  The names
            of the columns with the poi ids will be poi1, poi2, etc - it
            will take roughly twice as long to include these ids as to not
            include them

        Returns
        -------
        d : Pandas DataFrame
            Like aggregate, this series has an index of all the node ids for
            the network.  Unlike aggregate, this method returns a dataframe
            with the number of columns equal to the distances to the Nth
            closest poi.  For instance, if you ask for the 10 closest poi to
            each node, column d[1] wil be the distance to the 1st closest poi
            of that category while column d[2] will be the distance to the 2nd
            closest poi, and so on.
        """
        if max_distance is None:
            max_distance = distance

        if category not in self.poi_category_names:
            assert 0, "Need to call set_pois for this category"

        if num_pois > self.max_pois:
            assert 0, "Asking for more pois than set in init_pois"

        imp_num = self._imp_name_to_num(imp_name)

        dists, poi_ids = self.net.find_all_nearest_pois(
            distance,
            num_pois,
            category.encode('utf-8'),
            imp_num)
        dists[dists == -1] = max_distance

        df = pd.DataFrame(dists, index=self.node_ids)
        df.columns = list(range(1, num_pois+1))

        if include_poi_ids:
            df2 = pd.DataFrame(poi_ids, index=self.node_ids)
            df2.columns = ["poi%d" % i for i in range(1, num_pois+1)]
            for col in df2.columns:
                # if this is still all working according to plan at this point
                # the great magic trick is now to turn the integer position of
                # the poi, which is painstakingly returned from the c++ code,
                # and turn it into the actual index that was used when it was
                # initialized as a pandas series - this really is pandas-like
                # thinking.  it's complicated on the inside, but quite
                # intuitive to the user I think
                s = df2[col].astype('int')
                df2[col] = self.poi_category_indexes[category].values[s]
                df2.loc[s == -1, col] = np.nan

            df = pd.concat([df, df2], axis=1)

        return df

    def low_connectivity_nodes(self, impedance, count, imp_name=None):
        """
        Identify nodes that are connected to fewer than some threshold
        of other nodes within a given distance.

        Parameters
        ----------
        impedance : float
            Distance within which to search for other connected nodes. This
            will usually be a distance unit in meters however if you have
            customized the impedance this could be in other units such as
            utility or time etc.
        count : int
            Threshold for connectivity. If a node is connected to fewer
            than this many nodes within `impedance` it will be identified
            as "low connectivity".
        imp_name : string, optional
            The impedance name to use for the aggregation on this network.
            Must be one of the impedance names passed in the constructor of
            this object.  If not specified, there must be only one impedance
            passed in the constructor, which will be used.

        Returns
        -------
        node_ids : array
            List of "low connectivity" node IDs.

        """
        # set a counter variable on all nodes
        self.set(self.node_ids.to_series(), name='counter')

        # count nodes within impedance range
        agg = self.aggregate(
            impedance, type='count', imp_name=imp_name, name='counter')

        return np.array(agg[agg < count].index)


"""
Tools for creating Pandana networks from OpenStreetMap.

"""

import pandas as pd
import requests
from osmnet.load import network_from_bbox

from .. import Network

def pdna_network_from_bbox(
        lat_min=None, lng_min=None, lat_max=None, lng_max=None, bbox=None,
        network_type='walk', two_way=True,
        timeout=180, memory=None, max_query_area_size=50 * 1000 * 50 * 1000):
    """
    Make a Pandana network from a bounding lat/lon box
    request to the Overpass API. Distance will be in the default units meters.

    Parameters
    ----------
    lat_min, lng_min, lat_max, lng_max : float
    bbox : tuple
        Bounding box formatted as a 4 element tuple:
        (lng_max, lat_min, lng_min, lat_max)
    network_type : {'walk', 'drive'}, optional
        Specify whether the network will be used for walking or driving.
        A value of 'walk' attempts to exclude things like freeways,
        while a value of 'drive' attempts to exclude things like
        bike and walking paths.
    two_way : bool, optional
        Whether the routes are two-way. If True, node pairs will only
        occur once.
    timeout : int, optional
        the timeout interval for requests and to pass to Overpass API
    memory : int, optional
        server memory allocation size for the query, in bytes.
        If none, server will use its default allocation size
    max_query_area_size : float, optional
        max area for any part of the geometry, in the units the geometry is in

    Returns
    -------
    network : pandana.Network

    """

    nodes, edges = network_from_bbox(lat_min=lat_min, lng_min=lng_min,
                                     lat_max=lat_max, lng_max=lng_max,
                                     bbox=bbox, network_type=network_type,
                                     two_way=two_way, timeout=timeout,
                                     memory=memory,
                                     max_query_area_size=max_query_area_size)

    return Network(
        nodes['x'], nodes['y'],
        edges['from'], edges['to'], edges[['distance']])

def process_node(e):
    """
    Process a node element entry into a dict suitable for going into
    a Pandas DataFrame.

    Parameters
    ----------
    e : dict
    Returns
    -------
    node : dict
    """

    uninteresting_tags = {
        'source',
        'source_ref',
        'source:ref',
        'history',
        'attribution',
        'created_by',
        'tiger:tlid',
        'tiger:upload_uuid',
    }

    node = {
        'id': e['id'],
        'lat': e['lat'],
        'lon': e['lon']
    }

    if 'tags' in e:
        for t, v in list(e['tags'].items()):
            if t not in uninteresting_tags:
                node[t] = v

    return node

def make_osm_query(query):
    """
    Make a request to OSM and return the parsed JSON.

    Parameters
    ----------
    query : str
        A string in the Overpass QL format.

    Returns
    -------
    data : dict

    """
    osm_url = 'http://www.overpass-api.de/api/interpreter'
    req = requests.get(osm_url, params={'data': query})
    req.raise_for_status()

    return req.json()

def build_node_query(lat_min, lng_min, lat_max, lng_max, tags=None):
    """
    Build the string for a node-based OSM query.

    Parameters
    ----------
    lat_min, lng_min, lat_max, lng_max : float
    tags : str or list of str, optional
        Node tags that will be used to filter the search.
        See http://wiki.openstreetmap.org/wiki/Overpass_API/Language_Guide
        for information about OSM Overpass queries
        and http://wiki.openstreetmap.org/wiki/Map_Features
        for a list of tags.

    Returns
    -------
    query : str

    """
    if tags is not None:
        if isinstance(tags, str):
            tags = [tags]
        tags = ''.join('[{}]'.format(t) for t in tags)
    else:
        tags = ''

    query_fmt = (
        '[out:json];'
        '('
        '  node'
        '  {tags}'
        '  ({lat_min},{lng_min},{lat_max},{lng_max});'
        ');'
        'out;')

    return query_fmt.format(
        lat_min=lat_min, lng_min=lng_min, lat_max=lat_max, lng_max=lng_max,
        tags=tags)

def node_query(lat_min, lng_min, lat_max, lng_max, tags=None):
    """
    Search for OSM nodes within a bounding box that match given tags.

    Parameters
    ----------
    lat_min, lng_min, lat_max, lng_max : float
    tags : str or list of str, optional
        Node tags that will be used to filter the search.
        See http://wiki.openstreetmap.org/wiki/Overpass_API/Language_Guide
        for information about OSM Overpass queries
        and http://wiki.openstreetmap.org/wiki/Map_Features
        for a list of tags.

    Returns
    -------
    nodes : pandas.DataFrame
        Will have 'lat' and 'lon' columns, plus other columns for the
        tags associated with the node (these will vary based on the query).
        Index will be the OSM node IDs.

    """
    node_data = make_osm_query(build_node_query(
        lat_min, lng_min, lat_max, lng_max, tags=tags))

    if len(node_data['elements']) == 0:
        raise RuntimeError('OSM query results contain no data.')

    nodes = [process_node(n) for n in node_data['elements']]
    return pd.DataFrame.from_records(nodes, index='id')


import re
import sys
from unittest import TestCase
from .regex import isregex, strip_regex, isregex_expr

# If running Python 3
PY_3 = sys.version_info >= (3,)

def test_case():
    """
    Creates a new ``unittest.TestCase`` instance.

    Returns:
        unittest.TestCase
    """
    test = TestCase()
    test.maxDiff = None
    return test

def equal(x, y):
    """
    Shortcut function for ``unittest.TestCase.assertEqual()``.

    Arguments:
        x (mixed)
        y (mixed)

    Raises:
        AssertionError: in case of assertion error.

    Returns:
        bool
    """
    if PY_3:
        return test_case().assertEqual(x, y) or True

    assert x == y

def matches(x, y, regex_expr=False):
    """
    Tries to match a regular expression value ``x`` against ``y``.
    Aliast``unittest.TestCase.assertEqual()``

    Arguments:
        x (regex|str): regular expression to test.
        y (str): value to match.
        regex_expr (bool): enables regex string based expression matching.

    Raises:
        AssertionError: in case of mismatching.

    Returns:
        bool
    """
    # Parse regex expression, if needed
    x = strip_regex(x) if regex_expr and isregex_expr(x) else x

    # Run regex assertion
    if PY_3:
        # Retrieve original regex pattern
        x = x.pattern if isregex(x) else x
        # Assert regular expression via unittest matchers
        return test_case().assertRegex(y, x) or True

    # Primitive regex matching for Python 2.7
    if isinstance(x, str):
        x = re.compile(x, re.IGNORECASE)

    assert x.match(y) is not None

def test(x, y, regex_expr=False):
    """
    Compares to values based on regular expression matching or
    strict equality comparison.

    Arguments:
        x (regex|str): string or regular expression to test.
        y (str): value to match.
        regex_expr (bool): enables regex string based expression matching.

    Raises:
        AssertionError: in case of matching error.

    Returns:
        bool
    """
    return matches(x, y, regex_expr=regex_expr) if isregex(x) else equal(x, y)


import re

# Little hack to extra the regexp object type at runtime
retype = type(re.compile(''))

def isregex_expr(expr):
    """
    Returns ``True`` is the given expression value is a regular expression
    like string with prefix ``re/`` and suffix ``/``, otherwise ``False``.

    Arguments:
        expr (mixed): expression value to test.

    Returns:
        bool
    """
    if not isinstance(expr, str):
        return False

    return all([
        len(expr) > 3,
        expr.startswith('re/'),
        expr.endswith('/')
    ])

def isregex(value):
    """
    Returns ``True`` if the input argument object is a native
    regular expression object, otherwise ``False``.

    Arguments:
        value (mixed): input value to test.

    Returns:
        bool
    """
    if not value:
        return False
    return any((isregex_expr(value), isinstance(value, retype)))

def strip_regex(expr):
    """
    Strips regular expression notation syntax characters from the given
    string expression.

    Arguments:
        expr (str): regular expression expression to strip

    Returns:
        str
    """
    return expr.replace[3:-1] if isregex_expr(expr) else expr


import functools
from copy import deepcopy
from abc import abstractmethod, ABCMeta
from ..compare import compare

class BaseMatcher(object):
    """
    BaseMatcher implements the basic HTTP request matching interface.
    """

    __metaclass__ = ABCMeta

    # Negate matching if necessary
    negate = False

    def __init__(self, expectation, negate=False):
        if not expectation:
            raise ValueError('expectation argument cannot be empty')

        self.negate = negate
        self._expectation = expectation

    @property
    def name(self):
        return type(self).__name__

    @property
    def expectation(self):
        return self._expectation

    @expectation.setter
    def expectation(self, value):
        self._expectation = value

    @abstractmethod
    def match(self, request):
        """
        Match performs the value matching.
        This is an abstract method that must be implemented by child classes.

        Arguments:
            request (pook.Request): request object to match.
        """
        pass

    def compare(self, value, expectation, regex_expr=False):
        """
        Compares two values with regular expression matching support.

        Arguments:
            value (mixed): value to compare.
            expectation (mixed): value to match.
            regex_expr (bool, optional): enables string based regex matching.

        Returns:
            bool
        """
        return compare(value, expectation, regex_expr=regex_expr)

    def to_dict(self):
        """
        Returns the current matcher representation as dictionary.

        Returns:
            dict
        """
        return {self.name: deepcopy(self.expectation)}

    def __repr__(self):
        return '{}({})'.format(self.name, self.expectation)

    def __str__(self):
        return self.expectation

    @staticmethod
    def matcher(fn):
        @functools.wraps(fn)
        def wrapper(self, *args):
            result = fn(self, *args)
            return not result if self.negate else result
        return wrapper


import functools

def fluent(fn):
    """
    Simple function decorator allowing easy method chaining.

    Arguments:
        fn (function): target function to decorate.
    """
    @functools.wraps(fn)
    def wrapper(self, *args, **kw):
        # Trigger method proxy
        result = fn(self, *args, **kw)
        # Return self instance or method result
        return self if result is None else result
    return wrapper


import re
from .assertion import test

# Negate is used a reserved token identifier to negate matching
NEGATE = '!!'

def compile(expr):
    try:
        return re.compile(expr, re.IGNORECASE)
    except Exception:
        pass

def match(expr, value):
    regex = compile(expr)
    if not regex:
        return False
    return regex.match(value) is not None

def strip_negate(value):
    return value[len(NEGATE):].lstrip()

def compare(expr, value, regex_expr=False):
    """
    Compares an string or regular expression againast a given value.

    Arguments:
        expr (str|regex): string or regular expression value to compare.
        value (str): value to compare against to.
        regex_expr (bool, optional): enables string based regex matching.

    Raises:
        AssertionError: in case of assertion error.

    Returns:
        bool
    """
    # Strict equality comparison
    if expr == value:
        return True

    # Infer negate expression to match, if needed
    negate = False
    if isinstance(expr, str):
        negate = expr.startswith(NEGATE)
        expr = strip_negate(expr) if negate else expr

    try:
        # RegExp or strict equality comparison
        test(expr, value, regex_expr=regex_expr)
    except Exception as err:
        if negate:
            return True
        else:
            raise err

    return True


from inspect import ismethod, isfunction
from .exceptions import PookInvalidArgument

def trigger_methods(instance, args):
    """"
    Triggers specific class methods using a simple reflection
    mechanism based on the given input dictionary params.

    Arguments:
        instance (object): target instance to dynamically trigger methods.
        args (iterable): input arguments to trigger objects to

    Returns:
        None
    """
    # Start the magic
    for name in sorted(args):
        value = args[name]
        target = instance

        # If response attibutes
        if name.startswith('response_') or name.startswith('reply_'):
            name = name.replace('response_', '').replace('reply_', '')
            # If instance has response attribute, use it
            if hasattr(instance, '_response'):
                target = instance._response

        # Retrieve class member for inspection and future use
        member = getattr(target, name, None)

        # Is attribute
        isattr = name in dir(target)
        iscallable = ismethod(member) and not isfunction(member)

        if not iscallable and not isattr:
            raise PookInvalidArgument('Unsupported argument: {}'.format(name))

        # Set attribute or trigger method
        if iscallable:
            member(value)
        else:
            setattr(target, name, value)


class MatcherEngine(list):
    """
    HTTP request matcher engine used by `pook.Mock` to test if an
    intercepted outgoing HTTP request must be mocked out or not.
    """

    def add(self, matcher):
        """
        Adds a new matcher function to the current engine.

        Arguments:
            matcher (function): matcher function to be added.
        """
        self.append(matcher)

    def flush(self):
        """
        Flushes the current matcher engine, removing all the registered
        matcher functions.
        """
        self.clear()

    def match(self, request):
        """
        Match the given HTTP request instance against the registered
        matcher functions in the current engine.

        Arguments:
            request (pook.Request): outgoing request to match.

        Returns:
            tuple(bool, list[Exception]): ``True`` if all matcher tests
                passes, otherwise ``False``. Also returns an optional list
                of error exceptions.
        """
        errors = []

        def match(matcher):
            try:
                return matcher.match(request)
            except Exception as err:
                err = '{}: {}'.format(type(matcher).__name__, err)
                errors.append(err)
                return False

        return all([match(matcher) for matcher in self]), errors

    def __repr__(self):
        """
        Returns an human friendly readable instance data representation.

        Returns:
            str
        """
        matchers = [repr(matcher) for matcher in self]
        return 'MatcherEngine([\n  {}\n])'.format(',\n  '.join(matchers))


from .base import BaseMatcher
from .url import URLMatcher
from .body import BodyMatcher
from .query import QueryMatcher
from .method import MethodMatcher
from .headers import HeadersMatcher
from .path import PathMatcher
from .xml import XMLMatcher
from .json import JSONMatcher
from .json_schema import JSONSchemaMatcher

# Explicit symbols to export
__all__ = (
    'init',
    'add',
    'get',
    'matchers',
    'BaseMatcher',
    'MethodMatcher',
    'URLMatcher',
    'HeadersMatcher',
    'QueryMatcher',
    'PathMatcher',
    'BodyMatcher',
    'XMLMatcher',
    'JSONMatcher',
    'JSONSchemaMatcher',
    'QueryMatcher',
)

# List of built-in matchers
# This is intended to be mutable.
matchers = [
    MethodMatcher,
    URLMatcher,
    HeadersMatcher,
    QueryMatcher,
    PathMatcher,
    BodyMatcher,
    XMLMatcher,
    JSONMatcher,
    JSONSchemaMatcher,
    QueryMatcher,
]

def add(*matcher):
    """
    Registers one or multiple matchers to be used by default from
    mocking engine.

    Arguments:
        *matcher (list[pook.BaseMatcher]): variadic matchers to add.
    """
    [matchers.append(m) for m in matcher]

def get(name):
    """
    Returns a matcher instance by class or alias name.

    Arguments:
        name (str): matcher class name or alias.

    Returns:
        matcher: found matcher instance, otherwise ``None``.
    """
    for matcher in matchers:
        if matcher.__name__ == name or getattr(matcher, 'name', None) == name:
            return matcher

def init(name, *args):
    """
    Initializes a matcher instance passing variadic arguments to
    its constructor. Acts as a delegator proxy.

    Arguments:
        name (str): matcher class name or alias to execute.
        *args (mixed): variadic argument

    Returns:
        matcher: matcher instance.

    Raises:
        ValueError: if matcher was not found.
    """
    matcher = get(name)
    if not matcher:
        raise ValueError('Cannot find matcher: {}'.format(name))
    return matcher(*args)