#!/usr/bin/env python

import argparse
import logging
import time
import pandas as pd
import numpy as np

import msaf
from msaf import input_output as io

def process(in_path, annot_beats=False, feature="mfcc", framesync=False,
            boundaries_id="gt", labels_id=None, n_jobs=4, config=None):
    """Sweeps parameters across the specified algorithm."""

    results_file = "results_sweep_boundsE%s_labelsE%s.csv" % (boundaries_id,
                                                              labels_id)

    if labels_id == "cnmf3" or boundaries_id == "cnmf3":
        config = io.get_configuration(feature, annot_beats, framesync,
                                      boundaries_id, labels_id)

        hh = range(15, 33)
        RR = range(15, 40)
        ranks = range(3, 6)
        RR_labels = range(11, 12)
        ranks_labels = range(6, 7)
        all_results = pd.DataFrame()
        for rank in ranks:
            for h in hh:
                for R in RR:
                    for rank_labels in ranks_labels:
                        for R_labels in RR_labels:
                            config["h"] = h
                            config["R"] = R
                            config["rank"] = rank
                            config["rank_labels"] = rank_labels
                            config["R_labels"] = R_labels
                            config["features"] = None

                            # Run process
                            msaf.run.process(
                                in_path, n_jobs=n_jobs,
                                boundaries_id=boundaries_id,
                                labels_id=labels_id, config=config)

                            # Compute evaluations
                            results = msaf.eval.process(
                                in_path, boundaries_id, labels_id,
                                save=True, n_jobs=n_jobs, config=config)

                            # Save avg results
                            new_columns = {"config_h": h, "config_R": R,
                                           "config_rank": rank,
                                           "config_R_labels": R_labels,
                                           "config_rank_labels": rank_labels}
                            results = results.append([new_columns],
                                                     ignore_index=True)
                            all_results = all_results.append(results.mean(),
                                                             ignore_index=True)
                            all_results.to_csv(results_file)

    elif labels_id is None and boundaries_id == "sf":
        config = io.get_configuration(feature, annot_beats, framesync,
                                      boundaries_id, labels_id)

        MM = range(20, 32)
        mm = range(3, 4)
        kk = np.arange(0.03, 0.1, 0.01)
        Mpp = range(16, 32)
        ott = np.arange(0.02, 0.1, 0.01)
        all_results = pd.DataFrame()
        for k in kk:
            for ot in ott:
                for m in mm:
                    for M in MM:
                        for Mp in Mpp:
                            config["M_gaussian"] = M
                            config["m_embedded"] = m
                            config["k_nearest"] = k
                            config["Mp_adaptive"] = Mp
                            config["offset_thres"] = ot
                            config["features"] = None

                            # Run process
                            msaf.run.process(
                                in_path, n_jobs=n_jobs,
                                boundaries_id=boundaries_id,
                                labels_id=labels_id, config=config)

                            # Compute evaluations
                            results = msaf.eval.process(
                                in_path, boundaries_id, labels_id,
                                save=True, n_jobs=n_jobs, config=config)

                            # Save avg results
                            new_columns = {"config_M": M, "config_m": m,
                                           "config_k": k, "config_Mp": Mp,
                                           "config_ot": ot}
                            results = results.append([new_columns],
                                                     ignore_index=True)
                            all_results = all_results.append(results.mean(),
                                                             ignore_index=True)
                            all_results.to_csv(results_file)

    else:
        logging.error("Can't sweep parameters for %s algorithm. "
                      "Implement me! :D")

def main():
    """Main function to sweep parameters of a certain algorithm."""
    parser = argparse.ArgumentParser(
        description="Runs the speficied algorithm(s) on the MSAF "
        "formatted dataset.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("in_path",
                        action="store",
                        help="Input dataset")
    parser.add_argument("-f",
                        action="store",
                        dest="feature",
                        default="pcp",
                        type=str,
                        help="Type of features",
                        choices=["pcp", "tonnetz", "mfcc", "cqt", "tempogram"])
    parser.add_argument("-b",
                        action="store_true",
                        dest="annot_beats",
                        help="Use annotated beats",
                        default=False)
    parser.add_argument("-fs",
                        action="store_true",
                        dest="framesync",
                        help="Use frame-synchronous features",
                        default=False)
    parser.add_argument("-bid",
                        action="store",
                        help="Boundary algorithm identifier",
                        dest="boundaries_id",
                        default="gt",
                        choices=["gt"] +
                        io.get_all_boundary_algorithms())
    parser.add_argument("-lid",
                        action="store",
                        help="Label algorithm identifier",
                        dest="labels_id",
                        default=None,
                        choices=io.get_all_label_algorithms())
    parser.add_argument("-j",
                        action="store",
                        dest="n_jobs",
                        default=4,
                        type=int,
                        help="The number of threads to use")
    args = parser.parse_args()
    start_time = time.time()

    # Run the algorithm(s)
    process(args.in_path, annot_beats=args.annot_beats, feature=args.feature,
            framesync=args.framesync, boundaries_id=args.boundaries_id,
            labels_id=args.labels_id, n_jobs=args.n_jobs)

    # Done!
    logging.info("Done! Took %.2f seconds." % (time.time() - start_time))

if __name__ == "__main__":
    main()


#!/usr/bin/env python
"""
Runs one boundary algorithm and a label algorithm on a specified audio file
and outputs the results using the MIREX format.
"""
import argparse
import logging
import time

# MSAF import
import msaf

def main():
    """Main function to parse the arguments and call the main process."""
    parser = argparse.ArgumentParser(
        description="Runs the speficied algorithm(s) on the input file and "
        "the results using the MIREX format.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("-bid",
                        action="store",
                        help="Boundary algorithm identifier",
                        dest="boundaries_id",
                        default=msaf.config.default_bound_id,
                        choices=["gt"] +
                        msaf.io.get_all_boundary_algorithms())
    parser.add_argument("-lid",
                        action="store",
                        help="Label algorithm identifier",
                        dest="labels_id",
                        default=msaf.config.default_label_id,
                        choices=msaf.io.get_all_label_algorithms())
    parser.add_argument("-i",
                        action="store",
                        dest="in_file",
                        help="Input audio file")
    parser.add_argument("-o",
                        action="store",
                        dest="out_file",
                        help="Output file with the results",
                        default="out.txt")

    args = parser.parse_args()
    start_time = time.time()

    # Setup the logger
    logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s',
                        level=logging.INFO)

    # Run MSAF
    params = {
        "annot_beats": False,
        "feature": "cqt",
        "framesync": False,
        "boundaries_id": args.boundaries_id,
        "labels_id": args.labels_id,
        "n_jobs": 1,
        "hier": False,
        "sonify_bounds": False,
        "plot": False
    }
    res = msaf.run.process(args.in_file, **params)
    msaf.io.write_mirex(res[0], res[1], args.out_file)

    # Done!
    logging.info("Done! Took %.2f seconds." % (time.time() - start_time))

if __name__ == '__main__':
    main()


"""
Evaluates the estimated results of the Segmentation dataset against the
ground truth (human annotated data).

.. autosummary::
    :toctree: generated/

    process
"""
import jams
from joblib import Parallel, delayed
import logging
import mir_eval
import numpy as np
import os
import pandas as pd
import six

# Local stuff
import msaf
from msaf.exceptions import NoReferencesError
import msaf.input_output as io
from msaf import utils

def print_results(results):
    """Print all the results.

    Parameters
    ----------
    results: pd.DataFrame
        Dataframe with all the results
    """
    if len(results) == 0:
        logging.warning("No results to print!")
        return
    res = results.mean()
    logging.info("Results:\n%s" % res)

def compute_results(ann_inter, est_inter, ann_labels, est_labels, bins,
                    est_file, weight=0.58):
    """Compute the results using all the available evaluations.

    Parameters
    ----------
    ann_inter : np.array
        Annotated intervals in seconds.
    est_inter : np.array
        Estimated intervals in seconds.
    ann_labels : np.array
        Annotated labels.
    est_labels : np.array
        Estimated labels
    bins : int
        Number of bins for the information gain.
    est_file : str
        Path to the output file to store results.
    weight: float
        Weight the Precision and Recall values of the hit rate boundaries
        differently (<1 will weight Precision higher, >1 will weight Recall
        higher).
        The default parameter (0.58) is the one proposed in (Nieto et al. 2014)

    Return
    ------
    results : dict
        Contains the results of all the evaluations for the given file.
        Keys are the following:
            track_id: Name of the track
            HitRate_3F: F-measure of hit rate at 3 seconds
            HitRate_3P: Precision of hit rate at 3 seconds
            HitRate_3R: Recall of hit rate at 3 seconds
            HitRate_0.5F: F-measure of hit rate at 0.5 seconds
            HitRate_0.5P: Precision of hit rate at 0.5 seconds
            HitRate_0.5R: Recall of hit rate at 0.5 seconds
            HitRate_w3F: F-measure of hit rate at 3 seconds weighted
            HitRate_w0.5F: F-measure of hit rate at 0.5 seconds weighted
            HitRate_wt3F: F-measure of hit rate at 3 seconds weighted and
                          trimmed
            HitRate_wt0.5F: F-measure of hit rate at 0.5 seconds weighted
                            and trimmed
            HitRate_t3F: F-measure of hit rate at 3 seconds (trimmed)
            HitRate_t3P: Precision of hit rate at 3 seconds (trimmed)
            HitRate_t3F: Recall of hit rate at 3 seconds (trimmed)
            HitRate_t0.5F: F-measure of hit rate at 0.5 seconds (trimmed)
            HitRate_t0.5P: Precision of hit rate at 0.5 seconds (trimmed)
            HitRate_t0.5R: Recall of hit rate at 0.5 seconds (trimmed)
            DevA2E: Median deviation of annotation to estimation
            DevE2A: Median deviation of estimation to annotation
            D: Information gain
            PWF: F-measure of pair-wise frame clustering
            PWP: Precision of pair-wise frame clustering
            PWR: Recall of pair-wise frame clustering
            Sf: F-measure normalized entropy score
            So: Oversegmentation normalized entropy score
            Su: Undersegmentation normalized entropy score
    """
    res = {}

    # --Boundaries-- #
    # Hit Rate standard
    res["HitRate_3P"], res["HitRate_3R"], res["HitRate_3F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=3, trim=False)
    res["HitRate_0.5P"], res["HitRate_0.5R"], res["HitRate_0.5F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=.5, trim=False)

    # Hit rate trimmed
    res["HitRate_t3P"], res["HitRate_t3R"], res["HitRate_t3F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=3, trim=True)
    res["HitRate_t0.5P"], res["HitRate_t0.5R"], res["HitRate_t0.5F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=.5, trim=True)

    # Hit rate weighted
    _, _, res["HitRate_w3F"] = mir_eval.segment.detection(
        ann_inter, est_inter, window=3, trim=False, beta=weight)
    _, _, res["HitRate_w0.5F"] = mir_eval.segment.detection(
        ann_inter, est_inter, window=.5, trim=False, beta=weight)

    # Hit rate weighted and trimmed
    _, _, res["HitRate_wt3F"] = mir_eval.segment.detection(
        ann_inter, est_inter, window=3, trim=True, beta=weight)
    _, _, res["HitRate_wt0.5F"] = mir_eval.segment.detection(
        ann_inter, est_inter, window=.5, trim=True, beta=weight)

    # Information gain
    res["D"] = compute_information_gain(ann_inter, est_inter, est_file,
                                        bins=bins)

    # Median Deviations
    res["DevR2E"], res["DevE2R"] = mir_eval.segment.deviation(
        ann_inter, est_inter, trim=False)
    res["DevtR2E"], res["DevtE2R"] = mir_eval.segment.deviation(
        ann_inter, est_inter, trim=True)

    # --Labels-- #
    if est_labels is not None and ("-1" in est_labels or "@" in est_labels):
        est_labels = None
    if est_labels is not None and len(est_labels) != 0:
        # Align labels with intervals
        ann_labels = list(ann_labels)
        est_labels = list(est_labels)
        ann_inter, ann_labels = mir_eval.util.adjust_intervals(ann_inter,
                                                               ann_labels)
        est_inter, est_labels = mir_eval.util.adjust_intervals(
            est_inter, est_labels, t_min=0.0, t_max=ann_inter.max())

        # Pair-wise frame clustering
        res["PWP"], res["PWR"], res["PWF"] = mir_eval.segment.pairwise(
            ann_inter, ann_labels, est_inter, est_labels)

        # Normalized Conditional Entropies
        res["So"], res["Su"], res["Sf"] = mir_eval.segment.nce(
            ann_inter, ann_labels, est_inter, est_labels)

    # Names
    base = os.path.basename(est_file)
    res["track_id"] = base[:-5]
    res["ds_name"] = base.split("_")[0]

    return res

def compute_gt_results(est_file, ref_file, boundaries_id, labels_id, config,
                       bins=251, annotator_id=0):
    """Computes the results by using the ground truth dataset identified by
    the annotator parameter.

    Return
    ------
    results : dict
        Dictionary of the results (see function compute_results).
    """
    if config["hier"]:
        ref_times, ref_labels, ref_levels = \
            msaf.io.read_hier_references(
                ref_file, annotation_id=annotator_id,
                exclude_levels=["segment_salami_function"])
    else:
        jam = jams.load(ref_file, validate=False)
        ann = jam.search(namespace='segment_.*')[annotator_id]
        ref_inter, ref_labels = ann.to_interval_values()

    # Read estimations with correct configuration
    est_inter, est_labels = io.read_estimations(est_file, boundaries_id,
                                                labels_id, **config)

    # Compute the results and return
    logging.info("Evaluating %s" % os.path.basename(est_file))
    if config["hier"]:
        # Hierarchical
        assert len(est_inter) == len(est_labels), "Same number of levels " \
            "are required in the boundaries and labels for the hierarchical " \
            "evaluation."
        est_times = []
        est_labels = []

        # Sort based on how many segments per level
        est_inter = sorted(est_inter, key=lambda level: len(level))

        for inter in est_inter:
            est_times.append(msaf.utils.intervals_to_times(inter))
            # Add fake labels (hierarchical eval does not use labels --yet--)
            est_labels.append(np.ones(len(est_times[-1]) - 1) * -1)

        # Align the times
        utils.align_end_hierarchies(est_times, ref_times, thres=1)

        # To intervals
        est_hier = [utils.times_to_intervals(times) for times in est_times]
        ref_hier = [utils.times_to_intervals(times) for times in ref_times]

        # Compute evaluations
        res = {}
        res["t_recall10"], res["t_precision10"], res["t_measure10"] = \
            mir_eval.hierarchy.tmeasure(ref_hier, est_hier, window=10)
        res["t_recall15"], res["t_precision15"], res["t_measure15"] = \
            mir_eval.hierarchy.tmeasure(ref_hier, est_hier, window=15)

        res["track_id"] = os.path.basename(est_file)[:-5]
        return res
    else:
        # Flat
        return compute_results(ref_inter, est_inter, ref_labels, est_labels,
                               bins, est_file)

def compute_information_gain(ann_inter, est_inter, est_file, bins):
    """Computes the information gain of the est_file from the annotated
    intervals and the estimated intervals."""
    ann_times = utils.intervals_to_times(ann_inter)
    est_times = utils.intervals_to_times(est_inter)
    return mir_eval.beat.information_gain(ann_times, est_times, bins=bins)

def process_track(file_struct, boundaries_id, labels_id, config,
                  annotator_id=0):
    """Processes a single track.

    Parameters
    ----------
    file_struct : object (FileStruct) or str
        File struct or full path of the audio file to be evaluated.
    boundaries_id : str
        Identifier of the boundaries algorithm.
    labels_id : str
        Identifier of the labels algorithm.
    config : dict
        Configuration of the algorithms to be evaluated.
    annotator_id : int
        Number identifiying the annotator.

    Returns
    -------
    one_res : dict
        Dictionary of the results (see function compute_results).
    """
    # Convert to file_struct if string is passed
    if isinstance(file_struct, six.string_types):
        file_struct = io.FileStruct(file_struct)

    est_file = file_struct.est_file
    ref_file = file_struct.ref_file

    # Sanity check
    assert os.path.basename(est_file)[:-4] == \
        os.path.basename(ref_file)[:-4], "File names are different %s --- %s" \
        % (os.path.basename(est_file)[:-4], os.path.basename(ref_file)[:-4])

    if not os.path.isfile(ref_file):
        raise NoReferencesError("Reference file %s does not exist. You must "
                                "have annotated references to run "
                                "evaluations." % ref_file)

    one_res = compute_gt_results(est_file, ref_file, boundaries_id, labels_id,
                                 config, annotator_id=annotator_id)

    return one_res

def get_results_file_name(boundaries_id, labels_id, config,
                          annotator_id):
    """Based on the config and the dataset, get the file name to store the
    results."""
    utils.ensure_dir(msaf.config.results_dir)
    file_name = os.path.join(msaf.config.results_dir, "results")
    file_name += "_boundsE%s_labelsE%s" % (boundaries_id, labels_id)
    file_name += "_annotatorE%d" % (annotator_id)
    sorted_keys = sorted(config.keys(), key=str.lower)
    for key in sorted_keys:
        file_name += "_%sE%s" % (key, str(config[key]).replace("/", "_"))

    # Check for max file length
    if len(file_name) > 255 - len(msaf.config.results_ext):
        file_name = file_name[:255 - len(msaf.config.results_ext)]

    return file_name + msaf.config.results_ext

def process(in_path, boundaries_id=msaf.config.default_bound_id,
            labels_id=msaf.config.default_label_id, annot_beats=False,
            framesync=False, feature="pcp", hier=False, save=False,
            out_file=None, n_jobs=4, annotator_id=0, config=None):
    """Main process to evaluate algorithms' results.

    Parameters
    ----------
    in_path : str
        Path to the dataset root folder.
    boundaries_id : str
        Boundaries algorithm identifier (e.g. siplca, cnmf)
    labels_id : str
        Labels algorithm identifier (e.g. siplca, cnmf)
    ds_name : str
        Name of the dataset to be evaluated (e.g. SALAMI). * stands for all.
    annot_beats : boolean
        Whether to use the annotated beats or not.
    framesync: str
        Whether to use framesync features or not (default: False -> beatsync)
    feature: str
        String representing the feature to be used (e.g. pcp, mfcc, tonnetz)
    hier : bool
        Whether to compute a hierarchical or flat segmentation.
    save: boolean
        Whether to save the results into the `out_file` csv file.
    out_file: str
        Path to the csv file to save the results (if `None` and `save = True`
        it will save the results in the default file name obtained by
        calling `get_results_file_name`).
    n_jobs: int
        Number of processes to run in parallel. Only available in collection
        mode.
    annotator_id : int
        Number identifiying the annotator.
    config: dict
        Dictionary containing custom configuration parameters for the
        algorithms.  If None, the default parameters are used.

    Return
    ------
    results : pd.DataFrame
        DataFrame containing the evaluations for each file.
    """

    # Set up configuration based on algorithms parameters
    if config is None:
        config = io.get_configuration(feature, annot_beats, framesync,
                                      boundaries_id, labels_id)

    # Hierarchical segmentation
    config["hier"] = hier

    # Remove actual features
    config.pop("features", None)

    # Get out file in case we want to save results
    if out_file is None:
        out_file = get_results_file_name(boundaries_id, labels_id, config,
                                         annotator_id)

    # If out_file already exists, read and return them
    if os.path.exists(out_file):
        logging.warning("Results already exists, reading from file %s" %
                        out_file)
        results = pd.read_csv(out_file)
        print_results(results)
        return results

    # Perform actual evaluations
    if os.path.isfile(in_path):
        # Single File mode
        evals = [process_track(in_path, boundaries_id, labels_id, config,
                               annotator_id=annotator_id)]
    else:
        # Collection mode
        # Get files
        file_structs = io.get_dataset_files(in_path)

        # Evaluate in parallel
        logging.info("Evaluating %d tracks..." % len(file_structs))
        evals = Parallel(n_jobs=n_jobs)(delayed(process_track)(
            file_struct, boundaries_id, labels_id, config,
            annotator_id=annotator_id) for file_struct in file_structs[:])

    # Aggregate evaluations in pandas format
    results = pd.DataFrame()
    for e in evals:
        if e != []:
            results = results.append(e, ignore_index=True)
    logging.info("%d tracks analyzed" % len(results))

    # Print results
    print_results(results)

    # Save all results
    if save:
        logging.info("Writing results in %s" % out_file)
        results.to_csv(out_file)

    return results


"""This script contains relevant functions to read the configuration for
MSAF.
A bunch of it is basically shamefully copy pasted from the almighty theano."""

from __future__ import absolute_import, print_function, division
try:
    from configparser import (ConfigParser, NoOptionError, NoSectionError,
                              InterpolationError)
except ImportError:
    from six.moves.configparser import (ConfigParser, NoOptionError, NoSectionError,
                                        InterpolationError)
import os
import shlex
from six import StringIO
from six import string_types
import sys
import warnings

import msaf

MSAF_FLAGS = os.getenv(msaf.MSAF_FLAGS_VAR, "")
# The MSAF_FLAGS environment variable should be a list of comma-separated
# [section.]option=value entries. If the section part is omitted, there should
# be only one section that contains the given option.

class MsafConfigWarning(Warning):
    def warn(cls, message, stacklevel=0):
        warnings.warn(message, cls, stacklevel=stacklevel + 3)
    warn = classmethod(warn)

def parse_config_string(config_string, issue_warnings=True):
    """
    Parses a config string (comma-separated key=value components) into a dict.
    """
    config_dict = {}
    my_splitter = shlex.shlex(config_string, posix=True)
    my_splitter.whitespace = ','
    my_splitter.whitespace_split = True
    for kv_pair in my_splitter:
        kv_pair = kv_pair.strip()
        if not kv_pair:
            continue
        kv_tuple = kv_pair.split('=', 1)
        if len(kv_tuple) == 1:
            if issue_warnings:
                MsafConfigWarning.warn(
                    ("Config key '%s' has no value, ignoring it" %
                     kv_tuple[0]), stacklevel=1)
        else:
            k, v = kv_tuple
            # subsequent values for k will override earlier ones
            config_dict[k] = v
    return config_dict

MSAF_FLAGS_DICT = parse_config_string(MSAF_FLAGS, issue_warnings=True)

# MSAFRC can contain a colon-delimited list of config files, like
# MSAFRC=~rincewind/.msafrc:~/.msafrc
# In that case, definitions in files on the right (here, ~/.msafrc) have
# precedence over those in files on the left.
def config_files_from_msafrc():
    rval = [os.path.expanduser(s) for s in
            os.getenv(msaf.MSAFRC_VAR, msaf.MSAFRC_FILE).split(os.pathsep)]
    if os.getenv(msaf.MSAFRC_VAR) is None and sys.platform == "win32":
        # to don't need to change the filename and make it open easily
        rval.append(os.path.expanduser(msaf.MSAFRC_WIN_FILE))
    return rval

config_files = config_files_from_msafrc()
msaf_cfg = ConfigParser(
    {'USER': os.getenv("USER", os.path.split(os.path.expanduser('~'))[-1]),
     'LSCRATCH': os.getenv("LSCRATCH", ""),
     'TMPDIR': os.getenv("TMPDIR", ""),
     'TEMP': os.getenv("TEMP", ""),
     'TMP': os.getenv("TMP", ""),
     'PID': str(os.getpid()),
     }
)
msaf_cfg.read(config_files)
# Having a raw version of the config around as well enables us to pass
# through config values that contain format strings.
# The time required to parse the config twice is negligible.
msaf_raw_cfg = ConfigParser()
msaf_raw_cfg.read(config_files)

def fetch_val_for_key(key, delete_key=False):
    """Return the overriding config value for a key.
    A successful search returns a string value.
    An unsuccessful search raises a KeyError
    The (decreasing) priority order is:
    - MSAF_FLAGS
    - ~./msafrc
    """

    # first try to find it in the FLAGS
    try:
        if delete_key:
            return MSAF_FLAGS_DICT.pop(key)
        return MSAF_FLAGS_DICT[key]
    except KeyError:
        pass

    # next try to find it in the config file

    # config file keys can be of form option, or section.option
    key_tokens = key.rsplit('.', 1)
    if len(key_tokens) == 2:
        section, option = key_tokens
    else:
        section, option = 'global', key
    try:
        try:
            return msaf_cfg.get(section, option)
        except InterpolationError:
            return msaf_raw_cfg.get(section, option)
    except (NoOptionError, NoSectionError):
        raise KeyError(key)

_config_var_list = []

def _config_print(thing, buf, print_doc=True):
    for cv in _config_var_list:
        print(cv, file=buf)
        if print_doc:
            print("    Doc: ", cv.doc, file=buf)
        print("    Value: ", cv.__get__(True, None), file=buf)
        print("", file=buf)

class MsafConfigParser(object):
    # properties are installed by AddConfigVar
    _i_am_a_config_class = True

    def __str__(self, print_doc=True):
        sio = StringIO()
        _config_print(self.__class__, sio, print_doc=print_doc)
        return sio.getvalue()

# N.B. all instances of MsafConfigParser give access to the same properties.
config = MsafConfigParser()

# The data structure at work here is a tree of CLASSES with
# CLASS ATTRIBUTES/PROPERTIES that are either a) INSTANTIATED
# dynamically-generated CLASSES, or b) ConfigParam instances.  The root
# of this tree is the MsafConfigParser CLASS, and the internal nodes
# are the SubObj classes created inside of AddConfigVar().
# Why this design ?
# - The config object is a true singleton.  Every instance of
#   MsafConfigParser is an empty instance that looks up attributes/properties
#   in the [single] MsafConfigParser.__dict__
# - The subtrees provide the same interface as the root
# - ConfigParser subclasses control get/set of config properties to guard
#   against craziness.
def AddConfigVar(name, doc, configparam, root=config):
    """Add a new variable to msaf.config

    Parameters
    ----------
    name: str
        String of the form "[section0.[section1.[etc]]]option", containing the
        full name for this configuration variable.
    string: str
        What does this variable specify?
    configparam: `ConfigParam`
        An object for getting and setting this configuration parameter.
    root: object
        Used for recursive calls -- do not provide an argument for this
        parameter.
    """

    # This method also performs some of the work of initializing ConfigParam
    # instances

    if root is config:
        # only set the name in the first call, not the recursive ones
        configparam.fullname = name
    sections = name.split('.')
    if len(sections) > 1:
        # set up a subobject
        if not hasattr(root, sections[0]):
            # every internal node in the config tree is an instance of its own
            # unique class
            class SubObj(object):
                _i_am_a_config_class = True
            setattr(root.__class__, sections[0], SubObj())
        newroot = getattr(root, sections[0])
        if (not getattr(newroot, '_i_am_a_config_class', False) or
                isinstance(newroot, type)):
            raise TypeError(
                'Internal config nodes must be config class instances',
                newroot)
        return AddConfigVar('.'.join(sections[1:]), doc, configparam,
                            root=newroot)
    else:
        if hasattr(root, name):
            raise AttributeError('This name is already taken',
                                 configparam.fullname)
        configparam.doc = doc
        # Trigger a read of the value from config files and env vars
        # This allow to filter wrong value from the user.
        if not callable(configparam.default):
            configparam.__get__(root, type(root), delete_key=True)
        else:
            # We do not want to evaluate now the default value
            # when it is a callable.
            try:
                fetch_val_for_key(configparam.fullname)
                # The user provided a value, filter it now.
                configparam.__get__(root, type(root), delete_key=True)
            except KeyError:
                pass
        setattr(root.__class__, sections[0], configparam)
        _config_var_list.append(configparam)

class ConfigParam(object):

    def __init__(self, default, filter=None, allow_override=True):
        """
        If allow_override is False, we can't change the value after the import
        of Theano. So the value should be the same during all the execution.
        """
        self.default = default
        self.filter = filter
        self.allow_override = allow_override
        self.is_default = True
        # N.B. --
        # self.fullname  # set by AddConfigVar
        # self.doc       # set by AddConfigVar

        # Note that we do not call `self.filter` on the default value: this
        # will be done automatically in AddConfigVar, potentially with a
        # more appropriate user-provided default value.
        # Calling `filter` here may actually be harmful if the default value is
        # invalid and causes a crash or has unwanted side effects.

    def __get__(self, cls, type_, delete_key=False):
        if cls is None:
            return self
        if not hasattr(self, 'val'):
            try:
                val_str = fetch_val_for_key(self.fullname,
                                            delete_key=delete_key)
                self.is_default = False
            except KeyError:
                if callable(self.default):
                    val_str = self.default()
                else:
                    val_str = self.default
            self.__set__(cls, val_str)
        # print "RVAL", self.val
        return self.val

    def __set__(self, cls, val):
        if not self.allow_override and hasattr(self, 'val'):
            raise Exception(
                "Can't change the value of this config parameter "
                "after initialization!")
        # print "SETTING PARAM", self.fullname,(cls), val
        if self.filter:
            self.val = self.filter(val)
        else:
            self.val = val

class EnumStr(ConfigParam):
    def __init__(self, default, *options, **kwargs):
        self.default = default
        self.all = (default,) + options

        # All options should be strings
        for val in self.all:
            if not isinstance(val, string_types) and val is not None:
                raise ValueError('Valid values for an EnumStr parameter '
                                 'should be strings or `None`', val, type(val))

        convert = kwargs.get("convert", None)

        def filter(val):
            # uri: We want to keep None values
            if val is None:
                return val

            if convert:
                val = convert(val)
            if val in self.all:
                return val
            else:
                raise ValueError((
                    'Invalid value ("%s") for configuration variable "%s". '
                    'Valid options are %s'
                    % (val, self.fullname, self.all)))
        over = kwargs.get("allow_override", True)
        super(EnumStr, self).__init__(default, filter, over)

    def __str__(self):
        return '%s (%s) ' % (self.fullname, self.all)

class ListParam(ConfigParam):
    def __init__(self, default, *options, **kwargs):
        self.default = default
        try:
            assert len(default) > 0
        except AssertionError:
            raise ValueError("List is empty")
        except TypeError:
            raise ValueError("The parameter is not a list.")

        over = kwargs.get("allow_override", True)
        super(ListParam, self).__init__(default, None, over)

    def __str__(self):
        return '%s (%s) ' % (self.fullname, self.default)

class TypedParam(ConfigParam):
    def __init__(self, default, mytype, is_valid=None, allow_override=True):
        self.mytype = mytype

        def filter(val):
            # uri: We want to keep None values
            if val is None:
                return val
            cast_val = mytype(val)
            if callable(is_valid):
                if is_valid(cast_val):
                    return cast_val
                else:
                    raise ValueError(
                        'Invalid value (%s) for configuration variable '
                        '"%s".'
                        % (val, self.fullname), val)
            return cast_val

        super(TypedParam, self).__init__(default, filter,
                                         allow_override=allow_override)

    def __str__(self):
        return '%s (%s) ' % (self.fullname, self.mytype)

def StrParam(default, is_valid=None, allow_override=True):
    return TypedParam(default, str, is_valid, allow_override=allow_override)

def IntParam(default, is_valid=None, allow_override=True):
    return TypedParam(default, int, is_valid, allow_override=allow_override)

def FloatParam(default, is_valid=None, allow_override=True):
    return TypedParam(default, float, is_valid, allow_override=allow_override)

def BoolParam(default, is_valid=None, allow_override=True):
    # see comment at the beginning of this file.

    def booltype(s):
        if s in ['False', 'false', '0', False]:
            return False
        elif s in ['True', 'true', '1', True]:
            return True

    def is_valid_bool(s):
        if s in ['False', 'false', '0', 'True', 'true', '1', False, True]:
            return True
        else:
            return False

    if is_valid is None:
        is_valid = is_valid_bool

    return TypedParam(default, booltype, is_valid,
                      allow_override=allow_override)


"""
Variable Markov Oracle algorithm
"""
from msaf.algorithms.interface import SegmenterInterface
import librosa
from . import main

class Segmenter(SegmenterInterface):
    """
    This script identifies the structure of a given track using the Variable
    Markov Oracle technique described here:

    Wang, C., Mysore, J. G., Structural Segmentation With The Variable Markov
    Oracle And Boundary Adjustment. Proc. of the 41st IEEE International
    Conference on Acoustics, Speech, and Signal Processing (ICASSP).
    Shanghai, China, 2016 (`PDF`_).

    .. _PDF: https://ccrma.stanford.edu/~gautham/Site/Publications_files/segmentation-icassp_2016.pdf
    """
    def processFlat(self):
        """Main process.
        Returns
        -------
        est_idxs : np.array(N)
            Estimated indeces the segment boundaries in frame indeces.
        est_labels : np.array(N-1)
            Estimated labels for the segments.
        """
        # Preprocess to obtain features (array(n_frames, n_features))

        F = self._preprocess()
        F = librosa.util.normalize(F, axis=0)
        F = librosa.feature.stack_memory(F.T).T

        self.config["hier"] = False
        my_bounds, my_labels, _ = main.scluster_segment(F, self.config, self.in_bound_idxs)

        # Post process estimations
        est_idxs, est_labels = self._postprocess(my_bounds, my_labels)

        assert est_idxs[0] == 0 and est_idxs[-1] == F.shape[0] - 1
        # We're done!
        return est_idxs, est_labels

    def processHierarchical(self):
        """Main process.for hierarchial segmentation.
        Returns
        -------
        est_idxs : list
            List with np.arrays for each layer of segmentation containing
            the estimated indeces for the segment boundaries.
        est_labels : list
            List with np.arrays containing the labels for each layer of the
            hierarchical segmentation.
        """
        F = self._preprocess()
        F = librosa.util.normalize(F, axis=0)
        F = librosa.feature.stack_memory(F.T).T

        self.config["hier"] = True
        est_idxs, est_labels, F = main.scluster_segment(F, self.config, self.in_bound_idxs)
        for layer in range(len(est_idxs)):
            assert est_idxs[layer][0] == 0 and \
                est_idxs[layer][-1] == F.shape[1] - 1
            est_idxs[layer], est_labels[layer] = \
                self._postprocess(est_idxs[layer], est_labels[layer])
        return est_idxs, est_labels


#!/usr/bin/python
#
# Copyright (C) Christian Thurau, 2010.
# Licensed under the GNU General Public License (GPL).
# http://www.gnu.org/licenses/gpl.txt
"""
PyMF Non-negative Matrix Factorization.

    NMF: Class for Non-negative Matrix Factorization

[1] Lee, D. D. and Seung, H. S. (1999), Learning the Parts of Objects by Non-negative
Matrix Factorization, Nature 401(6755), 788-799.
"""

import numpy as np
import logging
import logging.config
import scipy.sparse

__all__ = ["NMF"]

class NMF():
    """
    NMF(data, num_bases=4)

    Non-negative Matrix Factorization. Factorize a data matrix into two matrices
    s.t. F = | data - W*H | = | is minimal. H, and W are restricted to non-negative
    data. Uses the classicial multiplicative update rule.

    Parameters
    ----------
    data : array_like, shape (_data_dimension, _num_samples)
        the input data
    num_bases: int, optional
        Number of bases to compute (column rank of W and row rank of H).
        4 (default)

    Attributes
    ----------
    W : "data_dimension x num_bases" matrix of basis vectors
    H : "num bases x num_samples" matrix of coefficients
    ferr : frobenius norm (after calling .factorize())

    Example
    -------
    Applying NMF to some rather stupid data set:

    >>> import numpy as np
    >>> data = np.array([[1.0, 0.0, 2.0], [0.0, 1.0, 1.0]])
    >>> nmf_mdl = NMF(data, num_bases=2, niter=10)
    >>> nmf_mdl.factorize()

    The basis vectors are now stored in nmf_mdl.W, the coefficients in nmf_mdl.H.
    To compute coefficients for an existing set of basis vectors simply    copy W
    to nmf_mdl.W, and set compute_w to False:

    >>> data = np.array([[1.5], [1.2]])
    >>> W = np.array([[1.0, 0.0], [0.0, 1.0]])
    >>> nmf_mdl = NMF(data, num_bases=2)
    >>> nmf_mdl.W = W
    >>> nmf_mdl.factorize(niter=20, compute_w=False)

    The result is a set of coefficients nmf_mdl.H, s.t. data = W * nmf_mdl.H.
    """

    # some small value
    _EPS = 10**-8

    def __init__(self, data, num_bases=4):

        def setup_logging():
            # create logger
            self._logger = logging.getLogger("pymf")

            # add ch to logger
            if len(self._logger.handlers) < 1:
                # create console handler and set level to debug
                ch = logging.StreamHandler()
                ch.setLevel(logging.DEBUG)
                # create formatter
                formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")

                # add formatter to ch
                ch.setFormatter(formatter)

                self._logger.addHandler(ch)

        setup_logging()

        # set variables
        self.data = data
        self._num_bases = num_bases

        # initialize H and W to random values
        (self._data_dimension, self._num_samples) = self.data.shape

    def frobenius_norm(self):
        """ Frobenius norm (||data - WH||) of a data matrix and a low rank
        approximation given by WH

        Returns:
            frobenius norm: F = ||data - WH||
        """

        # check if W and H exist
        if hasattr(self,'H') and hasattr(self,'W') and not scipy.sparse.issparse(self.data):
            err = np.sqrt( np.sum((self.data[:,:] - np.dot(self.W, self.H))**2 ))
        else:
            err = -123456

        return err

    def init_w(self):
        self.W = np.random.random((self._data_dimension, self._num_bases))

    def init_h(self):
        self.H = np.random.random((self._num_bases, self._num_samples))

    def update_h(self):
            # pre init H1, and H2 (necessary for storing matrices on disk)
            H2 = np.dot(np.dot(self.W.T, self.W), self.H) + 10**-9
            self.H *= np.dot(self.W.T, self.data[:,:])
            self.H /= H2

    def update_w(self):
            # pre init W1, and W2 (necessary for storing matrices on disk)
            W2 = np.dot(np.dot(self.W, self.H), self.H.T) + 10**-9
            self.W *= np.dot(self.data[:,:], self.H.T)
            self.W /= W2

    def converged(self, i):
        derr = np.abs(self.ferr[i] - self.ferr[i-1])/self._num_samples
        if derr < self._EPS:
            return True
        else:
            return False

    def factorize(self, niter=1, show_progress=False,
                  compute_w=True, compute_h=True, compute_err=True):
        """ Factorize s.t. WH = data

            Parameters
            ----------
            niter : int
                    number of iterations.
            show_progress : bool
                    print some extra information to stdout.
            compute_h : bool
                    iteratively update values for H.
            compute_w : bool
                    iteratively update values for W.
            compute_err : bool
                    compute Frobenius norm |data-WH| after each update and store
                    it to .ferr[k].

            Updated Values
            --------------
            .W : updated values for W.
            .H : updated values for H.
            .ferr : Frobenius norm |data-WH| for each iteration.
        """

        if show_progress:
            self._logger.setLevel(logging.INFO)
        else:
            self._logger.setLevel(logging.ERROR)

        # create W and H if they don't already exist
        # -> any custom initialization to W,H should be done before
        if not hasattr(self,'W'):
               self.init_w()

        if not hasattr(self,'H'):
                self.init_h()

        if compute_err:
            self.ferr = np.zeros(niter)

        for i in range(niter):
            if compute_w:
                self.update_w()

            if compute_h:
                self.update_h()

            if compute_err:
                self.ferr[i] = self.frobenius_norm()
                self._logger.info('Iteration ' + str(i+1) + '/' + str(niter) +
                ' FN:' + str(self.ferr[i]))
            else:
                self._logger.info('Iteration ' + str(i+1) + '/' + str(niter))

            # check if the err is not changing anymore
            if i > 1 and compute_err:
                if self.converged(i):
                    # adjust the error measure
                    self.ferr = self.ferr[:i]
                    break

if __name__ == "__main__":
    import doctest
    doctest.testmod()


#!/usr/bin/env python
"""
This script computes the features for a single or multiple audio files.
It uses the default parameters. These parameters can be changed the
`.msafrc` config file.

Examples:

    Single file mode:
        >> ./compute_features.py path_to_audio.mp3 -o my_features.json

    Collection mode:
        Run on 12 cores on the specified dataset:
            >> ./compute_features.py path_to_dataset/ -j 12
"""

import argparse
from joblib import Parallel, delayed
import logging
import os
import time

# Local stuff
import msaf
from msaf.features import Features

def compute_all_features(file_struct, framesync):
    """Computes all features for the given file."""
    for feature_id in msaf.features_registry:
        logging.info("Computing %s for file %s" % (feature_id,
                                                   file_struct.audio_file))
        feats = Features.select_features(feature_id, file_struct, False, framesync)
        feats.features

def process(in_path, out_file, n_jobs, framesync):
    """Computes the features for the selected dataset or file."""
    if os.path.isfile(in_path):
        # Single file mode
        # Get (if they exitst) or compute features
        file_struct = msaf.io.FileStruct(in_path)
        file_struct.features_file = out_file
        compute_all_features(file_struct, framesync)
    else:
        # Collection mode
        file_structs = msaf.io.get_dataset_files(in_path)

        # Call in parallel
        return Parallel(n_jobs=n_jobs)(delayed(compute_all_features)(
            file_struct, framesync) for file_struct in file_structs)

def main():
    """Main function to parse the arguments and call the main process."""
    parser = argparse.ArgumentParser(
        description="Extracts a set of features from a given dataset "
        "or audio file and saves them into the 'features' folder of "
        "the dataset or the specified single file.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("in_path",
                        action="store",
                        help="Input dataset dir or audio file")
    parser.add_argument("-j",
                        action="store",
                        dest="n_jobs",
                        type=int,
                        help="Number of jobs (only for collection mode)",
                        default=4)
    parser.add_argument("-o",
                        action="store",
                        dest="out_file",
                        type=str,
                        help="Output file (only for single file mode)",
                        default="out.json")
    parser.add_argument("-d",
                        action="store",
                        dest="ds_name",
                        default="*",
                        help="The prefix of the dataset to use "
                        "(e.g. Isophonics, SALAMI)")
    parser.add_argument("-fs",
                        action="store_true",
                        dest="framesync",
                        help="Use frame-synchronous features",
                        default=False)
    args = parser.parse_args()
    start_time = time.time()

    # Setup the logger
    logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s',
        level=logging.INFO)

    # Run the main process
    process(args.in_path, out_file=args.out_file, n_jobs=args.n_jobs, framesync=args.framesync)

    # Done!
    logging.info("Done! Took %.2f seconds." % (time.time() - start_time))

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# CREATED:2013-08-22 12:20:01 by Brian McFee <brm2132@columbia.edu>
'''Music segmentation using timbre, pitch, repetition and time.
'''
import argparse
import logging
import sys

import numpy as np
import scipy.signal
import scipy.linalg

import librosa
import msaf

from msaf.algorithms.interface import SegmenterInterface
from msaf.base import Features

REP_WIDTH = 3
REP_FILTER = 7
N_MFCC = 32
N_CHROMA = 12
N_REP = 32

# mfcc, chroma, repetitions for each, and 4 time features
__DIMENSION = N_MFCC + N_CHROMA + 2 * N_REP + 4

def features(file_struct, annot_beats=False, framesync=False):
    '''Feature-extraction for audio segmentation
    Arguments:
        file_struct -- msaf.io.FileStruct
        paths to the input files in the Segmentation dataset

    Returns:
        - X -- ndarray

            beat-synchronous feature matrix:
            MFCC (mean-aggregated)
            Chroma (median-aggregated)
            Latent timbre repetition
            Latent chroma repetition
            Time index
            Beat index

        - dur -- float
            duration of the track in seconds

    '''
    def compress_data(X, k):
        Xtemp = X.dot(X.T)
        if len(Xtemp) == 0:
            return None
        e_vals, e_vecs = np.linalg.eig(Xtemp)

        e_vals = np.maximum(0.0, np.real(e_vals))
        e_vecs = np.real(e_vecs)

        idx = np.argsort(e_vals)[::-1]

        e_vals = e_vals[idx]
        e_vecs = e_vecs[:, idx]

        # Truncate to k dimensions
        if k < len(e_vals):
            e_vals = e_vals[:k]
            e_vecs = e_vecs[:, :k]

        # Normalize by the leading singular value of X
        Z = np.sqrt(e_vals.max())

        if Z > 0:
            e_vecs = e_vecs / Z

        return e_vecs.T.dot(X)

    # Latent factor repetition features
    def repetition(X, metric='euclidean'):
        R = librosa.segment.recurrence_matrix(
            X, k=2 * int(np.ceil(np.sqrt(X.shape[1]))),
            width=REP_WIDTH, metric=metric, sym=False).astype(np.float32)

        P = scipy.signal.medfilt2d(librosa.segment.recurrence_to_lag(R),
                                   [1, REP_FILTER])

        # Discard empty rows.
        # This should give an equivalent SVD, but resolves some numerical
        # instabilities.
        P = P[P.any(axis=1)]

        return compress_data(P, N_REP)

    #########
    # '\tloading annotations and features of ', audio_path
    pcp_obj = Features.select_features("pcp", file_struct, annot_beats,
                                       framesync)
    mfcc_obj = Features.select_features("mfcc", file_struct, annot_beats,
                                        framesync)
    chroma = pcp_obj.features
    mfcc = mfcc_obj.features
    beats = pcp_obj.frame_times
    dur = pcp_obj.dur

    # Sampling Rate
    sr = msaf.config.sample_rate

    ##########
    # print '\treading beats'
    B = beats[:chroma.shape[0]]
    # beat_frames = librosa.time_to_frames(B, sr=sr,
                                         #hop_length=msaf.config.hop_size)
    #print beat_frames, len(beat_frames), uidx

    #########
    M = mfcc.T
    #plt.imshow(M, interpolation="nearest", aspect="auto"); plt.show()

    #########
    # Get the beat-sync chroma
    C = chroma.T
    C += C.min() + 0.1
    C = C / C.max(axis=0)
    C = 80 * np.log10(C)  # Normalize from -80 to 0
    #plt.imshow(C, interpolation="nearest", aspect="auto"); plt.show()

    # Time-stamp features
    N = np.arange(float(chroma.shape[0]))

    #########
    #print '\tgenerating structure features'

    # TODO:  This might fail if audio file (or number of beats) is too small
    R_timbre = repetition(librosa.feature.stack_memory(M))
    R_chroma = repetition(librosa.feature.stack_memory(C))
    if R_timbre is None or R_chroma is None:
        return None, dur

    R_timbre += R_timbre.min()
    R_timbre /= R_timbre.max()
    R_chroma += R_chroma.min()
    R_chroma /= R_chroma.max()
    #plt.imshow(R_chroma, interpolation="nearest", aspect="auto"); plt.show()

    # Stack it all up
    #print M.shape, C.shape, R_timbre.shape, R_chroma.shape, len(B), len(N)
    X = np.vstack([M, C, R_timbre, R_chroma, B, B / dur, N,
                   N / float(chroma.shape[0])])

    #plt.imshow(X, interpolation="nearest", aspect="auto"); plt.show()

    return X, dur

def gaussian_cost(X):
    '''Return the average log-likelihood of data under a standard normal
    '''

    d, n = X.shape

    if n < 2:
        return 0

    sigma = np.var(X, axis=1, ddof=1)

    cost = -0.5 * d * n * np.log(2. * np.pi) - 0.5 * (n - 1.) * np.sum(sigma)
    return cost

def clustering_cost(X, boundaries):

    # Boundaries include beginning and end frames, so k is one less
    k = len(boundaries) - 1

    d, n = map(float, X.shape)

    # Compute the average log-likelihood of each cluster
    cost = [gaussian_cost(X[:, start:end]) for
            (start, end) in zip(boundaries[:-1], boundaries[1:])]

    cost = - 2 * np.sum(cost) / n + 2 * (d * k)

    return cost

def get_k_segments(X, k):

    # Step 1: run ward
    boundaries = librosa.segment.agglomerative(X, k)

    # Add first and last boundary indeces
    boundaries = np.unique(np.concatenate(([0], boundaries, [X.shape[1]-1])))

    # Step 2: compute cost
    cost = clustering_cost(X, boundaries)

    return boundaries, cost

def get_segments(X, kmin=8, kmax=32):

    cost_min = np.inf
    S_best = []
    best_k = -1
    for k in range(kmax, kmin, -1):
        S, cost = get_k_segments(X, k)
        if cost < cost_min:
            cost_min = cost
            S_best = S
            best_k = k
        else:
            break

    return S_best

def process_arguments():
    parser = argparse.ArgumentParser(description='Music segmentation')

    parser.add_argument('-t',
                        '--transform',
                        dest='transform',
                        required=False,
                        type=str,
                        help='npy file containing the linear projection',
                        default=None)

    parser.add_argument('input_song',
                        action='store',
                        help='path to input audio data')

    parser.add_argument('output_file',
                        action='store',
                        help='path to output segment file')

    return vars(parser.parse_args(sys.argv[1:]))

def load_transform(transform_file):

    if transform_file is None:
        W = np.eye(__DIMENSION)
    else:
        W = np.load(transform_file)

    return W

def get_num_segs(duration, MIN_SEG=10.0, MAX_SEG=45.0):
    kmin = max(1, np.floor(duration / MAX_SEG).astype(int))
    kmax = max(2, np.ceil(duration / MIN_SEG).astype(int))

    return kmin, kmax

class Segmenter(SegmenterInterface):
    """
    This class implements the algorithm described here:

    McFee, B. and Ellis, D.P.W., Learning to segment songs with ordinal linear
    discriminant analysis. International conference on acoustics, speech and
    signal processing (ICASSP). 2014 (`PDF`_).

    .. _PDF: https://bmcfee.github.io/papers/icassp2014_segments.pdf
    """
    def processFlat(self):
        """Main process for flat segmentation.
        Returns
        -------
        est_idxs : np.array(N)
            Estimated times for the segment boundaries in frame indeces.
        est_labels : np.array(N-1)
            Estimated labels for the segments.
        """
        # Preprocess to obtain features and duration
        F, dur = features(self.file_struct, self.annot_beats, self.framesync)

        try:
            # Load and apply transform
            W = load_transform(self.config["transform"])
            F = W.dot(F)

            # Get Segments
            kmin, kmax = get_num_segs(dur)
            est_idxs = get_segments(F, kmin=kmin, kmax=kmax)
        except:
            # The audio file is too short, only beginning and end
            logging.warning("Audio file too short! "
                            "Only start and end boundaries.")
            est_idxs = [0, F.shape[1] - 1]

        # Make sure that the first and last boundaries are included
        assert est_idxs[0] == 0 and est_idxs[-1] == F.shape[1] - 1

        # Empty labels
        est_labels = np.ones(len(est_idxs) - 1) * -1

        # Post process estimations
        est_idxs, est_labels = self._postprocess(est_idxs, est_labels)

        return est_idxs, est_labels

    def processHierarchical(self):
        """Main process for hierarchical segmentation.
        Returns
        -------
        est_idxs : list
            List containing estimated times for each layer in the hierarchy
            as np.arrays
        est_labels : list
            List containing estimated labels for each layer in the hierarchy
            as np.arrays
        """
        # Preprocess to obtain features, times, and input boundary indeces
        F, dur = features(self.file_struct, self.annot_beats, self.framesync)

        try:
            # Load and apply transform
            W = load_transform(self.config["transform"])
            F = W.dot(F)

            # Get Segments
            kmin, kmax = get_num_segs(dur)

            # Run algorithm layer by layer
            est_idxs = []
            est_labels = []
            for k in range(kmin, kmax):
                S, cost = get_k_segments(F, k)
                est_idxs.append(S)
                est_labels.append(np.ones(len(S) - 1) * -1)

                # Make sure that the first and last boundaries are included
                assert est_idxs[-1][0] == 0 and \
                    est_idxs[-1][-1] == F.shape[1] - 1, "Layer %d does not " \
                    "start or end in the right frame(s)." % k

                # Post process layer
                est_idxs[-1], est_labels[-1] = \
                        self._postprocess(est_idxs[-1], est_labels[-1])
        except:
            # The audio file is too short, only beginning and end
            logging.warning("Audio file too short! "
                            "Only start and end boundaries.")
            est_idxs = [np.array([0, F.shape[1] - 1])]
            est_labels = [np.ones(1) * -1]

        return est_idxs, est_labels


"""
Useful functions that are common in MSAF
"""
import librosa
import mir_eval
import numpy as np
import os
import scipy.io.wavfile
import six

def lognormalize(F, floor=0.1, min_db=-80):
    """Log-normalizes features such that each vector is between min_db to 0."""
    assert min_db < 0
    F = min_max_normalize(F, floor=floor)
    F = np.abs(min_db) * np.log10(F)  # Normalize from min_db to 0
    return F

def min_max_normalize(F, floor=0.001):
    """Normalizes features such that each vector is between floor to 1."""
    F += -F.min() + floor
    F = F / F.max(axis=0)
    return F

def normalize(X, norm_type, floor=0.0, min_db=-80):
    """Normalizes the given matrix of features.

    Parameters
    ----------
    X: np.array
        Each row represents a feature vector.
    norm_type: {"min_max", "log", np.inf, -np.inf, 0, float > 0, None}
        - `"min_max"`: Min/max scaling is performed
        - `"log"`: Logarithmic scaling is performed
        - `np.inf`: Maximum absolute value
        - `-np.inf`: Mininum absolute value
        - `0`: Number of non-zeros
        - float: Corresponding l_p norm.
        - None : No normalization is performed

    Returns
    -------
    norm_X: np.array
        Normalized `X` according the the input parameters.
    """
    if isinstance(norm_type, six.string_types):
        if norm_type == "min_max":
            return min_max_normalize(X, floor=floor)
        if norm_type == "log":
            return lognormalize(X, floor=floor, min_db=min_db)
    return librosa.util.normalize(X, norm=norm_type, axis=1)

def ensure_dir(directory):
    """Makes sure that the given directory exists."""
    if not os.path.exists(directory):
        os.makedirs(directory)

def times_to_intervals(times):
    """Given a set of times, convert them into intervals.

    Parameters
    ----------
    times: np.array(N)
        A set of times.

    Returns
    -------
    inters: np.array(N-1, 2)
        A set of intervals.
    """
    return np.asarray(list(zip(times[:-1], times[1:])))

def intervals_to_times(inters):
    """Given a set of intervals, convert them into times.

    Parameters
    ----------
    inters: np.array(N-1, 2)
        A set of intervals.

    Returns
    -------
    times: np.array(N)
        A set of times.
    """
    return np.concatenate((inters.flatten()[::2], [inters[-1, -1]]), axis=0)

def get_num_frames(dur, anal):
    """Given the duration of a track and a dictionary containing analysis
    info, return the number of frames."""
    total_samples = dur * anal["sample_rate"]
    return int(total_samples / anal["hop_size"])

def get_time_frames(dur, anal):
    """Gets the time frames and puts them in a numpy array."""
    n_frames = get_num_frames(dur, anal)
    return np.linspace(0, dur, num=n_frames)

def remove_empty_segments(times, labels):
    """Removes empty segments if needed."""
    assert len(times) - 1 == len(labels)
    inters = times_to_intervals(times)
    new_inters = []
    new_labels = []
    for inter, label in zip(inters, labels):
        if inter[0] < inter[1]:
            new_inters.append(inter)
            new_labels.append(label)
    return intervals_to_times(np.asarray(new_inters)), new_labels

def sonify_clicks(audio, clicks, out_file, fs, offset=0):
    """Sonifies the estimated times into the output file.

    Parameters
    ----------
    audio: np.array
        Audio samples of the input track.
    clicks: np.array
        Click positions in seconds.
    out_file: str
        Path to the output file.
    fs: int
        Sample rate.
    offset: float
        Offset of the clicks with respect to the audio.
    """
    # Generate clicks (this should be done by mir_eval, but its
    # latest release is not compatible with latest numpy)
    times = clicks + offset
    # 1 kHz tone, 100ms
    click = np.sin(2 * np.pi * np.arange(fs * .1) * 1000 / (1. * fs))
    # Exponential decay
    click *= np.exp(-np.arange(fs * .1) / (fs * .01))
    length = int(times.max() * fs + click.shape[0] + 1)
    audio_clicks = mir_eval.sonify.clicks(times, fs, length=length)

    # Create array to store the audio plus the clicks
    out_audio = np.zeros(max(len(audio), len(audio_clicks)))

    # Assign the audio and the clicks
    out_audio[:len(audio)] = audio
    out_audio[:len(audio_clicks)] += audio_clicks

    # Write to file
    scipy.io.wavfile.write(out_file, fs, out_audio)

def synchronize_labels(new_bound_idxs, old_bound_idxs, old_labels, N):
    """Synchronizes the labels from the old_bound_idxs to the new_bound_idxs.

    Parameters
    ----------
    new_bound_idxs: np.array
        New indeces to synchronize with.
    old_bound_idxs: np.array
        Old indeces, same shape as labels + 1.
    old_labels: np.array
        Labels associated to the old_bound_idxs.
    N: int
        Total number of frames.

    Returns
    -------
    new_labels: np.array
        New labels, synchronized to the new boundary indeces.
    """
    assert len(old_bound_idxs) - 1 == len(old_labels)

    # Construct unfolded labels array
    unfold_labels = np.zeros(N)
    for i, (bound_idx, label) in enumerate(
            zip(old_bound_idxs[:-1], old_labels)):
        unfold_labels[bound_idx:old_bound_idxs[i + 1]] = label

    # Constuct new labels
    new_labels = np.zeros(len(new_bound_idxs) - 1)
    for i, bound_idx in enumerate(new_bound_idxs[:-1]):
        new_labels[i] = np.median(
            unfold_labels[bound_idx:new_bound_idxs[i + 1]])

    return new_labels

def process_segmentation_level(est_idxs, est_labels, N, frame_times, dur):
    """Processes a level of segmentation, and converts it into times.

    Parameters
    ----------
    est_idxs: np.array
        Estimated boundaries in frame indeces.
    est_labels: np.array
        Estimated labels.
    N: int
        Number of frames in the whole track.
    frame_times: np.array
        Time stamp for each frame.
    dur: float
        Duration of the audio track.

    Returns
    -------
    est_times: np.array
        Estimated segment boundaries in seconds.
    est_labels: np.array
        Estimated labels for each segment.
    """
    assert est_idxs[0] == 0 and est_idxs[-1] == N - 1
    assert len(est_idxs) - 1 == len(est_labels)

    # Add silences, if needed
    est_times = np.concatenate(([0], frame_times[est_idxs], [dur]))
    silence_label = np.max(est_labels) + 1
    est_labels = np.concatenate(([silence_label], est_labels, [silence_label]))

    # Remove empty segments if needed
    est_times, est_labels = remove_empty_segments(est_times, est_labels)

    # Make sure that the first and last times are 0 and duration, respectively
    assert np.allclose([est_times[0]], [0]) and \
        np.allclose([est_times[-1]], [dur])

    return est_times, est_labels

def align_end_hierarchies(hier1, hier2, thres=0.5):
    """Align the end of the hierarchies such that they end at the same exact
    second as long they have the same duration within a certain threshold.

    Parameters
    ----------
    hier1: list
        List containing hierarchical segment boundaries.
    hier2: list
        List containing hierarchical segment boundaries.
    thres: float > 0
        Threshold to decide whether two values are the same.
    """
    # Make sure we have correctly formatted hierarchies
    dur_h1 = hier1[0][-1]
    for hier in hier1:
        assert hier[-1] == dur_h1, "hier1 is not correctly " \
            "formatted {} {}".format(hier[-1], dur_h1)
    dur_h2 = hier2[0][-1]
    for hier in hier2:
        assert hier[-1] == dur_h2, "hier2 is not correctly formatted"

    # If durations are different, do nothing
    if abs(dur_h1 - dur_h2) > thres:
        return

    # Align h1 with h2
    for hier in hier1:
        hier[-1] = dur_h2


#!/usr/bin/python
#
# Copyright (C) Christian Thurau, 2010.
# Licensed under the GNU General Public License (GPL).
# http://www.gnu.org/licenses/gpl.txt
"""
PyMF Simplex Volume Maximization [1]

    SIVM: class for SiVM

[1] C. Thurau, K. Kersting, and C. Bauckhage. Yes We Can - Simplex Volume
Maximization for Descriptive Web-Scale Matrix Factorization. In Proc. Int.
Conf. on Information and Knowledge Management. ACM. 2010.
"""

import scipy.sparse
import numpy as np

from .dist import *
from .aa import AA

__all__ = ["SIVM"]

class SIVM(AA):
    """
    SIVM(data, num_bases=4, dist_measure='l2')

    Simplex Volume Maximization. Factorize a data matrix into two matrices s.t.
    F = | data - W*H | is minimal. H is restricted to convexity. W is iteratively
    found by maximizing the volume of the resulting simplex (see [1]).

    Parameters
    ----------
    data : array_like, shape (_data_dimension, _num_samples)
        the input data
    num_bases: int, optional
        Number of bases to compute (column rank of W and row rank of H).
        4 (default)
    dist_measure : one of 'l2' ,'cosine', 'l1', 'kl'
        Standard is 'l2' which maximizes the volume of the simplex. In contrast,
        'cosine' maximizes the volume of a cone (see [1] for details).
     init : string (default: 'fastmap')
        'fastmap' or 'origin'. Sets the method used for finding the very first
        basis vector. 'Origin' assumes the zero vector, 'Fastmap' picks one of
        the two vectors that have the largest pairwise distance.
    Attributes
    ----------
    W : "data_dimension x num_bases" matrix of basis vectors
    H : "num bases x num_samples" matrix of coefficients
    ferr : frobenius norm (after calling .factorize())

    Example
    -------
    Applying SIVM to some rather stupid data set:

    >>> import numpy as np
    >>> data = np.array([[1.0, 0.0, 2.0], [0.0, 1.0, 1.0]])
    >>> sivm_mdl = SIVM(data, num_bases=2)
    >>> sivm_mdl.factorize()

    The basis vectors are now stored in sivm_mdl.W, the coefficients in sivm_mdl.H.
    To compute coefficients for an existing set of basis vectors simply copy W
    to sivm_mdl.W, and set compute_w to False:

    >>> data = np.array([[1.5, 1.3], [1.2, 0.3]])
    >>> W = np.array([[1.0, 0.0], [0.0, 1.0]])
    >>> sivm_mdl = SIVM(data, num_bases=2)
    >>> sivm_mdl.W = W
    >>> sivm_mdl.factorize(compute_w=False)

    The result is a set of coefficients sivm_mdl.H, s.t. data = W * sivm_mdl.H.
    """

    # always overwrite the default number of iterations
    # -> any value other does not make sense.
    _NITER = 1

    def __init__(self, data, num_bases=4, dist_measure='l2',  init='fastmap'):

        AA.__init__(self, data, num_bases=num_bases)

        self._dist_measure = dist_measure
        self._init = init

        # assign the correct distance function
        if self._dist_measure == 'l1':
            self._distfunc = l1_distance

        elif self._dist_measure == 'l2':
            self._distfunc = l2_distance

        elif self._dist_measure == 'cosine':
            self._distfunc = cosine_distance

        elif self._dist_measure == 'abs_cosine':
            self._distfunc = abs_cosine_distance

        elif self._dist_measure == 'weighted_abs_cosine':
            self._distfunc = weighted_abs_cosine_distance

        elif self._dist_measure == 'kl':
            self._distfunc = kl_divergence

    def _distance(self, idx):
        """ compute distances of a specific data point to all other samples"""

        if scipy.sparse.issparse(self.data):
            step = self.data.shape[1]
        else:
            step = 50000

        d = np.zeros((self.data.shape[1]))
        if idx == -1:
            # set vec to origin if idx=-1
            vec = np.zeros((self.data.shape[0], 1))
            if scipy.sparse.issparse(self.data):
                vec = scipy.sparse.csc_matrix(vec)
        else:
            vec = self.data[:, idx:idx+1]

        self._logger.info('compute distance to node ' + str(idx))

        # slice data into smaller chunks
        for idx_start in range(0, self.data.shape[1], step):
            if idx_start + step > self.data.shape[1]:
                idx_end = self.data.shape[1]
            else:
                idx_end = idx_start + step

            d[idx_start:idx_end] = self._distfunc(
                self.data[:,idx_start:idx_end], vec)
            self._logger.info('completed:' +
                str(idx_end/(self.data.shape[1]/100.0)) + "%")
        return d

    def init_h(self):
        self.H = np.zeros((self._num_bases, self._num_samples))

    def init_w(self):
        self.W = np.zeros((self._data_dimension, self._num_bases))

    def init_sivm(self):
        self.select = []
        if self._init == 'fastmap':
            # Fastmap like initialization
            # set the starting index for fastmap initialization
            cur_p = 0

            # after 3 iterations the first "real" index is found
            for i in range(3):
                d = self._distance(cur_p)
                cur_p = np.argmax(d)

            # store maximal found distance -> later used for "a" (->update_w)
            self._maxd = np.max(d)
            self.select.append(cur_p)

        elif self._init == 'origin':
            # set first vertex to origin
            cur_p = -1
            d = self._distance(cur_p)
            self._maxd = np.max(d)
            self.select.append(cur_p)

    def update_w(self):
        """ compute new W """
        EPS = 10**-8
        self.init_sivm()

        # initialize some of the recursively updated distance measures ....
        d_square = np.zeros((self.data.shape[1]))
        d_sum = np.zeros((self.data.shape[1]))
        d_i_times_d_j = np.zeros((self.data.shape[1]))
        distiter = np.zeros((self.data.shape[1]))
        a = np.log(self._maxd)
        a_inc = a.copy()

        for l in range(1, self._num_bases):
            d = self._distance(self.select[l-1])

            # take the log of d (sually more stable that d)
            d = np.log(d + EPS)

            d_i_times_d_j += d * d_sum
            d_sum += d
            d_square += d**2
            distiter = d_i_times_d_j + a*d_sum - (l/2.0) * d_square

            # detect the next best data point
            self.select.append(np.argmax(distiter))

            self._logger.info('cur_nodes: ' + str(self.select))

        # sort indices, otherwise h5py won't work
        self.W = self.data[:, np.sort(self.select)]

        # "unsort" it again to keep the correct order
        self.W = self.W[:, np.argsort(np.argsort(self.select))]

    def factorize(self, show_progress=False, compute_w=True, compute_h=True,
                  compute_err=True, niter=1):
        """ Factorize s.t. WH = data

            Parameters
            ----------
            show_progress : bool
                    print some extra information to stdout.
            compute_h : bool
                    iteratively update values for H.
            compute_w : bool
                    iteratively update values for W.
            compute_err : bool
                    compute Frobenius norm |data-WH| after each update and store
                    it to .ferr[k].

            Updated Values
            --------------
            .W : updated values for W.
            .H : updated values for H.
            .ferr : Frobenius norm |data-WH|.
        """

        AA.factorize(self, niter=1, show_progress=show_progress,
                  compute_w=compute_w, compute_h=compute_h,
                  compute_err=compute_err)

if __name__ == "__main__":
    import doctest
    doctest.testmod()


#!/usr/bin/env python
"""Class that implements X-means."""

import argparse
import numpy as np
import time
import pylab as plt
import scipy.cluster.vq as vq
from scipy.spatial import distance

class XMeans:
    def __init__(self, X, init_K=2, plot=False):
        self.X = X
        self.init_K = init_K
        self.plot = plot

    def estimate_K_xmeans(self, th=0.2, maxK = 10):
        """Estimates K running X-means algorithm (Pelleg & Moore, 2000)."""

        # Run initial K-means
        means, labels = self.run_kmeans(self.X, self.init_K)

        # Run X-means algorithm
        stop = False
        curr_K = self.init_K
        while not stop:
            stop = True
            final_means = []
            for k in range(curr_K):
                # Find the data that corresponds to the k-th cluster
                D = self.get_clustered_data(self.X, labels, k)
                if len(D) == 0 or D.shape[0] == 1:
                    continue

                # Whiten and find whitened mean
                stdD = np.std(D, axis=0)
                #D = vq.whiten(D)
                D /= float(stdD)  # Same as line above
                mean = D.mean(axis=0)

                # Cluster this subspace by half (K=2)
                half_means, half_labels = self.run_kmeans(D, K=2)

                # Compute BICs
                bic1 = self.compute_bic(D, [mean], K=1,
                                        labels=np.zeros(D.shape[0]),
                                        R=D.shape[0])
                bic2 = self.compute_bic(D, half_means, K=2,
                                        labels=half_labels, R=D.shape[0])

                # Split or not
                max_bic = np.max([np.abs(bic1), np.abs(bic2)])
                norm_bic1 = bic1 / float(max_bic)
                norm_bic2 = bic2 / float(max_bic)
                diff_bic = np.abs(norm_bic1 - norm_bic2)

                # Split!
                #print "diff_bic", diff_bic
                if diff_bic > th:
                    final_means.append(half_means[0] * stdD)
                    final_means.append(half_means[1] * stdD)
                    curr_K += 1
                    stop = False
                # Don't split
                else:
                    final_means.append(mean * stdD)

            final_means = np.asarray(final_means)

            #print "Estimated K: ", curr_K
            if self.plot:
                plt.scatter(self.X[:, 0], self.X[:, 1])
                plt.scatter(final_means[:, 0], final_means[:, 1], color="y")
                plt.show()

            if curr_K >= maxK or self.X.shape[-1] != final_means.shape[-1]:
                stop = True
            else:
                labels, dist = vq.vq(self.X, final_means)

        return curr_K

    def estimate_K_knee(self, th=.015, maxK=12):
        """Estimates the K using K-means and BIC, by sweeping various K and
            choosing the optimal BIC."""
        # Sweep K-means
        if self.X.shape[0] < maxK:
            maxK = self.X.shape[0]
        if maxK < 2:
            maxK = 2
        K = np.arange(1, maxK)
        bics = []
        for k in K:
            means, labels = self.run_kmeans(self.X, k)
            bic = self.compute_bic(self.X, means, labels, K=k,
                                   R=self.X.shape[0])
            bics.append(bic)
        diff_bics = np.diff(bics)
        finalK = K[-1]

        if len(bics) == 1:
            finalK = 2
        else:
            # Normalize
            bics = np.asarray(bics)
            bics -= bics.min()
            #bics /= bics.max()
            diff_bics -= diff_bics.min()
            #diff_bics /= diff_bics.max()

            #print bics, diff_bics

            # Find optimum K
            for i in range(len(K[:-1])):
                #if bics[i] > diff_bics[i]:
                if diff_bics[i] < th and K[i] != 1:
                    finalK = K[i]
                    break

        #print "Estimated K: ", finalK
        if self.plot:
            plt.subplot(2, 1, 1)
            plt.plot(K, bics, label="BIC")
            plt.plot(K[:-1], diff_bics, label="BIC diff")
            plt.legend(loc=2)
            plt.subplot(2, 1, 2)
            plt.scatter(self.X[:, 0], self.X[:, 1])
            plt.show()

        return finalK

    def get_clustered_data(self, X, labels, label_index):
        """Returns the data with a specific label_index, using the previously
         learned labels."""
        D = X[np.argwhere(labels == label_index)]
        return D.reshape((D.shape[0], D.shape[-1]))

    def run_kmeans(self, X, K):
        """Runs k-means and returns the labels assigned to the data."""
        wX = vq.whiten(X)
        means, dist = vq.kmeans(wX, K, iter=100)
        labels, dist = vq.vq(wX, means)
        return means, labels

    def compute_bic(self, D, means, labels, K, R):
        """Computes the Bayesian Information Criterion."""
        D = vq.whiten(D)
        Rn = D.shape[0]
        M = D.shape[1]

        if R == K:
            return 1

        # Maximum likelihood estimate (MLE)
        mle_var = 0
        for k in range(len(means)):
            X = D[np.argwhere(labels == k)]
            X = X.reshape((X.shape[0], X.shape[-1]))
            for x in X:
                mle_var += distance.euclidean(x, means[k])
                #print x, means[k], mle_var
        mle_var /= float(R - K)

        # Log-likelihood of the data
        l_D = - Rn/2. * np.log(2*np.pi) - (Rn * M)/2. * np.log(mle_var) - \
            (Rn - K) / 2. + Rn * np.log(Rn) - Rn * np.log(R)

        # Params of BIC
        p = (K-1) + M * K + mle_var

        #print "BIC:", l_D, p, R, K

        # Return the bic
        return l_D - p / 2. * np.log(R)

    @classmethod
    def generate_2d_data(self, N=100, K=5):
        """Generates N*K 2D data points with K means and N data points
            for each mean."""
        # Seed the random
        np.random.seed(seed=int(time.time()))

        # Amount of spread of the centroids
        spread = 30

        # Generate random data
        X = np.empty((0, 2))
        for i in range(K):
            mean = np.array([np.random.random()*spread,
                             np.random.random()*spread])
            x = np.random.normal(0.0, scale=1.0, size=(N, 2)) + mean
            X = np.append(X, x, axis=0)

        return X

def test_kmeans(K=5):
    """Test k-means with the synthetic data."""
    X = XMeans.generate_2d_data(K=4)
    wX = vq.whiten(X)
    dic, dist = vq.kmeans(wX, K, iter=100)

    plt.scatter(wX[:, 0], wX[:, 1])
    plt.scatter(dic[:, 0], dic[:, 1], color="m")
    plt.show()

def main(args):
    #test_kmeans(6)
    X = XMeans.generate_2d_data(K=args.k)
    xmeans = XMeans(X, init_K=2, plot=args.plot)
    est_K = xmeans.estimate_K_xmeans()
    est_K_knee = xmeans.estimate_K_knee()
    #print "Estimated x-means K:", est_K
    #print "Estimated Knee Point Detection K:", est_K_knee

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Runs x-means")
    parser.add_argument("k",
                        metavar="k", type=int,
                        help="Number of clusters to estimate.")
    parser.add_argument("-p", action="store_true", default=False,
                        dest="plot", help="Plot the results")
    main(parser.parse_args())


#!/usr/bin/python
#
# Copyright (C) Christian Thurau, 2010.
# Licensed under the GNU General Public License (GPL).
# http://www.gnu.org/licenses/gpl.txt
"""
PyMF Matrix sampling methods

    SUB: apply one of the matrix factorization methods of PyMF
         on sampled data for computing W, then compute H.

Copyright (C) Christian Thurau, 2010. GNU General Public License (GPL).
"""

import numpy as np
import random
#from itertools import combinations
from .chnmf import combinations

from . import dist
from .chnmf import quickhull
from .nmf import NMF
from .pca import PCA
from .kmeans import Kmeans
from .laesa import LAESA
from .sivm import SIVM

__all__ = ["SUB"]

class SUB(NMF):
    """
    SUB(data, mfmethod, sstrategy='rand', nsub=20, show_progress=True, mapW=False,
    base_sel=2,    num_bases=3 , niterH=1, niter=100, compute_h=True, compute_w=True, )

    Evaluate a matrix factorization method "mfmethod" for a certain sampling
    strategy "sstrategy". This is particular useful for very large datasets.

    Parameters
    ----------
    todo ...

    Attributes
    ----------
    todo ....
    """

    def __init__(self, data, mfmethod, nsub=20, show_progress=True, mapW=False, base_sel=2,
                num_bases=3 , niterH=1, compute_h=True, compute_w=True, sstrategy='rand'):
        NMF.__init__(self, data, num_bases=num_bases, compute_h=compute_h, show_progress=show_progress, compute_w=compute_w)

        self._niterH = niterH
        self._nsub = nsub
        self.data = data
        self._mfmethod = mfmethod
        self._mapW = mapW
        self._sstrategy = sstrategy
        self._base_sel = base_sel

        # assign the correct distance function
        if self._sstrategy == 'cur':
            self._subfunc = self.curselect

        elif self._sstrategy == 'kmeans':
            self._subfunc = self.kmeansselect

        elif self._sstrategy == 'hull':
            self._subfunc = self.hullselect

        elif self._sstrategy == 'laesa':
            self._subfunc = self.laesaselect

        elif self._sstrategy == 'sivm':
            self._subfunc = self.sivmselect

        else:
            self._subfunc = self.randselect

    def hullselect(self):

        def selectHullPoints(data, n=20):
            """ select data points for pairwise projections of the first n
            dimensions """

            # iterate over all projections and select data points
            idx = np.array([])

            # iterate over some pairwise combinations of dimensions
            for i in combinations(range(n), 2):

                # sample convex hull points in 2D projection
                convex_hull_d = quickhull(data[i, :].T)

                # get indices for convex hull data points
                idx = np.append(idx, dist.vq(data[i, :], convex_hull_d.T))
                idx = np.unique(idx)

            return np.int32(idx)

        # determine convex hull data points only if the total
        # amount of available data is >50
        #if self.data.shape[1] > 50:
        pcamodel = PCA(self.data, show_progress=self._show_progress)
        pcamodel.factorize()

        idx = selectHullPoints(pcamodel.H, n=self._base_sel)

        # set the number of subsampled data
        self.nsub = len(idx)

        return idx

    def kmeansselect(self):
            kmeans_mdl = Kmeans(self.data, num_bases=self._nsub)
            kmeans_mdl.initialization()
            kmeans_mdl.factorize()

            # pick data samples closest to the centres
            idx = dist.vq(kmeans_mdl.data, kmeans_mdl.W)
            return idx

    def curselect(self):
        def sample_probability():
            dsquare = self.data[:,:]**2

            pcol = np.array(dsquare.sum(axis=0))
            pcol /= pcol.sum()

            return (pcol.reshape(-1,1))

        probs = sample_probability()
        prob_cols = np.cumsum(probs.flatten()) #.flatten()
        temp_ind = np.zeros(self._nsub, np.int32)

        for i in range(self._nsub):
            tempI = np.where(prob_cols >= np.random.rand())[0]
            temp_ind[i] = tempI[0]

        return np.sort(temp_ind)

    def sivmselect(self):
        sivmmdl = SIVM(self.data, num_bases=self._nsub, compute_w=True, compute_h=False, dist_measure='cosine')

        sivmmdl.initialization()
        sivmmdl.factorize()
        idx = sivmmdl.select
        return idx

    def laesaselect(self):
        laesamdl = LAESA(self.data, num_bases=self._nsub, compute_w=True, compute_h=False, dist_measure='cosine')
        laesamdl.initialization()
        laesamdl.factorize()
        idx = laesamdl.select
        return idx

    def randselect(self):
        idx = random.sample(range(self._num_samples), self._nsub)
        return np.sort(np.int32(idx))

    def update_w(self):

        idx = self._subfunc()
        idx = np.sort(np.int32(idx))

        mdl_small = self._mfmethod(self.data[:, idx],
                                num_bases=self._num_bases,
                                show_progress=self._show_progress,
                                compute_w=True)

        # initialize W, H, and beta
        mdl_small.initialization()

        # determine W
        mdl_small.factorize()

        self.mdl = self._mfmethod(self.data[:, :],
                                    num_bases=self._num_bases ,
                                    show_progress=self._show_progress,
                                    compute_w=False)

        self.mdl.initialization()

        if self._mapW:
            # compute pairwise distances
            #distance = vq(self.data, self.W)
            _Wmapped_index = dist.vq(self.mdl.data, mdl_small.W)

            # do not directly assign, i.e. Wdist = self.data[:,sel]
            # as self might be unsorted (in non ascending order)
            # -> sorting sel would screw the matching to W if
            # self.data is stored as a hdf5 table (see h5py)
            for i,s in enumerate(_Wmapped_index):
                self.mdl.W[:,i] = self.mdl.data[:,s]
        else:
            self.mdl.W = np.copy(mdl_small.W)

    def update_h(self):
        self.mdl.factorize()

    def factorize(self):
        """Do factorization s.t. data = dot(dot(data,beta),H), under the convexity constraint
            beta >=0, sum(beta)=1, H >=0, sum(H)=1
        """
        # compute new coefficients for reconstructing data points
        self.update_w()

        # for CHNMF it is sometimes useful to only compute
        # the basis vectors
        if self._compute_h:
            self.update_h()

        self.W = self.mdl.W
        self.H = self.mdl.H

        self.ferr = np.zeros(1)
        self.ferr[0] = self.mdl.frobenius_norm()
        self._print_cur_status(' Fro:' + str(self.ferr[0]))

if __name__ == "__main__":
    import doctest
    doctest.testmod()


"""
Set of util functions for the section similarity project.
"""

import copy
import numpy as np
import json
import scipy.fftpack
import pylab as plt

def resample_mx(X, incolpos, outcolpos):
    """
    Y = resample_mx(X, incolpos, outcolpos)
    X is taken as a set of columns, each starting at 'time'
    colpos, and continuing until the start of the next column.
    Y is a similar matrix, with time boundaries defined by
    outcolpos.  Each column of Y is a duration-weighted average of
    the overlapping columns of X.
    2010-04-14 Dan Ellis dpwe@ee.columbia.edu  based on samplemx/beatavg
    -> python: TBM, 2011-11-05, TESTED
    """
    noutcols = len(outcolpos)
    Y = np.zeros((X.shape[0], noutcols))
    # assign 'end times' to final columns
    if outcolpos.max() > incolpos.max():
        incolpos = np.concatenate([incolpos,[outcolpos.max()]])
        X = np.concatenate([X, X[:,-1].reshape(X.shape[0],1)], axis=1)
    outcolpos = np.concatenate([outcolpos, [outcolpos[-1]]])
    # durations (default weights) of input columns)
    incoldurs = np.concatenate([np.diff(incolpos), [1]])

    for c in range(noutcols):
        firstincol = np.where(incolpos <= outcolpos[c])[0][-1]
        firstincolnext = np.where(incolpos < outcolpos[c+1])[0][-1]
        lastincol = max(firstincol,firstincolnext)
        # default weights
        wts = copy.deepcopy(incoldurs[firstincol:lastincol+1])
        # now fix up by partial overlap at ends
        if len(wts) > 1:
            wts[0] = wts[0] - (outcolpos[c] - incolpos[firstincol])
            wts[-1] = wts[-1] - (incolpos[lastincol+1] - outcolpos[c+1])
        wts = wts * 1. / float(sum(wts))
        Y[:,c] = np.dot(X[:,firstincol:lastincol+1], wts)
    # done
    return Y

def magnitude(X):
    """Magnitude of a complex matrix."""
    r = np.real(X)
    i = np.imag(X)
    return np.sqrt(r * r + i * i);

def json_to_bounds(segments_json):
    """Extracts the boundaries from a json file and puts them into
        an np array."""
    f = open(segments_json)
    segments = json.load(f)["segments"]
    bounds = []
    for segment in segments:
        bounds.append(segment["start"])
    bounds.append(bounds[-1] + segments[-1]["duration"]) # Add last boundary
    f.close()
    return np.asarray(bounds)

def json_bounds_to_bounds(bounds_json):
    """Extracts the boundaries from a bounds json file and puts them into
        an np array."""
    f = open(bounds_json)
    segments = json.load(f)["bounds"]
    bounds = []
    for segment in segments:
        bounds.append(segment["start"])
    f.close()
    return np.asarray(bounds)

def json_to_labels(segments_json):
    """Extracts the labels from a json file and puts them into
        an np array."""
    f = open(segments_json)
    segments = json.load(f)["segments"]
    labels = []
    str_labels = []
    for segment in segments:
        if not segment["label"] in str_labels:
            str_labels.append(segment["label"])
            labels.append(len(str_labels)-1)
        else:
            label_idx = np.where(np.asarray(str_labels) == segment["label"])[0][0]
            labels.append(label_idx)
    f.close()
    return np.asarray(labels)

def json_to_beats(beats_json_file):
    """Extracts the beats from the beats_json_file and puts them into
        an np array."""
    f = open(beats_json_file, "r")
    beats_json = json.load(f)
    beats = []
    for beat in beats_json["beats"]:
        beats.append(beat["start"])
    f.close()
    return np.asarray(beats)

def analyze_results(file):
    f = open(file, "r")
    lines = f.readlines()
    F = []
    for line in lines:
        F.append(float(line.split("\t")[0]))
    f.close()
    #print np.mean(F)

def compute_ffmc2d(X):
    """Computes the 2D-Fourier Magnitude Coefficients."""
    # 2d-fft
    fft2 = scipy.fftpack.fft2(X)

    # Magnitude
    fft2m = magnitude(fft2)

    # FFTshift and flatten
    fftshift = scipy.fftpack.fftshift(fft2m).flatten()

    #cmap = plt.cm.get_cmap('hot')
    #plt.imshow(np.log1p(scipy.fftpack.fftshift(fft2m)).T, interpolation="nearest",
    #    aspect="auto", cmap=cmap)
    #plt.show()

    # Take out redundant components
    return fftshift[:fftshift.shape[0] // 2 + 1]