<commit_before>//=============================================================================================================
/**
* @file     main.cpp
* @author   Lorenz Esch Lorenz Esch <Lorenz.Esch@tu-ilmenau.de>;
*           Lorenz Esch <lorenz.esch@tu-ilmenau.de>;
*           Matti Hamalainen <msh@nmr.mgh.harvard.edu>
* @version  1.0
* @date     July, 2016
*
* @section  LICENSE
*
* Copyright (C) 2016, Lorenz Esch and Matti Hamalainen. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided that
* the following conditions are met:
*     * Redistributions of source code must retain the above copyright notice, this list of conditions and the
*       following disclaimer.
*     * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
*       the following disclaimer in the documentation and/or other materials provided with the distribution.
*     * Neither the name of MNE-CPP authors nor the names of its contributors may be used
*       to endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*
* @brief    Example of using the MNE-CPP Disp3D library
*
*/

//*************************************************************************************************************
//=============================================================================================================
// INCLUDES
//=============================================================================================================

#include <disp3D/view3D.h>
#include <disp3D/control/control3dwidget.h>
#include <disp3D/3DObjects/brain/brainrtsourcelocdatatreeitem.h>

#include <fs/label.h>
#include <fs/surfaceset.h>
#include <fs/annotationset.h>

#include <fiff/fiff_evoked.h>
#include <fiff/fiff.h>
#include <fiff/fiff_dig_point_set.h>
#include <mne/mne.h>

#include <mne/mne.h>
#include <mne/mne_epoch_data_list.h>
#include <mne/mne_sourceestimate.h>

#include <inverse/minimumNorm/minimumnorm.h>

#include <utils/mnemath.h>

#include <iostream>

//*************************************************************************************************************
//=============================================================================================================
// QT INCLUDES
//=============================================================================================================

#include <QApplication>
#include <QMainWindow>
#include <QCommandLineParser>

//*************************************************************************************************************
//=============================================================================================================
// USED NAMESPACES
//=============================================================================================================

using namespace DISP3DLIB;
using namespace MNELIB;
using namespace FSLIB;
using namespace FIFFLIB;
using namespace INVERSELIB;
using namespace UTILSLIB;

//*************************************************************************************************************
//=============================================================================================================
// MAIN
//=============================================================================================================

//=============================================================================================================
/**
* The function main marks the entry point of the program.
* By default, main has the storage class extern.
*
* @param [in] argc (argument count) is an integer that indicates how many arguments were entered on the command line when the program was started.
* @param [in] argv (argument vector) is an array of pointers to arrays of character objects. The array objects are null-terminated strings, representing the arguments that were entered on the command line when the program was started.
* @return the value that was set to exit() (which is 0 if exit() is called via quit()).
*/
int main(int argc, char *argv[])
{
    QApplication a(argc, argv);

    // Command Line Parser
    QCommandLineParser parser;
    parser.setApplicationDescription("Disp3D Example");
    parser.addHelpOption();
    QCommandLineOption surfOption("surfType", "Surface type <type>.", "type", "orig");
    QCommandLineOption annotOption("annotType", "Annotation type <type>.", "type", "aparc.a2009s");
    QCommandLineOption hemiOption("hemi", "Selected hemisphere <hemi>.", "hemi", "2");
    QCommandLineOption subjectOption("subject", "Selected subject <subject>.", "subject", "sample");
    QCommandLineOption subjectPathOption("subjectPath", "Selected subject path <subjectPath>.", "subjectPath", "./MNE-sample-data/subjects");
    QCommandLineOption sourceLocOption("doSourceLoc", "Do real time source localization <doSourceLoc>.", "doSourceLoc", "true");
    QCommandLineOption fwdOption("fwd", "Path to forwad solution <file>.", "file", "./MNE-sample-data/MEG/sample/sample_audvis-meg-eeg-oct-6-fwd.fif");
    QCommandLineOption invOpOption("inv", "Path to inverse operator <file>.", "file", "");
    QCommandLineOption clustOption("doClust", "Path to clustered inverse operator <doClust>.", "doClust", "true");
    QCommandLineOption covFileOption("cov", "Path to the covariance <file>.", "file", "./MNE-sample-data/MEG/sample/sample_audvis-cov.fif");
    QCommandLineOption evokedFileOption("ave", "Path to the evoked/average <file>.", "file", "./MNE-sample-data/MEG/sample/sample_audvis-ave.fif");
    QCommandLineOption methodOption("method", "Inverse estimation <method>, i.e., 'MNE', 'dSPM' or 'sLORETA'.", "method", "dSPM");//"MNE" | "dSPM" | "sLORETA"
    QCommandLineOption snrOption("snr", "The SNR value used for computation <snr>.", "snr", "3.0");//3.0f;//0.1f;//3.0f;
    QCommandLineOption evokedIndexOption("aveIdx", "The average <index> to choose from the average file.", "index", "0");

    parser.addOption(surfOption);
    parser.addOption(annotOption);
    parser.addOption(hemiOption);
    parser.addOption(subjectOption);
    parser.addOption(subjectPathOption);
    parser.addOption(sourceLocOption);
    parser.addOption(fwdOption);
    parser.addOption(invOpOption);
    parser.addOption(clustOption);
    parser.addOption(covFileOption);
    parser.addOption(evokedFileOption);
    parser.addOption(methodOption);
    parser.addOption(snrOption);
    parser.addOption(evokedIndexOption);
    parser.process(a);

    bool bAddRtSourceLoc = parser.value(sourceLocOption) == "false" ? false : true;
    bool bDoClustering = parser.value(clustOption) == "false" ? false : true;

    //Inits
    SurfaceSet tSurfSet (parser.value(subjectOption), parser.value(hemiOption).toInt(), parser.value(surfOption), parser.value(subjectPathOption));
    AnnotationSet tAnnotSet (parser.value(subjectOption), parser.value(hemiOption).toInt(), parser.value(annotOption), parser.value(subjectPathOption));

    QFile t_fileFwd(parser.value(fwdOption));
    MNEForwardSolution t_Fwd(t_fileFwd);
    MNEForwardSolution t_clusteredFwd;

    QString t_sFileClusteredInverse(parser.value(invOpOption));

    QFile t_fileCov(parser.value(covFileOption));
    QFile t_fileEvoked(parser.value(evokedFileOption));

    //########################################################################################
    //
    // Source Estimate START
    //
    //########################################################################################

    // Load data
    QPair<QVariant, QVariant> baseline(QVariant(), 0);
    MNESourceEstimate sourceEstimate;
    FiffEvoked evoked(t_fileEvoked, parser.value(evokedIndexOption).toInt(), baseline);

    if(bAddRtSourceLoc) {
        double snr = parser.value(snrOption).toDouble();
        double lambda2 = 1.0 / pow(snr, 2);
        QString method(parser.value(methodOption));

        // Load data
        QPair<QVariant, QVariant> baseline(QVariant(), 0);
        FiffEvoked evoked(t_fileEvoked, 0, baseline);
        t_fileEvoked.close();
        if(evoked.isEmpty())
            return 1;

        std::cout << std::endl;
        std::cout << "Evoked description: " << evoked.comment.toLatin1().constData() << std::endl;

        if(t_Fwd.isEmpty())
            return 1;

        FiffCov noise_cov(t_fileCov);

        // regularize noise covariance
        noise_cov = noise_cov.regularize(evoked.info, 0.05, 0.05, 0.1, true);

        //
        // Cluster forward solution;
        //
        if(bDoClustering) {
            t_clusteredFwd = t_Fwd.cluster_forward_solution(tAnnotSet, 40);
        } else {
            t_clusteredFwd = t_Fwd;
        }

        //
        // make an inverse operators
        //
        FiffInfo info = evoked.info;

        MNEInverseOperator inverse_operator(info, t_clusteredFwd, noise_cov, 0.2f, 0.8f);

        if(!t_sFileClusteredInverse.isEmpty())
        {
            QFile t_fileClusteredInverse(t_sFileClusteredInverse);
            inverse_operator.write(t_fileClusteredInverse);
        }

        //
        // Compute inverse solution
        //
        MinimumNorm minimumNorm(inverse_operator, lambda2, method);
        sourceEstimate = minimumNorm.calculateInverse(evoked);

        if(sourceEstimate.isEmpty())
            return 1;

        // View activation time-series
        std::cout << "\nsourceEstimate:\n" << sourceEstimate.data.block(0,0,10,10) << std::endl;
        std::cout << "time\n" << sourceEstimate.times.block(0,0,1,10) << std::endl;
        std::cout << "timeMin\n" << sourceEstimate.times[0] << std::endl;
        std::cout << "timeMax\n" << sourceEstimate.times[sourceEstimate.times.size()-1] << std::endl;
        std::cout << "time step\n" << sourceEstimate.tstep << std::endl;
    }

    //########################################################################################
    //
    //Source Estimate END
    //
    //########################################################################################

    //########################################################################################
    //
    // Create the test view START
    //
    //########################################################################################

    std::cout<<"Creating BrainView"<<std::endl;

    //Create the 3D view
    View3D::SPtr testWindow = View3D::SPtr(new View3D());

    //Add fressurfer surface set including both hemispheres
    testWindow->addSurfaceSet(parser.value(subjectOption), evoked.comment, tSurfSet, tAnnotSet);

    //Read and show BEM
    QFile t_fileBem("./MNE-sample-data/subjects/sample/bem/sample-5120-5120-5120-bem.fif");
    MNEBem t_Bem(t_fileBem);
    testWindow->addBemData(parser.value(subjectOption), "BEM", t_Bem);

    //Read and show sensor helmets
    QFile t_filesensorSurfaceVV("./resources/sensorSurfaces/306m_rt.fif");
    MNEBem t_sensorSurfaceVV(t_filesensorSurfaceVV);
    testWindow->addBemData("Sensors", "VectorView", t_sensorSurfaceVV);

    // Read & show digitizer points
    QFile t_fileDig("./MNE-sample-data/MEG/sample/sample_audvis-ave.fif");
    FiffDigPointSet t_Dig(t_fileDig);
    testWindow->addDigitizerData(parser.value(subjectOption), evoked.comment, t_Dig);

    if(bAddRtSourceLoc) {
        //Add rt source loc data
        QList<BrainRTSourceLocDataTreeItem*> rtItemList_RV = testWindow->addSourceData(parser.value(subjectOption), evoked.comment, sourceEstimate, t_clusteredFwd);

        //Init some rt related values for right visual data
        for(int i = 0; i < rtItemList_RV.size(); ++i) {
            rtItemList_RV.at(i)->setLoopState(true);
            rtItemList_RV.at(i)->setTimeInterval(17);
            rtItemList_RV.at(i)->setNumberAverages(1);
            rtItemList_RV.at(i)->setStreamingActive(true);
            rtItemList_RV.at(i)->setNormalization(QVector3D(0.0,5.5,10));
            rtItemList_RV.at(i)->setVisualizationType("Annotation based");
            rtItemList_RV.at(i)->setColortable("Hot");
        }
    }

    testWindow->show();

    Control3DWidget::SPtr control3DWidget = Control3DWidget::SPtr(new Control3DWidget());
    control3DWidget->setView3D(testWindow);
    control3DWidget->show();

    //########################################################################################
    //
    // Create the test view END
    //
    //########################################################################################

    return a.exec();
}
<commit_msg>[SC-128] Fix disp3DTutorial<commit_after>//=============================================================================================================
/**
* @file     main.cpp
* @author   Lorenz Esch Lorenz Esch <Lorenz.Esch@tu-ilmenau.de>;
*           Lorenz Esch <lorenz.esch@tu-ilmenau.de>;
*           Matti Hamalainen <msh@nmr.mgh.harvard.edu>
* @version  1.0
* @date     July, 2016
*
* @section  LICENSE
*
* Copyright (C) 2016, Lorenz Esch and Matti Hamalainen. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided that
* the following conditions are met:
*     * Redistributions of source code must retain the above copyright notice, this list of conditions and the
*       following disclaimer.
*     * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
*       the following disclaimer in the documentation and/or other materials provided with the distribution.
*     * Neither the name of MNE-CPP authors nor the names of its contributors may be used
*       to endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*
* @brief    Example of using the MNE-CPP Disp3D library
*
*/

//*************************************************************************************************************
//=============================================================================================================
// INCLUDES
//=============================================================================================================

#include <disp3D/view3D.h>
#include <disp3D/control/control3dwidget.h>
#include <disp3D/3DObjects/brain/brainrtsourcelocdatatreeitem.h>

#include <fs/label.h>
#include <fs/surfaceset.h>
#include <fs/annotationset.h>

#include <fiff/fiff_evoked.h>
#include <fiff/fiff.h>
#include <fiff/fiff_dig_point_set.h>
#include <mne/mne.h>

#include <mne/mne.h>
#include <mne/mne_epoch_data_list.h>
#include <mne/mne_sourceestimate.h>

#include <inverse/minimumNorm/minimumnorm.h>

#include <utils/mnemath.h>

#include <iostream>

//*************************************************************************************************************
//=============================================================================================================
// QT INCLUDES
//=============================================================================================================

#include <QApplication>
#include <QMainWindow>
#include <QCommandLineParser>

//*************************************************************************************************************
//=============================================================================================================
// USED NAMESPACES
//=============================================================================================================

using namespace DISP3DLIB;
using namespace MNELIB;
using namespace FSLIB;
using namespace FIFFLIB;
using namespace INVERSELIB;
using namespace UTILSLIB;

//*************************************************************************************************************
//=============================================================================================================
// MAIN
//=============================================================================================================

//=============================================================================================================
/**
* The function main marks the entry point of the program.
* By default, main has the storage class extern.
*
* @param [in] argc (argument count) is an integer that indicates how many arguments were entered on the command line when the program was started.
* @param [in] argv (argument vector) is an array of pointers to arrays of character objects. The array objects are null-terminated strings, representing the arguments that were entered on the command line when the program was started.
* @return the value that was set to exit() (which is 0 if exit() is called via quit()).
*/
int main(int argc, char *argv[])
{
    QApplication a(argc, argv);

    // Command Line Parser
    QCommandLineParser parser;
    parser.setApplicationDescription("Disp3D Example");
    parser.addHelpOption();
    QCommandLineOption surfOption("surfType", "Surface type <type>.", "type", "orig");
    QCommandLineOption annotOption("annotType", "Annotation type <type>.", "type", "aparc.a2009s");
    QCommandLineOption hemiOption("hemi", "Selected hemisphere <hemi>.", "hemi", "2");
    QCommandLineOption subjectOption("subject", "Selected subject <subject>.", "subject", "sample");
    QCommandLineOption subjectPathOption("subjectPath", "Selected subject path <subjectPath>.", "subjectPath", "./MNE-sample-data/subjects");
    QCommandLineOption sourceLocOption("doSourceLoc", "Do real time source localization <doSourceLoc>.", "doSourceLoc", "true");
    QCommandLineOption fwdOption("fwd", "Path to forwad solution <file>.", "file", "./MNE-sample-data/MEG/sample/sample_audvis-meg-eeg-oct-6-fwd.fif");
    QCommandLineOption invOpOption("inv", "Path to inverse operator <file>.", "file", "");
    QCommandLineOption clustOption("doClust", "Path to clustered inverse operator <doClust>.", "doClust", "true");
    QCommandLineOption covFileOption("cov", "Path to the covariance <file>.", "file", "./MNE-sample-data/MEG/sample/sample_audvis-cov.fif");
    QCommandLineOption evokedFileOption("ave", "Path to the evoked/average <file>.", "file", "./MNE-sample-data/MEG/sample/sample_audvis-ave.fif");
    QCommandLineOption methodOption("method", "Inverse estimation <method>, i.e., 'MNE', 'dSPM' or 'sLORETA'.", "method", "dSPM");//"MNE" | "dSPM" | "sLORETA"
    QCommandLineOption snrOption("snr", "The SNR value used for computation <snr>.", "snr", "3.0");//3.0f;//0.1f;//3.0f;
    QCommandLineOption evokedIndexOption("aveIdx", "The average <index> to choose from the average file.", "index", "0");

    parser.addOption(surfOption);
    parser.addOption(annotOption);
    parser.addOption(hemiOption);
    parser.addOption(subjectOption);
    parser.addOption(subjectPathOption);
    parser.addOption(sourceLocOption);
    parser.addOption(fwdOption);
    parser.addOption(invOpOption);
    parser.addOption(clustOption);
    parser.addOption(covFileOption);
    parser.addOption(evokedFileOption);
    parser.addOption(methodOption);
    parser.addOption(snrOption);
    parser.addOption(evokedIndexOption);
    parser.process(a);

    bool bAddRtSourceLoc = parser.value(sourceLocOption) == "false" ? false : true;
    bool bDoClustering = parser.value(clustOption) == "false" ? false : true;

    //Inits
    SurfaceSet tSurfSet (parser.value(subjectOption), parser.value(hemiOption).toInt(), parser.value(surfOption), parser.value(subjectPathOption));
    AnnotationSet tAnnotSet (parser.value(subjectOption), parser.value(hemiOption).toInt(), parser.value(annotOption), parser.value(subjectPathOption));

    QFile t_fileFwd(parser.value(fwdOption));
    MNEForwardSolution t_Fwd(t_fileFwd);
    MNEForwardSolution t_clusteredFwd;

    QString t_sFileClusteredInverse(parser.value(invOpOption));

    QFile t_fileCov(parser.value(covFileOption));
    QFile t_fileEvoked(parser.value(evokedFileOption));

    //########################################################################################
    //
    // Source Estimate START
    //
    //########################################################################################

    // Load data
    QPair<QVariant, QVariant> baseline(QVariant(), 0);
    MNESourceEstimate sourceEstimate;
    FiffEvoked evoked(t_fileEvoked, parser.value(evokedIndexOption).toInt(), baseline);

    if(bAddRtSourceLoc) {
        double snr = parser.value(snrOption).toDouble();
        double lambda2 = 1.0 / pow(snr, 2);
        QString method(parser.value(methodOption));

        // Load data
        t_fileEvoked.close();
        if(evoked.isEmpty())
            return 1;

        std::cout << std::endl;
        std::cout << "Evoked description: " << evoked.comment.toLatin1().constData() << std::endl;

        if(t_Fwd.isEmpty())
            return 1;

        FiffCov noise_cov(t_fileCov);

        // regularize noise covariance
        noise_cov = noise_cov.regularize(evoked.info, 0.05, 0.05, 0.1, true);

        //
        // Cluster forward solution;
        //
        if(bDoClustering) {
            t_clusteredFwd = t_Fwd.cluster_forward_solution(tAnnotSet, 40);
        } else {
            t_clusteredFwd = t_Fwd;
        }

        //
        // make an inverse operators
        //
        FiffInfo info = evoked.info;

        MNEInverseOperator inverse_operator(info, t_clusteredFwd, noise_cov, 0.2f, 0.8f);

        if(!t_sFileClusteredInverse.isEmpty())
        {
            QFile t_fileClusteredInverse(t_sFileClusteredInverse);
            inverse_operator.write(t_fileClusteredInverse);
        }

        //
        // Compute inverse solution
        //
        MinimumNorm minimumNorm(inverse_operator, lambda2, method);
        sourceEstimate = minimumNorm.calculateInverse(evoked);

        if(sourceEstimate.isEmpty())
            return 1;

        // View activation time-series
        std::cout << "\nsourceEstimate:\n" << sourceEstimate.data.block(0,0,10,10) << std::endl;
        std::cout << "time\n" << sourceEstimate.times.block(0,0,1,10) << std::endl;
        std::cout << "timeMin\n" << sourceEstimate.times[0] << std::endl;
        std::cout << "timeMax\n" << sourceEstimate.times[sourceEstimate.times.size()-1] << std::endl;
        std::cout << "time step\n" << sourceEstimate.tstep << std::endl;
    }

    //########################################################################################
    //
    //Source Estimate END
    //
    //########################################################################################

    //########################################################################################
    //
    // Create the test view START
    //
    //########################################################################################

    std::cout<<"Creating BrainView"<<std::endl;

    //Create the 3D view
    View3D::SPtr testWindow = View3D::SPtr(new View3D());

    //Add fressurfer surface set including both hemispheres
    testWindow->addSurfaceSet(parser.value(subjectOption), evoked.comment, tSurfSet, tAnnotSet);

    //Read and show BEM
    QFile t_fileBem("./MNE-sample-data/subjects/sample/bem/sample-5120-5120-5120-bem.fif");
    MNEBem t_Bem(t_fileBem);
    testWindow->addBemData(parser.value(subjectOption), "BEM", t_Bem);

    //Read and show sensor helmets
    QFile t_filesensorSurfaceVV("./resources/sensorSurfaces/306m_rt.fif");
    MNEBem t_sensorSurfaceVV(t_filesensorSurfaceVV);
    testWindow->addBemData("Sensors", "VectorView", t_sensorSurfaceVV);

    // Read & show digitizer points
    QFile t_fileDig("./MNE-sample-data/MEG/sample/sample_audvis-ave.fif");
    FiffDigPointSet t_Dig(t_fileDig);
    testWindow->addDigitizerData(parser.value(subjectOption), evoked.comment, t_Dig);

    if(bAddRtSourceLoc) {
        //Add rt source loc data
        QList<BrainRTSourceLocDataTreeItem*> rtItemList_RV = testWindow->addSourceData(parser.value(subjectOption), evoked.comment, sourceEstimate, t_clusteredFwd);

        //Init some rt related values for right visual data
        for(int i = 0; i < rtItemList_RV.size(); ++i) {
            rtItemList_RV.at(i)->setLoopState(true);
            rtItemList_RV.at(i)->setTimeInterval(17);
            rtItemList_RV.at(i)->setNumberAverages(1);
            rtItemList_RV.at(i)->setStreamingActive(true);
            rtItemList_RV.at(i)->setNormalization(QVector3D(0.0,5.5,10));
            rtItemList_RV.at(i)->setVisualizationType("Annotation based");
            rtItemList_RV.at(i)->setColortable("Hot");
        }
    }

    testWindow->show();

    Control3DWidget::SPtr control3DWidget = Control3DWidget::SPtr(new Control3DWidget());
    control3DWidget->setView3D(testWindow);
    control3DWidget->show();

    //########################################################################################
    //
    // Create the test view END
    //
    //########################################################################################

    return a.exec();
}
<|endoftext|>


<commit_before>#ifndef TIMING_UTIL_HPP
#define TIMING_UTIL_HPP

#include <atomic>
#include <chrono>
#include <cstdint>
#include <mutex>
#include <unordered_map>

namespace osrm
{
namespace util
{

struct GlobalTimer
{
    GlobalTimer() : time(0) {}
    std::atomic<uint64_t> time;
};

class GlobalTimerFactory
{
  public:
    static GlobalTimerFactory &get()
    {
        static GlobalTimerFactory instance;
        return instance;
    }

    GlobalTimer &getGlobalTimer(const std::string &name)
    {
        std::lock_guard<std::mutex> lock(map_mutex);
        return timer_map[name];
    }

  private:
    std::mutex map_mutex;
    std::unordered_map<std::string, GlobalTimer> timer_map;
};

#define GLOBAL_TIMER_AQUIRE(_X)                                                                    \
    auto &_X##_global_timer = GlobalTimerFactory::get().getGlobalTimer(#_X)
#define GLOBAL_TIMER_RESET(_X) _X##_global_timer.time = 0
#define GLOBAL_TIMER_START(_X) TIMER_START(_X)
#define GLOBAL_TIMER_STOP(_X)                                                                      \
    TIMER_STOP(_X);                                                                                \
    _X##_global_timer.time += TIMER_NSEC(_X)
#define GLOBAL_TIMER_NSEC(_X) static_cast<double>(_X##_global_timer.time)
#define GLOBAL_TIMER_USEC(_X) (_X##_global_timer.time / 1000.0)
#define GLOBAL_TIMER_MSEC(_X) (_X##_global_timer.time / 1000.0 / 1000.0)
#define GLOBAL_TIMER_SEC(_X) (_X##_global_timer.time / 1000.0 / 1000.0 / 1000.0)

#define TIMER_START(_X) auto _X##_start = std::chrono::steady_clock::now(), _X##_stop = _X##_start
#define TIMER_STOP(_X) _X##_stop = std::chrono::steady_clock::now()
#define TIMER_NSEC(_X)                                                                             \
    std::chrono::duration_cast<std::chrono::nanoseconds>(_X##_stop - _X##_start).count()
#define TIMER_USEC(_X)                                                                             \
    std::chrono::duration_cast<std::chrono::microseconds>(_X##_stop - _X##_start).count()
#define TIMER_MSEC(_X)                                                                             \
    (0.000001 *                                                                                \
     std::chrono::duration_cast<std::chrono::nanoseconds>(_X##_stop - _X##_start).count())
#define TIMER_SEC(_X)                                                                              \
    (0.000001 *                                                                                \
     std::chrono::duration_cast<std::chrono::microseconds>(_X##_stop - _X##_start).count())
#define TIMER_MIN(_X)                                                                              \
    std::chrono::duration_cast<std::chrono::minutes>(_X##_stop - _X##_start).count()
}
}

#endif // TIMING_UTIL_HPP
<commit_msg>Remove obsolete timer code<commit_after>#ifndef TIMING_UTIL_HPP
#define TIMING_UTIL_HPP

#include <chrono>
#include <cstdint>

namespace osrm
{
namespace util
{

#define TIMER_START(_X) auto _X##_start = std::chrono::steady_clock::now(), _X##_stop = _X##_start
#define TIMER_STOP(_X) _X##_stop = std::chrono::steady_clock::now()
#define TIMER_NSEC(_X)                                                                             \
    std::chrono::duration_cast<std::chrono::nanoseconds>(_X##_stop - _X##_start).count()
#define TIMER_USEC(_X)                                                                             \
    std::chrono::duration_cast<std::chrono::microseconds>(_X##_stop - _X##_start).count()
#define TIMER_MSEC(_X)                                                                             \
    (0.000001 *                                                                                \
     std::chrono::duration_cast<std::chrono::nanoseconds>(_X##_stop - _X##_start).count())
#define TIMER_SEC(_X)                                                                              \
    (0.000001 *                                                                                \
     std::chrono::duration_cast<std::chrono::microseconds>(_X##_stop - _X##_start).count())
#define TIMER_MIN(_X)                                                                              \
    std::chrono::duration_cast<std::chrono::minutes>(_X##_stop - _X##_start).count()
}
}

#endif // TIMING_UTIL_HPP
<|endoftext|>


<commit_before>////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Simon Grätzer
////////////////////////////////////////////////////////////////////////////////

#include "PregelFeature.h"
#include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/MutexLocker.h"
#include "Basics/ThreadPool.h"
#include "Cluster/ClusterFeature.h"
#include "Cluster/ClusterInfo.h"
#include "Pregel/Conductor.h"
#include "Pregel/Recovery.h"
#include "Pregel/Worker.h"

using namespace arangodb::pregel;

static PregelFeature* Instance;

uint64_t PregelFeature::createExecutionNumber() {
  return ClusterInfo::instance()->uniqid();
}

PregelFeature::PregelFeature(application_features::ApplicationServer* server)
    : ApplicationFeature(server, "Pregel") {
  setOptional(false);
  requiresElevatedPrivileges(false);
  startsAfter("Logger");
  startsAfter("Database");
  startsAfter("Endpoint");
  startsAfter("Cluster");
}

PregelFeature::~PregelFeature() {
  if (_recoveryManager) {
    _recoveryManager.reset();
  }
  cleanupAll();
}

PregelFeature* PregelFeature::instance() { return Instance; }

static size_t _approxThreadNumber(){
  const size_t procNum = TRI_numberProcessors();
  if (procNum <= 1) return 1;
  else if (procNum <= 16) return procNum / 2;
  else return procNum;// use full performance on cluster
}

void PregelFeature::start() {
  Instance = this;

  const size_t threadNum = _approxThreadNumber();
  _threadPool.reset(new basics::ThreadPool(threadNum, "Pregel"));

  ClusterFeature* cluster =
      application_features::ApplicationServer::getFeature<ClusterFeature>(
          "Cluster");
  if (cluster != nullptr) {
    AgencyCallbackRegistry* registry = cluster->agencyCallbackRegistry();
    if (registry != nullptr) {
      _recoveryManager.reset(new RecoveryManager(registry));
    }
  }
}

void PregelFeature::beginShutdown() { cleanupAll(); }

void PregelFeature::addExecution(Conductor* const exec,
                                 uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  //_executions.
  _conductors[executionNumber] = exec;
}

Conductor* PregelFeature::conductor(uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  auto it = _conductors.find(executionNumber);
  return it != _conductors.end() ? it->second : nullptr;
}

void PregelFeature::addWorker(IWorker* const worker, uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  _workers[executionNumber] = worker;
}

IWorker* PregelFeature::worker(uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  auto it = _workers.find(executionNumber);
  return it != _workers.end() ? it->second : nullptr;
}

void PregelFeature::cleanup(uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  auto cit = _conductors.find(executionNumber);
  if (cit != _conductors.end()) {
    delete (cit->second);
    _conductors.erase(executionNumber);
  }
  auto wit = _workers.find(executionNumber);
  if (wit != _workers.end()) {
    delete (wit->second);
    _workers.erase(executionNumber);
  }
}

void PregelFeature::cleanupAll() {
  MUTEX_LOCKER(guard, _mutex);
  for (auto it : _conductors) {
    delete (it.second);
  }
  _conductors.clear();
  for (auto it : _workers) {
    it.second->cancelGlobalStep(VPackSlice());
    delete (it.second);
  }
  _workers.clear();
}
<commit_msg>adjusted number of threads used per default<commit_after>////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Simon Grätzer
////////////////////////////////////////////////////////////////////////////////

#include "PregelFeature.h"
#include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/MutexLocker.h"
#include "Basics/ThreadPool.h"
#include "Cluster/ClusterFeature.h"
#include "Cluster/ClusterInfo.h"
#include "Pregel/Conductor.h"
#include "Pregel/Recovery.h"
#include "Pregel/Worker.h"

using namespace arangodb::pregel;

static PregelFeature* Instance;

uint64_t PregelFeature::createExecutionNumber() {
  return ClusterInfo::instance()->uniqid();
}

PregelFeature::PregelFeature(application_features::ApplicationServer* server)
    : ApplicationFeature(server, "Pregel") {
  setOptional(false);
  requiresElevatedPrivileges(false);
  startsAfter("Logger");
  startsAfter("Database");
  startsAfter("Endpoint");
  startsAfter("Cluster");
}

PregelFeature::~PregelFeature() {
  if (_recoveryManager) {
    _recoveryManager.reset();
  }
  cleanupAll();
}

PregelFeature* PregelFeature::instance() { return Instance; }

static size_t _approxThreadNumber(){
  const size_t procNum = TRI_numberProcessors();
  if (procNum <= 1) return 1;
  else return procNum / 2;
  //else return procNum;// use full performance on cluster
}

void PregelFeature::start() {
  Instance = this;

  const size_t threadNum = _approxThreadNumber();
  LOG(INFO) << "Pregel uses " << threadNum << " threads";
  _threadPool.reset(new basics::ThreadPool(threadNum, "Pregel"));

  ClusterFeature* cluster =
      application_features::ApplicationServer::getFeature<ClusterFeature>(
          "Cluster");
  if (cluster != nullptr) {
    AgencyCallbackRegistry* registry = cluster->agencyCallbackRegistry();
    if (registry != nullptr) {
      _recoveryManager.reset(new RecoveryManager(registry));
    }
  }
}

void PregelFeature::beginShutdown() { cleanupAll(); }

void PregelFeature::addExecution(Conductor* const exec,
                                 uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  //_executions.
  _conductors[executionNumber] = exec;
}

Conductor* PregelFeature::conductor(uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  auto it = _conductors.find(executionNumber);
  return it != _conductors.end() ? it->second : nullptr;
}

void PregelFeature::addWorker(IWorker* const worker, uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  _workers[executionNumber] = worker;
}

IWorker* PregelFeature::worker(uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  auto it = _workers.find(executionNumber);
  return it != _workers.end() ? it->second : nullptr;
}

void PregelFeature::cleanup(uint64_t executionNumber) {
  MUTEX_LOCKER(guard, _mutex);
  auto cit = _conductors.find(executionNumber);
  if (cit != _conductors.end()) {
    delete (cit->second);
    _conductors.erase(executionNumber);
  }
  auto wit = _workers.find(executionNumber);
  if (wit != _workers.end()) {
    delete (wit->second);
    _workers.erase(executionNumber);
  }
}

void PregelFeature::cleanupAll() {
  MUTEX_LOCKER(guard, _mutex);
  for (auto it : _conductors) {
    delete (it.second);
  }
  _conductors.clear();
  for (auto it : _workers) {
    it.second->cancelGlobalStep(VPackSlice());
    delete (it.second);
  }
  _workers.clear();
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2009 Yahoo! Inc.  All rights reserved.  The copyrights
embodied in the content of this file are licensed under the BSD
(revised) open source license
 */

#include <fstream>
#include <iostream>
using namespace std;

#include <unistd.h>
#include <stdlib.h>
#include <stdint.h>
#include <math.h>
#include "parse_regressor.h"
#include "loss_functions.h"
#include "global_data.h"
#include "io.h"

void initialize_regressor(regressor &r)
{
  size_t length = ((size_t)1) << global.num_bits;
  global.thread_mask = (global.stride * (length >> global.thread_bits)) - 1;
  size_t num_threads = global.num_threads();
  r.weight_vectors = (weight **)malloc(num_threads * sizeof(weight*));
  if (global.per_feature_regularizer_input != "")
    r.regularizers = (weight **)malloc(num_threads * sizeof(weight*));
  else
    r.regularizers = NULL;
  for (size_t i = 0; i < num_threads; i++)
    {
      r.weight_vectors[i] = (weight *)calloc(global.stride*length/num_threads, sizeof(weight));
      if (r.regularizers != NULL)
	r.regularizers[i] = (weight *)calloc(length/num_threads, sizeof(weight));
      if (r.weight_vectors[i] == NULL || (r.regularizers != NULL && r.regularizers[i] == NULL))
        {
          cerr << global.program_name << ": Failed to allocate weight array: try decreasing -b <bits>" << endl;
          exit (1);
        }
      if (global.initial_weight != 0.)
	for (size_t j = 0; j < global.stride*length/num_threads; j+=global.stride)
	  r.weight_vectors[i][j] = global.initial_weight;
      if (global.random_weights)
	for (size_t j = 0; j < length/num_threads; j++) {
          r.weight_vectors[i][j] = drand48() - 0.5;
        }
      if (global.lda)
	{
	  size_t stride = global.stride;

          for (size_t j = 0; j < stride*length/num_threads; j+=stride)
	    {
	      for (size_t k = 0; k < global.lda; k++) {
                r.weight_vectors[i][j+k] = -log(drand48()) + 1.0;
//                 r.weight_vectors[i][j+k] *= r.weight_vectors[i][j+k];
//                 r.weight_vectors[i][j+k] *= r.weight_vectors[i][j+k];
		r.weight_vectors[i][j+k] *= (float)global.lda_D / (float)global.lda
		  / global.length() * 200;
              }
	      r.weight_vectors[i][j+global.lda] = global.initial_t;
	    }
	}
      if(global.adaptive)
        for (size_t j = 1; j < global.stride*length/num_threads; j+=global.stride)
	  r.weight_vectors[i][j] = 1;
    }
}

void read_vector(const char* file, regressor& r, bool& initialized, bool reg_vector)
{
  ifstream source(file);
  if (!source.is_open())
    {
      cout << "can't open " << file << endl << " ... exiting." << endl;
      exit(1);
    }

  size_t v_length;
  source.read((char*)&v_length, sizeof(v_length));
  char t[v_length];
  source.read(t,v_length);
  if (strcmp(t,version.c_str()) != 0)
    {
      cout << "source has possibly incompatible version!" << endl;
      exit(1);
    }

  source.read((char*)&global.min_label, sizeof(global.min_label));
  source.read((char*)&global.max_label, sizeof(global.max_label));

  size_t local_num_bits;
  source.read((char *)&local_num_bits, sizeof(local_num_bits));
  if (!initialized){
    if (global.default_bits != true && global.num_bits != local_num_bits)
      {
	cout << "Wrong number of bits for source!" << endl;
	exit (1);
      }
    global.default_bits = false;
    global.num_bits = local_num_bits;
  }
  else
    if (local_num_bits != global.num_bits)
      {
	cout << "can't combine sources with different feature number!" << endl;
	exit (1);
      }

  size_t local_thread_bits;
  source.read((char*)&local_thread_bits, sizeof(local_thread_bits));
  if (!initialized){
    global.thread_bits = local_thread_bits;
    global.partition_bits = global.thread_bits;
  }
  else
    if (local_thread_bits != global.thread_bits)
      {
	cout << "can't combine sources trained with different numbers of threads!" << endl;
	exit (1);
      }

  int len;
  source.read((char *)&len, sizeof(len));

  vector<string> local_pairs;
  for (; len > 0; len--)
    {
      char pair[2];
      source.read(pair, sizeof(char)*2);
      string temp(pair, 2);
      local_pairs.push_back(temp);
    }
  if (!initialized)
    {
      global.pairs = local_pairs;
      initialize_regressor(r);
    }
  else
    if (local_pairs != global.pairs)
      {
	cout << "can't combine sources with different features!" << endl;
	for (size_t i = 0; i < local_pairs.size(); i++)
	  cout << local_pairs[i] << " " << local_pairs[i].size() << " ";
	cout << endl;
	for (size_t i = 0; i < global.pairs.size(); i++)
	  cout << global.pairs[i] << " " << global.pairs[i].size() << " ";
	cout << endl;
	exit (1);
      }
  size_t local_ngram;
  source.read((char*)&local_ngram, sizeof(local_ngram));
  size_t local_skips;
  source.read((char*)&local_skips, sizeof(local_skips));
  if (!initialized)
    {
      global.ngram = local_ngram;
      global.skips = local_skips;
      initialized = true;
    }
  else
    if (global.ngram != local_ngram || global.skips != local_skips)
      {
	cout << "can't combine sources with different ngram features!" << endl;
	exit(1);
      }
  size_t stride = global.stride;
  while (source.good())
    {
      uint32_t hash;
      source.read((char *)&hash, sizeof(hash));
      weight w = 0.;
      source.read((char *)&w, sizeof(float));

      size_t num_threads = global.num_threads();
      if (source.good())
	{
	  if (global.lda == 0)
	    if (reg_vector)
	      r.regularizers[hash % num_threads][hash/num_threads] = w;
	    else
	      r.weight_vectors[hash % num_threads][(hash*stride)/num_threads]
		= r.weight_vectors[hash % num_threads][(hash*stride)/num_threads] + w;
	  else
	      r.weight_vectors[hash % num_threads][hash/num_threads]
		= r.weight_vectors[hash % num_threads][hash/num_threads] + w;
	}
    }
  source.close();
}

void parse_regressor_args(po::variables_map& vm, regressor& r, string& final_regressor_name, bool quiet)
{
  if (vm.count("final_regressor")) {
    final_regressor_name = vm["final_regressor"].as<string>();
    if (!quiet)
      cerr << "final_regressor = " << vm["final_regressor"].as<string>() << endl;
  }
  else
    final_regressor_name = "";

  vector<string> regs;
  if (vm.count("initial_regressor"))
    regs = vm["initial_regressor"].as< vector<string> >();

  /*
     Read in regressors.  If multiple regressors are specified, do a weighted
     average.  If none are specified, initialize according to global_seg &
     numbits.
  */
  bool initialized = false;

  for (size_t i = 0; i < regs.size(); i++)
    read_vector(regs[i].c_str(), r, initialized, false);

  if (global.per_feature_regularizer_input != "")
    read_vector(global.per_feature_regularizer_input.c_str(), r, initialized, true);

  if (!initialized)
    {
      if(vm.count("noop") || vm.count("sendto"))
	r.weight_vectors = NULL;
      else
	initialize_regressor(r);
    }
}

void free_regressor(regressor &r)
{
  if (r.weight_vectors != NULL)
    {
      for (size_t i = 0; i < global.num_threads(); i++)
	if (r.weight_vectors[i] != NULL)
	  free(r.weight_vectors[i]);
      free(r.weight_vectors);
    }
  if (r.regularizers != NULL)
    {
      for (size_t i = 0; i < global.num_threads(); i++)
	if (r.regularizers[i] != NULL)
	  free(r.regularizers[i]);
      free(r.regularizers);
    }
}

void dump_regressor(string reg_name, regressor &r, bool as_text, bool reg_vector)
{
  if (reg_name == string(""))
    return;
  string start_name = reg_name+string(".writing");
  io_buf io_temp;

  int f = io_temp.open_file(start_name.c_str(),io_buf::WRITE);

  if (f<0)
    {
      cout << "can't open: " << start_name << " for writing, exiting" << endl;
      exit(1);
    }
  size_t v_length = version.length()+1;
  if (!as_text) {
    io_temp.write_file(f,(char*)&v_length, sizeof(v_length));
    io_temp.write_file(f,version.c_str(),v_length);

    io_temp.write_file(f,(char*)&global.min_label, sizeof(global.min_label));
    io_temp.write_file(f,(char*)&global.max_label, sizeof(global.max_label));

    io_temp.write_file(f,(char *)&global.num_bits, sizeof(global.num_bits));
    io_temp.write_file(f,(char *)&global.thread_bits, sizeof(global.thread_bits));
    int len = global.pairs.size();
    io_temp.write_file(f,(char *)&len, sizeof(len));
    for (vector<string>::iterator i = global.pairs.begin(); i != global.pairs.end();i++)
      io_temp.write_file(f,i->c_str(),2);

    io_temp.write_file(f,(char*)&global.ngram, sizeof(global.ngram));
    io_temp.write_file(f,(char*)&global.skips, sizeof(global.skips));
  }
  else {
    char buff[512];
    int len;
    len = sprintf(buff, "Version %s\n", version.c_str());
    io_temp.write_file(f, buff, len);
    len = sprintf(buff, "Min label:%f max label:%f\n", global.min_label, global.max_label);
    io_temp.write_file(f, buff, len);
    len = sprintf(buff, "bits:%d thread_bits:%d\n", (int)global.num_bits, (int)global.thread_bits);
    io_temp.write_file(f, buff, len);
    for (vector<string>::iterator i = global.pairs.begin(); i != global.pairs.end();i++) {
      len = sprintf(buff, "%s ", i->c_str());
      io_temp.write_file(f, buff, len);
    }
    if (global.pairs.size() > 0)
      {
	len = sprintf(buff, "\n");
	io_temp.write_file(f, buff, len);
      }
    len = sprintf(buff, "ngram:%d skips:%d\nindex:weight pairs:\n", (int)global.ngram, (int)global.skips);
    io_temp.write_file(f, buff, len);
  }

  uint32_t length = 1 << global.num_bits;
  size_t num_threads = global.num_threads();
  size_t stride = global.stride;
  for(uint32_t i = 0; i < length; i++)
    {
      if (global.lda == 0)
	{
	  weight v;
	  if (reg_vector)
	    v = r.regularizers[i%num_threads][(i/num_threads)];
	  else
	    v = r.weight_vectors[i%num_threads][stride*(i/num_threads)];
	  if (v != 0.)
	    {
              if (!as_text) {
                io_temp.write_file(f,(char *)&i, sizeof (i));
                io_temp.write_file(f,(char *)&v, sizeof (v));
              } else {
                char buff[512];
                int len = sprintf(buff, "%d:%f\n", i, v);
                io_temp.write_file(f, buff, len);
              }
	    }
	}
      else
	{
	  size_t K;

	  if (global.lda != 0)
	    K = global.lda;

          for (size_t k = 0; k < K; k++)
            {
              weight v = r.weight_vectors[i%num_threads][(stride*i+k)/num_threads];
              uint32_t ndx = stride*i+k;
              if (!as_text) {
                io_temp.write_file(f,(char *)&ndx, sizeof (ndx));
                io_temp.write_file(f,(char *)&v, sizeof (v));
              } else {
                char buff[512];
                int len = sprintf(buff, "%f ", v + global.lda_rho);
                io_temp.write_file(f, buff, len);
              }
            }
          if (as_text)
            io_temp.write_file(f, "\n", 1);
	}
    }

  rename(start_name.c_str(),reg_name.c_str());

  io_temp.close_file();
}

void finalize_regressor(string reg_name, regressor &r)
{
  dump_regressor(reg_name, r, false);
  dump_regressor(global.text_regressor_name, r, true);
  dump_regressor(global.per_feature_regularizer_output, r, false, true);
  dump_regressor(global.per_feature_regularizer_output, r, true, true);
  free_regressor(r);
}
<commit_msg>fix rename/close order bug from Raja<commit_after>/*
Copyright (c) 2009 Yahoo! Inc.  All rights reserved.  The copyrights
embodied in the content of this file are licensed under the BSD
(revised) open source license
 */

#include <fstream>
#include <iostream>
using namespace std;

#include <unistd.h>
#include <stdlib.h>
#include <stdint.h>
#include <math.h>
#include "parse_regressor.h"
#include "loss_functions.h"
#include "global_data.h"
#include "io.h"

void initialize_regressor(regressor &r)
{
  size_t length = ((size_t)1) << global.num_bits;
  global.thread_mask = (global.stride * (length >> global.thread_bits)) - 1;
  size_t num_threads = global.num_threads();
  r.weight_vectors = (weight **)malloc(num_threads * sizeof(weight*));
  if (global.per_feature_regularizer_input != "")
    r.regularizers = (weight **)malloc(num_threads * sizeof(weight*));
  else
    r.regularizers = NULL;
  for (size_t i = 0; i < num_threads; i++)
    {
      r.weight_vectors[i] = (weight *)calloc(global.stride*length/num_threads, sizeof(weight));
      if (r.regularizers != NULL)
	r.regularizers[i] = (weight *)calloc(length/num_threads, sizeof(weight));
      if (r.weight_vectors[i] == NULL || (r.regularizers != NULL && r.regularizers[i] == NULL))
        {
          cerr << global.program_name << ": Failed to allocate weight array: try decreasing -b <bits>" << endl;
          exit (1);
        }
      if (global.initial_weight != 0.)
	for (size_t j = 0; j < global.stride*length/num_threads; j+=global.stride)
	  r.weight_vectors[i][j] = global.initial_weight;
      if (global.random_weights)
	for (size_t j = 0; j < length/num_threads; j++) {
          r.weight_vectors[i][j] = drand48() - 0.5;
        }
      if (global.lda)
	{
	  size_t stride = global.stride;

          for (size_t j = 0; j < stride*length/num_threads; j+=stride)
	    {
	      for (size_t k = 0; k < global.lda; k++) {
                r.weight_vectors[i][j+k] = -log(drand48()) + 1.0;
//                 r.weight_vectors[i][j+k] *= r.weight_vectors[i][j+k];
//                 r.weight_vectors[i][j+k] *= r.weight_vectors[i][j+k];
		r.weight_vectors[i][j+k] *= (float)global.lda_D / (float)global.lda
		  / global.length() * 200;
              }
	      r.weight_vectors[i][j+global.lda] = global.initial_t;
	    }
	}
      if(global.adaptive)
        for (size_t j = 1; j < global.stride*length/num_threads; j+=global.stride)
	  r.weight_vectors[i][j] = 1;
    }
}

void read_vector(const char* file, regressor& r, bool& initialized, bool reg_vector)
{
  ifstream source(file);
  if (!source.is_open())
    {
      cout << "can't open " << file << endl << " ... exiting." << endl;
      exit(1);
    }

  size_t v_length;
  source.read((char*)&v_length, sizeof(v_length));
  char t[v_length];
  source.read(t,v_length);
  if (strcmp(t,version.c_str()) != 0)
    {
      cout << "source has possibly incompatible version!" << endl;
      exit(1);
    }

  source.read((char*)&global.min_label, sizeof(global.min_label));
  source.read((char*)&global.max_label, sizeof(global.max_label));

  size_t local_num_bits;
  source.read((char *)&local_num_bits, sizeof(local_num_bits));
  if (!initialized){
    if (global.default_bits != true && global.num_bits != local_num_bits)
      {
	cout << "Wrong number of bits for source!" << endl;
	exit (1);
      }
    global.default_bits = false;
    global.num_bits = local_num_bits;
  }
  else
    if (local_num_bits != global.num_bits)
      {
	cout << "can't combine sources with different feature number!" << endl;
	exit (1);
      }

  size_t local_thread_bits;
  source.read((char*)&local_thread_bits, sizeof(local_thread_bits));
  if (!initialized){
    global.thread_bits = local_thread_bits;
    global.partition_bits = global.thread_bits;
  }
  else
    if (local_thread_bits != global.thread_bits)
      {
	cout << "can't combine sources trained with different numbers of threads!" << endl;
	exit (1);
      }

  int len;
  source.read((char *)&len, sizeof(len));

  vector<string> local_pairs;
  for (; len > 0; len--)
    {
      char pair[2];
      source.read(pair, sizeof(char)*2);
      string temp(pair, 2);
      local_pairs.push_back(temp);
    }
  if (!initialized)
    {
      global.pairs = local_pairs;
      initialize_regressor(r);
    }
  else
    if (local_pairs != global.pairs)
      {
	cout << "can't combine sources with different features!" << endl;
	for (size_t i = 0; i < local_pairs.size(); i++)
	  cout << local_pairs[i] << " " << local_pairs[i].size() << " ";
	cout << endl;
	for (size_t i = 0; i < global.pairs.size(); i++)
	  cout << global.pairs[i] << " " << global.pairs[i].size() << " ";
	cout << endl;
	exit (1);
      }
  size_t local_ngram;
  source.read((char*)&local_ngram, sizeof(local_ngram));
  size_t local_skips;
  source.read((char*)&local_skips, sizeof(local_skips));
  if (!initialized)
    {
      global.ngram = local_ngram;
      global.skips = local_skips;
      initialized = true;
    }
  else
    if (global.ngram != local_ngram || global.skips != local_skips)
      {
	cout << "can't combine sources with different ngram features!" << endl;
	exit(1);
      }
  size_t stride = global.stride;
  while (source.good())
    {
      uint32_t hash;
      source.read((char *)&hash, sizeof(hash));
      weight w = 0.;
      source.read((char *)&w, sizeof(float));

      size_t num_threads = global.num_threads();
      if (source.good())
	{
	  if (global.lda == 0)
	    if (reg_vector)
	      r.regularizers[hash % num_threads][hash/num_threads] = w;
	    else
	      r.weight_vectors[hash % num_threads][(hash*stride)/num_threads]
		= r.weight_vectors[hash % num_threads][(hash*stride)/num_threads] + w;
	  else
	      r.weight_vectors[hash % num_threads][hash/num_threads]
		= r.weight_vectors[hash % num_threads][hash/num_threads] + w;
	}
    }
  source.close();
}

void parse_regressor_args(po::variables_map& vm, regressor& r, string& final_regressor_name, bool quiet)
{
  if (vm.count("final_regressor")) {
    final_regressor_name = vm["final_regressor"].as<string>();
    if (!quiet)
      cerr << "final_regressor = " << vm["final_regressor"].as<string>() << endl;
  }
  else
    final_regressor_name = "";

  vector<string> regs;
  if (vm.count("initial_regressor"))
    regs = vm["initial_regressor"].as< vector<string> >();

  /*
     Read in regressors.  If multiple regressors are specified, do a weighted
     average.  If none are specified, initialize according to global_seg &
     numbits.
  */
  bool initialized = false;

  for (size_t i = 0; i < regs.size(); i++)
    read_vector(regs[i].c_str(), r, initialized, false);

  if (global.per_feature_regularizer_input != "")
    read_vector(global.per_feature_regularizer_input.c_str(), r, initialized, true);

  if (!initialized)
    {
      if(vm.count("noop") || vm.count("sendto"))
	r.weight_vectors = NULL;
      else
	initialize_regressor(r);
    }
}

void free_regressor(regressor &r)
{
  if (r.weight_vectors != NULL)
    {
      for (size_t i = 0; i < global.num_threads(); i++)
	if (r.weight_vectors[i] != NULL)
	  free(r.weight_vectors[i]);
      free(r.weight_vectors);
    }
  if (r.regularizers != NULL)
    {
      for (size_t i = 0; i < global.num_threads(); i++)
	if (r.regularizers[i] != NULL)
	  free(r.regularizers[i]);
      free(r.regularizers);
    }
}

void dump_regressor(string reg_name, regressor &r, bool as_text, bool reg_vector)
{
  if (reg_name == string(""))
    return;
  string start_name = reg_name+string(".writing");
  io_buf io_temp;

  int f = io_temp.open_file(start_name.c_str(),io_buf::WRITE);

  if (f<0)
    {
      cout << "can't open: " << start_name << " for writing, exiting" << endl;
      exit(1);
    }
  size_t v_length = version.length()+1;
  if (!as_text) {
    io_temp.write_file(f,(char*)&v_length, sizeof(v_length));
    io_temp.write_file(f,version.c_str(),v_length);

    io_temp.write_file(f,(char*)&global.min_label, sizeof(global.min_label));
    io_temp.write_file(f,(char*)&global.max_label, sizeof(global.max_label));

    io_temp.write_file(f,(char *)&global.num_bits, sizeof(global.num_bits));
    io_temp.write_file(f,(char *)&global.thread_bits, sizeof(global.thread_bits));
    int len = global.pairs.size();
    io_temp.write_file(f,(char *)&len, sizeof(len));
    for (vector<string>::iterator i = global.pairs.begin(); i != global.pairs.end();i++)
      io_temp.write_file(f,i->c_str(),2);

    io_temp.write_file(f,(char*)&global.ngram, sizeof(global.ngram));
    io_temp.write_file(f,(char*)&global.skips, sizeof(global.skips));
  }
  else {
    char buff[512];
    int len;
    len = sprintf(buff, "Version %s\n", version.c_str());
    io_temp.write_file(f, buff, len);
    len = sprintf(buff, "Min label:%f max label:%f\n", global.min_label, global.max_label);
    io_temp.write_file(f, buff, len);
    len = sprintf(buff, "bits:%d thread_bits:%d\n", (int)global.num_bits, (int)global.thread_bits);
    io_temp.write_file(f, buff, len);
    for (vector<string>::iterator i = global.pairs.begin(); i != global.pairs.end();i++) {
      len = sprintf(buff, "%s ", i->c_str());
      io_temp.write_file(f, buff, len);
    }
    if (global.pairs.size() > 0)
      {
	len = sprintf(buff, "\n");
	io_temp.write_file(f, buff, len);
      }
    len = sprintf(buff, "ngram:%d skips:%d\nindex:weight pairs:\n", (int)global.ngram, (int)global.skips);
    io_temp.write_file(f, buff, len);
  }

  uint32_t length = 1 << global.num_bits;
  size_t num_threads = global.num_threads();
  size_t stride = global.stride;
  for(uint32_t i = 0; i < length; i++)
    {
      if (global.lda == 0)
	{
	  weight v;
	  if (reg_vector)
	    v = r.regularizers[i%num_threads][(i/num_threads)];
	  else
	    v = r.weight_vectors[i%num_threads][stride*(i/num_threads)];
	  if (v != 0.)
	    {
              if (!as_text) {
                io_temp.write_file(f,(char *)&i, sizeof (i));
                io_temp.write_file(f,(char *)&v, sizeof (v));
              } else {
                char buff[512];
                int len = sprintf(buff, "%d:%f\n", i, v);
                io_temp.write_file(f, buff, len);
              }
	    }
	}
      else
	{
	  size_t K;

	  if (global.lda != 0)
	    K = global.lda;

          for (size_t k = 0; k < K; k++)
            {
              weight v = r.weight_vectors[i%num_threads][(stride*i+k)/num_threads];
              uint32_t ndx = stride*i+k;
              if (!as_text) {
                io_temp.write_file(f,(char *)&ndx, sizeof (ndx));
                io_temp.write_file(f,(char *)&v, sizeof (v));
              } else {
                char buff[512];
                int len = sprintf(buff, "%f ", v + global.lda_rho);
                io_temp.write_file(f, buff, len);
              }
            }
          if (as_text)
            io_temp.write_file(f, "\n", 1);
	}
    }

  io_temp.close_file();
  rename(start_name.c_str(),reg_name.c_str());
}

void finalize_regressor(string reg_name, regressor &r)
{
  dump_regressor(reg_name, r, false);
  dump_regressor(global.text_regressor_name, r, true);
  dump_regressor(global.per_feature_regularizer_output, r, false, true);
  dump_regressor(global.per_feature_regularizer_output, r, true, true);
  free_regressor(r);
}
<|endoftext|>


<commit_before>#include "Timer.hpp"
#include "Cpu.hpp"
#include "registerAddr.hpp"
#include "interrupt.hpp"

/*
** ############################################################################
** CREATE Singleton
** ############################################################################
*/

Timer::Timer(Memory *memory) :
	_memory(memory),
	_divider(0),
	_cyclesAcc(0)
{
	this->_memory->write_byte(REGISTER_TIMA, 0x00);
	this->_memory->write_byte(REGISTER_TMA, 0x00);
	this->_memory->write_byte(REGISTER_TAC, 0x00);
	this->reset();
}

/*
** ############################################################################
** Methodes SETTEUR
** ############################################################################
*/

void Timer::reset(void)
{
	this->_divider = 0;
	this->_cyclesAcc = 0;
}

void Timer::step(unsigned int cycles)
{
	uint8_t			tac = _memory->read_byte(REGISTER_TAC);
	uint8_t			freqId = tac & 0x3;
	unsigned int	cyclesAccTima = Cpu_z80::getClockSpeed() / _arrayFrequency[freqId];

	_divider += cycles;
	if (_divider >= Cpu_z80::getClockSpeed() / 16384)
	{
		incDivider();
		_divider -= Cpu_z80::getClockSpeed() / 16384;
	}
	if ((tac & 0x4) == 0x4) // Flag timer stop
	{
		_cyclesAcc += cycles;
		if (_cyclesAcc >= cyclesAccTima)
		{
			incTima();
			_cyclesAcc -= cyclesAccTima;
		}
	}
}

void	Timer::incTima(void)
{
	uint8_t			tima = _memory->read_byte(REGISTER_TIMA);
	uint8_t			tma = _memory->read_byte(REGISTER_TMA);

	if (tima == 0xFF)
	{
		_memory->write_byte(REGISTER_TIMA, tma);
		_memory->write_byte(REGISTER_IF, INTER_TOVERF);
	}
	else
		_memory->write_byte(REGISTER_TIMA, tima + 1);
}

void	Timer::incDivider(void)
{
	uint8_t			div = _memory->read_byte(REGISTER_DIV);

	if (div == 0xFF)
		_memory->write_byte(REGISTER_DIV, 0x00);
	else
		_memory->write_byte(REGISTER_DIV, div + 1, true);
}
<commit_msg>Fix interrupt Timer overflow<commit_after>#include "Timer.hpp"
#include "Cpu.hpp"
#include "registerAddr.hpp"
#include "interrupt.hpp"

/*
** ############################################################################
** CREATE Singleton
** ############################################################################
*/

Timer::Timer(Memory *memory) :
	_memory(memory),
	_divider(0),
	_cyclesAcc(0)
{
	this->_memory->write_byte(REGISTER_TIMA, 0x00);
	this->_memory->write_byte(REGISTER_TMA, 0x00);
	this->_memory->write_byte(REGISTER_TAC, 0x00);
	this->reset();
}

/*
** ############################################################################
** Methodes SETTEUR
** ############################################################################
*/

void Timer::reset(void)
{
	this->_divider = 0;
	this->_cyclesAcc = 0;
}

void Timer::step(unsigned int cycles)
{
	uint8_t			tac = _memory->read_byte(REGISTER_TAC);
	uint8_t			freqId = tac & 0x3;
	unsigned int	cyclesAccTima = Cpu_z80::getClockSpeed() / _arrayFrequency[freqId];

	_divider += cycles;
	if (_divider >= Cpu_z80::getClockSpeed() / 16384)
	{
		incDivider();
		_divider -= Cpu_z80::getClockSpeed() / 16384;
	}
	if ((tac & 0x4) == 0x4) // Flag timer stop
	{
		_cyclesAcc += cycles;
		if (_cyclesAcc >= cyclesAccTima)
		{
			incTima();
			_cyclesAcc -= cyclesAccTima;
		}
	}
}

void	Timer::incTima(void)
{
	uint8_t			tima = _memory->read_byte(REGISTER_TIMA);
	uint8_t			tma = _memory->read_byte(REGISTER_TMA);

	if (tima == 0xFF)
	{
		_memory->write_byte(REGISTER_TIMA, tma);
		_memory->write_byte(REGISTER_IF, _memory->read_byte(REGISTER_IF) | INTER_TOVERF);
	}
	else
		_memory->write_byte(REGISTER_TIMA, tima + 1);
}

void	Timer::incDivider(void)
{
	uint8_t			div = _memory->read_byte(REGISTER_DIV);

	if (div == 0xFF)
		_memory->write_byte(REGISTER_DIV, 0x00);
	else
		_memory->write_byte(REGISTER_DIV, div + 1, true);
}
<|endoftext|>


<commit_before>#include "decompose_imf_lib/optimization_task.h"

#include "decompose_imf_lib/calculations.h"
#include "decompose_imf_lib/processing.h"

#include "cpp_utils/math_constants.h"
#include "cpp_utils/optimize.h"
#include "cpp_utils/parallel_executor.h"
#include "cpp_utils/std_make_unique.h"

#include "qt_utils/exception_handling.h"

#include <opencv/cv.h>

namespace dimf
{

//void OptimizationTask::start( OptimizationParams params )
std::vector<std::vector<double> > runOptimization(
        const OptimizationParams & params )
{
    using cu::pi;

    auto done = false;
    auto currentImf = size_t{0};

    const auto processingFunctions = createProcessingFunctions();
    const auto samples = processSamples(
                params.samples,
                params.preprocessing,
                processingFunctions );
    auto imfs = std::vector<std::vector<double> >{};

    // This variable is shared between 'shallTerminate' and 'sendBestFit'.
    auto nIter = 0;

    while ( !done )
    {
        const auto oldImf = currentImf;
        std::cout << "Optimizing IMF " << currentImf << "." << std::endl;
        auto f = samples;
        for( const auto & imf : imfs )
            cu::subAssign( begin(f), end(f), 1., begin(imf), end(imf) );
        f = processSamples( f, params.interprocessing, processingFunctions );
        const auto nSamples = f.size();

        // calculate an initial approximation and swarm
        CU_ASSERT_THROW( params.initializer,
                         "The algorithm for the initial "
                         "approximation has not been specified." );
        const auto initApprox = params.initializer(f);

        // calculate base with equidistant center points of logistic functions
        auto logisticBase = std::vector<std::vector<double>>{};
        auto nodes = std::vector<double>{};
        const auto factor = nSamples / params.xIntervalWidth;
        const auto initSigma = params.initSigmaUnits * factor;
        for ( auto i = size_t{0}; i < params.nParams; ++i )
        {
            nodes.push_back( (i+.5)*initApprox.size()/params.nParams );
            logisticBase.push_back( getSamplesFromLogisticFunctionBase(
                { 1., nodes.back() }, initSigma, initApprox.size() ) );
        }
        auto logisticBaseMat = cv::Mat(
                    logisticBase.front().size(), logisticBase.size(),
                    CV_64FC1, cv::Scalar::all(0) );
        for ( auto row = 0; row < logisticBaseMat.rows; ++row )
        {
            for ( auto col = 0; col < logisticBaseMat.cols; ++col )
            {
                logisticBaseMat.at<double>(row,col) =
                        logisticBase.at(col).at(row);
            }
        }

        // calculate the element closest to initApprox that lies in the
        // space spanned by the base elements. Also calculate the
        // coefficients of that minimum element with respect to the given
        // base
        auto initApproxMat = cv::Mat(
                    initApprox.size(), 1, CV_64FC1, cv::Scalar::all(0) );
        for ( auto row = 0; row < initApproxMat.rows; ++row )
        {
            initApproxMat.at<double>( row ) = initApprox.at(row).imag();
        }
        const auto invLogisticBase =
                cv::Mat{ logisticBaseMat.inv( cv::DECOMP_SVD ) };
        const auto bestApproxMat =
                cv::Mat{ invLogisticBase * initApproxMat };

        auto swarm = std::vector<std::vector<double>>( params.swarmSize );
        auto rng = std::mt19937{};
        {
            auto normal_dist = std::normal_distribution<>{};
            auto uniform = std::uniform_real_distribution<double>{-1,1};
            for ( auto & x : swarm )
            {
                const auto sigmaDev  = params.sigmaDevUnits  * factor;
                const auto tauDev    = params.tauDevUnits    * factor;
                const auto initTau   = params.initTauUnits   * factor;
                const auto nodeDev   = params.nodeDevUnits   * factor;
                const auto angleDev  = params.angleDevDegs/180*cu::pi;
                for ( auto i = size_t{0}; i < nodes.size(); ++i )
                {
                    x.push_back( params.amplitudeDev*normal_dist(rng) );
                    x.push_back( nodes[i] + nodeDev*normal_dist(rng) );
                }
                for ( auto i = size_t{0}; i < nodes.size(); ++i )
                {
                    x.push_back( bestApproxMat.at<double>( i ) +
                                 angleDev*uniform(rng) );
                    x.push_back( nodes[i] );
                }
                x.push_back( initSigma + sigmaDev*normal_dist(rng) );
                x.push_back( initTau + tauDev*normal_dist(rng) );
            }
        }

        // cost function for optimization
        const auto cost = [&f, nSamples]( std::vector<double> v ) -> double
        {
            return costFunction( f,
                getSamplesFromParams( std::move(v), nSamples ) );
        };

        // function which returns whether the
        // optimization algorithm shall terminate.
        const auto shallTerminate = [&]( const decltype(swarm) & )-> bool
        {
            ++nIter;
            const auto nextImf = params.howToContinue( nIter );
            if ( nextImf == ~size_t{0} )
            {
                done = true;
                return true;
            }
            if ( nextImf == currentImf )
                return false;
            currentImf = nextImf;
            return true;
        };

        // function which is called by the optimization algorithm
        // each time the best fit is improved.
        auto bestParams = std::vector<double>{};
        const auto sendBestFit = [&](
                const std::vector<double> & v_, double cost )
        {
            bestParams = v_;
            params.receiveBestFit(bestParams,cost,nSamples,nIter,f);
        };

        // perform the optimization.
        swarm = cu::differentialEvolution(
            std::move(swarm), params.crossOverProb, params.diffWeight,
            cost, shallTerminate, sendBestFit, rng );

        const auto bestImf = calculateImfFromPairsOfReals(
                    getSamplesFromParams( bestParams, nSamples ) );
        if ( oldImf >= imfs.size() )
        {
            CU_ASSERT_THROW( oldImf == imfs.size(),
                "Invalid IMF index. Have some indexes been skipped?" );
            imfs.push_back({});
        }
        imfs[oldImf] = bestImf;
    } // while loop
    return imfs;
}

} // namespace dimf
<commit_msg>Bugfix: Improvement of already optimized IMFs can now not yield worse results than previously.<commit_after>#include "decompose_imf_lib/optimization_task.h"

#include "decompose_imf_lib/calculations.h"
#include "decompose_imf_lib/processing.h"

#include "cpp_utils/math_constants.h"
#include "cpp_utils/optimize.h"
#include "cpp_utils/parallel_executor.h"
#include "cpp_utils/std_make_unique.h"

#include "qt_utils/exception_handling.h"

#include <opencv/cv.h>

namespace dimf
{

//void OptimizationTask::start( OptimizationParams params )
std::vector<std::vector<double> > runOptimization(
        const OptimizationParams & params )
{
    using cu::pi;

    auto done = false;
    auto currentImf = size_t{0};

    const auto processingFunctions = createProcessingFunctions();
    const auto samples = processSamples(
                params.samples,
                params.preprocessing,
                processingFunctions );
    auto imfs = std::vector<std::vector<double> >{};
    auto bestParamSets = std::vector<std::vector<double> >{};

    // This variable is shared between 'shallTerminate' and 'sendBestFit'.
    auto nIter = 0;

    while ( !done )
    {
        const auto oldImf = currentImf;
        std::cout << "Optimizing IMF " << currentImf << "." << std::endl;
        auto f = samples;
        for ( auto i = size_t(); i < imfs.size(); ++i )
        {
            if ( i == currentImf )
                continue;
            const auto & imf = imfs[i];
            cu::subAssign( begin(f), end(f), 1., begin(imf), end(imf) );
        }
        f = processSamples( f,
                params.interprocessing,
                processingFunctions );
        const auto nSamples = f.size();

        // calculate an initial approximation and swarm
        CU_ASSERT_THROW( params.initializer,
                         "The algorithm for the initial "
                         "approximation has not been specified." );
        const auto initApprox = params.initializer(f);

        // calculate base with equidistant center points of logistic functions
        auto logisticBase = std::vector<std::vector<double>>{};
        auto nodes = std::vector<double>{};
        const auto factor = nSamples / params.xIntervalWidth;
        const auto initSigma = params.initSigmaUnits * factor;
        for ( auto i = size_t{0}; i < params.nParams; ++i )
        {
            nodes.push_back( (i+.5)*initApprox.size()/params.nParams );
            logisticBase.push_back( getSamplesFromLogisticFunctionBase(
                { 1., nodes.back() }, initSigma, initApprox.size() ) );
        }
        auto logisticBaseMat = cv::Mat(
                    logisticBase.front().size(), logisticBase.size(),
                    CV_64FC1, cv::Scalar::all(0) );
        for ( auto row = 0; row < logisticBaseMat.rows; ++row )
        {
            for ( auto col = 0; col < logisticBaseMat.cols; ++col )
            {
                logisticBaseMat.at<double>(row,col) =
                        logisticBase.at(col).at(row);
            }
        }

        // calculate the element closest to initApprox that lies in the
        // space spanned by the base elements. Also calculate the
        // coefficients of that minimum element with respect to the given
        // base
        auto initApproxMat = cv::Mat(
                    initApprox.size(), 1, CV_64FC1, cv::Scalar::all(0) );
        for ( auto row = 0; row < initApproxMat.rows; ++row )
        {
            initApproxMat.at<double>( row ) = initApprox.at(row).imag();
        }
        const auto invLogisticBase =
                cv::Mat{ logisticBaseMat.inv( cv::DECOMP_SVD ) };
        const auto bestApproxMat =
                cv::Mat{ invLogisticBase * initApproxMat };

        auto swarm = std::vector<std::vector<double> >( params.swarmSize );
        auto rng = std::mt19937{};
        {
            auto normal_dist = std::normal_distribution<>{};
            auto uniform = std::uniform_real_distribution<double>{-1,1};
            for ( auto & x : swarm )
            {
                const auto sigmaDev  = params.sigmaDevUnits  * factor;
                const auto tauDev    = params.tauDevUnits    * factor;
                const auto initTau   = params.initTauUnits   * factor;
                const auto nodeDev   = params.nodeDevUnits   * factor;
                const auto angleDev  = params.angleDevDegs/180*cu::pi;
                for ( auto i = size_t{0}; i < nodes.size(); ++i )
                {
                    x.push_back( params.amplitudeDev*normal_dist(rng) );
                    x.push_back( nodes[i] + nodeDev*normal_dist(rng) );
                }
                for ( auto i = size_t{0}; i < nodes.size(); ++i )
                {
                    x.push_back( bestApproxMat.at<double>( i ) +
                                 angleDev*uniform(rng) );
                    x.push_back( nodes[i] );
                }
                x.push_back( initSigma + sigmaDev*normal_dist(rng) );
                x.push_back( initTau + tauDev*normal_dist(rng) );
            }
            if ( currentImf < bestParamSets.size() )
                swarm.front() = bestParamSets.at(currentImf);
        }

        // cost function for optimization
        const auto cost = [&f, nSamples]( std::vector<double> v ) -> double
        {
            return costFunction( f,
                getSamplesFromParams( std::move(v), nSamples ) );
        };

        // function which returns whether the
        // optimization algorithm shall terminate.
        const auto shallTerminate = [&]( const decltype(swarm) & )-> bool
        {
            ++nIter;
            const auto nextImf = params.howToContinue( nIter );
            if ( nextImf == ~size_t{0} )
            {
                done = true;
                return true;
            }
            if ( nextImf == currentImf )
                return false;
            currentImf = nextImf;
            return true;
        };

        // function which is called by the optimization algorithm
        // each time the best fit is improved.
        auto bestParams = std::vector<double>{};
        const auto sendBestFit = [&](
                const std::vector<double> & v_, double cost )
        {
            bestParams = v_;
            params.receiveBestFit(bestParams,cost,nSamples,nIter,f);
        };

        // perform the optimization.
        swarm = cu::differentialEvolution(
            std::move(swarm), params.crossOverProb, params.diffWeight,
            cost, shallTerminate, sendBestFit, rng );

        const auto bestImf = calculateImfFromPairsOfReals(
                    getSamplesFromParams( bestParams, nSamples ) );
        if ( oldImf >= imfs.size() )
        {
            CU_ASSERT_THROW( oldImf == imfs.size(),
                "Invalid IMF index. Have some indexes been skipped?" );
            imfs.push_back({});
            bestParamSets.push_back({});
        }
        imfs[oldImf] = bestImf;
        bestParamSets[oldImf] = bestParams;
    } // while loop
    return imfs;
}

} // namespace dimf
<|endoftext|>


<commit_before>// Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "open_spiel/abseil-cpp/absl/flags/flag.h"
#include "open_spiel/abseil-cpp/absl/flags/usage.h"
#include "open_spiel/abseil-cpp/absl/flags/parse.h"
#include "open_spiel/higc/referee.h"

ABSL_FLAG(std::string, bots_dir, "higc/bots",
          "Directory containing the competition bots.");

namespace open_spiel {
namespace higc {
namespace {

void PlaySingleMatchIIGS() {
  std::string bot_first_action = absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                              "/test_bot_first_action.sh");
  open_spiel::higc::Referee
      ref("goofspiel(imp_info=True,points_order=descending)",
          {bot_first_action, bot_first_action}, /*seed=*/42,
      // Increase times for Python scripts.
          TournamentSettings{
              .timeout_ready = 2000,
              .timeout_start = 500,
          });
  std::unique_ptr<TournamentResults> results = ref.PlayTournament(1);
  SPIEL_CHECK_EQ(results->num_matches(), 1);
  SPIEL_CHECK_TRUE(results->matches[0].terminal->IsTerminal());
  SPIEL_CHECK_EQ(results->matches[0].terminal->HistoryString(),
                 "0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, "
                 "6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11");
}

void TestInvalidBots() {
  std::string bot_first_action = absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                              "/test_bot_first_action.sh");
  std::vector<std::string> failing_cases = {
      "/non_existing_bot",
      "/test_bot_with_non_exec_flag"
  };

  for (const std::string& failing_case : failing_cases) {
    std::cout << "Invalid bot: " << failing_case << std::endl;
    std::string invalid_bot = absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                           failing_case);
    bool thrown = false;
    try {
      open_spiel::higc::Referee ref("tic_tac_toe",
                                    {invalid_bot, bot_first_action});
    } catch (std::runtime_error& e) {
      std::cout << e.what() << std::endl;
      thrown = true;
    }
    SPIEL_CHECK_TRUE(thrown);
  }
}

void PlayWithFailingBots() {
  std::vector<std::string> failing_cases = {
      "/test_bot_break_pipe.sh",
      "/test_bot_sleep.sh",
      "/test_bot_ready.sh",
      "/test_bot_start.sh",
      "/test_bot_illegal_action.sh",
      "/test_bot_buffer_overflow.sh",
      "/test_bot_fail_after_few_actions.sh",
  };

  for (int i = 0; i < failing_cases.size(); ++i) {
    const std::string& failing_case = failing_cases[i];
    std::string failing_bot = absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                           failing_case);
    std::cout << "\n\nFailing bot: " << failing_bot << std::endl;

    // Use a single-player game.
    open_spiel::higc::Referee ref("cliff_walking", {failing_bot}, /*seed=*/42,
        /*settings=*/TournamentSettings{
            // Disqualify after the 2nd failing match.
            .disqualification_rate = 0.5
        });
    std::unique_ptr<TournamentResults> results = ref.PlayTournament(2);
    SPIEL_CHECK_EQ(results->disqualified[0], true);
    if (i < 2) {
      // No matches are played, if the bot can't even start properly.
      SPIEL_CHECK_EQ(results->num_matches(), 0);
    } else {
      SPIEL_CHECK_EQ(results->num_matches(), 2);
    }
  }
}

void PonderActTimeout() {
  open_spiel::higc::Referee ref(
      "leduc_poker",
      {absl::StrCat(absl::GetFlag(FLAGS_bots_dir), "/random_bot_py.sh"),
       absl::StrCat(absl::GetFlag(FLAGS_bots_dir), "/test_bot_start.sh")},
       /*seed=*/42,
       // Increase times for Python scripts.
       TournamentSettings{
         .timeout_ready = 2000,
         .timeout_start = 500,
         });
  std::unique_ptr<TournamentResults> results = ref.PlayTournament(1);
  SPIEL_CHECK_EQ(results->num_matches(), 1);
}

void PlayManyRandomMatches(int num_matches = 5) {
  open_spiel::higc::Referee ref(
      "leduc_poker",
      {absl::StrCat(absl::GetFlag(FLAGS_bots_dir), "/random_bot_py.sh"),
       absl::StrCat(absl::GetFlag(FLAGS_bots_dir), "/random_bot_cpp.sh")},
      /*seed=*/42,
      // Increase times for Python scripts.
      TournamentSettings{
          .timeout_ready = 2000,
          .timeout_start = 500,
      });
  std::unique_ptr<TournamentResults> results = ref.PlayTournament(num_matches);
  SPIEL_CHECK_EQ(results->num_matches(), num_matches);
  results->PrintCsv(std::cout, /*print_header=*/true);
}

void PlayWithManyPlayers() {
  constexpr const int num_bots = 8;
  std::vector<std::string> bots;
  for (int i = 0; i < num_bots; ++i) {
    bots.push_back(absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                "/random_bot_cpp.sh"));
  }
  open_spiel::higc::Referee ref(
      absl::StrCat("goofspiel(players=",num_bots,
                   ",imp_info=True,points_order=descending)"),
      bots,
      /*seed=*/42,
      // Increase times for Python scripts.
      TournamentSettings{
          .timeout_ready = 2000,
          .timeout_start = 500,
      });
  std::unique_ptr<TournamentResults> results = ref.PlayTournament(1);
  SPIEL_CHECK_EQ(results->num_matches(), 1);
}

}  // namespace
}  // namespace higc
}  // namespace open_spiel

int main(int argc, char** argv) {
  absl::ParseCommandLine(argc, argv);
  open_spiel::higc::TestInvalidBots();
  open_spiel::higc::PlayWithFailingBots();
  open_spiel::higc::PonderActTimeout();
  open_spiel::higc::PlayWithManyPlayers();
  open_spiel::higc::PlaySingleMatchIIGS();
  open_spiel::higc::PlayManyRandomMatches();
}
<commit_msg>Catch SIGPIPE signals so the ctest does not abort.<commit_after>// Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "open_spiel/abseil-cpp/absl/flags/flag.h"
#include "open_spiel/abseil-cpp/absl/flags/usage.h"
#include "open_spiel/abseil-cpp/absl/flags/parse.h"
#include "open_spiel/higc/referee.h"

ABSL_FLAG(std::string, bots_dir, "higc/bots",
          "Directory containing the competition bots.");

namespace open_spiel {
namespace higc {
namespace {

void PlaySingleMatchIIGS() {
  std::string bot_first_action = absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                              "/test_bot_first_action.sh");
  open_spiel::higc::Referee
      ref("goofspiel(imp_info=True,points_order=descending)",
          {bot_first_action, bot_first_action}, /*seed=*/42,
      // Increase times for Python scripts.
          TournamentSettings{
              .timeout_ready = 2000,
              .timeout_start = 500,
          });
  std::unique_ptr<TournamentResults> results = ref.PlayTournament(1);
  SPIEL_CHECK_EQ(results->num_matches(), 1);
  SPIEL_CHECK_TRUE(results->matches[0].terminal->IsTerminal());
  SPIEL_CHECK_EQ(results->matches[0].terminal->HistoryString(),
                 "0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, "
                 "6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11");
}

void TestInvalidBots() {
  std::string bot_first_action = absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                              "/test_bot_first_action.sh");
  std::vector<std::string> failing_cases = {
      "/non_existing_bot",
      "/test_bot_with_non_exec_flag"
  };

  for (const std::string& failing_case : failing_cases) {
    std::cout << "Invalid bot: " << failing_case << std::endl;
    std::string invalid_bot = absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                           failing_case);
    bool thrown = false;
    try {
      open_spiel::higc::Referee ref("tic_tac_toe",
                                    {invalid_bot, bot_first_action});
    } catch (std::runtime_error& e) {
      std::cout << e.what() << std::endl;
      thrown = true;
    }
    SPIEL_CHECK_TRUE(thrown);
  }
}

void PlayWithFailingBots() {
  std::vector<std::string> failing_cases = {
      "/test_bot_break_pipe.sh",
      "/test_bot_sleep.sh",
      "/test_bot_ready.sh",
      "/test_bot_start.sh",
      "/test_bot_illegal_action.sh",
      "/test_bot_buffer_overflow.sh",
      "/test_bot_fail_after_few_actions.sh",
  };

  for (int i = 0; i < failing_cases.size(); ++i) {
    const std::string& failing_case = failing_cases[i];
    std::string failing_bot = absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                           failing_case);
    std::cout << "\n\nFailing bot: " << failing_bot << std::endl;

    // Use a single-player game.
    open_spiel::higc::Referee ref("cliff_walking", {failing_bot}, /*seed=*/42,
        /*settings=*/TournamentSettings{
            // Disqualify after the 2nd failing match.
            .disqualification_rate = 0.5
        });
    std::unique_ptr<TournamentResults> results = ref.PlayTournament(2);
    SPIEL_CHECK_EQ(results->disqualified[0], true);
    if (i < 2) {
      // No matches are played, if the bot can't even start properly.
      SPIEL_CHECK_EQ(results->num_matches(), 0);
    } else {
      SPIEL_CHECK_EQ(results->num_matches(), 2);
    }
  }
}

void PonderActTimeout() {
  open_spiel::higc::Referee ref(
      "leduc_poker",
      {absl::StrCat(absl::GetFlag(FLAGS_bots_dir), "/random_bot_py.sh"),
       absl::StrCat(absl::GetFlag(FLAGS_bots_dir), "/test_bot_start.sh")},
       /*seed=*/42,
       // Increase times for Python scripts.
       TournamentSettings{
         .timeout_ready = 2000,
         .timeout_start = 500,
         });
  std::unique_ptr<TournamentResults> results = ref.PlayTournament(1);
  SPIEL_CHECK_EQ(results->num_matches(), 1);
}

void PlayManyRandomMatches(int num_matches = 5) {
  open_spiel::higc::Referee ref(
      "leduc_poker",
      {absl::StrCat(absl::GetFlag(FLAGS_bots_dir), "/random_bot_py.sh"),
       absl::StrCat(absl::GetFlag(FLAGS_bots_dir), "/random_bot_cpp.sh")},
      /*seed=*/42,
      // Increase times for Python scripts.
      TournamentSettings{
          .timeout_ready = 2000,
          .timeout_start = 500,
      });
  std::unique_ptr<TournamentResults> results = ref.PlayTournament(num_matches);
  SPIEL_CHECK_EQ(results->num_matches(), num_matches);
  results->PrintCsv(std::cout, /*print_header=*/true);
}

void PlayWithManyPlayers() {
  constexpr const int num_bots = 8;
  std::vector<std::string> bots;
  for (int i = 0; i < num_bots; ++i) {
    bots.push_back(absl::StrCat(absl::GetFlag(FLAGS_bots_dir),
                                "/random_bot_cpp.sh"));
  }
  open_spiel::higc::Referee ref(
      absl::StrCat("goofspiel(players=",num_bots,
                   ",imp_info=True,points_order=descending)"),
      bots,
      /*seed=*/42,
      // Increase times for Python scripts.
      TournamentSettings{
          .timeout_ready = 2000,
          .timeout_start = 500,
      });
  std::unique_ptr<TournamentResults> results = ref.PlayTournament(1);
  SPIEL_CHECK_EQ(results->num_matches(), 1);
}

}  // namespace
}  // namespace higc
}  // namespace open_spiel

// Reroute the SIGPIPE signall here, so the test pass ok.
void signal_callback_handler(int signum) {
  std::cout << "Caught signal SIGPIPE " << signum << std::endl;
}

int main(int argc, char** argv) {
  absl::ParseCommandLine(argc, argv);
  signal(SIGPIPE, signal_callback_handler);

  open_spiel::higc::TestInvalidBots();
  open_spiel::higc::PlayWithFailingBots();
  open_spiel::higc::PonderActTimeout();
  open_spiel::higc::PlayWithManyPlayers();
  open_spiel::higc::PlaySingleMatchIIGS();
  open_spiel::higc::PlayManyRandomMatches();
}
<|endoftext|>