<commit_before>// FONcTransmitter.cc

// This file is part of BES Netcdf File Out Module

// Copyright (c) 2004,2005 University Corporation for Atmospheric Research
// Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//
// You can contact University Corporation for Atmospheric Research at
// 3080 Center Green Drive, Boulder, CO 80301

// (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
// Please read the full copyright statement in the file COPYRIGHT_UCAR.
//
// Authors:
//      pwest       Patrick West <pwest@ucar.edu>
//      jgarcia     Jose Garcia <jgarcia@ucar.edu>

#include "config.h"

#include <stdio.h>
#include <stdlib.h>

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#include <sys/types.h>                  // For umask
#include <sys/stat.h>

#include <iostream>
#include <fstream>
#include <exception>

#include <DataDDS.h>
#include <BaseType.h>
#include <escaping.h>
#include <ConstraintEvaluator.h>

#include <BESInternalError.h>
#include <BESDapError.h>
#include <TheBESKeys.h>
#include <BESContextManager.h>
#include <BESDataDDSResponse.h>
#include <BESDapNames.h>
#include <BESDataNames.h>
#include <BESDebug.h>
#include <DapFunctionUtils.h>

#include "FONcBaseType.h"
#include "FONcRequestHandler.h"
#include "FONcTransmitter.h"
#include "FONcTransform.h"

using namespace ::libdap;
using namespace std;

// size of the buffer used to read from the temporary file built on disk and
// send data to the client over the network connection (socket/stream)
#define OUTPUT_FILE_BLOCK_SIZE 4096

/** @brief Construct the FONcTransmitter, adding it with name netcdf to be
 * able to transmit a data response
 *
 * The transmitter is created to add the ability to return OPeNDAP data
 * objects (DataDDS) as a netcdf file.
 *
 * The OPeNDAP data object is written to a netcdf file locally in a
 * temporary directory specified by the BES configuration parameter
 * FONc.Tempdir. If this variable is not found or is not set then it
 * defaults to the macro definition FONC_TEMP_DIR.
 */
FONcTransmitter::FONcTransmitter() :
    BESBasicTransmitter()
{
    add_method(DATA_SERVICE, FONcTransmitter::send_data);
}

#if 0
/**
 * Hack to ensure the file descriptor for the temporary file is closed.
 */
struct wrap_file_descriptor {
    int fd;
    wrap_file_descriptor(int i) : fd(i) { }
    ~wrap_file_descriptor() { cerr << "*** Closing fd" << endl; close(fd); }
};
#endif

/** @brief The static method registered to transmit OPeNDAP data objects as
 * a netcdf file.
 *
 * This function takes the OPeNDAP DataDDS object, reads in the data (can be
 * used with any data handler), transforms the data into a netcdf file, and
 * streams back that netcdf file back to the requester using the stream
 * specified in the BESDataHandlerInterface.
 *
 * @param obj The BESResponseObject containing the OPeNDAP DataDDS object
 * @param dhi BESDataHandlerInterface containing information about the
 * request and response
 * @throws BESInternalError if the response is not an OPeNDAP DataDDS or if
 * there are any problems reading the data, writing to a netcdf file, or
 * streaming the netcdf file
 */
void FONcTransmitter::send_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
{
    BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(obj);
    if (!bdds) throw BESInternalError("cast error", __FILE__, __LINE__);

    libdap::DataDDS *dds = bdds->get_dds();
    if (!dds) throw BESInternalError("No DataDDS has been created for transmit", __FILE__, __LINE__);

    BESDEBUG("fonc", "FONcTransmitter::send_data() - Parsing the constraint" << endl);

    ConstraintEvaluator &eval = bdds->get_ce();

    string ncVersion = dhi.data[RETURN_CMD];

    ostream &strm = dhi.get_output_stream();
    if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);

    // ticket 1248 jhrg 2/23/09
    string ce = www2id(dhi.data[POST_CONSTRAINT], "%", "%20%26");
    try {
        eval.parse_constraint(ce, *dds);
    }
    catch (Error &e) {
        throw BESDapError("Failed to parse the constraint expression: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
    }
    catch (...) {
        throw BESInternalError("Failed to parse the constraint expression: Unknown exception caught", __FILE__,
            __LINE__);
    }

    // The dataset_name is no longer used in the constraint evaluator, so no
    // need to get here. Plus, just getting the first container's dataset
    // name would not have worked with multiple containers.
    // pwest Jan 4, 2009
    // string dataset_name = "";

    // now we need to read the data
    BESDEBUG("fonc", "FONcTransmitter::send_data() - Reading data into DataDDS" << endl);

    // ADB: remember when we're using a temp DDS
    // bool using_temp_dds = false; See comment below about set_dds(). jhrg 8/8/14

    try {
        // Handle *functional* constraint expressions specially
        if (eval.function_clauses()) {
            BESDEBUG("fonc", "FONcTransmitter::send_data() - Processing functional constraint clause(s)." << endl);
            DataDDS *tmp_dds = eval.eval_function_clauses(*dds);
            delete dds;
            dds = tmp_dds;
            bdds->set_dds(dds);

            // This next step utilizes a well known function, promote_function_output_structures()
            // to look for one or more top level Structures whose name indicates (by way of ending
            // with "_uwrap") that their contents should be promoted (aka moved) to the top level.
            // This is in support of a hack around the current API where server side functions
            // may only return a single DAP object and not a collection of objects. The name suffix
            // "_unwrap" is used as a signal from the function to the the various response
            // builders and transmitters that the representation needs to be altered before
            // transmission, and that in fact is what happens in our friend
            // promote_function_output_structures()
            promote_function_output_structures(dds);

        }
        else {
            // Iterate through the variables in the DataDDS and read
            // in the data if the variable has the send flag set.

            for (DDS::Vars_iter i = dds->var_begin(); i != dds->var_end(); i++) {
                if ((*i)->send_p()) {
                    BESDEBUG("fonc", "FONcTransmitter::send_data() - Interning data for variable: '" << (*i)->name() << "'" << endl);
                    (*i)->intern_data(eval, *dds);
                }
            }
        }
    }
    catch (Error &e) {
        throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
    }
    catch (BESError &e) {
        throw;
    }
    catch (std::exception &e) {
        throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
    }
    catch (...) {
        throw BESInternalError("Failed to read data. Unknown Error", __FILE__, __LINE__);
    }

    //string temp_file_name = FONcTransmitter::temp_dir + '/' + "ncXXXXXX";
    string temp_file_name = FONcRequestHandler::temp_dir + '/' + "ncXXXXXX";
    vector<char> temp_full(temp_file_name.length() + 1);
    string::size_type len = temp_file_name.copy(&temp_full[0], temp_file_name.length());
    temp_full[len] = '\0';
    // cover the case where older versions of mkstemp() create the file using
    // a mode of 666.
    mode_t original_mode = umask(077);
    int fd = mkstemp(&temp_full[0]);
    umask(original_mode);
#if 0
    // Trick: We can unlink the file right here - it will persist until the file descriptor
    // is closed. This means we don't have to litter the code with call to unlink(). jhrg 11/25/15
    (void) unlink(&temp_full[0]);
    // Hack: Use this simple class to 'wrap' the file descriptor so that we can be sure it
    // is closed no matter how this code is exited. jhrg 11/25/15
    wrap_file_descriptor wrapped_fd(fd);
#endif
    if (fd == -1) throw BESInternalError("Failed to open the temporary file: " + temp_file_name, __FILE__, __LINE__);

    // transform the OPeNDAP DataDDS to the netcdf file
    BESDEBUG("fonc", "FONcTransmitter::send_data - Transforming into temporary file: " << &temp_full[0] << endl);

    try {
        FONcTransform ft(dds, dhi, &temp_full[0], ncVersion);
        ft.transform();

        BESDEBUG("fonc", "FONcTransmitter::send_data - Transmitting temp file " << &temp_full[0] << endl);
        FONcTransmitter::return_temp_stream(&temp_full[0], strm, ncVersion);
    }
    catch (Error &e) {
        (void) unlink(&temp_full[0]);
        close(fd);
        throw BESDapError("Failed to Transform data to NetCDF: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
    }
    catch (BESError &e) {
        (void) unlink(&temp_full[0]);
        close(fd);
        throw;
    }
    catch (std::exception &e) {
        (void) unlink(&temp_full[0]);
        close(fd);
        throw BESInternalError("Failed to Transform data to NetCDF: STL Error: " + string(e.what()), __FILE__, __LINE__);
    }
    catch (...) {
        (void) unlink(&temp_full[0]);
        close(fd);
        throw BESInternalError("Failed to Transform data to NetCDF. Unknown Error", __FILE__, __LINE__);
    }

    (void) unlink(&temp_full[0]);
    close(fd);

    BESDEBUG("fonc", "FONcTransmitter::send_data - done transmitting to netcdf" << endl);
}

/** @brief stream the temporary netcdf file back to the requester
 *
 * Streams the temporary netcdf file specified by filename to the specified
 * C++ ostream
 *
 * @param filename The name of the file to stream back to the requester
 * @param strm C++ ostream to write the contents of the file to
 * @throws BESInternalError if problem opening the file
 */
void FONcTransmitter::return_temp_stream(const string &filename, ostream &strm, const string &ncVersion)
{
    ifstream os;
    os.open(filename.c_str(), ios::binary | ios::in);
    if (!os)
        throw BESInternalError("Fileout netcdf: Cannot connect to netcdf file.", __FILE__, __LINE__);;

    char block[OUTPUT_FILE_BLOCK_SIZE];

    os.read(block, sizeof block);
    int nbytes = os.gcount();
    if (nbytes > 0) {
        bool found = false;
        string context = "transmit_protocol";
        string protocol = BESContextManager::TheManager()->get_context(context, found);
        if (protocol == "HTTP") {
            strm << "HTTP/1.0 200 OK\n";
            strm << "Content-type: application/octet-stream\n";
            strm << "Content-Description: " << "BES dataset" << "\n";
            if (ncVersion == RETURNAS_NETCDF4) {
                strm << "Content-Disposition: filename=" << filename << ".nc4;\n\n";
            }
            else {
                strm << "Content-Disposition: filename=" << filename << ".nc;\n\n";
            }
            strm << flush;
        }
        strm.write(block, nbytes);
    }
    else {
        // close the stream before we leave.
        os.close();
        throw BESInternalError("Fileout netcdf: Failed to stream the response back to the client, got zero count on stream buffer.", __FILE__, __LINE__);
    }

    while (os) {
        os.read(block, sizeof block);
        strm.write(block, os.gcount());
    }

    os.close();
}

<commit_msg>Dropped use of DataDDS in favor of the DDS class.<commit_after>// FONcTransmitter.cc

// This file is part of BES Netcdf File Out Module

// Copyright (c) 2004,2005 University Corporation for Atmospheric Research
// Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//
// You can contact University Corporation for Atmospheric Research at
// 3080 Center Green Drive, Boulder, CO 80301

// (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
// Please read the full copyright statement in the file COPYRIGHT_UCAR.
//
// Authors:
//      pwest       Patrick West <pwest@ucar.edu>
//      jgarcia     Jose Garcia <jgarcia@ucar.edu>

#include "config.h"

#include <stdio.h>
#include <stdlib.h>

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#include <sys/types.h>                  // For umask
#include <sys/stat.h>

#include <iostream>
#include <fstream>
#include <exception>

#include <DataDDS.h>
#include <BaseType.h>
#include <escaping.h>
#include <ConstraintEvaluator.h>

#include <BESInternalError.h>
#include <BESDapError.h>
#include <TheBESKeys.h>
#include <BESContextManager.h>
#include <BESDataDDSResponse.h>
#include <BESDapNames.h>
#include <BESDataNames.h>
#include <BESDebug.h>
#include <DapFunctionUtils.h>

#include "FONcBaseType.h"
#include "FONcRequestHandler.h"
#include "FONcTransmitter.h"
#include "FONcTransform.h"

using namespace ::libdap;
using namespace std;

// size of the buffer used to read from the temporary file built on disk and
// send data to the client over the network connection (socket/stream)
#define OUTPUT_FILE_BLOCK_SIZE 4096

/** @brief Construct the FONcTransmitter, adding it with name netcdf to be
 * able to transmit a data response
 *
 * The transmitter is created to add the ability to return OPeNDAP data
 * objects (DataDDS) as a netcdf file.
 *
 * The OPeNDAP data object is written to a netcdf file locally in a
 * temporary directory specified by the BES configuration parameter
 * FONc.Tempdir. If this variable is not found or is not set then it
 * defaults to the macro definition FONC_TEMP_DIR.
 */
FONcTransmitter::FONcTransmitter() :
    BESBasicTransmitter()
{
    add_method(DATA_SERVICE, FONcTransmitter::send_data);
}

#if 0
/**
 * Hack to ensure the file descriptor for the temporary file is closed.
 */
struct wrap_file_descriptor {
    int fd;
    wrap_file_descriptor(int i) : fd(i) { }
    ~wrap_file_descriptor() { cerr << "*** Closing fd" << endl; close(fd); }
};
#endif

/** @brief The static method registered to transmit OPeNDAP data objects as
 * a netcdf file.
 *
 * This function takes the OPeNDAP DataDDS object, reads in the data (can be
 * used with any data handler), transforms the data into a netcdf file, and
 * streams back that netcdf file back to the requester using the stream
 * specified in the BESDataHandlerInterface.
 *
 * @param obj The BESResponseObject containing the OPeNDAP DataDDS object
 * @param dhi BESDataHandlerInterface containing information about the
 * request and response
 * @throws BESInternalError if the response is not an OPeNDAP DataDDS or if
 * there are any problems reading the data, writing to a netcdf file, or
 * streaming the netcdf file
 */
void FONcTransmitter::send_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
{
    BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(obj);
    if (!bdds) throw BESInternalError("cast error", __FILE__, __LINE__);

    libdap::DDS *dds = bdds->get_dds();
    if (!dds) throw BESInternalError("No DataDDS has been created for transmit", __FILE__, __LINE__);

    BESDEBUG("fonc", "FONcTransmitter::send_data() - Parsing the constraint" << endl);

    ConstraintEvaluator &eval = bdds->get_ce();

    string ncVersion = dhi.data[RETURN_CMD];

    ostream &strm = dhi.get_output_stream();
    if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);

    // ticket 1248 jhrg 2/23/09
    string ce = www2id(dhi.data[POST_CONSTRAINT], "%", "%20%26");
    try {
        eval.parse_constraint(ce, *dds);
    }
    catch (Error &e) {
        throw BESDapError("Failed to parse the constraint expression: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
    }
    catch (...) {
        throw BESInternalError("Failed to parse the constraint expression: Unknown exception caught", __FILE__,
            __LINE__);
    }

    // The dataset_name is no longer used in the constraint evaluator, so no
    // need to get here. Plus, just getting the first container's dataset
    // name would not have worked with multiple containers.
    // pwest Jan 4, 2009
    // string dataset_name = "";

    // now we need to read the data
    BESDEBUG("fonc", "FONcTransmitter::send_data() - Reading data into DataDDS" << endl);

    // ADB: remember when we're using a temp DDS
    // bool using_temp_dds = false; See comment below about set_dds(). jhrg 8/8/14

    try {
        // Handle *functional* constraint expressions specially
        if (eval.function_clauses()) {
            BESDEBUG("fonc", "FONcTransmitter::send_data() - Processing functional constraint clause(s)." << endl);
            DDS *tmp_dds = eval.eval_function_clauses(*dds);
            delete dds;
            dds = tmp_dds;
            bdds->set_dds(dds);

            // This next step utilizes a well known function, promote_function_output_structures()
            // to look for one or more top level Structures whose name indicates (by way of ending
            // with "_uwrap") that their contents should be promoted (aka moved) to the top level.
            // This is in support of a hack around the current API where server side functions
            // may only return a single DAP object and not a collection of objects. The name suffix
            // "_unwrap" is used as a signal from the function to the the various response
            // builders and transmitters that the representation needs to be altered before
            // transmission, and that in fact is what happens in our friend
            // promote_function_output_structures()
            promote_function_output_structures(dds);

        }
        else {
            // Iterate through the variables in the DataDDS and read
            // in the data if the variable has the send flag set.

            for (DDS::Vars_iter i = dds->var_begin(); i != dds->var_end(); i++) {
                if ((*i)->send_p()) {
                    BESDEBUG("fonc", "FONcTransmitter::send_data() - Interning data for variable: '" << (*i)->name() << "'" << endl);
                    (*i)->intern_data(eval, *dds);
                }
            }
        }
    }
    catch (Error &e) {
        throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
    }
    catch (BESError &e) {
        throw;
    }
    catch (std::exception &e) {
        throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
    }
    catch (...) {
        throw BESInternalError("Failed to read data. Unknown Error", __FILE__, __LINE__);
    }

    //string temp_file_name = FONcTransmitter::temp_dir + '/' + "ncXXXXXX";
    string temp_file_name = FONcRequestHandler::temp_dir + '/' + "ncXXXXXX";
    vector<char> temp_full(temp_file_name.length() + 1);
    string::size_type len = temp_file_name.copy(&temp_full[0], temp_file_name.length());
    temp_full[len] = '\0';
    // cover the case where older versions of mkstemp() create the file using
    // a mode of 666.
    mode_t original_mode = umask(077);
    int fd = mkstemp(&temp_full[0]);
    umask(original_mode);
#if 0
    // Trick: We can unlink the file right here - it will persist until the file descriptor
    // is closed. This means we don't have to litter the code with call to unlink(). jhrg 11/25/15
    (void) unlink(&temp_full[0]);
    // Hack: Use this simple class to 'wrap' the file descriptor so that we can be sure it
    // is closed no matter how this code is exited. jhrg 11/25/15
    wrap_file_descriptor wrapped_fd(fd);
#endif
    if (fd == -1) throw BESInternalError("Failed to open the temporary file: " + temp_file_name, __FILE__, __LINE__);

    // transform the OPeNDAP DataDDS to the netcdf file
    BESDEBUG("fonc", "FONcTransmitter::send_data - Transforming into temporary file: " << &temp_full[0] << endl);

    try {
        FONcTransform ft(dds, dhi, &temp_full[0], ncVersion);
        ft.transform();

        BESDEBUG("fonc", "FONcTransmitter::send_data - Transmitting temp file " << &temp_full[0] << endl);
        FONcTransmitter::return_temp_stream(&temp_full[0], strm, ncVersion);
    }
    catch (Error &e) {
        (void) unlink(&temp_full[0]);
        close(fd);
        throw BESDapError("Failed to Transform data to NetCDF: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
    }
    catch (BESError &e) {
        (void) unlink(&temp_full[0]);
        close(fd);
        throw;
    }
    catch (std::exception &e) {
        (void) unlink(&temp_full[0]);
        close(fd);
        throw BESInternalError("Failed to Transform data to NetCDF: STL Error: " + string(e.what()), __FILE__, __LINE__);
    }
    catch (...) {
        (void) unlink(&temp_full[0]);
        close(fd);
        throw BESInternalError("Failed to Transform data to NetCDF. Unknown Error", __FILE__, __LINE__);
    }

    (void) unlink(&temp_full[0]);
    close(fd);

    BESDEBUG("fonc", "FONcTransmitter::send_data - done transmitting to netcdf" << endl);
}

/** @brief stream the temporary netcdf file back to the requester
 *
 * Streams the temporary netcdf file specified by filename to the specified
 * C++ ostream
 *
 * @param filename The name of the file to stream back to the requester
 * @param strm C++ ostream to write the contents of the file to
 * @throws BESInternalError if problem opening the file
 */
void FONcTransmitter::return_temp_stream(const string &filename, ostream &strm, const string &ncVersion)
{
    ifstream os;
    os.open(filename.c_str(), ios::binary | ios::in);
    if (!os)
        throw BESInternalError("Fileout netcdf: Cannot connect to netcdf file.", __FILE__, __LINE__);;

    char block[OUTPUT_FILE_BLOCK_SIZE];

    os.read(block, sizeof block);
    int nbytes = os.gcount();
    if (nbytes > 0) {
        bool found = false;
        string context = "transmit_protocol";
        string protocol = BESContextManager::TheManager()->get_context(context, found);
        if (protocol == "HTTP") {
            strm << "HTTP/1.0 200 OK\n";
            strm << "Content-type: application/octet-stream\n";
            strm << "Content-Description: " << "BES dataset" << "\n";
            if (ncVersion == RETURNAS_NETCDF4) {
                strm << "Content-Disposition: filename=" << filename << ".nc4;\n\n";
            }
            else {
                strm << "Content-Disposition: filename=" << filename << ".nc;\n\n";
            }
            strm << flush;
        }
        strm.write(block, nbytes);
    }
    else {
        // close the stream before we leave.
        os.close();
        throw BESInternalError("Fileout netcdf: Failed to stream the response back to the client, got zero count on stream buffer.", __FILE__, __LINE__);
    }

    while (os) {
        os.read(block, sizeof block);
        strm.write(block, os.gcount());
    }

    os.close();
}

<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: viewstrategy.hxx,v $
 *
 *  $Revision: 1.6 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-08 04:37:43 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef CONFIGMGR_VIEWBEHAVIOR_HXX_
#define CONFIGMGR_VIEWBEHAVIOR_HXX_

#ifndef CONFIGMGR_VIEWNODE_HXX_
#include "viewnode.hxx"
#endif

#ifndef CONFIGMGR_GROUPNODEBEHAVIOR_HXX_
#include "groupnodeimpl.hxx"
#endif
#ifndef CONFIGMGR_SETNODEBEHAVIOR_HXX_
#include "setnodeimpl.hxx"
#endif

#ifndef _SALHELPER_SIMPLEREFERENCEOBJECT_HXX_
#include <salhelper/simplereferenceobject.hxx>
#endif
#ifndef _RTL_REF_HXX_
#include <rtl/ref.hxx>
#endif

namespace configmgr
{
//-----------------------------------------------------------------------------
    namespace memory { class Segment; }
//-----------------------------------------------------------------------------
    namespace configuration
    {
        class SetElementChangeImpl;
        class ValueChangeImpl;
    }
//-----------------------------------------------------------------------------
    namespace view
    {
//-----------------------------------------------------------------------------
        struct NodeFactory;
//-----------------------------------------------------------------------------
        using configuration::Name;
        using configuration::NodeOffset;
        using configuration::TreeDepth;

        typedef com::sun::star::uno::Any    UnoAny;
        typedef com::sun::star::uno::Type   UnoType;
//-----------------------------------------------------------------------------
        class ViewStrategy : public salhelper::SimpleReferenceObject
        {
        // node attributes
        public:
            /// retrieve the attributes of the node
            Name getName(Node const& _aNode)    const;

            /// retrieve the attributes of the node
            node::Attributes getAttributes(Node const& _aNode)  const;

        // tracking pending changes
        public:
            typedef configuration::NodeChanges NodeChanges;

            void collectChanges(Tree const& _aTree, NodeChanges& rChanges) const;

            bool hasChanges(Tree const& _aTree) const;

            bool hasChanges(Node const& _aNode) const;

            void markChanged(Node const& _aNode);

        // commit protocol
        public:
            std::auto_ptr<SubtreeChange> preCommitChanges(Tree const& _aTree, configuration::ElementList& _rRemovedElements);

            void finishCommit(Tree const& _aTree, SubtreeChange& rRootChange);

            void revertCommit(Tree const& _aTree, SubtreeChange& rRootChange);

            void recoverFailedCommit(Tree const& _aTree, SubtreeChange& rRootChange);

        // notification protocol
        public:
            typedef configuration::NodeChangesInformation NodeChangesInformation;

            /// Adjust the internal representation after external changes to the original data - build NodeChangeInformation objects for notification
            void    adjustToChanges(NodeChangesInformation& rLocalChanges, Node const & _aNode, SubtreeChange const& aExternalChange);

        // visitor dispatch
        public:
            typedef configuration::GroupMemberVisitor GroupMemberVisitor;
            typedef configuration::SetNodeVisitor SetNodeVisitor;

            GroupMemberVisitor::Result dispatchToValues(GroupNode const& _aNode, GroupMemberVisitor& _aVisitor);

            /// Call aVisitor.visit(aElement) for each element in this set until SetNodeVisitor::DONE is returned.
            SetNodeVisitor::Result dispatchToElements(SetNode const& _aNode, SetNodeVisitor& _aVisitor);

        // value (element) node specific operations
        public:
            /// Does this node assume its default value
            /// retrieve the current value of this node
            UnoAny  getValue(ValueNode const& _aNode) const;

            /// get the type of this value
            UnoType getValueType(ValueNode const& _aNode)   const;

        // group node specific operations
        public:
            typedef configuration::ValueMemberNode   ValueMemberNode;
            typedef configuration::ValueMemberUpdate ValueMemberUpdate;

            /// does this hold a child value of the given name
            bool hasValue(GroupNode const& _aNode, Name const& _aName) const;

            /// does this hold a child value
            bool hasValue(GroupNode const& _aNode) const;

            /// are defaults for this node available ?
            bool areValueDefaultsAvailable(GroupNode const& _aNode) const;

            /// retrieve data for the child value of the given name
            ValueMemberNode getValue(GroupNode const& _aNode, Name const& _aName) const;

            /// retrieve data for updating the child value of the given name
            ValueMemberUpdate getValueForUpdate(GroupNode const & _aNode, Name const& _aName);

        // set node specific operations
        public:
            typedef configuration::ElementTreeData  SetNodeElement;
            typedef configuration::SetEntry         SetNodeEntry;

            /// does this set contain any elements (loads elements if needed)
            bool isEmpty(SetNode const& _aNode) const;

            /// does this set contain an element named <var>aName</var> (loads elements if needed)
            SetNodeEntry    findElement(SetNode const& _aNode, Name const& aName) const;

            /// does this set contain an element named <var>aName</var> (and is that element loaded ?)
            SetNodeEntry    findAvailableElement(SetNode const& _aNode, Name const& aName) const;

            /// insert a new entry into this set
            void        insertElement(SetNode const& _aNode, Name const& aName, SetNodeEntry const& aNewEntry);

            /// remove an existing entry into this set
            void        removeElement(SetNode const& _aNode, Name const& aName);

            /** Create a Subtree change as 'diff' which allows transforming the set to its default state
                (given that <var>_rDefaultTree</var> points to a default instance of this set)

Ownership of added trees should be transferred to the SubtreeChange.

            */
            std::auto_ptr<SubtreeChange> differenceToDefaultState(SetNode const& _aNode, ISubtree& _rDefaultTree) const;

            /// Get the template that describes elements of this set
            configuration::TemplateHolder getElementTemplate(SetNode const& _aNode) const;

            /// Get a template provider that can create new elements for this set
            configuration::TemplateProvider getTemplateProvider(SetNode const& _aNode) const;

            // create a view::Tree from a configuration::SetEntry
            Tree extractTree(SetNodeEntry const& _anEntry);

        // creating/changing state/strategy
        public:
            NodeFactory& getNodeFactory();

        // direct update access to data
        public:
            void releaseDataSegment();

            memory::Segment const * getDataSegment() const;
            memory::Segment  * getDataSegmentForUpdate();

            data::NodeAddress   ::DataType * getDataForUpdate(data::NodeAccessRef const & _aNode);
            data::SetNodeAddress::DataType * getDataForUpdate(data::SetNodeAccess const & _aNode);
            data::GroupNodeAddress::DataType * getDataForUpdate(data::GroupNodeAccess const & _aNode);
            data::ValueNodeAddress::DataType * getDataForUpdate(data::ValueNodeAccess const & _aNode);

        // access to node innards
        protected:
            /// provide access to the data of the underlying node
            data::NodeAccessRef getNodeAccessRef(Node const& _aNode) const;

            /// provide access to the address of the underlying node
            data::NodeAddress getNodeAddress(Node const& _aNode) const;

            /// retrieve the name of the underlying node
            Name getNodeName(Node const& _aNode) const;

            /// retrieve the attributes of the underlying node
            node::Attributes getNodeAttributes(Node const& _aNode) const;

        protected:
            //helper for migration to new (info based) model for adjusting to changes
            static void addLocalChangeHelper( NodeChangesInformation& rLocalChanges, configuration::NodeChange const& aChange);

        private:
            void implAdjustToValueChanges(NodeChangesInformation& rLocalChanges, GroupNode const& _aGroupNode, SubtreeChange const& rExternalChanges);
            void implAdjustToSubChanges(NodeChangesInformation& rLocalChanges, GroupNode const & _aGroupNode, SubtreeChange const& rExternalChanges);
            void implAdjustToElementChanges(NodeChangesInformation& rLocalChanges, SetNode const& _aNode, SubtreeChange const& rExternalChanges, TreeDepth nDepth);
            void implAdjustToElementChange (NodeChangesInformation& rLocalChanges, SetNode const& _aNode, Change const& rElementChange, TreeDepth nElementDepth);
            void implCommitDirectIn(data::TreeAccessor const& _aPlaceHolder, Node const& _aNode);

        protected:
            void            checkInstance(Tree const& _aTreeForThis) const;
            SetNodeEntry    implFindElement(SetNode const& _aNode, Name const& aName) const;
            SetNodeElement  implMakeElement(SetNode const& _aNode, SetNodeEntry const& anEntry) const;

        // virtual interface - these functions must be provided
        private:
            // change handling
            virtual bool doHasChanges(Node const& _aNode) const     = 0;
            virtual void doMarkChanged(Node const& _aNode)          = 0;

            virtual NodeFactory& doGetNodeFactory()  = 0;

        // virtual interface - these functions all have default implementations without support for pending changes
        protected:
            virtual void doReleaseDataSegment() = 0;

            // special support for direct changes to underlying data - default is no support
            virtual data::NodeAddress::DataType * implAccessForUpdate(data::NodeAccessRef const & _aDataAccess);
            virtual memory::Segment const * doGetDataSegment() const = 0;
            virtual memory::Segment       * doGetDataSegmentForUpdate();

            // change handling
            virtual void doCollectChanges(Node const& _aNode, NodeChanges& rChanges) const;

            // commit protocol
            virtual std::auto_ptr<SubtreeChange> doPreCommitChanges(Tree const& _aTree, configuration::ElementList& _rRemovedElements);
            virtual void doFailedCommit(Tree const& _aTree, SubtreeChange& rChanges);
            virtual void doFinishCommit(Tree const& _aTree, SubtreeChange& rChanges);
            virtual void doRevertCommit(Tree const& _aTree, SubtreeChange& rChanges);

            // notification protocol
            virtual configuration::ValueChangeImpl* doAdjustToValueChange(GroupNode const& _aGroupNode, Name const& aName, ValueChange const& rExternalChange);

            // common attributes
            virtual node::Attributes doAdjustAttributes(node::Attributes const& _aAttributes) const = 0;

            // group member access
            virtual ValueMemberNode doGetValueMember(GroupNode const& _aNode, Name const& _aName, bool _bForUpdate) const = 0;

            // set element access
            virtual void doInsertElement(SetNode const& _aNode, Name const& aName, SetNodeEntry const& aNewEntry) = 0;
            virtual void doRemoveElement(SetNode const& _aNode, Name const& aName) = 0;

            // strategy change support
/*          virtual void doCommitChanges(Node const& _aNode);
            virtual rtl::Reference<ViewStrategy> doCloneDirect() = 0;
            virtual rtl::Reference<ViewStrategy> doCloneIndirect() = 0;
*/      };

//-----------------------------------------------------------------------------
        inline Name ViewStrategy::getName(Node const& _aNode)   const
        { return getNodeName(_aNode); }

        inline node::Attributes ViewStrategy::getAttributes(Node const& _aNode) const
        { return doAdjustAttributes(getNodeAttributes(_aNode)); }

        inline bool ViewStrategy::hasChanges(Node const& _aNode) const
        { return doHasChanges(_aNode); }

        inline NodeFactory& ViewStrategy::getNodeFactory()
        { return doGetNodeFactory(); }

        inline void ViewStrategy::releaseDataSegment()
        { doReleaseDataSegment(); }

        inline memory::Segment const * ViewStrategy::getDataSegment()   const
        { return doGetDataSegment(); }

        inline memory::Segment       * ViewStrategy::getDataSegmentForUpdate()
        { return doGetDataSegmentForUpdate(); }

//-----------------------------------------------------------------------------
    }
//-----------------------------------------------------------------------------
}

#endif // CONFIGMGR_CONFIGNODEBEHAVIOR_HXX_
<commit_msg>INTEGRATION: CWS configrefactor01 (1.6.84); FILE MERGED 2007/02/07 12:14:55 mmeeks 1.6.84.5: remove obsolete memory::Segment forward decls. 2007/01/16 12:18:28 mmeeks 1.6.84.4: Submitted by: mmeeks Kill 'memory::Segment' - no longer needed. Bin some other (empty / redundant) headers. 2007/01/11 20:16:06 mmeeks 1.6.84.3: Submitted by: mmeeks More re-factoring, lots of locking rationalized, drastically reduced the mutex count, also removed ~300k interlocked increments with a non-interlocking SimpleReferencedObject base 2007/01/11 10:35:41 mmeeks 1.6.84.2: Submitted by: mmeeks<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: viewstrategy.hxx,v $
 *
 *  $Revision: 1.7 $
 *
 *  last change: $Author: ihi $ $Date: 2007-11-23 14:50:21 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef CONFIGMGR_VIEWBEHAVIOR_HXX_
#define CONFIGMGR_VIEWBEHAVIOR_HXX_

#ifndef CONFIGMGR_VIEWNODE_HXX_
#include "viewnode.hxx"
#endif

#ifndef CONFIGMGR_GROUPNODEBEHAVIOR_HXX_
#include "groupnodeimpl.hxx"
#endif
#ifndef CONFIGMGR_SETNODEBEHAVIOR_HXX_
#include "setnodeimpl.hxx"
#endif
#ifndef CONFIGMGR_UTILITY_HXX_
#include "utility.hxx"
#endif

#ifndef _RTL_REF_HXX_
#include <rtl/ref.hxx>
#endif

namespace configmgr
{
//-----------------------------------------------------------------------------
    namespace configuration
    {
        class SetElementChangeImpl;
        class ValueChangeImpl;
    }
//-----------------------------------------------------------------------------
    namespace view
    {
//-----------------------------------------------------------------------------
        struct NodeFactory;
//-----------------------------------------------------------------------------
        using configuration::Name;
        using configuration::NodeOffset;
        using configuration::TreeDepth;

        typedef com::sun::star::uno::Any    UnoAny;
        typedef com::sun::star::uno::Type   UnoType;
//-----------------------------------------------------------------------------
        class ViewStrategy : public configmgr::SimpleReferenceObject
        {
        // node attributes
        public:
            /// retrieve the attributes of the node
            Name getName(Node const& _aNode)    const;

            /// retrieve the attributes of the node
            node::Attributes getAttributes(Node const& _aNode)  const;

        // tracking pending changes
        public:
            typedef configuration::NodeChanges NodeChanges;

            void collectChanges(Tree const& _aTree, NodeChanges& rChanges) const;

            bool hasChanges(Tree const& _aTree) const;

            bool hasChanges(Node const& _aNode) const;

            void markChanged(Node const& _aNode);

        // commit protocol
        public:
            std::auto_ptr<SubtreeChange> preCommitChanges(Tree const& _aTree, configuration::ElementList& _rRemovedElements);

            void finishCommit(Tree const& _aTree, SubtreeChange& rRootChange);

            void revertCommit(Tree const& _aTree, SubtreeChange& rRootChange);

            void recoverFailedCommit(Tree const& _aTree, SubtreeChange& rRootChange);

        // notification protocol
        public:
            typedef configuration::NodeChangesInformation NodeChangesInformation;

            /// Adjust the internal representation after external changes to the original data - build NodeChangeInformation objects for notification
            void    adjustToChanges(NodeChangesInformation& rLocalChanges, Node const & _aNode, SubtreeChange const& aExternalChange);

        // visitor dispatch
        public:
            typedef configuration::GroupMemberVisitor GroupMemberVisitor;
            typedef configuration::SetNodeVisitor SetNodeVisitor;

            GroupMemberVisitor::Result dispatchToValues(GroupNode const& _aNode, GroupMemberVisitor& _aVisitor);

            /// Call aVisitor.visit(aElement) for each element in this set until SetNodeVisitor::DONE is returned.
            SetNodeVisitor::Result dispatchToElements(SetNode const& _aNode, SetNodeVisitor& _aVisitor);

        // value (element) node specific operations
        public:
            /// Does this node assume its default value
            /// retrieve the current value of this node
            UnoAny  getValue(ValueNode const& _aNode) const;

            /// get the type of this value
            UnoType getValueType(ValueNode const& _aNode)   const;

        // group node specific operations
        public:
            typedef configuration::ValueMemberNode   ValueMemberNode;
            typedef configuration::ValueMemberUpdate ValueMemberUpdate;

            /// does this hold a child value of the given name
            bool hasValue(GroupNode const& _aNode, Name const& _aName) const;

            /// does this hold a child value
            bool hasValue(GroupNode const& _aNode) const;

            /// are defaults for this node available ?
            bool areValueDefaultsAvailable(GroupNode const& _aNode) const;

            /// retrieve data for the child value of the given name
            ValueMemberNode getValue(GroupNode const& _aNode, Name const& _aName) const;

            /// retrieve data for updating the child value of the given name
            ValueMemberUpdate getValueForUpdate(GroupNode const & _aNode, Name const& _aName);

        // set node specific operations
        public:
            typedef configuration::ElementTreeData  SetNodeElement;
            typedef configuration::SetEntry         SetNodeEntry;

            /// does this set contain any elements (loads elements if needed)
            bool isEmpty(SetNode const& _aNode) const;

            /// does this set contain an element named <var>aName</var> (loads elements if needed)
            SetNodeEntry    findElement(SetNode const& _aNode, Name const& aName) const;

            /// does this set contain an element named <var>aName</var> (and is that element loaded ?)
            SetNodeEntry    findAvailableElement(SetNode const& _aNode, Name const& aName) const;

            /// insert a new entry into this set
            void        insertElement(SetNode const& _aNode, Name const& aName, SetNodeEntry const& aNewEntry);

            /// remove an existing entry into this set
            void        removeElement(SetNode const& _aNode, Name const& aName);

            /** Create a Subtree change as 'diff' which allows transforming the set to its default state
                (given that <var>_rDefaultTree</var> points to a default instance of this set)

Ownership of added trees should be transferred to the SubtreeChange.

            */
            std::auto_ptr<SubtreeChange> differenceToDefaultState(SetNode const& _aNode, ISubtree& _rDefaultTree) const;

            /// Get the template that describes elements of this set
            configuration::TemplateHolder getElementTemplate(SetNode const& _aNode) const;

            /// Get a template provider that can create new elements for this set
            configuration::TemplateProvider getTemplateProvider(SetNode const& _aNode) const;

            // create a view::Tree from a configuration::SetEntry
            Tree extractTree(SetNodeEntry const& _anEntry);

        // creating/changing state/strategy
        public:
            NodeFactory& getNodeFactory();

        // access to node innards
        protected:
            /// provide access to the data of the underlying node
            data::NodeAccess getNodeAccess(Node const& _aNode) const;

            /// provide access to the address of the underlying node
            data::NodeAddress getNodeAddress(Node const& _aNode) const;

            /// retrieve the name of the underlying node
            Name getNodeName(Node const& _aNode) const;

            /// retrieve the attributes of the underlying node
            node::Attributes getNodeAttributes(Node const& _aNode) const;

        protected:
            //helper for migration to new (info based) model for adjusting to changes
            static void addLocalChangeHelper( NodeChangesInformation& rLocalChanges, configuration::NodeChange const& aChange);

        private:
            void implAdjustToValueChanges(NodeChangesInformation& rLocalChanges, GroupNode const& _aGroupNode, SubtreeChange const& rExternalChanges);
            void implAdjustToSubChanges(NodeChangesInformation& rLocalChanges, GroupNode const & _aGroupNode, SubtreeChange const& rExternalChanges);
            void implAdjustToElementChanges(NodeChangesInformation& rLocalChanges, SetNode const& _aNode, SubtreeChange const& rExternalChanges, TreeDepth nDepth);
            void implAdjustToElementChange (NodeChangesInformation& rLocalChanges, SetNode const& _aNode, Change const& rElementChange, TreeDepth nElementDepth);
            void implCommitDirectIn(data::TreeAccessor const& _aPlaceHolder, Node const& _aNode);

        protected:
            void            checkInstance(Tree const& _aTreeForThis) const;
            SetNodeEntry    implFindElement(SetNode const& _aNode, Name const& aName) const;
            SetNodeElement  implMakeElement(SetNode const& _aNode, SetNodeEntry const& anEntry) const;

        // virtual interface - these functions must be provided
        private:
            // change handling
            virtual bool doHasChanges(Node const& _aNode) const     = 0;
            virtual void doMarkChanged(Node const& _aNode)          = 0;

            virtual NodeFactory& doGetNodeFactory()  = 0;

        // virtual interface - these functions all have default implementations without support for pending changes
        protected:
            // change handling
            virtual void doCollectChanges(Node const& _aNode, NodeChanges& rChanges) const;

            // commit protocol
            virtual std::auto_ptr<SubtreeChange> doPreCommitChanges(Tree const& _aTree, configuration::ElementList& _rRemovedElements);
            virtual void doFailedCommit(Tree const& _aTree, SubtreeChange& rChanges);
            virtual void doFinishCommit(Tree const& _aTree, SubtreeChange& rChanges);
            virtual void doRevertCommit(Tree const& _aTree, SubtreeChange& rChanges);

            // notification protocol
            virtual configuration::ValueChangeImpl* doAdjustToValueChange(GroupNode const& _aGroupNode, Name const& aName, ValueChange const& rExternalChange);

            // common attributes
            virtual node::Attributes doAdjustAttributes(node::Attributes const& _aAttributes) const = 0;

            // group member access
            virtual ValueMemberNode doGetValueMember(GroupNode const& _aNode, Name const& _aName, bool _bForUpdate) const = 0;

            // set element access
            virtual void doInsertElement(SetNode const& _aNode, Name const& aName, SetNodeEntry const& aNewEntry) = 0;
            virtual void doRemoveElement(SetNode const& _aNode, Name const& aName) = 0;
        };

//-----------------------------------------------------------------------------
        inline Name ViewStrategy::getName(Node const& _aNode)   const
        { return getNodeName(_aNode); }

        inline node::Attributes ViewStrategy::getAttributes(Node const& _aNode) const
        { return doAdjustAttributes(getNodeAttributes(_aNode)); }

        inline bool ViewStrategy::hasChanges(Node const& _aNode) const
        { return doHasChanges(_aNode); }

        inline NodeFactory& ViewStrategy::getNodeFactory()
        { return doGetNodeFactory(); }

//-----------------------------------------------------------------------------
    }
//-----------------------------------------------------------------------------
}

#endif // CONFIGMGR_CONFIGNODEBEHAVIOR_HXX_
<|endoftext|>


<commit_before>#include "../common/certs.hpp"
#include "../common/mock_connector.hpp"

#include "root_path.hpp"

#include <pxp-agent/external_module.hpp>
#include <pxp-agent/request_processor.hpp>
#include <pxp-agent/configuration.hpp>

#include <leatherman/json_container/json_container.hpp>

#include <cpp-pcp-client/util/thread.hpp>   // this_thread::sleep_for
#include <cpp-pcp-client/util/chrono.hpp>

#include <boost/filesystem.hpp>
#include <boost/filesystem/operations.hpp>

#include <catch.hpp>

namespace PXPAgent {

#ifdef TEST_VIRTUAL

namespace lth_jc = leatherman::json_container;
namespace pcp_util = PCPClient::Util;
namespace fs = boost::filesystem;

static const std::string REVERSE_VALID_MODULE_NAME { "reverse_valid" };

TEST_CASE("Valid External Module Configuration", "[component]") {
    AGENT_CONFIGURATION.modules_config_dir = VALID_MODULES_CONFIG;
    auto c_ptr = std::make_shared<MockConnector>();
    RequestProcessor r_p { c_ptr, AGENT_CONFIGURATION };

    SECTION("retrieves the configuration from file if valid JSON format") {
        REQUIRE(r_p.hasModuleConfig(REVERSE_VALID_MODULE_NAME));
    }

    SECTION("does load the module when the configuration is valid") {
        REQUIRE(r_p.hasModuleConfig(REVERSE_VALID_MODULE_NAME));
        REQUIRE(r_p.hasModule(REVERSE_VALID_MODULE_NAME));
    }
}

TEST_CASE("Badly formatted External Module Configuration", "[component]") {
    AGENT_CONFIGURATION.modules_config_dir = BAD_FORMAT_MODULES_CONFIG;
    auto c_ptr = std::make_shared<MockConnector>();
    RequestProcessor r_p { c_ptr, AGENT_CONFIGURATION };

    SECTION("stores a null JSON value as the configuration, if it's in an "
            "invalid JSON format") {
        REQUIRE(r_p.getModuleConfig(REVERSE_VALID_MODULE_NAME) == "null");
    }

    SECTION("does not load the module when the configuration is in "
            "bad JSON format") {
        REQUIRE(r_p.getModuleConfig(REVERSE_VALID_MODULE_NAME) == "null");
        REQUIRE_FALSE(r_p.hasModule(REVERSE_VALID_MODULE_NAME));
    }
}

TEST_CASE("Invalid (by metadata) External Module Configuration", "[component]") {
    SECTION("does not load the module when the configuration is invalid") {
        AGENT_CONFIGURATION.modules_config_dir = BROKEN_MODULES_CONFIG;
        auto c_ptr = std::make_shared<MockConnector>();
        RequestProcessor r_p { c_ptr, AGENT_CONFIGURATION };
        REQUIRE(r_p.hasModuleConfig(REVERSE_VALID_MODULE_NAME));
        REQUIRE_FALSE(r_p.hasModule(REVERSE_VALID_MODULE_NAME));
    }
}

TEST_CASE("Process correctly requests for external modules", "[component]") {
    AGENT_CONFIGURATION.modules_config_dir = "";
    auto c_ptr = std::make_shared<MockConnector>();
    RequestProcessor r_p { c_ptr, AGENT_CONFIGURATION };
    lth_jc::JsonContainer envelope { VALID_ENVELOPE_TXT };
    std::vector<lth_jc::JsonContainer> debug {};

    lth_jc::JsonContainer data {};
    data.set<std::string>("transaction_id", "42");

    SECTION("correctly process nonblocking requests") {
        SECTION("send a blocking response when the requested action succeeds") {
            REQUIRE(!c_ptr->sent_blocking_response);
            data.set<std::string>("module", "reverse_valid");
            data.set<std::string>("action", "string");
            lth_jc::JsonContainer params {};
            params.set<std::string>("argument", "was");
            data.set<lth_jc::JsonContainer>("params", params);
            const PCPClient::ParsedChunks p_c { envelope, data, debug, 0 };

            REQUIRE_NOTHROW(r_p.processRequest(RequestType::Blocking, p_c));

            // Wait a bit to let the execution thread finish
            pcp_util::this_thread::sleep_for(
                pcp_util::chrono::microseconds(100000));

            REQUIRE(c_ptr->sent_blocking_response);
        }

        SECTION("send a PXP error in case of action failure") {
            data.set<std::string>("module", "failures_test");
            data.set<std::string>("action", "broken_action");
            lth_jc::JsonContainer params {};
            params.set<std::string>("argument", "bikini");
            data.set<lth_jc::JsonContainer>("params", params);
            const PCPClient::ParsedChunks p_c { envelope, data, debug, 0 };

            REQUIRE_THROWS_AS(r_p.processRequest(RequestType::Blocking, p_c),
                              MockConnector::pxpError_msg);
        }
    }

    SECTION("correctly process non-blocking requests") {
        SECTION("send a provisional response when the requested starts successfully") {
            REQUIRE(!c_ptr->sent_provisional_response);

            data.set<std::string>("module", "reverse_valid");
            data.set<bool>("notify_outcome", false);
            data.set<std::string>("action", "string");
            lth_jc::JsonContainer params {};
            params.set<std::string>("argument", "lemon");
            data.set<lth_jc::JsonContainer>("params", params);
            const PCPClient::ParsedChunks p_c { envelope, data, debug, 0 };

            REQUIRE_NOTHROW(r_p.processRequest(RequestType::NonBlocking, p_c));

            // Wait a bit to let the execution thread finish
            pcp_util::this_thread::sleep_for(
                pcp_util::chrono::microseconds(100000));

            REQUIRE(c_ptr->sent_provisional_response);
        }

        SECTION("send a blocking response when the requested action succeeds") {
            REQUIRE(!c_ptr->sent_provisional_response);
            REQUIRE(!c_ptr->sent_non_blocking_response);

            data.set<std::string>("module", "reverse_valid");
            data.set<bool>("notify_outcome", true);
            data.set<std::string>("action", "string");
            lth_jc::JsonContainer params {};
            params.set<std::string>("argument", "kondgbia");
            data.set<lth_jc::JsonContainer>("params", params);
            const PCPClient::ParsedChunks p_c { envelope, data, debug, 0 };

            REQUIRE_NOTHROW(r_p.processRequest(RequestType::NonBlocking, p_c));

            // Wait a bit to let the execution thread finish
            pcp_util::this_thread::sleep_for(
                pcp_util::chrono::microseconds(100000));

            REQUIRE(c_ptr->sent_provisional_response);
            REQUIRE(c_ptr->sent_non_blocking_response);
        }
    }

    fs::remove_all(SPOOL);
}

#endif  // TEST_VIRTUAL

}  // namespace PXPAgent
<commit_msg>(maint) Correct test title<commit_after>#include "../common/certs.hpp"
#include "../common/mock_connector.hpp"

#include "root_path.hpp"

#include <pxp-agent/external_module.hpp>
#include <pxp-agent/request_processor.hpp>
#include <pxp-agent/configuration.hpp>

#include <leatherman/json_container/json_container.hpp>

#include <cpp-pcp-client/util/thread.hpp>   // this_thread::sleep_for
#include <cpp-pcp-client/util/chrono.hpp>

#include <boost/filesystem.hpp>
#include <boost/filesystem/operations.hpp>

#include <catch.hpp>

namespace PXPAgent {

#ifdef TEST_VIRTUAL

namespace lth_jc = leatherman::json_container;
namespace pcp_util = PCPClient::Util;
namespace fs = boost::filesystem;

static const std::string REVERSE_VALID_MODULE_NAME { "reverse_valid" };

TEST_CASE("Valid External Module Configuration", "[component]") {
    AGENT_CONFIGURATION.modules_config_dir = VALID_MODULES_CONFIG;
    auto c_ptr = std::make_shared<MockConnector>();
    RequestProcessor r_p { c_ptr, AGENT_CONFIGURATION };

    SECTION("retrieves the configuration from file if valid JSON format") {
        REQUIRE(r_p.hasModuleConfig(REVERSE_VALID_MODULE_NAME));
    }

    SECTION("does load the module when the configuration is valid") {
        REQUIRE(r_p.hasModuleConfig(REVERSE_VALID_MODULE_NAME));
        REQUIRE(r_p.hasModule(REVERSE_VALID_MODULE_NAME));
    }
}

TEST_CASE("Badly formatted External Module Configuration", "[component]") {
    AGENT_CONFIGURATION.modules_config_dir = BAD_FORMAT_MODULES_CONFIG;
    auto c_ptr = std::make_shared<MockConnector>();
    RequestProcessor r_p { c_ptr, AGENT_CONFIGURATION };

    SECTION("stores a null JSON value as the configuration, if it's in an "
            "invalid JSON format") {
        REQUIRE(r_p.getModuleConfig(REVERSE_VALID_MODULE_NAME) == "null");
    }

    SECTION("does not load the module when the configuration is in "
            "bad JSON format") {
        REQUIRE(r_p.getModuleConfig(REVERSE_VALID_MODULE_NAME) == "null");
        REQUIRE_FALSE(r_p.hasModule(REVERSE_VALID_MODULE_NAME));
    }
}

TEST_CASE("Invalid (by metadata) External Module Configuration", "[component]") {
    SECTION("does not load the module when the configuration is invalid") {
        AGENT_CONFIGURATION.modules_config_dir = BROKEN_MODULES_CONFIG;
        auto c_ptr = std::make_shared<MockConnector>();
        RequestProcessor r_p { c_ptr, AGENT_CONFIGURATION };
        REQUIRE(r_p.hasModuleConfig(REVERSE_VALID_MODULE_NAME));
        REQUIRE_FALSE(r_p.hasModule(REVERSE_VALID_MODULE_NAME));
    }
}

TEST_CASE("Process correctly requests for external modules", "[component]") {
    AGENT_CONFIGURATION.modules_config_dir = "";
    auto c_ptr = std::make_shared<MockConnector>();
    RequestProcessor r_p { c_ptr, AGENT_CONFIGURATION };
    lth_jc::JsonContainer envelope { VALID_ENVELOPE_TXT };
    std::vector<lth_jc::JsonContainer> debug {};

    lth_jc::JsonContainer data {};
    data.set<std::string>("transaction_id", "42");

    SECTION("correctly process nonblocking requests") {
        SECTION("send a blocking response when the requested action succeeds") {
            REQUIRE(!c_ptr->sent_blocking_response);
            data.set<std::string>("module", "reverse_valid");
            data.set<std::string>("action", "string");
            lth_jc::JsonContainer params {};
            params.set<std::string>("argument", "was");
            data.set<lth_jc::JsonContainer>("params", params);
            const PCPClient::ParsedChunks p_c { envelope, data, debug, 0 };

            REQUIRE_NOTHROW(r_p.processRequest(RequestType::Blocking, p_c));

            // Wait a bit to let the execution thread finish
            pcp_util::this_thread::sleep_for(
                pcp_util::chrono::microseconds(100000));

            REQUIRE(c_ptr->sent_blocking_response);
        }

        SECTION("send a PXP error in case of action failure") {
            data.set<std::string>("module", "failures_test");
            data.set<std::string>("action", "broken_action");
            lth_jc::JsonContainer params {};
            params.set<std::string>("argument", "bikini");
            data.set<lth_jc::JsonContainer>("params", params);
            const PCPClient::ParsedChunks p_c { envelope, data, debug, 0 };

            REQUIRE_THROWS_AS(r_p.processRequest(RequestType::Blocking, p_c),
                              MockConnector::pxpError_msg);
        }
    }

    SECTION("correctly process non-blocking requests") {
        SECTION("send a provisional response when the requested action starts "
                "successfully") {
            REQUIRE(!c_ptr->sent_provisional_response);

            data.set<std::string>("module", "reverse_valid");
            data.set<bool>("notify_outcome", false);
            data.set<std::string>("action", "string");
            lth_jc::JsonContainer params {};
            params.set<std::string>("argument", "lemon");
            data.set<lth_jc::JsonContainer>("params", params);
            const PCPClient::ParsedChunks p_c { envelope, data, debug, 0 };

            REQUIRE_NOTHROW(r_p.processRequest(RequestType::NonBlocking, p_c));

            // Wait a bit to let the execution thread finish
            pcp_util::this_thread::sleep_for(
                pcp_util::chrono::microseconds(100000));

            REQUIRE(c_ptr->sent_provisional_response);
        }

        SECTION("send a blocking response when the requested action succeeds") {
            REQUIRE(!c_ptr->sent_provisional_response);
            REQUIRE(!c_ptr->sent_non_blocking_response);

            data.set<std::string>("module", "reverse_valid");
            data.set<bool>("notify_outcome", true);
            data.set<std::string>("action", "string");
            lth_jc::JsonContainer params {};
            params.set<std::string>("argument", "kondgbia");
            data.set<lth_jc::JsonContainer>("params", params);
            const PCPClient::ParsedChunks p_c { envelope, data, debug, 0 };

            REQUIRE_NOTHROW(r_p.processRequest(RequestType::NonBlocking, p_c));

            // Wait a bit to let the execution thread finish
            pcp_util::this_thread::sleep_for(
                pcp_util::chrono::microseconds(100000));

            REQUIRE(c_ptr->sent_provisional_response);
            REQUIRE(c_ptr->sent_non_blocking_response);
        }
    }

    fs::remove_all(SPOOL);
}

#endif  // TEST_VIRTUAL

}  // namespace PXPAgent
<|endoftext|>


<commit_before>#include "drake/util/SystemIdentification.h"

#include <algorithm>

namespace drake {
namespace util {

template<typename T>
std::set<typename SystemIdentification<T>::MonomialType>
SystemIdentification<T>::GetAllCombinationsOfVars(
    const std::vector<PolyType>& polys,
    const std::set<VarType>& vars_of_interest) {
  std::set<MonomialType> interest_monomials;
  for (const PolyType& poly : polys) {
    for (const MonomialType& monomial : poly.getMonomials()) {
      typename PolyType::Monomial interest_monomial;
      interest_monomial.coefficient = 1;
      for (const TermType& term : monomial.terms) {
        if (vars_of_interest.count(term.var)) {
          interest_monomial.terms.push_back(term);
        }
      }
      interest_monomials.insert(interest_monomial);
    }
  }
  return interest_monomials;
}

template<typename T>
bool SystemIdentification<T>::MonomialMatches(
    const MonomialType& haystack,
    const MonomialType& needle,
    const std::set<VarType>& vars_of_interest) {
  const MonomialType residue = haystack.factor(needle);
  if (residue.coefficient == 0) {
    return false;
  }
  for (const VarType& var : vars_of_interest) {
    if (residue.getDegreeOf(var) > 0) {
      return false;
    }
  }
  return true;
}

template<typename T>
std::pair<T, typename SystemIdentification<T>::PolyType>
SystemIdentification<T>::NormalizePolynomial(const PolyType& poly) {
  std::vector<MonomialType> monomials = poly.getMonomials();
  const T min_coefficient = std::min_element(
      monomials.begin(), monomials.end(),
      [&](const MonomialType& l, const MonomialType& r){
        return l.coefficient < r.coefficient; })->coefficient;
  for (MonomialType& monomial : monomials) {
    monomial.coefficient /= min_coefficient;
  }
  return std::make_pair(min_coefficient, PolyType(monomials.begin(),
                                                  monomials.end()));
}

template<typename T>
typename SystemIdentification<T>::LumpingMapType
SystemIdentification<T>::GetLumpedParametersFromPolynomial(
    const PolyType& poly,
    const std::set<VarType>& vars_of_interest) {
  // Just dispatch to the set version.
  const std::vector<Polynomial<T>> polys = {poly};
  return SystemIdentification<T>::GetLumpedParametersFromPolynomials(
      polys, vars_of_interest);
}

template<typename T>
typename SystemIdentification<T>::LumpingMapType
SystemIdentification<T>::GetLumpedParametersFromPolynomials(
    const std::vector<PolyType>& polys,
    const std::set<VarType>& vars_of_interest) {
  // Before we begin, check that we can reserve some names (VarType values)
  // for our lumped parameters.
  std::set<VarType> all_vars;
  for (const PolyType& poly : polys) {
    const auto& poly_vars = poly.getVariables();
    all_vars.insert(poly_vars.begin(), poly_vars.end());
  }
  const VarType reservation_start = Polynomiald("lump", 1).getSimpleVariable();
  const VarType reservation_end = Polynomiald("lump", 1000).getSimpleVariable();
  for (const VarType& var : all_vars) {
    if ((var >= reservation_start) && (var <= reservation_end)) {
      throw std::runtime_error(
          "Lumped parameters failed because variable name already in use");
    }
  }

  // First, extract every combination of the vars_of_interest.
  const std::set<typename PolyType::Monomial> interest_monomials =
      GetAllCombinationsOfVars(polys, vars_of_interest);

  // Second, for each of those combinations, find the corresponding
  // polynomials of parameter (ie, non of-interest) variables in each
  // polynomial.
  std::set<PolyType> lumped_parameters;
  for (const MonomialType& interest_monomial : interest_monomials) {
    for (const PolyType& poly : polys) {
      std::vector<MonomialType> lumped_parameter;
      for (const MonomialType& monomial : poly.getMonomials()) {
        if (MonomialMatches(monomial, interest_monomial, vars_of_interest)) {
          lumped_parameter.push_back(monomial.factor(interest_monomial));
        }
      }
      if (!lumped_parameter.size()) { continue; }
      // Factor out any coefficients, so that 'a' and '2*a' are not both
      // considered lumped parameters.
      PolyType lumped_parameter_polynomial(lumped_parameter.begin(),
                                           lumped_parameter.end());
      PolyType normalized =
          NormalizePolynomial(lumped_parameter_polynomial).second;
      lumped_parameters.insert(normalized);
    }
  }

  // Third, for the set of such parameter polynomials, create a lumped
  // parameter for each.
  int lump_index = 1;
  typename SystemIdentification<T>::LumpingMapType lumping_map;

  for (const PolyType& lump : lumped_parameters) {
    VarType lump_var = PolyType("lump", lump_index).getSimpleVariable();
    lumping_map[lump] = lump_var;
    lump_index++;
  }

  return lumping_map;
}

template<typename T>
typename SystemIdentification<T>::PolyType
SystemIdentification<T>::RewritePolynomialWithLumpedParameters(
    const PolyType& poly,
    const LumpingMapType& lumped_parameters) {
  std::set<VarType> vars_of_interest = poly.getVariables();
  for (auto lump_name_pair : lumped_parameters) {
    std::set<VarType> parameters_in_lump = lump_name_pair.first.getVariables();
    for (const VarType& var : parameters_in_lump) {
      vars_of_interest.erase(var);
    }
  }
  std::set<typename PolyType::Monomial> interest_monomials =
      GetAllCombinationsOfVars({poly}, vars_of_interest);
  std::vector<MonomialType> working_monomials = poly.getMonomials();
  for (const MonomialType& interest_monomial : interest_monomials) {
    std::vector<MonomialType> new_working_monomials;
    std::vector<int> indices_to_erase;
    std::vector<MonomialType> factor_monomials;
    for (const MonomialType& working_monomial : working_monomials) {
      if (MonomialMatches(working_monomial, interest_monomial,
                          vars_of_interest)) {
        factor_monomials.push_back(working_monomial.factor(interest_monomial));
      } else {
        new_working_monomials.push_back(working_monomial);
      }
    }
    const PolyType factor_polynomial(factor_monomials.begin(),
                               factor_monomials.end());
    const auto& normalization = NormalizePolynomial(factor_polynomial);
    const T factor = normalization.first;
    const PolyType& normalized = normalization.second;
    if (!lumped_parameters.count(normalized)) {
      // No lumping possible for this interest monomial.
      continue;
    }
    TermType lump_term;
    lump_term.var = lumped_parameters.at(normalized);
    lump_term.power = 1;
    MonomialType lumped_monomial;
    lumped_monomial.terms = interest_monomial.terms;
    lumped_monomial.terms.push_back(lump_term);
    lumped_monomial.coefficient = factor;
    new_working_monomials.push_back(lumped_monomial);
    working_monomials = new_working_monomials;
  }

  return PolyType(working_monomials.begin(), working_monomials.end());
}

};
};

template class DRAKEPOLYNOMIAL_EXPORT drake::util::SystemIdentification<double>;
<commit_msg>checklisting:  Implementation comments, variable name uniqueness, dead code removal<commit_after>#include "drake/util/SystemIdentification.h"

#include <algorithm>

namespace drake {
namespace util {

template<typename T>
std::set<typename SystemIdentification<T>::MonomialType>
SystemIdentification<T>::GetAllCombinationsOfVars(
    const std::vector<PolyType>& polys,
    const std::set<VarType>& vars_of_interest) {
  std::set<MonomialType> interest_monomials;
  for (const PolyType& poly : polys) {
    for (const MonomialType& monomial : poly.getMonomials()) {
      typename PolyType::Monomial interest_monomial;
      interest_monomial.coefficient = 1;
      for (const TermType& term : monomial.terms) {
        if (vars_of_interest.count(term.var)) {
          interest_monomial.terms.push_back(term);
        }
      }
      interest_monomials.insert(interest_monomial);
    }
  }
  return interest_monomials;
}

template<typename T>
bool SystemIdentification<T>::MonomialMatches(
    const MonomialType& haystack,
    const MonomialType& needle,
    const std::set<VarType>& vars_of_interest) {
  const MonomialType residue = haystack.factor(needle);
  if (residue.coefficient == 0) {
    return false;
  }
  for (const VarType& var : vars_of_interest) {
    if (residue.getDegreeOf(var) > 0) {
      return false;
    }
  }
  return true;
}

template<typename T>
std::pair<T, typename SystemIdentification<T>::PolyType>
SystemIdentification<T>::NormalizePolynomial(const PolyType& poly) {
  std::vector<MonomialType> monomials = poly.getMonomials();
  const T min_coefficient = std::min_element(
      monomials.begin(), monomials.end(),
      [&](const MonomialType& l, const MonomialType& r){
        return l.coefficient < r.coefficient; })->coefficient;
  for (MonomialType& monomial : monomials) {
    monomial.coefficient /= min_coefficient;
  }
  return std::make_pair(min_coefficient, PolyType(monomials.begin(),
                                                  monomials.end()));
}

template<typename T>
typename SystemIdentification<T>::LumpingMapType
SystemIdentification<T>::GetLumpedParametersFromPolynomial(
    const PolyType& poly,
    const std::set<VarType>& vars_of_interest) {
  // Just dispatch to the set version.
  const std::vector<Polynomial<T>> polys = {poly};
  return SystemIdentification<T>::GetLumpedParametersFromPolynomials(
      polys, vars_of_interest);
}

template<typename T>
typename SystemIdentification<T>::LumpingMapType
SystemIdentification<T>::GetLumpedParametersFromPolynomials(
    const std::vector<PolyType>& polys,
    const std::set<VarType>& vars_of_interest) {
  // Before we begin, check that we can reserve some names (VarType values)
  // for our lumped parameters.
  std::set<VarType> all_vars;
  for (const PolyType& poly : polys) {
    const auto& poly_vars = poly.getVariables();
    all_vars.insert(poly_vars.begin(), poly_vars.end());
  }
  const VarType reservation_start = Polynomiald("lump", 1).getSimpleVariable();
  const VarType reservation_end = Polynomiald("lump", 1000).getSimpleVariable();
  for (const VarType& var : all_vars) {
    if ((var >= reservation_start) && (var <= reservation_end)) {
      throw std::runtime_error(
          "Lumped parameters failed because variable name already in use");
    }
  }

  // Extract every combination of the vars_of_interest.
  const std::set<typename PolyType::Monomial> interest_monomials =
      GetAllCombinationsOfVars(polys, vars_of_interest);

  // For each of those combinations, find the corresponding polynomials of
  // parameter (ie, non of-interest) variables in each polynomial.
  std::set<PolyType> lumped_parameters;
  for (const MonomialType& interest_monomial : interest_monomials) {
    for (const PolyType& poly : polys) {
      std::vector<MonomialType> lumped_parameter;
      for (const MonomialType& monomial : poly.getMonomials()) {
        if (MonomialMatches(monomial, interest_monomial, vars_of_interest)) {
          lumped_parameter.push_back(monomial.factor(interest_monomial));
        }
      }
      if (!lumped_parameter.size()) { continue; }
      // Factor out any coefficients, so that 'a' and '2*a' are not both
      // considered lumped parameters.
      PolyType lumped_parameter_polynomial(lumped_parameter.begin(),
                                           lumped_parameter.end());
      PolyType normalized =
          NormalizePolynomial(lumped_parameter_polynomial).second;
      lumped_parameters.insert(normalized);
    }
  }

  // For each such parameter polynomial, create a lumped parameter.
  int lump_index = 1;
  typename SystemIdentification<T>::LumpingMapType lumping_map;

  for (const PolyType& lump : lumped_parameters) {
    VarType lump_var = PolyType("lump", lump_index).getSimpleVariable();
    lumping_map[lump] = lump_var;
    lump_index++;
  }

  return lumping_map;
}

template<typename T>
typename SystemIdentification<T>::PolyType
SystemIdentification<T>::RewritePolynomialWithLumpedParameters(
    const PolyType& poly,
    const LumpingMapType& lumped_parameters) {
  // Reconstruct vars_of_interest, the variables in poly that are not
  // mentioned by the lumped_parameters.
  std::set<VarType> vars_of_interest = poly.getVariables();
  for (auto lump_name_pair : lumped_parameters) {
    std::set<VarType> parameters_in_lump = lump_name_pair.first.getVariables();
    for (const VarType& var : parameters_in_lump) {
      vars_of_interest.erase(var);
    }
  }

  // Loop over the combinations of the variables-of-interest, constructing the
  // polynomial of parameters that multiply by each combination; if that
  // polynomial is a lumped variable, substitute in a new monomial of the
  // lumped variable times the combination instead.
  std::set<typename PolyType::Monomial> interest_monomials =
      GetAllCombinationsOfVars({poly}, vars_of_interest);
  std::vector<MonomialType> working_monomials = poly.getMonomials();
  for (const MonomialType& interest_monomial : interest_monomials) {
    std::vector<MonomialType> new_working_monomials;
    std::vector<MonomialType> factor_monomials;
    for (const MonomialType& working_monomial : working_monomials) {
      if (MonomialMatches(working_monomial, interest_monomial,
                          vars_of_interest)) {
        // This monomial matches our interest monomial; we will factor it by
        // the interest monomial and add the resulting monomial of parameters
        // to our factor list.
        factor_monomials.push_back(working_monomial.factor(interest_monomial));
      } else {
        // This monomial does not match our interest monomial; copy it
        // unchanged.
        new_working_monomials.push_back(working_monomial);
      }
    }
    const PolyType factor_polynomial(factor_monomials.begin(),
                               factor_monomials.end());
    const auto& normalization = NormalizePolynomial(factor_polynomial);
    const T coefficient = normalization.first;
    const PolyType& normalized = normalization.second;

    if (!lumped_parameters.count(normalized)) {
      // Factoring out this combination yielded a parameter polynomial that
      // does not correspond to a lumped variable.  Ignore it.
      continue;
    }

    // We have a lumped parameter, so construct a new monomial and replace the
    // working monomials list.
    TermType lump_term;
    lump_term.var = lumped_parameters.at(normalized);
    lump_term.power = 1;
    MonomialType lumped_monomial;
    lumped_monomial.terms = interest_monomial.terms;
    lumped_monomial.terms.push_back(lump_term);
    lumped_monomial.coefficient = coefficient;
    new_working_monomials.push_back(lumped_monomial);
    working_monomials = new_working_monomials;
  }

  return PolyType(working_monomials.begin(), working_monomials.end());
}

};
};

template class DRAKEPOLYNOMIAL_EXPORT drake::util::SystemIdentification<double>;
<|endoftext|>


<commit_before>#include "nanocv.h"
#include "models/forward_network.h"
#include <boost/program_options.hpp>
#include <set>

using namespace ncv;

static void test_grad(const string_t& header, const string_t& loss_id, const model_t& model, scalar_t lambda)
{
        random_t<size_t> rand(2, 16);
        const size_t n_threads = 1 + (rand() % 2);

        accumulator_t acc_params(model, n_threads, accumulator_t::type::vgrad, lambda);
        rmodel_t rmodel_inputs = model.clone();

        const size_t n_tests = 64;
        const size_t n_samples = rand();

        const rloss_t rloss = loss_manager_t::instance().get(loss_id);
        const loss_t& loss = *rloss;

        const size_t psize = model.psize();
        const size_t isize = model.isize();
        const size_t osize = model.osize();

        vector_t params(psize);
        vectors_t targets(n_samples, vector_t(osize));
        tensors_t inputs(n_samples, tensor_t(model.idims(), model.irows(), model.icols()));

        // optimization problem (wrt parameters & inputs): size
        auto fn_params_size = [&] ()
        {
                return psize;
        };

        auto fn_inputs_size = [&] ()
        {
                return isize;
        };

        // optimization problem (wrt parameters & inputs): function value
        auto fn_params_fval = [&] (const vector_t& x)
        {
                acc_params.reset(x);
                acc_params.update(inputs, targets, loss);

                return acc_params.value();
        };

        auto fn_inputs_fval = [&] (const vector_t& x)
        {
                rmodel_inputs->load_params(params);

                const vector_t target = targets[0];
                const vector_t output = rmodel_inputs->forward(x).vector();

                return loss.value(target, output);
        };

        // optimization problem (wrt parameters & inputs): function value & gradient
        auto fn_params_grad = [&] (const vector_t& x, vector_t& gx)
        {
                acc_params.reset(x);
                acc_params.update(inputs, targets, loss);

                gx = acc_params.vgrad();
                return acc_params.value();
        };

        auto fn_inputs_grad = [&] (const vector_t& x, vector_t& gx)
        {
                rmodel_inputs->load_params(params);

                const vector_t target = targets[0];
                const vector_t output = rmodel_inputs->forward(x).vector();

                gx = rmodel_inputs->backward(loss.vgrad(target, output)).vector();
                return loss.value(target, output);
        };

        // construct optimization problem: analytic gradient and finite difference approximation
        const opt_problem_t problem_analytic_params(fn_params_size, fn_params_fval, fn_params_grad);
        const opt_problem_t problem_aproxdif_params(fn_params_size, fn_params_fval);

        const opt_problem_t problem_analytic_inputs(fn_inputs_size, fn_inputs_fval, fn_inputs_grad);
        const opt_problem_t problem_aproxdif_inputs(fn_inputs_size, fn_inputs_fval);

        for (size_t t = 0; t < n_tests; t ++)
        {
                random_t<scalar_t> prgen(-1.0, +1.0);
                random_t<scalar_t> irgen(-0.1, +0.1);
                random_t<size_t> trgen(0, osize - 1);

                prgen(params.data(), params.data() + psize);
                for (vector_t& target : targets)
                {
                        target = ncv::class_target(trgen(), osize);
                }
                for (tensor_t& input : inputs)
                {
                        irgen(input.data(), input.data() + isize);
                }

                vector_t analytic_params_grad, aproxdif_params_grad;
                vector_t analytic_inputs_grad, aproxdif_inputs_grad;

                problem_analytic_params(params, analytic_params_grad);
                problem_aproxdif_params(params, aproxdif_params_grad);

                problem_analytic_inputs(inputs[0].vector(), analytic_inputs_grad);
                problem_aproxdif_inputs(inputs[0].vector(), aproxdif_inputs_grad);

                const scalar_t dg_params = (analytic_params_grad - aproxdif_params_grad).lpNorm<Eigen::Infinity>();
                const scalar_t dg_inputs = (analytic_inputs_grad - aproxdif_inputs_grad).lpNorm<Eigen::Infinity>();

                const scalar_t eps = 1e-6;

                log_info() << header << " [" << (t + 1) << "/" << n_tests
                           << "]: samples = " << n_samples
                           << ", dg_params = " << dg_params << " (" << (dg_params > eps ? "ERROR" : "OK") << ")"
                           << ", dg_inputs = " << dg_inputs << " (" << (dg_inputs > eps ? "ERROR" : "OK") << ").";
        }
}

static void test_grad(const string_t& header, const string_t& loss_id, const model_t& model)
{
        const scalars_t lambdas = { 0.0, 1e-3, 1e-2, 1e-1, 1.0 };
        for (scalar_t lambda : lambdas)
        {
                test_grad(header, loss_id, model, lambda);
        }
}

int main(int argc, char *argv[])
{
        ncv::init();

        const strings_t conv_layer_ids { "", "conv" };
        const strings_t pool_layer_ids { "", "pool-max", "pool-min", "pool-avg" };
        const strings_t full_layer_ids { "", "linear" };
        const strings_t actv_layer_ids { "", "act-unit", "act-tanh", "act-snorm", "act-splus" };
        const strings_t loss_ids = loss_manager_t::instance().ids();

        const color_mode cmd_color = color_mode::luma;
        const size_t cmd_irows = 10;
        const size_t cmd_icols = 10;
        const size_t cmd_outputs = 4;
        const size_t cmd_max_layers = 2;

        // evaluate the analytical gradient vs. the finite difference approximation for various:
        //      * convolution layers
        //      * pooling layers
        //      * fully connected layers
        //      * activation layers
        std::set<string_t> descs;
        for (size_t n_layers = 0; n_layers <= cmd_max_layers; n_layers ++)
        {
                for (const string_t& actv_layer_id : actv_layer_ids)
                {
                        for (const string_t& pool_layer_id : pool_layer_ids)
                        {
                                for (const string_t& conv_layer_id : conv_layer_ids)
                                {
                                        for (const string_t& full_layer_id : full_layer_ids)
                                        {
                                                string_t desc;

                                                // convolution part
                                                for (size_t l = 0; l < n_layers && !conv_layer_id.empty(); l ++)
                                                {
                                                        random_t<size_t> rgen(2, 3);

                                                        string_t params;
                                                        params += "dims=" + text::to_string(rgen());
                                                        params += (rgen() % 2 == 0) ? ",rows=3,cols=3" : ",rows=4,cols=4";

                                                        desc += conv_layer_id + ":" + params + ";";
                                                        desc += pool_layer_id + ";";
                                                        desc += actv_layer_id + ";";
                                                }

                                                // fully-connected part
                                                for (size_t l = 0; l < n_layers && !full_layer_id.empty(); l ++)
                                                {
                                                        random_t<size_t> rgen(1, 8);

                                                        string_t params;
                                                        params += "dims=" + text::to_string(rgen());

                                                        desc += full_layer_id + ":" + params + ";";
                                                        desc += actv_layer_id + ";";
                                                }

                                                desc += "linear:dims=" + text::to_string(cmd_outputs) + ";";
                                                desc += "softmax:type=global;";

                                                descs.insert(desc);
                                        }
                                }
                        }
                }
        }

        for (const string_t& desc : descs)
        {
                // create network
                forward_network_t network(desc);
                network.resize(cmd_irows, cmd_icols, cmd_outputs, cmd_color, true);

                // test network
                for (const string_t& loss_id : loss_ids)
                {
                        test_grad("[loss = " + loss_id + "]", loss_id, network);
                }
        }

        // OK
        log_info() << done;
        return EXIT_SUCCESS;
}
<commit_msg>split gradient checks for params & inputs<commit_after>#include "nanocv.h"
#include "models/forward_network.h"
#include <boost/program_options.hpp>
#include <set>

using namespace ncv;

static void test_grad_params(const string_t& header, const string_t& loss_id, const model_t& model, scalar_t lambda)
{
        random_t<size_t> rand(2, 16);
        const size_t n_threads = 1 + (rand() % 2);

        accumulator_t acc_params(model, n_threads, accumulator_t::type::vgrad, lambda);

        const size_t n_tests = 64;
        const size_t n_samples = rand();

        const rloss_t rloss = loss_manager_t::instance().get(loss_id);
        const loss_t& loss = *rloss;

        const size_t psize = model.psize();
        const size_t isize = model.isize();
        const size_t osize = model.osize();

        vector_t params(psize);
        vectors_t targets(n_samples, vector_t(osize));
        tensors_t inputs(n_samples, tensor_t(model.idims(), model.irows(), model.icols()));

        // optimization problem (wrt parameters & inputs): size
        auto fn_params_size = [&] ()
        {
                return psize;
        };

        // optimization problem (wrt parameters & inputs): function value
        auto fn_params_fval = [&] (const vector_t& x)
        {
                acc_params.reset(x);
                acc_params.update(inputs, targets, loss);

                return acc_params.value();
        };

        // optimization problem (wrt parameters & inputs): function value & gradient
        auto fn_params_grad = [&] (const vector_t& x, vector_t& gx)
        {
                acc_params.reset(x);
                acc_params.update(inputs, targets, loss);

                gx = acc_params.vgrad();
                return acc_params.value();
        };

        // construct optimization problem: analytic gradient and finite difference approximation
        const opt_problem_t problem_analytic_params(fn_params_size, fn_params_fval, fn_params_grad);
        const opt_problem_t problem_aproxdif_params(fn_params_size, fn_params_fval);

        for (size_t t = 0; t < n_tests; t ++)
        {
                random_t<scalar_t> prgen(-1.0, +1.0);
                random_t<scalar_t> irgen(-0.1, +0.1);
                random_t<size_t> trgen(0, osize - 1);

                prgen(params.data(), params.data() + psize);
                for (vector_t& target : targets)
                {
                        target = ncv::class_target(trgen(), osize);
                }
                for (tensor_t& input : inputs)
                {
                        irgen(input.data(), input.data() + isize);
                }

                vector_t analytic_params_grad, aproxdif_params_grad;

                problem_analytic_params(params, analytic_params_grad);
                problem_aproxdif_params(params, aproxdif_params_grad);

                const scalar_t dg_params = (analytic_params_grad - aproxdif_params_grad).lpNorm<Eigen::Infinity>();
                const scalar_t eps = 1e-6;

                log_info() << header << " [" << (t + 1) << "/" << n_tests
                           << "]: samples = " << n_samples
                           << ", dg_params = " << dg_params << " (" << (dg_params > eps ? "ERROR" : "OK") << ").";
        }
}

static void test_grad_inputs(const string_t& header, const string_t& loss_id, const model_t& model)
{
        rmodel_t rmodel_inputs = model.clone();

        const size_t n_tests = 64;
        const size_t n_samples = rand();

        const rloss_t rloss = loss_manager_t::instance().get(loss_id);
        const loss_t& loss = *rloss;

        const size_t psize = model.psize();
        const size_t isize = model.isize();
        const size_t osize = model.osize();

        vector_t params(psize);
        vector_t target(osize);
        tensor_t input(model.idims(), model.irows(), model.icols());

        // optimization problem (wrt parameters & inputs): size
        auto fn_inputs_size = [&] ()
        {
                return isize;
        };

        // optimization problem (wrt parameters & inputs): function value
        auto fn_inputs_fval = [&] (const vector_t& x)
        {
                rmodel_inputs->load_params(params);

                const vector_t output = rmodel_inputs->forward(x).vector();

                return loss.value(target, output);
        };

        // optimization problem (wrt parameters & inputs): function value & gradient
        auto fn_inputs_grad = [&] (const vector_t& x, vector_t& gx)
        {
                rmodel_inputs->load_params(params);

                const vector_t output = rmodel_inputs->forward(x).vector();

                gx = rmodel_inputs->backward(loss.vgrad(target, output)).vector();
                return loss.value(target, output);
        };

        // construct optimization problem: analytic gradient and finite difference approximation
        const opt_problem_t problem_analytic_inputs(fn_inputs_size, fn_inputs_fval, fn_inputs_grad);
        const opt_problem_t problem_aproxdif_inputs(fn_inputs_size, fn_inputs_fval);

        for (size_t t = 0; t < n_tests; t ++)
        {
                random_t<scalar_t> prgen(-1.0, +1.0);
                random_t<scalar_t> irgen(-0.1, +0.1);
                random_t<size_t> trgen(0, osize - 1);

                prgen(params.data(), params.data() + psize);
                target = ncv::class_target(trgen(), osize);
                irgen(input.data(), input.data() + isize);

                vector_t analytic_inputs_grad, aproxdif_inputs_grad;

                problem_analytic_inputs(input.vector(), analytic_inputs_grad);
                problem_aproxdif_inputs(input.vector(), aproxdif_inputs_grad);

                const scalar_t dg_inputs = (analytic_inputs_grad - aproxdif_inputs_grad).lpNorm<Eigen::Infinity>();
                const scalar_t eps = 1e-6;

                log_info() << header << " [" << (t + 1) << "/" << n_tests
                           << "]: samples = " << n_samples
                           << ", dg_inputs = " << dg_inputs << " (" << (dg_inputs > eps ? "ERROR" : "OK") << ").";
        }
}

static void test_grad(const string_t& header, const string_t& loss_id, const model_t& model)
{
        const scalars_t lambdas = { 0.0, 1e-3, 1e-2, 1e-1, 1.0 };
        for (scalar_t lambda : lambdas)
        {
                test_grad_params(header, loss_id, model, lambda);
        }

        test_grad_inputs(header, loss_id, model);
}

int main(int argc, char *argv[])
{
        ncv::init();

        const strings_t conv_layer_ids { "", "conv" };
        const strings_t pool_layer_ids { "", "pool-max", "pool-min", "pool-avg" };
        const strings_t full_layer_ids { "", "linear" };
        const strings_t actv_layer_ids { "", "act-unit", "act-tanh", "act-snorm", "act-splus" };
        const strings_t loss_ids = loss_manager_t::instance().ids();

        const color_mode cmd_color = color_mode::luma;
        const size_t cmd_irows = 10;
        const size_t cmd_icols = 10;
        const size_t cmd_outputs = 4;
        const size_t cmd_max_layers = 2;

        // evaluate the analytical gradient vs. the finite difference approximation for various:
        //      * convolution layers
        //      * pooling layers
        //      * fully connected layers
        //      * activation layers
        std::set<string_t> descs;
        for (size_t n_layers = 0; n_layers <= cmd_max_layers; n_layers ++)
        {
                for (const string_t& actv_layer_id : actv_layer_ids)
                {
                        for (const string_t& pool_layer_id : pool_layer_ids)
                        {
                                for (const string_t& conv_layer_id : conv_layer_ids)
                                {
                                        for (const string_t& full_layer_id : full_layer_ids)
                                        {
                                                string_t desc;

                                                // convolution part
                                                for (size_t l = 0; l < n_layers && !conv_layer_id.empty(); l ++)
                                                {
                                                        random_t<size_t> rgen(2, 3);

                                                        string_t params;
                                                        params += "dims=" + text::to_string(rgen());
                                                        params += (rgen() % 2 == 0) ? ",rows=3,cols=3" : ",rows=4,cols=4";

                                                        desc += conv_layer_id + ":" + params + ";";
                                                        desc += pool_layer_id + ";";
                                                        desc += actv_layer_id + ";";
                                                }

                                                // fully-connected part
                                                for (size_t l = 0; l < n_layers && !full_layer_id.empty(); l ++)
                                                {
                                                        random_t<size_t> rgen(1, 8);

                                                        string_t params;
                                                        params += "dims=" + text::to_string(rgen());

                                                        desc += full_layer_id + ":" + params + ";";
                                                        desc += actv_layer_id + ";";
                                                }

                                                desc += "linear:dims=" + text::to_string(cmd_outputs) + ";";
                                                desc += "softmax:type=global;";

                                                descs.insert(desc);
                                        }
                                }
                        }
                }
        }

        for (const string_t& desc : descs)
        {
                // create network
                forward_network_t network(desc);
                network.resize(cmd_irows, cmd_icols, cmd_outputs, cmd_color, true);

                // test network
                for (const string_t& loss_id : loss_ids)
                {
                        test_grad("[loss = " + loss_id + "]", loss_id, network);
                }
        }

        // OK
        log_info() << done;
        return EXIT_SUCCESS;
}
<|endoftext|>