<commit_before>#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include <Python.h>
#include "numpy/arrayobject.h"
#include "part_int.h"
#include <set>

/*Wraps the flux_extractor into a python module called spectra_priv. Don't call this directly, call the python wrapper.*/

/*Check whether the passed array has type typename. Returns 1 if it doesn't, 0 if it does.*/
int check_type(PyArrayObject * arr, int npy_typename)
{
  return !PyArray_EquivTypes(PyArray_DESCR(arr), PyArray_DescrFromType(npy_typename));
}

int check_float(PyArrayObject * arr)
{
  return check_type(arr, NPY_FLOAT);
}

/* When handed a list of particles,
 * return a list of bools with True for those nearby to a sightline*/
extern "C" PyObject * Py_near_lines(PyObject *self, PyObject *args)
{
    int NumLos;
    long long Npart;
    double box100;
    PyArrayObject *cofm, *axis, *pos, *hh, *is_a_line;
    PyObject *out;

    if(!PyArg_ParseTuple(args, "dO!O!O!O!",&box100,  &PyArray_Type, &pos, &PyArray_Type, &hh, &PyArray_Type, &axis, &PyArray_Type, &cofm) )
      return NULL;

    NumLos = PyArray_DIM(cofm,0);
    Npart = PyArray_DIM(pos,0);

    if(NumLos != PyArray_DIM(axis,0) || 3 != PyArray_DIM(cofm,1)){
      PyErr_SetString(PyExc_ValueError, "cofm must have dimensions (np.size(axis),3) \n");
      return NULL;
    }
    if(check_type(cofm, NPY_DOUBLE) || check_type(axis,NPY_INT)){
      PyErr_SetString(PyExc_ValueError, "cofm must have 64-bit float type and axis must be a 32-bit integer\n");
      return NULL;
    }
    if(check_float(pos) || check_float(hh)){
       PyErr_SetString(PyExc_TypeError, "pos and h must have 32-bit float type\n");
       return NULL;
    }

    //Setup los_tables
    //PyArray_GETCONTIGUOUS increments the reference count of the object,
    //so to avoid leaking we need to save the PyArrayObject pointer.
    cofm = PyArray_GETCONTIGUOUS(cofm);
    axis = PyArray_GETCONTIGUOUS(axis);
    double * Cofm =(double *) PyArray_DATA(cofm);
    int32_t * Axis =(int32_t *) PyArray_DATA(axis);
    IndexTable sort_los_table(Cofm, Axis, NumLos, box100);

    //Set of particles near a line
    std::set<int> near_lines;
    //find lists
    //DANGER: potentially huge allocation
    pos = PyArray_GETCONTIGUOUS(pos);
    hh = PyArray_GETCONTIGUOUS(hh);
    const float * Pos =(float *) PyArray_DATA(pos);
    const float * h = (float *) PyArray_DATA(hh);
    #pragma omp parallel for
    for(long long i=0; i < Npart; i++){
	    std::map<int, double> nearby=sort_los_table.get_near_lines(&(Pos[3*i]),h[i]);
        if(nearby.size()>0){
           #pragma omp critical
           {
              near_lines.insert(i);
           }
        }
    }
    //Copy data into python
    npy_intp size = near_lines.size();
    is_a_line = (PyArrayObject *) PyArray_SimpleNew(1, &size, NPY_INT);
    int i=0;
    for (std::set<int>::const_iterator it = near_lines.begin(); it != near_lines.end() && i < size; ++it, ++i){
            *(npy_int *)PyArray_GETPTR1(is_a_line,i) = (*it);
    }
    out = Py_BuildValue("O", is_a_line);
    Py_DECREF(is_a_line);
    //Because PyArray_GETCONTIGUOUS incremented the reference count,
    //and may have made an allocation, in which case this does not point to what it used to.
    Py_DECREF(pos);
    Py_DECREF(hh);
    Py_DECREF(cofm);
    Py_DECREF(axis);
    return out;
}

/*****************************************************************************/
/*Interface for SPH interpolation*/
extern "C" PyObject * Py_Particle_Interpolation(PyObject *self, PyObject *args)
{
    //Things which should be from input
    int nbins, NumLos, compute_tau;
    long long Npart;
    double box100, velfac, lambda, gamma, fosc, amumass, atime;
    npy_intp size[2];
    //Input variables in np format
    PyArrayObject *pos, *vel, *dens, *temp, *h;
    PyArrayObject *cofm, *axis;

    //Get our input
    if(!PyArg_ParseTuple(args, "iidddddddO!O!O!O!O!O!O!", &compute_tau, &nbins, &box100,  &velfac, &atime, &lambda, &gamma, &fosc, &amumass, &PyArray_Type, &pos, &PyArray_Type, &vel, &PyArray_Type, &dens, &PyArray_Type, &temp, &PyArray_Type, &h, &PyArray_Type, &axis, &PyArray_Type, &cofm) )
    {
      PyErr_SetString(PyExc_AttributeError, "Incorrect arguments: use compute_tau, nbins, boxsize, velfac, atime, lambda, gamma, fosc, species mass (amu), pos, vel, dens, temp, h, axis, cofm\n");
      return NULL;
    }

    //Check that our input has the right types
    if(check_float(pos) || check_float(vel) || check_float(dens) || check_float(temp) || check_float(h)){
       PyErr_SetString(PyExc_TypeError, "One of the data arrays does not have 32-bit float type\n");
       return NULL;
    }
    if(check_type(cofm,NPY_DOUBLE)){
      PyErr_SetString(PyExc_TypeError, "Sightline positions must have 64-bit float type\n");
      return NULL;
    }
    if(check_type(axis, NPY_INT32)){
      PyErr_SetString(PyExc_TypeError, "Axis must be a 32-bit integer\n");
      return NULL;
    }

    NumLos = PyArray_DIM(cofm,0);
    Npart = PyArray_DIM(pos,0);
    //Malloc stuff
    size[0] = NumLos;
    size[1] = nbins;

    if(NumLos != PyArray_DIM(axis,0) || 3 != PyArray_DIM(cofm,1))
    {
      PyErr_SetString(PyExc_ValueError, "cofm must have dimensions (np.size(axis),3) \n");
      return NULL;
    }

    //Initialise P from the data in the input numpy arrays.
    //Note: better be sure they are float32 in the calling function.
    //PyArray_GETCONTIGUOUS increments the reference count of the object,
    //so to avoid leaking we need to save the PyArrayObject pointer.
    pos = PyArray_GETCONTIGUOUS(pos);
    dens = PyArray_GETCONTIGUOUS(dens);
    h = PyArray_GETCONTIGUOUS(h);
    float * Pos =(float *) PyArray_DATA(pos);
    float * Hh= (float *) PyArray_DATA(h);
    float * Dens =(float *) PyArray_DATA(dens);

    cofm = PyArray_GETCONTIGUOUS(cofm);
    axis = PyArray_GETCONTIGUOUS(axis);
    double * Cofm =(double *) PyArray_DATA(cofm);
    int32_t * Axis =(int32_t *) PyArray_DATA(axis);
    if( !Pos || !Dens || !Hh || !Cofm || !Axis ){
        PyErr_SetString(PyExc_MemoryError, "Getting contiguous copies of input arrays failed\n");
        return NULL;
    }
    ParticleInterp pint(nbins, lambda, gamma, fosc, amumass, box100, velfac, atime, Cofm, Axis ,NumLos);

    PyObject * for_return;
    /* Allocate array space. This is (I hope) contiguous.
     * Note: for an array of shape (a,b), element (i,j) can be accessed as
     * [i*b+j] */
    if (compute_tau){
        vel = PyArray_GETCONTIGUOUS(vel);
        temp = PyArray_GETCONTIGUOUS(temp);

        float * Vel =(float *) PyArray_DATA(vel);
        float * Temp =(float *) PyArray_DATA(temp);

        if( !Vel || !Temp ){
          PyErr_SetString(PyExc_MemoryError, "Getting contiguous copies of Vel and Temp failed\n");
          return NULL;
        }

        PyArrayObject * tau_out = (PyArrayObject *) PyArray_SimpleNew(2, size, NPY_DOUBLE);
        double * tau = (double *) PyArray_DATA(tau_out);
        if ( !tau_out ){
          PyErr_SetString(PyExc_MemoryError, "Could not allocate memory for tau\n");
          return NULL;
        }
        PyArray_FILLWBYTE(tau_out, 0);
        //Do the work
        pint.compute_tau(tau, Pos, Vel, Dens, Temp, Hh, Npart);

        //Build a tuple from the interp struct
        for_return = Py_BuildValue("O", tau_out);
        Py_DECREF(tau_out);
        Py_DECREF(vel);
        Py_DECREF(temp);

    }
    else{
        PyArrayObject * colden_out = (PyArrayObject *) PyArray_SimpleNew(2, size, NPY_DOUBLE);
        double * colden = (double *) PyArray_DATA(colden_out);
        if ( !colden_out ){
          PyErr_SetString(PyExc_MemoryError, "Could not allocate memory for colden\n");
          return NULL;
        }
        //Initialise output arrays to 0.
        PyArray_FILLWBYTE(colden_out, 0);

        //Do the work
        pint.compute_colden(colden, Pos, Dens, Hh, Npart);

        //Build a tuple from the interp struct
        for_return = Py_BuildValue("O", colden_out);
        Py_DECREF(colden_out);
    }

    //Because PyArray_GETCONTIGUOUS incremented the reference count,
    //and may have made an allocation, in which case this does not point to what it used to.
    Py_DECREF(pos);
    Py_DECREF(dens);
    Py_DECREF(h);
    Py_DECREF(cofm);
    Py_DECREF(axis);

    return for_return;
}

static PyMethodDef spectrae[] = {
  {"_Particle_Interpolate", Py_Particle_Interpolation, METH_VARARGS,
   "Find absorption or column density by interpolating particles. "
   "    Arguments: compute_tau nbins, boxsize, velfac, atime, lambda, gamma, fosc, species mass (amu), pos, vel, dens, temp, h, axis, cofm"
   "    "},
  {"_near_lines", Py_near_lines,METH_VARARGS,
   "Give a list of particles and sightlines, "
   "return a list of booleans for those particles near "
   "a sightline."
   "   Arguments: box, pos, h, axis, cofm"},
  {NULL, NULL, 0, NULL},
};

PyMODINIT_FUNC
init_spectra_priv(void)
{
  Py_InitModule("_spectra_priv", spectrae);
  import_array();
}
<commit_msg>An extra check<commit_after>#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include <Python.h>
#include "numpy/arrayobject.h"
#include "part_int.h"
#include <set>

/*Wraps the flux_extractor into a python module called spectra_priv. Don't call this directly, call the python wrapper.*/

/*Check whether the passed array has type typename. Returns 1 if it doesn't, 0 if it does.*/
int check_type(PyArrayObject * arr, int npy_typename)
{
  return !PyArray_EquivTypes(PyArray_DESCR(arr), PyArray_DescrFromType(npy_typename));
}

int check_float(PyArrayObject * arr)
{
  return check_type(arr, NPY_FLOAT);
}

/* When handed a list of particles,
 * return a list of bools with True for those nearby to a sightline*/
extern "C" PyObject * Py_near_lines(PyObject *self, PyObject *args)
{
    int NumLos;
    long long Npart;
    double box100;
    PyArrayObject *cofm, *axis, *pos, *hh, *is_a_line;
    PyObject *out;

    if(!PyArg_ParseTuple(args, "dO!O!O!O!",&box100,  &PyArray_Type, &pos, &PyArray_Type, &hh, &PyArray_Type, &axis, &PyArray_Type, &cofm) )
      return NULL;

    if(2 > PyArray_NDIM(cofm) || 1 >  PyArray_NDIM(axis)){
      PyErr_SetString(PyExc_ValueError, "cofm must have dimensions (np.size(axis),3) \n");
      return NULL;
    }
    NumLos = PyArray_DIM(cofm,0);
    Npart = PyArray_DIM(pos,0);

    if(NumLos != PyArray_DIM(axis,0) || 3 != PyArray_DIM(cofm,1)){
      PyErr_SetString(PyExc_ValueError, "cofm must have dimensions (np.size(axis),3) \n");
      return NULL;
    }
    if(check_type(cofm, NPY_DOUBLE) || check_type(axis,NPY_INT)){
      PyErr_SetString(PyExc_ValueError, "cofm must have 64-bit float type and axis must be a 32-bit integer\n");
      return NULL;
    }
    if(check_float(pos) || check_float(hh)){
       PyErr_SetString(PyExc_TypeError, "pos and h must have 32-bit float type\n");
       return NULL;
    }

    //Setup los_tables
    //PyArray_GETCONTIGUOUS increments the reference count of the object,
    //so to avoid leaking we need to save the PyArrayObject pointer.
    cofm = PyArray_GETCONTIGUOUS(cofm);
    axis = PyArray_GETCONTIGUOUS(axis);
    double * Cofm =(double *) PyArray_DATA(cofm);
    int32_t * Axis =(int32_t *) PyArray_DATA(axis);
    IndexTable sort_los_table(Cofm, Axis, NumLos, box100);

    //Set of particles near a line
    std::set<int> near_lines;
    //find lists
    //DANGER: potentially huge allocation
    pos = PyArray_GETCONTIGUOUS(pos);
    hh = PyArray_GETCONTIGUOUS(hh);
    const float * Pos =(float *) PyArray_DATA(pos);
    const float * h = (float *) PyArray_DATA(hh);
    #pragma omp parallel for
    for(long long i=0; i < Npart; i++){
	    std::map<int, double> nearby=sort_los_table.get_near_lines(&(Pos[3*i]),h[i]);
        if(nearby.size()>0){
           #pragma omp critical
           {
              near_lines.insert(i);
           }
        }
    }
    //Copy data into python
    npy_intp size = near_lines.size();
    is_a_line = (PyArrayObject *) PyArray_SimpleNew(1, &size, NPY_INT);
    int i=0;
    for (std::set<int>::const_iterator it = near_lines.begin(); it != near_lines.end() && i < size; ++it, ++i){
            *(npy_int *)PyArray_GETPTR1(is_a_line,i) = (*it);
    }
    out = Py_BuildValue("O", is_a_line);
    Py_DECREF(is_a_line);
    //Because PyArray_GETCONTIGUOUS incremented the reference count,
    //and may have made an allocation, in which case this does not point to what it used to.
    Py_DECREF(pos);
    Py_DECREF(hh);
    Py_DECREF(cofm);
    Py_DECREF(axis);
    return out;
}

/*****************************************************************************/
/*Interface for SPH interpolation*/
extern "C" PyObject * Py_Particle_Interpolation(PyObject *self, PyObject *args)
{
    //Things which should be from input
    int nbins, NumLos, compute_tau;
    long long Npart;
    double box100, velfac, lambda, gamma, fosc, amumass, atime;
    npy_intp size[2];
    //Input variables in np format
    PyArrayObject *pos, *vel, *dens, *temp, *h;
    PyArrayObject *cofm, *axis;

    //Get our input
    if(!PyArg_ParseTuple(args, "iidddddddO!O!O!O!O!O!O!", &compute_tau, &nbins, &box100,  &velfac, &atime, &lambda, &gamma, &fosc, &amumass, &PyArray_Type, &pos, &PyArray_Type, &vel, &PyArray_Type, &dens, &PyArray_Type, &temp, &PyArray_Type, &h, &PyArray_Type, &axis, &PyArray_Type, &cofm) )
    {
      PyErr_SetString(PyExc_AttributeError, "Incorrect arguments: use compute_tau, nbins, boxsize, velfac, atime, lambda, gamma, fosc, species mass (amu), pos, vel, dens, temp, h, axis, cofm\n");
      return NULL;
    }

    //Check that our input has the right types
    if(check_float(pos) || check_float(vel) || check_float(dens) || check_float(temp) || check_float(h)){
       PyErr_SetString(PyExc_TypeError, "One of the data arrays does not have 32-bit float type\n");
       return NULL;
    }
    if(check_type(cofm,NPY_DOUBLE)){
      PyErr_SetString(PyExc_TypeError, "Sightline positions must have 64-bit float type\n");
      return NULL;
    }
    if(check_type(axis, NPY_INT32)){
      PyErr_SetString(PyExc_TypeError, "Axis must be a 32-bit integer\n");
      return NULL;
    }

    NumLos = PyArray_DIM(cofm,0);
    Npart = PyArray_DIM(pos,0);
    //Malloc stuff
    size[0] = NumLos;
    size[1] = nbins;

    if(NumLos != PyArray_DIM(axis,0) || 3 != PyArray_DIM(cofm,1))
    {
      PyErr_SetString(PyExc_ValueError, "cofm must have dimensions (np.size(axis),3) \n");
      return NULL;
    }

    //Initialise P from the data in the input numpy arrays.
    //Note: better be sure they are float32 in the calling function.
    //PyArray_GETCONTIGUOUS increments the reference count of the object,
    //so to avoid leaking we need to save the PyArrayObject pointer.
    pos = PyArray_GETCONTIGUOUS(pos);
    dens = PyArray_GETCONTIGUOUS(dens);
    h = PyArray_GETCONTIGUOUS(h);
    float * Pos =(float *) PyArray_DATA(pos);
    float * Hh= (float *) PyArray_DATA(h);
    float * Dens =(float *) PyArray_DATA(dens);

    cofm = PyArray_GETCONTIGUOUS(cofm);
    axis = PyArray_GETCONTIGUOUS(axis);
    double * Cofm =(double *) PyArray_DATA(cofm);
    int32_t * Axis =(int32_t *) PyArray_DATA(axis);
    if( !Pos || !Dens || !Hh || !Cofm || !Axis ){
        PyErr_SetString(PyExc_MemoryError, "Getting contiguous copies of input arrays failed\n");
        return NULL;
    }
    ParticleInterp pint(nbins, lambda, gamma, fosc, amumass, box100, velfac, atime, Cofm, Axis ,NumLos);

    PyObject * for_return;
    /* Allocate array space. This is (I hope) contiguous.
     * Note: for an array of shape (a,b), element (i,j) can be accessed as
     * [i*b+j] */
    if (compute_tau){
        vel = PyArray_GETCONTIGUOUS(vel);
        temp = PyArray_GETCONTIGUOUS(temp);

        float * Vel =(float *) PyArray_DATA(vel);
        float * Temp =(float *) PyArray_DATA(temp);

        if( !Vel || !Temp ){
          PyErr_SetString(PyExc_MemoryError, "Getting contiguous copies of Vel and Temp failed\n");
          return NULL;
        }

        PyArrayObject * tau_out = (PyArrayObject *) PyArray_SimpleNew(2, size, NPY_DOUBLE);
        double * tau = (double *) PyArray_DATA(tau_out);
        if ( !tau_out ){
          PyErr_SetString(PyExc_MemoryError, "Could not allocate memory for tau\n");
          return NULL;
        }
        PyArray_FILLWBYTE(tau_out, 0);
        //Do the work
        pint.compute_tau(tau, Pos, Vel, Dens, Temp, Hh, Npart);

        //Build a tuple from the interp struct
        for_return = Py_BuildValue("O", tau_out);
        Py_DECREF(tau_out);
        Py_DECREF(vel);
        Py_DECREF(temp);

    }
    else{
        PyArrayObject * colden_out = (PyArrayObject *) PyArray_SimpleNew(2, size, NPY_DOUBLE);
        double * colden = (double *) PyArray_DATA(colden_out);
        if ( !colden_out ){
          PyErr_SetString(PyExc_MemoryError, "Could not allocate memory for colden\n");
          return NULL;
        }
        //Initialise output arrays to 0.
        PyArray_FILLWBYTE(colden_out, 0);

        //Do the work
        pint.compute_colden(colden, Pos, Dens, Hh, Npart);

        //Build a tuple from the interp struct
        for_return = Py_BuildValue("O", colden_out);
        Py_DECREF(colden_out);
    }

    //Because PyArray_GETCONTIGUOUS incremented the reference count,
    //and may have made an allocation, in which case this does not point to what it used to.
    Py_DECREF(pos);
    Py_DECREF(dens);
    Py_DECREF(h);
    Py_DECREF(cofm);
    Py_DECREF(axis);

    return for_return;
}

static PyMethodDef spectrae[] = {
  {"_Particle_Interpolate", Py_Particle_Interpolation, METH_VARARGS,
   "Find absorption or column density by interpolating particles. "
   "    Arguments: compute_tau nbins, boxsize, velfac, atime, lambda, gamma, fosc, species mass (amu), pos, vel, dens, temp, h, axis, cofm"
   "    "},
  {"_near_lines", Py_near_lines,METH_VARARGS,
   "Give a list of particles and sightlines, "
   "return a list of booleans for those particles near "
   "a sightline."
   "   Arguments: box, pos, h, axis, cofm"},
  {NULL, NULL, 0, NULL},
};

PyMODINIT_FUNC
init_spectra_priv(void)
{
  Py_InitModule("_spectra_priv", spectrae);
  import_array();
}
<|endoftext|>


<commit_before>#include "stdafx.h"
#include "ComputeDistance.h"

#include "DTW.hpp"

#include "glm/glm.hpp"

#include "mmstd_datatools/misc/FrechetDistance.h"

megamol::probe::ComputeDistance::ComputeDistance()
        : _out_table_slot("outTable", ""), _in_probes_slot("inProbes", "") {
    _out_table_slot.SetCallback(stdplugin::datatools::table::TableDataCall::ClassName(),
        stdplugin::datatools::table::TableDataCall::FunctionName(0), &ComputeDistance::get_data_cb);
    _out_table_slot.SetCallback(stdplugin::datatools::table::TableDataCall::ClassName(),
        stdplugin::datatools::table::TableDataCall::FunctionName(1), &ComputeDistance::get_extent_cb);
    MakeSlotAvailable(&_out_table_slot);

    _in_probes_slot.SetCompatibleCall<CallProbesDescription>();
    MakeSlotAvailable(&_in_probes_slot);
}

megamol::probe::ComputeDistance::~ComputeDistance() {
    this->Release();
}

bool megamol::probe::ComputeDistance::create() {
    return true;
}

void megamol::probe::ComputeDistance::release() {}

bool megamol::probe::ComputeDistance::get_data_cb(core::Call& c) {
    auto out_table = dynamic_cast<stdplugin::datatools::table::TableDataCall*>(&c);
    if (out_table == nullptr)
        return false;
    auto in_probes = _in_probes_slot.CallAs<CallProbes>();
    if (in_probes == nullptr)
        return false;

    if (!(*in_probes)(CallProbes::CallGetMetaData))
        return false;
    if (!(*in_probes)(CallProbes::CallGetData))
        return false;

    auto const& meta_data = in_probes->getMetaData();

    if (in_probes->hasUpdate() || meta_data.m_frame_ID != _frame_id) {
        auto const& probe_data = in_probes->getData();
        auto const probe_count = probe_data->getProbeCount();

        _row_count = probe_count;
        _col_count = probe_count;
        _col_infos.clear();
        _col_infos.resize(probe_count);
        _dis_mat.clear();
        _dis_mat.resize(probe_count * probe_count, 0.0f);

        if (probe_count == 0)
            return false;

        bool vec_probe = false;
        {
            auto test_probe = probe_data->getGenericProbe(0);
            vec_probe = std::holds_alternative<Vec4Probe>(test_probe);
        }

        if (vec_probe) {
            core::utility::log::Log::DefaultLog.WriteInfo("[ComputeDistance] Computing distances for vector probes");
#pragma omp parallel for
            for (std::int64_t a_pidx = 0; a_pidx < probe_count; ++a_pidx) {
                std::vector<glm::vec4> a_samples;
                {
                    auto const a_probe = probe_data->getProbe<Vec4Probe>(a_pidx);
                    auto const& a_samples_tmp = a_probe.getSamplingResult()->samples;
                    a_samples.resize(a_samples_tmp.size());
                    std::transform(a_samples_tmp.begin(), a_samples_tmp.end(), a_samples.begin(),
                        [](auto const& el) { return glm::vec4(el[0], el[1], el[2], el[3]); });
                    std::for_each(a_samples.begin(), a_samples.end(), [](auto& el) { el = glm::normalize(el); });
                }
                for (std::int64_t b_pidx = a_pidx; b_pidx < probe_count; ++b_pidx) {
                    std::vector<glm::vec4> b_samples;
                    {
                        auto const b_probe = probe_data->getProbe<Vec4Probe>(b_pidx);
                        auto const& b_samples_tmp = b_probe.getSamplingResult()->samples;
                        b_samples.resize(b_samples_tmp.size());
                        std::transform(b_samples_tmp.begin(), b_samples_tmp.end(), b_samples.begin(),
                            [](auto const& el) { return glm::vec4(el[0], el[1], el[2], el[3]); });
                        std::for_each(b_samples.begin(), b_samples.end(), [](auto& el) { el = glm::normalize(el); });
                    }
                    std::vector<float> scores(a_samples.size());
                    std::transform(a_samples.begin(), a_samples.end(), b_samples.begin(), scores.begin(),
                        [](auto const& lhs, auto const& rhs) {
                            if (std::isnan(lhs.x) || std::isnan(lhs.y) || std::isnan(lhs.z) || std::isnan(lhs.w) ||
                                std::isnan(rhs.x) || std::isnan(rhs.y) || std::isnan(rhs.z) || std::isnan(rhs.w)) {
                                return 0.0f;
                            }
                            // return glm::dot(lhs, rhs);
                            auto const angle = std::acos(glm::dot(lhs, rhs));
                            auto const angle_dis = angle / 3.14f;
                            return 1.0f - angle_dis;
                        });
                    auto const score = std::accumulate(scores.begin(), scores.end(), 0.0f, std::plus<float>());
                    _dis_mat[a_pidx + b_pidx * probe_count] = score;
                    _dis_mat[b_pidx + a_pidx * probe_count] = score;
                }
            }
        } else {
            core::utility::log::Log::DefaultLog.WriteInfo("[ComputeDistance] Computing distances for scalar probes");
#pragma omp parallel for
            for (std::int64_t a_pidx = 0; a_pidx < probe_count; ++a_pidx) {
                /*std::vector<std::vector<double>> a_samples;
                {
                    auto const a_probe = probe_data->getProbe<FloatProbe>(a_pidx);
                    auto const& a_samples_tmp = a_probe.getSamplingResult()->samples;
                    a_samples.resize(a_samples_tmp.size());
                    double tmp_cnt = 0.0;
                    std::transform(
                        a_samples_tmp.cbegin(), a_samples_tmp.cend(), a_samples.begin(), [&tmp_cnt](auto const val) {
                            return std::vector<double>{tmp_cnt++, static_cast<double>(val)};
                        });
                    auto const it = std::stable_partition(
                        a_samples.begin(), a_samples.end(), [](auto const& el) { return !std::isnan(el[1]); });
                    std::for_each(it, a_samples.end(), [](auto& el) { el[1] = 0.0; });
                }*/
                std::vector<double> a_samples;
                {
                    auto const a_probe = probe_data->getProbe<FloatProbe>(a_pidx);
                    auto const& a_samples_tmp = a_probe.getSamplingResult()->samples;
                    a_samples.resize(a_samples_tmp.size());
                    std::transform(
                        a_samples_tmp.cbegin(), a_samples_tmp.cend(), a_samples.begin(), [](auto const val) {
                            return static_cast<double>(val);
                        });
                    auto const it = std::stable_partition(
                        a_samples.begin(), a_samples.end(), [](auto const& el) { return !std::isnan(el); });
                    std::for_each(it, a_samples.end(), [](auto& el) { el = 0.0; });
                }
                for (std::int64_t b_pidx = a_pidx; b_pidx < probe_count; ++b_pidx) {
                    /*std::vector<std::vector<double>> b_samples;
                    {
                        auto const b_probe = probe_data->getProbe<FloatProbe>(b_pidx);
                        auto const b_samples_tmp = b_probe.getSamplingResult()->samples;
                        b_samples.resize(b_samples_tmp.size());
                        double tmp_cnt = 0.0;
                        std::transform(b_samples_tmp.cbegin(), b_samples_tmp.cend(), b_samples.begin(),
                            [&tmp_cnt](auto const val) {
                                return std::vector<double>{tmp_cnt++, static_cast<double>(val)};
                            });
                        auto const it = std::stable_partition(
                            b_samples.begin(), b_samples.end(), [](auto const& el) { return !std::isnan(el[1]); });
                        std::for_each(it, b_samples.end(), [](auto& el) { el[1] = 0.0; });
                    }*/
                    std::vector<double> b_samples;
                    {
                        auto const b_probe = probe_data->getProbe<FloatProbe>(b_pidx);
                        auto const b_samples_tmp = b_probe.getSamplingResult()->samples;
                        b_samples.resize(b_samples_tmp.size());
                        std::transform(b_samples_tmp.cbegin(), b_samples_tmp.cend(), b_samples.begin(),
                            [](auto const val) {
                                return static_cast<double>(val);
                            });
                        auto const it = std::stable_partition(
                            b_samples.begin(), b_samples.end(), [](auto const& el) { return !std::isnan(el); });
                        std::for_each(it, b_samples.end(), [](auto& el) { el = 0.0; });
                    }
                    auto const dis = stdplugin::datatools::misc::frechet_distance<double>(a_samples, b_samples);
                    //auto const dis = DTW::dtw_distance_only(a_samples, b_samples, 2);
                    _dis_mat[a_pidx + b_pidx * probe_count] = dis;
                    _dis_mat[b_pidx + a_pidx * probe_count] = dis;
                }
            }
        }

        for (std::int64_t a_pidx = 0; a_pidx < probe_count; ++a_pidx) {
            auto const minmax =
                std::minmax_element(&_dis_mat[a_pidx * probe_count], &_dis_mat[a_pidx * probe_count + probe_count]);
            _col_infos[a_pidx].SetName("p" + std::to_string(a_pidx));
            _col_infos[a_pidx].SetType(stdplugin::datatools::table::TableDataCall::ColumnType::QUANTITATIVE);
            _col_infos[a_pidx].SetMinimumValue(*minmax.first);
            _col_infos[a_pidx].SetMaximumValue(*minmax.second);
        }

        _frame_id = meta_data.m_frame_ID;
        ++_out_data_hash;
    }

    out_table->SetFrameCount(meta_data.m_frame_cnt);
    out_table->SetFrameID(_frame_id);
    out_table->SetDataHash(_out_data_hash);
    out_table->Set(_col_count, _row_count, _col_infos.data(), _dis_mat.data());

    return true;
}

bool megamol::probe::ComputeDistance::get_extent_cb(core::Call& c) {
    auto ctd = dynamic_cast<stdplugin::datatools::table::TableDataCall*>(&c);
    if (ctd == nullptr)
        return false;

    auto cpd = _in_probes_slot.CallAs<CallProbes>();
    if (cpd == nullptr)
        return false;

    // get metadata from probes
    auto meta_data = cpd->getMetaData();
    meta_data.m_frame_ID = ctd->GetFrameID();
    cpd->setMetaData(meta_data);

    if (!(*cpd)(1))
        return false;

    // put metadata in table call
    meta_data = cpd->getMetaData();
    ctd->SetFrameCount(meta_data.m_frame_cnt);

    return true;
}
<commit_msg>now just skip the NaNs<commit_after>#include "stdafx.h"
#include "ComputeDistance.h"

#include "DTW.hpp"

#include "glm/glm.hpp"

#include "mmstd_datatools/misc/FrechetDistance.h"
#include "mmstd_datatools/misc/KDE.h"

megamol::probe::ComputeDistance::ComputeDistance()
        : _out_table_slot("outTable", ""), _in_probes_slot("inProbes", "") {
    _out_table_slot.SetCallback(stdplugin::datatools::table::TableDataCall::ClassName(),
        stdplugin::datatools::table::TableDataCall::FunctionName(0), &ComputeDistance::get_data_cb);
    _out_table_slot.SetCallback(stdplugin::datatools::table::TableDataCall::ClassName(),
        stdplugin::datatools::table::TableDataCall::FunctionName(1), &ComputeDistance::get_extent_cb);
    MakeSlotAvailable(&_out_table_slot);

    _in_probes_slot.SetCompatibleCall<CallProbesDescription>();
    MakeSlotAvailable(&_in_probes_slot);
}

megamol::probe::ComputeDistance::~ComputeDistance() {
    this->Release();
}

bool megamol::probe::ComputeDistance::create() {
    return true;
}

void megamol::probe::ComputeDistance::release() {}

bool megamol::probe::ComputeDistance::get_data_cb(core::Call& c) {
    auto out_table = dynamic_cast<stdplugin::datatools::table::TableDataCall*>(&c);
    if (out_table == nullptr)
        return false;
    auto in_probes = _in_probes_slot.CallAs<CallProbes>();
    if (in_probes == nullptr)
        return false;

    if (!(*in_probes)(CallProbes::CallGetMetaData))
        return false;
    if (!(*in_probes)(CallProbes::CallGetData))
        return false;

    auto const& meta_data = in_probes->getMetaData();

    if (in_probes->hasUpdate() || meta_data.m_frame_ID != _frame_id) {
        auto const& probe_data = in_probes->getData();
        auto const probe_count = probe_data->getProbeCount();

        _row_count = probe_count;
        _col_count = probe_count;
        _col_infos.clear();
        _col_infos.resize(probe_count);
        _dis_mat.clear();
        _dis_mat.resize(probe_count * probe_count, 0.0f);

        if (probe_count == 0)
            return false;

        bool vec_probe = false;
        {
            auto test_probe = probe_data->getGenericProbe(0);
            vec_probe = std::holds_alternative<Vec4Probe>(test_probe);
        }

        if (vec_probe) {
            core::utility::log::Log::DefaultLog.WriteInfo("[ComputeDistance] Computing distances for vector probes");
#pragma omp parallel for
            for (std::int64_t a_pidx = 0; a_pidx < probe_count; ++a_pidx) {
                std::vector<glm::vec4> a_samples;
                {
                    auto const a_probe = probe_data->getProbe<Vec4Probe>(a_pidx);
                    auto const& a_samples_tmp = a_probe.getSamplingResult()->samples;
                    a_samples.resize(a_samples_tmp.size());
                    std::transform(a_samples_tmp.begin(), a_samples_tmp.end(), a_samples.begin(),
                        [](auto const& el) { return glm::vec4(el[0], el[1], el[2], el[3]); });
                    std::for_each(a_samples.begin(), a_samples.end(), [](auto& el) { el = glm::normalize(el); });
                }
                for (std::int64_t b_pidx = a_pidx; b_pidx < probe_count; ++b_pidx) {
                    std::vector<glm::vec4> b_samples;
                    {
                        auto const b_probe = probe_data->getProbe<Vec4Probe>(b_pidx);
                        auto const& b_samples_tmp = b_probe.getSamplingResult()->samples;
                        b_samples.resize(b_samples_tmp.size());
                        std::transform(b_samples_tmp.begin(), b_samples_tmp.end(), b_samples.begin(),
                            [](auto const& el) { return glm::vec4(el[0], el[1], el[2], el[3]); });
                        std::for_each(b_samples.begin(), b_samples.end(), [](auto& el) { el = glm::normalize(el); });
                    }
                    std::vector<float> scores(a_samples.size());
                    std::transform(a_samples.begin(), a_samples.end(), b_samples.begin(), scores.begin(),
                        [](auto const& lhs, auto const& rhs) {
                            if (std::isnan(lhs.x) || std::isnan(lhs.y) || std::isnan(lhs.z) || std::isnan(lhs.w) ||
                                std::isnan(rhs.x) || std::isnan(rhs.y) || std::isnan(rhs.z) || std::isnan(rhs.w)) {
                                return 0.0f;
                            }
                            // return glm::dot(lhs, rhs);
                            auto const angle = std::acos(glm::dot(lhs, rhs));
                            auto const angle_dis = angle / 3.14f;
                            return 1.0f - angle_dis;
                        });
                    auto const score = std::accumulate(scores.begin(), scores.end(), 0.0f, std::plus<float>());
                    _dis_mat[a_pidx + b_pidx * probe_count] = score;
                    _dis_mat[b_pidx + a_pidx * probe_count] = score;
                }
            }
        } else {
            core::utility::log::Log::DefaultLog.WriteInfo("[ComputeDistance] Computing distances for scalar probes");
            std::size_t base_skip = 0;
            for (std::int64_t a_pidx = 0; a_pidx < probe_count; ++a_pidx) {
                auto const a_probe = probe_data->getProbe<FloatProbe>(a_pidx);
                auto const& a_samples_tmp = a_probe.getSamplingResult()->samples;
                auto first_not_nan = std::find_if_not(a_samples_tmp.begin(), a_samples_tmp.end(), std::isnan<float>);
                if (first_not_nan != a_samples_tmp.end()) {
                    auto first_not_nan_idx = std::distance(a_samples_tmp.begin(), first_not_nan);
                    if (first_not_nan_idx > base_skip) {
                        base_skip = first_not_nan_idx;
                    }
                }
            }
#pragma omp parallel for
            for (std::int64_t a_pidx = 0; a_pidx < probe_count; ++a_pidx) {
                /*std::vector<std::vector<double>> a_samples;
                {
                    auto const a_probe = probe_data->getProbe<FloatProbe>(a_pidx);
                    auto const& a_samples_tmp = a_probe.getSamplingResult()->samples;
                    a_samples.resize(a_samples_tmp.size());
                    double tmp_cnt = 0.0;
                    std::transform(
                        a_samples_tmp.cbegin(), a_samples_tmp.cend(), a_samples.begin(), [&tmp_cnt](auto const val) {
                            return std::vector<double>{tmp_cnt++, static_cast<double>(val)};
                        });
                    auto const it = std::stable_partition(
                        a_samples.begin(), a_samples.end(), [](auto const& el) { return !std::isnan(el[1]); });
                    std::for_each(it, a_samples.end(), [](auto& el) { el[1] = 0.0; });
                }*/
                std::vector<double> a_samples;
                {
                    auto const a_probe = probe_data->getProbe<FloatProbe>(a_pidx);
                    auto const& a_samples_tmp = a_probe.getSamplingResult()->samples;
                    a_samples.resize(a_samples_tmp.size());
                    std::transform(
                        a_samples_tmp.cbegin(), a_samples_tmp.cend(), a_samples.begin(), [](auto const val) {
                            return static_cast<double>(val);
                        });
                    /*auto const it = std::stable_partition(
                        a_samples.begin(), a_samples.end(), [](auto const& el) { return !std::isnan(el); });
                    std::for_each(it, a_samples.end(), [](auto& el) { el = 0.0; });*/
                    /*auto first_val = std::find_if_not(a_samples.begin(), a_samples.end(), std::isnan<double>);
                    auto last_val = std::find_if_not(a_samples.rbegin(), a_samples.rend(), std::isnan<double>);
                    std::for_each(a_samples.begin(), first_val, [&first_val](auto& el){
                        el = *first_val;});
                    std::for_each(a_samples.rbegin(), last_val, [&last_val](auto& el) { el = *last_val; });
                    auto a_tmp = a_samples;
                    for (size_t idx = 0; idx < a_samples.size(); ++idx) {
                        auto a = idx - 1;
                        if (idx == 0)
                            a = 0;
                        auto b = idx;
                        auto c = idx + 1;
                        if (idx == a_samples.size() - 1)
                            c = a_samples.size() - 1;

                        a_samples[idx] = 0.5 * (0.5 * a_tmp[a] + a_tmp[b] + 0.5 * a_tmp[c]);
                    }*/
                    a_samples.erase(a_samples.begin(), a_samples.begin() + base_skip);
                }
                for (std::int64_t b_pidx = a_pidx; b_pidx < probe_count; ++b_pidx) {
                    /*std::vector<std::vector<double>> b_samples;
                    {
                        auto const b_probe = probe_data->getProbe<FloatProbe>(b_pidx);
                        auto const b_samples_tmp = b_probe.getSamplingResult()->samples;
                        b_samples.resize(b_samples_tmp.size());
                        double tmp_cnt = 0.0;
                        std::transform(b_samples_tmp.cbegin(), b_samples_tmp.cend(), b_samples.begin(),
                            [&tmp_cnt](auto const val) {
                                return std::vector<double>{tmp_cnt++, static_cast<double>(val)};
                            });
                        auto const it = std::stable_partition(
                            b_samples.begin(), b_samples.end(), [](auto const& el) { return !std::isnan(el[1]); });
                        std::for_each(it, b_samples.end(), [](auto& el) { el[1] = 0.0; });
                    }*/
                    std::vector<double> b_samples;
                    {
                        auto const b_probe = probe_data->getProbe<FloatProbe>(b_pidx);
                        auto const b_samples_tmp = b_probe.getSamplingResult()->samples;
                        b_samples.resize(b_samples_tmp.size());
                        std::transform(b_samples_tmp.cbegin(), b_samples_tmp.cend(), b_samples.begin(),
                            [](auto const val) {
                                return static_cast<double>(val);
                            });
                        /*auto const it = std::stable_partition(
                            b_samples.begin(), b_samples.end(), [](auto const& el) { return !std::isnan(el); });
                        std::for_each(it, b_samples.end(), [](auto& el) { el = 0.0; });*/
                        /*auto first_val = std::find_if_not(b_samples.begin(), b_samples.end(), std::isnan<double>);
                        auto last_val = std::find_if_not(b_samples.rbegin(), b_samples.rend(), std::isnan<double>);
                        std::for_each(b_samples.begin(), first_val, [&first_val](auto& el) { el = *first_val; });
                        std::for_each(b_samples.rbegin(), last_val, [&last_val](auto& el) { el = *last_val; });
                        auto b_tmp = b_samples;
                        for (size_t idx = 0; idx < b_samples.size(); ++idx) {
                            auto a = idx - 1;
                            if (idx == 0)
                                a = 0;
                            auto b = idx;
                            auto c = idx + 1;
                            if (idx == b_samples.size() - 1)
                                c = b_samples.size() - 1;

                            b_samples[idx] = 0.5 * (0.5 * b_tmp[a] + b_tmp[b] + 0.5 * b_tmp[c]);
                        }*/
                        b_samples.erase(b_samples.begin(), b_samples.begin() + base_skip);
                    }
                    auto const dis = stdplugin::datatools::misc::frechet_distance<double>(a_samples, b_samples);
                    //auto const dis = DTW::dtw_distance_only(a_samples, b_samples, 2);
                    _dis_mat[a_pidx + b_pidx * probe_count] = dis;
                    _dis_mat[b_pidx + a_pidx * probe_count] = dis;
                }
            }
        }

        for (std::int64_t a_pidx = 0; a_pidx < probe_count; ++a_pidx) {
            auto const minmax =
                std::minmax_element(&_dis_mat[a_pidx * probe_count], &_dis_mat[a_pidx * probe_count + probe_count]);
            _col_infos[a_pidx].SetName("p" + std::to_string(a_pidx));
            _col_infos[a_pidx].SetType(stdplugin::datatools::table::TableDataCall::ColumnType::QUANTITATIVE);
            _col_infos[a_pidx].SetMinimumValue(*minmax.first);
            _col_infos[a_pidx].SetMaximumValue(*minmax.second);
        }

        _frame_id = meta_data.m_frame_ID;
        ++_out_data_hash;
    }

    out_table->SetFrameCount(meta_data.m_frame_cnt);
    out_table->SetFrameID(_frame_id);
    out_table->SetDataHash(_out_data_hash);
    out_table->Set(_col_count, _row_count, _col_infos.data(), _dis_mat.data());

    return true;
}

bool megamol::probe::ComputeDistance::get_extent_cb(core::Call& c) {
    auto ctd = dynamic_cast<stdplugin::datatools::table::TableDataCall*>(&c);
    if (ctd == nullptr)
        return false;

    auto cpd = _in_probes_slot.CallAs<CallProbes>();
    if (cpd == nullptr)
        return false;

    // get metadata from probes
    auto meta_data = cpd->getMetaData();
    meta_data.m_frame_ID = ctd->GetFrameID();
    cpd->setMetaData(meta_data);

    if (!(*cpd)(1))
        return false;

    // put metadata in table call
    meta_data = cpd->getMetaData();
    ctd->SetFrameCount(meta_data.m_frame_cnt);

    return true;
}
<|endoftext|>


<commit_before>/***************************************************************
 *
 * Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

/*
  This file holds utility functions that rely on ClassAds.
*/

#include "condor_common.h"
#include "condor_classad.h"
#include "condor_attributes.h"
#include "sig_name.h"
#include "exit.h"
#include "enum_utils.h"
#include "condor_adtypes.h"
#include "condor_config.h"
#include "filename_tools.h"

/*
  This method is static to this file and shouldn't be used directly.
  it just does the actual work for findSoftKillSig() and
  findRmKillSig().  In here, we lookup the given attribute name, which
  specifies a signal we want to use.  The signal could either be
  stored as an int, or as a string.  if it's a string, we want to
  translate it into the appropriate signal number for this platform.
*/
static int
findSignal( ClassAd* ad, const char* attr_name )
{
	if( ! ad ) {
		return -1;
	}
	char* name = NULL;

	ExprTree *tree, *rhs;
	tree = ad->Lookup( attr_name );
	if(  tree ) {
		rhs = tree->RArg();
		if( ! rhs ) {
				// invalid!
			return -1;
		}
		switch( rhs->MyType() ) {
		case LX_STRING:
				// translate the string version to the local number
				// we'll need to use
			name = ((String *)rhs)->Value();
			return signalNumber( name );
			break;
		case LX_INTEGER:
			return ((Integer *)rhs)->Value();
			break;
		default:
			return -1;
			break;
		}
	}
	return -1;
}

int
findSoftKillSig( ClassAd* ad )
{
	return findSignal( ad, ATTR_KILL_SIG );
}

int
findRmKillSig( ClassAd* ad )
{
	return findSignal( ad, ATTR_REMOVE_KILL_SIG );
}

int
findHoldKillSig( ClassAd* ad )
{
	return findSignal( ad, ATTR_HOLD_KILL_SIG );
}

bool
printExitString( ClassAd* ad, int exit_reason, MyString &str )
{
		// first handle a bunch of cases that don't really need any
		// info from the ClassAd at all.

	switch ( exit_reason ) {

	case JOB_KILLED:
		str += "was removed by the user";
		return true;
		break;

	case JOB_NOT_CKPTED:
		str += "was evicted by condor, without a checkpoint";
		return true;
		break;

	case JOB_NOT_STARTED:
		str += "was never started";
		return true;
		break;

	case JOB_SHADOW_USAGE:
		str += "had incorrect arguments to the condor_shadow ";
		str += "(internal error)";
		return true;
		break;

	case JOB_COREDUMPED:
	case JOB_EXITED:
			// these two need more work...  we handle both below
		break;

	default:
		str += "has a strange exit reason code of ";
		str += exit_reason;
		return true;
		break;

	} // switch()

		// if we're here, it's because we hit JOB_EXITED or
		// JOB_COREDUMPED in the above switch.  now we've got to pull
		// a bunch of info out of the ClassAd to finish our task...

	int int_value;
	bool exited_by_signal = false;
	int exit_value = -1;

		// first grab everything from the ad we must have for this to
		// work at all...
	if( ad->LookupBool(ATTR_ON_EXIT_BY_SIGNAL, int_value) ) {
		if( int_value ) {
			exited_by_signal = true;
		} else {
			exited_by_signal = false;
		}
	} else {
		dprintf( D_ALWAYS, "ERROR in printExitString: %s not found in ad\n",
				 ATTR_ON_EXIT_BY_SIGNAL );
		return false;
	}

	if( exited_by_signal ) {
		if( ad->LookupInteger(ATTR_ON_EXIT_SIGNAL, int_value) ) {
			exit_value = int_value;
		} else {
			dprintf( D_ALWAYS, "ERROR in printExitString: %s is true but "
					 "%s not found in ad\n", ATTR_ON_EXIT_BY_SIGNAL,
					 ATTR_ON_EXIT_SIGNAL);
			return false;
		}
	} else {
		if( ad->LookupInteger(ATTR_ON_EXIT_CODE, int_value) ) {
			exit_value = int_value;
		} else {
			dprintf( D_ALWAYS, "ERROR in printExitString: %s is false but "
					 "%s not found in ad\n", ATTR_ON_EXIT_BY_SIGNAL,
					 ATTR_ON_EXIT_CODE);
			return false;
		}
	}

		// now we can grab all the optional stuff that might help...
	char* ename = NULL;
	int got_exception = ad->LookupString(ATTR_EXCEPTION_NAME, &ename);
	char* reason_str = NULL;
	ad->LookupString( ATTR_EXIT_REASON, &reason_str );

		// finally, construct the right string
	if( exited_by_signal ) {
		if( got_exception ) {
			str += "died with exception ";
			str += ename;
		} else {
			if( reason_str ) {
				str += reason_str;
			} else {
				str += "died on signal ";
				str += exit_value;
			}
		}
	} else {
		str += "exited normally with status ";
		str += exit_value;
	}

	if( ename ) {
		free( ename );
	}
	if( reason_str ) {
		free( reason_str );
	}
	return true;
}

/* Utility function to create a generic job ad. The caller can then fill
 * in the relevant details.
 */
ClassAd *CreateJobAd( const char *owner, int universe, const char *cmd )
{
	ClassAd *job_ad = new ClassAd();

	job_ad->SetMyTypeName(JOB_ADTYPE);
	job_ad->SetTargetTypeName(STARTD_ADTYPE);

	if ( owner ) {
		job_ad->Assign( ATTR_OWNER, owner );
	} else {
		job_ad->AssignExpr( ATTR_OWNER, "Undefined" );
	}
	job_ad->Assign( ATTR_JOB_UNIVERSE, universe );
	job_ad->Assign( ATTR_JOB_CMD, cmd );

	job_ad->Assign( ATTR_Q_DATE, (int)time(NULL) );
	job_ad->Assign( ATTR_COMPLETION_DATE, 0 );

	job_ad->Assign( ATTR_JOB_REMOTE_WALL_CLOCK, (float)0.0 );
	job_ad->Assign( ATTR_JOB_LOCAL_USER_CPU, (float)0.0 );
	job_ad->Assign( ATTR_JOB_LOCAL_SYS_CPU, (float)0.0 );
	job_ad->Assign( ATTR_JOB_REMOTE_USER_CPU, (float)0.0 );
	job_ad->Assign( ATTR_JOB_REMOTE_SYS_CPU, (float)0.0 );

		// This is a magic cookie, see how condor_submit sets it
	job_ad->Assign( ATTR_CORE_SIZE, -1 );

		// Are these ones really necessary?
	job_ad->Assign( ATTR_JOB_EXIT_STATUS, 0 );
	job_ad->Assign( ATTR_ON_EXIT_BY_SIGNAL, false );

	job_ad->Assign( ATTR_NUM_CKPTS, 0 );
	job_ad->Assign( ATTR_NUM_JOB_STARTS, 0 );
	job_ad->Assign( ATTR_NUM_RESTARTS, 0 );
	job_ad->Assign( ATTR_NUM_SYSTEM_HOLDS, 0 );
	job_ad->Assign( ATTR_JOB_COMMITTED_TIME, 0 );
	job_ad->Assign( ATTR_TOTAL_SUSPENSIONS, 0 );
	job_ad->Assign( ATTR_LAST_SUSPENSION_TIME, 0 );
	job_ad->Assign( ATTR_CUMULATIVE_SUSPENSION_TIME, 0 );

	job_ad->Assign( ATTR_JOB_ROOT_DIR, "/" );

	job_ad->Assign( ATTR_MIN_HOSTS, 1 );
	job_ad->Assign( ATTR_MAX_HOSTS, 1 );
	job_ad->Assign( ATTR_CURRENT_HOSTS, 0 );

	job_ad->Assign( ATTR_WANT_REMOTE_SYSCALLS, false );
	job_ad->Assign( ATTR_WANT_CHECKPOINT, false );
	job_ad->Assign( ATTR_WANT_REMOTE_IO, true );

	job_ad->Assign( ATTR_JOB_STATUS, IDLE );
	job_ad->Assign( ATTR_ENTERED_CURRENT_STATUS, (int)time(NULL) );

	job_ad->Assign( ATTR_JOB_PRIO, 0 );
	job_ad->Assign( ATTR_NICE_USER, false );

	job_ad->Assign( ATTR_JOB_ENVIRONMENT1, "" );

	job_ad->Assign( ATTR_JOB_NOTIFICATION, NOTIFY_NEVER );

	job_ad->Assign( ATTR_KILL_SIG, "SIGTERM" );

	job_ad->Assign( ATTR_IMAGE_SIZE, 0 );

	job_ad->Assign( ATTR_JOB_IWD, "/tmp" );
	job_ad->Assign( ATTR_JOB_INPUT, NULL_FILE );
	job_ad->Assign( ATTR_JOB_OUTPUT, NULL_FILE );
	job_ad->Assign( ATTR_JOB_ERROR, NULL_FILE );

		// Not sure what to do with these. If stdin/out/err is unset in the
		// submit file, condor_submit sets In/Out/Err to the NULL_FILE and
		// these transfer attributes to false. Otherwise, it leaves the
		// transfer attributes unset (which is treated as true). If we
		// explicitly set these to false here, our caller needs to reset
		// them to true if it changes In/Out/Err and wants the default
		// behavior of transfering them. This will probably be a common
		// oversite. Leaving them unset should be safe if our caller doesn't
		// change In/Out/Err.
	//job_ad->Assign( ATTR_TRANSFER_INPUT, false );
	//job_ad->Assign( ATTR_TRANSFER_OUTPUT, false );
	//job_ad->Assign( ATTR_TRANSFER_ERROR, false );
	//job_ad->Assign( ATTR_TRANSFER_EXECUTABLE, false );

	job_ad->Assign( ATTR_BUFFER_SIZE, 512*1024 );
	job_ad->Assign( ATTR_BUFFER_BLOCK_SIZE, 32*1024 );

	job_ad->Assign( ATTR_SHOULD_TRANSFER_FILES,
					getShouldTransferFilesString( STF_YES ) );
	job_ad->Assign( ATTR_TRANSFER_FILES, "ONEXIT" );
	job_ad->Assign( ATTR_WHEN_TO_TRANSFER_OUTPUT,
					getFileTransferOutputString( FTO_ON_EXIT ) );

	job_ad->Assign( ATTR_REQUIREMENTS, true );

	job_ad->Assign( ATTR_PERIODIC_HOLD_CHECK, false );
	job_ad->Assign( ATTR_PERIODIC_REMOVE_CHECK, false );
	job_ad->Assign( ATTR_PERIODIC_RELEASE_CHECK, false );

	job_ad->Assign( ATTR_ON_EXIT_HOLD_CHECK, false );
	job_ad->Assign( ATTR_ON_EXIT_REMOVE_CHECK, true );

	job_ad->Assign( ATTR_JOB_ARGUMENTS1, "" );

	job_ad->Assign( ATTR_JOB_LEAVE_IN_QUEUE, false );

	return job_ad;
}

bool getPathToUserLog(ClassAd *job_ad, MyString &result,
					   const char* ulog_path_attr = ATTR_ULOG_FILE)
{
	bool ret_val = true;
	char *global_log = NULL;

	if ( job_ad == NULL ||
	     job_ad->LookupString(ulog_path_attr,result) == 0 )
	{
		// failed to find attribute, check config file
		global_log = param("EVENT_LOG");
		if ( global_log ) {
			// canonicalize to UNIX_NULL_FILE even on Win32
			result = UNIX_NULL_FILE;
		} else {
			ret_val = false;
		}
	}

	if ( global_log ) free(global_log);

	if( ret_val && is_relative_to_cwd(result.Value()) ) {
		MyString iwd;
		if( job_ad->LookupString(ATTR_JOB_IWD,iwd) ) {
			iwd += "/";
			iwd += result;
			result = iwd;
		}
	}

	return ret_val;
}
<commit_msg>fix coverity found bugs<commit_after>/***************************************************************
 *
 * Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

/*
  This file holds utility functions that rely on ClassAds.
*/

#include "condor_common.h"
#include "condor_classad.h"
#include "condor_attributes.h"
#include "sig_name.h"
#include "exit.h"
#include "enum_utils.h"
#include "condor_adtypes.h"
#include "condor_config.h"
#include "filename_tools.h"

/*
  This method is static to this file and shouldn't be used directly.
  it just does the actual work for findSoftKillSig() and
  findRmKillSig().  In here, we lookup the given attribute name, which
  specifies a signal we want to use.  The signal could either be
  stored as an int, or as a string.  if it's a string, we want to
  translate it into the appropriate signal number for this platform.
*/
static int
findSignal( ClassAd* ad, const char* attr_name )
{
	if( ! ad ) {
		return -1;
	}
	char* name = NULL;

	ExprTree *tree, *rhs;
	tree = ad->Lookup( attr_name );
	if(  tree ) {
		rhs = tree->RArg();
		if( ! rhs ) {
				// invalid!
			return -1;
		}
		switch( rhs->MyType() ) {
		case LX_STRING:
				// translate the string version to the local number
				// we'll need to use
			name = ((String *)rhs)->Value();
			return signalNumber( name );
			break;
		case LX_INTEGER:
			return ((Integer *)rhs)->Value();
			break;
		default:
			return -1;
			break;
		}
	}
	return -1;
}

int
findSoftKillSig( ClassAd* ad )
{
	return findSignal( ad, ATTR_KILL_SIG );
}

int
findRmKillSig( ClassAd* ad )
{
	return findSignal( ad, ATTR_REMOVE_KILL_SIG );
}

int
findHoldKillSig( ClassAd* ad )
{
	return findSignal( ad, ATTR_HOLD_KILL_SIG );
}

bool
printExitString( ClassAd* ad, int exit_reason, MyString &str )
{
		// first handle a bunch of cases that don't really need any
		// info from the ClassAd at all.

	switch ( exit_reason ) {

	case JOB_KILLED:
		str += "was removed by the user";
		return true;
		break;

	case JOB_NOT_CKPTED:
		str += "was evicted by condor, without a checkpoint";
		return true;
		break;

	case JOB_NOT_STARTED:
		str += "was never started";
		return true;
		break;

	case JOB_SHADOW_USAGE:
		str += "had incorrect arguments to the condor_shadow ";
		str += "(internal error)";
		return true;
		break;

	case JOB_COREDUMPED:
	case JOB_EXITED:
			// these two need more work...  we handle both below
		break;

	default:
		str += "has a strange exit reason code of ";
		str += exit_reason;
		return true;
		break;

	} // switch()

		// if we're here, it's because we hit JOB_EXITED or
		// JOB_COREDUMPED in the above switch.  now we've got to pull
		// a bunch of info out of the ClassAd to finish our task...

	int int_value;
	bool exited_by_signal = false;
	int exit_value = -1;

		// first grab everything from the ad we must have for this to
		// work at all...
	if( ad->LookupBool(ATTR_ON_EXIT_BY_SIGNAL, int_value) ) {
		if( int_value ) {
			exited_by_signal = true;
		} else {
			exited_by_signal = false;
		}
	} else {
		dprintf( D_ALWAYS, "ERROR in printExitString: %s not found in ad\n",
				 ATTR_ON_EXIT_BY_SIGNAL );
		return false;
	}

	if( exited_by_signal ) {
		if( ad->LookupInteger(ATTR_ON_EXIT_SIGNAL, int_value) ) {
			exit_value = int_value;
		} else {
			dprintf( D_ALWAYS, "ERROR in printExitString: %s is true but "
					 "%s not found in ad\n", ATTR_ON_EXIT_BY_SIGNAL,
					 ATTR_ON_EXIT_SIGNAL);
			return false;
		}
	} else {
		if( ad->LookupInteger(ATTR_ON_EXIT_CODE, int_value) ) {
			exit_value = int_value;
		} else {
			dprintf( D_ALWAYS, "ERROR in printExitString: %s is false but "
					 "%s not found in ad\n", ATTR_ON_EXIT_BY_SIGNAL,
					 ATTR_ON_EXIT_CODE);
			return false;
		}
	}

		// now we can grab all the optional stuff that might help...
	char* ename = NULL;
	int got_exception = ad->LookupString(ATTR_EXCEPTION_NAME, &ename);
	char* reason_str = NULL;
	ad->LookupString( ATTR_EXIT_REASON, &reason_str );

		// finally, construct the right string
	if( exited_by_signal ) {
		if( got_exception ) {
			str += "died with exception ";
			str += ename;
		} else {
			if( reason_str ) {
				str += reason_str;
			} else {
				str += "died on signal ";
				str += exit_value;
			}
		}
	} else {
		str += "exited normally with status ";
		str += exit_value;
	}

	if( ename ) {
		free( ename );
	}
	if( reason_str ) {
		free( reason_str );
	}
	return true;
}

/* Utility function to create a generic job ad. The caller can then fill
 * in the relevant details.
 */
ClassAd *CreateJobAd( const char *owner, int universe, const char *cmd )
{
	ClassAd *job_ad = new ClassAd();

	job_ad->SetMyTypeName(JOB_ADTYPE);
	job_ad->SetTargetTypeName(STARTD_ADTYPE);

	if ( owner ) {
		job_ad->Assign( ATTR_OWNER, owner );
	} else {
		job_ad->AssignExpr( ATTR_OWNER, "Undefined" );
	}
	job_ad->Assign( ATTR_JOB_UNIVERSE, universe );
	job_ad->Assign( ATTR_JOB_CMD, cmd );

	job_ad->Assign( ATTR_Q_DATE, (int)time(NULL) );
	job_ad->Assign( ATTR_COMPLETION_DATE, 0 );

	job_ad->Assign( ATTR_JOB_REMOTE_WALL_CLOCK, (float)0.0 );
	job_ad->Assign( ATTR_JOB_LOCAL_USER_CPU, (float)0.0 );
	job_ad->Assign( ATTR_JOB_LOCAL_SYS_CPU, (float)0.0 );
	job_ad->Assign( ATTR_JOB_REMOTE_USER_CPU, (float)0.0 );
	job_ad->Assign( ATTR_JOB_REMOTE_SYS_CPU, (float)0.0 );

		// This is a magic cookie, see how condor_submit sets it
	job_ad->Assign( ATTR_CORE_SIZE, -1 );

		// Are these ones really necessary?
	job_ad->Assign( ATTR_JOB_EXIT_STATUS, 0 );
	job_ad->Assign( ATTR_ON_EXIT_BY_SIGNAL, false );

	job_ad->Assign( ATTR_NUM_CKPTS, 0 );
	job_ad->Assign( ATTR_NUM_JOB_STARTS, 0 );
	job_ad->Assign( ATTR_NUM_RESTARTS, 0 );
	job_ad->Assign( ATTR_NUM_SYSTEM_HOLDS, 0 );
	job_ad->Assign( ATTR_JOB_COMMITTED_TIME, 0 );
	job_ad->Assign( ATTR_TOTAL_SUSPENSIONS, 0 );
	job_ad->Assign( ATTR_LAST_SUSPENSION_TIME, 0 );
	job_ad->Assign( ATTR_CUMULATIVE_SUSPENSION_TIME, 0 );

	job_ad->Assign( ATTR_JOB_ROOT_DIR, "/" );

	job_ad->Assign( ATTR_MIN_HOSTS, 1 );
	job_ad->Assign( ATTR_MAX_HOSTS, 1 );
	job_ad->Assign( ATTR_CURRENT_HOSTS, 0 );

	job_ad->Assign( ATTR_WANT_REMOTE_SYSCALLS, false );
	job_ad->Assign( ATTR_WANT_CHECKPOINT, false );
	job_ad->Assign( ATTR_WANT_REMOTE_IO, true );

	job_ad->Assign( ATTR_JOB_STATUS, IDLE );
	job_ad->Assign( ATTR_ENTERED_CURRENT_STATUS, (int)time(NULL) );

	job_ad->Assign( ATTR_JOB_PRIO, 0 );
	job_ad->Assign( ATTR_NICE_USER, false );

	job_ad->Assign( ATTR_JOB_ENVIRONMENT1, "" );

	job_ad->Assign( ATTR_JOB_NOTIFICATION, NOTIFY_NEVER );

	job_ad->Assign( ATTR_KILL_SIG, "SIGTERM" );

	job_ad->Assign( ATTR_IMAGE_SIZE, 0 );

	job_ad->Assign( ATTR_JOB_IWD, "/tmp" );
	job_ad->Assign( ATTR_JOB_INPUT, NULL_FILE );
	job_ad->Assign( ATTR_JOB_OUTPUT, NULL_FILE );
	job_ad->Assign( ATTR_JOB_ERROR, NULL_FILE );

		// Not sure what to do with these. If stdin/out/err is unset in the
		// submit file, condor_submit sets In/Out/Err to the NULL_FILE and
		// these transfer attributes to false. Otherwise, it leaves the
		// transfer attributes unset (which is treated as true). If we
		// explicitly set these to false here, our caller needs to reset
		// them to true if it changes In/Out/Err and wants the default
		// behavior of transfering them. This will probably be a common
		// oversite. Leaving them unset should be safe if our caller doesn't
		// change In/Out/Err.
	//job_ad->Assign( ATTR_TRANSFER_INPUT, false );
	//job_ad->Assign( ATTR_TRANSFER_OUTPUT, false );
	//job_ad->Assign( ATTR_TRANSFER_ERROR, false );
	//job_ad->Assign( ATTR_TRANSFER_EXECUTABLE, false );

	job_ad->Assign( ATTR_BUFFER_SIZE, 512*1024 );
	job_ad->Assign( ATTR_BUFFER_BLOCK_SIZE, 32*1024 );

	job_ad->Assign( ATTR_SHOULD_TRANSFER_FILES,
					getShouldTransferFilesString( STF_YES ) );
	job_ad->Assign( ATTR_TRANSFER_FILES, "ONEXIT" );
	job_ad->Assign( ATTR_WHEN_TO_TRANSFER_OUTPUT,
					getFileTransferOutputString( FTO_ON_EXIT ) );

	job_ad->Assign( ATTR_REQUIREMENTS, true );

	job_ad->Assign( ATTR_PERIODIC_HOLD_CHECK, false );
	job_ad->Assign( ATTR_PERIODIC_REMOVE_CHECK, false );
	job_ad->Assign( ATTR_PERIODIC_RELEASE_CHECK, false );

	job_ad->Assign( ATTR_ON_EXIT_HOLD_CHECK, false );
	job_ad->Assign( ATTR_ON_EXIT_REMOVE_CHECK, true );

	job_ad->Assign( ATTR_JOB_ARGUMENTS1, "" );

	job_ad->Assign( ATTR_JOB_LEAVE_IN_QUEUE, false );

	return job_ad;
}

bool getPathToUserLog(ClassAd *job_ad, MyString &result,
					   const char* ulog_path_attr = ATTR_ULOG_FILE)
{
	bool ret_val = true;
	char *global_log = NULL;

	if ( job_ad == NULL ||
	     job_ad->LookupString(ulog_path_attr,result) == 0 )
	{
		// failed to find attribute, check config file
		global_log = param("EVENT_LOG");
		if ( global_log ) {
			// canonicalize to UNIX_NULL_FILE even on Win32
			result = UNIX_NULL_FILE;
		} else {
			ret_val = false;
		}
	}

	if ( global_log ) free(global_log);

	if( ret_val && is_relative_to_cwd(result.Value()) ) {
		MyString iwd;
		if( job_ad && job_ad->LookupString(ATTR_JOB_IWD,iwd) ) {
			iwd += "/";
			iwd += result;
			result = iwd;
		}
	}

	return ret_val;
}
<|endoftext|>


<commit_before>
#include "condor_common.h"
#include "network_matching.h"
#include <classad/matchClassad.h>

#include "condor_attributes.h"
#include "condor_debug.h"
#include "condor_config.h"
#include "../condor_sysapi/sysapi.h"

/*
 * A slight twist on classad's MatchClassAd.
 * This is meant to be a RAII-style class for matchmaking.
 * Once the TemporaryMatchAd is destroyed, the left and right
 * ads are put back into their original state.
 */
class TemporaryMatchAd : public classad::MatchClassAd
{
public:
	TemporaryMatchAd(ClassAd &lad, ClassAd &rad) : MatchClassAd(&lad, &rad)
	{
		if ( !compat_classad::ClassAd::m_strictEvaluation ) {
			lad.alternateScope = &rad;
			rad.alternateScope = &lad;
		}
	}

	virtual ~TemporaryMatchAd()
	{
		classad::ClassAd *ad = RemoveLeftAd();
		ad->alternateScope = NULL;
		ad = RemoveRightAd();
		ad->alternateScope = NULL;
	}
};

/*
 * A RAII-style object for managing the scope of a classad ExprTree.
 */
class ScopeGuard
{
public:
	ScopeGuard(classad::ExprTree &expr, const classad::ClassAd *scope_ptr)
	  : m_orig(expr.GetParentScope()), m_expr(expr), m_new(scope_ptr)
	{
		if (m_new) m_expr.SetParentScope(scope_ptr);
	}
	~ScopeGuard()
	{
		if (m_new) m_expr.SetParentScope(m_orig);
	}

private:
	const classad::ClassAd *m_orig;
	classad::ExprTree &m_expr;
	const classad::ClassAd *m_new;
};

/*
 * Iterate through the list of ClassAds in the schedd ad's attribute "NetworkAds".
 * For each ad, see if its requirements match the given machine ad.
 * If they do, then use the value of BandwidthDownMbps and BandwidthUpMbps in the
 * network ad as the bandwidth estiamte.
 *
 * If there are multiple matches, use the smallest value for bandwidth up/down.
 *
 * After evaluating the network ads in the schedd, evaluate the network ads
 * given in the machine using the same algorithm.
 *
 */
bool
EstimateNetworkBandwidth(const classad::ClassAd &scheddAd, const classad::ClassAd &machineAd, long &bandwidth_down_mbps, long &bandwidth_up_mbps)
{
	// TODO: In the future, use Network ads to estimate
	classad::Value val;
	classad_shared_ptr<classad::ExprList> list;
	bandwidth_down_mbps = -1;
	bandwidth_up_mbps = -1;
	long tmp_bandwidth_down_mbps, tmp_bandwidth_up_mbps;
	if (scheddAd.EvaluateAttr("NetworkAds", val) && val.IsSListValue(list)) {
		for (classad::ExprList::const_iterator it = list->begin(); it != list->end(); it++) {
			if (!(*it)->isClassad()) { continue; }
			TemporaryMatchAd match_ad(*static_cast<ClassAd*>(*it), const_cast<classad::ClassAd&>(machineAd));
			if (!match_ad.leftMatchesRight()) {continue;}
			classad::ClassAd *network_ad = match_ad.GetLeftContext();
			if (network_ad->EvaluateAttrNumber("BandwidthDownMbps", tmp_bandwidth_down_mbps) )
			{
				if ( (bandwidth_down_mbps == -1) || (bandwidth_down_mbps > tmp_bandwidth_down_mbps) ) {
					bandwidth_down_mbps = tmp_bandwidth_down_mbps;
				}
			}
			if (network_ad->EvaluateAttrNumber("BandwidthUpMbps", tmp_bandwidth_up_mbps) )
			{
				if ( (bandwidth_up_mbps == -1) || (bandwidth_up_mbps > tmp_bandwidth_up_mbps) ) {
					bandwidth_up_mbps = tmp_bandwidth_up_mbps;
				}
			}
		}
	}
	if (machineAd.EvaluateAttr("NetworkAds", val) && val.IsSListValue(list)) {
		for (classad::ExprList::const_iterator it = list->begin(); it != list->end(); it++) {
			if (!(*it)->isClassad()) { continue; }
			TemporaryMatchAd match_ad(*static_cast<ClassAd*>(*it), const_cast<classad::ClassAd&>(scheddAd));
			if (!match_ad.leftMatchesRight()) {continue;}
			classad::ClassAd *network_ad = match_ad.GetLeftContext();
			if (network_ad->EvaluateAttrNumber("BandwidthDownMbps", tmp_bandwidth_down_mbps) )
			{
				if ( (bandwidth_down_mbps == -1) || (bandwidth_down_mbps > tmp_bandwidth_down_mbps) ) {
					bandwidth_down_mbps = tmp_bandwidth_down_mbps;
				}
			}
			if (network_ad->EvaluateAttrNumber("BandwidthUpMbps", tmp_bandwidth_up_mbps) )
			{
				if ( (bandwidth_up_mbps == -1) || (bandwidth_up_mbps > tmp_bandwidth_up_mbps) ) {
					bandwidth_up_mbps = tmp_bandwidth_up_mbps;
				}
			}
		}
	}
	return (bandwidth_up_mbps >= 0) && (bandwidth_down_mbps >= 0);
}

bool
IsANetworkMatch(classad::ClassAd &jobAd, classad::ClassAd &machineAd, const classad::ClassAd &scheddAd)
{
	TemporaryMatchAd match_ad(jobAd, machineAd);

	classad::ExprTree *expr = scheddAd.Lookup(ATTR_TRANSFER_QUEUE_USER_EXPR);
	if (!expr) {
		dprintf(D_FULLDEBUG, "Unable to determine the transfer queue expression from the schedd ad.\n");
		return true;
	}
        classad::ClassAd *ad_for_eval = match_ad.GetLeftContext();
	ScopeGuard scope(*expr, ad_for_eval);
	std::string transfer_queue;
	classad::Value v;
	if (!expr->Evaluate(v) || !v.IsStringValue(transfer_queue)) {
		dprintf(D_FULLDEBUG, "Unable to evaluate the transfer queue expression to a string.\n");
		return true;
	}

	long mb_for_download, mb_for_upload;
	if (!scheddAd.EvaluateAttrNumber(transfer_queue + "FileTransferMBWaitingToDownload", mb_for_download))
	{
		dprintf(D_FULLDEBUG, "Unable to determine MB waiting to download for transfer queue %s.\n", transfer_queue.c_str());
		return true;
	}
	if (!scheddAd.EvaluateAttrNumber(transfer_queue + "FileTransferMBWaitingToUpload", mb_for_upload))
	{
		dprintf(D_FULLDEBUG, "Unable to determine MB waiting to upload for transfer queue %s.\n", transfer_queue.c_str());
		return true;
	}
	long bandwidth_down_mbps, bandwidth_up_mbps;
	if (!EstimateNetworkBandwidth(scheddAd, machineAd, bandwidth_down_mbps, bandwidth_up_mbps))
	{
		bandwidth_down_mbps = param_integer("ESTIMATED_BANDWIDTH_DOWN_MBPS", 1000);
		bandwidth_up_mbps = param_integer("ESTIMATED_BANDWIDTH_UP_MBPS", 1000);
		dprintf(D_FULLDEBUG, "Unable to estimate network bandwidth; using defaults of %ld mbps down / %ld mbps up.\n", bandwidth_down_mbps, bandwidth_up_mbps);
	}
	long max_wait = param_integer("MAX_ESTIMATED_NETWORK_WAIT", 600);
	long est_download_wait = mb_for_download * 8 / bandwidth_down_mbps;
	long est_upload_wait = mb_for_upload * 8 / bandwidth_up_mbps;
	if (max_wait < est_download_wait) {
		dprintf(D_FULLDEBUG, "Rejecting match due to insufficient download bandwidth.  Estimated wait is %ld of max %ld.\n", est_download_wait, max_wait);
		return false;
	}
	if (max_wait < est_upload_wait) {
		dprintf(D_FULLDEBUG, "Rejecting match due to insufficient upload bandwidth.  Estimated wait is %ld of max %ld.\n", est_upload_wait, max_wait);
		return false;
	}
	dprintf(D_FULLDEBUG, "Match has sufficient bandwidth.  Estimated bandwidth is %ld Mbps down / %ld Mbps up.  There are %ld and %ld MB in queue %s, for an estimated wait time of %lds for downloads and %lds for uploads, respectively.\n", bandwidth_down_mbps, bandwidth_up_mbps, mb_for_download, mb_for_upload, transfer_queue.c_str(), est_download_wait, est_upload_wait);
	return true;
}

bool PopulateDefaultNetworkAd(classad::ClassAd &ad)
{
	std::vector<NetworkDeviceInfo> devices;
	if (!sysapi_get_network_device_info(devices))
	{
		dprintf(D_ALWAYS, "Unable to determine local network devices\n");
		return false;
	}
	std::vector<NetworkDeviceInfo>::const_iterator it;
	int min_speed = -1;
	for (it = devices.begin(); it != devices.end(); it++)
	{
		if (strcmp(it->name(), "lo")) { continue; }
		unsigned speed = it->link_speed_mbps();
		if (!speed) { continue; }
		if ( (min_speed == -1) || (speed < min_speed)) { min_speed = speed; }
	}
	unsigned up_speed = param_integer("ESTIMATED_BANDWIDTH_UP_MBPS", 0);
	unsigned down_speed = param_integer("ESTIMATED_BANDWIDTH_DOWN_MBPS", 0);
	ad.InsertAttr("BandwidthUpMbps", static_cast<int>((up_speed > 0) ? up_speed : ((min_speed > 0) ? min_speed: 1000)));
	ad.InsertAttr("BandwidthDownMbps", static_cast<int>((down_speed > 0) ? down_speed : ((min_speed > 0) ? min_speed: 1000)));

	return true;
}

<commit_msg>combine job and machine classad as scope guard when evaluating classad attributes & minor fix on attribute names<commit_after>
#include "condor_common.h"
#include "network_matching.h"
#include <classad/matchClassad.h>

#include "condor_attributes.h"
#include "condor_debug.h"
#include "condor_config.h"
#include "../condor_sysapi/sysapi.h"

/*
 * A slight twist on classad's MatchClassAd.
 * This is meant to be a RAII-style class for matchmaking.
 * Once the TemporaryMatchAd is destroyed, the left and right
 * ads are put back into their original state.
 */
class TemporaryMatchAd : public classad::MatchClassAd
{
public:
	TemporaryMatchAd(ClassAd &lad, ClassAd &rad) : MatchClassAd(&lad, &rad)
	{
		if ( !compat_classad::ClassAd::m_strictEvaluation ) {
			lad.alternateScope = &rad;
			rad.alternateScope = &lad;
		}
	}

	virtual ~TemporaryMatchAd()
	{
		classad::ClassAd *ad = RemoveLeftAd();
		ad->alternateScope = NULL;
		ad = RemoveRightAd();
		ad->alternateScope = NULL;
	}
};

/*
 * A RAII-style object for managing the scope of a classad ExprTree.
 */
class ScopeGuard
{
public:
	ScopeGuard(classad::ExprTree &expr, const classad::ClassAd *scope_ptr)
	  : m_orig(expr.GetParentScope()), m_expr(expr), m_new(scope_ptr)
	{
		if (m_new) m_expr.SetParentScope(scope_ptr);
	}
	~ScopeGuard()
	{
		if (m_new) m_expr.SetParentScope(m_orig);
	}

private:
	const classad::ClassAd *m_orig;
	classad::ExprTree &m_expr;
	const classad::ClassAd *m_new;
};

/*
 * Iterate through the list of ClassAds in the schedd ad's attribute "NetworkAds".
 * For each ad, see if its requirements match the given machine ad.
 * If they do, then use the value of BandwidthDownMbps and BandwidthUpMbps in the
 * network ad as the bandwidth estiamte.
 *
 * If there are multiple matches, use the smallest value for bandwidth up/down.
 *
 * After evaluating the network ads in the schedd, evaluate the network ads
 * given in the machine using the same algorithm.
 *
 */
bool
EstimateNetworkBandwidth(const classad::ClassAd &scheddAd, const classad::ClassAd &machineAd, long &bandwidth_down_mbps, long &bandwidth_up_mbps)
{
	// TODO: In the future, use Network ads to estimate
	classad::Value val;
	classad_shared_ptr<classad::ExprList> list;
	bandwidth_down_mbps = -1;
	bandwidth_up_mbps = -1;
	long tmp_bandwidth_down_mbps, tmp_bandwidth_up_mbps;
	if (scheddAd.EvaluateAttr("NetworkAds", val) && val.IsSListValue(list)) {
		for (classad::ExprList::const_iterator it = list->begin(); it != list->end(); it++) {
			if (!(*it)->isClassad()) { continue; }
			TemporaryMatchAd match_ad(*static_cast<ClassAd*>(*it), const_cast<classad::ClassAd&>(machineAd));
			if (!match_ad.leftMatchesRight()) {continue;}
			classad::ClassAd *network_ad = match_ad.GetLeftContext();
			if (network_ad->EvaluateAttrNumber("BandwidthDownMbps", tmp_bandwidth_down_mbps) )
			{
				if ( (bandwidth_down_mbps == -1) || (bandwidth_down_mbps > tmp_bandwidth_down_mbps) ) {
					bandwidth_down_mbps = tmp_bandwidth_down_mbps;
				}
			}
			if (network_ad->EvaluateAttrNumber("BandwidthUpMbps", tmp_bandwidth_up_mbps) )
			{
				if ( (bandwidth_up_mbps == -1) || (bandwidth_up_mbps > tmp_bandwidth_up_mbps) ) {
					bandwidth_up_mbps = tmp_bandwidth_up_mbps;
				}
			}
		}
	}
	if (machineAd.EvaluateAttr("NetworkAds", val) && val.IsSListValue(list)) {
		for (classad::ExprList::const_iterator it = list->begin(); it != list->end(); it++) {
			if (!(*it)->isClassad()) { continue; }
			TemporaryMatchAd match_ad(*static_cast<ClassAd*>(*it), const_cast<classad::ClassAd&>(scheddAd));
			if (!match_ad.leftMatchesRight()) {continue;}
			classad::ClassAd *network_ad = match_ad.GetLeftContext();
			if (network_ad->EvaluateAttrNumber("BandwidthDownMbps", tmp_bandwidth_down_mbps) )
			{
				if ( (bandwidth_down_mbps == -1) || (bandwidth_down_mbps > tmp_bandwidth_down_mbps) ) {
					bandwidth_down_mbps = tmp_bandwidth_down_mbps;
				}
			}
			if (network_ad->EvaluateAttrNumber("BandwidthUpMbps", tmp_bandwidth_up_mbps) )
			{
				if ( (bandwidth_up_mbps == -1) || (bandwidth_up_mbps > tmp_bandwidth_up_mbps) ) {
					bandwidth_up_mbps = tmp_bandwidth_up_mbps;
				}
			}
		}
	}
	return (bandwidth_up_mbps >= 0) && (bandwidth_down_mbps >= 0);
}

bool
IsANetworkMatch(classad::ClassAd &jobAd, classad::ClassAd &machineAd, const classad::ClassAd &scheddAd)
{
	TemporaryMatchAd match_ad(jobAd, machineAd);

	classad::ExprTree *expr = scheddAd.Lookup(ATTR_TRANSFER_QUEUE_USER_EXPR);
	if (!expr) {
		dprintf(D_FULLDEBUG, "Unable to determine the transfer queue expression from the schedd ad.\n");
		return true;
	}
  classad::ClassAd *ad_for_eval = match_ad.GetLeftContext();
  jobAd.ChainToAd(&machineAd);
	ScopeGuard scope(*expr, &jobAd);
	std::string transfer_queue;
	classad::Value v;
	if (!expr->Evaluate(v) || !v.IsStringValue(transfer_queue)) {
		dprintf(D_FULLDEBUG, "Unable to evaluate the transfer queue expression to a string.\n");
    jobAd.Unchain();
		return true;
	}
  jobAd.Unchain();

	long mb_for_download, mb_for_upload;
	if (!scheddAd.EvaluateAttrNumber(transfer_queue + "_FileTransferMBWaitingToDownload", mb_for_download))
	{
		dprintf(D_FULLDEBUG, "Unable to determine MB waiting to download for transfer queue %s.\n", transfer_queue.c_str());
		return true;
	}
	if (!scheddAd.EvaluateAttrNumber(transfer_queue + "_FileTransferMBWaitingToUpload", mb_for_upload))
	{
		dprintf(D_FULLDEBUG, "Unable to determine MB waiting to upload for transfer queue %s.\n", transfer_queue.c_str());
		return true;
	}
	long bandwidth_down_mbps, bandwidth_up_mbps;
	if (!EstimateNetworkBandwidth(scheddAd, machineAd, bandwidth_down_mbps, bandwidth_up_mbps))
	{
		bandwidth_down_mbps = param_integer("ESTIMATED_BANDWIDTH_DOWN_MBPS", 1000);
		bandwidth_up_mbps = param_integer("ESTIMATED_BANDWIDTH_UP_MBPS", 1000);
		dprintf(D_FULLDEBUG, "Unable to estimate network bandwidth; using defaults of %ld mbps down / %ld mbps up.\n", bandwidth_down_mbps, bandwidth_up_mbps);
	}
	long max_wait = param_integer("MAX_ESTIMATED_NETWORK_WAIT", 600);
	long est_download_wait = mb_for_download * 8 / bandwidth_down_mbps;
	long est_upload_wait = mb_for_upload * 8 / bandwidth_up_mbps;
	if (max_wait < est_download_wait) {
		dprintf(D_FULLDEBUG, "Rejecting match due to insufficient download bandwidth.  Estimated wait is %ld of max %ld.\n", est_download_wait, max_wait);
		return false;
	}
	if (max_wait < est_upload_wait) {
		dprintf(D_FULLDEBUG, "Rejecting match due to insufficient upload bandwidth.  Estimated wait is %ld of max %ld.\n", est_upload_wait, max_wait);
		return false;
	}
	dprintf(D_FULLDEBUG, "Match has sufficient bandwidth.  Estimated bandwidth is %ld Mbps down / %ld Mbps up.  There are %ld and %ld MB in queue %s, for an estimated wait time of %lds for downloads and %lds for uploads, respectively.\n", bandwidth_down_mbps, bandwidth_up_mbps, mb_for_download, mb_for_upload, transfer_queue.c_str(), est_download_wait, est_upload_wait);
	return true;
}

bool PopulateDefaultNetworkAd(classad::ClassAd &ad)
{
	std::vector<NetworkDeviceInfo> devices;
	if (!sysapi_get_network_device_info(devices))
	{
		dprintf(D_ALWAYS, "Unable to determine local network devices\n");
		return false;
	}
	std::vector<NetworkDeviceInfo>::const_iterator it;
	int min_speed = -1;
	for (it = devices.begin(); it != devices.end(); it++)
	{
		if (strcmp(it->name(), "lo")) { continue; }
		unsigned speed = it->link_speed_mbps();
		if (!speed) { continue; }
		if ( (min_speed == -1) || (speed < min_speed)) { min_speed = speed; }
	}
	unsigned up_speed = param_integer("ESTIMATED_BANDWIDTH_UP_MBPS", 0);
	unsigned down_speed = param_integer("ESTIMATED_BANDWIDTH_DOWN_MBPS", 0);
	ad.InsertAttr("BandwidthUpMbps", static_cast<int>((up_speed > 0) ? up_speed : ((min_speed > 0) ? min_speed: 1000)));
	ad.InsertAttr("BandwidthDownMbps", static_cast<int>((down_speed > 0) ? down_speed : ((min_speed > 0) ? min_speed: 1000)));

	return true;
}

<|endoftext|>