<commit_before>#ifndef EBITEN_GAME_GRAPHICS_AFFINE_MATRIX_HPP
#define EBITEN_GAME_GRAPHICS_AFFINE_MATRIX_HPP

#include <boost/mpl/size_t.hpp>
#include <boost/noncopyable.hpp>
#include <boost/static_assert.hpp>
#include <array>
#include <cassert>

namespace ebiten {
namespace game {
namespace graphics {

template<class Float, std::size_t Dimension>
class affine_matrix : private boost::noncopyable {
  BOOST_STATIC_ASSERT(0 < Dimension);
private:
  static std::size_t const size = Dimension * (Dimension - 1);
  typedef std::array<Float, size> elements_type;
  elements_type elements_;
public:
  // TODO: accepts iterators
  // TODO: constructor's arugments?
  template<class Elements>
  affine_matrix(Elements const& elements) {
    assert(static_cast<std::size_t>(boost::size(elements)) <= size);
    this->elements_.fill(0);
    std::copy(std::begin(elements), std::end(elements), this->elements_.begin());
  }
  template<std::size_t I, std::size_t J>
  Float
  element() const {
    BOOST_STATIC_ASSERT(I < Dimension);
    BOOST_STATIC_ASSERT(J < Dimension);
    return this->element(boost::mpl::size_t<I>(), boost::mpl::size_t<J>());
  }
  template<std::size_t I, std::size_t J>
  Float
  set_element(Float element) {
    BOOST_STATIC_ASSERT(I < Dimension - 1);
    BOOST_STATIC_ASSERT(J < Dimension);
    return this->elements_[I * Dimension + J] = element;
  }
  bool
  is_identity() const {
    typename elements_type::const_iterator it = this->elements_.begin();
    for (std::size_t i = 0; i < Dimension - 1; ++i) {
      for (std::size_t j = 0; j < Dimension; ++j, ++it) {
        if (i == j) {
          if (*it != 1) {
            return false;
          }
        } else {
          if (*it != 0) {
            return false;
          }
        }
      }
    }
    return true;
  }
private:
  Float
  element(std::size_t i, std::size_t j) const {
    return this->elements_[i * Dimension + j];
  }
  Float
  element(boost::mpl::size_t<Dimension - 1>, std::size_t) const {
    return 0;
  }
  Float
  element(boost::mpl::size_t<Dimension - 1>, boost::mpl::size_t<Dimension - 1>) const {
    return 1;
  }
};

}
}
}

#ifdef EBITEN_TEST

namespace ebiten {
namespace game {
namespace graphics {

BOOST_AUTO_TEST_CASE(affine_matrix_element) {
  affine_matrix<double, 4> m((double[]){1, 2, 3});
  BOOST_CHECK_EQUAL(1, (m.element<0, 0>()));
  BOOST_CHECK_EQUAL(2, (m.element<0, 1>()));
  BOOST_CHECK_EQUAL(3, (m.element<0, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<0, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 2>()));
  BOOST_CHECK_EQUAL(1, (m.element<3, 3>()));
  m.set_element<1, 0>(4);
  m.set_element<1, 1>(5);
  m.set_element<2, 3>(6);
  BOOST_CHECK_EQUAL(1, (m.element<0, 0>()));
  BOOST_CHECK_EQUAL(2, (m.element<0, 1>()));
  BOOST_CHECK_EQUAL(3, (m.element<0, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<0, 3>()));
  BOOST_CHECK_EQUAL(4, (m.element<1, 0>()));
  BOOST_CHECK_EQUAL(5, (m.element<1, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 2>()));
  BOOST_CHECK_EQUAL(6, (m.element<2, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 2>()));
  BOOST_CHECK_EQUAL(1, (m.element<3, 3>()));
}

BOOST_AUTO_TEST_CASE(affine_matrix_is_identity) {
  affine_matrix<double, 4> m1((double[]){1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0});
  BOOST_CHECK(m1.is_identity());
  affine_matrix<double, 4> m2((double[]){1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0});
  BOOST_CHECK(!m2.is_identity());
}

}
}
}

#endif

#endif
<commit_msg>Removed #include<boost/mpl...<commit_after>#ifndef EBITEN_GAME_GRAPHICS_AFFINE_MATRIX_HPP
#define EBITEN_GAME_GRAPHICS_AFFINE_MATRIX_HPP

#include <boost/noncopyable.hpp>
#include <boost/static_assert.hpp>
#include <array>
#include <cassert>

namespace ebiten {
namespace game {
namespace graphics {

template<class Float, std::size_t Dimension>
class affine_matrix : private boost::noncopyable {
  BOOST_STATIC_ASSERT(0 < Dimension);
private:
  static std::size_t const size = Dimension * (Dimension - 1);
  typedef std::array<Float, size> elements_type;
  elements_type elements_;
public:
  // TODO: accepts iterators
  // TODO: constructor's arugments?
  template<class Elements>
  explicit
  affine_matrix(Elements const& elements) {
    assert(static_cast<std::size_t>(boost::size(elements)) <= size);
    this->elements_.fill(0);
    std::copy(std::begin(elements), std::end(elements), this->elements_.begin());
  }
  template<std::size_t I, std::size_t J>
  Float
  element() const {
    BOOST_STATIC_ASSERT(I < Dimension);
    BOOST_STATIC_ASSERT(J < Dimension);
    return this->element_<I, J>();
  }
  template<std::size_t I, std::size_t J>
  Float
  set_element(Float element) {
    BOOST_STATIC_ASSERT(I < Dimension - 1);
    BOOST_STATIC_ASSERT(J < Dimension);
    return this->elements_[I * Dimension + J] = element;
  }
  bool
  is_identity() const {
    typename elements_type::const_iterator it = this->elements_.begin();
    for (std::size_t i = 0; i < Dimension - 1; ++i) {
      for (std::size_t j = 0; j < Dimension; ++j, ++it) {
        if (i == j) {
          if (*it != 1) {
            return false;
          }
        } else {
          if (*it != 0) {
            return false;
          }
        }
      }
    }
    return true;
  }
private:
  template<std::size_t I, std::size_t J>
  Float
  element_() const {
    if (I == Dimension - 1) {
      if (J == Dimension - 1) {
        return 1;
      }
      return 0;
    }
    return this->elements_[I * Dimension + J];
  }
};

}
}
}

#ifdef EBITEN_TEST

namespace ebiten {
namespace game {
namespace graphics {

BOOST_AUTO_TEST_CASE(affine_matrix_element) {
  affine_matrix<double, 4> m((double[]){1, 2, 3});
  BOOST_CHECK_EQUAL(1, (m.element<0, 0>()));
  BOOST_CHECK_EQUAL(2, (m.element<0, 1>()));
  BOOST_CHECK_EQUAL(3, (m.element<0, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<0, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 2>()));
  BOOST_CHECK_EQUAL(1, (m.element<3, 3>()));
  m.set_element<1, 0>(4);
  m.set_element<1, 1>(5);
  m.set_element<2, 3>(6);
  BOOST_CHECK_EQUAL(1, (m.element<0, 0>()));
  BOOST_CHECK_EQUAL(2, (m.element<0, 1>()));
  BOOST_CHECK_EQUAL(3, (m.element<0, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<0, 3>()));
  BOOST_CHECK_EQUAL(4, (m.element<1, 0>()));
  BOOST_CHECK_EQUAL(5, (m.element<1, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 2>()));
  BOOST_CHECK_EQUAL(0, (m.element<1, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<2, 2>()));
  BOOST_CHECK_EQUAL(6, (m.element<2, 3>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 0>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 1>()));
  BOOST_CHECK_EQUAL(0, (m.element<3, 2>()));
  BOOST_CHECK_EQUAL(1, (m.element<3, 3>()));
}

BOOST_AUTO_TEST_CASE(affine_matrix_is_identity) {
  affine_matrix<double, 4> m1((double[]){1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0});
  BOOST_CHECK(m1.is_identity());
  affine_matrix<double, 4> m2((double[]){1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0});
  BOOST_CHECK(!m2.is_identity());
}

}
}
}

#endif

#endif
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: share.hxx,v $
 *
 *  $Revision: 1.5 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-07 22:23:20 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#include "uno/mapping.h"

#include <typeinfo>
#include <exception>
#include <cstddef>

namespace CPPU_CURRENT_NAMESPACE
{

  void dummy_can_throw_anything( char const * );

// ----- following decl from libstdc++-v3/libsupc++/unwind-cxx.h and unwind.h

struct _Unwind_Exception
{
    unsigned exception_class __attribute__((__mode__(__DI__)));
    void * exception_cleanup;
    unsigned private_1 __attribute__((__mode__(__word__)));
    unsigned private_2 __attribute__((__mode__(__word__)));
} __attribute__((__aligned__));

struct __cxa_exception
{
    ::std::type_info *exceptionType;
    void (*exceptionDestructor)(void *);

    ::std::unexpected_handler unexpectedHandler;
    ::std::terminate_handler terminateHandler;

    __cxa_exception *nextException;

    int handlerCount;

    int handlerSwitchValue;
    const unsigned char *actionRecord;
    const unsigned char *languageSpecificData;
    void *catchTemp;
    void *adjustedPtr;

    _Unwind_Exception unwindHeader;
};

extern "C" void *__cxa_allocate_exception(
    std::size_t thrown_size ) throw();
extern "C" void __cxa_throw (
    void *thrown_exception, std::type_info *tinfo, void (*dest) (void *) ) __attribute__((noreturn));

struct __cxa_eh_globals
{
    __cxa_exception *caughtExceptions;
    unsigned int uncaughtExceptions;
};
extern "C" __cxa_eh_globals *__cxa_get_globals () throw();

// -----

//==================================================================================================
void raiseException(
    uno_Any * pUnoExc, uno_Mapping * pUno2Cpp );
//==================================================================================================
void fillUnoException(
    __cxa_exception * header, uno_Any *, uno_Mapping * pCpp2Uno );
}
<commit_msg>INTEGRATION: CWS changefileheader (1.5.170); FILE MERGED 2008/03/28 16:30:13 rt 1.5.170.1: #i87441# Change license header to LPGL v3.<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: share.hxx,v $
 * $Revision: 1.6 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

#include "uno/mapping.h"

#include <typeinfo>
#include <exception>
#include <cstddef>

namespace CPPU_CURRENT_NAMESPACE
{

  void dummy_can_throw_anything( char const * );

// ----- following decl from libstdc++-v3/libsupc++/unwind-cxx.h and unwind.h

struct _Unwind_Exception
{
    unsigned exception_class __attribute__((__mode__(__DI__)));
    void * exception_cleanup;
    unsigned private_1 __attribute__((__mode__(__word__)));
    unsigned private_2 __attribute__((__mode__(__word__)));
} __attribute__((__aligned__));

struct __cxa_exception
{
    ::std::type_info *exceptionType;
    void (*exceptionDestructor)(void *);

    ::std::unexpected_handler unexpectedHandler;
    ::std::terminate_handler terminateHandler;

    __cxa_exception *nextException;

    int handlerCount;

    int handlerSwitchValue;
    const unsigned char *actionRecord;
    const unsigned char *languageSpecificData;
    void *catchTemp;
    void *adjustedPtr;

    _Unwind_Exception unwindHeader;
};

extern "C" void *__cxa_allocate_exception(
    std::size_t thrown_size ) throw();
extern "C" void __cxa_throw (
    void *thrown_exception, std::type_info *tinfo, void (*dest) (void *) ) __attribute__((noreturn));

struct __cxa_eh_globals
{
    __cxa_exception *caughtExceptions;
    unsigned int uncaughtExceptions;
};
extern "C" __cxa_eh_globals *__cxa_get_globals () throw();

// -----

//==================================================================================================
void raiseException(
    uno_Any * pUnoExc, uno_Mapping * pUno2Cpp );
//==================================================================================================
void fillUnoException(
    __cxa_exception * header, uno_Any *, uno_Mapping * pCpp2Uno );
}
<|endoftext|>


<commit_before>// UjoImro, 2013
// Experimental code for the CARP Project
// Copyright (c) RealEyes, 2013
// This version tests the responseMap calculation with input dumps

#include <chrono>
#include <string>
#include <stdlib.h>
#include <opencv2/core/core.hpp>
#include <boost/preprocessor.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/archive/binary_iarchive.hpp>

#include "cast.h"
#include "mlp.hpp"
#include "mlp_impl.h"

/*
extern int EF_ALIGNMENT = 0;
extern int EF_PROTECT_BELOW = 0;
extern int EF_PROTECT_FREE = 0;
extern int EF_ALLOW_MALLOC_0 = 1;
extern int EF_FILL = 1922;
*/

namespace { struct hack_t; }
MatChar  convertCVToMatChar ( const cv::Mat_<uint8_t> & input );
mlp *    convertHackToMlp ( const hack_t & hack );
void     freeClassifiers( mlp * classifiers[], int size );
MatChar  convertCVToMatChar ( const cv::Mat_<uint8_t> & input );
MatFloat convertCVToMatFloat (  const cv::Mat_<double> & input );
cv::Mat_<double> convertMatFloatToCV( MatFloat input );
void     allocateResponseMaps( int mapSize, int size, MatFloat * responseMaps[] );
void     freeResponseMaps( MatFloat * responseMaps[], int size );

namespace {

    struct hack_t {
        int m_visibleLandmarks_size;
        int m_mapSize;

        cv::Mat_<double> shape;
        cv::Mat_<uint8_t> alignedImage;
        std::vector<gel::MLP<double> > m_classifiers;
        std::vector<cv::Mat_<double> > responseMaps;

        template <class MT0>
        void serialize( MT0 & archiver, unsigned int ) {
            GEL_EXPORT_VAR( archiver, m_visibleLandmarks_size );
            GEL_EXPORT_VAR( archiver, m_mapSize );
            GEL_EXPORT_VAR( archiver, shape );
            GEL_EXPORT_VAR( archiver, alignedImage );
            GEL_EXPORT_VAR( archiver, m_classifiers );
            GEL_EXPORT_VAR( archiver, responseMaps );
        } // serialize

    }; // struct hack_t

    class conductor_t {
    public:
        int id;
        hack_t hack;
        std::ifstream dumpStream;
        boost::archive::binary_iarchive importer;

    public:

        conductor_t() : id(0), dumpStream("response_dumps.bin", std::ios::in | std::ios::binary ), importer(dumpStream) {

        }; // conductor_t

    }; // conductor_t

} // unnamed namespace

mlp *
convertHackToMlp ( const hack_t & hack )
{
    assert(hack.m_visibleLandmarks_size==hack.m_classifiers.size());
    assert(hack.m_visibleLandmarks_size==hack.responseMaps.size());

    mlp * result;
    result = reinterpret_cast<mlp*>( malloc( sizeof(mlp) * hack.m_visibleLandmarks_size ) );

    // we export each classifier
    for (int q=0; q<hack.m_visibleLandmarks_size; q++)
    {
        result[q].m_patchSize = hack.m_classifiers[q].m_patchSize;
        result[q].m_wIn       = convertCVToMatFloat(hack.m_classifiers[q].m_wIn);
        result[q].m_wOut      = convertCVToMatFloat(hack.m_classifiers[q].m_wOut);
        result[q].m_U         = convertCVToMatFloat(hack.m_classifiers[q].m_U);
        result[q].hidden_num  = hack.m_classifiers[q].hidden_num;
        result[q].rho2        = hack.m_classifiers[q].rho2;
    } // for q in m_visibleLandmarks_size

    return result;
} // convertHackToMlp

void
freeClassifiers( mlp * classifiers[], int size )
{
    mlp * result = *classifiers;
    for (int q=0; q<size; q++ )
        freeMLP( &(result[q]) );

    free(*classifiers);
    *classifiers=NULL;

    return;
} // freeClassifiers

MatChar convertCVToMatChar ( const cv::Mat_<uint8_t> & input )
{
    MatChar result = CreateMatChar( input.rows, input.cols );

    for ( int q=0; q<input.rows; q++)
        for ( int w=0; w<input.cols; w++ )
            result.data[ q * result.step + w + result.start ] = input(q,w);

    return result;
} // convertCVToMatChar

MatFloat convertCVToMatFloat (  const cv::Mat_<double> & input )
{
    MatFloat result = CreateMatFloat( input.rows, input.cols );

    for ( int q=0; q<input.rows; q++)
        for ( int w=0; w<input.cols; w++ )
            result.data[ q * result.step + w + result.start ] = input(q,w);

    return result;
} // convertCVToMatFloat

cv::Mat_<double> convertMatFloatToCV( MatFloat input )
{
    cv::Mat_<double> result( input.rows, input.cols );

    for ( int q=0; q<input.rows; q++)
        for ( int w=0; w<input.cols; w++ )
            result(q,w) = input.data[ q * input.step + w + input.start ];

    return result;
} // convertMatFloatToCV

void allocateResponseMaps( int mapSize, int size, MatFloat * responseMaps[] )
{
    *responseMaps = (MatFloat*)malloc( sizeof(MatFloat) * size );
    assert(*responseMaps);
    MatFloat * result = *responseMaps;

    for ( int q=0; q<size; q++ )
        result[q] = CreateMatFloat( 2 * mapSize + 1, 2 * mapSize + 1 );

    return;
}

void freeResponseMaps( MatFloat * responseMaps[], int size )
{
    MatFloat * result = *responseMaps;
    assert(result);
    for ( int q=0; q<size; q++ )
        freeMatFloat(&(result[q]));

    free(result);
    *responseMaps=NULL;
    return;
}

template <class T0>
auto
microseconds( T0 t0 ) -> decltype(std::chrono::duration_cast<std::chrono::microseconds>(t0).count())
{
    return std::chrono::duration_cast<std::chrono::microseconds>(t0).count();
}

int main()
{
    conductor_t conductor;
    int fail = 0;
    long int elapsed_time = 0;

    for ( conductor.importer >> BOOST_SERIALIZATION_NVP(conductor.id);
          ((conductor.id != -1) and (conductor.id != 25));
          // conductor.id != -1;
          conductor.importer >> BOOST_SERIALIZATION_NVP(conductor.id)
        )
    {
        PRINT(conductor.id);
        conductor.importer >> BOOST_SERIALIZATION_NVP(conductor.hack);

        // here comes the function call
        {
            // preparing the inputs
            MatChar alignedImage = convertCVToMatChar(conductor.hack.alignedImage);
            MatFloat shape = convertCVToMatFloat(conductor.hack.shape);
            mlp * m_classifiers = convertHackToMlp(conductor.hack);
            MatFloat * responseMaps;
            allocateResponseMaps( conductor.hack.m_mapSize, conductor.hack.m_visibleLandmarks_size, &responseMaps );

            auto start = std::chrono::high_resolution_clock::now();
            calculateMaps(
                conductor.hack.m_visibleLandmarks_size,
                conductor.hack.m_mapSize,
                alignedImage,
                shape,
                m_classifiers,
                &responseMaps
                );
            auto end = std::chrono::high_resolution_clock::now();
            elapsed_time = microseconds(end - start);

            // releasing the inputs
            freeMatChar(&alignedImage);
            freeMatFloat(&shape);
            freeClassifiers(&m_classifiers, conductor.hack.m_classifiers.size());

            // converting the outputs
            std::vector< cv::Mat_<double> > calculatedResults;
            for (int q=0; q<conductor.hack.m_visibleLandmarks_size; q++)
            {
                cv::Mat_<double> nextResult;
                nextResult = convertMatFloatToCV( responseMaps[q] );
                calculatedResults.push_back(nextResult);
            }

            // testing the output
            for (int q=0; q<conductor.hack.m_visibleLandmarks_size; q++)
            {
              // std::cout << "cv::norm( conductor.hack.responseMaps[" << q << "] - calculatedResults[" << q << "] ) = "
              //           << cv::norm( conductor.hack.responseMaps[q] - calculatedResults[q] ) << std::endl;
                assert(cv::norm( conductor.hack.responseMaps[q] - calculatedResults[q] ) < 1.5);
                if (cv::norm( conductor.hack.responseMaps[q] - calculatedResults[q] ) > 0.5 )
                {
                  fail++;
                  std::cout << "cv::norm( conductor.hack.responseMaps[" << q << "] - calculatedResults[" << q << "] ) = "
                            << cv::norm( conductor.hack.responseMaps[q] - calculatedResults[q] ) << std::endl;

                }

                // PRINT(fail);
                int tolerance = 49 * conductor.id * 0.005 + 3;
                // PRINT(tolerance);
                assert(fail < tolerance);
            }

            // releasing the outputs
            freeResponseMaps( &responseMaps, conductor.hack.m_visibleLandmarks_size );

        }
        // here comes the test
        // PRINT(cv::norm( ));
    }

    std::cout << "total elapsed time = " << elapsed_time / 1000000. << " s." << std::endl;
    //conductor.importer >> BOOST_SERIALIZATION_NVP(conductor.hack);

    return EXIT_SUCCESS;
}

// LuM end of file
<commit_msg>testing the mlp<commit_after>// UjoImro, 2013
// Experimental code for the CARP Project
// Copyright (c) RealEyes, 2013
// This version tests the responseMap calculation with input dumps

#include <chrono>
#include <string>
#include <stdlib.h>
#include <opencv2/core/core.hpp>
#include <boost/preprocessor.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/archive/binary_iarchive.hpp>

#include "cast.h"
#include "mlp.hpp"
#include "mlp_impl.h"

extern int EF_ALIGNMENT = 0;
extern int EF_PROTECT_BELOW = 0;
extern int EF_PROTECT_FREE = 0;
extern int EF_ALLOW_MALLOC_0 = 1;
extern int EF_FILL = 1922;

namespace { struct hack_t; }
MatChar  convertCVToMatChar ( const cv::Mat_<uint8_t> & input );
mlp *    convertHackToMlp ( const hack_t & hack );
void     freeClassifiers( mlp * classifiers[], int size );
MatChar  convertCVToMatChar ( const cv::Mat_<uint8_t> & input );
MatFloat convertCVToMatFloat (  const cv::Mat_<double> & input );
cv::Mat_<double> convertMatFloatToCV( MatFloat input );
void     allocateResponseMaps( int mapSize, int size, MatFloat * responseMaps[] );
void     freeResponseMaps( MatFloat * responseMaps[], int size );

namespace {

    struct hack_t {
        int m_visibleLandmarks_size;
        int m_mapSize;

        cv::Mat_<double> shape;
        cv::Mat_<uint8_t> alignedImage;
        std::vector<gel::MLP<double> > m_classifiers;
        std::vector<cv::Mat_<double> > responseMaps;

        template <class MT0>
        void serialize( MT0 & archiver, unsigned int ) {
            GEL_EXPORT_VAR( archiver, m_visibleLandmarks_size );
            GEL_EXPORT_VAR( archiver, m_mapSize );
            GEL_EXPORT_VAR( archiver, shape );
            GEL_EXPORT_VAR( archiver, alignedImage );
            GEL_EXPORT_VAR( archiver, m_classifiers );
            GEL_EXPORT_VAR( archiver, responseMaps );
        } // serialize

    }; // struct hack_t

    class conductor_t {
    public:
        int id;
        hack_t hack;
        std::ifstream dumpStream;
        boost::archive::binary_iarchive importer;

    public:

        conductor_t() : id(0), dumpStream("response_dumps.bin", std::ios::in | std::ios::binary ), importer(dumpStream) {

        }; // conductor_t

    }; // conductor_t

} // unnamed namespace

mlp *
convertHackToMlp ( const hack_t & hack )
{
    assert(hack.m_visibleLandmarks_size==hack.m_classifiers.size());
    assert(hack.m_visibleLandmarks_size==hack.responseMaps.size());

    mlp * result;
    result = reinterpret_cast<mlp*>( malloc( sizeof(mlp) * hack.m_visibleLandmarks_size ) );

    // we export each classifier
    for (int q=0; q<hack.m_visibleLandmarks_size; q++)
    {
        result[q].m_patchSize = hack.m_classifiers[q].m_patchSize;
        result[q].m_wIn       = convertCVToMatFloat(hack.m_classifiers[q].m_wIn);
        result[q].m_wOut      = convertCVToMatFloat(hack.m_classifiers[q].m_wOut);
        result[q].m_U         = convertCVToMatFloat(hack.m_classifiers[q].m_U);
        result[q].hidden_num  = hack.m_classifiers[q].hidden_num;
        result[q].rho2        = hack.m_classifiers[q].rho2;
    } // for q in m_visibleLandmarks_size

    return result;
} // convertHackToMlp

void
freeClassifiers( mlp * classifiers[], int size )
{
    mlp * result = *classifiers;
    for (int q=0; q<size; q++ )
        freeMLP( &(result[q]) );

    free(*classifiers);
    *classifiers=NULL;

    return;
} // freeClassifiers

MatChar convertCVToMatChar ( const cv::Mat_<uint8_t> & input )
{
    MatChar result = CreateMatChar( input.rows, input.cols );

    for ( int q=0; q<input.rows; q++)
        for ( int w=0; w<input.cols; w++ )
            result.data[ q * result.step + w + result.start ] = input(q,w);

    return result;
} // convertCVToMatChar

MatFloat convertCVToMatFloat (  const cv::Mat_<double> & input )
{
    MatFloat result = CreateMatFloat( input.rows, input.cols );

    for ( int q=0; q<input.rows; q++)
        for ( int w=0; w<input.cols; w++ )
            result.data[ q * result.step + w + result.start ] = input(q,w);

    return result;
} // convertCVToMatFloat

cv::Mat_<double> convertMatFloatToCV( MatFloat input )
{
    cv::Mat_<double> result( input.rows, input.cols );

    for ( int q=0; q<input.rows; q++)
        for ( int w=0; w<input.cols; w++ )
            result(q,w) = input.data[ q * input.step + w + input.start ];

    return result;
} // convertMatFloatToCV

void allocateResponseMaps( int mapSize, int size, MatFloat * responseMaps[] )
{
    *responseMaps = (MatFloat*)malloc( sizeof(MatFloat) * size );
    assert(*responseMaps);
    MatFloat * result = *responseMaps;

    for ( int q=0; q<size; q++ )
        result[q] = CreateMatFloat( 2 * mapSize + 1, 2 * mapSize + 1 );

    return;
}

void freeResponseMaps( MatFloat * responseMaps[], int size )
{
    MatFloat * result = *responseMaps;
    assert(result);
    for ( int q=0; q<size; q++ )
        freeMatFloat(&(result[q]));

    free(result);
    *responseMaps=NULL;
    return;
}

template <class T0>
auto
microseconds( T0 t0 ) -> decltype(std::chrono::duration_cast<std::chrono::microseconds>(t0).count())
{
    return std::chrono::duration_cast<std::chrono::microseconds>(t0).count();
}

int main()
{
    conductor_t conductor;
    int fail = 0;
    long int elapsed_time = 0;

    for ( conductor.importer >> BOOST_SERIALIZATION_NVP(conductor.id);
          ((conductor.id != -1) and (conductor.id != 25));
          // conductor.id != -1;
          conductor.importer >> BOOST_SERIALIZATION_NVP(conductor.id)
        )
    {
        PRINT(conductor.id);
        conductor.importer >> BOOST_SERIALIZATION_NVP(conductor.hack);

        // here comes the function call
        {
            // preparing the inputs
            MatChar alignedImage = convertCVToMatChar(conductor.hack.alignedImage);
            MatFloat shape = convertCVToMatFloat(conductor.hack.shape);
            mlp * m_classifiers = convertHackToMlp(conductor.hack);
            MatFloat * responseMaps;
            allocateResponseMaps( conductor.hack.m_mapSize, conductor.hack.m_visibleLandmarks_size, &responseMaps );

            auto start = std::chrono::high_resolution_clock::now();
            calculateMaps(
                conductor.hack.m_visibleLandmarks_size,
                conductor.hack.m_mapSize,
                alignedImage,
                shape,
                m_classifiers,
                &responseMaps
                );
            auto end = std::chrono::high_resolution_clock::now();
            elapsed_time = microseconds(end - start);

            // releasing the inputs
            freeMatChar(&alignedImage);
            freeMatFloat(&shape);
            freeClassifiers(&m_classifiers, conductor.hack.m_classifiers.size());

            // converting the outputs
            std::vector< cv::Mat_<double> > calculatedResults;
            for (int q=0; q<conductor.hack.m_visibleLandmarks_size; q++)
            {
                cv::Mat_<double> nextResult;
                nextResult = convertMatFloatToCV( responseMaps[q] );
                calculatedResults.push_back(nextResult);
            }

            // testing the output
            for (int q=0; q<conductor.hack.m_visibleLandmarks_size; q++)
            {
                 std::cout << "cv::norm( conductor.hack.responseMaps[" << q << "] - calculatedResults[" << q << "] ) = "
                           << cv::norm( conductor.hack.responseMaps[q] - calculatedResults[q] ) << std::endl;
                assert(cv::norm( conductor.hack.responseMaps[q] - calculatedResults[q] ) < 10);
                // if (cv::norm( conductor.hack.responseMaps[q] - calculatedResults[q] ) > 0.5 )
                // {
                //   fail++;
                //   std::cout << "cv::norm( conductor.hack.responseMaps[" << q << "] - calculatedResults[" << q << "] ) = "
                //             << cv::norm( conductor.hack.responseMaps[q] - calculatedResults[q] ) << std::endl;

                // }

                // PRINT(fail);
                int tolerance = 49 * conductor.id * 0.005 + 3;
                // PRINT(tolerance);
                assert(fail < tolerance);
            }

            // releasing the outputs
            freeResponseMaps( &responseMaps, conductor.hack.m_visibleLandmarks_size );

        }
        // here comes the test
        // PRINT(cv::norm( ));
    }

    std::cout << "total elapsed time = " << elapsed_time / 1000000. << " s." << std::endl;
    //conductor.importer >> BOOST_SERIALIZATION_NVP(conductor.hack);

    return EXIT_SUCCESS;
}

// LuM end of file
<|endoftext|>


<commit_before>#include <QString>
#include <functional>
#include <future>
#include <memory>
#include "kernel.h"
#include "coverage.h"
#include "columndefinition.h"
#include "table.h"
#include "attributerecord.h"
#include "polygon.h"
#include "geometry.h"
#include "feature.h"
#include "featurecoverage.h"
#include "symboltable.h"
#include "OperationExpression.h"
#include "operationmetadata.h"
#include "operation.h"
#include "operationhelper.h"
#include "operationhelperfeatures.h"
#include "commandhandler.h"
#include "featureiterator.h"
#include "selectionfeatures.h"

using namespace Ilwis;
using namespace BaseOperations;

SelectionFeatures::SelectionFeatures()
{
}

SelectionFeatures::SelectionFeatures(quint64 metaid, const Ilwis::OperationExpression &expr) : OperationImplementation(metaid, expr)
{
}

SelectionFeatures::~SelectionFeatures()
{
}

bool SelectionFeatures::execute(ExecutionContext *ctx, SymbolTable &symTable)
{
    if (_prepState == sNOTPREPARED)
        if((_prepState = prepare(ctx, symTable)) != sPREPARED)
            return false;

    IFeatureCoverage outputFC = _outputObj.get<FeatureCoverage>();
    IFeatureCoverage inputFC = _inputObj.get<FeatureCoverage>();

    SubSetAsyncFunc selection = [&](const std::vector<quint32>& subset ) -> bool {
        FeatureIterator iterIn(inputFC, subset);

        AttributeRecord rec;
        if ( _attribColumn != "")
            rec = AttributeRecord(inputFC->attributeTable(), FEATUREIDCOLUMN);
        SPAttributeRecord attrib (new AttributeRecord(_attTable, FEATUREIDCOLUMN));
        quint64 v_in = 0;
        for_each(iterIn, iterIn.end(), [&](SPFeatureI feature){
            QVariant v;
            if ( rec.isValid()) {
                SPFeatureI feature = *iterIn;
                v_in = feature->value(FEATUREIDCOLUMN).toULongLong();
                v = rec.cellByKey(v_in,_attribColumn);
            } else {
                v = v_in;
            }
            SPFeatureI newFeature = outputFC->newFeatureFrom(feature);
            if ( !newFeature.isNull()) {
                newFeature->attributeRecord(attrib);
                _attTable->record(NEW_RECORD,{newFeature->featureid(), v});
            }

            ++iterIn;
        }
        );
        return true;
    };
    ctx->_threaded = false;
    bool resource = OperationHelperFeatures::execute(ctx,selection, inputFC, outputFC, _attTable);

    if ( resource && ctx != 0) {
        outputFC->attributeTable(_attTable);
        QVariant value;
        value.setValue<IFeatureCoverage>(outputFC);
        ctx->addOutput(symTable, value, outputFC->name(), itFEATURE,outputFC->source());
    }
    return true;
}

Ilwis::OperationImplementation *SelectionFeatures::create(quint64 metaid, const Ilwis::OperationExpression &expr)
{
    return new SelectionFeatures(metaid, expr);
}

Ilwis::OperationImplementation::State SelectionFeatures::prepare(ExecutionContext *, const SymbolTable &)
{
    IlwisTypes inputType = itFEATURE;
    QString fc = _expression.parm(0).value();
    if (!_inputObj.prepare(fc, inputType)) {
        ERROR2(ERR_COULD_NOT_LOAD_2,fc,"");
        return sPREPAREFAILED;
    }
    IFeatureCoverage inputFC = _inputObj.get<FeatureCoverage>();
    quint64 copylist = itCOORDSYSTEM;

    QString selector = _expression.parm(1).value();
    selector = selector.remove('"');

    int index = selector.indexOf("box=");
    Box2D<double> box;
    if ( index != -1) {
        QString crdlist = "box(" + selector.mid(index+4) + ")";
        _box = Box3D<double>(crdlist);
        copylist |= itDOMAIN | itTABLE;
    }
    index = selector.indexOf("polygon=");
    if ( index != -1)
    {
        //TODO
        copylist |= itDOMAIN | itTABLE;
    }
    index = selector.indexOf("attribute=");
    if ( index != -1 ) {
        if (! inputFC->attributeTable().isValid()) {
            ERROR2(ERR_NO_FOUND2,"attribute-table", "coverage");
            return sPREPAREFAILED;
        }
        _attribColumn =  selector.mid(index+10);
        copylist |= itENVELOPE;
    }

     _outputObj = OperationHelperFeatures::initialize(_inputObj,inputType, copylist);
     if ( !_outputObj.isValid()) {
         ERROR1(ERR_NO_INITIALIZED_1, "output coverage");
         return sPREPAREFAILED;
     }
     QString outputName = _expression.parm(0,false).value();
     if ( outputName != sUNDEF)
         _outputObj->setName(outputName);

     if ( _attribColumn != "") {
         QString url = "ilwis://internal/" + outputName;
         Resource resource(url, itFLATTABLE);
         _attTable.prepare(resource);
         IDomain covdom;
         if (!covdom.prepare("count")){
             return sPREPAREFAILED;
         }
         _attTable->addColumn(FEATUREIDCOLUMN,covdom);
         _attTable->addColumn(_attribColumn, inputFC->attributeTable()->columndefinition(_attribColumn).datadef().domain());
     }
     if ( (_box.isValid() && !_box.isNull()) == 0) {
        //TODO selections in features on bounding box
     }
     return sPREPARED;
}

quint64 SelectionFeatures::createMetadata()
{
    QString url = QString("ilwis://operations/selection");
    Resource resource(QUrl(url), itOPERATIONMETADATA);
    resource.addProperty("namespace","ilwis");
    resource.addProperty("longname","selection");
    resource.addProperty("syntax","selection(featurecoverage,selection-definition)");
    resource.addProperty("description",TR("the operation select parts of the spatial extent or attributes to create a 'smaller' coverage"));
    resource.addProperty("inparameters","2");
    resource.addProperty("pin_1_type", itFEATURE);
    resource.addProperty("pin_1_name", TR("input rastercoverage"));
    resource.addProperty("pin_1_desc",TR("input rastercoverage with a domain as specified by the selection"));
    resource.addProperty("pin_2_type", itSTRING);
    resource.addProperty("pin_2_name", TR("selection-definition"));
    resource.addProperty("pin_2_desc",TR("Selection can either be attribute, layer index or area definition (e.g. box)"));
    resource.addProperty("pout_1_type", itFEATURE);
    resource.addProperty("pout_1_name", TR("rastercoverage were the selection has been applied"));
    resource.addProperty("pout_1_desc",TR(""));
    resource.prepare();
    url += "=" + QString::number(resource.id());
    resource.setUrl(url);

    mastercatalog()->addItems({resource});
    return resource.id();

}
<commit_msg>changes due to changes in the way how features work<commit_after>#include <QString>
#include <functional>
#include <future>
#include <memory>
#include "kernel.h"
#include "coverage.h"
#include "columndefinition.h"
#include "table.h"
#include "attributerecord.h"
#include "polygon.h"
#include "geometry.h"
#include "feature.h"
#include "featurecoverage.h"
#include "symboltable.h"
#include "OperationExpression.h"
#include "operationmetadata.h"
#include "operation.h"
#include "operationhelper.h"
#include "operationhelperfeatures.h"
#include "commandhandler.h"
#include "featureiterator.h"
#include "selectionfeatures.h"

using namespace Ilwis;
using namespace BaseOperations;

SelectionFeatures::SelectionFeatures()
{
}

SelectionFeatures::SelectionFeatures(quint64 metaid, const Ilwis::OperationExpression &expr) : OperationImplementation(metaid, expr)
{
}

SelectionFeatures::~SelectionFeatures()
{
}

bool SelectionFeatures::execute(ExecutionContext *ctx, SymbolTable &symTable)
{
    if (_prepState == sNOTPREPARED)
        if((_prepState = prepare(ctx, symTable)) != sPREPARED)
            return false;

    IFeatureCoverage outputFC = _outputObj.get<FeatureCoverage>();
    IFeatureCoverage inputFC = _inputObj.get<FeatureCoverage>();

    SubSetAsyncFunc selection = [&](const std::vector<quint32>& subset ) -> bool {
        FeatureIterator iterIn(inputFC, subset);

        for_each(iterIn, iterIn.end(), [&](SPFeatureI feature){
            QVariant v = feature->cell(_attribColumn);
            SPFeatureI newFeature = outputFC->newFeatureFrom(feature);
            _attTable->record(NEW_RECORD,{newFeature->featureid(), v});

            ++iterIn;
        }
        );
        return true;
    };
    ctx->_threaded = false;
    bool resource = OperationHelperFeatures::execute(ctx,selection, inputFC, outputFC, _attTable);

    if ( resource && ctx != 0) {
        outputFC->attributeTable(_attTable);
        QVariant value;
        value.setValue<IFeatureCoverage>(outputFC);
        ctx->addOutput(symTable, value, outputFC->name(), itFEATURE,outputFC->source());
    }
    return true;
}

Ilwis::OperationImplementation *SelectionFeatures::create(quint64 metaid, const Ilwis::OperationExpression &expr)
{
    return new SelectionFeatures(metaid, expr);
}

Ilwis::OperationImplementation::State SelectionFeatures::prepare(ExecutionContext *, const SymbolTable &)
{
    IlwisTypes inputType = itFEATURE;
    QString fc = _expression.parm(0).value();
    if (!_inputObj.prepare(fc, inputType)) {
        ERROR2(ERR_COULD_NOT_LOAD_2,fc,"");
        return sPREPAREFAILED;
    }
    IFeatureCoverage inputFC = _inputObj.get<FeatureCoverage>();
    quint64 copylist = itCOORDSYSTEM;

    QString selector = _expression.parm(1).value();
    selector = selector.remove('"');

    int index = selector.indexOf("box=");
    Box2D<double> box;
    if ( index != -1) {
        QString crdlist = "box(" + selector.mid(index+4) + ")";
        _box = Box3D<double>(crdlist);
        copylist |= itDOMAIN | itTABLE;
    }
    index = selector.indexOf("polygon=");
    if ( index != -1)
    {
        //TODO
        copylist |= itDOMAIN | itTABLE;
    }
    index = selector.indexOf("attribute=");
    if ( index != -1 ) {
        if (! inputFC->attributeTable().isValid()) {
            ERROR2(ERR_NO_FOUND2,"attribute-table", "coverage");
            return sPREPAREFAILED;
        }
        _attribColumn =  selector.mid(index+10);
        copylist |= itENVELOPE;
    }

     _outputObj = OperationHelperFeatures::initialize(_inputObj,inputType, copylist);
     if ( !_outputObj.isValid()) {
         ERROR1(ERR_NO_INITIALIZED_1, "output coverage");
         return sPREPAREFAILED;
     }
     QString outputName = _expression.parm(0,false).value();
     if ( outputName != sUNDEF)
         _outputObj->setName(outputName);

     if ( _attribColumn != "") {
         QString url = "ilwis://internal/" + outputName;
         Resource resource(url, itFLATTABLE);
         _attTable.prepare(resource);
         IDomain covdom;
         if (!covdom.prepare("count")){
             return sPREPAREFAILED;
         }
         _attTable->addColumn(FEATUREIDCOLUMN,covdom);
         _attTable->addColumn(_attribColumn, inputFC->attributeTable()->columndefinition(_attribColumn).datadef().domain());
     }
     if ( (_box.isValid() && !_box.isNull()) == 0) {
        //TODO selections in features on bounding box
     }
     return sPREPARED;
}

quint64 SelectionFeatures::createMetadata()
{
    QString url = QString("ilwis://operations/selection");
    Resource resource(QUrl(url), itOPERATIONMETADATA);
    resource.addProperty("namespace","ilwis");
    resource.addProperty("longname","selection");
    resource.addProperty("syntax","selection(featurecoverage,selection-definition)");
    resource.addProperty("description",TR("the operation select parts of the spatial extent or attributes to create a 'smaller' coverage"));
    resource.addProperty("inparameters","2");
    resource.addProperty("pin_1_type", itFEATURE);
    resource.addProperty("pin_1_name", TR("input rastercoverage"));
    resource.addProperty("pin_1_desc",TR("input rastercoverage with a domain as specified by the selection"));
    resource.addProperty("pin_2_type", itSTRING);
    resource.addProperty("pin_2_name", TR("selection-definition"));
    resource.addProperty("pin_2_desc",TR("Selection can either be attribute, layer index or area definition (e.g. box)"));
    resource.addProperty("pout_1_type", itFEATURE);
    resource.addProperty("pout_1_name", TR("rastercoverage were the selection has been applied"));
    resource.addProperty("pout_1_desc",TR(""));
    resource.prepare();
    url += "=" + QString::number(resource.id());
    resource.setUrl(url);

    mastercatalog()->addItems({resource});
    return resource.id();

}
<|endoftext|>


<commit_before>// Copyright 2014 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "scheduling/request-pool-service.h"

#include <list>
#include <string>

#include "common/logging.h"
#include "rpc/thrift-util.h"
#include "util/jni-util.h"
#include "util/parse-util.h"

using namespace std;
using namespace impala;

DEFINE_string(fair_scheduler_allocation_path, "", "Path to the fair scheduler "
    "allocation file (fair-scheduler.xml).");
DEFINE_string(llama_site_path, "", "Path to the Llama configuration file "
    "(llama-site.xml). If set, fair_scheduler_allocation_path must also be set.");

// The default_pool parameters are used if fair scheduler allocation and Llama
// configuration files are not provided.
DEFINE_int64(default_pool_max_requests, -1, "Maximum number of concurrent outstanding "
    "requests allowed to run before queueing incoming requests. A negative value "
    "indicates no limit. 0 indicates no requests will be admitted. Ignored if "
    "fair_scheduler_config_path and llama_site_path are set.");
DEFINE_string(default_pool_mem_limit, "", "Maximum amount of memory that all "
    "outstanding requests in this pool may use before new requests to this pool"
    " are queued. Specified as a number of bytes ('<int>[bB]?'), megabytes "
    "('<float>[mM]'), gigabytes ('<float>[gG]'), or percentage of the physical memory "
    "('<int>%'). 0 or not setting indicates no limit. Defaults to bytes if no unit is "
    "given. Ignored if fair_scheduler_config_path and llama_site_path are set.");
DEFINE_int64(default_pool_max_queued, 0, "Maximum number of requests allowed to be "
    "queued before rejecting requests. A negative value or 0 indicates requests "
    "will always be rejected once the maximum number of concurrent requests are "
    "executing. Ignored if fair_scheduler_config_path and "
    "llama_site_path are set.");

// Flags to disable the pool limits for all pools.
DEFINE_bool(disable_pool_mem_limits, false, "Disables all per-pool mem limits.");
DEFINE_bool(disable_pool_max_requests, false, "Disables all per-pool limits on the "
    "maximum number of running requests.");

DECLARE_bool(enable_rm);

// Pool name used when the configuration files are not specified.
const string DEFAULT_POOL_NAME = "default-pool";

RequestPoolService::RequestPoolService() {
  if (FLAGS_fair_scheduler_allocation_path.empty() &&
      FLAGS_llama_site_path.empty()) {
    if (FLAGS_enable_rm) {
      LOG(ERROR) << "If resource management is enabled, -fair_scheduler_allocation_path "
                 << "is required.";
      exit(1);
    }
    default_pool_only_ = true;
    bool is_percent; // not used
    int64_t bytes_limit = ParseUtil::ParseMemSpec(FLAGS_default_pool_mem_limit,
        &is_percent);
    // -1 indicates an error occurred
    if (bytes_limit < 0) {
      LOG(ERROR) << "Unable to parse default pool mem limit from '"
                 << FLAGS_default_pool_mem_limit << "'.";
      exit(1);
    }
    // 0 indicates no limit or not set
    if (bytes_limit == 0) {
      default_pool_mem_limit_ = -1;
    } else {
      default_pool_mem_limit_ = bytes_limit;
    }
    return;
  }
  default_pool_only_ = false;

  jmethodID start_id; // RequestPoolService.start(), only called in this method.
  JniMethodDescriptor methods[] = {
    {"<init>", "(Ljava/lang/String;Ljava/lang/String;)V", &ctor_},
    {"start", "()V", &start_id},
    {"resolveRequestPool", "([B)[B", &resolve_request_pool_id_},
    {"getPoolConfig", "([B)[B", &get_pool_config_id_}};

  JNIEnv* jni_env = getJNIEnv();
  request_pool_service_class_ =
    jni_env->FindClass("com/cloudera/impala/util/RequestPoolService");
  EXIT_IF_EXC(jni_env);
  uint32_t num_methods = sizeof(methods) / sizeof(methods[0]);
  for (int i = 0; i < num_methods; ++i) {
    EXIT_IF_ERROR(JniUtil::LoadJniMethod(jni_env, request_pool_service_class_,
        &(methods[i])));
  }

  jstring fair_scheduler_config_path =
      jni_env->NewStringUTF(FLAGS_fair_scheduler_allocation_path.c_str());
  EXIT_IF_EXC(jni_env);
  jstring llama_site_path =
      jni_env->NewStringUTF(FLAGS_llama_site_path.c_str());
  EXIT_IF_EXC(jni_env);

  jobject request_pool_service = jni_env->NewObject(request_pool_service_class_, ctor_,
      fair_scheduler_config_path, llama_site_path);
  EXIT_IF_EXC(jni_env);
  EXIT_IF_ERROR(JniUtil::LocalToGlobalRef(jni_env, request_pool_service,
      &request_pool_service_));
  jni_env->CallObjectMethod(request_pool_service_, start_id);
  EXIT_IF_EXC(jni_env);
}

Status RequestPoolService::ResolveRequestPool(const string& requested_pool_name,
    const string& user, TResolveRequestPoolResult* resolved_pool) {
  if (default_pool_only_) {
    resolved_pool->__set_resolved_pool(DEFAULT_POOL_NAME);
    resolved_pool->__set_has_access(true);
    return Status::OK;
  }

  TResolveRequestPoolParams params;
  params.__set_user(user);
  params.__set_requested_pool(requested_pool_name);
  return JniUtil::CallJniMethod(request_pool_service_, resolve_request_pool_id_, params,
      resolved_pool);
}

Status RequestPoolService::GetPoolConfig(const string& pool_name,
    TPoolConfigResult* pool_config) {
  if (default_pool_only_) {
    pool_config->__set_max_requests(
        FLAGS_disable_pool_max_requests ? -1 : FLAGS_default_pool_max_requests);
    pool_config->__set_mem_limit(
        FLAGS_disable_pool_mem_limits ? -1 : default_pool_mem_limit_);
    pool_config->__set_max_queued(FLAGS_default_pool_max_queued);
    return Status::OK;
  }

  TPoolConfigParams params;
  params.__set_pool(pool_name);
  RETURN_IF_ERROR(JniUtil::CallJniMethod(
        request_pool_service_, get_pool_config_id_, params, pool_config));
  if (FLAGS_disable_pool_max_requests) pool_config->__set_max_requests(-1);
  if (FLAGS_disable_pool_mem_limits) pool_config->__set_mem_limit(-1);
  return Status::OK;
}
<commit_msg>Admission controller: Change default values for the "default pool"<commit_after>// Copyright 2014 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "scheduling/request-pool-service.h"

#include <list>
#include <string>

#include "common/logging.h"
#include "rpc/thrift-util.h"
#include "util/jni-util.h"
#include "util/parse-util.h"

using namespace std;
using namespace impala;

DEFINE_string(fair_scheduler_allocation_path, "", "Path to the fair scheduler "
    "allocation file (fair-scheduler.xml).");
DEFINE_string(llama_site_path, "", "Path to the Llama configuration file "
    "(llama-site.xml). If set, fair_scheduler_allocation_path must also be set.");

// The default_pool parameters are used if fair scheduler allocation and Llama
// configuration files are not provided. The default values for this 'default pool'
// are the same as the default values for pools defined via the fair scheduler
// allocation file and Llama configurations.
DEFINE_int64(default_pool_max_requests, 20, "Maximum number of concurrent outstanding "
    "requests allowed to run before queueing incoming requests. A negative value "
    "indicates no limit. 0 indicates no requests will be admitted. Ignored if "
    "fair_scheduler_config_path and llama_site_path are set.");
DEFINE_string(default_pool_mem_limit, "", "Maximum amount of memory that all "
    "outstanding requests in this pool may use before new requests to this pool"
    " are queued. Specified as a number of bytes ('<int>[bB]?'), megabytes "
    "('<float>[mM]'), gigabytes ('<float>[gG]'), or percentage of the physical memory "
    "('<int>%'). 0 or not setting indicates no limit. Defaults to bytes if no unit is "
    "given. Ignored if fair_scheduler_config_path and llama_site_path are set.");
DEFINE_int64(default_pool_max_queued, 50, "Maximum number of requests allowed to be "
    "queued before rejecting requests. A negative value or 0 indicates requests "
    "will always be rejected once the maximum number of concurrent requests are "
    "executing. Ignored if fair_scheduler_config_path and "
    "llama_site_path are set.");

// Flags to disable the pool limits for all pools.
DEFINE_bool(disable_pool_mem_limits, false, "Disables all per-pool mem limits.");
DEFINE_bool(disable_pool_max_requests, false, "Disables all per-pool limits on the "
    "maximum number of running requests.");

DECLARE_bool(enable_rm);

// Pool name used when the configuration files are not specified.
const string DEFAULT_POOL_NAME = "default-pool";

RequestPoolService::RequestPoolService() {
  if (FLAGS_fair_scheduler_allocation_path.empty() &&
      FLAGS_llama_site_path.empty()) {
    if (FLAGS_enable_rm) {
      LOG(ERROR) << "If resource management is enabled, -fair_scheduler_allocation_path "
                 << "is required.";
      exit(1);
    }
    default_pool_only_ = true;
    bool is_percent; // not used
    int64_t bytes_limit = ParseUtil::ParseMemSpec(FLAGS_default_pool_mem_limit,
        &is_percent);
    // -1 indicates an error occurred
    if (bytes_limit < 0) {
      LOG(ERROR) << "Unable to parse default pool mem limit from '"
                 << FLAGS_default_pool_mem_limit << "'.";
      exit(1);
    }
    // 0 indicates no limit or not set
    if (bytes_limit == 0) {
      default_pool_mem_limit_ = -1;
    } else {
      default_pool_mem_limit_ = bytes_limit;
    }
    return;
  }
  default_pool_only_ = false;

  jmethodID start_id; // RequestPoolService.start(), only called in this method.
  JniMethodDescriptor methods[] = {
    {"<init>", "(Ljava/lang/String;Ljava/lang/String;)V", &ctor_},
    {"start", "()V", &start_id},
    {"resolveRequestPool", "([B)[B", &resolve_request_pool_id_},
    {"getPoolConfig", "([B)[B", &get_pool_config_id_}};

  JNIEnv* jni_env = getJNIEnv();
  request_pool_service_class_ =
    jni_env->FindClass("com/cloudera/impala/util/RequestPoolService");
  EXIT_IF_EXC(jni_env);
  uint32_t num_methods = sizeof(methods) / sizeof(methods[0]);
  for (int i = 0; i < num_methods; ++i) {
    EXIT_IF_ERROR(JniUtil::LoadJniMethod(jni_env, request_pool_service_class_,
        &(methods[i])));
  }

  jstring fair_scheduler_config_path =
      jni_env->NewStringUTF(FLAGS_fair_scheduler_allocation_path.c_str());
  EXIT_IF_EXC(jni_env);
  jstring llama_site_path =
      jni_env->NewStringUTF(FLAGS_llama_site_path.c_str());
  EXIT_IF_EXC(jni_env);

  jobject request_pool_service = jni_env->NewObject(request_pool_service_class_, ctor_,
      fair_scheduler_config_path, llama_site_path);
  EXIT_IF_EXC(jni_env);
  EXIT_IF_ERROR(JniUtil::LocalToGlobalRef(jni_env, request_pool_service,
      &request_pool_service_));
  jni_env->CallObjectMethod(request_pool_service_, start_id);
  EXIT_IF_EXC(jni_env);
}

Status RequestPoolService::ResolveRequestPool(const string& requested_pool_name,
    const string& user, TResolveRequestPoolResult* resolved_pool) {
  if (default_pool_only_) {
    resolved_pool->__set_resolved_pool(DEFAULT_POOL_NAME);
    resolved_pool->__set_has_access(true);
    return Status::OK;
  }

  TResolveRequestPoolParams params;
  params.__set_user(user);
  params.__set_requested_pool(requested_pool_name);
  return JniUtil::CallJniMethod(request_pool_service_, resolve_request_pool_id_, params,
      resolved_pool);
}

Status RequestPoolService::GetPoolConfig(const string& pool_name,
    TPoolConfigResult* pool_config) {
  if (default_pool_only_) {
    pool_config->__set_max_requests(
        FLAGS_disable_pool_max_requests ? -1 : FLAGS_default_pool_max_requests);
    pool_config->__set_mem_limit(
        FLAGS_disable_pool_mem_limits ? -1 : default_pool_mem_limit_);
    pool_config->__set_max_queued(FLAGS_default_pool_max_queued);
    return Status::OK;
  }

  TPoolConfigParams params;
  params.__set_pool(pool_name);
  RETURN_IF_ERROR(JniUtil::CallJniMethod(
        request_pool_service_, get_pool_config_id_, params, pool_config));
  if (FLAGS_disable_pool_max_requests) pool_config->__set_max_requests(-1);
  if (FLAGS_disable_pool_mem_limits) pool_config->__set_mem_limit(-1);
  return Status::OK;
}
<|endoftext|>


<commit_before>#include "stochastic_trainer.h"
#include "common/timer.h"
#include "common/logger.h"
#include "common/math.hpp"
#include "common/thread_pool.h"
#include "common/random.hpp"
#include "common/log_search.hpp"
#include "sampler.h"
#include "accumulator.h"

namespace ncv
{
        namespace detail
        {
                ///
                /// \brief tune the regularization factor for a given criterion (if needed)
                ///
                template
                <
                        typename toperator
                >
                static trainer_result_t tune(const toperator& op, const string_t& criterion)
                {
                        if (accumulator_t::can_regularize(criterion))
                        {
                                return log_min_search<toperator, scalar_t>(op, -1.0, +6.0, 0.2);
                        }

                        else
                        {
                                return op(0.0);
                        }
                }

                struct rnd_t
                {
                        rnd_t(random_t<size_t>& gen)
                                :       m_gen(gen)
                        {
                        }

                        size_t operator()(size_t i)
                        {
                                return m_gen() % i;
                        }

                        random_t<size_t>&  m_gen;
                };

                static void stochastic_train(
                        trainer_data_t& data,
                        stochastic_optimizer type, size_t epochs, scalar_t alpha0, scalar_t beta,
                        trainer_result_t& result, thread_pool_t::mutex_t& mutex)
                {
                        samples_t tsamples = data.m_tsampler.get();
                        samples_t vsamples = data.m_vsampler.get();

                        random_t<size_t> xrng(0, tsamples.size());
                        rnd_t xrnd(xrng);

                        timer_t timer;

                        vector_t x = data.m_x0, xparam = x, xavg = x;

                        vector_t gavg(x.size());
                        gavg.setZero();

                        scalar_t alpha = alpha0;
                        scalar_t sumb = 1.0 / alpha;

                        for (size_t e = 0; e < epochs; e ++)
                        {
                                std::random_shuffle(tsamples.begin(), tsamples.end(), xrnd);

                                // one epoch: a pass through all training samples
                                switch (type)
                                {
                                        // stochastic gradient
                                case stochastic_optimizer::SG:
                                        for (size_t i = 0; i < tsamples.size(); i ++, alpha *= beta)
                                        {
                                                data.m_gacc.reset(x);
                                                data.m_gacc.update(data.m_task, tsamples[i], data.m_loss);

                                                x.noalias() -= alpha * data.m_gacc.vgrad();
                                        }
                                        xparam = x;
                                        break;

                                        // stochastic gradient average
                                case stochastic_optimizer::SGA:
                                        for (size_t i = 0; i < tsamples.size(); i ++, alpha *= beta)
                                        {
                                                data.m_gacc.reset(x);
                                                data.m_gacc.update(data.m_task, tsamples[i], data.m_loss);

                                                const vector_t g = data.m_gacc.vgrad();

                                                const scalar_t b = 1.0 / alpha;
                                                gavg = (gavg * sumb + g * b) / (sumb + b);
                                                sumb = sumb + b;

                                                x.noalias() -= alpha * gavg;
                                        }
                                        xparam = x;
                                        break;

                                        // stochastic iterative average
                                case stochastic_optimizer::SIA:
                                default:
                                        for (size_t i = 0; i < tsamples.size(); i ++, alpha *= beta)
                                        {
                                                data.m_gacc.reset(x);
                                                data.m_gacc.update(data.m_task, tsamples[i], data.m_loss);

                                                x.noalias() -= alpha * data.m_gacc.vgrad();

                                                const scalar_t b = 1.0 / alpha;
                                                xavg = (xavg * sumb + x * b) / (sumb + b);
                                                sumb = sumb + b;
                                        }
                                        xparam = xavg;
                                        break;
                                }

                                // evaluate training samples
                                data.m_lacc.reset(xparam);
                                data.m_lacc.update(data.m_task, tsamples, data.m_loss);
                                const scalar_t tvalue = data.m_lacc.value();
                                const scalar_t terror = data.m_lacc.error();

                                // evaluate validation samples
                                data.m_lacc.reset(xparam);
                                data.m_lacc.update(data.m_task, vsamples, data.m_loss);
                                const scalar_t vvalue = data.m_lacc.value();
                                const scalar_t verror = data.m_lacc.error();

                                // OK, update the optimum solution
                                const thread_pool_t::lock_t lock(mutex);

                                result.update(xparam, tvalue, terror, vvalue, verror, e,
                                              scalars_t({ alpha0, data.m_lacc.lambda() }));

                                log_info()
                                        << "[train = " << tvalue << "/" << terror
                                        << ", valid = " << vvalue << "/" << verror
                                        << ", rate = " << alpha << "/" << alpha0
                                        << ", epoch = " << e << "/" << epochs
                                        << ", dims = " << data.m_lacc.psize()
                                        << ", lambda = " << data.m_lacc.lambda()
                                        << "] done in " << timer.elapsed() << ".";
                        }
                }
        }

        trainer_result_t stochastic_train(
                const model_t& model, const task_t& task, const sampler_t& tsampler, const sampler_t& vsampler, size_t nthreads,
                const loss_t& loss, const string_t& criterion,
                stochastic_optimizer optimizer, size_t epochs)
        {
                const auto op = [&] (scalar_t lambda)
                {
                        // prepare workers
                        thread_pool_t wpool(nthreads);
                        thread_pool_t::mutex_t mutex;

                        const size_t iterations = epochs * tsampler.size();             // SGD iterations
                        const scalar_t beta = std::pow(0.01, 1.0 / iterations);         // Learning rate decay rate

                        vector_t x0;
                        model.save_params(x0);

                        trainer_result_t result;

                        // tune the learning rate
                        const scalars_t alphas = { 0.001, 0.010, 0.100 };
                        for (scalar_t alpha : alphas)
                        {
                                wpool.enqueue([=, &task, &loss, &model, &x0, &result, &mutex]()
                                {
                                        accumulator_t lacc(model, 1, criterion, criterion_t::type::value, lambda);
                                        accumulator_t gacc(model, 1, criterion, criterion_t::type::vgrad, lambda);

                                        trainer_data_t data(task, tsampler, vsampler, loss, x0, lacc, gacc);

                                        detail::stochastic_train(data, optimizer, epochs, alpha, beta, result, mutex);
                                });
                        }

                        wpool.wait();

                        // OK
                        return result;
                };

                // tune the regularization factor (if needed)
                return detail::tune(op, criterion);
        }

        stochastic_trainer_t::stochastic_trainer_t(const string_t& parameters)
                :       trainer_t(parameters,
                                  "parameters: opt=sg[,sga,sia],epoch=16[1,1024]")
        {
        }

        trainer_result_t stochastic_trainer_t::train(
                const task_t& task, const fold_t& fold, const loss_t& loss, size_t nthreads, const string_t& criterion,
                model_t& model) const
        {
                if (fold.second != protocol::train)
                {
                        log_error() << "stochastic trainer: can only train models with training samples!";
                        return trainer_result_t();
                }

                // initialize the model
                model.resize(task, true);
                model.random_params();

                // prune training & validation data
                sampler_t tsampler(task);
                tsampler.setup(fold).setup(sampler_t::atype::annotated);

                sampler_t vsampler(task);
                tsampler.split(90, vsampler);

                if (tsampler.empty() || vsampler.empty())
                {
                        log_error() << "stochastic trainer: no annotated training samples!";
                        return trainer_result_t();
                }

                // parameters
                const size_t epochs = math::clamp(text::from_params<size_t>(configuration(), "epoch", 16), 1, 1024);

                const stochastic_optimizer optimizer = text::from_string<stochastic_optimizer>
                        (text::from_params<string_t>(configuration(), "opt", "sg"));

                // train the model
                const trainer_result_t result = ncv::stochastic_train(
                        model, task, tsampler, vsampler, nthreads,
                        loss, criterion, optimizer, epochs);

                log_info() << "optimum [train = " << result.m_opt_state.m_tvalue << "/" << result.m_opt_state.m_terror
                           << ", valid = " << result.m_opt_state.m_vvalue << "/" << result.m_opt_state.m_verror
                           << ", epoch = " << result.m_opt_epoch
                           << ", config = " << text::concatenate(result.m_opt_config, "/")
                           << "].";

                // OK
                if (result.valid())
                {
                        model.load_params(result.m_opt_params);
                }
                return result;
        }
}
<commit_msg>more efficient tuning of the learning rate for stochastic trainer<commit_after>#include "stochastic_trainer.h"
#include "common/timer.h"
#include "common/logger.h"
#include "common/math.hpp"
#include "common/thread_pool.h"
#include "common/random.hpp"
#include "common/log_search.hpp"
#include "sampler.h"
#include "accumulator.h"

namespace ncv
{
        namespace detail
        {
                ///
                /// \brief tune the regularization factor for a given criterion (if needed)
                ///
                template
                <
                        typename toperator
                >
                static trainer_result_t tune(const toperator& op, const string_t& criterion)
                {
                        if (accumulator_t::can_regularize(criterion))
                        {
                                return log_min_search<toperator, scalar_t>(op, -1.0, +6.0, 0.2);
                        }

                        else
                        {
                                return op(0.0);
                        }
                }

                struct rnd_t
                {
                        rnd_t(random_t<size_t>& gen)
                                :       m_gen(gen)
                        {
                        }

                        size_t operator()(size_t i)
                        {
                                return m_gen() % i;
                        }

                        random_t<size_t>&  m_gen;
                };

                static void stochastic_train(
                        trainer_data_t& data,
                        stochastic_optimizer type, size_t epochs, scalar_t alpha0, scalar_t beta,
                        trainer_result_t& result, thread_pool_t::mutex_t& mutex)
                {
                        samples_t tsamples = data.m_tsampler.get();
                        samples_t vsamples = data.m_vsampler.get();

                        random_t<size_t> xrng(0, tsamples.size());
                        rnd_t xrnd(xrng);

                        timer_t timer;

                        vector_t x = data.m_x0, xparam = x, xavg = x;

                        vector_t gavg(x.size());
                        gavg.setZero();

                        scalar_t alpha = alpha0;
                        scalar_t sumb = 1.0 / alpha;

                        for (size_t e = 0; e < epochs; e ++)
                        {
                                std::random_shuffle(tsamples.begin(), tsamples.end(), xrnd);

                                // one epoch: a pass through all training samples
                                switch (type)
                                {
                                        // stochastic gradient
                                case stochastic_optimizer::SG:
                                        for (size_t i = 0; i < tsamples.size(); i ++, alpha *= beta)
                                        {
                                                data.m_gacc.reset(x);
                                                data.m_gacc.update(data.m_task, tsamples[i], data.m_loss);

                                                x.noalias() -= alpha * data.m_gacc.vgrad();
                                        }
                                        xparam = x;
                                        break;

                                        // stochastic gradient average
                                case stochastic_optimizer::SGA:
                                        for (size_t i = 0; i < tsamples.size(); i ++, alpha *= beta)
                                        {
                                                data.m_gacc.reset(x);
                                                data.m_gacc.update(data.m_task, tsamples[i], data.m_loss);

                                                const vector_t g = data.m_gacc.vgrad();

                                                const scalar_t b = 1.0 / alpha;
                                                gavg = (gavg * sumb + g * b) / (sumb + b);
                                                sumb = sumb + b;

                                                x.noalias() -= alpha * gavg;
                                        }
                                        xparam = x;
                                        break;

                                        // stochastic iterative average
                                case stochastic_optimizer::SIA:
                                default:
                                        for (size_t i = 0; i < tsamples.size(); i ++, alpha *= beta)
                                        {
                                                data.m_gacc.reset(x);
                                                data.m_gacc.update(data.m_task, tsamples[i], data.m_loss);

                                                x.noalias() -= alpha * data.m_gacc.vgrad();

                                                const scalar_t b = 1.0 / alpha;
                                                xavg = (xavg * sumb + x * b) / (sumb + b);
                                                sumb = sumb + b;
                                        }
                                        xparam = xavg;
                                        break;
                                }

                                // evaluate training samples
                                data.m_lacc.reset(xparam);
                                data.m_lacc.update(data.m_task, tsamples, data.m_loss);
                                const scalar_t tvalue = data.m_lacc.value();
                                const scalar_t terror = data.m_lacc.error();

                                // evaluate validation samples
                                data.m_lacc.reset(xparam);
                                data.m_lacc.update(data.m_task, vsamples, data.m_loss);
                                const scalar_t vvalue = data.m_lacc.value();
                                const scalar_t verror = data.m_lacc.error();

                                // OK, update the optimum solution
                                const thread_pool_t::lock_t lock(mutex);

                                result.update(xparam, tvalue, terror, vvalue, verror, e,
                                              scalars_t({ alpha0, data.m_lacc.lambda() }));

                                log_info()
                                        << "[train = " << tvalue << "/" << terror
                                        << ", valid = " << vvalue << "/" << verror
                                        << ", rate = " << alpha << "/" << alpha0
                                        << ", epoch = " << e << "/" << epochs
                                        << ", dims = " << data.m_lacc.psize()
                                        << ", lambda = " << data.m_lacc.lambda()
                                        << "] done in " << timer.elapsed() << ".";
                        }
                }
        }

        trainer_result_t stochastic_train(
                const model_t& model, const task_t& task, const sampler_t& tsampler, const sampler_t& vsampler, size_t nthreads,
                const loss_t& loss, const string_t& criterion,
                stochastic_optimizer optimizer, size_t epochs)
        {
                const auto op = [&] (scalar_t lambda)
                {
                        const size_t iterations = epochs * tsampler.size();             // SGD iterations
                        const scalar_t beta = std::pow(0.01, 1.0 / iterations);         // Learning rate decay rate

                        const scalar_t min_log_alpha = -3;
                        const scalar_t max_log_alpha = +0;
                        const scalar_t dif_log_alpha = (max_log_alpha - min_log_alpha) / std::min(size_t(8), nthreads);

                        scalars_t alphas;
                        for (scalar_t log_alpha = min_log_alpha; log_alpha < max_log_alpha + 1e-6; log_alpha += dif_log_alpha)
                        {
                                alphas.push_back(std::exp(log_alpha));
                        }

                        // tune the learning rate (single epoch)
                        vector_t x0;
                        model.save_params(x0);

                        thread_pool_t wpool(nthreads);
                        thread_pool_t::mutex_t mutex;

                        trainer_result_t result;
                        for (scalar_t alpha : alphas)
                        {
                                wpool.enqueue([=, &task, &loss, &model, &x0, &result, &mutex]()
                                {
                                        accumulator_t lacc(model, 1, criterion, criterion_t::type::value, lambda);
                                        accumulator_t gacc(model, 1, criterion, criterion_t::type::vgrad, lambda);

                                        trainer_data_t data(task, tsampler, vsampler, loss, x0, lacc, gacc);

                                        detail::stochastic_train(data, optimizer, 1, alpha, beta, result, mutex);
                                });
                        }

                        wpool.wait();

                        // train with the optimum learning rate (multiple epochs)
                        const scalar_t opt_alpha = result.m_opt_config[0];
                        log_info() << "optimum learning rate = " << opt_alpha << ".";

                        result = trainer_result_t();

                        accumulator_t lacc(model, 1, criterion, criterion_t::type::value, lambda);
                        accumulator_t gacc(model, 1, criterion, criterion_t::type::vgrad, lambda);

                        trainer_data_t data(task, tsampler, vsampler, loss, x0, lacc, gacc);

                        detail::stochastic_train(data, optimizer, epochs, opt_alpha, beta, result, mutex);

                        // OK
                        return result;
                };

                // tune the regularization factor (if needed)
                return detail::tune(op, criterion);
        }

        stochastic_trainer_t::stochastic_trainer_t(const string_t& parameters)
                :       trainer_t(parameters,
                                  "parameters: opt=sg[,sga,sia],epoch=16[1,1024]")
        {
        }

        trainer_result_t stochastic_trainer_t::train(
                const task_t& task, const fold_t& fold, const loss_t& loss, size_t nthreads, const string_t& criterion,
                model_t& model) const
        {
                if (fold.second != protocol::train)
                {
                        log_error() << "stochastic trainer: can only train models with training samples!";
                        return trainer_result_t();
                }

                // initialize the model
                model.resize(task, true);
                model.random_params();

                // prune training & validation data
                sampler_t tsampler(task);
                tsampler.setup(fold).setup(sampler_t::atype::annotated);

                sampler_t vsampler(task);
                tsampler.split(90, vsampler);

                if (tsampler.empty() || vsampler.empty())
                {
                        log_error() << "stochastic trainer: no annotated training samples!";
                        return trainer_result_t();
                }

                // parameters
                const size_t epochs = math::clamp(text::from_params<size_t>(configuration(), "epoch", 16), 1, 1024);

                const stochastic_optimizer optimizer = text::from_string<stochastic_optimizer>
                        (text::from_params<string_t>(configuration(), "opt", "sg"));

                // train the model
                const trainer_result_t result = ncv::stochastic_train(
                        model, task, tsampler, vsampler, nthreads,
                        loss, criterion, optimizer, epochs);

                log_info() << "optimum [train = " << result.m_opt_state.m_tvalue << "/" << result.m_opt_state.m_terror
                           << ", valid = " << result.m_opt_state.m_vvalue << "/" << result.m_opt_state.m_verror
                           << ", epoch = " << result.m_opt_epoch
                           << ", config = " << text::concatenate(result.m_opt_config, "/")
                           << "].";

                // OK
                if (result.valid())
                {
                        model.load_params(result.m_opt_params);
                }
                return result;
        }
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "pyiree/rt/status_utils.h"

#include "absl/strings/str_cat.h"

namespace iree {
namespace python {

namespace {

PyObject* ApiStatusToPyExcClass(iree_status_t status) {
  switch (iree_status_code(status)) {
    case IREE_STATUS_INVALID_ARGUMENT:
      return PyExc_ValueError;
    case IREE_STATUS_OUT_OF_RANGE:
      return PyExc_IndexError;
    case IREE_STATUS_UNIMPLEMENTED:
      return PyExc_NotImplementedError;
    default:
      return PyExc_RuntimeError;
  }
}

}  // namespace

pybind11::error_already_set ApiStatusToPyExc(iree_status_t status,
                                             const char* message) {
  assert(!iree_status_is_ok(status));
  auto full_message = absl::StrCat(
      message, ": ", iree_status_code_string(iree_status_code(status)));
  PyErr_SetString(ApiStatusToPyExcClass(status), full_message.c_str());
  iree_status_ignore(status);
  return pybind11::error_already_set();
}

pybind11::error_already_set RaisePyError(PyObject* exc_class,
                                         const char* message) {
  PyErr_SetString(exc_class, message);
  return pybind11::error_already_set();
}

}  // namespace python
}  // namespace iree
<commit_msg>Python error messages print the full IREE status if available.<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "pyiree/rt/status_utils.h"

#include "absl/strings/str_cat.h"

namespace iree {
namespace python {

namespace {

PyObject* ApiStatusToPyExcClass(iree_status_t status) {
  switch (iree_status_code(status)) {
    case IREE_STATUS_INVALID_ARGUMENT:
      return PyExc_ValueError;
    case IREE_STATUS_OUT_OF_RANGE:
      return PyExc_IndexError;
    case IREE_STATUS_UNIMPLEMENTED:
      return PyExc_NotImplementedError;
    default:
      return PyExc_RuntimeError;
  }
}

}  // namespace

pybind11::error_already_set ApiStatusToPyExc(iree_status_t status,
                                             const char* message) {
  assert(!iree_status_is_ok(status));
  std::string full_message;

  char* iree_message;
  size_t iree_message_length;
  if (iree_status_to_string(status, &iree_message, &iree_message_length)) {
    full_message = absl::StrCat(
        message, ": ", absl::string_view(iree_message, iree_message_length));
    iree_allocator_free(iree_allocator_system(), iree_message);
  } else {
    full_message = absl::StrCat(
        message, ": ", iree_status_code_string(iree_status_code(status)));
  }

  PyErr_SetString(ApiStatusToPyExcClass(status), full_message.c_str());
  iree_status_ignore(status);
  return pybind11::error_already_set();
}

pybind11::error_already_set RaisePyError(PyObject* exc_class,
                                         const char* message) {
  PyErr_SetString(exc_class, message);
  return pybind11::error_already_set();
}

}  // namespace python
}  // namespace iree
<|endoftext|>