<commit_before>/*
 * opencog/learning/moses/moses/scoring.cc
 *
 * Copyright (C) 2002-2008 Novamente LLC
 * All Rights Reserved
 *
 * Written by Moshe Looks, Nil Geisweiller
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
#include "scoring.h"

#include <cmath>

#include <boost/range/algorithm/sort.hpp>
#include <boost/range/algorithm/for_each.hpp>
#include <boost/range/algorithm_ext/for_each.hpp>
#include <boost/range/algorithm/transform.hpp>
#include <boost/range/adaptor/map.hpp>
#include <boost/range/adaptor/filtered.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>

#include <opencog/util/numeric.h>
#include <opencog/util/KLD.h>
#include <opencog/util/MannWhitneyU.h>

namespace opencog { namespace moses {

using namespace std;
using boost::adaptors::map_values;
using boost::adaptors::map_keys;
using boost::adaptors::filtered;
using boost::transform;
using namespace boost::phoenix;
using boost::phoenix::arg_names::arg1;
using namespace boost::accumulators;

// helper to log a combo_tree and its behavioral score
inline void log_candidate_bscore(const combo_tree& tr,
                                 const behavioral_score& bs)
{
    if (logger().getLevel() < Logger::FINE)
        return;

    stringstream ss;
    ss << "Evaluate candidate: " << tr << "\n";
    ss << "\tBScored: " << bs;
    logger().fine(ss.str());
}

////////////////////
// logical_bscore //
////////////////////

behavioral_score logical_bscore::operator()(const combo_tree& tr) const
{
    combo::complete_truth_table tt(tr, arity);
    behavioral_score bs(target.size());

    boost::transform(tt, target, bs.begin(), [](bool b1, bool b2) {
            return -score_t(b1 != b2); });

    return bs;
}

behavioral_score logical_bscore::best_possible_bscore() const {
    return behavioral_score(target.size(), 0);
}

///////////////////
// contin_bscore //
///////////////////

score_t contin_complexity_coef(unsigned alphabet_size, double stdev) {
    return log(alphabet_size) * 2 * sq(stdev);
}

behavioral_score contin_bscore::operator()(const combo_tree& tr) const
{
    OTable ct(tr, cti, rng);
    behavioral_score bs(target.size() + (occam?1:0));
    boost::transform(ct, target, bs.begin(),
                     [](const vertex& vl, const vertex& vr) {
                         return -sq(get_contin(vl) - get_contin(vr)); });
    // add the Occam's razor feature
    if(occam)
        bs.back() = complexity(tr) * complexity_coef;

    // Logger
    log_candidate_bscore(tr, bs);
    // ~Logger

    return bs;
}

behavioral_score contin_bscore::best_possible_bscore() const {
    return behavioral_score(target.size() + (occam?1:0), 0);
}

void contin_bscore::set_complexity_coef(float alphabet_size, float stdev)
{
    if(occam)
        complexity_coef = contin_complexity_coef(alphabet_size, stdev);
}

//////////////////////////////
// discretize_contin_bscore //
//////////////////////////////

score_t discrete_complexity_coef(unsigned alphabet_size, double p) {
    return -log((double)alphabet_size) / log(p/(1-p));
}

discretize_contin_bscore::discretize_contin_bscore(const OTable& ot,
                                                   const ITable& it,
                                                   const vector<contin_t>& thres,
                                                   bool wa,
                                                   float alphabet_size,
                                                   float p,
                                                   RandGen& _rng)
    : target(ot), cit(it), thresholds(thres), weighted_accuracy(wa), rng(_rng),
      classes(ot.size()), weights(thresholds.size() + 1, 1) {
    // enforce that thresholds is sorted
    boost::sort(thresholds);
    // precompute classes
    boost::transform(target, classes.begin(), [&](const vertex& v) {
            return this->class_idx(get_contin(v)); });
    // precompute weights
    multiset<size_t> cs(classes.begin(), classes.end());
    if(weighted_accuracy)
        for(size_t i = 0; i < weights.size(); ++i)
            weights[i] = classes.size() / (float)(weights.size() * cs.count(i));
    // precompute Occam's razor coefficient
    occam = p > 0 && p < 0.5;
    if(occam)
        complexity_coef = discrete_complexity_coef(alphabet_size, p);
}

behavioral_score discretize_contin_bscore::best_possible_bscore() const {
    return behavioral_score(target.size(), 0);
}

size_t discretize_contin_bscore::class_idx(contin_t v) const {
    if(v < thresholds[0])
        return 0;
    size_t s = thresholds.size();
    if(v >= thresholds[s - 1])
        return s;
    return class_idx_within(v, 1, s);
}

size_t discretize_contin_bscore::class_idx_within(contin_t v,
                                                  size_t l_idx,
                                                  size_t u_idx) const
{
    // base case
    if(u_idx - l_idx == 1)
        return l_idx;
    // recursive case
    size_t m_idx = l_idx + (u_idx - l_idx) / 2;
    contin_t t = thresholds[m_idx - 1];
    if(v < t)
        return class_idx_within(v, l_idx, m_idx);
    else
        return class_idx_within(v, m_idx, u_idx);
}

behavioral_score discretize_contin_bscore::operator()(const combo_tree& tr) const
{
    OTable ct(tr, cit, rng);
    behavioral_score bs(target.size() + (occam?1:0));
    boost::transform(ct, classes, bs.begin(), [&](const vertex& v, size_t c_idx) {
            return (c_idx != this->class_idx(get_contin(v))) * this->weights[c_idx];
        });
    // add the Occam's razor feature
    if(occam)
        bs.back() = complexity(tr) * complexity_coef;

    // Logger
    log_candidate_bscore(tr, bs);
    // ~Logger

    return bs;
}

/////////////////////////
// ctruth_table_bscore //
/////////////////////////

ctruth_table_bscore::ctruth_table_bscore(const CTable& _ctt,
                                         float alphabet_size, float p,
                                         RandGen& _rng)
    : ctt(_ctt), rng(_rng)
{
    // Both p==0.0 and p==0.5 are singularity points in the Occam's
    // razor formula for discrete outputs (see the explanation in the
    // comment above ctruth_table_bscore)
    occam = p > 0.0f && p < 0.5f;
    if (occam)
        complexity_coef = discrete_complexity_coef(alphabet_size, p);
}

behavioral_score ctruth_table_bscore::operator()(const combo_tree& tr) const
{
    // The OTable constructor will take the combo tree tr and evaluate
    // it for every row of input in ctt. (The rng is passed straight
    // through to the combo evaluator).
    OTable ptt(tr, ctt, rng);
    behavioral_score bs(ctt.size() + (occam?1:0));

    transform(ptt, ctt | map_values, bs.begin(),
              [](const vertex& v, CTable::mapped_type& vd) {
                  return -score_t(vd[negate_vertex(v)]); });

    // Add the Occam's razor feature
    if (occam)
        bs.back() = complexity(tr) * complexity_coef;

    log_candidate_bscore(tr, bs);

    return bs;
}

behavioral_score ctruth_table_bscore::best_possible_bscore() const
{
    behavioral_score bs(ctt.size() + (occam?1:0));
    transform(ctt | map_values, bs.begin(), [](CTable::mapped_type& vd) {
            return -score_t(min(vd[id::logical_true], vd[id::logical_false])); });
    // add the Occam's razor feature
    if(occam)
        bs.back() = 0;
    return bs;
}

//////////////////////////////////
// interesting_predicate_bscore //
//////////////////////////////////

interesting_predicate_bscore::interesting_predicate_bscore(const CTable& ctable_,
                                                           float alphabet_size,
                                                           float stdev,
                                                           RandGen& _rng,
                                                           weight_t kld_w_,
                                                           weight_t skewness_w_,
                                                           weight_t stdU_w_,
                                                           weight_t skew_U_w_,
                                                           weight_t log_entropy_w_,
                                                           bool decompose_kld_)
    : ctable(ctable_), rng(_rng),
      kld_w(kld_w_), skewness_w(skewness_w_), stdU_w(stdU_w_),
      skew_U_w(skew_U_w_), log_entropy_w(log_entropy_w_),
      decompose_kld(decompose_kld_)
{
    // initialize Occam's razor
    occam = stdev > 0;
    if (occam)
        set_complexity_coef(alphabet_size, stdev);
    // define counter (mapping between observation and its number of occurences)
    boost::for_each(ctable | map_values, [this](const CTable::mapped_type& mv) {
            boost::for_each(mv, [this](const CTable::mapped_type::value_type& v) {
                    counter[get_contin(v.first)] += v.second; }); });
    // precompute pdf
    if (kld_w > 0) {
        pdf = counter;
        klds.set_p_pdf(pdf);
    }
    // compute the skewness of pdf
    accumulator_t acc;
    foreach(const auto& v, pdf)
        acc(v.first, weight = v.second);
    skewness = weighted_skewness(acc);
    logger().fine("skewness = %f", skewness);
}

behavioral_score interesting_predicate_bscore::operator()(const combo_tree& tr) const
{
    static const double entropy_threshold = 0.1; /// @todo should be a parameter

    OTable pred_ot(tr, ctable, rng);

    // compute the entropy of the output of the predicate
    vector<double> prob;
    Counter<vertex, unsigned> vc;
    boost::for_each(ctable | map_values, pred_ot,
                    [&](const CTable::counter_t& c, const vertex& v) {
                        vc[v] += boost::accumulate(c | map_values, 0); });
    double total = klds.p_size();
    transform(vc | map_values, back_inserter(prob), arg1 / total);
    double pred_entropy = entropy(prob);
    logger().fine("pred_entropy = %f", pred_entropy);

    behavioral_score bs;
    if (pred_entropy > entropy_threshold) {
        // filter the output according to tr
        counter_t pred_counter;     // map obvervation to occurence
                                    // conditioned by predicate truth
        boost::for_each(ctable | map_values, pred_ot,
                        [&](const CTable::counter_t& c, const vertex& v) {
                            if (vertex_to_bool(v)) {
                                foreach(const auto& mv, c)
                                    pred_counter[get_contin(mv.first)] = mv.second;
                            }});

        logger().fine("pred_ot.size() = %u", pred_ot.size());
        logger().fine("pred_counter.size() = %u", pred_counter.size());

        if (pred_counter.size() > 1) { // otherwise the statistics are
                                       // messed up (for instance
                                       // pred_skewness can be inf)
            // compute KLD
            if (kld_w > 0) {
                if(decompose_kld) {
                    klds(pred_counter, back_inserter(bs));
                    boost::transform(bs, bs.begin(), kld_w * arg1);
                } else {
                    score_t pred_klds = klds(pred_counter);
                    logger().fine("klds = %f", pred_klds);
                    bs.push_back(kld_w * pred_klds);
                }
            }

            if (skewness_w > 0 || stdU_w > 0 || skew_U_w > 0) {

                // gather statistics with a boost accumulator
                accumulator_t acc;
                foreach(const auto& v, pred_counter)
                    acc(v.first, weight = v.second);

                score_t diff_skewness = 0;
                if (skewness_w > 0 || skew_U_w > 0) {
                    // push the absolute difference between the
                    // unconditioned skewness and conditioned one
                    score_t pred_skewness = weighted_skewness(acc);
                    diff_skewness = pred_skewness - skewness;
                    logger().fine("pred_skewness = %f", pred_skewness);
                    if (skewness_w > 0)
                        bs.push_back(skewness_w * abs(diff_skewness));
                }

                score_t stdU = 0;
                if (stdU_w > 0 || skew_U_w > 0) {

                    // compute the standardized Mann–Whitney U
                    stdU = standardizedMannWhitneyU(counter, pred_counter);
                    logger().fine("stdU = %f", stdU);
                    if (stdU_w > 0)
                        bs.push_back(stdU_w * abs(stdU));
                }

                // push the product of the relative differences of the
                // shift (stdU) and the skewness (so that if both go
                // in the same direction the value if positive, and
                // negative otherwise)
                if (skew_U_w > 0)
                    bs.push_back(skew_U_w * stdU * diff_skewness);
            }

            // add log entropy, this is completely heuristic, no
            // justification except that the log of the entropy is
            // gonna add a large penalty when the entropy tends to
            // 0. Not sure here what we really need, a perhaps
            // something like 1 - p-value, or some form of indefinite
            // TV confidence on the calculation of KLD.
            logger().fine("log(pred_entropy) = %f", log(pred_entropy));
            if (log_entropy_w)
                bs.push_back(log_entropy_w * log(pred_entropy));

            // add the Occam's razor feature
            if(occam)
                bs.push_back(complexity(tr) * complexity_coef);
        }
    }

    // Logger
    log_candidate_bscore(tr, bs);
    // ~Logger

    return bs;
}

behavioral_score interesting_predicate_bscore::best_possible_bscore() const
{
    return behavioral_score(1, best_score);
}

void interesting_predicate_bscore::set_complexity_coef(float alphabet_size,
                                                       float stdev)
{
    if(occam)
        complexity_coef = contin_complexity_coef(alphabet_size, stdev);
}

} // ~namespace moses
} // ~namespace opencog
<commit_msg>moses: in interesting_predicate_bscore assign worse_score if not within an acceptable entropy range<commit_after>/*
 * opencog/learning/moses/moses/scoring.cc
 *
 * Copyright (C) 2002-2008 Novamente LLC
 * All Rights Reserved
 *
 * Written by Moshe Looks, Nil Geisweiller
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
#include "scoring.h"

#include <cmath>

#include <boost/range/algorithm/sort.hpp>
#include <boost/range/algorithm/for_each.hpp>
#include <boost/range/algorithm_ext/for_each.hpp>
#include <boost/range/algorithm/transform.hpp>
#include <boost/range/adaptor/map.hpp>
#include <boost/range/adaptor/filtered.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>

#include <opencog/util/numeric.h>
#include <opencog/util/KLD.h>
#include <opencog/util/MannWhitneyU.h>

namespace opencog { namespace moses {

using namespace std;
using boost::adaptors::map_values;
using boost::adaptors::map_keys;
using boost::adaptors::filtered;
using boost::transform;
using namespace boost::phoenix;
using boost::phoenix::arg_names::arg1;
using namespace boost::accumulators;

// helper to log a combo_tree and its behavioral score
inline void log_candidate_bscore(const combo_tree& tr,
                                 const behavioral_score& bs)
{
    if (logger().getLevel() < Logger::FINE)
        return;

    stringstream ss;
    ss << "Evaluate candidate: " << tr << "\n";
    ss << "\tBScored: " << bs;
    logger().fine(ss.str());
}

////////////////////
// logical_bscore //
////////////////////

behavioral_score logical_bscore::operator()(const combo_tree& tr) const
{
    combo::complete_truth_table tt(tr, arity);
    behavioral_score bs(target.size());

    boost::transform(tt, target, bs.begin(), [](bool b1, bool b2) {
            return -score_t(b1 != b2); });

    return bs;
}

behavioral_score logical_bscore::best_possible_bscore() const {
    return behavioral_score(target.size(), 0);
}

///////////////////
// contin_bscore //
///////////////////

score_t contin_complexity_coef(unsigned alphabet_size, double stdev) {
    return log(alphabet_size) * 2 * sq(stdev);
}

behavioral_score contin_bscore::operator()(const combo_tree& tr) const
{
    OTable ct(tr, cti, rng);
    behavioral_score bs(target.size() + (occam?1:0));
    boost::transform(ct, target, bs.begin(),
                     [](const vertex& vl, const vertex& vr) {
                         return -sq(get_contin(vl) - get_contin(vr)); });
    // add the Occam's razor feature
    if(occam)
        bs.back() = complexity(tr) * complexity_coef;

    // Logger
    log_candidate_bscore(tr, bs);
    // ~Logger

    return bs;
}

behavioral_score contin_bscore::best_possible_bscore() const {
    return behavioral_score(target.size() + (occam?1:0), 0);
}

void contin_bscore::set_complexity_coef(float alphabet_size, float stdev)
{
    if(occam)
        complexity_coef = contin_complexity_coef(alphabet_size, stdev);
}

//////////////////////////////
// discretize_contin_bscore //
//////////////////////////////

score_t discrete_complexity_coef(unsigned alphabet_size, double p) {
    return -log((double)alphabet_size) / log(p/(1-p));
}

discretize_contin_bscore::discretize_contin_bscore(const OTable& ot,
                                                   const ITable& it,
                                                   const vector<contin_t>& thres,
                                                   bool wa,
                                                   float alphabet_size,
                                                   float p,
                                                   RandGen& _rng)
    : target(ot), cit(it), thresholds(thres), weighted_accuracy(wa), rng(_rng),
      classes(ot.size()), weights(thresholds.size() + 1, 1) {
    // enforce that thresholds is sorted
    boost::sort(thresholds);
    // precompute classes
    boost::transform(target, classes.begin(), [&](const vertex& v) {
            return this->class_idx(get_contin(v)); });
    // precompute weights
    multiset<size_t> cs(classes.begin(), classes.end());
    if(weighted_accuracy)
        for(size_t i = 0; i < weights.size(); ++i)
            weights[i] = classes.size() / (float)(weights.size() * cs.count(i));
    // precompute Occam's razor coefficient
    occam = p > 0 && p < 0.5;
    if(occam)
        complexity_coef = discrete_complexity_coef(alphabet_size, p);
}

behavioral_score discretize_contin_bscore::best_possible_bscore() const {
    return behavioral_score(target.size(), 0);
}

size_t discretize_contin_bscore::class_idx(contin_t v) const {
    if(v < thresholds[0])
        return 0;
    size_t s = thresholds.size();
    if(v >= thresholds[s - 1])
        return s;
    return class_idx_within(v, 1, s);
}

size_t discretize_contin_bscore::class_idx_within(contin_t v,
                                                  size_t l_idx,
                                                  size_t u_idx) const
{
    // base case
    if(u_idx - l_idx == 1)
        return l_idx;
    // recursive case
    size_t m_idx = l_idx + (u_idx - l_idx) / 2;
    contin_t t = thresholds[m_idx - 1];
    if(v < t)
        return class_idx_within(v, l_idx, m_idx);
    else
        return class_idx_within(v, m_idx, u_idx);
}

behavioral_score discretize_contin_bscore::operator()(const combo_tree& tr) const
{
    OTable ct(tr, cit, rng);
    behavioral_score bs(target.size() + (occam?1:0));
    boost::transform(ct, classes, bs.begin(), [&](const vertex& v, size_t c_idx) {
            return (c_idx != this->class_idx(get_contin(v))) * this->weights[c_idx];
        });
    // add the Occam's razor feature
    if(occam)
        bs.back() = complexity(tr) * complexity_coef;

    // Logger
    log_candidate_bscore(tr, bs);
    // ~Logger

    return bs;
}

/////////////////////////
// ctruth_table_bscore //
/////////////////////////

ctruth_table_bscore::ctruth_table_bscore(const CTable& _ctt,
                                         float alphabet_size, float p,
                                         RandGen& _rng)
    : ctt(_ctt), rng(_rng)
{
    // Both p==0.0 and p==0.5 are singularity points in the Occam's
    // razor formula for discrete outputs (see the explanation in the
    // comment above ctruth_table_bscore)
    occam = p > 0.0f && p < 0.5f;
    if (occam)
        complexity_coef = discrete_complexity_coef(alphabet_size, p);
}

behavioral_score ctruth_table_bscore::operator()(const combo_tree& tr) const
{
    // The OTable constructor will take the combo tree tr and evaluate
    // it for every row of input in ctt. (The rng is passed straight
    // through to the combo evaluator).
    OTable ptt(tr, ctt, rng);
    behavioral_score bs(ctt.size() + (occam?1:0));

    transform(ptt, ctt | map_values, bs.begin(),
              [](const vertex& v, CTable::mapped_type& vd) {
                  return -score_t(vd[negate_vertex(v)]); });

    // Add the Occam's razor feature
    if (occam)
        bs.back() = complexity(tr) * complexity_coef;

    log_candidate_bscore(tr, bs);

    return bs;
}

behavioral_score ctruth_table_bscore::best_possible_bscore() const
{
    behavioral_score bs(ctt.size() + (occam?1:0));
    transform(ctt | map_values, bs.begin(), [](CTable::mapped_type& vd) {
            return -score_t(min(vd[id::logical_true], vd[id::logical_false])); });
    // add the Occam's razor feature
    if(occam)
        bs.back() = 0;
    return bs;
}

//////////////////////////////////
// interesting_predicate_bscore //
//////////////////////////////////

interesting_predicate_bscore::interesting_predicate_bscore(const CTable& ctable_,
                                                           float alphabet_size,
                                                           float stdev,
                                                           RandGen& _rng,
                                                           weight_t kld_w_,
                                                           weight_t skewness_w_,
                                                           weight_t stdU_w_,
                                                           weight_t skew_U_w_,
                                                           weight_t log_entropy_w_,
                                                           bool decompose_kld_)
    : ctable(ctable_), rng(_rng),
      kld_w(kld_w_), skewness_w(skewness_w_), stdU_w(stdU_w_),
      skew_U_w(skew_U_w_), log_entropy_w(log_entropy_w_),
      decompose_kld(decompose_kld_)
{
    // initialize Occam's razor
    occam = stdev > 0;
    if (occam)
        set_complexity_coef(alphabet_size, stdev);
    // define counter (mapping between observation and its number of occurences)
    boost::for_each(ctable | map_values, [this](const CTable::mapped_type& mv) {
            boost::for_each(mv, [this](const CTable::mapped_type::value_type& v) {
                    counter[get_contin(v.first)] += v.second; }); });
    // precompute pdf
    if (kld_w > 0) {
        pdf = counter;
        klds.set_p_pdf(pdf);
    }
    // compute the skewness of pdf
    accumulator_t acc;
    foreach(const auto& v, pdf)
        acc(v.first, weight = v.second);
    skewness = weighted_skewness(acc);
    logger().fine("skewness = %f", skewness);
}

behavioral_score interesting_predicate_bscore::operator()(const combo_tree& tr) const
{
    static const double entropy_threshold = 0.1; /// @todo should be a parameter

    OTable pred_ot(tr, ctable, rng);

    // compute the entropy of the output of the predicate
    vector<double> prob;
    Counter<vertex, unsigned> vc;
    boost::for_each(ctable | map_values, pred_ot,
                    [&](const CTable::counter_t& c, const vertex& v) {
                        vc[v] += boost::accumulate(c | map_values, 0); });
    double total = klds.p_size();
    transform(vc | map_values, back_inserter(prob), arg1 / total);
    double pred_entropy = entropy(prob);
    logger().fine("pred_entropy = %f", pred_entropy);

    behavioral_score bs;
    auto push_worst = [&bs](){ bs.push_back(worst_score); };
    if (pred_entropy > entropy_threshold) {
        // filter the output according to tr
        counter_t pred_counter;     // map obvervation to occurence
                                    // conditioned by predicate truth
        boost::for_each(ctable | map_values, pred_ot,
                        [&](const CTable::counter_t& c, const vertex& v) {
                            if (vertex_to_bool(v)) {
                                foreach(const auto& mv, c)
                                    pred_counter[get_contin(mv.first)] = mv.second;
                            }});

        logger().fine("pred_ot.size() = %u", pred_ot.size());
        logger().fine("pred_counter.size() = %u", pred_counter.size());

        if (pred_counter.size() > 1) { // otherwise the statistics are
                                       // messed up (for instance
                                       // pred_skewness can be inf)
            // compute KLD
            if (kld_w > 0) {
                if(decompose_kld) {
                    klds(pred_counter, back_inserter(bs));
                    boost::transform(bs, bs.begin(), kld_w * arg1);
                } else {
                    score_t pred_klds = klds(pred_counter);
                    logger().fine("klds = %f", pred_klds);
                    bs.push_back(kld_w * pred_klds);
                }
            }

            if (skewness_w > 0 || stdU_w > 0 || skew_U_w > 0) {

                // gather statistics with a boost accumulator
                accumulator_t acc;
                foreach(const auto& v, pred_counter)
                    acc(v.first, weight = v.second);

                score_t diff_skewness = 0;
                if (skewness_w > 0 || skew_U_w > 0) {
                    // push the absolute difference between the
                    // unconditioned skewness and conditioned one
                    score_t pred_skewness = weighted_skewness(acc);
                    diff_skewness = pred_skewness - skewness;
                    logger().fine("pred_skewness = %f", pred_skewness);
                    if (skewness_w > 0)
                        bs.push_back(skewness_w * abs(diff_skewness));
                }

                score_t stdU = 0;
                if (stdU_w > 0 || skew_U_w > 0) {

                    // compute the standardized Mann–Whitney U
                    stdU = standardizedMannWhitneyU(counter, pred_counter);
                    logger().fine("stdU = %f", stdU);
                    if (stdU_w > 0)
                        bs.push_back(stdU_w * abs(stdU));
                }

                // push the product of the relative differences of the
                // shift (stdU) and the skewness (so that if both go
                // in the same direction the value if positive, and
                // negative otherwise)
                if (skew_U_w > 0)
                    bs.push_back(skew_U_w * stdU * diff_skewness);
            }

            // add log entropy, this is completely heuristic, no
            // justification except that the log of the entropy is
            // gonna add a large penalty when the entropy tends to
            // 0. Not sure here what we really need, a perhaps
            // something like 1 - p-value, or some form of indefinite
            // TV confidence on the calculation of KLD.
            logger().fine("log(pred_entropy) = %f", log(pred_entropy));
            if (log_entropy_w)
                bs.push_back(log_entropy_w * log(pred_entropy));

            // add the Occam's razor feature
            if(occam)
                bs.push_back(complexity(tr) * complexity_coef);
        }
        else {
            push_worst();
        }
    } else {
        push_worst();
    }

    // Logger
    log_candidate_bscore(tr, bs);
    // ~Logger

    return bs;
}

behavioral_score interesting_predicate_bscore::best_possible_bscore() const
{
    return behavioral_score(1, best_score);
}

void interesting_predicate_bscore::set_complexity_coef(float alphabet_size,
                                                       float stdev)
{
    if(occam)
        complexity_coef = contin_complexity_coef(alphabet_size, stdev);
}

} // ~namespace moses
} // ~namespace opencog
<|endoftext|>


<commit_before>//  illarionserver - server for the game Illarion
//  Copyright 2011 Illarion e.V.
//
//  This file is part of illarionserver.
//
//  illarionserver is free software: you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.
//
//  illarionserver is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.
//
//  You should have received a copy of the GNU General Public License
//  along with illarionserver.  If not, see <http://www.gnu.org/licenses/>.

#ifndef __FUSE_PTR_HPP__
#define __FUSE_PTR_HPP__

#include <map>
//#include <iostream>
#include <stdexcept>

template<class T>
class fuse_ptr {
private:
    // pointer to the object
    T **ptr;

    // container for all references from pointers to associated fuse_ptrs
    static std::multimap<T *,fuse_ptr<T> * > fusebox;

public:
    // contructor for null pointer
    fuse_ptr() {
        ptr = new T*;
        *ptr = 0;
    }

    // constructor for objects
    fuse_ptr(T *p) {
        ptr = new T*;
        *ptr = p;

        //std::cout << this << " is now fuse for " << *ptr << std::endl;
        if (p != 0) {
            fusebox.insert(std::pair<T *,fuse_ptr<T> * >(p,this));
        }
    }

    // copy constructor
    fuse_ptr(fuse_ptr const &p) {
        ptr = new T*;
        *ptr = *(p.ptr);
        //std::cout << this << " is now fuse for " << *ptr << std::endl;
        fusebox.insert(std::pair<T *,fuse_ptr<T> * >(*ptr,this));
    }

    // pointer access
    T *get() const {
        if (*ptr == 0) {
            throw std::logic_error("Usage of invalid Character! Use isCharValid( char ) to check if a Character is still valid.");
        } else {
            //std::cout << "Getting ptr of " << this << ": " << *ptr << std::endl;
            return *ptr;
        }
    }

    operator T*() const {
        return get();
    }

    T *operator->() const {
        return get();
    }

    operator bool() const {
        return *ptr != 0;
    }

    // virtual destructor
    virtual ~fuse_ptr() {
        if (*ptr != 0) {
            std::pair< typename std::multimap<T *,fuse_ptr<T> * >::iterator, typename std::multimap<T *,fuse_ptr<T> * >::iterator > range = fusebox.equal_range(*ptr);
            typename std::multimap<T *,fuse_ptr<T> * >::iterator it;

            for (it = range.first; it != range.second && it->second != this; ++it);

            if (it != range.second) {
                //std::cout << this << " is not any longer fuse for " << it->first << std::endl;
                fusebox.erase(it);
            }
        }

        delete ptr;
        ptr = 0;
    }

    // sets all associated fuse_ptrs' contents to null and removes them from fusebox
    static void blow_fuse(T *p) {
        std::pair< typename std::multimap<T *,fuse_ptr<T> * >::iterator, typename std::multimap<T *,fuse_ptr<T> * >::iterator > range = fusebox.equal_range(p);
        typename std::multimap<T *,fuse_ptr<T> * >::iterator it;

        for (it = range.first; it != range.second; ++it) {
            //std::cout << "Setting *ptr of " << it->second << " = 0" << std::endl;
            *(it->second->ptr) = 0;
        }

        fusebox.erase(p);
    }
};

template<class T>
std::multimap<T *,fuse_ptr<T> * > fuse_ptr<T>::fusebox;

// returns p.get()
template<class T>
T *get_pointer(fuse_ptr<T> const &p) {
    return p.get();
}

template<class T>
bool isValid(fuse_ptr<T> const &p) {
    return (bool)p;
}

#endif
<commit_msg>increase readability with auto<commit_after>/*
 *  illarionserver - server for the game Illarion
 *  Copyright 2011 Illarion e.V.
 *
 *  This file is part of illarionserver.
 *
 *  illarionserver is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  illarionserver is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with illarionserver.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef _FUSE_PTR_HPP_
#define _FUSE_PTR_HPP_

#include <map>
#include <stdexcept>

template<class T>
class fuse_ptr {
private:
    typedef std::pair<T*, fuse_ptr<T>*> Fuse;

    T **ptr;
    static std::multimap<T*, fuse_ptr<T>*> fusebox;

public:
    fuse_ptr() {
        ptr = new T*;
        *ptr = 0;
    }

    fuse_ptr(T *p) {
        ptr = new T*;
        *ptr = p;

        if (p != 0) {
            fusebox.insert(Fuse(p,this));
        }
    }

    fuse_ptr(fuse_ptr const &p) {
        ptr = new T*;
        *ptr = *(p.ptr);
        fusebox.insert(Fuse(*ptr,this));
    }

    T *get() const {
        if (*ptr == 0) {
            throw std::logic_error("Usage of invalid Character! Use isCharValid( char ) to check if a Character is still valid.");
        } else {
            return *ptr;
        }
    }

    operator T*() const {
        return get();
    }

    T *operator->() const {
        return get();
    }

    operator bool() const {
        return *ptr != 0;
    }

    virtual ~fuse_ptr() {
        if (*ptr != 0) {
            auto range = fusebox.equal_range(*ptr);
            auto it = range.first;

            for (; it != range.second && it->second != this; ++it);

            if (it != range.second) {
                fusebox.erase(it);
            }
        }

        delete ptr;
        ptr = 0;
    }

    static void blow_fuse(T *p) {
        auto range = fusebox.equal_range(p);

        for (auto it = range.first; it != range.second; ++it) {
            *(it->second->ptr) = 0;
        }

        fusebox.erase(p);
    }
};

template<class T>
std::multimap<T*, fuse_ptr<T>*> fuse_ptr<T>::fusebox;

template<class T>
T *get_pointer(fuse_ptr<T> const &p) {
    return p.get();
}

template<class T>
bool isValid(fuse_ptr<T> const &p) {
    return (bool)p;
}

#endif

<|endoftext|>


<commit_before>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "content/public/browser/android/devtools_auth.h"

#include "base/logging.h"

namespace content {

bool CanUserConnectToDevTools(uid_t uid, gid_t gid) {
  struct passwd* creds = getpwuid(uid);
  if (!creds || !creds->pw_name) {
    LOG(WARNING) << "DevTools: can't obtain creds for uid " << uid;
    return false;
  }
  if (gid == uid &&
      (strcmp("root", creds->pw_name) == 0 ||  // For rooted devices
       strcmp("shell", creds->pw_name) == 0)) {  // For non-rooted devices
    return true;
  }
  LOG(WARNING) << "DevTools: connection attempt from " << creds->pw_name;
  return false;
}

}  // namespace content
<commit_msg>[Android] Allow connections from the same user to DevTools server socket<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "content/public/browser/android/devtools_auth.h"

#include <unistd.h>
#include <sys/types.h>

#include "base/logging.h"

namespace content {

bool CanUserConnectToDevTools(uid_t uid, gid_t gid) {
  struct passwd* creds = getpwuid(uid);
  if (!creds || !creds->pw_name) {
    LOG(WARNING) << "DevTools: can't obtain creds for uid " << uid;
    return false;
  }
  if (gid == uid &&
      (strcmp("root", creds->pw_name) == 0 ||   // For rooted devices
       strcmp("shell", creds->pw_name) == 0 ||  // For non-rooted devices
       uid == getuid())) {  // From processes signed with the same key
    return true;
  }
  LOG(WARNING) << "DevTools: connection attempt from " << creds->pw_name;
  return false;
}

}  // namespace content
<|endoftext|>


<commit_before>#include "io_service_pool.h"

using namespace std;
namespace asio = boost::asio;

namespace wsgi_boost
{
	IoServicePool::IoServicePool(unsigned int pool_size)
	{
		if (pool_size == 0)
		{
			unsigned int threads_hint = thread::hardware_concurrency();
			if (threads_hint > 0)
				pool_size = threads_hint;
			else
				pool_size = 1;
		}
		for (unsigned int i = 0; i < pool_size; ++i)
		{
			// make_shared does not work with an explicit io_service constructor
			io_service_ptr io_service{ new asio::io_service{ 1 } };
			auto work = asio::io_service::work{ *io_service };
			m_io_services.emplace_back(io_service);
			m_works.emplace_back(work);
		}
	}

	void IoServicePool::run()
	{
		m_threads.clear();
		for (size_t i = 1; i < size(); ++i)
		{
			m_threads.emplace_back([this, i]()
			{
				this->m_io_services[i]->run();
			});
		}
		m_io_services[0]->run();
		for (auto& t : m_threads)
			t.join();
	}

	void IoServicePool::stop()
	{
		for (auto& io : m_io_services)
			io->stop();
	}

	void IoServicePool::reset()
	{
		for (auto& io : m_io_services)
		{
			if (io->stopped())
				io->reset();
		}
	}

	io_service_ptr IoServicePool::get_io_service()
	{
		io_service_ptr io_service = m_io_services[m_next_io_service];
		++m_next_io_service;
		if (m_next_io_service == m_io_services.size())
			m_next_io_service = 0;
		return io_service;
	}

	size_t IoServicePool::size() const
	{
		return m_io_services.size();
	}
}
<commit_msg>Call m_io_services.size() directly<commit_after>#include "io_service_pool.h"

using namespace std;
namespace asio = boost::asio;

namespace wsgi_boost
{
	IoServicePool::IoServicePool(unsigned int pool_size)
	{
		if (pool_size == 0)
		{
			unsigned int threads_hint = thread::hardware_concurrency();
			if (threads_hint > 0)
				pool_size = threads_hint;
			else
				pool_size = 1;
		}
		for (unsigned int i = 0; i < pool_size; ++i)
		{
			// make_shared does not work with an explicit io_service constructor
			io_service_ptr io_service{ new asio::io_service{ 1 } };
			auto work = asio::io_service::work{ *io_service };
			m_io_services.emplace_back(io_service);
			m_works.emplace_back(work);
		}
	}

	void IoServicePool::run()
	{
		m_threads.clear();
		for (size_t i = 1; i < m_io_services.size(); ++i)
		{
			m_threads.emplace_back([this, i]()
			{
				this->m_io_services[i]->run();
			});
		}
		m_io_services[0]->run();
		for (auto& t : m_threads)
			t.join();
	}

	void IoServicePool::stop()
	{
		for (auto& io : m_io_services)
			io->stop();
	}

	void IoServicePool::reset()
	{
		for (auto& io : m_io_services)
		{
			if (io->stopped())
				io->reset();
		}
	}

	io_service_ptr IoServicePool::get_io_service()
	{
		io_service_ptr io_service = m_io_services[m_next_io_service];
		++m_next_io_service;
		if (m_next_io_service == m_io_services.size())
			m_next_io_service = 0;
		return io_service;
	}

	size_t IoServicePool::size() const
	{
		return m_io_services.size();
	}
}
<|endoftext|>


<commit_before>#include <cstdio>
#include <cstdlib>
#include <mutex>

#include <eigen3/Eigen/Dense>

#include "cereal/messaging/messaging.h"
#include "cereal/visionipc/visionipc_client.h"
#include "selfdrive/common/clutil.h"
#include "selfdrive/common/params.h"
#include "selfdrive/common/swaglog.h"
#include "selfdrive/common/util.h"
#include "selfdrive/hardware/hw.h"
#include "selfdrive/modeld/models/driving.h"

ExitHandler do_exit;
// globals
bool live_calib_seen;
mat3 cur_transform;
std::mutex transform_lock;

void calibration_thread(bool wide_camera) {
  set_thread_name("calibration");
  set_realtime_priority(50);

  SubMaster sm({"liveCalibration"});

  /*
     import numpy as np
     from common.transformations.model import medmodel_frame_from_road_frame
     medmodel_frame_from_ground = medmodel_frame_from_road_frame[:, (0, 1, 3)]
     ground_from_medmodel_frame = np.linalg.inv(medmodel_frame_from_ground)
  */
  Eigen::Matrix<float, 3, 3> ground_from_medmodel_frame;
  ground_from_medmodel_frame <<
    0.00000000e+00, 0.00000000e+00, 1.00000000e+00,
    -1.09890110e-03, 0.00000000e+00, 2.81318681e-01,
    -1.84808520e-20, 9.00738606e-04,-4.28751576e-02;

  Eigen::Matrix<float, 3, 3> cam_intrinsics = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>(wide_camera ? ecam_intrinsic_matrix.v : fcam_intrinsic_matrix.v);
  const mat3 yuv_transform = get_model_yuv_transform();

  while (!do_exit) {
    sm.update(100);
    if(sm.updated("liveCalibration")) {
      auto extrinsic_matrix = sm["liveCalibration"].getLiveCalibration().getExtrinsicMatrix();
      Eigen::Matrix<float, 3, 4> extrinsic_matrix_eigen;
      for (int i = 0; i < 4*3; i++) {
        extrinsic_matrix_eigen(i / 4, i % 4) = extrinsic_matrix[i];
      }

      auto camera_frame_from_road_frame = cam_intrinsics * extrinsic_matrix_eigen;
      Eigen::Matrix<float, 3, 3> camera_frame_from_ground;
      camera_frame_from_ground.col(0) = camera_frame_from_road_frame.col(0);
      camera_frame_from_ground.col(1) = camera_frame_from_road_frame.col(1);
      camera_frame_from_ground.col(2) = camera_frame_from_road_frame.col(3);

      auto warp_matrix = camera_frame_from_ground * ground_from_medmodel_frame;
      mat3 transform = {};
      for (int i=0; i<3*3; i++) {
        transform.v[i] = warp_matrix(i / 3, i % 3);
      }
      mat3 model_transform = matmul3(yuv_transform, transform);
      std::lock_guard lk(transform_lock);
      cur_transform = model_transform;
      live_calib_seen = true;
    }
  }
}

void run_model(ModelState &model, VisionIpcClient &vipc_client) {
  // messaging
  PubMaster pm({"modelV2", "cameraOdometry"});
  SubMaster sm({"lateralPlan", "roadCameraState"});

  // setup filter to track dropped frames
  FirstOrderFilter frame_dropped_filter(0., 10., 1. / MODEL_FREQ);

  uint32_t frame_id = 0, last_vipc_frame_id = 0;
  double last = 0;
  int desire = -1;
  uint32_t run_count = 0;

  while (!do_exit) {
    VisionIpcBufExtra extra = {};
    VisionBuf *buf = vipc_client.recv(&extra);
    if (buf == nullptr) continue;

    transform_lock.lock();
    mat3 model_transform = cur_transform;
    const bool run_model_this_iter = live_calib_seen;
    transform_lock.unlock();

    // TODO: path planner timeout?
    sm.update(0);
    desire = ((int)sm["lateralPlan"].getLateralPlan().getDesire());
    frame_id = sm["roadCameraState"].getRoadCameraState().getFrameId();

    if (run_model_this_iter) {
      run_count++;

      float vec_desire[DESIRE_LEN] = {0};
      if (desire >= 0 && desire < DESIRE_LEN) {
        vec_desire[desire] = 1.0;
      }

      double mt1 = millis_since_boot();
      ModelDataRaw model_buf = model_eval_frame(&model, buf->buf_cl, buf->width, buf->height,
                                                model_transform, vec_desire);
      double mt2 = millis_since_boot();
      float model_execution_time = (mt2 - mt1) / 1000.0;

      // tracked dropped frames
      uint32_t vipc_dropped_frames = extra.frame_id - last_vipc_frame_id - 1;
      float frames_dropped = frame_dropped_filter.update((float)std::min(vipc_dropped_frames, 10U));
      if (run_count < 10) { // let frame drops warm up
        frame_dropped_filter.reset(0);
        frames_dropped = 0.;
      }

      float frame_drop_ratio = frames_dropped / (1 + frames_dropped);

      model_publish(pm, extra.frame_id, frame_id, frame_drop_ratio, model_buf, extra.timestamp_eof, model_execution_time,
                    kj::ArrayPtr<const float>(model.output.data(), model.output.size()));
      posenet_publish(pm, extra.frame_id, vipc_dropped_frames, model_buf, extra.timestamp_eof);

      //printf("model process: %.2fms, from last %.2fms, vipc_frame_id %u, frame_id, %u, frame_drop %.3f\n", mt2 - mt1, mt1 - last, extra.frame_id, frame_id, frame_drop_ratio);
      last = mt1;
      last_vipc_frame_id = extra.frame_id;
    }
  }
}

int main(int argc, char **argv) {
  set_realtime_priority(54);

  if (Hardware::EON()) {
    set_core_affinity(2);
  } else if (Hardware::TICI()) {
    set_core_affinity(7);
  }
  bool wide_camera = Hardware::TICI() ? Params().getBool("EnableWideCamera") : false;

  // start calibration thread
  std::thread thread = std::thread(calibration_thread, wide_camera);

  // cl init
  cl_device_id device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT);
  cl_context context = CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));

  // init the models
  ModelState model;
  model_init(&model, device_id, context);
  LOGW("models loaded, modeld starting");

  VisionIpcClient vipc_client = VisionIpcClient("camerad", wide_camera ? VISION_STREAM_YUV_WIDE : VISION_STREAM_YUV_BACK, true, device_id, context);
  while (!do_exit && !vipc_client.connect(false)) {
    util::sleep_for(100);
  }

  // run the models
  // vipc_client.connected is false only when do_exit is true
  if (vipc_client.connected) {
    const VisionBuf *b = &vipc_client.buffers[0];
    LOGW("connected with buffer size: %d (%d x %d)", b->len, b->width, b->height);
    run_model(model, vipc_client);
  }

  model_free(&model);
  LOG("joining calibration thread");
  thread.join();
  CL_CHECK(clReleaseContext(context));
  return 0;
}
<commit_msg>modeld.cc: no need to initialize desire to -1 (#21424)<commit_after>#include <cstdio>
#include <cstdlib>
#include <mutex>

#include <eigen3/Eigen/Dense>

#include "cereal/messaging/messaging.h"
#include "cereal/visionipc/visionipc_client.h"
#include "selfdrive/common/clutil.h"
#include "selfdrive/common/params.h"
#include "selfdrive/common/swaglog.h"
#include "selfdrive/common/util.h"
#include "selfdrive/hardware/hw.h"
#include "selfdrive/modeld/models/driving.h"

ExitHandler do_exit;
// globals
bool live_calib_seen;
mat3 cur_transform;
std::mutex transform_lock;

void calibration_thread(bool wide_camera) {
  set_thread_name("calibration");
  set_realtime_priority(50);

  SubMaster sm({"liveCalibration"});

  /*
     import numpy as np
     from common.transformations.model import medmodel_frame_from_road_frame
     medmodel_frame_from_ground = medmodel_frame_from_road_frame[:, (0, 1, 3)]
     ground_from_medmodel_frame = np.linalg.inv(medmodel_frame_from_ground)
  */
  Eigen::Matrix<float, 3, 3> ground_from_medmodel_frame;
  ground_from_medmodel_frame <<
    0.00000000e+00, 0.00000000e+00, 1.00000000e+00,
    -1.09890110e-03, 0.00000000e+00, 2.81318681e-01,
    -1.84808520e-20, 9.00738606e-04,-4.28751576e-02;

  Eigen::Matrix<float, 3, 3> cam_intrinsics = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>(wide_camera ? ecam_intrinsic_matrix.v : fcam_intrinsic_matrix.v);
  const mat3 yuv_transform = get_model_yuv_transform();

  while (!do_exit) {
    sm.update(100);
    if(sm.updated("liveCalibration")) {
      auto extrinsic_matrix = sm["liveCalibration"].getLiveCalibration().getExtrinsicMatrix();
      Eigen::Matrix<float, 3, 4> extrinsic_matrix_eigen;
      for (int i = 0; i < 4*3; i++) {
        extrinsic_matrix_eigen(i / 4, i % 4) = extrinsic_matrix[i];
      }

      auto camera_frame_from_road_frame = cam_intrinsics * extrinsic_matrix_eigen;
      Eigen::Matrix<float, 3, 3> camera_frame_from_ground;
      camera_frame_from_ground.col(0) = camera_frame_from_road_frame.col(0);
      camera_frame_from_ground.col(1) = camera_frame_from_road_frame.col(1);
      camera_frame_from_ground.col(2) = camera_frame_from_road_frame.col(3);

      auto warp_matrix = camera_frame_from_ground * ground_from_medmodel_frame;
      mat3 transform = {};
      for (int i=0; i<3*3; i++) {
        transform.v[i] = warp_matrix(i / 3, i % 3);
      }
      mat3 model_transform = matmul3(yuv_transform, transform);
      std::lock_guard lk(transform_lock);
      cur_transform = model_transform;
      live_calib_seen = true;
    }
  }
}

void run_model(ModelState &model, VisionIpcClient &vipc_client) {
  // messaging
  PubMaster pm({"modelV2", "cameraOdometry"});
  SubMaster sm({"lateralPlan", "roadCameraState"});

  // setup filter to track dropped frames
  FirstOrderFilter frame_dropped_filter(0., 10., 1. / MODEL_FREQ);

  uint32_t frame_id = 0, last_vipc_frame_id = 0;
  double last = 0;
  uint32_t run_count = 0;

  while (!do_exit) {
    VisionIpcBufExtra extra = {};
    VisionBuf *buf = vipc_client.recv(&extra);
    if (buf == nullptr) continue;

    transform_lock.lock();
    mat3 model_transform = cur_transform;
    const bool run_model_this_iter = live_calib_seen;
    transform_lock.unlock();

    // TODO: path planner timeout?
    sm.update(0);
    int desire = ((int)sm["lateralPlan"].getLateralPlan().getDesire());
    frame_id = sm["roadCameraState"].getRoadCameraState().getFrameId();

    if (run_model_this_iter) {
      run_count++;

      float vec_desire[DESIRE_LEN] = {0};
      if (desire >= 0 && desire < DESIRE_LEN) {
        vec_desire[desire] = 1.0;
      }

      double mt1 = millis_since_boot();
      ModelDataRaw model_buf = model_eval_frame(&model, buf->buf_cl, buf->width, buf->height,
                                                model_transform, vec_desire);
      double mt2 = millis_since_boot();
      float model_execution_time = (mt2 - mt1) / 1000.0;

      // tracked dropped frames
      uint32_t vipc_dropped_frames = extra.frame_id - last_vipc_frame_id - 1;
      float frames_dropped = frame_dropped_filter.update((float)std::min(vipc_dropped_frames, 10U));
      if (run_count < 10) { // let frame drops warm up
        frame_dropped_filter.reset(0);
        frames_dropped = 0.;
      }

      float frame_drop_ratio = frames_dropped / (1 + frames_dropped);

      model_publish(pm, extra.frame_id, frame_id, frame_drop_ratio, model_buf, extra.timestamp_eof, model_execution_time,
                    kj::ArrayPtr<const float>(model.output.data(), model.output.size()));
      posenet_publish(pm, extra.frame_id, vipc_dropped_frames, model_buf, extra.timestamp_eof);

      //printf("model process: %.2fms, from last %.2fms, vipc_frame_id %u, frame_id, %u, frame_drop %.3f\n", mt2 - mt1, mt1 - last, extra.frame_id, frame_id, frame_drop_ratio);
      last = mt1;
      last_vipc_frame_id = extra.frame_id;
    }
  }
}

int main(int argc, char **argv) {
  set_realtime_priority(54);

  if (Hardware::EON()) {
    set_core_affinity(2);
  } else if (Hardware::TICI()) {
    set_core_affinity(7);
  }
  bool wide_camera = Hardware::TICI() ? Params().getBool("EnableWideCamera") : false;

  // start calibration thread
  std::thread thread = std::thread(calibration_thread, wide_camera);

  // cl init
  cl_device_id device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT);
  cl_context context = CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));

  // init the models
  ModelState model;
  model_init(&model, device_id, context);
  LOGW("models loaded, modeld starting");

  VisionIpcClient vipc_client = VisionIpcClient("camerad", wide_camera ? VISION_STREAM_YUV_WIDE : VISION_STREAM_YUV_BACK, true, device_id, context);
  while (!do_exit && !vipc_client.connect(false)) {
    util::sleep_for(100);
  }

  // run the models
  // vipc_client.connected is false only when do_exit is true
  if (vipc_client.connected) {
    const VisionBuf *b = &vipc_client.buffers[0];
    LOGW("connected with buffer size: %d (%d x %d)", b->len, b->width, b->height);
    run_model(model, vipc_client);
  }

  model_free(&model);
  LOG("joining calibration thread");
  thread.join();
  CL_CHECK(clReleaseContext(context));
  return 0;
}
<|endoftext|>


<commit_before><commit_msg>Suppress loplugin:implicitboolconversion warnings in Objective-C code<commit_after><|endoftext|>


<commit_before>/*
 *  nextpnr -- Next Generation Place and Route
 *
 *  Copyright (C) 2018  David Shah <david@symbioticeda.com>
 *
 *  Permission to use, copy, modify, and/or distribute this software for any
 *  purpose with or without fee is hereby granted, provided that the above
 *  copyright notice and this permission notice appear in all copies.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 *  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 *  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 *  ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 *  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 *  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 *  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *
 */

#include <boost/algorithm/string.hpp>
#include <sstream>
#include "log.h"

NEXTPNR_NAMESPACE_BEGIN

bool Arch::applyLPF(std::string filename, std::istream &in)
{
    auto isempty = [](const std::string &str) {
        return std::all_of(str.begin(), str.end(), [](char c) { return isblank(c) || c == '\r' || c == '\n'; });
    };
    auto strip_quotes = [](const std::string &str) {
        if (str.at(0) == '"') {
            NPNR_ASSERT(str.back() == '"');
            return str.substr(1, str.size() - 2);
        } else {
            return str;
        }
    };

    try {
        if (!in)
            log_error("failed to open LPF file\n");
        std::string line;
        std::string linebuf;
        int lineno = 0;
        while (std::getline(in, line)) {
            ++lineno;
            size_t cstart = line.find('#');
            if (cstart != std::string::npos)
                line = line.substr(0, cstart);
            if (isempty(line))
                continue;
            linebuf += line;
            // Look for a command up to a semicolon
            size_t scpos = linebuf.find(';');
            while (scpos != std::string::npos) {
                std::string command = linebuf.substr(0, scpos);
                // Split command into words
                std::stringstream ss(command);
                std::vector<std::string> words;
                std::string tmp;
                while (ss >> tmp)
                    words.push_back(tmp);
                if (words.size() >= 0) {
                    std::string verb = words.at(0);
                    if (verb == "BLOCK" || verb == "SYSCONFIG") {
                        if (words.size() != 2 || (words.at(1) != "ASYNCPATHS" && words.at(1) != "RESETPATHS"))
                            log_warning("    ignoring unsupported LPF command '%s' (on line %d)\n", command.c_str(),
                                        lineno);
                    } else if (verb == "FREQUENCY") {
                        if (words.size() < 2)
                            log_error("expected object type after FREQUENCY (on line %d)\n", lineno);
                        std::string etype = words.at(1);
                        if (etype == "PORT" || etype == "NET") {
                            if (words.size() < 4)
                                log_error("expected frequency value and unit after 'FREQUENCY %s' (on line %d)\n",
                                          etype.c_str(), lineno);
                            std::string target = strip_quotes(words.at(2));
                            float freq = std::stof(words.at(3));
                            std::string unit = words.at(4);
                            boost::algorithm::to_upper(unit);
                            if (unit == "MHZ")
                                ;
                            else if (unit == "KHZ")
                                freq /= 1.0e3;
                            else if (unit == "HZ")
                                freq /= 1.0e6;
                            else
                                log_error("unsupported frequency unit '%s' (on line %d)\n", unit.c_str(), lineno);
                            addClock(id(target), freq);
                        } else {
                            log_warning("    ignoring unsupported LPF command '%s %s' (on line %d)\n", command.c_str(),
                                        etype.c_str(), lineno);
                        }
                    } else if (verb == "LOCATE") {
                        if (words.size() < 5)
                            log_error("expected syntax 'LOCATE COMP <port name> SITE <pin>' (on line %d)\n", lineno);
                        if (words.at(1) != "COMP")
                            log_error("expected 'COMP' after 'LOCATE' (on line %d)\n", lineno);
                        std::string cell = strip_quotes(words.at(2));
                        if (words.at(3) != "SITE")
                            log_error("expected 'SITE' after 'LOCATE COMP %s' (on line %d)\n", cell.c_str(), lineno);
                        auto fnd_cell = cells.find(id(cell));
                        if (fnd_cell != cells.end()) {
                            fnd_cell->second->attrs[id("LOC")] = strip_quotes(words.at(4));
                        }
                    } else if (verb == "IOBUF") {
                        if (words.size() < 3)
                            log_error("expected syntax 'IOBUF PORT <port name> <attr>=<value>...' (on line %d)\n",
                                      lineno);
                        if (words.at(1) != "PORT")
                            log_error("expected 'PORT' after 'IOBUF' (on line %d)\n", lineno);
                        std::string cell = strip_quotes(words.at(2));
                        auto fnd_cell = cells.find(id(cell));
                        if (fnd_cell != cells.end()) {
                            for (size_t i = 3; i < words.size(); i++) {
                                std::string setting = words.at(i);
                                size_t eqpos = setting.find('=');
                                if (eqpos == std::string::npos)
                                    log_error(
                                            "expected syntax 'IOBUF PORT <port name> <attr>=<value>...' (on line %d)\n",
                                            lineno);
                                std::string key = setting.substr(0, eqpos), value = setting.substr(eqpos + 1);
                                fnd_cell->second->attrs[id(key)] = value;
                            }
                        }
                    }
                }

                linebuf = linebuf.substr(scpos + 1);
                scpos = linebuf.find(';');
            }
        }
        if (!isempty(linebuf))
            log_error("unexpected end of LPF file\n");
        settings[id("input/lpf")] = filename;
        return true;
    } catch (log_execution_error_exception) {
        return false;
    }
}

NEXTPNR_NAMESPACE_END
<commit_msg>Fix assertion failure on invalid LOCATE input.<commit_after>/*
 *  nextpnr -- Next Generation Place and Route
 *
 *  Copyright (C) 2018  David Shah <david@symbioticeda.com>
 *
 *  Permission to use, copy, modify, and/or distribute this software for any
 *  purpose with or without fee is hereby granted, provided that the above
 *  copyright notice and this permission notice appear in all copies.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 *  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 *  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 *  ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 *  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 *  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 *  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *
 */

#include <boost/algorithm/string.hpp>
#include <sstream>
#include "log.h"

NEXTPNR_NAMESPACE_BEGIN

bool Arch::applyLPF(std::string filename, std::istream &in)
{
    auto isempty = [](const std::string &str) {
        return std::all_of(str.begin(), str.end(), [](char c) { return isblank(c) || c == '\r' || c == '\n'; });
    };
    auto strip_quotes = [](const std::string &str) {
        if (str.at(0) == '"') {
            NPNR_ASSERT(str.back() == '"');
            return str.substr(1, str.size() - 2);
        } else {
            return str;
        }
    };

    try {
        if (!in)
            log_error("failed to open LPF file\n");
        std::string line;
        std::string linebuf;
        int lineno = 0;
        while (std::getline(in, line)) {
            ++lineno;
            size_t cstart = line.find('#');
            if (cstart != std::string::npos)
                line = line.substr(0, cstart);
            if (isempty(line))
                continue;
            linebuf += line;
            // Look for a command up to a semicolon
            size_t scpos = linebuf.find(';');
            while (scpos != std::string::npos) {
                std::string command = linebuf.substr(0, scpos);
                // Split command into words
                std::stringstream ss(command);
                std::vector<std::string> words;
                std::string tmp;
                while (ss >> tmp)
                    words.push_back(tmp);
                if (words.size() >= 0) {
                    std::string verb = words.at(0);
                    if (verb == "BLOCK" || verb == "SYSCONFIG") {
                        if (words.size() != 2 || (words.at(1) != "ASYNCPATHS" && words.at(1) != "RESETPATHS"))
                            log_warning("    ignoring unsupported LPF command '%s' (on line %d)\n", command.c_str(),
                                        lineno);
                    } else if (verb == "FREQUENCY") {
                        if (words.size() < 2)
                            log_error("expected object type after FREQUENCY (on line %d)\n", lineno);
                        std::string etype = words.at(1);
                        if (etype == "PORT" || etype == "NET") {
                            if (words.size() < 4)
                                log_error("expected frequency value and unit after 'FREQUENCY %s' (on line %d)\n",
                                          etype.c_str(), lineno);
                            std::string target = strip_quotes(words.at(2));
                            float freq = std::stof(words.at(3));
                            std::string unit = words.at(4);
                            boost::algorithm::to_upper(unit);
                            if (unit == "MHZ")
                                ;
                            else if (unit == "KHZ")
                                freq /= 1.0e3;
                            else if (unit == "HZ")
                                freq /= 1.0e6;
                            else
                                log_error("unsupported frequency unit '%s' (on line %d)\n", unit.c_str(), lineno);
                            addClock(id(target), freq);
                        } else {
                            log_warning("    ignoring unsupported LPF command '%s %s' (on line %d)\n", command.c_str(),
                                        etype.c_str(), lineno);
                        }
                    } else if (verb == "LOCATE") {
                        if (words.size() < 5)
                            log_error("expected syntax 'LOCATE COMP <port name> SITE <pin>' (on line %d)\n", lineno);
                        if (words.at(1) != "COMP")
                            log_error("expected 'COMP' after 'LOCATE' (on line %d)\n", lineno);
                        std::string cell = strip_quotes(words.at(2));
                        if (words.at(3) != "SITE")
                            log_error("expected 'SITE' after 'LOCATE COMP %s' (on line %d)\n", cell.c_str(), lineno);
                        auto fnd_cell = cells.find(id(cell));
                        if (words.size() > 5)
                            log_error("unexpected input following LOCATE clause (on line %d)\n", lineno);
                        if (fnd_cell != cells.end()) {
                            fnd_cell->second->attrs[id("LOC")] = strip_quotes(words.at(4));
                        }
                    } else if (verb == "IOBUF") {
                        if (words.size() < 3)
                            log_error("expected syntax 'IOBUF PORT <port name> <attr>=<value>...' (on line %d)\n",
                                      lineno);
                        if (words.at(1) != "PORT")
                            log_error("expected 'PORT' after 'IOBUF' (on line %d)\n", lineno);
                        std::string cell = strip_quotes(words.at(2));
                        auto fnd_cell = cells.find(id(cell));
                        if (fnd_cell != cells.end()) {
                            for (size_t i = 3; i < words.size(); i++) {
                                std::string setting = words.at(i);
                                size_t eqpos = setting.find('=');
                                if (eqpos == std::string::npos)
                                    log_error(
                                            "expected syntax 'IOBUF PORT <port name> <attr>=<value>...' (on line %d)\n",
                                            lineno);
                                std::string key = setting.substr(0, eqpos), value = setting.substr(eqpos + 1);
                                fnd_cell->second->attrs[id(key)] = value;
                            }
                        }
                    }
                }

                linebuf = linebuf.substr(scpos + 1);
                scpos = linebuf.find(';');
            }
        }
        if (!isempty(linebuf))
            log_error("unexpected end of LPF file\n");
        settings[id("input/lpf")] = filename;
        return true;
    } catch (log_execution_error_exception) {
        return false;
    }
}

NEXTPNR_NAMESPACE_END
<|endoftext|>


<commit_before>/* Copyright (c) 2017 Nguyen Viet Giang. All rights reserved. */
#include <ProgramOptions.hxx>
#include <fstream>
#include <vector>
#include <stdexcept>
#include <memory>

#include <any/version.h>
#include <any/asm.h>
#include <any/scheduler.h>
#include <any/db.h>
#include <any/loader.h>
#include <any/actor.h>
#include <any/errno.h>
#include <any/std.h>
#include <any/std_io.h>
#include <any/std_string.h>
#include <any/std_buffer.h>
#include <any/std_buffer_bits.h>
#include <any/std_array.h>
#include <any/std_tuple.h>
#include <any/std_table.h>

#if defined(ALINUX) || defined(AAPPLE)
#include <unistd.h>
#endif

#include "compiler.h"

struct address_t
{
    std::string host;
    uint16_t port;
};

static void* myalloc(void*, void* old, aint_t sz)
{
    return realloc(old, (size_t)sz);
}

static void error(const char* fmt, ...)
{
    va_list args;
    char buf[512];
    va_start(args, fmt);
    vsnprintf(buf, sizeof(buf), fmt, args);
    va_end(args);
    throw std::logic_error(buf);
}

static std::string file_name_without_extension(const std::string& path)
{
    auto base_filename = path.substr(path.find_last_of("/\\") + 1);
    std::string::size_type const p(base_filename.find_last_of('.'));
    return base_filename.substr(0, p);
}

static void print_version()
{
    std::cout <<
        "amlc version 1.0" << ", " <<
        "target AVM " << AVERSION_MAJOR << "." << AVERSION_MINOR <<
        " [" << AVERSION_NAME << "]\n";
}

static void compile(const std::string& i, const std::string& o, bool verbose)
{
    std::cout << "compiling " << i << "\n";

    std::ifstream is;
    is.open(i, std::fstream::in);
    if (!is.is_open()) {
        error("failed to open `%s`", i.c_str());
    }
    is.seekg(0, std::fstream::end);
    auto sz = (size_t)is.tellg();
    is.seekg(0, std::fstream::beg);
    std::vector<char> buf;
    buf.resize(sz + 1);
    is.read(buf.data(), sz);
    buf[sz] = '\0';
    is.close();

    aasm_t a;
    aasm_init(&a, &myalloc, NULL);
    if (aasm_load(&a, NULL) != AERR_NONE) {
        error("failed to load aasm_t");
    }
    amlc_compile(&a, buf.data(), i.c_str(), verbose);
    aasm_save(&a);
    std::ofstream os;
    os.open(o, std::fstream::out | std::fstream::binary | std::fstream::trunc);
    os.write((const char*)a.chunk, a.chunk_size);
    os.close();
    aasm_cleanup(&a);

    std::cout << " -> " << o << "\n";
}

static std::string pid_string(aactor_t* a, apid_t pid)
{
    std::stringstream ss;
    apid_idx_t idx = apid_idx(a->owner->idx_bits, pid);
    apid_gen_t gen = apid_gen(a->owner->idx_bits, a->owner->gen_bits, pid);
    ss << "<" << idx << "." << gen << ">";
    return ss.str();
}

static void on_panic(aactor_t* a, void*)
{
    aint_t ev_idx = any_top(a);
    std::cout << pid_string(a, ascheduler_pid(a->owner, a)) << " ";
    if (a->frame->pt) {
        aprototype_t& chunk = a->frame->pt->chunk->prototypes[0];
        std::cout << chunk.strings + chunk.header->source << ":";
        if (a->frame->ip < a->frame->pt->header->num_instructions) {
            std::cout << a->frame->pt->source_lines[a->frame->ip] << " ";
        } else {
            std::cout << a->frame->pt->source_lines[a->frame->ip - 1] << " ";
        }
    }
    std::cout << "panic - ";
    if (any_type(a, ev_idx).type == AVT_STRING) {
        std::cout << any_to_string(a, ev_idx) << "\n";
    } else {
        std::cout << "unknown fatal error\n";
    }
}

static void on_unresolved(
    aloader_t*, const char* module, const char* name, void*)
{
    std::cout << "unresolved import `" << module << ':' << name << "`\n";
}

static void execute(
    const std::string& module, const std::string& name,
    int8_t idx_bits, int8_t gen_bits, aint_t cstack_sz,
    const std::vector<std::string>& chunks,
    address_t* debug, int32_t max_conns, bool alive,
    int32_t realtime_resolution)
{
    aerror_t ec;
    ascheduler_t s;
    adb_t db;

    ec = ascheduler_init(&s, idx_bits, gen_bits, &myalloc, NULL);
    if (ec != AERR_NONE) {
        error("failed to init scheduler %d", ec);
    }
    ascheduler_on_panic(&s, &on_panic, NULL);
    aloader_on_unresolved(&s.loader, &on_unresolved, NULL);

    if (debug) {
        ec = adb_init(
            &db, &myalloc, NULL, &s,
            debug->host.c_str(), debug->port, (aint_t)max_conns);
        if (ec != AERR_NONE) {
            error("failed to init debug service %d", ec);
        } else {
            std::cout <<
                "debug service attached\n" <<
                " -> at " << debug->host << ":" << debug->port << "\n";
        }
    }

    astd_lib_add(&s.loader);
    astd_lib_add_io(
        &s.loader, [](void*, const char* str) { std::cout << str; }, NULL);
    astd_lib_add_string(&s.loader);
    astd_lib_add_buffer(&s.loader);
	astd_lib_add_buffer_bits(&s.loader);
    astd_lib_add_array(&s.loader);
    astd_lib_add_tuple(&s.loader);
    astd_lib_add_table(&s.loader);

    for (size_t i = 0; i < chunks.size(); ++i) {
        auto& c = chunks[i];
        std::ifstream is;
        is.open(c, std::fstream::in | std::fstream::binary);
        if (!is.is_open()) {
            error("failed to open `%s`", c.c_str());
        }
        is.seekg(0, std::fstream::end);
        auto sz = (size_t)is.tellg();
        is.seekg(0, std::fstream::beg);
        auto* chunk = (achunk_header_t*)myalloc(NULL, NULL, sz);
        is.read((char*)chunk, sz);
        is.close();
        std::cout << "add " << c << "\n";
        ec = aloader_add_chunk(&s.loader, chunk, sz, &myalloc, NULL);
        if (ec != AERR_NONE) {
            error("failed to add chunk %d", ec);
        }
    }

    ec = aloader_link(&s.loader, TRUE);
    if (ec != AERR_NONE) {
        error("failed to link %d", ec);
    }
    std::cout << "linking success\n";

    aactor_t* a;
    std::cout << "spawn " << module << ":" << name << "\n";
    ec = ascheduler_new_actor(&s, cstack_sz, &a);
    if (ec != AERR_NONE) {
        error("failed to create actor %d", ec);
    }
    std::cout << " -> pid = " << pid_string(a, ascheduler_pid(&s, a)) << "\n";
    any_import(a, module.c_str(), name.c_str());
    ascheduler_start(&s, a, 0);

    while (alive || ascheduler_num_processes(&s) > 0) {
        if (debug) adb_run_once(&db);
        ascheduler_run_once(&s);
#ifdef AWINDOWS
        Sleep(std::max<DWORD>(realtime_resolution/1000, 1));
#else
        usleep((useconds_t)realtime_resolution);
#endif
    }

    if (debug) adb_cleanup(&db);
    ascheduler_cleanup(&s);
}

int entry(int argc, char** argv)
{
    try {
        po::parser p;

        p["help"]
            .abbreviation('h')
            .description("print this help screen")
            .type(po::void_)
            .callback([&] { std::cout << p << "\n"; });

        p["version"]
            .abbreviation('v')
            .description("display compiler version information")
            .type(po::void_)
            .callback(&print_version);

        p["compile"]
            .abbreviation('c')
            .description("compile an AML source file")
            .type(po::string);

        p["execute"]
            .abbreviation('e')
            .description("run with entry point")
            .type(po::string);

        p["debug"]
            .abbreviation('d')
            .description("run with debug service at host:port")
            .type(po::string);

        p["verbose"]
            .abbreviation('V')
            .description("display debugging log")
            .type(po::void_);

        p["output"]
            .abbreviation('o')
            .description("place the output into file")
            .type(po::string);

        p["idx_bits"]
            .description("number of index bits")
            .type(po::i32)
            .fallback(8);

        p["gen_bits"]
            .description("number of generation bits")
            .type(po::i32)
            .fallback(24);

        p["cstack_sz"]
            .description("size of entry point native stack in bytes")
            .type(po::i32)
            .fallback(4096000);

        p["max_conns"]
            .description("maximum number of debug connections")
            .type(po::i32)
            .fallback(64);

        p["alive"]
            .description("keep AMLC execution session alive")
            .type(po::void_);

        p["rt_res"]
            .description("real-time resolution hint in us")
            .type(po::i32)
            .fallback(1);

        p[""];

        if (!p(argc, argv)) {
            return 1;
        } else {
            if (p["compile"].was_set()) {
                auto i = p["compile"].get().string;
                if (i.length() <= 0) {
                    error("input missing");
                }
                std::string o;
                if (p["output"].was_set()) {
                    o = p["output"].get().string;
                } else {
                    o = file_name_without_extension(i) + ".avmc";
                }
                compile(i, o, p["verbose"].available());
            }
            if (p["execute"].was_set()) {
                auto e = p["execute"].get().string;
                if (e.length() <= 0) {
                    error("entry point missing");
                }
                auto sep = e.find_first_of(':');
                if (sep == std::string::npos) {
                    error(
                        ("bad entry `" + e + "`, missing `:`").c_str());
                }
                if (sep == e.length() - 1) {
                    error(
                        ("bad entry `" + e + "`, missing function").c_str());
                }
                std::unique_ptr<address_t> debug;
                if (p["debug"].was_set()) {
                    debug.reset(new address_t());
                    auto d = p["debug"].get().string;
                    auto colon = d.find_first_of(':');
                    debug->host = d.substr(0, colon);
                    auto port_str = d.substr(colon + 1);
                    debug->port = (uint16_t)atoi(port_str.c_str());
                }
                execute(
                    e.substr(0, sep), e.substr(sep + 1),
                    (int8_t)p["idx_bits"].get().i32,
                    (int8_t)p["gen_bits"].get().i32,
                    (aint_t)p["cstack_sz"].get().i32,
                    p[""].to_vector<po::string>(),
                    debug.get(), p["max_conns"].get().i32,
                    p["alive"].was_set(),
                    p["rt_res"].get().i32);
            }
            return 0;
        }
    } catch (const std::exception& e) {
        std::cerr << "uncaught exception: " << e.what() << "\n";
        return -1;
    }
}

int main(int argc, char** argv)
{
#ifdef AWINDOWS
    WSADATA wsa_data;
    int ws_err = WSAStartup(MAKEWORD(2, 2), &wsa_data);
    if (ws_err != 0) {
        error("WSAStartup failed %d", ws_err);
    }
#endif
    entry(argc, argv);
#ifdef AWINDOWS
    WSACleanup();
#endif
}
<commit_msg>amlc support alive without execute<commit_after>/* Copyright (c) 2017 Nguyen Viet Giang. All rights reserved. */
#include <ProgramOptions.hxx>
#include <fstream>
#include <vector>
#include <stdexcept>
#include <memory>

#include <any/version.h>
#include <any/asm.h>
#include <any/scheduler.h>
#include <any/db.h>
#include <any/loader.h>
#include <any/actor.h>
#include <any/errno.h>
#include <any/std.h>
#include <any/std_io.h>
#include <any/std_string.h>
#include <any/std_buffer.h>
#include <any/std_buffer_bits.h>
#include <any/std_array.h>
#include <any/std_tuple.h>
#include <any/std_table.h>

#if defined(ALINUX) || defined(AAPPLE)
#include <unistd.h>
#endif

#include "compiler.h"

struct address_t
{
    std::string host;
    uint16_t port;
};

static void* myalloc(void*, void* old, aint_t sz)
{
    return realloc(old, (size_t)sz);
}

static void error(const char* fmt, ...)
{
    va_list args;
    char buf[512];
    va_start(args, fmt);
    vsnprintf(buf, sizeof(buf), fmt, args);
    va_end(args);
    throw std::logic_error(buf);
}

static std::string file_name_without_extension(const std::string& path)
{
    auto base_filename = path.substr(path.find_last_of("/\\") + 1);
    std::string::size_type const p(base_filename.find_last_of('.'));
    return base_filename.substr(0, p);
}

static void print_version()
{
    std::cout <<
        "amlc version 1.0" << ", " <<
        "target AVM " << AVERSION_MAJOR << "." << AVERSION_MINOR <<
        " [" << AVERSION_NAME << "]\n";
}

static void compile(const std::string& i, const std::string& o, bool verbose)
{
    std::cout << "compiling " << i << "\n";

    std::ifstream is;
    is.open(i, std::fstream::in);
    if (!is.is_open()) {
        error("failed to open `%s`", i.c_str());
    }
    is.seekg(0, std::fstream::end);
    auto sz = (size_t)is.tellg();
    is.seekg(0, std::fstream::beg);
    std::vector<char> buf;
    buf.resize(sz + 1);
    is.read(buf.data(), sz);
    buf[sz] = '\0';
    is.close();

    aasm_t a;
    aasm_init(&a, &myalloc, NULL);
    if (aasm_load(&a, NULL) != AERR_NONE) {
        error("failed to load aasm_t");
    }
    amlc_compile(&a, buf.data(), i.c_str(), verbose);
    aasm_save(&a);
    std::ofstream os;
    os.open(o, std::fstream::out | std::fstream::binary | std::fstream::trunc);
    os.write((const char*)a.chunk, a.chunk_size);
    os.close();
    aasm_cleanup(&a);

    std::cout << " -> " << o << "\n";
}

static std::string pid_string(aactor_t* a, apid_t pid)
{
    std::stringstream ss;
    apid_idx_t idx = apid_idx(a->owner->idx_bits, pid);
    apid_gen_t gen = apid_gen(a->owner->idx_bits, a->owner->gen_bits, pid);
    ss << "<" << idx << "." << gen << ">";
    return ss.str();
}

static void on_panic(aactor_t* a, void*)
{
    aint_t ev_idx = any_top(a);
    std::cout << pid_string(a, ascheduler_pid(a->owner, a)) << " ";
    if (a->frame->pt) {
        aprototype_t& chunk = a->frame->pt->chunk->prototypes[0];
        std::cout << chunk.strings + chunk.header->source << ":";
        if (a->frame->ip < a->frame->pt->header->num_instructions) {
            std::cout << a->frame->pt->source_lines[a->frame->ip] << " ";
        } else {
            std::cout << a->frame->pt->source_lines[a->frame->ip - 1] << " ";
        }
    }
    std::cout << "panic - ";
    if (any_type(a, ev_idx).type == AVT_STRING) {
        std::cout << any_to_string(a, ev_idx) << "\n";
    } else {
        std::cout << "unknown fatal error\n";
    }
}

static void on_unresolved(
    aloader_t*, const char* module, const char* name, void*)
{
    std::cout << "unresolved import `" << module << ':' << name << "`\n";
}

static void execute(
    const std::string& module, const std::string& name,
    int8_t idx_bits, int8_t gen_bits, aint_t cstack_sz,
    const std::vector<std::string>& chunks,
    address_t* debug, int32_t max_conns, bool alive,
    int32_t realtime_resolution)
{
    aerror_t ec;
    ascheduler_t s;
    adb_t db;

    ec = ascheduler_init(&s, idx_bits, gen_bits, &myalloc, NULL);
    if (ec != AERR_NONE) {
        error("failed to init scheduler %d", ec);
    }
    ascheduler_on_panic(&s, &on_panic, NULL);
    aloader_on_unresolved(&s.loader, &on_unresolved, NULL);

    if (debug) {
        ec = adb_init(
            &db, &myalloc, NULL, &s,
            debug->host.c_str(), debug->port, (aint_t)max_conns);
        if (ec != AERR_NONE) {
            error("failed to init debug service %d", ec);
        } else {
            std::cout <<
                "debug service attached\n" <<
                " -> at " << debug->host << ":" << debug->port << "\n";
        }
    }

    astd_lib_add(&s.loader);
    astd_lib_add_io(
        &s.loader, [](void*, const char* str) { std::cout << str; }, NULL);
    astd_lib_add_string(&s.loader);
    astd_lib_add_buffer(&s.loader);
	astd_lib_add_buffer_bits(&s.loader);
    astd_lib_add_array(&s.loader);
    astd_lib_add_tuple(&s.loader);
    astd_lib_add_table(&s.loader);

    for (size_t i = 0; i < chunks.size(); ++i) {
        auto& c = chunks[i];
        std::ifstream is;
        is.open(c, std::fstream::in | std::fstream::binary);
        if (!is.is_open()) {
            error("failed to open `%s`", c.c_str());
        }
        is.seekg(0, std::fstream::end);
        auto sz = (size_t)is.tellg();
        is.seekg(0, std::fstream::beg);
        auto* chunk = (achunk_header_t*)myalloc(NULL, NULL, sz);
        is.read((char*)chunk, sz);
        is.close();
        std::cout << "add " << c << "\n";
        ec = aloader_add_chunk(&s.loader, chunk, sz, &myalloc, NULL);
        if (ec != AERR_NONE) {
            error("failed to add chunk %d", ec);
        }
    }

    ec = aloader_link(&s.loader, TRUE);
    if (ec != AERR_NONE) {
        error("failed to link %d", ec);
    }
    std::cout << "linking success\n";

    if (module.length() > 0) {
        aactor_t* a;
        std::cout << "spawn " << module << ":" << name << "\n";
        ec = ascheduler_new_actor(&s, cstack_sz, &a);
        if (ec != AERR_NONE) {
            error("failed to create actor %d", ec);
        }

        std::cout << " -> pid = " <<
            pid_string(a, ascheduler_pid(&s, a)) << "\n";
        any_import(a, module.c_str(), name.c_str());
        ascheduler_start(&s, a, 0);
    }

    while (alive || ascheduler_num_processes(&s) > 0) {
        if (debug) adb_run_once(&db);
        ascheduler_run_once(&s);
#ifdef AWINDOWS
        Sleep(std::max<DWORD>(realtime_resolution/1000, 1));
#else
        usleep((useconds_t)realtime_resolution);
#endif
    }

    if (debug) adb_cleanup(&db);
    ascheduler_cleanup(&s);
}

int entry(int argc, char** argv)
{
    try {
        po::parser p;

        p["help"]
            .abbreviation('h')
            .description("print this help screen")
            .type(po::void_)
            .callback([&] { std::cout << p << "\n"; });

        p["version"]
            .abbreviation('v')
            .description("display compiler version information")
            .type(po::void_)
            .callback(&print_version);

        p["compile"]
            .abbreviation('c')
            .description("compile an AML source file")
            .type(po::string);

        p["execute"]
            .abbreviation('e')
            .description("run with entry point")
            .type(po::string);

        p["debug"]
            .abbreviation('d')
            .description("run with debug service at host:port")
            .type(po::string);

        p["verbose"]
            .abbreviation('V')
            .description("display debugging log")
            .type(po::void_);

        p["output"]
            .abbreviation('o')
            .description("place the output into file")
            .type(po::string);

        p["idx_bits"]
            .description("number of index bits")
            .type(po::i32)
            .fallback(8);

        p["gen_bits"]
            .description("number of generation bits")
            .type(po::i32)
            .fallback(24);

        p["cstack_sz"]
            .description("size of entry point native stack in bytes")
            .type(po::i32)
            .fallback(4096000);

        p["max_conns"]
            .description("maximum number of debug connections")
            .type(po::i32)
            .fallback(64);

        p["alive"]
            .description("keep AMLC execution session alive")
            .type(po::void_);

        p["rt_res"]
            .description("real-time resolution hint in us")
            .type(po::i32)
            .fallback(1);

        p[""];

        if (!p(argc, argv)) {
            return 1;
        } else {
            bool alive = p["alive"].was_set();
            bool exec = p["execute"].was_set();
            if (p["compile"].was_set()) {
                auto i = p["compile"].get().string;
                if (i.length() <= 0) {
                    error("input missing");
                }
                std::string o;
                if (p["output"].was_set()) {
                    o = p["output"].get().string;
                } else {
                    o = file_name_without_extension(i) + ".avmc";
                }
                compile(i, o, p["verbose"].available());
            }
            if (exec || alive) {
                std::string module;
                std::string name;
                if (exec) {
                    auto e = p["execute"].get().string;
                    if (e.length() <= 0) {
                        error("entry point missing");
                    }
                    auto sep = e.find_first_of(':');
                    if (sep == std::string::npos) {
                        error(
                            ("bad entry `" + e + "`, missing `:`").c_str());
                    }
                    if (sep == e.length() - 1) {
                        error(
                            ("bad entry `" + e + "`, missing function").c_str());
                    }
                    module = e.substr(0, sep);
                    name = e.substr(sep + 1);
                }
                std::unique_ptr<address_t> debug;
                if (p["debug"].was_set()) {
                    debug.reset(new address_t());
                    auto d = p["debug"].get().string;
                    auto colon = d.find_first_of(':');
                    debug->host = d.substr(0, colon);
                    auto port_str = d.substr(colon + 1);
                    debug->port = (uint16_t)atoi(port_str.c_str());
                }
                execute(
                    module, name,
                    (int8_t)p["idx_bits"].get().i32,
                    (int8_t)p["gen_bits"].get().i32,
                    (aint_t)p["cstack_sz"].get().i32,
                    p[""].to_vector<po::string>(),
                    debug.get(), p["max_conns"].get().i32,
                    alive,
                    p["rt_res"].get().i32);
            }
            return 0;
        }
    } catch (const std::exception& e) {
        std::cerr << "uncaught exception: " << e.what() << "\n";
        return -1;
    }
}

int main(int argc, char** argv)
{
#ifdef AWINDOWS
    WSADATA wsa_data;
    int ws_err = WSAStartup(MAKEWORD(2, 2), &wsa_data);
    if (ws_err != 0) {
        error("WSAStartup failed %d", ws_err);
    }
#endif
    entry(argc, argv);
#ifdef AWINDOWS
    WSACleanup();
#endif
}
<|endoftext|>