<commit_before>#include <algorithm>
#include <ctime>

#include "caffe2/core/operator.h"
#include "caffe2/core/net.h"
#include "caffe2/core/workspace.h"
#include "caffe2/proto/caffe2.pb.h"

namespace caffe2 {

Blob* Workspace::CreateBlob(const string& name) {
  if (HasBlob(name)) {
    CAFFE_VLOG(1) << "Blob " << name << " already exists. Skipping.";
  } else {
    CAFFE_VLOG(1) << "Creating blob " << name;
    (*blob_map_)[name] = unique_ptr<Blob>(new Blob());
  }
  return (*blob_map_)[name].get();
}

const Blob* Workspace::GetBlob(const string& name) const {
  if (!HasBlob(name)) {
    CAFFE_LOG_WARNING << "Blob " << name << " not in the workspace.";
    // TODO(Yangqing): do we want to always print out the list of blobs here?
    CAFFE_LOG_WARNING << "Current blobs:";
    for (const auto& entry : *blob_map_) {
      CAFFE_LOG_WARNING << entry.first;
    }
    return nullptr;
  } else {
    return (*blob_map_)[name].get();
  }
}

Blob* Workspace::GetBlob(const string& name) {
  return const_cast<Blob*>(
      static_cast<const Workspace*>(this)->GetBlob(name));
}

NetBase* Workspace::CreateNet(const NetDef& net_def) {
  CAFFE_CHECK(net_def.has_name()) << "Net definition should have a name.";
  if (net_map_.count(net_def.name()) > 0) {
    CAFFE_LOG_WARNING << "Overwriting existing network of the same name.";
    // Note(Yangqing): Why do we explicitly erase it here? Some components of
    // the old network, such as a opened LevelDB, may prevent us from creating a
    // new network before the old one is deleted. Thus we will need to first
    // erase the old one before the new one can be constructed.
    net_map_.erase(net_def.name());
  }
  // Create a new net with its name.
  CAFFE_LOG_INFO << "Initializing network " << net_def.name();
  net_map_[net_def.name()] =
      unique_ptr<NetBase>(caffe2::CreateNet(net_def, this));
  if (net_map_[net_def.name()].get() == nullptr) {
    CAFFE_LOG_ERROR << "Error when creating the network.";
    net_map_.erase(net_def.name());
    return nullptr;
  }
  if (!net_map_[net_def.name()]->Verify()) {
    CAFFE_LOG_ERROR << "Error when setting up network " << net_def.name();
    net_map_.erase(net_def.name());
    return nullptr;
  }
  return net_map_[net_def.name()].get();
}

void Workspace::DeleteNet(const string& name) {
  if (net_map_.count(name)) {
    net_map_.erase(name);
  }
}

bool Workspace::RunNet(const string& name) {
  if (!net_map_.count(name)) {
    CAFFE_LOG_ERROR << "Network " << name << " does not exist yet.";
    return false;
  }
  return net_map_[name]->Run();
}

bool Workspace::RunOperatorOnce(const OperatorDef& op_def) {
  std::unique_ptr<OperatorBase> op(CreateOperator(op_def, this));
  if (op.get() == nullptr) {
    CAFFE_LOG_ERROR << "Cannot create operator of type " << op_def.name();
    return false;
  }
  if (!op->Verify()) {
    CAFFE_LOG_ERROR << "Error when setting up operator " << op_def.name();
    return false;
  }
  if (!op->Run()) {
    CAFFE_LOG_ERROR << "Error when running operator " << op_def.name();
    return false;
  }
  return true;
}
bool Workspace::RunNetOnce(const NetDef& net_def) {
  std::unique_ptr<NetBase> net(caffe2::CreateNet(net_def, this));
  if (!net->Verify()) {
    CAFFE_LOG_ERROR << "Error when setting up network " << net_def.name();
    return false;
  }
  if (!net->Run()) {
    CAFFE_LOG_ERROR << "Error when running network " << net_def.name();
    return false;
  }
  return true;
}

bool Workspace::RunPlan(const PlanDef& plan) {
  CAFFE_LOG_INFO << "Started executing plan.";
  if (plan.network_size() == 0 || plan.execution_step_size() == 0) {
    CAFFE_LOG_WARNING << "Nothing to run - did you define a correct plan?";
    // We will do nothing, but the plan is still legal so we will return true.
    return true;
  }
  CAFFE_LOG_INFO << "Initializing networks.";

  for (const NetDef& net_def : plan.network()) {
    if (!CreateNet(net_def)) {
      CAFFE_LOG_ERROR << "Failed initializing the networks.";
      return false;
    }
  }
  clock_t start_time = clock();
  for (const ExecutionStep& step : plan.execution_step()) {
    clock_t step_start_time = clock();
    if (!ExecuteStepRecursive(step)) {
      CAFFE_LOG_ERROR << "Failed initializing step " << step.name();
      return false;
    }
    CAFFE_LOG_INFO << "Step " << step.name() << " took "
              << static_cast<float>(clock() - step_start_time) / CLOCKS_PER_SEC
              << " seconds.";
  }
  CAFFE_LOG_INFO << "Total plan took "
            << static_cast<float>(clock() - start_time) / CLOCKS_PER_SEC
            << " seconds.";
  CAFFE_LOG_INFO << "Plan executed successfully.";
  return true;
}

bool Workspace::ExecuteStepRecursive(const ExecutionStep& step) {
  CAFFE_LOG_INFO << "Running execution step " << step.name();
  if (!(step.substep_size() == 0 || step.network_size() == 0)) {
    CAFFE_LOG_ERROR << "An ExecutionStep should either have substep or networks "
               << "but not both.";
    return false;
  }

  int iterations = step.has_num_iter() ? step.num_iter() : 1;
  CAFFE_VLOG(1) << "Executing step for " << iterations << " iterations.";
  if (step.substep_size()) {
    for (int i = 0; i < iterations; ++i) {
      for (const ExecutionStep& substep : step.substep()) {
        if (!ExecuteStepRecursive(substep)) {
          return false;
        }
      }
    }
    return true;
  } else {
    // If this ExecutionStep just contains nets, we can directly run it.
    vector<NetBase*> networks;
    // Collect the networks to run.
    for (const string& network_name : step.network()) {
      if (!net_map_.count(network_name)) {
        CAFFE_LOG_ERROR << "Network " << network_name << " not found.";
        return false;
      }
      CAFFE_VLOG(1) << "Going to execute network " << network_name;
      networks.push_back(net_map_[network_name].get());
    }
    for (int iter = 0; iter < iterations; ++iter) {
      CAFFE_VLOG(1) << "Executing network iteration " << iter;
      for (NetBase* network : networks) {
        if (!network->Run()) {
          return false;
        }
      }
    }
  }
  return true;
}

}  // namespace caffe2
<commit_msg>minor bugfix for workspace<commit_after>#include <algorithm>
#include <ctime>

#include "caffe2/core/operator.h"
#include "caffe2/core/net.h"
#include "caffe2/core/workspace.h"
#include "caffe2/proto/caffe2.pb.h"

namespace caffe2 {

Blob* Workspace::CreateBlob(const string& name) {
  if (HasBlob(name)) {
    CAFFE_VLOG(1) << "Blob " << name << " already exists. Skipping.";
  } else {
    CAFFE_VLOG(1) << "Creating blob " << name;
    (*blob_map_)[name] = unique_ptr<Blob>(new Blob());
  }
  return (*blob_map_)[name].get();
}

const Blob* Workspace::GetBlob(const string& name) const {
  if (!HasBlob(name)) {
    CAFFE_LOG_WARNING << "Blob " << name << " not in the workspace.";
    // TODO(Yangqing): do we want to always print out the list of blobs here?
    CAFFE_LOG_WARNING << "Current blobs:";
    for (const auto& entry : *blob_map_) {
      CAFFE_LOG_WARNING << entry.first;
    }
    return nullptr;
  } else {
    return (*blob_map_)[name].get();
  }
}

Blob* Workspace::GetBlob(const string& name) {
  return const_cast<Blob*>(
      static_cast<const Workspace*>(this)->GetBlob(name));
}

NetBase* Workspace::CreateNet(const NetDef& net_def) {
  CAFFE_CHECK(net_def.has_name()) << "Net definition should have a name.";
  if (net_map_.count(net_def.name()) > 0) {
    CAFFE_LOG_WARNING << "Overwriting existing network of the same name.";
    // Note(Yangqing): Why do we explicitly erase it here? Some components of
    // the old network, such as a opened LevelDB, may prevent us from creating a
    // new network before the old one is deleted. Thus we will need to first
    // erase the old one before the new one can be constructed.
    net_map_.erase(net_def.name());
  }
  // Create a new net with its name.
  CAFFE_LOG_INFO << "Initializing network " << net_def.name();
  net_map_[net_def.name()] =
      unique_ptr<NetBase>(caffe2::CreateNet(net_def, this));
  if (net_map_[net_def.name()].get() == nullptr) {
    CAFFE_LOG_ERROR << "Error when creating the network.";
    net_map_.erase(net_def.name());
    return nullptr;
  }
  if (!net_map_[net_def.name()]->Verify()) {
    CAFFE_LOG_ERROR << "Error when setting up network " << net_def.name();
    net_map_.erase(net_def.name());
    return nullptr;
  }
  return net_map_[net_def.name()].get();
}

void Workspace::DeleteNet(const string& name) {
  if (net_map_.count(name)) {
    net_map_.erase(name);
  }
}

bool Workspace::RunNet(const string& name) {
  if (!net_map_.count(name)) {
    CAFFE_LOG_ERROR << "Network " << name << " does not exist yet.";
    return false;
  }
  return net_map_[name]->Run();
}

bool Workspace::RunOperatorOnce(const OperatorDef& op_def) {
  std::unique_ptr<OperatorBase> op(CreateOperator(op_def, this));
  if (op.get() == nullptr) {
    CAFFE_LOG_ERROR << "Cannot create operator of type " << op_def.type();
    return false;
  }
  if (!op->Verify()) {
    CAFFE_LOG_ERROR << "Error when setting up operator " << op_def.type();
    return false;
  }
  if (!op->Run()) {
    CAFFE_LOG_ERROR << "Error when running operator " << op_def.type();
    return false;
  }
  return true;
}
bool Workspace::RunNetOnce(const NetDef& net_def) {
  std::unique_ptr<NetBase> net(caffe2::CreateNet(net_def, this));
  if (!net->Verify()) {
    CAFFE_LOG_ERROR << "Error when setting up network " << net_def.name();
    return false;
  }
  if (!net->Run()) {
    CAFFE_LOG_ERROR << "Error when running network " << net_def.name();
    return false;
  }
  return true;
}

bool Workspace::RunPlan(const PlanDef& plan) {
  CAFFE_LOG_INFO << "Started executing plan.";
  if (plan.network_size() == 0 || plan.execution_step_size() == 0) {
    CAFFE_LOG_WARNING << "Nothing to run - did you define a correct plan?";
    // We will do nothing, but the plan is still legal so we will return true.
    return true;
  }
  CAFFE_LOG_INFO << "Initializing networks.";

  for (const NetDef& net_def : plan.network()) {
    if (!CreateNet(net_def)) {
      CAFFE_LOG_ERROR << "Failed initializing the networks.";
      return false;
    }
  }
  clock_t start_time = clock();
  for (const ExecutionStep& step : plan.execution_step()) {
    clock_t step_start_time = clock();
    if (!ExecuteStepRecursive(step)) {
      CAFFE_LOG_ERROR << "Failed initializing step " << step.name();
      return false;
    }
    CAFFE_LOG_INFO << "Step " << step.name() << " took "
              << static_cast<float>(clock() - step_start_time) / CLOCKS_PER_SEC
              << " seconds.";
  }
  CAFFE_LOG_INFO << "Total plan took "
            << static_cast<float>(clock() - start_time) / CLOCKS_PER_SEC
            << " seconds.";
  CAFFE_LOG_INFO << "Plan executed successfully.";
  return true;
}

bool Workspace::ExecuteStepRecursive(const ExecutionStep& step) {
  CAFFE_LOG_INFO << "Running execution step " << step.name();
  if (!(step.substep_size() == 0 || step.network_size() == 0)) {
    CAFFE_LOG_ERROR << "An ExecutionStep should either have substep or networks "
               << "but not both.";
    return false;
  }

  int iterations = step.has_num_iter() ? step.num_iter() : 1;
  CAFFE_VLOG(1) << "Executing step for " << iterations << " iterations.";
  if (step.substep_size()) {
    for (int i = 0; i < iterations; ++i) {
      for (const ExecutionStep& substep : step.substep()) {
        if (!ExecuteStepRecursive(substep)) {
          return false;
        }
      }
    }
    return true;
  } else {
    // If this ExecutionStep just contains nets, we can directly run it.
    vector<NetBase*> networks;
    // Collect the networks to run.
    for (const string& network_name : step.network()) {
      if (!net_map_.count(network_name)) {
        CAFFE_LOG_ERROR << "Network " << network_name << " not found.";
        return false;
      }
      CAFFE_VLOG(1) << "Going to execute network " << network_name;
      networks.push_back(net_map_[network_name].get());
    }
    for (int iter = 0; iter < iterations; ++iter) {
      CAFFE_VLOG(1) << "Executing network iteration " << iter;
      for (NetBase* network : networks) {
        if (!network->Run()) {
          return false;
        }
      }
    }
  }
  return true;
}

}  // namespace caffe2
<|endoftext|>


<commit_before>// Copyright 2010-2013 RethinkDB, all rights reserved.
#include <stdarg.h>

#include "errors.hpp"
#include <boost/bind.hpp>
#include <boost/tokenizer.hpp>

#include "arch/runtime/coroutines.hpp"
#include "containers/scoped.hpp"
#include "containers/scoped_regex.hpp"
#include "logger.hpp"
#include "perfmon/core.hpp"

/* Constructor and destructor register and deregister the perfmon. */
perfmon_t::perfmon_t() {
}

perfmon_t::~perfmon_t() {
}

struct stats_collection_context_t : public home_thread_mixin_t {
private:
    rwi_lock_t::read_acq_t lock_sentry;
public:
    DEBUG_ONLY(size_t size;)
    void **contexts;

    stats_collection_context_t(rwi_lock_t *constituents_lock,
                               const intrusive_list_t<perfmon_membership_t> &constituents) :
        lock_sentry(constituents_lock),
        DEBUG_ONLY(size(constituents.size()), )
        contexts(new void *[constituents.size()])
    { }

    ~stats_collection_context_t() {
        delete[] contexts;
    }
};

perfmon_collection_t::perfmon_collection_t() {
}

void *perfmon_collection_t::begin_stats() {
    stats_collection_context_t *ctx;
    {
        on_thread_t thread_switcher(home_thread());
        ctx = new stats_collection_context_t(&constituents_access, constituents);
    }

    size_t i = 0;
    for (perfmon_membership_t *p = constituents.head(); p != NULL; p = constituents.next(p), ++i) {
        rassert(i < ctx->size);
        ctx->contexts[i] = p->get()->begin_stats();
    }
    return ctx;
}

void perfmon_collection_t::visit_stats(void *_context) {
    stats_collection_context_t *ctx = reinterpret_cast<stats_collection_context_t*>(_context);
    size_t i = 0;
    for (perfmon_membership_t *p = constituents.head(); p != NULL; p = constituents.next(p), ++i) {
        rassert(i < ctx->size);
        p->get()->visit_stats(ctx->contexts[i]);
    }
}

scoped_ptr_t<perfmon_result_t> perfmon_collection_t::end_stats(void *_context) {
    stats_collection_context_t *ctx = reinterpret_cast<stats_collection_context_t*>(_context);

    scoped_ptr_t<perfmon_result_t> map = perfmon_result_t::alloc_map_result();

    size_t i = 0;
    for (perfmon_membership_t *p = constituents.head(); p != NULL; p = constituents.next(p), ++i) {
        rassert(i < ctx->size);
        scoped_ptr_t<perfmon_result_t> stat = p->get()->end_stats(ctx->contexts[i]);
        if (p->splice()) {
            stat->splice_into(map.get());
        } else {
            map->insert(p->name, std::move(stat));
        }
    }

    {
        on_thread_t thread_switcher(home_thread());
        delete ctx; // cleans up, unlocks
    }

    return map;
}

void perfmon_collection_t::add(perfmon_membership_t *perfmon) {
    scoped_ptr_t<on_thread_t> thread_switcher;
    if (coroutines_have_been_initialized()) {
        thread_switcher.init(new on_thread_t(home_thread()));
    }

    rwi_lock_t::write_acq_t write_acq(&constituents_access);
    constituents.push_back(perfmon);
}

void perfmon_collection_t::remove(perfmon_membership_t *perfmon) {
    scoped_ptr_t<on_thread_t> thread_switcher;
    if (coroutines_have_been_initialized()) {
        thread_switcher.init(new on_thread_t(home_thread()));
    }

    rwi_lock_t::write_acq_t write_acq(&constituents_access);
    constituents.remove(perfmon);
}

perfmon_membership_t::perfmon_membership_t(perfmon_collection_t *_parent, perfmon_t *_perfmon, const char *_name, bool _own_the_perfmon)
    : name(_name != NULL ? _name : ""), parent(_parent), perfmon(_perfmon), own_the_perfmon(_own_the_perfmon)
{
    parent->add(this);
}

perfmon_membership_t::perfmon_membership_t(perfmon_collection_t *_parent, perfmon_t *_perfmon, const std::string &_name, bool _own_the_perfmon)
    : name(_name), parent(_parent), perfmon(_perfmon), own_the_perfmon(_own_the_perfmon)
{
    parent->add(this);
}

perfmon_membership_t::~perfmon_membership_t() {
    parent->remove(this);
    if (own_the_perfmon)
        delete perfmon;
}

perfmon_t *perfmon_membership_t::get() {
    return perfmon;
}

bool perfmon_membership_t::splice() {
    return name.length() == 0;
}

perfmon_multi_membership_t::perfmon_multi_membership_t(perfmon_collection_t *collection, perfmon_t *perfmon, const char *name, ...) {
    // Create membership for the first provided perfmon first
    memberships.push_back(new perfmon_membership_t(collection, perfmon, name));

    va_list args;
    va_start(args, name);

    // Now go through varargs list until we read NULL
    while ((perfmon = va_arg(args, perfmon_t *)) != NULL) {
        name = va_arg(args, const char *);
        memberships.push_back(new perfmon_membership_t(collection, perfmon, name));
    }

    va_end(args);
}

perfmon_multi_membership_t::~perfmon_multi_membership_t() {
    for (std::vector<perfmon_membership_t*>::const_iterator it = memberships.begin(); it != memberships.end(); ++it) {
        delete *it;
    }
}

perfmon_result_t::perfmon_result_t() {
    type = type_value;
}

perfmon_result_t::perfmon_result_t(const perfmon_result_t &copyee)
    : type(copyee.type), value_(copyee.value_), map_() {
    for (auto it = copyee.map_.begin(); it != copyee.map_.end(); ++it) {
        map_.insert(std::make_pair(it->first,
                                   make_scoped<perfmon_result_t>(*it->second)));
    }
}

perfmon_result_t::perfmon_result_t(const std::string &s) {
    type = type_value;
    value_ = s;
}

perfmon_result_t::perfmon_result_t(std::map<std::string, scoped_ptr_t<perfmon_result_t> > &&m)
    : type(type_map), map_(std::move(m)) { }

perfmon_result_t::~perfmon_result_t() { }

void perfmon_result_t::erase(perfmon_result_t::iterator it) {
    map_.erase(it);
}

scoped_ptr_t<perfmon_result_t> perfmon_result_t::alloc_map_result() {
    return scoped_ptr_t<perfmon_result_t>(new perfmon_result_t(internal_map_t()));
}

std::string *perfmon_result_t::get_string() {
    rassert(type == type_value);
    return &value_;
}

const std::string *perfmon_result_t::get_string() const {
    rassert(type == type_value);
    return &value_;
}

perfmon_result_t::internal_map_t *perfmon_result_t::get_map() {
    rassert(type == type_map);
    return &map_;
}

const perfmon_result_t::internal_map_t *perfmon_result_t::get_map() const {
    rassert(type == type_map);
    return &map_;
}

size_t perfmon_result_t::get_map_size() const {
    rassert(type == type_map);
    return map_.size();
}

bool perfmon_result_t::is_string() const {
    return type == type_value;
}

bool perfmon_result_t::is_map() const {
    return type == type_map;
}

perfmon_result_t::perfmon_result_type_t perfmon_result_t::get_type() const {
    return type;
}

void perfmon_result_t::reset_type(perfmon_result_type_t new_type) {
    value_.clear();
    map_.clear();
    type = new_type;
}

std::pair<perfmon_result_t::iterator, bool> perfmon_result_t::insert(const std::string &name, scoped_ptr_t<perfmon_result_t> &&val) {
    std::string s(name);
    perfmon_result_t::internal_map_t *map = get_map();
    rassert(map->count(name) == 0, "Duplicate perfmons for: %s\n", name.c_str());
    return map->insert(std::make_pair(s, std::move(val)));
}

perfmon_result_t::iterator perfmon_result_t::begin() {
    return map_.begin();
}

perfmon_result_t::iterator perfmon_result_t::end() {
    return map_.end();
}

perfmon_result_t::const_iterator perfmon_result_t::begin() const {
    return map_.begin();
}

perfmon_result_t::const_iterator perfmon_result_t::end() const {
    return map_.end();
}

void perfmon_result_t::splice_into(perfmon_result_t *map) {
    rassert(type == type_map);

    // Transfer all elements from the internal map to the passed map.
    // Unfortunately we can't use here std::map::insert(InputIterator first, InputIterator last),
    // because that way we can overwrite an entry in the target map and thus leak a
    // perfmon_result_t value.
    for (auto it = map_.begin(); it != map_.end(); ++it) {
        map->insert(it->first, std::move(it->second));
    }
    map_.clear();
}

/* Construct a filter from a set of paths.  Paths are of the form foo/bar/baz,
   where each of those can be a regular expression.  They act a lot like XPath
   expressions, but for perfmon_t objects.  */
perfmon_filter_t::perfmon_filter_t(const std::set<std::string> &paths) {
    typedef boost::escaped_list_separator<char> separator_t;
    typedef boost::tokenizer<separator_t> tokenizer_t;
    separator_t slashes("\\", "/", "");

    for (std::set<std::string>::const_iterator
             str = paths.begin(); str != paths.end(); ++str) {
        regexps.push_back(std::vector<scoped_regex_t *>());
        std::vector<scoped_regex_t *> *path = &regexps.back();
        try {
            tokenizer_t t(*str, slashes);
            for (tokenizer_t::const_iterator it = t.begin(); it != t.end(); ++it) {
                path->push_back(new scoped_regex_t());
                if (!path->back()->compile("^"+(*it)+"$")) {
                    logWRN("Error: regex %s failed to compile (%s), treating as empty.",
                           sanitize_for_logger(*it).c_str(),
                           sanitize_for_logger(path->back()->get_error()).c_str());
                    if (!path->back()->compile("^$")) {
                        crash("Regex '^$' failed to compile (%s).\n",
                              sanitize_for_logger(path->back()->get_error()).c_str());
                    }
                }
            }
        } catch (const boost::escaped_list_error &e) {
            logWRN("Error: Could not parse %s (%s), skipping.",
                   sanitize_for_logger(*str).c_str(), e.what());
            continue; //Skip this path
        }
    }
}

perfmon_filter_t::~perfmon_filter_t() {
    for (std::vector<std::vector<scoped_regex_t *> >::const_iterator
             it = regexps.begin(); it != regexps.end(); ++it) {
        for (std::vector<scoped_regex_t *>::const_iterator
                 regexp = it->begin(); regexp != it->end(); ++regexp) {
            delete *regexp;
        }
    }
}

void perfmon_filter_t::filter(const scoped_ptr_t<perfmon_result_t> *p) const {
    guarantee(p->has(), "perfmon_filter_t::filter needs a perfmon_result_t");
    subfilter(const_cast<scoped_ptr_t<perfmon_result_t> *>(p),
              0, std::vector<bool>(regexps.size(), true));
    guarantee(p->has(), "subfilter is not supposed to delete the top-most node.");
}

/* Filter a [perfmon_result_t].  [depth] is how deep we are in the paths that
   the [perfmon_filter_t] was constructed from, and [active] is the set of paths
   that are still active (i.e. that haven't failed a match yet).  This should
   only be called by [filter]. */
void perfmon_filter_t::subfilter(
    scoped_ptr_t<perfmon_result_t> *p_ptr, const size_t depth,
    const std::vector<bool> active) const {

    perfmon_result_t *const p = p_ptr->get();

    bool keep_this_perfmon = true;
    if (p->is_string()) {
        std::string *str = p->get_string();
        for (size_t i = 0; i < regexps.size(); ++i) {
            if (!active[i]) {
                continue;
            }
            if (depth >= regexps[i].size()) {
                return;
            }
            if (depth == regexps[i].size() && regexps[i][depth]->matches(*str)) {
                return;
            }
        }
        keep_this_perfmon = false;
    } else if (p->is_map()) {
        perfmon_result_t::iterator it = p->begin();
        while (it != p->end()) {
            std::vector<bool> subactive = active;
            bool some_subpath = false;
            for (size_t i = 0; i < regexps.size(); ++i) {
                if (!active[i]) {
                    continue;
                }
                if (depth >= regexps[i].size()) {
                    return;
                }
                subactive[i] = regexps[i][depth]->matches(it->first);
                some_subpath |= subactive[i];
            }
            if (some_subpath) {
                subfilter(&it->second, depth + 1, subactive);
            }
            perfmon_result_t::iterator prev_it = it;
            ++it;
            if (!some_subpath || !it->second.has()) {
                p->erase(prev_it);
            }
        }

        if (p->get_map()->empty()) {
            keep_this_perfmon = false;
        }
    }

    if (!keep_this_perfmon && depth > 0) {  // Never delete the topmost node.
        p_ptr->reset();
    }
}

perfmon_collection_t &get_global_perfmon_collection() {
    // Getter function so that we can be sure that `collection` is initialized
    // before it is needed, as advised by the C++ FAQ. Otherwise, a `perfmon_t`
    // might be initialized before `collection` was initialized.

    // FIXME: probably use "new" to create the perfmon_collection_t. For more info check out C++ FAQ Lite answer 10.16.
    static perfmon_collection_t collection;
    return collection;
}

<commit_msg>Made perfmon_filter_t use prev_it instead of it, in a place, to fix a bug.<commit_after>// Copyright 2010-2013 RethinkDB, all rights reserved.
#include <stdarg.h>

#include "errors.hpp"
#include <boost/bind.hpp>
#include <boost/tokenizer.hpp>

#include "arch/runtime/coroutines.hpp"
#include "containers/scoped.hpp"
#include "containers/scoped_regex.hpp"
#include "logger.hpp"
#include "perfmon/core.hpp"

/* Constructor and destructor register and deregister the perfmon. */
perfmon_t::perfmon_t() {
}

perfmon_t::~perfmon_t() {
}

struct stats_collection_context_t : public home_thread_mixin_t {
private:
    rwi_lock_t::read_acq_t lock_sentry;
public:
    DEBUG_ONLY(size_t size;)
    void **contexts;

    stats_collection_context_t(rwi_lock_t *constituents_lock,
                               const intrusive_list_t<perfmon_membership_t> &constituents) :
        lock_sentry(constituents_lock),
        DEBUG_ONLY(size(constituents.size()), )
        contexts(new void *[constituents.size()])
    { }

    ~stats_collection_context_t() {
        delete[] contexts;
    }
};

perfmon_collection_t::perfmon_collection_t() {
}

void *perfmon_collection_t::begin_stats() {
    stats_collection_context_t *ctx;
    {
        on_thread_t thread_switcher(home_thread());
        ctx = new stats_collection_context_t(&constituents_access, constituents);
    }

    size_t i = 0;
    for (perfmon_membership_t *p = constituents.head(); p != NULL; p = constituents.next(p), ++i) {
        rassert(i < ctx->size);
        ctx->contexts[i] = p->get()->begin_stats();
    }
    return ctx;
}

void perfmon_collection_t::visit_stats(void *_context) {
    stats_collection_context_t *ctx = reinterpret_cast<stats_collection_context_t*>(_context);
    size_t i = 0;
    for (perfmon_membership_t *p = constituents.head(); p != NULL; p = constituents.next(p), ++i) {
        rassert(i < ctx->size);
        p->get()->visit_stats(ctx->contexts[i]);
    }
}

scoped_ptr_t<perfmon_result_t> perfmon_collection_t::end_stats(void *_context) {
    stats_collection_context_t *ctx = reinterpret_cast<stats_collection_context_t*>(_context);

    scoped_ptr_t<perfmon_result_t> map = perfmon_result_t::alloc_map_result();

    size_t i = 0;
    for (perfmon_membership_t *p = constituents.head(); p != NULL; p = constituents.next(p), ++i) {
        rassert(i < ctx->size);
        scoped_ptr_t<perfmon_result_t> stat = p->get()->end_stats(ctx->contexts[i]);
        if (p->splice()) {
            stat->splice_into(map.get());
        } else {
            map->insert(p->name, std::move(stat));
        }
    }

    {
        on_thread_t thread_switcher(home_thread());
        delete ctx; // cleans up, unlocks
    }

    return map;
}

void perfmon_collection_t::add(perfmon_membership_t *perfmon) {
    scoped_ptr_t<on_thread_t> thread_switcher;
    if (coroutines_have_been_initialized()) {
        thread_switcher.init(new on_thread_t(home_thread()));
    }

    rwi_lock_t::write_acq_t write_acq(&constituents_access);
    constituents.push_back(perfmon);
}

void perfmon_collection_t::remove(perfmon_membership_t *perfmon) {
    scoped_ptr_t<on_thread_t> thread_switcher;
    if (coroutines_have_been_initialized()) {
        thread_switcher.init(new on_thread_t(home_thread()));
    }

    rwi_lock_t::write_acq_t write_acq(&constituents_access);
    constituents.remove(perfmon);
}

perfmon_membership_t::perfmon_membership_t(perfmon_collection_t *_parent, perfmon_t *_perfmon, const char *_name, bool _own_the_perfmon)
    : name(_name != NULL ? _name : ""), parent(_parent), perfmon(_perfmon), own_the_perfmon(_own_the_perfmon)
{
    parent->add(this);
}

perfmon_membership_t::perfmon_membership_t(perfmon_collection_t *_parent, perfmon_t *_perfmon, const std::string &_name, bool _own_the_perfmon)
    : name(_name), parent(_parent), perfmon(_perfmon), own_the_perfmon(_own_the_perfmon)
{
    parent->add(this);
}

perfmon_membership_t::~perfmon_membership_t() {
    parent->remove(this);
    if (own_the_perfmon)
        delete perfmon;
}

perfmon_t *perfmon_membership_t::get() {
    return perfmon;
}

bool perfmon_membership_t::splice() {
    return name.length() == 0;
}

perfmon_multi_membership_t::perfmon_multi_membership_t(perfmon_collection_t *collection, perfmon_t *perfmon, const char *name, ...) {
    // Create membership for the first provided perfmon first
    memberships.push_back(new perfmon_membership_t(collection, perfmon, name));

    va_list args;
    va_start(args, name);

    // Now go through varargs list until we read NULL
    while ((perfmon = va_arg(args, perfmon_t *)) != NULL) {
        name = va_arg(args, const char *);
        memberships.push_back(new perfmon_membership_t(collection, perfmon, name));
    }

    va_end(args);
}

perfmon_multi_membership_t::~perfmon_multi_membership_t() {
    for (std::vector<perfmon_membership_t*>::const_iterator it = memberships.begin(); it != memberships.end(); ++it) {
        delete *it;
    }
}

perfmon_result_t::perfmon_result_t() {
    type = type_value;
}

perfmon_result_t::perfmon_result_t(const perfmon_result_t &copyee)
    : type(copyee.type), value_(copyee.value_), map_() {
    for (auto it = copyee.map_.begin(); it != copyee.map_.end(); ++it) {
        map_.insert(std::make_pair(it->first,
                                   make_scoped<perfmon_result_t>(*it->second)));
    }
}

perfmon_result_t::perfmon_result_t(const std::string &s) {
    type = type_value;
    value_ = s;
}

perfmon_result_t::perfmon_result_t(std::map<std::string, scoped_ptr_t<perfmon_result_t> > &&m)
    : type(type_map), map_(std::move(m)) { }

perfmon_result_t::~perfmon_result_t() { }

void perfmon_result_t::erase(perfmon_result_t::iterator it) {
    map_.erase(it);
}

scoped_ptr_t<perfmon_result_t> perfmon_result_t::alloc_map_result() {
    return scoped_ptr_t<perfmon_result_t>(new perfmon_result_t(internal_map_t()));
}

std::string *perfmon_result_t::get_string() {
    rassert(type == type_value);
    return &value_;
}

const std::string *perfmon_result_t::get_string() const {
    rassert(type == type_value);
    return &value_;
}

perfmon_result_t::internal_map_t *perfmon_result_t::get_map() {
    rassert(type == type_map);
    return &map_;
}

const perfmon_result_t::internal_map_t *perfmon_result_t::get_map() const {
    rassert(type == type_map);
    return &map_;
}

size_t perfmon_result_t::get_map_size() const {
    rassert(type == type_map);
    return map_.size();
}

bool perfmon_result_t::is_string() const {
    return type == type_value;
}

bool perfmon_result_t::is_map() const {
    return type == type_map;
}

perfmon_result_t::perfmon_result_type_t perfmon_result_t::get_type() const {
    return type;
}

void perfmon_result_t::reset_type(perfmon_result_type_t new_type) {
    value_.clear();
    map_.clear();
    type = new_type;
}

std::pair<perfmon_result_t::iterator, bool> perfmon_result_t::insert(const std::string &name, scoped_ptr_t<perfmon_result_t> &&val) {
    std::string s(name);
    perfmon_result_t::internal_map_t *map = get_map();
    rassert(map->count(name) == 0, "Duplicate perfmons for: %s\n", name.c_str());
    return map->insert(std::make_pair(s, std::move(val)));
}

perfmon_result_t::iterator perfmon_result_t::begin() {
    return map_.begin();
}

perfmon_result_t::iterator perfmon_result_t::end() {
    return map_.end();
}

perfmon_result_t::const_iterator perfmon_result_t::begin() const {
    return map_.begin();
}

perfmon_result_t::const_iterator perfmon_result_t::end() const {
    return map_.end();
}

void perfmon_result_t::splice_into(perfmon_result_t *map) {
    rassert(type == type_map);

    // Transfer all elements from the internal map to the passed map.
    // Unfortunately we can't use here std::map::insert(InputIterator first, InputIterator last),
    // because that way we can overwrite an entry in the target map and thus leak a
    // perfmon_result_t value.
    for (auto it = map_.begin(); it != map_.end(); ++it) {
        map->insert(it->first, std::move(it->second));
    }
    map_.clear();
}

/* Construct a filter from a set of paths.  Paths are of the form foo/bar/baz,
   where each of those can be a regular expression.  They act a lot like XPath
   expressions, but for perfmon_t objects.  */
perfmon_filter_t::perfmon_filter_t(const std::set<std::string> &paths) {
    typedef boost::escaped_list_separator<char> separator_t;
    typedef boost::tokenizer<separator_t> tokenizer_t;
    separator_t slashes("\\", "/", "");

    for (std::set<std::string>::const_iterator
             str = paths.begin(); str != paths.end(); ++str) {
        regexps.push_back(std::vector<scoped_regex_t *>());
        std::vector<scoped_regex_t *> *path = &regexps.back();
        try {
            tokenizer_t t(*str, slashes);
            for (tokenizer_t::const_iterator it = t.begin(); it != t.end(); ++it) {
                path->push_back(new scoped_regex_t());
                if (!path->back()->compile("^"+(*it)+"$")) {
                    logWRN("Error: regex %s failed to compile (%s), treating as empty.",
                           sanitize_for_logger(*it).c_str(),
                           sanitize_for_logger(path->back()->get_error()).c_str());
                    if (!path->back()->compile("^$")) {
                        crash("Regex '^$' failed to compile (%s).\n",
                              sanitize_for_logger(path->back()->get_error()).c_str());
                    }
                }
            }
        } catch (const boost::escaped_list_error &e) {
            logWRN("Error: Could not parse %s (%s), skipping.",
                   sanitize_for_logger(*str).c_str(), e.what());
            continue; //Skip this path
        }
    }
}

perfmon_filter_t::~perfmon_filter_t() {
    for (std::vector<std::vector<scoped_regex_t *> >::const_iterator
             it = regexps.begin(); it != regexps.end(); ++it) {
        for (std::vector<scoped_regex_t *>::const_iterator
                 regexp = it->begin(); regexp != it->end(); ++regexp) {
            delete *regexp;
        }
    }
}

void perfmon_filter_t::filter(const scoped_ptr_t<perfmon_result_t> *p) const {
    guarantee(p->has(), "perfmon_filter_t::filter needs a perfmon_result_t");
    subfilter(const_cast<scoped_ptr_t<perfmon_result_t> *>(p),
              0, std::vector<bool>(regexps.size(), true));
    guarantee(p->has(), "subfilter is not supposed to delete the top-most node.");
}

/* Filter a [perfmon_result_t].  [depth] is how deep we are in the paths that
   the [perfmon_filter_t] was constructed from, and [active] is the set of paths
   that are still active (i.e. that haven't failed a match yet).  This should
   only be called by [filter]. */
void perfmon_filter_t::subfilter(
    scoped_ptr_t<perfmon_result_t> *p_ptr, const size_t depth,
    const std::vector<bool> active) const {

    perfmon_result_t *const p = p_ptr->get();

    bool keep_this_perfmon = true;
    if (p->is_string()) {
        std::string *str = p->get_string();
        for (size_t i = 0; i < regexps.size(); ++i) {
            if (!active[i]) {
                continue;
            }
            if (depth >= regexps[i].size()) {
                return;
            }
            if (depth == regexps[i].size() && regexps[i][depth]->matches(*str)) {
                return;
            }
        }
        keep_this_perfmon = false;
    } else if (p->is_map()) {
        perfmon_result_t::iterator it = p->begin();
        while (it != p->end()) {
            std::vector<bool> subactive = active;
            bool some_subpath = false;
            for (size_t i = 0; i < regexps.size(); ++i) {
                if (!active[i]) {
                    continue;
                }
                if (depth >= regexps[i].size()) {
                    return;
                }
                subactive[i] = regexps[i][depth]->matches(it->first);
                some_subpath |= subactive[i];
            }
            if (some_subpath) {
                subfilter(&it->second, depth + 1, subactive);
            }
            perfmon_result_t::iterator prev_it = it;
            ++it;
            if (!some_subpath || !prev_it->second.has()) {
                p->erase(prev_it);
            }
        }

        if (p->get_map()->empty()) {
            keep_this_perfmon = false;
        }
    }

    if (!keep_this_perfmon && depth > 0) {  // Never delete the topmost node.
        p_ptr->reset();
    }
}

perfmon_collection_t &get_global_perfmon_collection() {
    // Getter function so that we can be sure that `collection` is initialized
    // before it is needed, as advised by the C++ FAQ. Otherwise, a `perfmon_t`
    // might be initialized before `collection` was initialized.

    // FIXME: probably use "new" to create the perfmon_collection_t. For more info check out C++ FAQ Lite answer 10.16.
    static perfmon_collection_t collection;
    return collection;
}

<|endoftext|>


<commit_before>#include <random>
#include <algorithm>
#include <cmath>
#include <string>
#include <fstream>
#include <sstream>
#include <chrono>

#include "bch.h"
#include "iterative.h"
#include "util.h"

static double sigma(const double eb_n0, const double R) {
  return sqrt(1.0f / (2 * R * pow(10, eb_n0 / 10.0)));
}

using decoder_t =
    std::function<std::tuple<std::vector<int>, std::vector<float>, unsigned>(
        std::vector<float>)>;

std::string generate_overview(std::mt19937 &generator, const decoder_t &decoder,
                              const size_t samples, const float eb_n0) {
  constexpr int fk = 3;
  constexpr size_t length = 31;
  constexpr float R = 16.0 / length;

  std::vector<float> b(length);

  std::normal_distribution<float> random(1.0, sigma(eb_n0, R));
  auto noise_gen = std::bind(std::ref(random), std::ref(generator));

  unsigned reconstruction_failures = 0;
  unsigned reconstruction_errors = 0;
  unsigned fk_corr = 0;
  unsigned bit_errors = 0;

  for (size_t sample = 0; sample < samples; sample++) {
    std::generate(std::begin(b), std::end(b), noise_gen);
    try {
      auto result = decoder(b);
      const auto &b_corr = std::get<0>(result);
      auto wrong_bits = std::count_if(std::cbegin(b_corr), std::cend(b_corr),
                                      [](const auto &bit) { return bit != 0; });
      if (wrong_bits) {
        reconstruction_errors++;
        bit_errors += wrong_bits;
      } else {
        auto d = std::count_if(std::cbegin(b), std::cend(b),
                               [](const auto &bit) { return bit < 0; });
        if (d > fk)
          fk_corr++;
      }
    }
    catch (...) {
      reconstruction_failures++;
    }
  }
  /* word error rate */
  std::ostringstream os;
  os << std::scientific;
  const double samples_ = samples;
  os << std::setprecision(12) << reconstruction_failures / samples_ << " ";
  os << std::setprecision(12) << reconstruction_errors / samples_ << " ";
  const auto wrong_words = reconstruction_failures + reconstruction_errors;
  os << std::setprecision(12) << wrong_words / samples_ << " ";
  os << std::setprecision(12) << fk_corr / samples_ << " ";
  os << std::setprecision(12) << bit_errors / (samples_ * length);

  return os.str();
}

std::tuple<std::vector<int>, std::vector<float>, unsigned>
pzg_wrapper(const bch &code, const std::vector<float> &b) {
  std::vector<int> hard;
  hard.reserve(b.size());
  std::transform(std::cbegin(b), std::cend(b), std::back_inserter(hard),
                 [](const auto &bit) { return -2 * bit + 1; });
  auto b_corr = code.correct_peterson(hard);
  return std::make_tuple(b_corr, std::vector<float>(), 0);
}

int main(int argc, const char *const argv[]) {
  constexpr size_t max_iterations = 50;
  constexpr float eb_no_max = 10;
  constexpr float eb_no_step = 0.1f;
  constexpr unsigned base_trials = 10000;

  bch code(5, 0x25, 7);
  std::mt19937 generator(
      std::chrono::high_resolution_clock::now().time_since_epoch().count());

  std::vector<std::pair<std::string, decoder_t> > algorithms;
  algorithms.emplace_back("ms", std::bind(min_sum<max_iterations, float, float>,
                                          std::cref(code.H()),
                                          std::placeholders::_1));
  algorithms.emplace_back(
      "scms1", std::bind(scms1<max_iterations, float, float>,
                         std::cref(code.H()), std::placeholders::_1));
  algorithms.emplace_back(
      "scms2", std::bind(scms2<max_iterations, float, float>,
                         std::cref(code.H()), std::placeholders::_1));
  algorithms.emplace_back(
      "pzg", std::bind(pzg_wrapper, std::cref(code), std::placeholders::_1));

  auto num_samples = [=](const double eb_no) {
    return base_trials * pow(10, eb_no / 3);
  };

  auto run_algorithm = [&](const auto &algorithm) {
    const auto &name = algorithm.first;
    const auto &decoder = algorithm.second;
    std::ofstream file(name.c_str(), std::ofstream::out);
    file << "eb_no "
         << "reconstruction_failures "
         << "reconstruction_errors "
         << "wer "
         << "fk_rate "
         << "ber" << std::endl;
    for (double eb_no = 0; eb_no < eb_no_max; eb_no += eb_no_step) {
      std::cout << "Calculating for E_b/N_0 = " << eb_no << " … ";
      std::cout.flush();
      auto start = std::chrono::high_resolution_clock::now();

      file << eb_no << " ";
      file << generate_overview(generator, decoder, num_samples(eb_no), eb_no)
           << " ";
      file << std::endl;

      auto end = std::chrono::high_resolution_clock::now();
      auto seconds =
          std::chrono::duration_cast<std::chrono::seconds>(end - start).count();

      std::cout << "took " << seconds << " s" << std::endl;
    }
  };

  if (argc < 2) {
    std::cout << "Running all algorithms" << std::endl;
    for (const auto &algorithm : algorithms) {
      run_algorithm(algorithm);
    }
  } else {
    unsigned long index = strtoul(argv[1], nullptr, 0);
    if (index < algorithms.size()) {
      std::cout << "Running algorithm " << algorithms.at(index).first
                << std::endl;
      run_algorithm(algorithms.at(index));
    } else {
      std::cout << "Index " << index << " is out of range. Choose from 0 to "
                << algorithms.size() - 1 << std::endl;
    }
  }
}
<commit_msg>Do not overwrite files<commit_after>#include <random>
#include <algorithm>
#include <cmath>
#include <string>
#include <fstream>
#include <sstream>
#include <chrono>

#include <sys/stat.h>

#include "bch.h"
#include "iterative.h"
#include "util.h"

static double sigma(const double eb_n0, const double R) {
  return sqrt(1.0f / (2 * R * pow(10, eb_n0 / 10.0)));
}

using decoder_t =
    std::function<std::tuple<std::vector<int>, std::vector<float>, unsigned>(
        std::vector<float>)>;

std::string generate_overview(std::mt19937 &generator, const decoder_t &decoder,
                              const size_t samples, const float eb_n0) {
  constexpr int fk = 3;
  constexpr size_t length = 31;
  constexpr float R = 16.0 / length;

  std::vector<float> b(length);

  std::normal_distribution<float> random(1.0, sigma(eb_n0, R));
  auto noise_gen = std::bind(std::ref(random), std::ref(generator));

  unsigned reconstruction_failures = 0;
  unsigned reconstruction_errors = 0;
  unsigned fk_corr = 0;
  unsigned bit_errors = 0;

  for (size_t sample = 0; sample < samples; sample++) {
    std::generate(std::begin(b), std::end(b), noise_gen);
    try {
      auto result = decoder(b);
      const auto &b_corr = std::get<0>(result);
      auto wrong_bits = std::count_if(std::cbegin(b_corr), std::cend(b_corr),
                                      [](const auto &bit) { return bit != 0; });
      if (wrong_bits) {
        reconstruction_errors++;
        bit_errors += wrong_bits;
      } else {
        auto d = std::count_if(std::cbegin(b), std::cend(b),
                               [](const auto &bit) { return bit < 0; });
        if (d > fk)
          fk_corr++;
      }
    }
    catch (...) {
      reconstruction_failures++;
    }
  }
  /* word error rate */
  std::ostringstream os;
  os << std::scientific;
  const double samples_ = samples;
  os << std::setprecision(12) << reconstruction_failures / samples_ << " ";
  os << std::setprecision(12) << reconstruction_errors / samples_ << " ";
  const auto wrong_words = reconstruction_failures + reconstruction_errors;
  os << std::setprecision(12) << wrong_words / samples_ << " ";
  os << std::setprecision(12) << fk_corr / samples_ << " ";
  os << std::setprecision(12) << bit_errors / (samples_ * length);

  return os.str();
}

std::tuple<std::vector<int>, std::vector<float>, unsigned>
pzg_wrapper(const bch &code, const std::vector<float> &b) {
  std::vector<int> hard;
  hard.reserve(b.size());
  std::transform(std::cbegin(b), std::cend(b), std::back_inserter(hard),
                 [](const auto &bit) { return -2 * bit + 1; });
  auto b_corr = code.correct_peterson(hard);
  return std::make_tuple(b_corr, std::vector<float>(), 0);
}

int main(int argc, const char *const argv[]) {
  constexpr size_t max_iterations = 50;
  constexpr float eb_no_max = 10;
  constexpr float eb_no_step = 0.1f;
  constexpr unsigned base_trials = 10000;

  bch code(5, 0x25, 7);
  std::mt19937 generator(
      std::chrono::high_resolution_clock::now().time_since_epoch().count());

  std::vector<std::pair<std::string, decoder_t> > algorithms;
  algorithms.emplace_back("ms", std::bind(min_sum<max_iterations, float, float>,
                                          std::cref(code.H()),
                                          std::placeholders::_1));
  algorithms.emplace_back(
      "scms1", std::bind(scms1<max_iterations, float, float>,
                         std::cref(code.H()), std::placeholders::_1));
  algorithms.emplace_back(
      "scms2", std::bind(scms2<max_iterations, float, float>,
                         std::cref(code.H()), std::placeholders::_1));
  algorithms.emplace_back(
      "pzg", std::bind(pzg_wrapper, std::cref(code), std::placeholders::_1));

  auto num_samples = [=](const double eb_no) {
    return base_trials * pow(10, eb_no / 3);
  };

  auto run_algorithm = [&](const auto &algorithm) {
    const auto &name = algorithm.first;
    const auto &decoder = algorithm.second;
    {
      struct stat buf;
      if (stat(name.c_str(), &buf) != -1) {
        std::cout << "File " << name << " already exists." << std::endl;
        return;
      }
    }
    std::ofstream file(name.c_str(), std::ofstream::out);
    file << "eb_no "
         << "reconstruction_failures "
         << "reconstruction_errors "
         << "wer "
         << "fk_rate "
         << "ber" << std::endl;
    for (double eb_no = 0; eb_no < eb_no_max; eb_no += eb_no_step) {
      std::cout << "Calculating for E_b/N_0 = " << eb_no << " … ";
      std::cout.flush();
      auto start = std::chrono::high_resolution_clock::now();

      file << eb_no << " ";
      file << generate_overview(generator, decoder, num_samples(eb_no), eb_no)
           << " ";
      file << std::endl;

      auto end = std::chrono::high_resolution_clock::now();
      auto seconds =
          std::chrono::duration_cast<std::chrono::seconds>(end - start).count();

      std::cout << "took " << seconds << " s" << std::endl;
    }
  };

  if (argc < 2) {
    std::cout << "Running all algorithms" << std::endl;
    for (const auto &algorithm : algorithms) {
      run_algorithm(algorithm);
    }
  } else {
    unsigned long index = strtoul(argv[1], nullptr, 0);
    if (index < algorithms.size()) {
      std::cout << "Running algorithm " << algorithms.at(index).first
                << std::endl;
      run_algorithm(algorithms.at(index));
    } else {
      std::cout << "Index " << index << " is out of range. Choose from 0 to "
                << algorithms.size() - 1 << std::endl;
    }
  }
}
<|endoftext|>


<commit_before>#include "grid.hpp"

#include <cmath>

std::deque<rectangle_t>
grid_t::arrange(const rectangle_t & screen, unsigned int nrects) const
{
  int gap = 5;

  int factor = std::round(std::sqrt(nrects));
  int rest = (factor * factor) - nrects;

  auto cells = decompose(factor, factor * factor);

  if (rest >= 0) {
    for (auto rit = cells.rbegin(); rit != cells.rend(); ) {
      if (rest == 0) {
        break;
      } else if (rest < *rit) {
        *rit -= rest;
        break;
      } else if (rest >= *rit) {
        rest -= *rit;
        ++rit;
        cells.pop_back();
      }
    }
  } else {
    cells.push_back((-1) * rest);
  }

  int ncol = cells.size();
  int colw = screen.width() / ncol;

  std::deque<rectangle_t> rects;
  for (int c = 0; c < ncol; ++c) {
    int nrow = cells[c];
    int rowh = screen.height() / nrow;
    for (int r = 0; r < nrow; ++r) {
      rects.push_back(rectangle_t(c * colw + screen.x() + gap,
                                  r * rowh + screen.y() + gap,
                                  colw - 2 * gap, rowh - 2 * gap));
    }
  }

  return rects;
}

std::deque<int>
grid_t::decompose(int f, int n) const
{
  std::deque<int> result;
  while (true) {
    n -= f;
    if (n > 0) {
      result.push_back(f);
    } else {
      result.push_back(n + f);
      break;
    }
  }
  return result;
}
<commit_msg>Rename variable factor -> radix<commit_after>#include "grid.hpp"

#include <cmath>

std::deque<rectangle_t>
grid_t::arrange(const rectangle_t & screen, unsigned int nrects) const
{
  int gap = 5;

  int radix = std::round(std::sqrt(nrects));
  int rest = (radix * radix) - nrects;

  auto cells = decompose(radix, radix * radix);

  if (rest >= 0) {
    for (auto rit = cells.rbegin(); rit != cells.rend(); ) {
      if (rest == 0) {
        break;
      } else if (rest < *rit) {
        *rit -= rest;
        break;
      } else if (rest >= *rit) {
        rest -= *rit;
        ++rit;
        cells.pop_back();
      }
    }
  } else {
    cells.push_back((-1) * rest);
  }

  int ncol = cells.size();
  int colw = screen.width() / ncol;

  std::deque<rectangle_t> rects;
  for (int c = 0; c < ncol; ++c) {
    int nrow = cells[c];
    int rowh = screen.height() / nrow;
    for (int r = 0; r < nrow; ++r) {
      rects.push_back(rectangle_t(c * colw + screen.x() + gap,
                                  r * rowh + screen.y() + gap,
                                  colw - 2 * gap, rowh - 2 * gap));
    }
  }

  return rects;
}

std::deque<int>
grid_t::decompose(int f, int n) const
{
  std::deque<int> result;
  while (true) {
    n -= f;
    if (n > 0) {
      result.push_back(f);
    } else {
      result.push_back(n + f);
      break;
    }
  }
  return result;
}
<|endoftext|>


<commit_before>#ifndef SRC_PORTS_PORTS_HPP_
#define SRC_PORTS_PORTS_HPP_

#include "event_ports.hpp"
#include "stream_ports.hpp"

#include "core/connection.hpp"

namespace fc
{

/**
 * \brief contains connections of streams during creation.
 *
 * stream_proxy is used to store the intermediate objects created,
 * when a connection of streams is build up.
 * Until the stream source is connected to a proper stream_sink,
 * the connection is not complete and no value can be pulled through.
 * Nonetheless, the connection needs to be stored to allow further connections.
 * the stream_proxy stores these temporary objects.
 *
 *
 * \tparam source_t the source of the connection, either a connectable or a stream_source
 * \tparam sink_t the stream_sink, which serves as the active part of the connection.
 *
 * ToDo merge this with the expected code duplication, when we make the proxy for events.
 */
template<class source_t, class sink_t>
struct stream_proxy
{
	stream_proxy(source_t source, sink_t sink_) :
			stored_source(source), sink(sink_)
	{
	}

	/**
	 * \brief connects a connectable, which is not a source_port to the stream_proxy.
	 *
	 * connects the new_source to the current stored source
	 *
	 * \pre new_source_t needs to be connectable
	 * \post new_source is connected to the old source.
	 * \returns a stream_proxy, which contains the new_source as the source.
	 */
	template<class new_source_t, class = typename std::enable_if<
	        !is_stream_source<new_source_t>::value>::type>
	auto connect(new_source_t new_source)
	{
		auto connection = fc::connect(new_source, stored_source);
		return stream_proxy<decltype(connection), sink_t>(connection, sink);
	}

	/**
	 * \brief connects a source_port to the stream_proxy.
	 *
	 * connects the new_source to the current stored source
	 * then connects this connection to the sink which completes the connection
	 *
	 * \pre new_source_t needs to a source_port
	 * \post new_source is now connected to sink via the connection stored in the proxy.
	 * \returns nothing, the connection is complete now
	 */
	template<class new_source_t, class enable = void>
	typename std::enable_if<is_stream_source<new_source_t>::value, void>::type
	connect(new_source_t new_source)
	{
		auto tmp = fc::connect(new_source, stored_source);
		sink.connect(tmp);
		return;
	}

	source_t stored_source;
	//access to connect method of sink port
	sink_t sink;
};

namespace detail
{

template<class T>
struct is_stream_proxy: std::false_type
{
};

template<class source_t, class sink_t>
struct is_stream_proxy<fc::stream_proxy<source_t, sink_t>> : std::true_type
{
};

/// Specialization of connect to call member connect of stream_proxy.
template<class sink_t, class source_t>
struct connect_impl<sink_t, source_t,
        typename std::enable_if<is_stream_proxy<sink_t>::value>::type>
{
	auto operator()(source_t source, sink_t sink)
	{
		return sink.connect(source);
	}
};

}  // namespace detail
}  // namespace fc

#endif /* SRC_PORTS_PORTS_HPP_ */<commit_msg>Fix comment on stream_proxy::connect.<commit_after>#ifndef SRC_PORTS_PORTS_HPP_
#define SRC_PORTS_PORTS_HPP_

#include "event_ports.hpp"
#include "stream_ports.hpp"

#include "core/connection.hpp"

namespace fc
{

/**
 * \brief contains connections of streams during creation.
 *
 * stream_proxy is used to store the intermediate objects created,
 * when a connection of streams is build up.
 * Until the stream source is connected to a proper stream_sink,
 * the connection is not complete and no value can be pulled through.
 * Nonetheless, the connection needs to be stored to allow further connections.
 * the stream_proxy stores these temporary objects.
 *
 *
 * \tparam source_t the source of the connection, either a connectable or a stream_source
 * \tparam sink_t the stream_sink, which serves as the active part of the connection.
 *
 * ToDo merge this with the expected code duplication, when we make the proxy for events.
 */
template<class source_t, class sink_t>
struct stream_proxy
{
	stream_proxy(source_t source, sink_t sink_) :
			stored_source(source), sink(sink_)
	{
	}

	/**
	 * \brief connects a connectable, which is not a source_port to the stream_proxy.
	 *
	 * connects the new_source to the current stored source
	 *
	 * \pre new_source_t needs to be connectable
	 * \post new_source is connected to the old source.
	 * \returns a stream_proxy, which contains the new_source as the source.
	 */
	template<class new_source_t, class = typename std::enable_if<
	        !is_stream_source<new_source_t>::value>::type>
	auto connect(new_source_t new_source)
	{
		auto connection = fc::connect(new_source, stored_source);
		return stream_proxy<decltype(connection), sink_t>(connection, sink);
	}

	/**
	 * \brief connects a source_port to the stream_proxy.
	 *
	 * connects the new_source to the current stored source
	 * then connects this connection to the sink which completes the connection
	 *
	 * \pre new_source_t needs to be a source_port
	 * \post new_source is now connected to sink via the connection stored in the proxy.
	 * \returns nothing, the connection is complete now
	 */
	template<class new_source_t, class enable = void>
	typename std::enable_if<is_stream_source<new_source_t>::value, void>::type
	connect(new_source_t new_source)
	{
		auto tmp = fc::connect(new_source, stored_source);
		sink.connect(tmp);
		return;
	}

	source_t stored_source;
	//access to connect method of sink port
	sink_t sink;
};

namespace detail
{

template<class T>
struct is_stream_proxy: std::false_type
{
};

template<class source_t, class sink_t>
struct is_stream_proxy<fc::stream_proxy<source_t, sink_t>> : std::true_type
{
};

/// Specialization of connect to call member connect of stream_proxy.
template<class sink_t, class source_t>
struct connect_impl<sink_t, source_t,
        typename std::enable_if<is_stream_proxy<sink_t>::value>::type>
{
	auto operator()(source_t source, sink_t sink)
	{
		return sink.connect(source);
	}
};

}  // namespace detail
}  // namespace fc

#endif /* SRC_PORTS_PORTS_HPP_ */<|endoftext|>


<commit_before>#include <pxp-agent/request_processor.hpp>
#include <pxp-agent/action_outcome.hpp>
#include <pxp-agent/pxp_schemas.hpp>
#include <pxp-agent/external_module.hpp>
#include <pxp-agent/modules/echo.hpp>
#include <pxp-agent/modules/ping.hpp>
#include <pxp-agent/modules/status.hpp>

#include <leatherman/json_container/json_container.hpp>
#include <leatherman/file_util/file.hpp>
#include <leatherman/file_util/directory.hpp>
#include <leatherman/util/strings.hpp>
#include <leatherman/util/timer.hpp>

#define LEATHERMAN_LOGGING_NAMESPACE "puppetlabs.pxp_agent.request_processor"
#include <leatherman/logging/logging.hpp>

#include <boost/filesystem/operations.hpp>

#include <vector>
#include <atomic>
#include <functional>
#include <stdexcept>  // out_of_range

namespace PXPAgent {

namespace fs = boost::filesystem;
namespace lth_jc = leatherman::json_container;
namespace lth_file = leatherman::file_util;
namespace lth_util = leatherman::util;

//
// Results Storage
//

class ResultsStorage {
  public:
    struct Error : public std::runtime_error {
        explicit Error(std::string const& msg) : std::runtime_error(msg) {}
    };

    // Throw a ResultsStorage::Error in case of failure while writing
    // to any of result files
    ResultsStorage(const ActionRequest& request, const std::string& results_dir)
            : module { request.module() },
              action { request.action() },
              out_path { results_dir + "/stdout" },
              err_path { results_dir + "/stderr" },
              status_path { results_dir + "/status" },
              action_status {} {
        initialize(request, results_dir);
    }

    void write(const ActionOutcome& outcome, const std::string& exec_error,
               const std::string& duration) {
        action_status.set<std::string>("status", "completed");
        action_status.set<std::string>("duration", duration);
        action_status.set<int>("exitcode", outcome.exitcode);
        lth_file::atomic_write_to_file(action_status.toString() + "\n", status_path);

        if (exec_error.empty()) {
            if (outcome.type == ActionOutcome::Type::External) {
                lth_file::atomic_write_to_file(outcome.std_out + "\n", out_path);
                if (!outcome.std_err.empty()) {
                    lth_file::atomic_write_to_file(outcome.std_err + "\n", err_path);
                }
            } else {
                // ActionOutcome::Type::Internal
                lth_file::atomic_write_to_file(outcome.results.toString()
                                               + "\n", out_path);
            }
        } else {
            lth_file::atomic_write_to_file(exec_error, err_path);
        }
    }

  private:
    std::string module;
    std::string action;
    std::string out_path;
    std::string err_path;
    std::string status_path;
    lth_jc::JsonContainer action_status;

    void initialize(const ActionRequest& request, const std::string results_dir) {
        if (!fs::exists(results_dir)) {
            LOG_DEBUG("Creating results directory for '%1% %2%', transaction "
                       "%3%, in '%4%'", request.module(), request.action(),
                       request.transactionId(), results_dir);
            try {
                fs::create_directories(results_dir);
            } catch (const fs::filesystem_error& e) {
                std::string err_msg { "failed to create results directory: " };
                throw Error { err_msg + e.what() };
            }
        }

        action_status.set<std::string>("module", module);
        action_status.set<std::string>("action", action);
        action_status.set<std::string>("status", "running");
        action_status.set<std::string>("duration", "0 s");
        action_status.set<int>("exitcode", EXIT_SUCCESS);

        if (!request.paramsTxt().empty()) {
            action_status.set<std::string>("input", request.paramsTxt());
        } else {
            action_status.set<std::string>("input", "none");
        }

        lth_file::atomic_write_to_file("", out_path);
        lth_file::atomic_write_to_file("", err_path);
        lth_file::atomic_write_to_file(action_status.toString() + "\n", status_path);
    }
};

//
// Non-blocking action task
//

void nonBlockingActionTask(std::shared_ptr<Module> module_ptr,
                           ActionRequest&& request,
                           std::string job_id,
                           ResultsStorage results_storage,
                           std::shared_ptr<PXPConnector> connector_ptr,
                           std::shared_ptr<std::atomic<bool>> done) {
    lth_util::Timer timer {};
    std::string exec_error {};
    ActionOutcome outcome {};

    try {
        outcome = module_ptr->executeAction(request);

        if (request.parsedChunks().data.get<bool>("notify_outcome")) {
            connector_ptr->sendNonBlockingResponse(request, outcome.results, job_id);
        }
    } catch (Module::ProcessingError& e) {
        connector_ptr->sendPXPError(request, e.what());
        exec_error = "Failed to execute '" + request.module() + " "
                     + request.action() + "': " + e.what() + "\n";
    } catch (PCPClient::connection_error& e) {
        exec_error = "Failed to send non blocking response for '"
                     + request.module() + " " + request.action() + "': "
                     + e.what() + "\n";
    }

    // Store results on disk
    auto duration = std::to_string(timer.elapsed_seconds()) + " s";
    results_storage.write(outcome, exec_error, duration);

    // Flag end of processing
    *done = true;
}

//
// Public interface
//

RequestProcessor::RequestProcessor(std::shared_ptr<PXPConnector> connector_ptr,
                                   const Configuration::Agent& agent_configuration)
        : thread_container_ { "Action Executer" },
          connector_ptr_ { connector_ptr },
          spool_dir_ { agent_configuration.spool_dir },
          modules_ {},
          modules_config_dir_ { agent_configuration.modules_config_dir },
          modules_config_ {} {
    assert(!spool_dir_.empty());

    // NB: certificate paths have been validated by HW

    loadModulesConfiguration();
    loadInternalModules();

    if (!agent_configuration.modules_dir.empty()) {
        loadExternalModulesFrom(agent_configuration.modules_dir);
    } else {
        LOG_WARNING("The modules directory was not provided; no external "
                    "module will be loaded");
    }

    logLoadedModules();
}

void RequestProcessor::processRequest(const RequestType& request_type,
                                      const PCPClient::ParsedChunks& parsed_chunks) {
    try {
        // Inspect and validate the request message format
        ActionRequest request { request_type, parsed_chunks };

        LOG_INFO("About to process %1% request %2% by %3%, transaction %4%",
                 requestTypeNames[request_type], request.id(), request.sender(),
                 request.transactionId());

        try {
            // We can access the request content; validate it
            validateRequestContent(request);
        } catch (RequestProcessor::Error& e) {
            // Invalid request; send *PXP error*

            LOG_ERROR("Invalid %1% request %2% by %3%, transaction %4%: %5%",
                      requestTypeNames[request_type], request.id(),
                      request.sender(), request.transactionId(), e.what());
            connector_ptr_->sendPXPError(request, e.what());
            return;
        }

        try {
            if (request.type() == RequestType::Blocking) {
                processBlockingRequest(request);
            } else {
                processNonBlockingRequest(request);
            }
        } catch (std::exception& e) {
            // Process failure; send *PXP error*
            LOG_ERROR("Failed to process %1% request %2% by %3%, transaction %4%: "
                      "%5%", requestTypeNames[request.type()], request.id(),
                      request.sender(), request.transactionId(), e.what());
            connector_ptr_->sendPXPError(request, e.what());
        }
    } catch (ActionRequest::Error& e) {
        // Failed to instantiate ActionRequest - bad message; send *PCP error*

        auto id = parsed_chunks.envelope.get<std::string>("id");
        auto sender = parsed_chunks.envelope.get<std::string>("sender");
        std::vector<std::string> endpoints { sender };
        LOG_ERROR("Invalid %1% request by %2%: %3%", id, sender, e.what());
        connector_ptr_->sendPCPError(id, e.what(), endpoints);
    }
}

//
// Private interface
//

void RequestProcessor::validateRequestContent(const ActionRequest& request) {
    // Validate requested module and action
    try {
        if (!modules_.at(request.module())->hasAction(request.action())) {
            throw RequestProcessor::Error { "unknown action '" + request.action()
                                            + "' for module " + request.module() };
        }
    } catch (std::out_of_range& e) {
        throw RequestProcessor::Error { "unknown module: " + request.module() };
    }

    // Validate request input params
    try {
        LOG_DEBUG("Validating input for parameters of '%1% %2%' request %3% "
                  "by %4%, transaction %5%", request.module(), request.action(),
                  request.id(), request.sender(), request.transactionId());

        // NB: the registred schemas have the same name as the action
        auto& validator = modules_.at(request.module())->input_validator_;
        validator.validate(request.params(), request.action());
    } catch (PCPClient::validation_error& e) {
        LOG_DEBUG("Invalid '%1% %2%' request %3%: %4%", request.module(),
                  request.action(), request.id(), e.what());
        throw RequestProcessor::Error { "invalid input for '" + request.module()
                                        + " " + request.action() + "'" };
    }
}

void RequestProcessor::processBlockingRequest(const ActionRequest& request) {
    // Execute action; possible request errors will be propagated
    auto outcome = modules_[request.module()]->executeAction(request);

    connector_ptr_->sendBlockingResponse(request, outcome.results);
}

void RequestProcessor::processNonBlockingRequest(const ActionRequest& request) {
    // HERE(ale): assuming spool_dir ends with '/' (up to Configuration)
    std::string results_dir { spool_dir_ + request.transactionId() };
    std::string err_msg {};

    LOG_DEBUG("Starting '%1% %2%' job with ID %3% for non-blocking request %4% "
              "by %5%", request.module(), request.action(),
              request.transactionId(), request.id(), request.sender());

    try {
        // Flag to enable signaling from task to thread_container
        auto done = std::make_shared<std::atomic<bool>>(false);

        thread_container_.add(std::thread(&nonBlockingActionTask,
                                          modules_[request.module()],
                                          request,
                                          request.transactionId(),
                                          ResultsStorage { request, results_dir },
                                          connector_ptr_,
                                          done),
                              done);
    } catch (ResultsStorage::Error& e) {
        // Failed to instantiate ResultsStorage
        LOG_ERROR("Failed to initialize the result files for '%1% %2%' action "
                  "job with ID %3%: %4%", request.module(), request.action(),
                  request.transactionId(), e.what());
        err_msg = std::string { "failed to initialize result files: " } + e.what();
    } catch (std::exception& e) {
        LOG_ERROR("Failed to spawn '%1% %2%' action job with ID %3%: %4%",
                  request.module(), request.action(), request.transactionId(),
                  e.what());
        err_msg = std::string { "failed to start action task: " } + e.what();
    }

    if (err_msg.empty()) {
        connector_ptr_->sendProvisionalResponse(request);
    } else {
        connector_ptr_->sendPXPError(request, err_msg);
    }
}

void RequestProcessor::loadModulesConfiguration() {
    LOG_INFO("Loading external modules configuration from %1%",
             modules_config_dir_);

    if (fs::is_directory(modules_config_dir_)) {
        lth_file::each_file(
            modules_config_dir_,
            [this](std::string const& s) -> bool {
                try {
                    fs::path s_path { s };
                    auto file_name = s_path.stem().string();
                    // NB: cfg suffix guaranteed by each_file()
                    auto pos_suffix = file_name.find(".cfg");
                    auto module_name = file_name.substr(0, pos_suffix);
                    modules_config_[module_name] =
                        lth_jc::JsonContainer(lth_file::read(s));
                    LOG_DEBUG("Loaded module configuration for module '%1%' "
                              "from %2%", module_name, s);
                } catch (lth_jc::data_parse_error& e) {
                    LOG_WARNING("Cannot load module config file '%1%'. File "
                                "contains invalid json: %2%", s, e.what());
                }
                return true;
                // naming convention for config files are .cfg. Don't
                // process files that don't end in this extension
            },
            "\\.cfg$");
    } else {
        LOG_WARNING("Directory '%1%' specified by modules-config-dir doesn't "
                    "exist", modules_config_dir_);
    }
}

void RequestProcessor::loadInternalModules() {
    // HERE(ale): no external configuration for internal modules
    modules_["echo"] = std::shared_ptr<Module>(new Modules::Echo);
    modules_["ping"] = std::shared_ptr<Module>(new Modules::Ping);
    modules_["status"] = std::shared_ptr<Module>(new Modules::Status);
}

void RequestProcessor::loadExternalModulesFrom(fs::path dir_path) {
    LOG_INFO("Loading external modules from %1%", dir_path.string());

    if (fs::is_directory(dir_path)) {
        fs::directory_iterator end;

        for (auto f = fs::directory_iterator(dir_path); f != end; ++f) {
            if (!fs::is_directory(f->status())) {
                auto f_p = f->path().string();

                try {
                    ExternalModule* e_m = new ExternalModule(f_p);
                    auto config_itr = modules_config_.find(e_m->module_name);

                    if (config_itr != modules_config_.end()) {
                        e_m->validateAndSetConfiguration(config_itr->second);
                        LOG_DEBUG("The '%1%' module configuration has been "
                                  "validated: %2%", e_m->module_name,
                                  config_itr->second.toString());
                    }

                    modules_[e_m->module_name] = std::shared_ptr<Module>(e_m);
                } catch (Module::LoadingError& e) {
                    LOG_ERROR("Failed to load %1%; %2%", f_p, e.what());
                } catch (PCPClient::validation_error& e) {
                    LOG_ERROR("Failed to configure %1%; %2%", f_p, e.what());
                } catch (std::exception& e) {
                    LOG_ERROR("Unexpected error when loading %1%; %2%",
                              f_p, e.what());
                } catch (...) {
                    LOG_ERROR("Unexpected error when loading %1%", f_p);
                }
            }
        }
    } else {
        LOG_WARNING("Failed to locate the modules directory; no external "
                    "module will be loaded");
    }
}

void RequestProcessor::logLoadedModules() const {
    for (auto& module : modules_) {
        std::string txt { "found no action" };
        std::string actions_list { "" };

        for (auto& action : module.second->actions) {
            if (actions_list.empty()) {
                txt = "action";
                actions_list += ": ";
            } else {
                actions_list += ", ";
            }
            actions_list += action;
        }

        auto txt_suffix = lth_util::plural(module.second->actions.size());
        LOG_INFO("Loaded '%1%' module - %2%%3%%4%",
                 module.first, txt, txt_suffix, actions_list);
    }
}

}  // namespace PXPAgent
<commit_msg>(maint) Fix result dir path<commit_after>#include <pxp-agent/request_processor.hpp>
#include <pxp-agent/action_outcome.hpp>
#include <pxp-agent/pxp_schemas.hpp>
#include <pxp-agent/external_module.hpp>
#include <pxp-agent/modules/echo.hpp>
#include <pxp-agent/modules/ping.hpp>
#include <pxp-agent/modules/status.hpp>

#include <leatherman/json_container/json_container.hpp>
#include <leatherman/file_util/file.hpp>
#include <leatherman/file_util/directory.hpp>
#include <leatherman/util/strings.hpp>
#include <leatherman/util/timer.hpp>

#define LEATHERMAN_LOGGING_NAMESPACE "puppetlabs.pxp_agent.request_processor"
#include <leatherman/logging/logging.hpp>

#include <boost/filesystem/operations.hpp>

#include <vector>
#include <atomic>
#include <functional>
#include <stdexcept>  // out_of_range

namespace PXPAgent {

namespace fs = boost::filesystem;
namespace lth_jc = leatherman::json_container;
namespace lth_file = leatherman::file_util;
namespace lth_util = leatherman::util;

//
// Results Storage
//

class ResultsStorage {
  public:
    struct Error : public std::runtime_error {
        explicit Error(std::string const& msg) : std::runtime_error(msg) {}
    };

    // Throw a ResultsStorage::Error in case of failure while writing
    // to any of result files
    ResultsStorage(const ActionRequest& request, const std::string& results_dir)
            : module { request.module() },
              action { request.action() },
              out_path { results_dir + "/stdout" },
              err_path { results_dir + "/stderr" },
              status_path { results_dir + "/status" },
              action_status {} {
        initialize(request, results_dir);
    }

    void write(const ActionOutcome& outcome, const std::string& exec_error,
               const std::string& duration) {
        action_status.set<std::string>("status", "completed");
        action_status.set<std::string>("duration", duration);
        action_status.set<int>("exitcode", outcome.exitcode);
        lth_file::atomic_write_to_file(action_status.toString() + "\n", status_path);

        if (exec_error.empty()) {
            if (outcome.type == ActionOutcome::Type::External) {
                lth_file::atomic_write_to_file(outcome.std_out + "\n", out_path);
                if (!outcome.std_err.empty()) {
                    lth_file::atomic_write_to_file(outcome.std_err + "\n", err_path);
                }
            } else {
                // ActionOutcome::Type::Internal
                lth_file::atomic_write_to_file(outcome.results.toString()
                                               + "\n", out_path);
            }
        } else {
            lth_file::atomic_write_to_file(exec_error, err_path);
        }
    }

  private:
    std::string module;
    std::string action;
    std::string out_path;
    std::string err_path;
    std::string status_path;
    lth_jc::JsonContainer action_status;

    void initialize(const ActionRequest& request, const std::string results_dir) {
        if (!fs::exists(results_dir)) {
            LOG_DEBUG("Creating results directory for '%1% %2%', transaction "
                       "%3%, in '%4%'", request.module(), request.action(),
                       request.transactionId(), results_dir);
            try {
                fs::create_directories(results_dir);
            } catch (const fs::filesystem_error& e) {
                std::string err_msg { "failed to create results directory: " };
                throw Error { err_msg + e.what() };
            }
        }

        action_status.set<std::string>("module", module);
        action_status.set<std::string>("action", action);
        action_status.set<std::string>("status", "running");
        action_status.set<std::string>("duration", "0 s");
        action_status.set<int>("exitcode", EXIT_SUCCESS);

        if (!request.paramsTxt().empty()) {
            action_status.set<std::string>("input", request.paramsTxt());
        } else {
            action_status.set<std::string>("input", "none");
        }

        lth_file::atomic_write_to_file("", out_path);
        lth_file::atomic_write_to_file("", err_path);
        lth_file::atomic_write_to_file(action_status.toString() + "\n", status_path);
    }
};

//
// Non-blocking action task
//

void nonBlockingActionTask(std::shared_ptr<Module> module_ptr,
                           ActionRequest&& request,
                           std::string job_id,
                           ResultsStorage results_storage,
                           std::shared_ptr<PXPConnector> connector_ptr,
                           std::shared_ptr<std::atomic<bool>> done) {
    lth_util::Timer timer {};
    std::string exec_error {};
    ActionOutcome outcome {};

    try {
        outcome = module_ptr->executeAction(request);

        if (request.parsedChunks().data.get<bool>("notify_outcome")) {
            connector_ptr->sendNonBlockingResponse(request, outcome.results, job_id);
        }
    } catch (Module::ProcessingError& e) {
        connector_ptr->sendPXPError(request, e.what());
        exec_error = "Failed to execute '" + request.module() + " "
                     + request.action() + "': " + e.what() + "\n";
    } catch (PCPClient::connection_error& e) {
        exec_error = "Failed to send non blocking response for '"
                     + request.module() + " " + request.action() + "': "
                     + e.what() + "\n";
    }

    // Store results on disk
    auto duration = std::to_string(timer.elapsed_seconds()) + " s";
    results_storage.write(outcome, exec_error, duration);

    // Flag end of processing
    *done = true;
}

//
// Public interface
//

RequestProcessor::RequestProcessor(std::shared_ptr<PXPConnector> connector_ptr,
                                   const Configuration::Agent& agent_configuration)
        : thread_container_ { "Action Executer" },
          connector_ptr_ { connector_ptr },
          spool_dir_ { agent_configuration.spool_dir },
          modules_ {},
          modules_config_dir_ { agent_configuration.modules_config_dir },
          modules_config_ {} {
    assert(!spool_dir_.empty());

    // NB: certificate paths have been validated by HW

    loadModulesConfiguration();
    loadInternalModules();

    if (!agent_configuration.modules_dir.empty()) {
        loadExternalModulesFrom(agent_configuration.modules_dir);
    } else {
        LOG_WARNING("The modules directory was not provided; no external "
                    "module will be loaded");
    }

    logLoadedModules();
}

void RequestProcessor::processRequest(const RequestType& request_type,
                                      const PCPClient::ParsedChunks& parsed_chunks) {
    try {
        // Inspect and validate the request message format
        ActionRequest request { request_type, parsed_chunks };

        LOG_INFO("About to process %1% request %2% by %3%, transaction %4%",
                 requestTypeNames[request_type], request.id(), request.sender(),
                 request.transactionId());

        try {
            // We can access the request content; validate it
            validateRequestContent(request);
        } catch (RequestProcessor::Error& e) {
            // Invalid request; send *PXP error*

            LOG_ERROR("Invalid %1% request %2% by %3%, transaction %4%: %5%",
                      requestTypeNames[request_type], request.id(),
                      request.sender(), request.transactionId(), e.what());
            connector_ptr_->sendPXPError(request, e.what());
            return;
        }

        try {
            if (request.type() == RequestType::Blocking) {
                processBlockingRequest(request);
            } else {
                processNonBlockingRequest(request);
            }
        } catch (std::exception& e) {
            // Process failure; send *PXP error*
            LOG_ERROR("Failed to process %1% request %2% by %3%, transaction %4%: "
                      "%5%", requestTypeNames[request.type()], request.id(),
                      request.sender(), request.transactionId(), e.what());
            connector_ptr_->sendPXPError(request, e.what());
        }
    } catch (ActionRequest::Error& e) {
        // Failed to instantiate ActionRequest - bad message; send *PCP error*

        auto id = parsed_chunks.envelope.get<std::string>("id");
        auto sender = parsed_chunks.envelope.get<std::string>("sender");
        std::vector<std::string> endpoints { sender };
        LOG_ERROR("Invalid %1% request by %2%: %3%", id, sender, e.what());
        connector_ptr_->sendPCPError(id, e.what(), endpoints);
    }
}

//
// Private interface
//

void RequestProcessor::validateRequestContent(const ActionRequest& request) {
    // Validate requested module and action
    try {
        if (!modules_.at(request.module())->hasAction(request.action())) {
            throw RequestProcessor::Error { "unknown action '" + request.action()
                                            + "' for module " + request.module() };
        }
    } catch (std::out_of_range& e) {
        throw RequestProcessor::Error { "unknown module: " + request.module() };
    }

    // Validate request input params
    try {
        LOG_DEBUG("Validating input for parameters of '%1% %2%' request %3% "
                  "by %4%, transaction %5%", request.module(), request.action(),
                  request.id(), request.sender(), request.transactionId());

        // NB: the registred schemas have the same name as the action
        auto& validator = modules_.at(request.module())->input_validator_;
        validator.validate(request.params(), request.action());
    } catch (PCPClient::validation_error& e) {
        LOG_DEBUG("Invalid '%1% %2%' request %3%: %4%", request.module(),
                  request.action(), request.id(), e.what());
        throw RequestProcessor::Error { "invalid input for '" + request.module()
                                        + " " + request.action() + "'" };
    }
}

void RequestProcessor::processBlockingRequest(const ActionRequest& request) {
    // Execute action; possible request errors will be propagated
    auto outcome = modules_[request.module()]->executeAction(request);

    connector_ptr_->sendBlockingResponse(request, outcome.results);
}

void RequestProcessor::processNonBlockingRequest(const ActionRequest& request) {
    fs::path spool_path { spool_dir_ };
    std::string results_dir { (spool_path / request.transactionId()).string() };
    std::string err_msg {};

    LOG_DEBUG("Starting '%1% %2%' job with ID %3% for non-blocking request %4% "
              "by %5%", request.module(), request.action(),
              request.transactionId(), request.id(), request.sender());

    try {
        // Flag to enable signaling from task to thread_container
        auto done = std::make_shared<std::atomic<bool>>(false);

        thread_container_.add(std::thread(&nonBlockingActionTask,
                                          modules_[request.module()],
                                          request,
                                          request.transactionId(),
                                          ResultsStorage { request, results_dir },
                                          connector_ptr_,
                                          done),
                              done);
    } catch (ResultsStorage::Error& e) {
        // Failed to instantiate ResultsStorage
        LOG_ERROR("Failed to initialize the result files for '%1% %2%' action "
                  "job with ID %3%: %4%", request.module(), request.action(),
                  request.transactionId(), e.what());
        err_msg = std::string { "failed to initialize result files: " } + e.what();
    } catch (std::exception& e) {
        LOG_ERROR("Failed to spawn '%1% %2%' action job with ID %3%: %4%",
                  request.module(), request.action(), request.transactionId(),
                  e.what());
        err_msg = std::string { "failed to start action task: " } + e.what();
    }

    if (err_msg.empty()) {
        connector_ptr_->sendProvisionalResponse(request);
    } else {
        connector_ptr_->sendPXPError(request, err_msg);
    }
}

void RequestProcessor::loadModulesConfiguration() {
    LOG_INFO("Loading external modules configuration from %1%",
             modules_config_dir_);

    if (fs::is_directory(modules_config_dir_)) {
        lth_file::each_file(
            modules_config_dir_,
            [this](std::string const& s) -> bool {
                try {
                    fs::path s_path { s };
                    auto file_name = s_path.stem().string();
                    // NB: cfg suffix guaranteed by each_file()
                    auto pos_suffix = file_name.find(".cfg");
                    auto module_name = file_name.substr(0, pos_suffix);
                    modules_config_[module_name] =
                        lth_jc::JsonContainer(lth_file::read(s));
                    LOG_DEBUG("Loaded module configuration for module '%1%' "
                              "from %2%", module_name, s);
                } catch (lth_jc::data_parse_error& e) {
                    LOG_WARNING("Cannot load module config file '%1%'. File "
                                "contains invalid json: %2%", s, e.what());
                }
                return true;
                // naming convention for config files are .cfg. Don't
                // process files that don't end in this extension
            },
            "\\.cfg$");
    } else {
        LOG_WARNING("Directory '%1%' specified by modules-config-dir doesn't "
                    "exist", modules_config_dir_);
    }
}

void RequestProcessor::loadInternalModules() {
    // HERE(ale): no external configuration for internal modules
    modules_["echo"] = std::shared_ptr<Module>(new Modules::Echo);
    modules_["ping"] = std::shared_ptr<Module>(new Modules::Ping);
    modules_["status"] = std::shared_ptr<Module>(new Modules::Status);
}

void RequestProcessor::loadExternalModulesFrom(fs::path dir_path) {
    LOG_INFO("Loading external modules from %1%", dir_path.string());

    if (fs::is_directory(dir_path)) {
        fs::directory_iterator end;

        for (auto f = fs::directory_iterator(dir_path); f != end; ++f) {
            if (!fs::is_directory(f->status())) {
                auto f_p = f->path().string();

                try {
                    ExternalModule* e_m = new ExternalModule(f_p);
                    auto config_itr = modules_config_.find(e_m->module_name);

                    if (config_itr != modules_config_.end()) {
                        e_m->validateAndSetConfiguration(config_itr->second);
                        LOG_DEBUG("The '%1%' module configuration has been "
                                  "validated: %2%", e_m->module_name,
                                  config_itr->second.toString());
                    }

                    modules_[e_m->module_name] = std::shared_ptr<Module>(e_m);
                } catch (Module::LoadingError& e) {
                    LOG_ERROR("Failed to load %1%; %2%", f_p, e.what());
                } catch (PCPClient::validation_error& e) {
                    LOG_ERROR("Failed to configure %1%; %2%", f_p, e.what());
                } catch (std::exception& e) {
                    LOG_ERROR("Unexpected error when loading %1%; %2%",
                              f_p, e.what());
                } catch (...) {
                    LOG_ERROR("Unexpected error when loading %1%", f_p);
                }
            }
        }
    } else {
        LOG_WARNING("Failed to locate the modules directory; no external "
                    "module will be loaded");
    }
}

void RequestProcessor::logLoadedModules() const {
    for (auto& module : modules_) {
        std::string txt { "found no action" };
        std::string actions_list { "" };

        for (auto& action : module.second->actions) {
            if (actions_list.empty()) {
                txt = "action";
                actions_list += ": ";
            } else {
                actions_list += ", ";
            }
            actions_list += action;
        }

        auto txt_suffix = lth_util::plural(module.second->actions.size());
        LOG_INFO("Loaded '%1%' module - %2%%3%%4%",
                 module.first, txt, txt_suffix, actions_list);
    }
}

}  // namespace PXPAgent
<|endoftext|>


<commit_before>#include "parser.h"

#include <cstddef>
#include <stdexcept>
#include <string>

#include "resyntax/resyntax.h"

namespace tinygrep {

// Find end of parenthesis subexpr in string:
// returns index s.t. [start_index, index) includes both parentheses.
std::size_t find_matching_parenthesis(const std::string& s, std::size_t start_index) {
  std::size_t index = start_index + 1;
  int unmatched_left_parens = 1;
  for (auto it = s.begin() + index; it < s.end() && unmatched_left_parens > 0; ++it, ++index) {
    if (*it == '(') {
      ++unmatched_left_parens;
    } else if (*it == ')') {
      --unmatched_left_parens;
    }
  }
  if (unmatched_left_parens != 0) {
    throw std::domain_error("String is not in the supported grammar: unmatched parenthesis.");
  }
  return index;
}

resyntax::RegExp parse(const std::string s) {
  // TODO: Validate s: check if it belongs to the grammar.
  // TODO: Implement the parsing function based on the pseudocode.

  //  init UNION-list
  //  init CONCAT-list
  //  while charstream is nonempty:
  //    c <- next char
  //    if c == ( :
  //      subexpr <- find subexpr
  //      CONCAT.append: recurse(subexpr)
  //    if c == . :
  //      CONCAT.append: DOT
  //    if c == literal :
  //      CONCAT.append: LITERAL(c)
  //    if c in {+,?,*} :
  //      CONCAT.top.modifywith(c)
  //    if c == | :
  //      cct <- CONCAT.CONCATENATEALL
  //      UNION.append: cct
  //      init CONCAT-list anew
  //    else
  //      throw a_fit
  //  cct <- CONCAT.CONCATENATEALL
  //  UNION.append: cct
  //  regexp <- UNION.UNITEALL
  //  return regexp

  // This is a temporary placeholder construction for verifying that testing is functional.
  resyntax::RegExp a = resyntax::createLiteral('a');
  resyntax::RegExp b = resyntax::createLiteral('b');
  resyntax::RegExp c = resyntax::createLiteral('c');
  resyntax::RegExp a_or_b = resyntax::createUnion(a, b);
  resyntax::RegExp a_or_b_star = resyntax::createClosure(a_or_b);
  resyntax::RegExp tree = resyntax::createConcatenation(a_or_b_star, c);
  return tree;
}

}  // namespace tinygrep
<commit_msg>Add recursion parameters to regex parsing functions.<commit_after>#include "parser.h"

#include <cstddef>
#include <stdexcept>
#include <string>
#include <vector>

#include "resyntax/resyntax.h"

namespace tinygrep {

// Find end of parenthesis subexpr in string:
// returns index s.t. [start_index, index) includes both parentheses.
std::size_t find_matching_parenthesis(const std::string& s, std::size_t start_index) {
  std::size_t index = start_index + 1;
  int unmatched_left_parens = 1;
  for (auto it = s.begin() + index; it < s.end() && unmatched_left_parens > 0; ++it, ++index) {
    if (*it == '(') {
      ++unmatched_left_parens;
    } else if (*it == ')') {
      --unmatched_left_parens;
    }
  }
  if (unmatched_left_parens != 0) {
    throw std::domain_error("String is not in the supported grammar: unmatched parenthesis.");
  }
  return index;
}

// Parse the sub-string s[start_index, end_index) to a RegExp AST.
resyntax::RegExp parse_subexpression(const std::string& s, int start_index, int end_index) {
  // TODO: Implement the parsing function based on the pseudocode.
  //  init UNION-list
  //  init CONCAT-list
  //  while charstream is nonempty:
  //    c <- next char
  //    if c == ( :
  //      subexpr <- find subexpr
  //      CONCAT.append: recurse(subexpr)
  //    if c == . :
  //      CONCAT.append: DOT
  //    if c == literal :
  //      CONCAT.append: LITERAL(c)
  //    if c in {+,?,*} :
  //      CONCAT.top.modifywith(c)
  //    if c == | :
  //      cct <- CONCAT.CONCATENATEALL
  //      UNION.append: cct
  //      init CONCAT-list anew
  //    else
  //      throw a_fit
  //  cct <- CONCAT.CONCATENATEALL
  //  UNION.append: cct
  //  regexp <- UNION.UNITEALL
  //  return regexp
  return resyntax::RegExp();
}

resyntax::RegExp parse(const std::string s) {
  // TODO: Validate s: check if it belongs to the grammar.
/*
  return parse_subexpression(s, 0, s.length());
}
*/
  // This is a temporary placeholder construction for verifying that testing is functional.
  resyntax::RegExp a = resyntax::createLiteral('a');
  resyntax::RegExp b = resyntax::createLiteral('b');
  resyntax::RegExp c = resyntax::createLiteral('c');
  resyntax::RegExp a_or_b = resyntax::createUnion(a, b);
  resyntax::RegExp a_or_b_star = resyntax::createClosure(a_or_b);
  resyntax::RegExp tree = resyntax::createConcatenation(a_or_b_star, c);
  return tree;
}

}  // namespace tinygrep
<|endoftext|>