<commit_before>/*=========================================================================
 *
 *  Copyright 2011-2013 The University of North Carolina at Chapel Hill
 *  All rights reserved.
 *
 *  Licensed under the MADAI Software License. You may obtain a copy of
 *  this license at
 *
 *         https://madai-public.cs.unc.edu/software/license/
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *=========================================================================*/

#include <cstdlib>
#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <algorithm>

#include "ApplicationUtilities.h"
#include "MetropolisHastingsSampler.h"
#include "GaussianProcessEmulatedModel.h"
#include "RuntimeParameterFileReader.h"
#include "Paths.h"
#include "Trace.h"

#include "madaisys/SystemTools.hxx"

/**
generateMCMCtrace
  Generate a trace of length N for a Gaussian Process Model Emulator
 */
static const int DEFAULT_NUMBER_ITERATIONS = 100;
static const int DEFAULT_BURN_IN = 0;
static const double DEFAULT_STEP_SIZE = 0.1;
const char useage [] =
  "Useage:\n"
  "    generateMCMCtrace StatisticsDirectory OutputFileName\n"
  "\n"
  "StatisticsDirectory is the directory in which all statistical data will\n"
  "be stored. Contains the parameter file stat_params.dat.\n"
  "\n"
  "OutputFileName is the name of the file the trace will be stored in. This file\n"
  "will be located in the StatisticsDirectory/trace/ directory.\n"
  "\n"
  "Format of stat_params.dat\n"
  "MODEL_OUTPUT_DIRECTORY <value>\n"
  "EXPERIMENTAL_RESULTS_DIRECTORY <value>\n"
  "MCMC_NUMBER_ITERATIONS <value>\n"
  "MCMC_NUMBER_BURN_IN <value>\n"
  "MCMC_USE_EMULATOR_COVARIANCE <value>\n"
  "MCMC_STEP_SIZE <value>\n"
  "\n"
  "Default values (if not specified) in order of listed:\n"
  "model_output\n"
  "experimental_results\n"
  "100\n"
  "0\n"
  "false\n"
  "0.1\n"
  "\n";

struct GaussianProcessMCMCRuntimeParameters
{
  int numberIter;
  int numberBurnIn;
  bool UseEmulatedCovariance;
  double StepSize;
  std::string ModelOutputDirectory;
  std::string ExperimentalResultsDirectory;
};

bool parseMCMCRuntimeParameters(
    int argc, char** argv,
    struct GaussianProcessMCMCRuntimeParameters & Opts )
{
  // Initialize as defaults
  Opts.ModelOutputDirectory = madai::Paths::MODEL_OUTPUT_DIRECTORY;
  Opts.ExperimentalResultsDirectory = madai::Paths::EXPERIMENTAL_RESULTS_DIRECTORY;
  Opts.numberIter = DEFAULT_NUMBER_ITERATIONS;
  Opts.numberBurnIn = DEFAULT_BURN_IN;
  Opts.UseEmulatedCovariance = false;
  Opts.StepSize = DEFAULT_STEP_SIZE;

  for ( unsigned int i = 0; i < argc; i++ ) {
    std::string argString( argv[i] );

    if ( argString == "MCMC_NUMBER_ITERATIONS" ) {
      Opts.numberIter = atoi(argv[i+1]);
      i++;
    } else if ( argString == "MODEL_OUTPUT_DIRECTORY" ) {
      Opts.ModelOutputDirectory = std::string( argv[i+1] );
      i++;
    } else if ( argString == "EXPERIMENTAL_RESULTS_DIRECTORY" ) {
      Opts.ExperimentalResultsDirectory = std::string( argv[i+1] );
      i++;
    } else if ( argString == "MCMC_NUMBER_BURN_IN" ) {
      Opts.numberBurnIn = atoi(argv[i+1]);
      i++;
    } else if ( argString == "MCMC_USE_EMULATOR_COVARIANCE" ) {
      std::string tstring(argv[i+1]);
      if ( tstring == "false" || tstring == "0" ) {
        Opts.UseEmulatedCovariance = false;
      } else if ( tstring == "true" || tstring == "1" ) {
        Opts.UseEmulatedCovariance = true;
      } else {
        std::cerr << "MCMC_USE_EMULATOR_COVARIANCE: " << tstring << " is invalid\n"
                  << "Setting to false\n";
        Opts.UseEmulatedCovariance = false;
      }
      i++;
    } else if ( argString == "MCMC_STEP_SIZE" ) {
      Opts.StepSize = atof(argv[i+1]);
      i++;
    }
  }
  return true;
}

int main(int argc, char ** argv) {

  if (argc < 3) {
    std::cerr << useage << "\n";
    return EXIT_FAILURE;
  }
  std::string StatisticsDirectory( argv[1] );
  std::string OutputFileName( argv[2] );

  madai::EnsurePathSeparatorAtEnd( StatisticsDirectory );
  madai::RuntimeParameterFileReader RPFR;
  RPFR.ParseFile( StatisticsDirectory + madai::Paths::RUNTIME_PARAMETER_FILE );
  char** Args = RPFR.GetArguments();
  int NArgs = RPFR.GetNumberOfArguments();
  struct GaussianProcessMCMCRuntimeParameters Opts;
  if ( !parseMCMCRuntimeParameters( NArgs, Args, Opts ) ) {
    std::cerr << "Error: Parsing configuration file for gaussian process mcmc.\n";
    return EXIT_FAILURE;
  }
  std::string observationsFile = Opts.ExperimentalResultsDirectory +
    madai::Paths::SEPARATOR + madai::Paths::RESULTS_FILE;

  madai::GaussianProcessEmulatedModel gpem;
  std::string MOD = StatisticsDirectory + Opts.ModelOutputDirectory;
  std::string ERD = StatisticsDirectory + Opts.ExperimentalResultsDirectory;
  if ( gpem.LoadConfiguration( StatisticsDirectory, MOD, ERD ) != madai::Model::NO_ERROR ) {
    std::cerr << "Error in GaussianProcessEmulatedModel::LoadConfiguration\n";
    return EXIT_FAILURE;
  }

  gpem.SetUseModelCovarianceToCalulateLogLikelihood(Opts.UseEmulatedCovariance);

  std::ifstream observations(observationsFile.c_str());
  if (madai::Model::NO_ERROR != gpem.LoadObservations(observations)) {
    std::cerr << "error loading observations.\n";
    return EXIT_FAILURE;
  }
  observations.close();

  madai::MetropolisHastingsSampler mcmc;
  mcmc.SetModel( &gpem );
  mcmc.SetStepSize(Opts.StepSize);

  std::vector< madai::Parameter > const & parameters
    = gpem.GetParameters();

  int t = gpem.GetNumberOfScalarOutputs();

  int step = Opts.numberBurnIn / 100, percent = 0;
  if (step < 1)
    step = 1; // avoid div-by-zero error;
  for ( int count = 0; count < Opts.numberBurnIn; count++ ) {
    if ( count % step == 0 )
      std::cerr << '\r' << "Burn in done: " << percent++ << "%";
    mcmc.NextSample();
  }
  step = Opts.numberIter / 100, percent = 0;

  std::vector< madai::Sample> samples;
  for (int count = 0; count < Opts.numberIter; count ++) {
    if (count % step == 0)
      std::cerr <<  '\r' << "MCMC percent done: " << percent++ << "%";
    samples.push_back(mcmc.NextSample());
  }
  std::cerr << "\r" ;

  madai::Trace trace;
  for (std::vector< madai::Sample >::const_iterator it = samples.begin() ;
       it != samples.end(); ++it) {
    trace.Add( *it );
  }
  std::string traceDirectory = StatisticsDirectory + madai::Paths::TRACE_DIRECTORY;
  madaisys::SystemTools::MakeDirectory( traceDirectory.c_str() );
  std::string OutputFile = traceDirectory+OutputFileName;
  std::ofstream Out( OutputFile.c_str() );
  trace.WriteCSVOutput(
      Out,
      gpem.GetParameters(),
      gpem.GetScalarOutputNames() );
  return EXIT_SUCCESS;
}
<commit_msg>Removed extra newline in help statement<commit_after>/*=========================================================================
 *
 *  Copyright 2011-2013 The University of North Carolina at Chapel Hill
 *  All rights reserved.
 *
 *  Licensed under the MADAI Software License. You may obtain a copy of
 *  this license at
 *
 *         https://madai-public.cs.unc.edu/software/license/
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *=========================================================================*/

#include <cstdlib>
#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <algorithm>

#include "ApplicationUtilities.h"
#include "MetropolisHastingsSampler.h"
#include "GaussianProcessEmulatedModel.h"
#include "RuntimeParameterFileReader.h"
#include "Paths.h"
#include "Trace.h"

#include "madaisys/SystemTools.hxx"

/**
generateMCMCtrace
  Generate a trace of length N for a Gaussian Process Model Emulator
 */
static const int DEFAULT_NUMBER_ITERATIONS = 100;
static const int DEFAULT_BURN_IN = 0;
static const double DEFAULT_STEP_SIZE = 0.1;
const char useage [] =
  "Useage:\n"
  "    generateMCMCtrace StatisticsDirectory OutputFileName\n"
  "\n"
  "StatisticsDirectory is the directory in which all statistical data will\n"
  "be stored. Contains the parameter file stat_params.dat.\n"
  "\n"
  "OutputFileName is the name of the file the trace will be stored in. This file\n"
  "will be located in the StatisticsDirectory/trace/ directory.\n"
  "\n"
  "Format of stat_params.dat\n"
  "MODEL_OUTPUT_DIRECTORY <value>\n"
  "EXPERIMENTAL_RESULTS_DIRECTORY <value>\n"
  "MCMC_NUMBER_ITERATIONS <value>\n"
  "MCMC_NUMBER_BURN_IN <value>\n"
  "MCMC_USE_EMULATOR_COVARIANCE <value>\n"
  "MCMC_STEP_SIZE <value>\n"
  "\n"
  "Default values (if not specified) in order of listed:\n"
  "model_output\n"
  "experimental_results\n"
  "100\n"
  "0\n"
  "false\n"
  "0.1\n";

struct GaussianProcessMCMCRuntimeParameters
{
  int numberIter;
  int numberBurnIn;
  bool UseEmulatedCovariance;
  double StepSize;
  std::string ModelOutputDirectory;
  std::string ExperimentalResultsDirectory;
};

bool parseMCMCRuntimeParameters(
    int argc, char** argv,
    struct GaussianProcessMCMCRuntimeParameters & Opts )
{
  // Initialize as defaults
  Opts.ModelOutputDirectory = madai::Paths::MODEL_OUTPUT_DIRECTORY;
  Opts.ExperimentalResultsDirectory = madai::Paths::EXPERIMENTAL_RESULTS_DIRECTORY;
  Opts.numberIter = DEFAULT_NUMBER_ITERATIONS;
  Opts.numberBurnIn = DEFAULT_BURN_IN;
  Opts.UseEmulatedCovariance = false;
  Opts.StepSize = DEFAULT_STEP_SIZE;

  for ( unsigned int i = 0; i < argc; i++ ) {
    std::string argString( argv[i] );

    if ( argString == "MCMC_NUMBER_ITERATIONS" ) {
      Opts.numberIter = atoi(argv[i+1]);
      i++;
    } else if ( argString == "MODEL_OUTPUT_DIRECTORY" ) {
      Opts.ModelOutputDirectory = std::string( argv[i+1] );
      i++;
    } else if ( argString == "EXPERIMENTAL_RESULTS_DIRECTORY" ) {
      Opts.ExperimentalResultsDirectory = std::string( argv[i+1] );
      i++;
    } else if ( argString == "MCMC_NUMBER_BURN_IN" ) {
      Opts.numberBurnIn = atoi(argv[i+1]);
      i++;
    } else if ( argString == "MCMC_USE_EMULATOR_COVARIANCE" ) {
      std::string tstring(argv[i+1]);
      if ( tstring == "false" || tstring == "0" ) {
        Opts.UseEmulatedCovariance = false;
      } else if ( tstring == "true" || tstring == "1" ) {
        Opts.UseEmulatedCovariance = true;
      } else {
        std::cerr << "MCMC_USE_EMULATOR_COVARIANCE: " << tstring << " is invalid\n"
                  << "Setting to false\n";
        Opts.UseEmulatedCovariance = false;
      }
      i++;
    } else if ( argString == "MCMC_STEP_SIZE" ) {
      Opts.StepSize = atof(argv[i+1]);
      i++;
    }
  }
  return true;
}

int main(int argc, char ** argv) {

  if (argc < 3) {
    std::cerr << useage << "\n";
    return EXIT_FAILURE;
  }
  std::string StatisticsDirectory( argv[1] );
  std::string OutputFileName( argv[2] );

  madai::EnsurePathSeparatorAtEnd( StatisticsDirectory );
  madai::RuntimeParameterFileReader RPFR;
  RPFR.ParseFile( StatisticsDirectory + madai::Paths::RUNTIME_PARAMETER_FILE );
  char** Args = RPFR.GetArguments();
  int NArgs = RPFR.GetNumberOfArguments();
  struct GaussianProcessMCMCRuntimeParameters Opts;
  if ( !parseMCMCRuntimeParameters( NArgs, Args, Opts ) ) {
    std::cerr << "Error: Parsing configuration file for gaussian process mcmc.\n";
    return EXIT_FAILURE;
  }
  std::string observationsFile = Opts.ExperimentalResultsDirectory +
    madai::Paths::SEPARATOR + madai::Paths::RESULTS_FILE;

  madai::GaussianProcessEmulatedModel gpem;
  std::string MOD = StatisticsDirectory + Opts.ModelOutputDirectory;
  std::string ERD = StatisticsDirectory + Opts.ExperimentalResultsDirectory;
  if ( gpem.LoadConfiguration( StatisticsDirectory, MOD, ERD ) != madai::Model::NO_ERROR ) {
    std::cerr << "Error in GaussianProcessEmulatedModel::LoadConfiguration\n";
    return EXIT_FAILURE;
  }

  gpem.SetUseModelCovarianceToCalulateLogLikelihood(Opts.UseEmulatedCovariance);

  std::ifstream observations(observationsFile.c_str());
  if (madai::Model::NO_ERROR != gpem.LoadObservations(observations)) {
    std::cerr << "error loading observations.\n";
    return EXIT_FAILURE;
  }
  observations.close();

  madai::MetropolisHastingsSampler mcmc;
  mcmc.SetModel( &gpem );
  mcmc.SetStepSize(Opts.StepSize);

  std::vector< madai::Parameter > const & parameters
    = gpem.GetParameters();

  int t = gpem.GetNumberOfScalarOutputs();

  int step = Opts.numberBurnIn / 100, percent = 0;
  if (step < 1)
    step = 1; // avoid div-by-zero error;
  for ( int count = 0; count < Opts.numberBurnIn; count++ ) {
    if ( count % step == 0 )
      std::cerr << '\r' << "Burn in done: " << percent++ << "%";
    mcmc.NextSample();
  }
  step = Opts.numberIter / 100, percent = 0;

  std::vector< madai::Sample> samples;
  for (int count = 0; count < Opts.numberIter; count ++) {
    if (count % step == 0)
      std::cerr <<  '\r' << "MCMC percent done: " << percent++ << "%";
    samples.push_back(mcmc.NextSample());
  }
  std::cerr << "\r" ;

  madai::Trace trace;
  for (std::vector< madai::Sample >::const_iterator it = samples.begin() ;
       it != samples.end(); ++it) {
    trace.Add( *it );
  }
  std::string traceDirectory = StatisticsDirectory + madai::Paths::TRACE_DIRECTORY;
  madaisys::SystemTools::MakeDirectory( traceDirectory.c_str() );
  std::string OutputFile = traceDirectory+OutputFileName;
  std::ofstream Out( OutputFile.c_str() );
  trace.WriteCSVOutput(
      Out,
      gpem.GetParameters(),
      gpem.GetScalarOutputNames() );
  return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>#pragma once
#include <Grappa.hpp>
#include <graph/Graph.hpp>
#include <Reducer.hpp>
#include <SmallLocalSet.hpp>

#include <unordered_set>
#include <unordered_map>
#include <vector>
using std::unordered_set;
using std::unordered_map;
using std::vector;

using CoreSet = SmallLocalSet<Core>;

// #include "cuckoo_set_pow2.hpp"
// using graphlab::cuckoo_set_pow2;
//using CoreSet = cuckoo_set_pow2<Core>;

GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, iteration_time);
DECLARE_int32(max_iterations);

using namespace Grappa;
using delegate::call;

using Empty = struct {};

namespace Graphlab {

  template< typename V, typename E >
  class Graph {
    GlobalAddress<Graph> self;
  public:
    static GlobalAddress<Graph> create(TupleGraph tg) {
      VLOG(1) << "Graphlab::Graph::create( undirected, greedy_oblivious )";
      auto g = symmetric_global_alloc<Graph>();

      on_all_cores([=]{
        g->self = g;

        // vertex placements that this core knows about (array of cores, each with a set of vertices mapped to it)
        unordered_map<VertexID,CoreSet> vplace;
        auto local_edges = iterate_local(tg.edges, tg.nedge);
        auto nlocal = local_edges.size();
        auto idx = [&](TupleGraph::Edge& e){ return &e - local_edges.begin(); };

        std::vector<Core> assignments; assignments.resize(local_edges.size());
        size_t* edge_cts = locale_alloc<size_t>(cores());
        Grappa::memset(edge_cts, 0, cores());

        /// cores this vertex has been placed on
        auto vcores = [&](VertexID i) -> CoreSet& {
          if (vplace.count(i) == 0) {
            vplace[i];
          }
          return vplace[i];
        };

        auto assign = [&](TupleGraph::Edge& e, Core c) {
          CHECK_LT(c, cores());
          CHECK_LT(idx(e), nlocal);
          assignments[idx(e)] = c;
          edge_cts[c]++;
          for (auto v : {e.v0, e.v1}) {
            if (vplace[v].count(c) == 0) {
              CHECK_GE(c, 0);
              vplace[v].insert(c);
            }
          }
        };

        auto cmp_load = [&](Core c0, Core c1) { return edge_cts[c0] < edge_cts[c1]; };

        for (auto& e : local_edges) {
          auto &vs0 = vcores(e.v0), &vs1 = vcores(e.v1);

          auto common = intersect_choose_random(vs0, vs1);
          if (common != CoreSet::INVALID) {
            // place this edge on 'common' because both vertices are already there
            assign(e, common);
          } else if (!vs0.empty() && !vs1.empty()) {
            // both assigned but no intersect, choose the least-loaded machine
            assign(e, min_element(vs0, vs1, cmp_load));
          } else if (!vs0.empty()) {
            // only v0 assigned, choose one of its cores
            assign(e, min_element(vs0, cmp_load));
          } else if (!vs1.empty()) {
            // only v1 assigned, choose one of its cores
            assign(e, min_element(vs1, cmp_load));
          } else {
            // neither assigned, so pick overall least-loaded core
            auto c = min_element(Range<Core>{0,cores()}, cmp_load);
            // prefer spreading out over all cores
            if (edge_cts[mycore()] <= edge_cts[c]) c = mycore();
            assign(e, c);
          }
        }

        allreduce_inplace<size_t,collective_add>(&edge_cts[0], cores());

        if (mycore() == 0) {
          std::cerr << util::array_str("edge_cts", edge_cts, cores()) << "\n";
        }
      });

      // forall(tg.edges, tg.nedge, [](TupleGraph::Edge& e){
      //
      // });

      return g;
    }
  } GRAPPA_BLOCK_ALIGNED;

}

template< typename G, typename GatherType >
struct GraphlabVertexProgram {
  using Vertex = typename G::Vertex;
  using Edge = typename G::Edge;
  using Gather = GatherType;

  GatherType cache;

  GraphlabVertexProgram(): cache() {}

  void post_delta(GatherType d){ cache += d; }
};

template< typename Subclass >
struct GraphlabVertexData {
  static Reducer<int64_t,ReducerType::Add> total_active;

  void* prog;
  bool active, active_minor_step;

  GraphlabVertexData(): active(false) {}
  void activate() { if (!active) { total_active++; active = true; } }
  void deactivate() { if (active) { total_active--; active = false; } }
};

template< typename Subclass >
Reducer<int64_t,ReducerType::Add> GraphlabVertexData<Subclass>::total_active;

template< typename V, typename E >
void activate_all(GlobalAddress<Graph<V,E>> g) {
  forall(g, [](typename Graph<V,E>::Vertex& v){ v->activate(); });
}

template< typename V >
void activate(GlobalAddress<V> v) {
  delegate::call(v, [](V& v){ v->activate(); });
}

////////////////////////////////////////////////////////
/// Synchronous GraphLab engine, assumes:
/// - Delta caching enabled
/// - (currently) gather_edges:IN_EDGES, scatter_edges:(OUT_EDGES || NONE)
///
/// Also requires that the Graph already contains the GraphlabVertexProgram
template< typename VertexProg, typename V, typename E >
void run_synchronous(GlobalAddress<Graph<V,E>> g) {
#define GVertex typename Graph<V,E>::Vertex
#define GEdge typename Graph<V,E>::Edge
  auto prog = [](GVertex& v) -> VertexProg& {
    return *static_cast<VertexProg*>(v->prog);
  };

  // // tack the VertexProg data onto the existing vertex data
  // struct VPlus : public V {
  //   VertexProg prog;
  //   VPlus(typename Graph<V,E>::Vertex& v): V(v.data), prog(v) {}
  // };
  // auto g = g->template transform<VPlus>([](typename Graph<V,E>::Vertex& v, VPlus& d){
  //   new (&d) VPlus(v);
  // });
  // using GPVertex = typename Graph<VPlus,E>::Vertex;
  // using GPEdge = typename Graph<VPlus,E>::Edge;

  // "gather" once to initialize cache (doing with a scatter)

  // TODO: find efficient way to skip 'gather' if 'gather_edges' is always false

  // initialize GraphlabVertexProgram
  forall(g, [=](GVertex& v){ v->prog = new VertexProg(v); });

  forall(g, [=](GVertex& v){
    forall<async>(adj(g,v), [=,&v](GEdge& e){
      // gather
      auto delta = prog(v).gather(v, e);

      call<async>(e.ga, [=](GVertex& ve){
        prog(ve).post_delta(delta);
      });
    });
  });

  int iteration = 0;

  while ( V::total_active > 0 && iteration < FLAGS_max_iterations )
      GRAPPA_TIME_REGION(iteration_time) {

    VLOG(1) << "iteration " << std::setw(3) << iteration
            << " -- active:" << V::total_active;
    double t = walltime();

    forall(g, [=](GVertex& v){
      if (v->active) {
        v->active_minor_step = true;
        v->deactivate();
      }
    });

    forall(g, [=](GVertex& v){
      if (!v->active_minor_step) return;

      auto& p = prog(v);

      // apply
      p.apply(v, p.cache);

      v->active_minor_step = p.scatter_edges(v);
    });

    forall(g, [=](GVertex& v){
      if (v->active_minor_step) {
        auto prog_copy = prog(v);
        // scatter
        forall<async>(adj(g,v), [=](GEdge& e){
          auto e_id = e.id;
          auto e_data = e.data;
          call<async>(e.ga, [=](GVertex& ve){
            auto local_e_data = e_data;
            GEdge e = { e_id, g->vs+e_id, local_e_data };
            auto gather_delta = prog_copy.scatter(e, ve);
            prog(ve).post_delta(gather_delta);
          });
        });
      }
    });

    iteration++;
    VLOG(1) << "> time: " << walltime()-t;
  }

  forall(g, [](GVertex& v){ delete static_cast<VertexProg*>(v->prog); });
}
<commit_msg>scatter edges, sort them, build vertex info on each core<commit_after>#pragma once
#include <Grappa.hpp>
#include <graph/Graph.hpp>
#include <Reducer.hpp>
#include <SmallLocalSet.hpp>

#include <unordered_set>
#include <unordered_map>
#include <vector>
using std::unordered_set;
using std::unordered_map;
using std::vector;

using CoreSet = SmallLocalSet<Core>;

// #include "cuckoo_set_pow2.hpp"
// using graphlab::cuckoo_set_pow2;
//using CoreSet = cuckoo_set_pow2<Core>;

GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, iteration_time);
DECLARE_int32(max_iterations);

using namespace Grappa;
using delegate::call;

using Empty = struct {};

namespace Graphlab {

  template< typename V, typename E >
  class Graph {

    struct Edge {
      VertexID src, dst;
      E data;
      Edge() = default;
      Edge(const TupleGraph::Edge& e): src(e.v0), dst(e.v1) {}

      friend std::ostream& operator<<(std::ostream& o, const Edge& e) {
        return o << "<" << e.src << "," << e.dst << ">";
      }
    };

    struct Vertex {
      VertexID id;
      V data;
      Edge * l_out;
      size_t l_nout;

      Vertex(VertexID id): id(id) {}
    };

    GlobalAddress<Graph> self;

    std::vector<Edge> l_edges;   ///< Local edges
    std::vector<Vertex> l_verts; ///< Local vertices (indexes into l_edges)
    size_t l_nsrc;             ///< Number of source vertices

    unordered_map<VertexID,Vertex*> l_vid;

    Graph(GlobalAddress<Graph> self)
      : self(self) , l_edges() , l_verts() , l_nsrc(0) , l_vid()
    { }

  public:
    Graph() = default;

    static GlobalAddress<Graph> create(TupleGraph tg) {
      VLOG(1) << "Graphlab::Graph::create( undirected, greedy_oblivious )";
      auto g = symmetric_global_alloc<Graph>();

      on_all_cores([=]{
        // intialize graph
        new (g.localize()) Graph(g);

        // vertex placements that this core knows about (array of cores, each with a set of vertices mapped to it)
        unordered_map<VertexID,CoreSet> vplace;
        auto local_edges = iterate_local(tg.edges, tg.nedge);
        auto nlocal = local_edges.size();
        auto idx = [&](TupleGraph::Edge& e){ return &e - local_edges.begin(); };

        std::vector<Core> assignments; assignments.resize(local_edges.size());
        size_t* edge_cts = locale_alloc<size_t>(cores());
        Grappa::memset(edge_cts, 0, cores());

        /// cores this vertex has been placed on
        auto vcores = [&](VertexID i) -> CoreSet& {
          if (vplace.count(i) == 0) {
            vplace[i];
          }
          return vplace[i];
        };

        auto assign = [&](TupleGraph::Edge& e, Core c) {
          CHECK_LT(c, cores());
          CHECK_LT(idx(e), nlocal);
          assignments[idx(e)] = c;
          edge_cts[c]++;
          for (auto v : {e.v0, e.v1}) {
            if (vplace[v].count(c) == 0) {
              CHECK_GE(c, 0);
              vplace[v].insert(c);
            }
          }
        };

        auto cmp_load = [&](Core c0, Core c1) { return edge_cts[c0] < edge_cts[c1]; };

        for (auto& e : local_edges) {
          auto &vs0 = vcores(e.v0), &vs1 = vcores(e.v1);

          auto common = intersect_choose_random(vs0, vs1);
          if (common != CoreSet::INVALID) {
            // place this edge on 'common' because both vertices are already there
            assign(e, common);
          } else if (!vs0.empty() && !vs1.empty()) {
            // both assigned but no intersect, choose the least-loaded machine
            assign(e, min_element(vs0, vs1, cmp_load));
          } else if (!vs0.empty()) {
            // only v0 assigned, choose one of its cores
            assign(e, min_element(vs0, cmp_load));
          } else if (!vs1.empty()) {
            // only v1 assigned, choose one of its cores
            assign(e, min_element(vs1, cmp_load));
          } else {
            // neither assigned, so pick overall least-loaded core
            auto c = min_element(Range<Core>{0,cores()}, cmp_load);
            // prefer spreading out over all cores
            if (edge_cts[mycore()] <= edge_cts[c]) c = mycore();
            assign(e, c);
          }
        }

        allreduce_inplace<size_t,collective_add>(&edge_cts[0], cores());

        if (mycore() == 0) {
          std::cerr << util::array_str("edge_cts", edge_cts, cores()) << "\n";
        }

        //////////////////////////
        // actually scatter edges
        auto& edges = g->l_edges;

        edges.reserve(edge_cts[mycore()]);
        barrier();

        finish([&]{
          for (auto& e : local_edges) {
            auto e_copy = e;
            call<async>(assignments[idx(e)], [e_copy,g]{
              g->l_edges.emplace_back(e_copy);
            });
          }
        });

        // if (mycore() == 0) global_free(tg.edges);
        std::sort(edges.begin(), edges.end(), [](const Edge& a, const Edge& b){
          if (a.src == b.src) return a.dst < b.dst;
          else return a.src < b.src;
        });

        auto it = std::unique(edges.begin(), edges.end(), [](const Edge& a, const Edge& b){
          return (a.src == b.src) && (a.dst == b.dst);
        });
        edges.resize(std::distance(edges.begin(), it));

        VLOG(0) << "l_edges: " << edges;

        auto& l_vid = g->l_vid;
        auto& lvs = g->l_verts;

        // so we can pre-size l_verts and get all the hashtable allocs over with
        for (auto& e : edges) {
          for (auto v : {e.src, e.dst}) {
            l_vid[v] = nullptr;
          }
        }
        lvs.reserve(l_vid.size());

        VertexID src = -1;
        Vertex* cv;

        for (Edge& e : edges) {
          if (src != e.src) {
            if (src >= 0) {
              // finish previous one
              cv->l_nout = &e - cv->l_out;
            }
            // start of new src
            lvs.emplace_back(e.src);
            cv = &lvs.back();
            l_vid[e.src] = cv;
            cv->l_out = &e;
            src = e.src;
          }
        }
        cv->l_nout = &*g->l_edges.end() - cv->l_out;
        g->l_nsrc = lvs.size(); // vertices that have at least one outgoing edge are first

        // now go add all the ones that are destination-only (on this core)
        for (auto& e : g->l_edges) {
          if (l_vid[e.dst] == nullptr) {
            lvs.emplace_back(e.dst);
            cv = &lvs.back();
            l_vid[e.dst] = cv;
            cv->l_out = nullptr;
            cv->l_nout = 0;
          }
        }

        // for (auto& v : lvs) {
        //   VLOG(0) << "{id:" << v.id << ", l_out:" << v.l_out << ", l_nout:" << v.l_nout << "}";
        // }

        // VLOG(0) << "vertices, edges => " << g->l_vid.size() << ", " << edge_cts[mycore()];

        size_t total_verts = allreduce<int64_t,collective_add>(g->l_vid.size());
        if (mycore() == 0) VLOG(0) << "total_vert_ct => " << total_verts;
      }); // on_all_cores

      // forall(tg.edges, tg.nedge, [](TupleGraph::Edge& e){
      //
      // });

      return g;
    }
  } GRAPPA_BLOCK_ALIGNED;

}

template< typename G, typename GatherType >
struct GraphlabVertexProgram {
  using Vertex = typename G::Vertex;
  using Edge = typename G::Edge;
  using Gather = GatherType;

  GatherType cache;

  GraphlabVertexProgram(): cache() {}

  void post_delta(GatherType d){ cache += d; }
};

template< typename Subclass >
struct GraphlabVertexData {
  static Reducer<int64_t,ReducerType::Add> total_active;

  void* prog;
  bool active, active_minor_step;

  GraphlabVertexData(): active(false) {}
  void activate() { if (!active) { total_active++; active = true; } }
  void deactivate() { if (active) { total_active--; active = false; } }
};

template< typename Subclass >
Reducer<int64_t,ReducerType::Add> GraphlabVertexData<Subclass>::total_active;

template< typename V, typename E >
void activate_all(GlobalAddress<Graph<V,E>> g) {
  forall(g, [](typename Graph<V,E>::Vertex& v){ v->activate(); });
}

template< typename V >
void activate(GlobalAddress<V> v) {
  delegate::call(v, [](V& v){ v->activate(); });
}

////////////////////////////////////////////////////////
/// Synchronous GraphLab engine, assumes:
/// - Delta caching enabled
/// - (currently) gather_edges:IN_EDGES, scatter_edges:(OUT_EDGES || NONE)
///
/// Also requires that the Graph already contains the GraphlabVertexProgram
template< typename VertexProg, typename V, typename E >
void run_synchronous(GlobalAddress<Graph<V,E>> g) {
#define GVertex typename Graph<V,E>::Vertex
#define GEdge typename Graph<V,E>::Edge
  auto prog = [](GVertex& v) -> VertexProg& {
    return *static_cast<VertexProg*>(v->prog);
  };

  // // tack the VertexProg data onto the existing vertex data
  // struct VPlus : public V {
  //   VertexProg prog;
  //   VPlus(typename Graph<V,E>::Vertex& v): V(v.data), prog(v) {}
  // };
  // auto g = g->template transform<VPlus>([](typename Graph<V,E>::Vertex& v, VPlus& d){
  //   new (&d) VPlus(v);
  // });
  // using GPVertex = typename Graph<VPlus,E>::Vertex;
  // using GPEdge = typename Graph<VPlus,E>::Edge;

  // "gather" once to initialize cache (doing with a scatter)

  // TODO: find efficient way to skip 'gather' if 'gather_edges' is always false

  // initialize GraphlabVertexProgram
  forall(g, [=](GVertex& v){ v->prog = new VertexProg(v); });

  forall(g, [=](GVertex& v){
    forall<async>(adj(g,v), [=,&v](GEdge& e){
      // gather
      auto delta = prog(v).gather(v, e);

      call<async>(e.ga, [=](GVertex& ve){
        prog(ve).post_delta(delta);
      });
    });
  });

  int iteration = 0;

  while ( V::total_active > 0 && iteration < FLAGS_max_iterations )
      GRAPPA_TIME_REGION(iteration_time) {

    VLOG(1) << "iteration " << std::setw(3) << iteration
            << " -- active:" << V::total_active;
    double t = walltime();

    forall(g, [=](GVertex& v){
      if (v->active) {
        v->active_minor_step = true;
        v->deactivate();
      }
    });

    forall(g, [=](GVertex& v){
      if (!v->active_minor_step) return;

      auto& p = prog(v);

      // apply
      p.apply(v, p.cache);

      v->active_minor_step = p.scatter_edges(v);
    });

    forall(g, [=](GVertex& v){
      if (v->active_minor_step) {
        auto prog_copy = prog(v);
        // scatter
        forall<async>(adj(g,v), [=](GEdge& e){
          auto e_id = e.id;
          auto e_data = e.data;
          call<async>(e.ga, [=](GVertex& ve){
            auto local_e_data = e_data;
            GEdge e = { e_id, g->vs+e_id, local_e_data };
            auto gather_delta = prog_copy.scatter(e, ve);
            prog(ve).post_delta(gather_delta);
          });
        });
      }
    });

    iteration++;
    VLOG(1) << "> time: " << walltime()-t;
  }

  forall(g, [](GVertex& v){ delete static_cast<VertexProg*>(v->prog); });
}
<|endoftext|>


<commit_before>#include <pcl/point_types.h>
#include <pcl/io/pcd_io.h>
#include <pcl/filters/voxel_grid.h>
#include <pcl/filters/statistical_outlier_removal.h>
#include <pcl/kdtree/kdtree_flann.h>
#include <pcl/features/normal_3d.h>
#include <pcl/surface/poisson.h>
#include <pcl/io/vtk_io.h>
#include <boost/thread/thread.hpp>
#include <pcl/common/common_headers.h>
#include <pcl/visualization/pcl_visualizer.h>

typedef pcl::Normal Normal;
//typedef pcl::PointXYZ PointType;
//typedef pcl::PointNormal PointTypeN;
typedef pcl::PointXYZRGB PointType;
typedef pcl::PointXYZRGBNormal PointTypeN;

void downsample (int, char*[]);
void remove_outliers (int, char* []);
void reconstruct_mesh (int, char* [], pcl::PolygonMesh&);
void show_mesh (const pcl::PolygonMesh&);

int main (int argc, char *argv[])
{
    downsample (argc, argv);
    remove_outliers (argc, argv);
    pcl::PolygonMesh mesh_of_triangles;
    reconstruct_mesh (argc, argv, mesh_of_triangles);
    show_mesh (mesh_of_triangles);

    return 0;
}

/* Downsampling with VoxelGrid class.
 * Voxel grid is a set of tiny 3D boxes in space.
 * In each voxel (i.e. 3D box) all points present will be approximated
 * (i.e. downsampled) with their centeroid.
 */

void downsample (int argc, char* argv[])
{
    std::cout << "Started - downsample() with VoxelGrid" << std::endl;

    pcl::PCLPointCloud2::Ptr cloud (new pcl::PCLPointCloud2());
    pcl::PCLPointCloud2::Ptr cloud_filtered (new pcl::PCLPointCloud2());

    // Fill in the cloud data
    pcl::PCDReader reader;
    if (argc < 2){
        reader.read ("pointcloud.pcd", *cloud);
    }
    else {
        reader.read (argv[1], *cloud);
    }

    std::cout << "PointCloud before filtering: ";
    std::cout << cloud->width * cloud->height << " data points (" ;
    std::cout << pcl::getFieldsList (*cloud) << ")." << std::endl;

    // Create the filtering object
    pcl::VoxelGrid<pcl::PCLPointCloud2> sor;
    sor.setInputCloud (cloud);
    // voxel size to be 1cm^3
    sor.setLeafSize (0.01f, 0.01f, 0.01f);
    sor.filter (*cloud_filtered);

    std::cout << "PointCloud after filtering: ";
    std::cout << cloud_filtered->width * cloud_filtered->height;
    std::cout << " data points ("  << pcl::getFieldsList (*cloud) << ").\n";

    pcl::PCDWriter writer;
    if (argc < 2){
        writer.write ("pointcloud-downsampled.pcd",
                *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd");
        writer.write (str, *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }

    std::cout << "Finished - downsample() with VoxelGrid\n" << std::endl;

}

/* Removing noisy data (i.e. outliers) from measurements using statistical
 * analysis technique
 */

void remove_outliers (int argc, char* argv[])
{
    std::cout << "Started - remove_outliers() with " <<
        "StatisticalOutlierRemoval" << std::endl;

    pcl::PCLPointCloud2::Ptr cloud (new pcl::PCLPointCloud2());
    pcl::PCLPointCloud2::Ptr cloud_filtered (new pcl::PCLPointCloud2());

    // Fill in the cloud data
    pcl::PCDReader reader;
    if (argc < 2){
        reader.read ("pointcloud-downsampled.pcd", *cloud);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd");
        reader.read (str, *cloud);
    }

    std::cout << "PointCloud before filtering: ";
    std::cout << cloud->width * cloud->height << " data points (" ;
    std::cout << pcl::getFieldsList (*cloud) << ")." << std::endl;

    // Create the filtering object
    pcl::StatisticalOutlierRemoval<pcl::PCLPointCloud2> sor2;
    sor2.setInputCloud (cloud);
    // Set number of neighbors to analyze
    sor2.setMeanK (50);
    sor2.setStddevMulThresh (1.0);
    sor2.filter (*cloud_filtered);

    std::cout << "PointCloud after filtering: ";
    std::cout << cloud_filtered->width * cloud_filtered->height;
    std::cout << " data points ("  << pcl::getFieldsList (*cloud) << ").\n";

    pcl::PCDWriter writer;

    // Write both inliers and outliers
    if (argc < 2){
        writer.write ("pointcloud-downsampled-inliers.pcd",
                *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);

        sor2.setNegative (true);
        sor2.filter (*cloud_filtered);
        writer.write ("pointcloud-downsampled-outliers.pcd",
                *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }
    else {
        std::string str1;
        str1.append(argv[1]).append("-downsampled.pcd-inliers.pcd");
        std::string str2;
        str2.append(argv[1]).append("-downsampled.pcd-outliers.pcd");
        writer.write (str1, *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);

        sor2.setNegative (true);
        sor2.filter (*cloud_filtered);
        writer.write (str2, *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }

    std::cout << "Finished - remove_outliers() with " <<
        "StatisticalOutlierRemoval\n" << std::endl;

}

/* Construct mesh of triangles from input pointcloud using Poisson's
 * surface reconstruction algorithm
 */

void reconstruct_mesh (int argc, char* argv[], pcl::PolygonMesh& triangles)
{
    std::cout << "Started - reconstruct_mesh() with " <<
        "Poisson" << std::endl;

    pcl::PCLPointCloud2::Ptr cloud_blob (new pcl::PCLPointCloud2());
    pcl::PointCloud<PointType>::Ptr cloud (new
            pcl::PointCloud<PointType>);

    pcl::PCDReader reader;
    if (argc < 2) {
        reader.read ("pointcloud-downsampled-inliers.pcd", *cloud_blob);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd-inliers.pcd");
        reader.read (str, *cloud_blob);
    }

    pcl::fromPCLPointCloud2 (*cloud_blob, *cloud);
    // the data should be available in cloud
    std::cout << "PointCloud loaded: " << cloud->size() << " data points\n";

    // Normal estimation
    pcl::NormalEstimation<PointType, Normal> normEst;
    pcl::PointCloud<Normal>::Ptr normals (new pcl::PointCloud<Normal>);

    // Create kdtree representation of cloud,
    // and pass it to the normal estimation object.
    pcl::search::KdTree<PointType>::Ptr tree (new
            pcl::search::KdTree<PointType>);
    tree->setInputCloud (cloud);
    normEst.setInputCloud (cloud);
    normEst.setSearchMethod (tree);
    // Use 20 neighbor points for estimating normal
    normEst.setKSearch (20);
    normEst.compute (*normals);
    // normals should not contain the point normals + surface
    // curvatures

    // Concatenate the XYZ and normal fields
    pcl::PointCloud<PointTypeN>::Ptr cloud_with_normals (new
            pcl::PointCloud<PointTypeN>);
    pcl::concatenateFields (*cloud, *normals, *cloud_with_normals);
    // cloud_with_normals = cloud + normals

    // Create search tree
    pcl::search::KdTree<PointTypeN>::Ptr tree2 (new
            pcl::search::KdTree<PointTypeN>);
    tree2->setInputCloud (cloud_with_normals);

    // Initialize objects
    // psn - for surface reconstruction algorithm
    // triangles - for storage of reconstructed triangles
    pcl::Poisson<PointTypeN> psn;
    //pcl::PolygonMesh triangles;

    psn.setInputCloud(cloud_with_normals);
    psn.setSearchMethod(tree2);
    psn.reconstruct (triangles);
    psn.setOutputPolygons(false);

    // Write reconstructed mesh
    if (argc < 2){
        pcl::io::saveVTKFile
            ("pointcloud-downsampled-outliers-mesh.vtk",
             triangles);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-mesh.vtk");
        pcl::io::saveVTKFile (str, triangles);
    }
    std::cout << "Finshed - reconstruct_mesh() with " <<
        "Poisson" << std::endl;

}

/* Simple use of PCLVisualizer class (same class is used for pcl_viewer)
 * for displaying constructed mesh
 */

void show_mesh (const pcl::PolygonMesh& mesh_of_triangles)
{
    // Create viewer object and show mesh
    boost::shared_ptr<pcl::visualization::PCLVisualizer> viewer (new
          pcl::visualization::PCLVisualizer ("3D Viewer"));
    viewer->setBackgroundColor (0, 0, 0);
    viewer->addPolygonMesh (mesh_of_triangles, "sample mesh");
    viewer->initCameraParameters ();
    while (!viewer->wasStopped ())
    {
        viewer->spinOnce (100); boost::this_thread::sleep
            (boost::posix_time::microseconds (100000));
    }

}
<commit_msg>added more info on stdout from show_mesh()<commit_after>#include <pcl/point_types.h>
#include <pcl/io/pcd_io.h>
#include <pcl/filters/voxel_grid.h>
#include <pcl/filters/statistical_outlier_removal.h>
#include <pcl/kdtree/kdtree_flann.h>
#include <pcl/features/normal_3d.h>
#include <pcl/surface/poisson.h>
#include <pcl/io/vtk_io.h>
#include <boost/thread/thread.hpp>
#include <pcl/common/common_headers.h>
#include <pcl/visualization/pcl_visualizer.h>

typedef pcl::Normal Normal;
//typedef pcl::PointXYZ PointType;
//typedef pcl::PointNormal PointTypeN;
typedef pcl::PointXYZRGB PointType;
typedef pcl::PointXYZRGBNormal PointTypeN;

void downsample (int, char*[]);
void remove_outliers (int, char* []);
void reconstruct_mesh (int, char* [], pcl::PolygonMesh&);
void show_mesh (const pcl::PolygonMesh&);

int main (int argc, char *argv[])
{
    downsample (argc, argv);
    remove_outliers (argc, argv);
    pcl::PolygonMesh mesh_of_triangles;
    reconstruct_mesh (argc, argv, mesh_of_triangles);
    show_mesh (mesh_of_triangles);

    return 0;
}

/* Downsampling with VoxelGrid class.
 * Voxel grid is a set of tiny 3D boxes in space.
 * In each voxel (i.e. 3D box) all points present will be approximated
 * (i.e. downsampled) with their centeroid.
 */

void downsample (int argc, char* argv[])
{
    std::cout << "Started - downsample() with VoxelGrid" << std::endl;

    pcl::PCLPointCloud2::Ptr cloud (new pcl::PCLPointCloud2());
    pcl::PCLPointCloud2::Ptr cloud_filtered (new pcl::PCLPointCloud2());

    // Fill in the cloud data
    pcl::PCDReader reader;
    if (argc < 2){
        reader.read ("pointcloud.pcd", *cloud);
    }
    else {
        reader.read (argv[1], *cloud);
    }

    std::cout << "PointCloud before filtering: ";
    std::cout << cloud->width * cloud->height << " data points (" ;
    std::cout << pcl::getFieldsList (*cloud) << ")." << std::endl;

    // Create the filtering object
    pcl::VoxelGrid<pcl::PCLPointCloud2> sor;
    sor.setInputCloud (cloud);
    // voxel size to be 1cm^3
    sor.setLeafSize (0.01f, 0.01f, 0.01f);
    sor.filter (*cloud_filtered);

    std::cout << "PointCloud after filtering: ";
    std::cout << cloud_filtered->width * cloud_filtered->height;
    std::cout << " data points ("  << pcl::getFieldsList (*cloud) << ").\n";

    pcl::PCDWriter writer;
    if (argc < 2){
        writer.write ("pointcloud-downsampled.pcd",
                *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd");
        writer.write (str, *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }

    std::cout << "Finished - downsample() with VoxelGrid\n" << std::endl;

}

/* Removing noisy data (i.e. outliers) from measurements using statistical
 * analysis technique
 */

void remove_outliers (int argc, char* argv[])
{
    std::cout << "Started - remove_outliers() with " <<
        "StatisticalOutlierRemoval" << std::endl;

    pcl::PCLPointCloud2::Ptr cloud (new pcl::PCLPointCloud2());
    pcl::PCLPointCloud2::Ptr cloud_filtered (new pcl::PCLPointCloud2());

    // Fill in the cloud data
    pcl::PCDReader reader;
    if (argc < 2){
        reader.read ("pointcloud-downsampled.pcd", *cloud);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd");
        reader.read (str, *cloud);
    }

    std::cout << "PointCloud before filtering: ";
    std::cout << cloud->width * cloud->height << " data points (" ;
    std::cout << pcl::getFieldsList (*cloud) << ")." << std::endl;

    // Create the filtering object
    pcl::StatisticalOutlierRemoval<pcl::PCLPointCloud2> sor2;
    sor2.setInputCloud (cloud);
    // Set number of neighbors to analyze
    sor2.setMeanK (50);
    sor2.setStddevMulThresh (1.0);
    sor2.filter (*cloud_filtered);

    std::cout << "PointCloud after filtering: ";
    std::cout << cloud_filtered->width * cloud_filtered->height;
    std::cout << " data points ("  << pcl::getFieldsList (*cloud) << ").\n";

    pcl::PCDWriter writer;

    // Write both inliers and outliers
    if (argc < 2){
        writer.write ("pointcloud-downsampled-inliers.pcd",
                *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);

        sor2.setNegative (true);
        sor2.filter (*cloud_filtered);
        writer.write ("pointcloud-downsampled-outliers.pcd",
                *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }
    else {
        std::string str1;
        str1.append(argv[1]).append("-downsampled.pcd-inliers.pcd");
        std::string str2;
        str2.append(argv[1]).append("-downsampled.pcd-outliers.pcd");
        writer.write (str1, *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);

        sor2.setNegative (true);
        sor2.filter (*cloud_filtered);
        writer.write (str2, *cloud_filtered, Eigen::Vector4f::Zero(),
                Eigen::Quaternionf::Identity(), false);
    }

    std::cout << "Finished - remove_outliers() with " <<
        "StatisticalOutlierRemoval\n" << std::endl;

}

/* Construct mesh of triangles from input pointcloud using Poisson's
 * surface reconstruction algorithm
 */

void reconstruct_mesh (int argc, char* argv[], pcl::PolygonMesh& triangles)
{
    std::cout << "Started - reconstruct_mesh() with " <<
        "Poisson" << std::endl;

    pcl::PCLPointCloud2::Ptr cloud_blob (new pcl::PCLPointCloud2());
    pcl::PointCloud<PointType>::Ptr cloud (new
            pcl::PointCloud<PointType>);

    pcl::PCDReader reader;
    if (argc < 2) {
        reader.read ("pointcloud-downsampled-inliers.pcd", *cloud_blob);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-downsampled.pcd-inliers.pcd");
        reader.read (str, *cloud_blob);
    }

    pcl::fromPCLPointCloud2 (*cloud_blob, *cloud);
    // the data should be available in cloud
    std::cout << "PointCloud loaded: " << cloud->size() << " data points\n";

    // Normal estimation
    pcl::NormalEstimation<PointType, Normal> normEst;
    pcl::PointCloud<Normal>::Ptr normals (new pcl::PointCloud<Normal>);

    // Create kdtree representation of cloud,
    // and pass it to the normal estimation object.
    pcl::search::KdTree<PointType>::Ptr tree (new
            pcl::search::KdTree<PointType>);
    tree->setInputCloud (cloud);
    normEst.setInputCloud (cloud);
    normEst.setSearchMethod (tree);
    // Use 20 neighbor points for estimating normal
    normEst.setKSearch (20);
    normEst.compute (*normals);
    // normals should not contain the point normals + surface
    // curvatures

    // Concatenate the XYZ and normal fields
    pcl::PointCloud<PointTypeN>::Ptr cloud_with_normals (new
            pcl::PointCloud<PointTypeN>);
    pcl::concatenateFields (*cloud, *normals, *cloud_with_normals);
    // cloud_with_normals = cloud + normals

    // Create search tree
    pcl::search::KdTree<PointTypeN>::Ptr tree2 (new
            pcl::search::KdTree<PointTypeN>);
    tree2->setInputCloud (cloud_with_normals);

    // Initialize objects
    // psn - for surface reconstruction algorithm
    // triangles - for storage of reconstructed triangles
    pcl::Poisson<PointTypeN> psn;
    //pcl::PolygonMesh triangles;

    psn.setInputCloud(cloud_with_normals);
    psn.setSearchMethod(tree2);
    psn.reconstruct (triangles);
    psn.setOutputPolygons(false);

    // Write reconstructed mesh
    if (argc < 2){
        pcl::io::saveVTKFile
            ("pointcloud-downsampled-outliers-mesh.vtk",
             triangles);
    }
    else {
        std::string str;
        str.append(argv[1]).append("-mesh.vtk");
        pcl::io::saveVTKFile (str, triangles);
    }
    std::cout << "Finshed - reconstruct_mesh() with " <<
        "Poisson\n" << std::endl;

}

/* Simple use of PCLVisualizer class (same class is used for pcl_viewer)
 * for displaying constructed mesh
 */

void show_mesh (const pcl::PolygonMesh& mesh_of_triangles)
{
    std::cout << "Started - show_mesh() with PCLVisualizer\n";
    // Create viewer object and show mesh
    boost::shared_ptr<pcl::visualization::PCLVisualizer> viewer (new
          pcl::visualization::PCLVisualizer ("3D Viewer"));
    viewer->setBackgroundColor (0, 0, 0);
    viewer->addPolygonMesh (mesh_of_triangles, "sample mesh");
    viewer->initCameraParameters ();
    while (!viewer->wasStopped ())
    {
        viewer->spinOnce (100); boost::this_thread::sleep
            (boost::posix_time::microseconds (100000));
    }
    std::cout << "Finshed - show_mesh() with PCLVisualizer\n";

}
<|endoftext|>


<commit_before>

#include "backendlayerhelper.hxx"

#ifndef _COM_SUN_STAR_CONFIGURATION_BACKEND_PROPERTYINFO_HPP_
#include <com/sun/star/configuration/backend/PropertyInfo.hpp>
#endif

namespace configmgr { namespace backendhelper {

//==============================================================================

//------------------------------------------------------------------------------
uno::Type toType(const ::rtl::OUString& _rType)
    {
        uno::Type aRet;

        if  (_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("boolean")))
            aRet = ::getBooleanCppuType();

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("short")))
            aRet = ::getCppuType(static_cast<sal_Int16 const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("int")))
            aRet = ::getCppuType(static_cast<sal_Int32 const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("integer")))
            aRet = ::getCppuType(static_cast<sal_Int32 const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("long")))
            aRet = ::getCppuType(static_cast<sal_Int64 const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("double")))
            aRet = ::getCppuType(static_cast<double const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("string")))
            aRet = ::getCppuType(static_cast<rtl::OUString const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("binary")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int8> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("any")))
            aRet = ::getCppuType(static_cast<uno::Any const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("boolean-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Bool> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("short-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int16> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("int-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int32> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("integer-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int32> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("long-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int64> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("double-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<double> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("string-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<rtl::OUString> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("binary-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<uno::Sequence<sal_Int8> > const*>(0));

        else
        {
            ::rtl::OString aStr("Unknown type! ");
            aStr += rtl::OUStringToOString(_rType,RTL_TEXTENCODING_ASCII_US);
            OSL_ENSURE(0,aStr.getStr());
        }

        return aRet;
    }

//------------------------------------------------------------------------------

IOONode::IOONode(const rtl::OUString& sName):
    mName(sName)
{
}

//------------------------------------------------------------------------------
OOProperty::OOProperty(
    const rtl::OUString& sName,const rtl::OUString& sPropType,
    const uno::Any& aPropValue,sal_Bool bProtected)
    :IOONode(sName), mPropType(sPropType), mPropValue(aPropValue),
    mbProtected(bProtected)
{
}
//------------------------------------------------------------------------------
OONode::OONode(const rtl::OUString& sName)
    :IOONode(sName)
{
}

OONode::OONode()
    :IOONode(rtl::OUString())
{
}
IOONode* OONode::addChild(IOONode* aChild)
{
    mChildList.push_back(aChild);
    return aChild;
}
const std::vector<IOONode*>& OONode::getChildren()
{
    return mChildList;
}

IOONode* OONode::getChild(const rtl::OUString& aChildName)
{
    for (sal_uInt32 i=0; i< mChildList.size();++i)
    {
        if (mChildList[i]->getName() == aChildName)
            return mChildList[i];
    }
    return NULL;
}
OONode::~OONode()
{
    for (sal_uInt32 i=0; i< mChildList.size();++i)
    {
        delete mChildList[i];
    }
    mChildList.clear();
}
//------------------------------------------------------------------------------
 sal_Bool addChildrenToNodeTree(
     OONode * aNode,
     sal_Int32 nNextToken,
     const backend::PropertyInfo& aPropInfo,
     const uno::Reference<uno::XInterface>& xContext)
{
    do
    {
        rtl::OUString aName = aPropInfo.Name.getToken(0, '/',nNextToken);
        if (aName.getLength() == 0)
        {
            throw backend::MalformedDataException(
                rtl::OUString(RTL_CONSTASCII_USTRINGPARAM(
                        "Malformed OpenOffice Key specified")),
                        xContext, uno::Any()) ;
        }
        //Check if Property -> nNextToken == -1
        if(nNextToken != -1)
        {
            //check if child already exists
            IOONode* aChildNode= aNode->getChild(aName);
            if (aChildNode == NULL)
            {
                aChildNode = new OONode(aName);
                if (aChildNode != 0)
                {
                    aNode->addChild( aChildNode);
                }
            }

            sal_Bool bFinished =addChildrenToNodeTree(
                aChildNode->getComposite(),
                nNextToken,
                aPropInfo,
                xContext);
            //Check that if you have finished parsing string therefore no
            //more children
            if (bFinished)
                break;
        }
        else
        {
            //Add Property
            IOONode* aProperty = new OOProperty(aName,
                                                aPropInfo.Type,
                                                aPropInfo.Value,
                                                aPropInfo.Protected);
            if (aProperty != 0)
            {
                aNode->addChild( aProperty);
            }
            //Return finished is true when you are finished parsing the string
            if( nNextToken == -1)
            {
                return sal_True;
            }
        }
    }
    while (nNextToken >= 0 ) ;
    return sal_True;
}
//------------------------------------------------------------------------------
void processChildren(
     std::vector<IOONode*> aChildList,
    const uno::Reference<backend::XLayerHandler>& xHandler)
{
    for(sal_uInt32 i=0; i <aChildList.size(); ++i)
    {
        OONode * aTestOONode = aChildList[i]->getComposite();
        if (aTestOONode)
        {
            xHandler->overrideNode(aTestOONode->getName(),0,false);
            processChildren(aTestOONode->getChildren(),xHandler);
            xHandler->endNode();
        }
        else
        {
            OOProperty* aProperty = aChildList[i]->asOOProperty();
            sal_Int16 aAttributes = aProperty->isProtected() ? 256:0;
            //Convert Type either simple or list
            uno::Type aType = toType( aProperty->getType());

            xHandler->overrideProperty(aProperty->getName(),
                                       aAttributes,
                                       aType,
                                       false);

            xHandler->setPropertyValue(aProperty->getValue());
            xHandler->endProperty();
        }

    }
}
//------------------------------------------------------------------------------
void buildNodeTree(
    const uno::Sequence< backend::PropertyInfo >& aPropertyInfos,
    const uno::Reference<uno::XInterface>& xContext,
    OONode& aNodeTree)
{
    sal_Int32 nNextToken =0;
    rtl::OUString aName = aPropertyInfos[0].Name.getToken(0, '/',nNextToken);
    if((nNextToken ==-1)||(aName.getLength()==0))
    {
         throw backend::MalformedDataException(
               rtl::OUString(RTL_CONSTASCII_USTRINGPARAM(
                       "Malformed OpenOffice Key specified")),
                    xContext, uno::Any()) ;

    }
    aNodeTree.setName(aName);
    sal_Int32 size = aPropertyInfos.getLength();
    for (sal_Int32 i =0; i < size; ++i)
    {
        addChildrenToNodeTree(&aNodeTree, nNextToken,aPropertyInfos[i],xContext);
    }

}
//------------------------------------------------------------------------------
BackendLayerHelper::BackendLayerHelper(
    const uno::Reference<uno::XComponentContext>& /*xContext*/)
    :BackendBase(mMutex)
{
}
//------------------------------------------------------------------------------

BackendLayerHelper::~BackendLayerHelper(void) {}

//------------------------------------------------------------------------------

rtl::OUString SAL_CALL BackendLayerHelper::
    getBackendLayerHelperName(void)
{
    static const rtl::OUString kImplementationName(
            RTL_CONSTASCII_USTRINGPARAM(
                "com.sun.star.comp.configuration.backend.LayerDescriber")) ;

    return kImplementationName ;
}
//------------------------------------------------------------------------------
void SAL_CALL BackendLayerHelper::describeLayer(
    const uno::Reference< backend::XLayerHandler >& xHandler,
    const uno::Sequence< backend::PropertyInfo >& aPropertyInfos )
        throw (lang::NullPointerException,
               backend::MalformedDataException,
               uno::RuntimeException)

{
    OONode aNodeTree;
    buildNodeTree(aPropertyInfos, *this, aNodeTree);

    //Descirbe the Layer to the XHandler Object
    xHandler->startLayer();
    xHandler->overrideNode(aNodeTree.getName(),0,false);
    std::vector<IOONode*> aChildList = aNodeTree.getChildren();
    processChildren(aChildList,xHandler);
    xHandler->endNode();
    xHandler->endLayer();

}
//------------------------------------------------------------------------------

rtl::OUString SAL_CALL BackendLayerHelper::getImplementationName(void)
    throw (uno::RuntimeException)
{
    return getBackendLayerHelperName() ;
}
//------------------------------------------------------------------------------

uno::Sequence<rtl::OUString> SAL_CALL BackendLayerHelper::
    getBackendLayerHelperServiceNames(void)
{
    uno::Sequence<rtl::OUString> aServices(1) ;
    aServices[0] = rtl::OUString( RTL_CONSTASCII_USTRINGPARAM("com.sun.star.configuration.backend.LayerDescriber")) ;
    return aServices ;
}
//------------------------------------------------------------------------------

sal_Bool SAL_CALL BackendLayerHelper::supportsService(
    const rtl::OUString& aServiceName)
    throw (uno::RuntimeException)
{
    uno::Sequence< rtl::OUString > const svc = getBackendLayerHelperServiceNames();

    for(sal_Int32 i = 0; i < svc.getLength(); ++i )
        if(svc[i] == aServiceName)
            return true;
    return false;
}
//------------------------------------------------------------------------------

uno::Sequence<rtl::OUString>
SAL_CALL BackendLayerHelper::getSupportedServiceNames(void)
    throw (uno::RuntimeException)
{
    return getBackendLayerHelperServiceNames() ;
}
//------------------------------------------------------------------------------

} // backendhelper
} // configmgr
<commit_msg>INTEGRATION: CWS pchfix02 (1.3.18); FILE MERGED 2006/09/01 17:20:32 kaib 1.3.18.1: #i68856# Added header markers and pch files<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: backendlayerhelper.cxx,v $
 *
 *  $Revision: 1.4 $
 *
 *  last change: $Author: obo $ $Date: 2006-09-16 15:08:08 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_configmgr.hxx"

#include "backendlayerhelper.hxx"

#ifndef _COM_SUN_STAR_CONFIGURATION_BACKEND_PROPERTYINFO_HPP_
#include <com/sun/star/configuration/backend/PropertyInfo.hpp>
#endif

namespace configmgr { namespace backendhelper {

//==============================================================================

//------------------------------------------------------------------------------
uno::Type toType(const ::rtl::OUString& _rType)
    {
        uno::Type aRet;

        if  (_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("boolean")))
            aRet = ::getBooleanCppuType();

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("short")))
            aRet = ::getCppuType(static_cast<sal_Int16 const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("int")))
            aRet = ::getCppuType(static_cast<sal_Int32 const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("integer")))
            aRet = ::getCppuType(static_cast<sal_Int32 const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("long")))
            aRet = ::getCppuType(static_cast<sal_Int64 const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("double")))
            aRet = ::getCppuType(static_cast<double const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("string")))
            aRet = ::getCppuType(static_cast<rtl::OUString const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("binary")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int8> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("any")))
            aRet = ::getCppuType(static_cast<uno::Any const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("boolean-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Bool> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("short-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int16> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("int-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int32> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("integer-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int32> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("long-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<sal_Int64> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("double-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<double> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("string-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<rtl::OUString> const*>(0));

        else if(_rType.equalsIgnoreAsciiCase(::rtl::OUString::createFromAscii("binary-list")))
            aRet = ::getCppuType(static_cast<uno::Sequence<uno::Sequence<sal_Int8> > const*>(0));

        else
        {
            ::rtl::OString aStr("Unknown type! ");
            aStr += rtl::OUStringToOString(_rType,RTL_TEXTENCODING_ASCII_US);
            OSL_ENSURE(0,aStr.getStr());
        }

        return aRet;
    }

//------------------------------------------------------------------------------

IOONode::IOONode(const rtl::OUString& sName):
    mName(sName)
{
}

//------------------------------------------------------------------------------
OOProperty::OOProperty(
    const rtl::OUString& sName,const rtl::OUString& sPropType,
    const uno::Any& aPropValue,sal_Bool bProtected)
    :IOONode(sName), mPropType(sPropType), mPropValue(aPropValue),
    mbProtected(bProtected)
{
}
//------------------------------------------------------------------------------
OONode::OONode(const rtl::OUString& sName)
    :IOONode(sName)
{
}

OONode::OONode()
    :IOONode(rtl::OUString())
{
}
IOONode* OONode::addChild(IOONode* aChild)
{
    mChildList.push_back(aChild);
    return aChild;
}
const std::vector<IOONode*>& OONode::getChildren()
{
    return mChildList;
}

IOONode* OONode::getChild(const rtl::OUString& aChildName)
{
    for (sal_uInt32 i=0; i< mChildList.size();++i)
    {
        if (mChildList[i]->getName() == aChildName)
            return mChildList[i];
    }
    return NULL;
}
OONode::~OONode()
{
    for (sal_uInt32 i=0; i< mChildList.size();++i)
    {
        delete mChildList[i];
    }
    mChildList.clear();
}
//------------------------------------------------------------------------------
 sal_Bool addChildrenToNodeTree(
     OONode * aNode,
     sal_Int32 nNextToken,
     const backend::PropertyInfo& aPropInfo,
     const uno::Reference<uno::XInterface>& xContext)
{
    do
    {
        rtl::OUString aName = aPropInfo.Name.getToken(0, '/',nNextToken);
        if (aName.getLength() == 0)
        {
            throw backend::MalformedDataException(
                rtl::OUString(RTL_CONSTASCII_USTRINGPARAM(
                        "Malformed OpenOffice Key specified")),
                        xContext, uno::Any()) ;
        }
        //Check if Property -> nNextToken == -1
        if(nNextToken != -1)
        {
            //check if child already exists
            IOONode* aChildNode= aNode->getChild(aName);
            if (aChildNode == NULL)
            {
                aChildNode = new OONode(aName);
                if (aChildNode != 0)
                {
                    aNode->addChild( aChildNode);
                }
            }

            sal_Bool bFinished =addChildrenToNodeTree(
                aChildNode->getComposite(),
                nNextToken,
                aPropInfo,
                xContext);
            //Check that if you have finished parsing string therefore no
            //more children
            if (bFinished)
                break;
        }
        else
        {
            //Add Property
            IOONode* aProperty = new OOProperty(aName,
                                                aPropInfo.Type,
                                                aPropInfo.Value,
                                                aPropInfo.Protected);
            if (aProperty != 0)
            {
                aNode->addChild( aProperty);
            }
            //Return finished is true when you are finished parsing the string
            if( nNextToken == -1)
            {
                return sal_True;
            }
        }
    }
    while (nNextToken >= 0 ) ;
    return sal_True;
}
//------------------------------------------------------------------------------
void processChildren(
     std::vector<IOONode*> aChildList,
    const uno::Reference<backend::XLayerHandler>& xHandler)
{
    for(sal_uInt32 i=0; i <aChildList.size(); ++i)
    {
        OONode * aTestOONode = aChildList[i]->getComposite();
        if (aTestOONode)
        {
            xHandler->overrideNode(aTestOONode->getName(),0,false);
            processChildren(aTestOONode->getChildren(),xHandler);
            xHandler->endNode();
        }
        else
        {
            OOProperty* aProperty = aChildList[i]->asOOProperty();
            sal_Int16 aAttributes = aProperty->isProtected() ? 256:0;
            //Convert Type either simple or list
            uno::Type aType = toType( aProperty->getType());

            xHandler->overrideProperty(aProperty->getName(),
                                       aAttributes,
                                       aType,
                                       false);

            xHandler->setPropertyValue(aProperty->getValue());
            xHandler->endProperty();
        }

    }
}
//------------------------------------------------------------------------------
void buildNodeTree(
    const uno::Sequence< backend::PropertyInfo >& aPropertyInfos,
    const uno::Reference<uno::XInterface>& xContext,
    OONode& aNodeTree)
{
    sal_Int32 nNextToken =0;
    rtl::OUString aName = aPropertyInfos[0].Name.getToken(0, '/',nNextToken);
    if((nNextToken ==-1)||(aName.getLength()==0))
    {
         throw backend::MalformedDataException(
               rtl::OUString(RTL_CONSTASCII_USTRINGPARAM(
                       "Malformed OpenOffice Key specified")),
                    xContext, uno::Any()) ;

    }
    aNodeTree.setName(aName);
    sal_Int32 size = aPropertyInfos.getLength();
    for (sal_Int32 i =0; i < size; ++i)
    {
        addChildrenToNodeTree(&aNodeTree, nNextToken,aPropertyInfos[i],xContext);
    }

}
//------------------------------------------------------------------------------
BackendLayerHelper::BackendLayerHelper(
    const uno::Reference<uno::XComponentContext>& /*xContext*/)
    :BackendBase(mMutex)
{
}
//------------------------------------------------------------------------------

BackendLayerHelper::~BackendLayerHelper(void) {}

//------------------------------------------------------------------------------

rtl::OUString SAL_CALL BackendLayerHelper::
    getBackendLayerHelperName(void)
{
    static const rtl::OUString kImplementationName(
            RTL_CONSTASCII_USTRINGPARAM(
                "com.sun.star.comp.configuration.backend.LayerDescriber")) ;

    return kImplementationName ;
}
//------------------------------------------------------------------------------
void SAL_CALL BackendLayerHelper::describeLayer(
    const uno::Reference< backend::XLayerHandler >& xHandler,
    const uno::Sequence< backend::PropertyInfo >& aPropertyInfos )
        throw (lang::NullPointerException,
               backend::MalformedDataException,
               uno::RuntimeException)

{
    OONode aNodeTree;
    buildNodeTree(aPropertyInfos, *this, aNodeTree);

    //Descirbe the Layer to the XHandler Object
    xHandler->startLayer();
    xHandler->overrideNode(aNodeTree.getName(),0,false);
    std::vector<IOONode*> aChildList = aNodeTree.getChildren();
    processChildren(aChildList,xHandler);
    xHandler->endNode();
    xHandler->endLayer();

}
//------------------------------------------------------------------------------

rtl::OUString SAL_CALL BackendLayerHelper::getImplementationName(void)
    throw (uno::RuntimeException)
{
    return getBackendLayerHelperName() ;
}
//------------------------------------------------------------------------------

uno::Sequence<rtl::OUString> SAL_CALL BackendLayerHelper::
    getBackendLayerHelperServiceNames(void)
{
    uno::Sequence<rtl::OUString> aServices(1) ;
    aServices[0] = rtl::OUString( RTL_CONSTASCII_USTRINGPARAM("com.sun.star.configuration.backend.LayerDescriber")) ;
    return aServices ;
}
//------------------------------------------------------------------------------

sal_Bool SAL_CALL BackendLayerHelper::supportsService(
    const rtl::OUString& aServiceName)
    throw (uno::RuntimeException)
{
    uno::Sequence< rtl::OUString > const svc = getBackendLayerHelperServiceNames();

    for(sal_Int32 i = 0; i < svc.getLength(); ++i )
        if(svc[i] == aServiceName)
            return true;
    return false;
}
//------------------------------------------------------------------------------

uno::Sequence<rtl::OUString>
SAL_CALL BackendLayerHelper::getSupportedServiceNames(void)
    throw (uno::RuntimeException)
{
    return getBackendLayerHelperServiceNames() ;
}
//------------------------------------------------------------------------------

} // backendhelper
} // configmgr
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-http/httpconnectionpool.h"
#include "fnord-mdb/MDB.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"
#include "logjoin/LogJoinTarget.h"
#include "logjoin/LogJoinUpload.h"
#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_logjoin_shutdown;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  /* shutdown hook */
  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "cmdata",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "clickmatcher app data dir",
      "<path>");

  flags.defineFlag(
      "feedserver_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "http://localhost:8000",
      "feedserver addr",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "commit_size",
      "<num>");

  flags.defineFlag(
      "commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "commit_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.defineFlag(
      "shard",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* schema... */
  auto joined_sessions_schema = joinedSessionsSchema();

  /* set up cmdata */
  auto cmdata_path = flags.getString("cmdata");
  if (!FileUtil::isDirectory(cmdata_path)) {
    RAISEF(kIOError, "no such directory: $0", cmdata_path);
  }

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);
  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* set up logjoin */
  auto dry_run = !flags.isSet("no_dryrun");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t commit_size = flags.getInt("commit_size");
  size_t commit_interval = flags.getInt("commit_interval");

  fnord::logInfo(
      "cm.logjoin",
      "Starting cm-logjoin with:\n    dry_run=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    commit_size=$3\n    commit_interval=$9\n"
      "    max_dbsize=$4MB\n" \
      "    shard=$5\n    shard_range=[$6, $7)\n    shard_modulo=$8",
      dry_run,
      batch_size,
      buffer_size,
      commit_size,
      flags.getInt("db_size"),
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo,
      commit_interval);

  /* open session db */
  auto sessdb_path = FileUtil::joinPaths(
      cmdata_path,
      StringUtil::format("logjoin/$0/sessions_db", shard.shard_name));

  FileUtil::mkdir_p(sessdb_path);
  auto sessdb = mdb::MDB::open(sessdb_path);
  sessdb->setMaxSize(1000000 * flags.getInt("db_size"));

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(10 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  input_feeds.emplace(
      "tracker_log.feedserver01.nue01.production.fnrd.net",
      URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* setup time backfill */
  feed_reader.setTimeBackfill([] (const feeds::FeedEntry& entry) -> DateTime {
    const auto& log_line = entry.data;

    auto c_end = log_line.find("|");
    if (c_end == std::string::npos) return 0;
    auto t_end = log_line.find("|", c_end + 1);
    if (t_end == std::string::npos) return 0;

    auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
    return std::stoul(timestr) * fnord::kMicrosPerSecond;
  });

  /* set up logjoin target */
  fnord::fts::Analyzer analyzer(flags.getString("conf"));
  cm::LogJoinTarget logjoin_target(joined_sessions_schema, &analyzer);

  /* set up logjoin upload */
  cm::LogJoinUpload logjoin_upload(
      sessdb,
      flags.getString("feedserver_addr"),
      &http);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, &logjoin_target);
  logjoin.exportStats("/cm-logjoin/global");
  logjoin.exportStats(StringUtil::format("/cm-logjoin/$0", shard.shard_name));

  fnord::stats::Counter<uint64_t> stat_stream_time_low;
  fnord::stats::Counter<uint64_t> stat_stream_time_high;
  fnord::stats::Counter<uint64_t> stat_active_sessions;
  fnord::stats::Counter<uint64_t> stat_dbsize;

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_low", shard.shard_name),
      &stat_stream_time_low,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_high", shard.shard_name),
      &stat_stream_time_high,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/active_sessions", shard.shard_name),
      &stat_active_sessions,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/dbsize", shard.shard_name),
      &stat_dbsize,
      fnord::stats::ExportMode::EXPORT_VALUE);

  /* upload pending q */
  logjoin_upload.upload();

  /* resume from last offset */
  auto txn = sessdb->startTransaction(true);
  try {
    logjoin.importTimeoutList(txn.get());

    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__logjoin_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.logjoin",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.logjoin", "Resuming logjoin...");

  DateTime last_flush;
  uint64_t rate_limit_micros = commit_interval * kMicrosPerSecond;

  for (;;) {
    auto begin = WallClock::unixMicros();
    auto turbo = FileUtil::exists("/tmp/logjoin_turbo.enabled");
    logjoin.setTurbo(turbo);

    feed_reader.fillBuffers();
    auto txn = sessdb->startTransaction();

    int i = 0;
    for (; i < commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        logjoin.insertLogline(entry.get().data, txn.get());
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid log line");
      }
    }

    auto watermarks = feed_reader.watermarks();

    auto etime = WallClock::now().unixMicros() - last_flush.unixMicros();
    if (etime > rate_limit_micros) {
      last_flush = WallClock::now();
      logjoin.flush(txn.get(), watermarks.first);
    }

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__logjoin_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.logjoin",
        "LogJoin comitting...\n    stream_time=<$0 ... $1>\n" \
        "    active_sessions=$2\n    flushed_sessions=$3\n    " \
        "flushed_queries=$4\n    flushed_item_visits=$5\n    turbo=$6\n    " \
        "cach_size=$7$8",
        watermarks.first,
        watermarks.second,
        logjoin.numSessions(),
        logjoin_target.num_sessions,
        logjoin_target.num_queries,
        logjoin_target.num_item_visits,
        turbo,
        logjoin.cacheSize(),
        stream_offsets_str);

    txn->commit();

    try {
      logjoin_upload.upload();
    } catch (const std::exception& e) {
      fnord::logError("cm.logjoin", e, "upload failed");
    }

    if (cm_logjoin_shutdown.load()) {
      break;
    }

    stat_stream_time_low.set(watermarks.first.unixMicros());
    stat_stream_time_high.set(watermarks.second.unixMicros());
    stat_active_sessions.set(logjoin.numSessions());
    stat_dbsize.set(FileUtil::du_c(sessdb_path));

    auto rtime = WallClock::unixMicros() - begin;
    if (rtime < kMicrosPerSecond) {
      begin = WallClock::unixMicros();
      usleep(kMicrosPerSecond - rtime);
    }
  }

  ev.shutdown();
  //evloop_thread.join();

  fnord::logInfo("cm.logjoin", "LogJoin exiting...");
  exit(0); // FIXPAUL

  return 0;
}

<commit_msg>typo fix<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-http/httpconnectionpool.h"
#include "fnord-mdb/MDB.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"
#include "logjoin/LogJoinTarget.h"
#include "logjoin/LogJoinUpload.h"
#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_logjoin_shutdown;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  /* shutdown hook */
  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "cmdata",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "clickmatcher app data dir",
      "<path>");

  flags.defineFlag(
      "feedserver_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "http://localhost:8000",
      "feedserver addr",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "commit_size",
      "<num>");

  flags.defineFlag(
      "commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "commit_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.defineFlag(
      "shard",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* schema... */
  auto joined_sessions_schema = joinedSessionsSchema();

  /* set up cmdata */
  auto cmdata_path = flags.getString("cmdata");
  if (!FileUtil::isDirectory(cmdata_path)) {
    RAISEF(kIOError, "no such directory: $0", cmdata_path);
  }

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);
  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* set up logjoin */
  auto dry_run = !flags.isSet("no_dryrun");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t commit_size = flags.getInt("commit_size");
  size_t commit_interval = flags.getInt("commit_interval");

  fnord::logInfo(
      "cm.logjoin",
      "Starting cm-logjoin with:\n    dry_run=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    commit_size=$3\n    commit_interval=$9\n"
      "    max_dbsize=$4MB\n" \
      "    shard=$5\n    shard_range=[$6, $7)\n    shard_modulo=$8",
      dry_run,
      batch_size,
      buffer_size,
      commit_size,
      flags.getInt("db_size"),
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo,
      commit_interval);

  /* open session db */
  auto sessdb_path = FileUtil::joinPaths(
      cmdata_path,
      StringUtil::format("logjoin/$0/sessions_db", shard.shard_name));

  FileUtil::mkdir_p(sessdb_path);
  auto sessdb = mdb::MDB::open(sessdb_path);
  sessdb->setMaxSize(1000000 * flags.getInt("db_size"));

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(10 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  input_feeds.emplace(
      "tracker_log.feedserver01.nue01.production.fnrd.net",
      URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* setup time backfill */
  feed_reader.setTimeBackfill([] (const feeds::FeedEntry& entry) -> DateTime {
    const auto& log_line = entry.data;

    auto c_end = log_line.find("|");
    if (c_end == std::string::npos) return 0;
    auto t_end = log_line.find("|", c_end + 1);
    if (t_end == std::string::npos) return 0;

    auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
    return std::stoul(timestr) * fnord::kMicrosPerSecond;
  });

  /* set up logjoin target */
  fnord::fts::Analyzer analyzer(flags.getString("conf"));
  cm::LogJoinTarget logjoin_target(joined_sessions_schema, &analyzer);

  /* set up logjoin upload */
  cm::LogJoinUpload logjoin_upload(
      sessdb,
      flags.getString("feedserver_addr"),
      &http);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, &logjoin_target);
  logjoin.exportStats("/cm-logjoin/global");
  logjoin.exportStats(StringUtil::format("/cm-logjoin/$0", shard.shard_name));

  fnord::stats::Counter<uint64_t> stat_stream_time_low;
  fnord::stats::Counter<uint64_t> stat_stream_time_high;
  fnord::stats::Counter<uint64_t> stat_active_sessions;
  fnord::stats::Counter<uint64_t> stat_dbsize;

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_low", shard.shard_name),
      &stat_stream_time_low,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_high", shard.shard_name),
      &stat_stream_time_high,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/active_sessions", shard.shard_name),
      &stat_active_sessions,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/dbsize", shard.shard_name),
      &stat_dbsize,
      fnord::stats::ExportMode::EXPORT_VALUE);

  /* upload pending q */
  logjoin_upload.upload();

  /* resume from last offset */
  auto txn = sessdb->startTransaction(true);
  try {
    logjoin.importTimeoutList(txn.get());

    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__logjoin_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.logjoin",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.logjoin", "Resuming logjoin...");

  DateTime last_flush;
  uint64_t rate_limit_micros = commit_interval * kMicrosPerSecond;

  for (;;) {
    auto begin = WallClock::unixMicros();
    auto turbo = FileUtil::exists("/tmp/logjoin_turbo.enabled");
    logjoin.setTurbo(turbo);

    feed_reader.fillBuffers();
    auto txn = sessdb->startTransaction();

    int i = 0;
    for (; i < commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        logjoin.insertLogline(entry.get().data, txn.get());
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid log line");
      }
    }

    auto watermarks = feed_reader.watermarks();

    auto etime = WallClock::now().unixMicros() - last_flush.unixMicros();
    if (etime > rate_limit_micros) {
      last_flush = WallClock::now();
      logjoin.flush(txn.get(), watermarks.first);
    }

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__logjoin_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.logjoin",
        "LogJoin comitting...\n    stream_time=<$0 ... $1>\n" \
        "    active_sessions=$2\n    flushed_sessions=$3\n    " \
        "flushed_queries=$4\n    flushed_item_visits=$5\n    turbo=$6\n    " \
        "cache_size=$7$8",
        watermarks.first,
        watermarks.second,
        logjoin.numSessions(),
        logjoin_target.num_sessions,
        logjoin_target.num_queries,
        logjoin_target.num_item_visits,
        turbo,
        logjoin.cacheSize(),
        stream_offsets_str);

    txn->commit();

    try {
      logjoin_upload.upload();
    } catch (const std::exception& e) {
      fnord::logError("cm.logjoin", e, "upload failed");
    }

    if (cm_logjoin_shutdown.load()) {
      break;
    }

    stat_stream_time_low.set(watermarks.first.unixMicros());
    stat_stream_time_high.set(watermarks.second.unixMicros());
    stat_active_sessions.set(logjoin.numSessions());
    stat_dbsize.set(FileUtil::du_c(sessdb_path));

    auto rtime = WallClock::unixMicros() - begin;
    if (rtime < kMicrosPerSecond) {
      begin = WallClock::unixMicros();
      usleep(kMicrosPerSecond - rtime);
    }
  }

  ev.shutdown();
  //evloop_thread.join();

  fnord::logInfo("cm.logjoin", "LogJoin exiting...");
  exit(0); // FIXPAUL

  return 0;
}

<|endoftext|>