<commit_before>#include "route.hpp"
#include "turns_generator.hpp"

#include "indexer/mercator.hpp"

#include "platform/location.hpp"

#include "geometry/angles.hpp"
#include "geometry/point2d.hpp"
#include "geometry/simplification.hpp"

#include "base/logging.hpp"

#include "std/numeric.hpp"
#include "std/utility.hpp"
#include "std/algorithm.hpp"

namespace routing
{
namespace
{
double constexpr kLocationTimeThreshold = 60.0 * 1.0;
double constexpr kOnEndToleranceM = 10.0;

}  //  namespace

Route::Route(string const & router, vector<m2::PointD> const & points, string const & name)
  : m_router(router), m_routingSettings(GetCarRoutingSettings()),
    m_name(name), m_poly(points.begin(), points.end())
{
  Update();
}

void Route::Swap(Route & rhs)
{
  m_router.swap(rhs.m_router);
  swap(m_routingSettings, rhs.m_routingSettings);
  m_poly.Swap(rhs.m_poly);
  m_simplifiedPoly.Swap(rhs.m_simplifiedPoly);
  m_name.swap(rhs.m_name);
  swap(m_currentTime, rhs.m_currentTime);
  swap(m_turns, rhs.m_turns);
  swap(m_times, rhs.m_times);
  m_absentCountries.swap(rhs.m_absentCountries);
}

double Route::GetTotalDistanceMeters() const
{
  return m_poly.GetTotalDistanceM();
}

double Route::GetCurrentDistanceFromBeginMeters() const
{
  return m_poly.GetDistanceFromBeginM();
}

void Route::GetTurnsDistances(vector<double> & distances) const
{
  double mercatorDistance = 0;
  distances.clear();
  auto const & polyline = m_poly.GetPolyline();
  for (auto currentTurn = m_turns.begin(); currentTurn != m_turns.end(); ++currentTurn)
  {
    // Skip turns at side points of the polyline geometry. We can't display them properly.
    if (currentTurn->m_index == 0 || currentTurn->m_index == (polyline.GetSize() - 1))
      continue;

    uint32_t formerTurnIndex = 0;
    if (currentTurn != m_turns.begin())
      formerTurnIndex = (currentTurn - 1)->m_index;

    //TODO (ldragunov) Extract CalculateMercatorDistance higher to avoid including turns generator.
    double const mercatorDistanceBetweenTurns =
      turns::CalculateMercatorDistanceAlongPath(formerTurnIndex,  currentTurn->m_index, polyline.GetPoints());
    mercatorDistance += mercatorDistanceBetweenTurns;

    distances.push_back(mercatorDistance);
   }
}

double Route::GetCurrentDistanceToEndMeters() const
{
  return m_poly.GetDistanceToEndM();
}

double Route::GetMercatorDistanceFromBegin() const
{
  //TODO Maybe better to return FollowedRoute and user will call GetMercatorDistance etc. by itself
  return m_poly.GetMercatorDistanceFromBegin();
}

uint32_t Route::GetTotalTimeSec() const
{
  return m_times.empty() ? 0 : m_times.back().second;
}

uint32_t Route::GetCurrentTimeToEndSec() const
{
  auto const & poly = m_poly.GetPolyline();
  size_t const polySz = poly.GetSize();
  if (m_times.empty() || poly.GetSize() == 0)
  {
    ASSERT(!m_times.empty(), ());
    ASSERT(polySz != 0, ());
    return 0;
  }

  TTimes::const_iterator it = upper_bound(m_times.begin(), m_times.end(), m_poly.GetCurrentIter().m_ind,
                                         [](size_t v, Route::TTimeItem const & item) { return v < item.first; });

  if (it == m_times.end())
    return 0;

  size_t idx = distance(m_times.begin(), it);
  double time = (*it).second;
  if (idx > 0)
    time -= m_times[idx - 1].second;

  auto distFn = [&](size_t start, size_t end)
  {
    return m_poly.GetDistanceM(m_poly.GetIterToIndex(start), m_poly.GetIterToIndex(end));
  };

  ASSERT_LESS_OR_EQUAL(m_times[idx].first, poly.GetSize(), ());
  double const dist = distFn(idx > 0 ? m_times[idx - 1].first : 0, m_times[idx].first);

  if (!my::AlmostEqualULPs(dist, 0.))
  {
    double const distRemain = distFn(m_poly.GetCurrentIter().m_ind, m_times[idx].first) -
                        MercatorBounds::DistanceOnEarth(m_poly.GetCurrentIter().m_pt, poly.GetPoint(m_poly.GetCurrentIter().m_ind));
    return (uint32_t)((GetTotalTimeSec() - (*it).second) + (double)time * (distRemain / dist));
  }
  else
    return (uint32_t)((GetTotalTimeSec() - (*it).second));
}

void Route::GetCurrentTurn(double & distanceToTurnMeters, turns::TurnItem & turn) const
{
  if (m_turns.empty())
  {
    ASSERT(!m_turns.empty(), ());
    distanceToTurnMeters = 0;
    turn = turns::TurnItem();
    return;
  }

  turns::TurnItem t;
  t.m_index = static_cast<uint32_t>(m_poly.GetCurrentIter().m_ind);
  auto it = upper_bound(m_turns.begin(), m_turns.end(), t,
            [](turns::TurnItem const & lhs, turns::TurnItem const & rhs)
            {
              return lhs.m_index < rhs.m_index;
            });

  ASSERT_GREATER_OR_EQUAL((*it).m_index - 1, 0, ());

  size_t const segIdx = (*it).m_index - 1;
  turn = (*it);
  distanceToTurnMeters = m_poly.GetDistanceM(m_poly.GetCurrentIter(),
                                             m_poly.GetIterToIndex(segIdx));
}

void Route::GetCurrentDirectionPoint(m2::PointD & pt) const
{
  if (m_routingSettings.m_keepPedestrianInfo && m_simplifiedPoly.IsValid())
    m_simplifiedPoly.GetCurrentDirectionPoint(pt, kOnEndToleranceM);
  else
    m_poly.GetCurrentDirectionPoint(pt, kOnEndToleranceM);
}

bool Route::MoveIterator(location::GpsInfo const & info) const
{
  double predictDistance = -1.0;
  if (m_currentTime > 0.0 && info.HasSpeed())
  {
    /// @todo Need to distinguish GPS and WiFi locations.
    /// They may have different time metrics in case of incorrect system time on a device.
    double const deltaT = info.m_timestamp - m_currentTime;
    if (deltaT > 0.0 && deltaT < kLocationTimeThreshold)
      predictDistance = info.m_speed * deltaT;
  }

  m2::RectD const rect = MercatorBounds::MetresToXY(
        info.m_longitude, info.m_latitude,
        max(m_routingSettings.m_matchingThresholdM, info.m_horizontalAccuracy));
  FollowedPolyline::Iter const res = m_poly.UpdateProjectionByPrediction(rect, predictDistance);
  if (m_simplifiedPoly.IsValid())
    m_simplifiedPoly.UpdateProjectionByPrediction(rect, predictDistance);
  return res.IsValid();
}

double Route::GetCurrentSqDistance(m2::PointD const & pt) const
{
  ASSERT(m_poly.IsValid(), ());
  return pt.SquareLength(m_poly.GetCurrentIter().m_pt);
}

double Route::GetPolySegAngle(size_t ind) const
{
  size_t const polySz = m_poly.GetPolyline().GetSize();

  if (ind + 1 >= polySz)
  {
    ASSERT(false, ());
    return 0;
  }

  m2::PointD const p1 = m_poly.GetPolyline().GetPoint(ind);
  m2::PointD p2;
  size_t i = ind + 1;
  do
  {
    p2 = m_poly.GetPolyline().GetPoint(i);
  }
  while (m2::AlmostEqualULPs(p1, p2) && ++i < polySz);
  return (i == polySz) ? 0 : my::RadToDeg(ang::AngleTo(p1, p2));
}

void Route::MatchLocationToRoute(location::GpsInfo & location, location::RouteMatchingInfo & routeMatchingInfo) const
{
  if (m_poly.IsValid())
  {
    auto const & iter = m_poly.GetCurrentIter();
    m2::PointD const locationMerc = MercatorBounds::FromLatLon(location.m_latitude, location.m_longitude);
    double const distFromRouteM = MercatorBounds::DistanceOnEarth(iter.m_pt, locationMerc);
    if (distFromRouteM < m_routingSettings.m_matchingThresholdM)
    {
      location.m_latitude = MercatorBounds::YToLat(iter.m_pt.y);
      location.m_longitude = MercatorBounds::XToLon(iter.m_pt.x);
      if (m_routingSettings.m_matchRoute)
        location.m_bearing = location::AngleToBearing(GetPolySegAngle(iter.m_ind));

      routeMatchingInfo.Set(iter.m_pt, iter.m_ind);
    }
  }
}

bool Route::IsCurrentOnEnd() const
{
  return (m_poly.GetDistanceToEndM() < kOnEndToleranceM);
}

void Route::Update()
{
  if (!m_poly.IsValid())
    return;
  if (m_routingSettings.m_keepPedestrianInfo)
  {
    vector<m2::PointD> points;
    auto distFn = m2::DistanceToLineSquare<m2::PointD>();
    // TODO (ldargunov) Rewrite dist f to distance in meters and avoid 0.00000 constants.
    SimplifyNearOptimal(20, m_poly.GetPolyline().Begin(), m_poly.GetPolyline().End(), 0.00000001, distFn,
                        MakeBackInsertFunctor(points));
    FollowedPolyline(points.begin(), points.end()).Swap(m_simplifiedPoly);
  }
  else
  {
    // Free memory if we don't need simplified geometry.
    FollowedPolyline().Swap(m_simplifiedPoly);
  }
  m_currentTime = 0.0;
}

string DebugPrint(Route const & r)
{
  return DebugPrint(r.m_poly.GetPolyline());
}

} // namespace routing
<commit_msg>Distance to turn calculation fix.<commit_after>#include "route.hpp"
#include "turns_generator.hpp"

#include "indexer/mercator.hpp"

#include "platform/location.hpp"

#include "geometry/angles.hpp"
#include "geometry/point2d.hpp"
#include "geometry/simplification.hpp"

#include "base/logging.hpp"

#include "std/numeric.hpp"
#include "std/utility.hpp"
#include "std/algorithm.hpp"

namespace routing
{
namespace
{
double constexpr kLocationTimeThreshold = 60.0 * 1.0;
double constexpr kOnEndToleranceM = 10.0;

}  //  namespace

Route::Route(string const & router, vector<m2::PointD> const & points, string const & name)
  : m_router(router), m_routingSettings(GetCarRoutingSettings()),
    m_name(name), m_poly(points.begin(), points.end())
{
  Update();
}

void Route::Swap(Route & rhs)
{
  m_router.swap(rhs.m_router);
  swap(m_routingSettings, rhs.m_routingSettings);
  m_poly.Swap(rhs.m_poly);
  m_simplifiedPoly.Swap(rhs.m_simplifiedPoly);
  m_name.swap(rhs.m_name);
  swap(m_currentTime, rhs.m_currentTime);
  swap(m_turns, rhs.m_turns);
  swap(m_times, rhs.m_times);
  m_absentCountries.swap(rhs.m_absentCountries);
}

double Route::GetTotalDistanceMeters() const
{
  return m_poly.GetTotalDistanceM();
}

double Route::GetCurrentDistanceFromBeginMeters() const
{
  return m_poly.GetDistanceFromBeginM();
}

void Route::GetTurnsDistances(vector<double> & distances) const
{
  double mercatorDistance = 0;
  distances.clear();
  auto const & polyline = m_poly.GetPolyline();
  for (auto currentTurn = m_turns.begin(); currentTurn != m_turns.end(); ++currentTurn)
  {
    // Skip turns at side points of the polyline geometry. We can't display them properly.
    if (currentTurn->m_index == 0 || currentTurn->m_index == (polyline.GetSize() - 1))
      continue;

    uint32_t formerTurnIndex = 0;
    if (currentTurn != m_turns.begin())
      formerTurnIndex = (currentTurn - 1)->m_index;

    //TODO (ldragunov) Extract CalculateMercatorDistance higher to avoid including turns generator.
    double const mercatorDistanceBetweenTurns =
      turns::CalculateMercatorDistanceAlongPath(formerTurnIndex,  currentTurn->m_index, polyline.GetPoints());
    mercatorDistance += mercatorDistanceBetweenTurns;

    distances.push_back(mercatorDistance);
   }
}

double Route::GetCurrentDistanceToEndMeters() const
{
  return m_poly.GetDistanceToEndM();
}

double Route::GetMercatorDistanceFromBegin() const
{
  //TODO Maybe better to return FollowedRoute and user will call GetMercatorDistance etc. by itself
  return m_poly.GetMercatorDistanceFromBegin();
}

uint32_t Route::GetTotalTimeSec() const
{
  return m_times.empty() ? 0 : m_times.back().second;
}

uint32_t Route::GetCurrentTimeToEndSec() const
{
  auto const & poly = m_poly.GetPolyline();
  size_t const polySz = poly.GetSize();
  if (m_times.empty() || poly.GetSize() == 0)
  {
    ASSERT(!m_times.empty(), ());
    ASSERT(polySz != 0, ());
    return 0;
  }

  TTimes::const_iterator it = upper_bound(m_times.begin(), m_times.end(), m_poly.GetCurrentIter().m_ind,
                                         [](size_t v, Route::TTimeItem const & item) { return v < item.first; });

  if (it == m_times.end())
    return 0;

  size_t idx = distance(m_times.begin(), it);
  double time = (*it).second;
  if (idx > 0)
    time -= m_times[idx - 1].second;

  auto distFn = [&](size_t start, size_t end)
  {
    return m_poly.GetDistanceM(m_poly.GetIterToIndex(start), m_poly.GetIterToIndex(end));
  };

  ASSERT_LESS_OR_EQUAL(m_times[idx].first, poly.GetSize(), ());
  double const dist = distFn(idx > 0 ? m_times[idx - 1].first : 0, m_times[idx].first);

  if (!my::AlmostEqualULPs(dist, 0.))
  {
    double const distRemain = distFn(m_poly.GetCurrentIter().m_ind, m_times[idx].first) -
                        MercatorBounds::DistanceOnEarth(m_poly.GetCurrentIter().m_pt, poly.GetPoint(m_poly.GetCurrentIter().m_ind));
    return (uint32_t)((GetTotalTimeSec() - (*it).second) + (double)time * (distRemain / dist));
  }
  else
    return (uint32_t)((GetTotalTimeSec() - (*it).second));
}

void Route::GetCurrentTurn(double & distanceToTurnMeters, turns::TurnItem & turn) const
{
  if (m_turns.empty())
  {
    ASSERT(!m_turns.empty(), ());
    distanceToTurnMeters = 0;
    turn = turns::TurnItem();
    return;
  }

  turns::TurnItem t;
  t.m_index = static_cast<uint32_t>(m_poly.GetCurrentIter().m_ind);
  auto it = upper_bound(m_turns.begin(), m_turns.end(), t,
            [](turns::TurnItem const & lhs, turns::TurnItem const & rhs)
            {
              return lhs.m_index < rhs.m_index;
            });

  ASSERT_GREATER_OR_EQUAL((*it).m_index, 0, ());

  size_t const segIdx = (*it).m_index;
  turn = (*it);
  distanceToTurnMeters = m_poly.GetDistanceM(m_poly.GetCurrentIter(),
                                             m_poly.GetIterToIndex(segIdx));
}

void Route::GetCurrentDirectionPoint(m2::PointD & pt) const
{
  if (m_routingSettings.m_keepPedestrianInfo && m_simplifiedPoly.IsValid())
    m_simplifiedPoly.GetCurrentDirectionPoint(pt, kOnEndToleranceM);
  else
    m_poly.GetCurrentDirectionPoint(pt, kOnEndToleranceM);
}

bool Route::MoveIterator(location::GpsInfo const & info) const
{
  double predictDistance = -1.0;
  if (m_currentTime > 0.0 && info.HasSpeed())
  {
    /// @todo Need to distinguish GPS and WiFi locations.
    /// They may have different time metrics in case of incorrect system time on a device.
    double const deltaT = info.m_timestamp - m_currentTime;
    if (deltaT > 0.0 && deltaT < kLocationTimeThreshold)
      predictDistance = info.m_speed * deltaT;
  }

  m2::RectD const rect = MercatorBounds::MetresToXY(
        info.m_longitude, info.m_latitude,
        max(m_routingSettings.m_matchingThresholdM, info.m_horizontalAccuracy));
  FollowedPolyline::Iter const res = m_poly.UpdateProjectionByPrediction(rect, predictDistance);
  if (m_simplifiedPoly.IsValid())
    m_simplifiedPoly.UpdateProjectionByPrediction(rect, predictDistance);
  return res.IsValid();
}

double Route::GetCurrentSqDistance(m2::PointD const & pt) const
{
  ASSERT(m_poly.IsValid(), ());
  return pt.SquareLength(m_poly.GetCurrentIter().m_pt);
}

double Route::GetPolySegAngle(size_t ind) const
{
  size_t const polySz = m_poly.GetPolyline().GetSize();

  if (ind + 1 >= polySz)
  {
    ASSERT(false, ());
    return 0;
  }

  m2::PointD const p1 = m_poly.GetPolyline().GetPoint(ind);
  m2::PointD p2;
  size_t i = ind + 1;
  do
  {
    p2 = m_poly.GetPolyline().GetPoint(i);
  }
  while (m2::AlmostEqualULPs(p1, p2) && ++i < polySz);
  return (i == polySz) ? 0 : my::RadToDeg(ang::AngleTo(p1, p2));
}

void Route::MatchLocationToRoute(location::GpsInfo & location, location::RouteMatchingInfo & routeMatchingInfo) const
{
  if (m_poly.IsValid())
  {
    auto const & iter = m_poly.GetCurrentIter();
    m2::PointD const locationMerc = MercatorBounds::FromLatLon(location.m_latitude, location.m_longitude);
    double const distFromRouteM = MercatorBounds::DistanceOnEarth(iter.m_pt, locationMerc);
    if (distFromRouteM < m_routingSettings.m_matchingThresholdM)
    {
      location.m_latitude = MercatorBounds::YToLat(iter.m_pt.y);
      location.m_longitude = MercatorBounds::XToLon(iter.m_pt.x);
      if (m_routingSettings.m_matchRoute)
        location.m_bearing = location::AngleToBearing(GetPolySegAngle(iter.m_ind));

      routeMatchingInfo.Set(iter.m_pt, iter.m_ind);
    }
  }
}

bool Route::IsCurrentOnEnd() const
{
  return (m_poly.GetDistanceToEndM() < kOnEndToleranceM);
}

void Route::Update()
{
  if (!m_poly.IsValid())
    return;
  if (m_routingSettings.m_keepPedestrianInfo)
  {
    vector<m2::PointD> points;
    auto distFn = m2::DistanceToLineSquare<m2::PointD>();
    // TODO (ldargunov) Rewrite dist f to distance in meters and avoid 0.00000 constants.
    SimplifyNearOptimal(20, m_poly.GetPolyline().Begin(), m_poly.GetPolyline().End(), 0.00000001, distFn,
                        MakeBackInsertFunctor(points));
    FollowedPolyline(points.begin(), points.end()).Swap(m_simplifiedPoly);
  }
  else
  {
    // Free memory if we don't need simplified geometry.
    FollowedPolyline().Swap(m_simplifiedPoly);
  }
  m_currentTime = 0.0;
}

string DebugPrint(Route const & r)
{
  return DebugPrint(r.m_poly.GetPolyline());
}

} // namespace routing
<|endoftext|>


<commit_before>// Based on TensorFlow Serving's gRPC server implementation from
// https://github.com/tensorflow/serving/blob/master/tensorflow_serving/model_servers/main.cc
//
// gRPC server implementation of
// tensorflow_serving/apis/prediction_service.proto.
//
// It bring up a standard server which connects to Apache Zookeeper and
// gets dynamic configuration from there; describing which versions of
// which models have to be loaded from what places.
//
// ModelServer has inter-request batching support built-in, by using the
// BatchingSession at:
//     tensorflow_serving/batching/batching_session.h
//
// For detailed description of example Zookeeper configuration, see
//     zookeeper_source.cc
//
// To specify port (default 8500): --port=my_port
// To enable batching (default disabled): --enable_batching

#include <unistd.h>
#include <iostream>
#include <memory>
#include <utility>
#include <vector>

#include "google/protobuf/wrappers.pb.h"
#include "grpc++/security/server_credentials.h"
#include "grpc++/server.h"
#include "grpc++/server_builder.h"
#include "grpc++/server_context.h"
#include "grpc++/support/status.h"
#include "grpc++/support/status_code_enum.h"
#include "grpc/grpc.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/protobuf/config.pb.h"
#include "tensorflow/core/util/command_line_flags.h"
#include "tensorflow_serving/apis/prediction_service.grpc.pb.h"
#include "tensorflow_serving/apis/prediction_service.pb.h"
#include "tensorflow_serving/config/model_server_config.pb.h"
#include "tensorflow_serving/core/availability_preserving_policy.h"
#include "tensorflow_serving/model_servers/model_platform_types.h"
#include "tensorflow_serving/model_servers/platform_config_util.h"
#include "tensorflow_serving/model_servers/server_core.h"
#include "tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h"
#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h"
#include "cranberries/model_server/predict_impl.h"
#include "cranberries/model_server/model_server_config.pb.h"
#include "zookeeper_cc_util/zookeeper_cc_util.h"
#include "cranberries/core/zookeeper_source.h"
#include "cranberries/core/zookeeper_state_reporter.h"

using tensorflow::serving::AspiredVersionsManager;
using tensorflow::serving::AspiredVersionPolicy;
using tensorflow::serving::AvailabilityPreservingPolicy;
using tensorflow::serving::BatchingParameters;
using tensorflow::serving::EventBus;
using tensorflow::serving::GetModelMetadataImpl;
using tensorflow::serving::SavedModelBundleSourceAdapter;
using tensorflow::serving::SessionBundleSourceAdapterConfig;
using tensorflow::serving::ServableState;
using tensorflow::serving::ServerCore;
using tensorflow::serving::SessionBundleConfig;
using tensorflow::serving::TensorflowPredictor;
using tensorflow::serving::UniquePtrWithDeps;
using tensorflow::string;

using grpc::InsecureServerCredentials;
using grpc::Server;
using grpc::ServerBuilder;
using grpc::ServerContext;
using tensorflow::serving::GetModelMetadataRequest;
using tensorflow::serving::GetModelMetadataResponse;
using tensorflow::serving::PredictRequest;
using tensorflow::serving::PredictResponse;
using tensorflow::serving::PredictionService;
using tensorflow::Status;

using cranberries::ModelServerConfig;
using zookeeper_cc_util::Zookeeper;
using tensorflow::serving::cranberries::ZookeeperSource;
using tensorflow::serving::cranberries::ZookeeperStateReporter;

namespace {

tensorflow::Status LoadCustomModelConfig(
    const ::google::protobuf::Any& any,
    EventBus<ServableState>* servable_event_bus,
    UniquePtrWithDeps<AspiredVersionsManager>* manager) {
  ModelServerConfig config;
  CHECK(any.UnpackTo(&config));

  std::unique_ptr<Zookeeper> zookeeper(
      new Zookeeper(config.zookeeper_hosts(), 100 /* recv_timeout */, config.zookeeper_base()));
  CHECK(zookeeper->Init()) << "Unable to init Zookeeper client";

  std::unique_ptr<ZookeeperStateReporter> state_reporter(
      new ZookeeperStateReporter(zookeeper.get()));
  std::unique_ptr<EventBus<ServableState>::Subscription> subscription =
      servable_event_bus->Subscribe(state_reporter->GetEventBusCallback());

  std::unique_ptr<SavedModelBundleSourceAdapter> bundle_adapter;
  SessionBundleSourceAdapterConfig bundle_adapter_config;
  TF_CHECK_OK(SavedModelBundleSourceAdapter::Create(
      bundle_adapter_config, &bundle_adapter));
  ConnectSourceToTarget(bundle_adapter.get(), manager->get());

  std::unique_ptr<ZookeeperSource> source(
      new ZookeeperSource(zookeeper.get()));
  ConnectSourceToTarget(source.get(), bundle_adapter.get());

  manager->AddDependency(std::move(zookeeper));
  manager->AddDependency(std::move(state_reporter));
  manager->AddDependency(std::move(subscription));
  manager->AddDependency(std::move(bundle_adapter));
  manager->AddDependency(std::move(source));
  return Status::OK();
}

grpc::Status ToGRPCStatus(const tensorflow::Status& status) {
  const int kErrorMessageLimit = 1024;
  string error_message;
  if (status.error_message().length() > kErrorMessageLimit) {
    error_message =
        status.error_message().substr(0, kErrorMessageLimit) + "...TRUNCATED";
  } else {
    error_message = status.error_message();
  }
  return grpc::Status(static_cast<grpc::StatusCode>(status.code()),
                      error_message);
}

class PredictionServiceImpl final : public PredictionService::Service {
 public:
  explicit PredictionServiceImpl(std::unique_ptr<ServerCore> core,
                                 bool use_saved_model)
      : core_(std::move(core)),
        predictor_(new TensorflowPredictor(use_saved_model)),
        use_saved_model_(use_saved_model) {}

  grpc::Status Predict(ServerContext* context, const PredictRequest* request,
                       PredictResponse* response) override {
    const grpc::Status status =
        ToGRPCStatus(predictor_->Predict(core_.get(), *request, response));
    if (!status.ok()) {
      VLOG(1) << "Predict failed: " << status.error_message();
    }
    return status;
  }

  grpc::Status GetModelMetadata(ServerContext* context,
                                const GetModelMetadataRequest* request,
                                GetModelMetadataResponse* response) override {
    if (!use_saved_model_) {
      return ToGRPCStatus(tensorflow::errors::InvalidArgument(
          "GetModelMetadata API is only available when use_saved_model is "
          "set to true"));
    }
    const grpc::Status status =
        ToGRPCStatus(GetModelMetadataImpl::GetModelMetadata(
            core_.get(), *request, response));
    if (!status.ok()) {
      VLOG(1) << "GetModelMetadata failed: " << status.error_message();
    }
    return status;
  }

 private:
  std::unique_ptr<ServerCore> core_;
  std::unique_ptr<TensorflowPredictor> predictor_;
  bool use_saved_model_;
};

void RunServer(int port, std::unique_ptr<ServerCore> core,
               bool use_saved_model) {
  // "0.0.0.0" is the way to listen on localhost in gRPC.
  const string server_address = "0.0.0.0:" + std::to_string(port);
  PredictionServiceImpl service(std::move(core), use_saved_model);
  ServerBuilder builder;
  std::shared_ptr<grpc::ServerCredentials> creds = InsecureServerCredentials();
  builder.AddListeningPort(server_address, creds);
  builder.RegisterService(&service);
  builder.SetMaxMessageSize(tensorflow::kint32max);
  std::unique_ptr<Server> server(builder.BuildAndStart());
  LOG(INFO) << "Running ModelServer at " << server_address << " ...";
  server->Wait();
}

}  // namespace

int main(int argc, char** argv) {
  tensorflow::int32 port = 8500;
  bool enable_batching = false;
  tensorflow::string zookeeper_hosts = "localhost:2181";
  tensorflow::string zookeeper_base;
  // Tensorflow session parallelism of zero means that both inter and intra op
  // thread pools will be auto configured.
  tensorflow::int64 tensorflow_session_parallelism = 0;
  std::vector<tensorflow::Flag> flag_list = {
      tensorflow::Flag("port", &port, "port to listen on"),
      tensorflow::Flag("enable_batching", &enable_batching, "enable batching"),
      tensorflow::Flag("zookeeper_hosts", &zookeeper_hosts,
                       "Specify list of Zookeeper servers in ensemble in format "
                       "host1:port1,host2:port2,host3:port3."),
      tensorflow::Flag("zookeeper_base", &zookeeper_base,
                       "Specify path to the base node for this instance of "
                       "TensorFlow Serving, e.g. /cranberries/servers/server-42 "
                       "(required)."),
      tensorflow::Flag("tensorflow_session_parallelism",
                       &tensorflow_session_parallelism,
                       "Number of threads to use for running a "
                       "Tensorflow session. Auto-configured by default.")};
  string usage = tensorflow::Flags::Usage(argv[0], flag_list);
  const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
  if (!parse_result || zookeeper_base.empty()) {
    std::cout << usage;
    return -1;
  }
  tensorflow::port::InitMain(argv[0], &argc, &argv);
  if (argc != 1) {
    std::cout << "unknown argument: " << argv[1] << "\n" << usage;
    return -1;
  }

  // For ServerCore Options, we leave servable_state_monitor_creator unspecified
  // so the default servable_state_monitor_creator will be used.
  ServerCore::Options options;
  {
    ModelServerConfig config;
    config.set_zookeeper_hosts(zookeeper_hosts);
    config.set_zookeeper_base(zookeeper_base);
    options.model_server_config.mutable_custom_model_config()->PackFrom(config);
  }

  SessionBundleConfig session_bundle_config;
  // Batching config
  if (enable_batching) {
    BatchingParameters* batching_parameters =
        session_bundle_config.mutable_batching_parameters();
    batching_parameters->mutable_thread_pool_name()->set_value(
        "model_server_batch_threads");
  }

  // use_saved_model is fixed to `true` because `LoadCustomModelConfig`
  // uses SavedModelBundleSourceAdapter only for now, and it's hard-coded.

  session_bundle_config.mutable_session_config()
      ->set_intra_op_parallelism_threads(tensorflow_session_parallelism);
  session_bundle_config.mutable_session_config()
      ->set_inter_op_parallelism_threads(tensorflow_session_parallelism);
  options.platform_config_map = CreateTensorFlowPlatformConfigMap(
      session_bundle_config, true /* use_saved_model */);

  options.custom_model_config_loader = &LoadCustomModelConfig;

  options.aspired_version_policy =
      std::unique_ptr<AspiredVersionPolicy>(new AvailabilityPreservingPolicy);

  std::unique_ptr<ServerCore> core;
  TF_CHECK_OK(ServerCore::Create(std::move(options), &core));
  RunServer(port, std::move(core), true /* use_saved_model */);

  return 0;
}
<commit_msg>model-server: raise hardcoded recv_timeout from 100ms to 2000ms<commit_after>// Based on TensorFlow Serving's gRPC server implementation from
// https://github.com/tensorflow/serving/blob/master/tensorflow_serving/model_servers/main.cc
//
// gRPC server implementation of
// tensorflow_serving/apis/prediction_service.proto.
//
// It bring up a standard server which connects to Apache Zookeeper and
// gets dynamic configuration from there; describing which versions of
// which models have to be loaded from what places.
//
// ModelServer has inter-request batching support built-in, by using the
// BatchingSession at:
//     tensorflow_serving/batching/batching_session.h
//
// For detailed description of example Zookeeper configuration, see
//     zookeeper_source.cc
//
// To specify port (default 8500): --port=my_port
// To enable batching (default disabled): --enable_batching

#include <unistd.h>
#include <iostream>
#include <memory>
#include <utility>
#include <vector>

#include "google/protobuf/wrappers.pb.h"
#include "grpc++/security/server_credentials.h"
#include "grpc++/server.h"
#include "grpc++/server_builder.h"
#include "grpc++/server_context.h"
#include "grpc++/support/status.h"
#include "grpc++/support/status_code_enum.h"
#include "grpc/grpc.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/protobuf/config.pb.h"
#include "tensorflow/core/util/command_line_flags.h"
#include "tensorflow_serving/apis/prediction_service.grpc.pb.h"
#include "tensorflow_serving/apis/prediction_service.pb.h"
#include "tensorflow_serving/config/model_server_config.pb.h"
#include "tensorflow_serving/core/availability_preserving_policy.h"
#include "tensorflow_serving/model_servers/model_platform_types.h"
#include "tensorflow_serving/model_servers/platform_config_util.h"
#include "tensorflow_serving/model_servers/server_core.h"
#include "tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h"
#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h"
#include "cranberries/model_server/predict_impl.h"
#include "cranberries/model_server/model_server_config.pb.h"
#include "zookeeper_cc_util/zookeeper_cc_util.h"
#include "cranberries/core/zookeeper_source.h"
#include "cranberries/core/zookeeper_state_reporter.h"

using tensorflow::serving::AspiredVersionsManager;
using tensorflow::serving::AspiredVersionPolicy;
using tensorflow::serving::AvailabilityPreservingPolicy;
using tensorflow::serving::BatchingParameters;
using tensorflow::serving::EventBus;
using tensorflow::serving::GetModelMetadataImpl;
using tensorflow::serving::SavedModelBundleSourceAdapter;
using tensorflow::serving::SessionBundleSourceAdapterConfig;
using tensorflow::serving::ServableState;
using tensorflow::serving::ServerCore;
using tensorflow::serving::SessionBundleConfig;
using tensorflow::serving::TensorflowPredictor;
using tensorflow::serving::UniquePtrWithDeps;
using tensorflow::string;

using grpc::InsecureServerCredentials;
using grpc::Server;
using grpc::ServerBuilder;
using grpc::ServerContext;
using tensorflow::serving::GetModelMetadataRequest;
using tensorflow::serving::GetModelMetadataResponse;
using tensorflow::serving::PredictRequest;
using tensorflow::serving::PredictResponse;
using tensorflow::serving::PredictionService;
using tensorflow::Status;

using cranberries::ModelServerConfig;
using zookeeper_cc_util::Zookeeper;
using tensorflow::serving::cranberries::ZookeeperSource;
using tensorflow::serving::cranberries::ZookeeperStateReporter;

namespace {

tensorflow::Status LoadCustomModelConfig(
    const ::google::protobuf::Any& any,
    EventBus<ServableState>* servable_event_bus,
    UniquePtrWithDeps<AspiredVersionsManager>* manager) {
  ModelServerConfig config;
  CHECK(any.UnpackTo(&config));

  std::unique_ptr<Zookeeper> zookeeper(
      new Zookeeper(config.zookeeper_hosts(), 2000 /* recv_timeout */, config.zookeeper_base()));
  CHECK(zookeeper->Init()) << "Unable to init Zookeeper client";

  std::unique_ptr<ZookeeperStateReporter> state_reporter(
      new ZookeeperStateReporter(zookeeper.get()));
  std::unique_ptr<EventBus<ServableState>::Subscription> subscription =
      servable_event_bus->Subscribe(state_reporter->GetEventBusCallback());

  std::unique_ptr<SavedModelBundleSourceAdapter> bundle_adapter;
  SessionBundleSourceAdapterConfig bundle_adapter_config;
  TF_CHECK_OK(SavedModelBundleSourceAdapter::Create(
      bundle_adapter_config, &bundle_adapter));
  ConnectSourceToTarget(bundle_adapter.get(), manager->get());

  std::unique_ptr<ZookeeperSource> source(
      new ZookeeperSource(zookeeper.get()));
  ConnectSourceToTarget(source.get(), bundle_adapter.get());

  manager->AddDependency(std::move(zookeeper));
  manager->AddDependency(std::move(state_reporter));
  manager->AddDependency(std::move(subscription));
  manager->AddDependency(std::move(bundle_adapter));
  manager->AddDependency(std::move(source));
  return Status::OK();
}

grpc::Status ToGRPCStatus(const tensorflow::Status& status) {
  const int kErrorMessageLimit = 1024;
  string error_message;
  if (status.error_message().length() > kErrorMessageLimit) {
    error_message =
        status.error_message().substr(0, kErrorMessageLimit) + "...TRUNCATED";
  } else {
    error_message = status.error_message();
  }
  return grpc::Status(static_cast<grpc::StatusCode>(status.code()),
                      error_message);
}

class PredictionServiceImpl final : public PredictionService::Service {
 public:
  explicit PredictionServiceImpl(std::unique_ptr<ServerCore> core,
                                 bool use_saved_model)
      : core_(std::move(core)),
        predictor_(new TensorflowPredictor(use_saved_model)),
        use_saved_model_(use_saved_model) {}

  grpc::Status Predict(ServerContext* context, const PredictRequest* request,
                       PredictResponse* response) override {
    const grpc::Status status =
        ToGRPCStatus(predictor_->Predict(core_.get(), *request, response));
    if (!status.ok()) {
      VLOG(1) << "Predict failed: " << status.error_message();
    }
    return status;
  }

  grpc::Status GetModelMetadata(ServerContext* context,
                                const GetModelMetadataRequest* request,
                                GetModelMetadataResponse* response) override {
    if (!use_saved_model_) {
      return ToGRPCStatus(tensorflow::errors::InvalidArgument(
          "GetModelMetadata API is only available when use_saved_model is "
          "set to true"));
    }
    const grpc::Status status =
        ToGRPCStatus(GetModelMetadataImpl::GetModelMetadata(
            core_.get(), *request, response));
    if (!status.ok()) {
      VLOG(1) << "GetModelMetadata failed: " << status.error_message();
    }
    return status;
  }

 private:
  std::unique_ptr<ServerCore> core_;
  std::unique_ptr<TensorflowPredictor> predictor_;
  bool use_saved_model_;
};

void RunServer(int port, std::unique_ptr<ServerCore> core,
               bool use_saved_model) {
  // "0.0.0.0" is the way to listen on localhost in gRPC.
  const string server_address = "0.0.0.0:" + std::to_string(port);
  PredictionServiceImpl service(std::move(core), use_saved_model);
  ServerBuilder builder;
  std::shared_ptr<grpc::ServerCredentials> creds = InsecureServerCredentials();
  builder.AddListeningPort(server_address, creds);
  builder.RegisterService(&service);
  builder.SetMaxMessageSize(tensorflow::kint32max);
  std::unique_ptr<Server> server(builder.BuildAndStart());
  LOG(INFO) << "Running ModelServer at " << server_address << " ...";
  server->Wait();
}

}  // namespace

int main(int argc, char** argv) {
  tensorflow::int32 port = 8500;
  bool enable_batching = false;
  tensorflow::string zookeeper_hosts = "localhost:2181";
  tensorflow::string zookeeper_base;
  // Tensorflow session parallelism of zero means that both inter and intra op
  // thread pools will be auto configured.
  tensorflow::int64 tensorflow_session_parallelism = 0;
  std::vector<tensorflow::Flag> flag_list = {
      tensorflow::Flag("port", &port, "port to listen on"),
      tensorflow::Flag("enable_batching", &enable_batching, "enable batching"),
      tensorflow::Flag("zookeeper_hosts", &zookeeper_hosts,
                       "Specify list of Zookeeper servers in ensemble in format "
                       "host1:port1,host2:port2,host3:port3."),
      tensorflow::Flag("zookeeper_base", &zookeeper_base,
                       "Specify path to the base node for this instance of "
                       "TensorFlow Serving, e.g. /cranberries/servers/server-42 "
                       "(required)."),
      tensorflow::Flag("tensorflow_session_parallelism",
                       &tensorflow_session_parallelism,
                       "Number of threads to use for running a "
                       "Tensorflow session. Auto-configured by default.")};
  string usage = tensorflow::Flags::Usage(argv[0], flag_list);
  const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
  if (!parse_result || zookeeper_base.empty()) {
    std::cout << usage;
    return -1;
  }
  tensorflow::port::InitMain(argv[0], &argc, &argv);
  if (argc != 1) {
    std::cout << "unknown argument: " << argv[1] << "\n" << usage;
    return -1;
  }

  // For ServerCore Options, we leave servable_state_monitor_creator unspecified
  // so the default servable_state_monitor_creator will be used.
  ServerCore::Options options;
  {
    ModelServerConfig config;
    config.set_zookeeper_hosts(zookeeper_hosts);
    config.set_zookeeper_base(zookeeper_base);
    options.model_server_config.mutable_custom_model_config()->PackFrom(config);
  }

  SessionBundleConfig session_bundle_config;
  // Batching config
  if (enable_batching) {
    BatchingParameters* batching_parameters =
        session_bundle_config.mutable_batching_parameters();
    batching_parameters->mutable_thread_pool_name()->set_value(
        "model_server_batch_threads");
  }

  // use_saved_model is fixed to `true` because `LoadCustomModelConfig`
  // uses SavedModelBundleSourceAdapter only for now, and it's hard-coded.

  session_bundle_config.mutable_session_config()
      ->set_intra_op_parallelism_threads(tensorflow_session_parallelism);
  session_bundle_config.mutable_session_config()
      ->set_inter_op_parallelism_threads(tensorflow_session_parallelism);
  options.platform_config_map = CreateTensorFlowPlatformConfigMap(
      session_bundle_config, true /* use_saved_model */);

  options.custom_model_config_loader = &LoadCustomModelConfig;

  options.aspired_version_policy =
      std::unique_ptr<AspiredVersionPolicy>(new AvailabilityPreservingPolicy);

  std::unique_ptr<ServerCore> core;
  TF_CHECK_OK(ServerCore::Create(std::move(options), &core));
  RunServer(port, std::move(core), true /* use_saved_model */);

  return 0;
}
<|endoftext|>


<commit_before>#include "kevlar/Converters/HDF5ParticleLabelVector.hh"
#include "kevlar/Services/HDF5File.hh"
#include "art/Framework/Principal/Event.h"
#include "art/Framework/Principal/Run.h"
#include "art/Framework/Principal/SubRun.h"
#include "fhiclcpp/ParameterSet.h"
#include "larcore/Geometry/Geometry.h"
#include "lardataobj/RawData/RawDigit.h"
#include "nusimdata/SimulationBase/MCTruth.h"
#include "messagefacility/MessageLogger/MessageLogger.h"
#include "TDatabasePDG.h"

#include <exception>
#include <fstream>
#include <string>
#include <iostream>
#include <algorithm>
#include <sstream>

namespace kevlar{

  class PDGNameNotFound: public std::exception
  {
    virtual const char* what() const throw()
    {
      return "Name in particle label vector is not in PDG DB";
    }
  };

  HDF5ParticleLabelVector::HDF5ParticleLabelVector(fhicl::ParameterSet const & pSet):
      art::EDAnalyzer(pSet),
      fProducerName(pSet.get<std::string>("ProducerLabel","largeant")),
      fDataSetName(pSet.get<std::string>("DataSetLabel","type")),
      fLabels(pSet.get< std::vector<std::string> >("Labels")),
      fDims{
        0,
        fLabels.size(),
      },
      fMaxDims{
        H5S_UNLIMITED, fLabels.size(),
      },
      fChunkDims{
        pSet.get<uint32_t>("ChunkSize",1), fLabels.size()
      },
      fDataSpace(2, fDims, fMaxDims),
      fParms(),
      fDataSet(NULL),
      fFillValue(pSet.get<uint32_t>("FillValue",0)),
      fNEvents(0),
      fBuffer(boost::extents[fChunkDims[0]][fChunkDims[1]]),
      fBufferCounter(0)
  {
      fParms.setChunk( 2, fChunkDims );
      fParms.setFillValue( H5::PredType::NATIVE_INT, &fFillValue);
      fParms.setDeflate(pSet.get<uint32_t>("CompressionLevel",5));
  }

  HDF5ParticleLabelVector::~HDF5ParticleLabelVector()
  {

  }

  void HDF5ParticleLabelVector::analyze(art::Event const & evt)
  {

    art::Handle< std::vector< simb::MCTruth > > mct_handle;
    std::cout<<"HDF5ParticleLabelVector:"<<this->fDataSetName<<" reading into buffer"<<std::endl;
    evt.getByLabel(fProducerName, mct_handle);
    for (auto truth: *mct_handle){
      for(int i=0; i<truth.NParticles(); ++i){
        int pdg = truth.GetParticle(i).PdgCode();
        if (abs(pdg) > 1E7) continue; // don't try GetName() on the Argon nucleus, e.g.
        std::cout<<"Found particle: "<<pdg<<" "<<truth.GetParticle(i).Process()<<std::endl;
        auto particle = TDatabasePDG::Instance()->GetParticle(pdg);
        if(!particle)
          continue;
        std::string name = particle->GetName();

        ptrdiff_t index = std::find(fLabels.begin(), fLabels.end(), name) - fLabels.begin();

        if(index < int(fLabels.size()) && index>=0 ){
          std::cout<<"Found Particle with name: "<<name<< " and output vector index: "<<index<<std::endl;

          fBuffer[fBufferCounter][index] = fBuffer[fBufferCounter][index] + 1;
        }
        else{
          std::cout<<"Found Particle Outside Label Table: "<<name<<std::endl;
        }
      }
    }

    (this->fBufferCounter)++;
    (this->fNEvents)++;
    std::cout<<"HDF5ParticleLabelVector Buffer for "<<this->fDataSetName<<" :"<<this->fBufferCounter<<" Events"<<std::endl;

    if (this->fBufferCounter == this->fChunkDims[0]){

      hsize_t newSize[2] = {this->fNEvents,fDims[1]};
      this->fDataSet->extend( newSize );

      H5::DataSpace filespace(this->fDataSet->getSpace());

      hsize_t offset[2]={this->fNEvents-fChunkDims[0],0};
      filespace.selectHyperslab( H5S_SELECT_SET, this->fChunkDims, offset );

      H5::DataSpace memspace(2, fChunkDims);
      std::cout<<"HDF5ParticleLabelVector:"<<this->fDataSetName<<" writing buffer to file"<<std::endl;
      this->fDataSet->write( fBuffer.data(), H5::PredType::NATIVE_INT,
                              memspace, filespace );
      this->fBufferCounter=0;
      fBuffer = boost::multi_array<int, 2>(boost::extents[fChunkDims[0]][fChunkDims[1]]);
      std::cout<<"HDF5ParticleLabelVector:"<<this->fDataSetName<<" finished resetting buffer"<<std::endl;
    }

  }

  void HDF5ParticleLabelVector::beginJob()
  {
    art::ServiceHandle<kevlar::HDF5File> _OutputFile;
    std::string group_name = "label";
    fDataSet = _OutputFile->CreateDataSet(this->fDataSetName,group_name,
      this->fDataSpace,
      this->fParms);

     H5::DataSpace attr_dataspace = H5::DataSpace(H5S_SCALAR);
     H5::StrType strdatatype(H5::PredType::C_S1, 256);

     {
      // Set up write buffer for attribute
      const std::string ATTR_NAME ("index1");
      const std::string strwritebuf ("particle_type");

      // Create attribute and write to it
      H5::Attribute myatt_in = this->fDataSet->createAttribute(ATTR_NAME, strdatatype, attr_dataspace);
      myatt_in.write(strdatatype, strwritebuf);
    }

    uint32_t label_index=0;
    for (auto label : fLabels){

      // Set up write buffer for attribute
      std::stringstream name;
      name<<"label_index"<<label_index;

      const std::string ATTR_NAME (name.str());
      const std::string strwritebuf (label);

      // Create attribute and write to it
      H5::Attribute myatt_in = this->fDataSet->createAttribute(ATTR_NAME, strdatatype, attr_dataspace);
      myatt_in.write(strdatatype, strwritebuf);

      label_index+=1;

    }
  }

  void HDF5ParticleLabelVector::endJob()
  {
    if(!(this->fBufferCounter==0)){
      // The new size is now the number of  events in the file
      hsize_t newSize[2] = {this->fNEvents,fDims[1]};
      this->fDataSet->extend( newSize );

      //The filespace is always the same.
      H5::DataSpace filespace(this->fDataSet->getSpace());

      // The offset is the number of events less the size of the buffer
      hsize_t offset[2] = {this->fNEvents-fBufferCounter,0};

      // Leftovers are the size of the buffer, not the chunk
      hsize_t leftover_size[2] = { this->fBufferCounter, fChunkDims[1] };
      filespace.selectHyperslab( H5S_SELECT_SET, leftover_size, offset );

      //Select a memoty space for the data to use
      H5::DataSpace memspace(2, leftover_size);

      // Now write and reset
      this->fDataSet->write( fBuffer.data(), H5::PredType::NATIVE_INT, memspace, filespace );
      this->fBufferCounter=0;
    }
  }
}
Contact GitHub API Training Shop Blog About
© 2017 GitHub, Inc. Terms Privacy Security Status Help<commit_msg>Bugfix<commit_after>#include "kevlar/Converters/HDF5ParticleLabelVector.hh"
#include "kevlar/Services/HDF5File.hh"
#include "art/Framework/Principal/Event.h"
#include "art/Framework/Principal/Run.h"
#include "art/Framework/Principal/SubRun.h"
#include "fhiclcpp/ParameterSet.h"
#include "larcore/Geometry/Geometry.h"
#include "lardataobj/RawData/RawDigit.h"
#include "nusimdata/SimulationBase/MCTruth.h"
#include "messagefacility/MessageLogger/MessageLogger.h"
#include "TDatabasePDG.h"

#include <exception>
#include <fstream>
#include <string>
#include <iostream>
#include <algorithm>
#include <sstream>

namespace kevlar{

  class PDGNameNotFound: public std::exception
  {
    virtual const char* what() const throw()
    {
      return "Name in particle label vector is not in PDG DB";
    }
  };

  HDF5ParticleLabelVector::HDF5ParticleLabelVector(fhicl::ParameterSet const & pSet):
      art::EDAnalyzer(pSet),
      fProducerName(pSet.get<std::string>("ProducerLabel","largeant")),
      fDataSetName(pSet.get<std::string>("DataSetLabel","type")),
      fLabels(pSet.get< std::vector<std::string> >("Labels")),
      fDims{
        0,
        fLabels.size(),
      },
      fMaxDims{
        H5S_UNLIMITED, fLabels.size(),
      },
      fChunkDims{
        pSet.get<uint32_t>("ChunkSize",1), fLabels.size()
      },
      fDataSpace(2, fDims, fMaxDims),
      fParms(),
      fDataSet(NULL),
      fFillValue(pSet.get<uint32_t>("FillValue",0)),
      fNEvents(0),
      fBuffer(boost::extents[fChunkDims[0]][fChunkDims[1]]),
      fBufferCounter(0)
  {
      fParms.setChunk( 2, fChunkDims );
      fParms.setFillValue( H5::PredType::NATIVE_INT, &fFillValue);
      fParms.setDeflate(pSet.get<uint32_t>("CompressionLevel",5));
  }

  HDF5ParticleLabelVector::~HDF5ParticleLabelVector()
  {

  }

  void HDF5ParticleLabelVector::analyze(art::Event const & evt)
  {

    art::Handle< std::vector< simb::MCTruth > > mct_handle;
    std::cout<<"HDF5ParticleLabelVector:"<<this->fDataSetName<<" reading into buffer"<<std::endl;
    evt.getByLabel(fProducerName, mct_handle);
    for (auto truth: *mct_handle){
      for(int i=0; i<truth.NParticles(); ++i){
        int pdg = truth.GetParticle(i).PdgCode();
        if (abs(pdg) > 1E7) continue; // don't try GetName() on the Argon nucleus, e.g.
        std::cout<<"Found particle: "<<pdg<<" "<<truth.GetParticle(i).Process()<<std::endl;
        auto particle = TDatabasePDG::Instance()->GetParticle(pdg);
        if(!particle)
          continue;
        std::string name = particle->GetName();

        ptrdiff_t index = std::find(fLabels.begin(), fLabels.end(), name) - fLabels.begin();

        if(index < int(fLabels.size()) && index>=0 ){
          std::cout<<"Found Particle with name: "<<name<< " and output vector index: "<<index<<std::endl;

          fBuffer[fBufferCounter][index] = fBuffer[fBufferCounter][index] + 1;
        }
        else{
          std::cout<<"Found Particle Outside Label Table: "<<name<<std::endl;
        }
      }
    }

    (this->fBufferCounter)++;
    (this->fNEvents)++;
    std::cout<<"HDF5ParticleLabelVector Buffer for "<<this->fDataSetName<<" :"<<this->fBufferCounter<<" Events"<<std::endl;

    if (this->fBufferCounter == this->fChunkDims[0]){

      hsize_t newSize[2] = {this->fNEvents,fDims[1]};
      this->fDataSet->extend( newSize );

      H5::DataSpace filespace(this->fDataSet->getSpace());

      hsize_t offset[2]={this->fNEvents-fChunkDims[0],0};
      filespace.selectHyperslab( H5S_SELECT_SET, this->fChunkDims, offset );

      H5::DataSpace memspace(2, fChunkDims);
      std::cout<<"HDF5ParticleLabelVector:"<<this->fDataSetName<<" writing buffer to file"<<std::endl;
      this->fDataSet->write( fBuffer.data(), H5::PredType::NATIVE_INT,
                              memspace, filespace );
      this->fBufferCounter=0;
      fBuffer = boost::multi_array<int, 2>(boost::extents[fChunkDims[0]][fChunkDims[1]]);
      std::cout<<"HDF5ParticleLabelVector:"<<this->fDataSetName<<" finished resetting buffer"<<std::endl;
    }

  }

  void HDF5ParticleLabelVector::beginJob()
  {
    art::ServiceHandle<kevlar::HDF5File> _OutputFile;
    std::string group_name = "label";
    fDataSet = _OutputFile->CreateDataSet(this->fDataSetName,group_name,
      this->fDataSpace,
      this->fParms);

     H5::DataSpace attr_dataspace = H5::DataSpace(H5S_SCALAR);
     H5::StrType strdatatype(H5::PredType::C_S1, 256);

     {
      // Set up write buffer for attribute
      const std::string ATTR_NAME ("index1");
      const std::string strwritebuf ("particle_type");

      // Create attribute and write to it
      H5::Attribute myatt_in = this->fDataSet->createAttribute(ATTR_NAME, strdatatype, attr_dataspace);
      myatt_in.write(strdatatype, strwritebuf);
    }

    uint32_t label_index=0;
    for (auto label : fLabels){

      // Set up write buffer for attribute
      std::stringstream name;
      name<<"label_index"<<label_index;

      const std::string ATTR_NAME (name.str());
      const std::string strwritebuf (label);

      // Create attribute and write to it
      H5::Attribute myatt_in = this->fDataSet->createAttribute(ATTR_NAME, strdatatype, attr_dataspace);
      myatt_in.write(strdatatype, strwritebuf);

      label_index+=1;

    }
  }

  void HDF5ParticleLabelVector::endJob()
  {
    if(!(this->fBufferCounter==0)){
      // The new size is now the number of  events in the file
      hsize_t newSize[2] = {this->fNEvents,fDims[1]};
      this->fDataSet->extend( newSize );

      //The filespace is always the same.
      H5::DataSpace filespace(this->fDataSet->getSpace());

      // The offset is the number of events less the size of the buffer
      hsize_t offset[2] = {this->fNEvents-fBufferCounter,0};

      // Leftovers are the size of the buffer, not the chunk
      hsize_t leftover_size[2] = { this->fBufferCounter, fChunkDims[1] };
      filespace.selectHyperslab( H5S_SELECT_SET, leftover_size, offset );

      //Select a memoty space for the data to use
      H5::DataSpace memspace(2, leftover_size);

      // Now write and reset
      this->fDataSet->write( fBuffer.data(), H5::PredType::NATIVE_INT, memspace, filespace );
      this->fBufferCounter=0;
    }
  }
}
<|endoftext|>


<commit_before>#include "acmacs-map-draw/mapi-settings.hh"
#include "acmacs-map-draw/draw.hh"

// ----------------------------------------------------------------------

bool acmacs::mapi::v1::Settings::apply_built_in(std::string_view name) // returns true if built-in command with that name found and applied
{
    using namespace std::string_view_literals;
    AD_LOG(acmacs::log::settings, "mapi::apply_built_in \"{}\"", name);
    AD_LOG_INDENT;
    try {
        // printenv();
        if (name == "antigens"sv)
            return apply_antigens();
        else if (name == "sera"sv)
            return apply_sera();
        else if (name == "vaccine"sv)
            return apply_vaccine();
        else if (name == "circle"sv)
            return apply_circle();
        else if (name == "path"sv)
            return apply_path();
        else if (name == "rotate"sv)
            return apply_rotate();
        else if (name == "flip"sv)
            return apply_flip();
        else if (name == "viewport"sv)
            return apply_viewport();
        else if (name == "background"sv)
            return apply_background();
        else if (name == "border"sv)
            return apply_border();
        else if (name == "grid"sv)
            return apply_grid();
        else if (name == "point-scale"sv || name == "point_scale"sv)
            return apply_point_scale();
        else if (name == "connection_lines"sv || name == "connection-lines"sv)
            return apply_connection_lines();
        else if (name == "error_lines"sv || name == "error-lines"sv)
            return apply_error_lines();
        else if (name == "legend"sv)
            return apply_legend();
        else if (name == "title"sv)
            return apply_title();
        else if (name == "serum-circles"sv || name == "serum-circle"sv || name == "serum_circles"sv || name == "serum_circle"sv)
            return apply_serum_circles();
        else if (name == "serum-coverage"sv || name == "serum_coverage"sv)
            return apply_serum_coverage();
        else if (name == "procrustes-arrows"sv || name == "procrustes_arrows"sv)
            return apply_procrustes();
        else if (name == "move"sv)
            return apply_move();
        else if (name == "reset"sv)
            return apply_reset();
        else if (name == "export"sv)
            return apply_export();
        else if (name == "pdf"sv)
            return apply_pdf();
        else if (name == "relax"sv)
            return apply_relax();
        else if (name == "compare-sequences"sv || name == "compare_sequences"sv)
            return apply_compare_sequences();
        else if (name == "time-series"sv || name == "time_series"sv)
            return apply_time_series();
        // else if (name == ""sv)
        //     return apply_();
        return acmacs::settings::Settings::apply_built_in(name);
    }
    catch (std::exception& err) {
        throw error{fmt::format("cannot apply \"{}\": {} while reading {}", name, err, getenv_toplevel())};
    }

} // acmacs::mapi::v1::Settings::apply_built_in

// ----------------------------------------------------------------------

// void acmacs::mapi::v1::Settings::chart_metadata(fmt::dynamic_format_arg_store<fmt::format_context>& store, const ChartAccess& chart_access) const
// {
//     using namespace std::string_view_literals;

//     const auto& chart{chart_access.chart()};
//     auto info{chart.info()};
//     auto titers{chart.titers()};
//     const auto& projection = chart_access.modified_projection();

//     const std::string virus_type{info->virus_type()}, lineage{chart.lineage()};
//     const auto assay{info->assay()};
//     const auto subset{info->subset(chart::Info::Compute::Yes)};

//     std::string virus_type_lineage;
//     if (lineage.empty())
//         virus_type_lineage = virus_type;
//     else
//         virus_type_lineage = fmt::format("{}/{}", virus_type, ::string::capitalize(lineage.substr(0, 3)));
//     std::string virus_type_lineage_subset_short;
//     if (virus_type == "A(H1N1)"sv) {
//         if (subset == "2009pdm"sv)
//             virus_type_lineage_subset_short = "h1pdm";
//         else
//             virus_type_lineage_subset_short = "h1";
//     }
//     else if (virus_type == "A(H3N2)"sv)
//         virus_type_lineage_subset_short = "h3";
//     else if (virus_type == "B"sv)
//         virus_type_lineage_subset_short = fmt::format("b{}", ::string::lower(lineage.substr(0, 3)));
//     else
//         virus_type_lineage_subset_short = ::string::lower(virus_type);

//     // https://stackoverflow.com/questions/61635042/c-how-can-i-formatting-stdstring-with-collection-efficiently
//     store.push_back(fmt::arg("virus", info->virus()));
//     store.push_back(fmt::arg("virus_type", virus_type));
//     store.push_back(fmt::arg("virus_type_lineage", virus_type_lineage));
//     store.push_back(fmt::arg("lineage", lineage));
//     store.push_back(fmt::arg("subset", subset));
//     store.push_back(fmt::arg("SUBSET", ::string::upper(subset)));
//     store.push_back(fmt::arg("virus_type_lineage_subset_short", virus_type_lineage_subset_short));
//     store.push_back(fmt::arg("assay", assay.hi_or_neut()));
//     store.push_back(fmt::arg("assay_full", assay));
//     store.push_back(fmt::arg("lab", info->lab()));
//     store.push_back(fmt::arg("lab_lower", ::string::lower(info->lab())));
//     store.push_back(fmt::arg("rbc", info->rbc_species()));
//     store.push_back(fmt::arg("table_date", info->date(chart::Info::Compute::Yes)));
//     store.push_back(fmt::arg("number_of_antigens", chart.number_of_antigens()));
//     store.push_back(fmt::arg("number_of_sera", chart.number_of_sera()));
//     store.push_back(fmt::arg("number_of_layers", titers->number_of_layers()));
//     store.push_back(fmt::arg("minimum_column_basis", static_cast<std::string>(projection.minimum_column_basis())));
//     store.push_back(fmt::arg("stress", projection.stress()));
//     store.push_back(fmt::arg("name", chart.make_name()));

// } // acmacs::mapi::v1::Settings::chart_metadata

// ----------------------------------------------------------------------

// std::string acmacs::mapi::v1::Settings::substitute_chart_metadata(std::string_view pattern, const ChartAccess& chart_access) const
// {
//     try {
//         fmt::dynamic_format_arg_store<fmt::format_context> store;
//         chart_metadata(store, chart_access);
//         return fmt::vformat(pattern, store);
//     }
//     catch (std::exception& err) {
//         AD_ERROR("fmt cannot substitute in \"{}\": {}", pattern, err);
//         throw;
//     }

// } // acmacs::mapi::v1::Settings::substitute_chart_metadata

// ----------------------------------------------------------------------

constexpr static const char* sUpdateEnvPattern = R"({{
"init": [
{{"N": "set",
"virus": "{virus}",
"virus-type": "{virus_type}",
"virus-type/lineage": "{virus_type_lineage}",
"lineage": "{lineage}",
"subset": "{subset}",
"SUBSET": "{SUBSET}",
"assay": "{assay}",
"assay-full": "{assay_full}",
"lab": "{lab}",
"rbc": "{rbc}",
"table-date": "{table_date}",
"number-of-antigens": {number_of_antigens},
"number-of-sera": {number_of_sera},
"number-of-layers": {number_of_layers}
}}
]}})";

void acmacs::mapi::v1::Settings::update_env()
{
    const auto json{chart_draw().chart(0).substitute_metadata(sUpdateEnvPattern)};
    // AD_DEBUG("update_env\n{}", json);
    load_from_string(json);

} // acmacs::mapi::v1::Settings::update_env

// ----------------------------------------------------------------------

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>chart metadata substitution refactored<commit_after>#include "acmacs-map-draw/mapi-settings.hh"
#include "acmacs-map-draw/draw.hh"

// ----------------------------------------------------------------------

bool acmacs::mapi::v1::Settings::apply_built_in(std::string_view name) // returns true if built-in command with that name found and applied
{
    using namespace std::string_view_literals;
    AD_LOG(acmacs::log::settings, "mapi::apply_built_in \"{}\"", name);
    AD_LOG_INDENT;
    try {
        // printenv();
        if (name == "antigens"sv)
            return apply_antigens();
        else if (name == "sera"sv)
            return apply_sera();
        else if (name == "vaccine"sv)
            return apply_vaccine();
        else if (name == "circle"sv)
            return apply_circle();
        else if (name == "path"sv)
            return apply_path();
        else if (name == "rotate"sv)
            return apply_rotate();
        else if (name == "flip"sv)
            return apply_flip();
        else if (name == "viewport"sv)
            return apply_viewport();
        else if (name == "background"sv)
            return apply_background();
        else if (name == "border"sv)
            return apply_border();
        else if (name == "grid"sv)
            return apply_grid();
        else if (name == "point-scale"sv || name == "point_scale"sv)
            return apply_point_scale();
        else if (name == "connection_lines"sv || name == "connection-lines"sv)
            return apply_connection_lines();
        else if (name == "error_lines"sv || name == "error-lines"sv)
            return apply_error_lines();
        else if (name == "legend"sv)
            return apply_legend();
        else if (name == "title"sv)
            return apply_title();
        else if (name == "serum-circles"sv || name == "serum-circle"sv || name == "serum_circles"sv || name == "serum_circle"sv)
            return apply_serum_circles();
        else if (name == "serum-coverage"sv || name == "serum_coverage"sv)
            return apply_serum_coverage();
        else if (name == "procrustes-arrows"sv || name == "procrustes_arrows"sv)
            return apply_procrustes();
        else if (name == "move"sv)
            return apply_move();
        else if (name == "reset"sv)
            return apply_reset();
        else if (name == "export"sv)
            return apply_export();
        else if (name == "pdf"sv)
            return apply_pdf();
        else if (name == "relax"sv)
            return apply_relax();
        else if (name == "compare-sequences"sv || name == "compare_sequences"sv)
            return apply_compare_sequences();
        else if (name == "time-series"sv || name == "time_series"sv)
            return apply_time_series();
        // else if (name == ""sv)
        //     return apply_();
        return acmacs::settings::Settings::apply_built_in(name);
    }
    catch (std::exception& err) {
        throw error{fmt::format("cannot apply \"{}\": {} while reading {}", name, err, getenv_toplevel())};
    }

} // acmacs::mapi::v1::Settings::apply_built_in

// ----------------------------------------------------------------------

constexpr static const char* sUpdateEnvPattern = R"({{
"init": [
{{"N": "set",
"virus": "{virus}",
"virus-type": "{virus_type}",
"virus-type/lineage": "{virus_type_lineage}",
"lineage": "{lineage}",
"subset": "{subset}",
"SUBSET": "{SUBSET}",
"assay": "{assay}",
"assay-full": "{assay_full}",
"lab": "{lab}",
"rbc": "{rbc}",
"table-date": "{table_date}",
"number-of-antigens": {number_of_antigens},
"number-of-sera": {number_of_sera},
"number-of-layers": {number_of_layers}
}}
]}})";

void acmacs::mapi::v1::Settings::update_env()
{
    const auto json{chart_draw().chart(0).substitute_metadata(sUpdateEnvPattern)};
    // AD_DEBUG("update_env\n{}", json);
    load_from_string(json);

} // acmacs::mapi::v1::Settings::update_env

// ----------------------------------------------------------------------

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>


<commit_before>/*
** Copyright 2011 Merethis
** This file is part of Centreon Broker.
**
** Centreon Broker is free software: you can redistribute it and/or
** modify it under the terms of the GNU General Public License version 2
** as published by the Free Software Foundation.
**
** Centreon Broker is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
** General Public License for more details.
**
** You should have received a copy of the GNU General Public License
** along with Centreon Broker. If not, see
** <http://www.gnu.org/licenses/>.
*/

#include <assert.h>
#include <sstream>
#include <stdlib.h>
#include "com/centreon/broker/exceptions/msg.hh"
#include "com/centreon/broker/logging/logging.hh"
#include "com/centreon/broker/rrd/cached.hh"
#include "com/centreon/broker/rrd/exceptions/open.hh"
#include "com/centreon/broker/rrd/exceptions/update.hh"
#include "com/centreon/broker/rrd/lib.hh"
#include "com/centreon/broker/rrd/output.hh"
#include "com/centreon/broker/storage/metric.hh"
#include "com/centreon/broker/storage/status.hh"

using namespace com::centreon::broker;
using namespace com::centreon::broker::rrd;

/**************************************
*                                     *
*           Private Methods           *
*                                     *
**************************************/

/**
 *  @brief Copy constructor.
 *
 *  Any call to this constructor will result in a call to abort().
 *
 *  @param[in] o Object to copy.
 */
output::output(output const& o) : io::stream(o) {
  assert(false);
  abort();
}

/**
 *  @brief Assignment operator.
 *
 *  Any call to this method will result in a call to abort().
 *
 *  @param[in] o Object to copy.
 *
 *  @return This object.
 */
output& output::operator=(output const& o) {
  (void)o;
  assert(false);
  abort();
  return (*this);
}

/**************************************
*                                     *
*           Public Methods            *
*                                     *
**************************************/

/**
 *  Standard constructor.
 *
 *  @param[in] metrics_path Path in which metrics RRD files should be
 *                          written.
 *  @param[in] status_path  Path in which status RRD files should be
 *                          written.
 */
output::output(QString const& metrics_path, QString const& status_path)
  : _backend(new lib),
    _metrics_path(metrics_path),
    _status_path(status_path) {}

/**
 *  Local socket constructor.
 *
 *  @param[in] metrics_path See standard constructor.
 *  @param[in] status_path  See standard constructor.
 *  @param[in] local        Local socket connection parameters.
 */
output::output(QString const& metrics_path,
               QString const& status_path,
               QString const& local)
  : _metrics_path(metrics_path),
    _status_path(status_path) {
  QScopedPointer<cached> rrdcached(new cached);
  rrdcached->connect_local(local);
  _backend.reset(rrdcached.take());
}

/**
 *  Network socket constructor.
 *
 *  @param[in] metrics_path See standard constructor.
 *  @param[in] status_path  See standard constructor.
 *  @param[in] port         rrdcached listening port.
 */
output::output(QString const& metrics_path,
               QString const& status_path,
               unsigned short port)
  : _metrics_path(metrics_path),
    _status_path(status_path) {
  QScopedPointer<cached> rrdcached(new cached);
  rrdcached->connect_remote("localhost", port);
  _backend.reset(rrdcached.take());
}

/**
 *  Destructor.
 */
output::~output() {}

/**
 *  Read data.
 *
 *  @return Does not return, throw an exception.
 */
QSharedPointer<io::data> output::read() {
  throw (broker::exceptions::msg()
           << "RRD: attempt to read data from an output endpoint");
  return (QSharedPointer<io::data>());
}

/**
 *  Write an event.
 *
 *  @param[in] d Data to write.
 */
void output::write(QSharedPointer<io::data> d) {
  if (d->type() == "com::centreon::broker::storage::metric") {
    // Debug message.
    logging::debug << logging::MEDIUM << "RRD: new metric data";
    QSharedPointer<storage::metric> e(d.staticCast<storage::metric>());

    // Write metrics RRD.
    std::ostringstream oss1;
    oss1 << _metrics_path.toStdString() << "/" << e->metric_id << ".rrd";
    try {
      _backend->open(oss1.str().c_str(), e->name);
    }
    catch (exceptions::open const& b) {
      _backend->open(oss1.str().c_str(),
        e->name,
        e->rrd_len / e->interval,
        0,
        e->interval);
    }
    std::ostringstream oss2;
    oss2 << e->value;
    try {
      _backend->update(e->ctime, oss2.str().c_str());
    }
    catch (exceptions::update const& b) {
      logging::error << logging::MEDIUM << b.what() << " (ignored)";
    }
  }
  else if (d->type() == "com::centreon::broker::storage::status") {
    // Debug message.
    logging::debug << logging::MEDIUM << "RRD: new status data";
    QSharedPointer<storage::status> e(d.staticCast<storage::status>());

    // Write status RRD.
    std::ostringstream oss1;
    oss1 << _status_path.toStdString() << "/" << e->index_id << ".rrd";
    try {
      _backend->open(oss1.str().c_str(), "status");
    }
    catch (exceptions::open const& b) {
      _backend->open(oss1.str().c_str(),
        "status",
        e->rrd_len / e->interval,
        0,
        e->interval);
    }
    std::ostringstream oss2;
    if (e->state == 0)
      oss2 << 100;
    else if (e->state == 1)
      oss2 << 75;
    else if (e->state == 2)
      oss2 << 0;
    try {
      _backend->update(e->ctime, oss2.str().c_str());
    }
    catch (exceptions::update const& b) {
      logging::error << logging::MEDIUM << b.what() << " (ignored)";
    }
  }
  return ;
}
<commit_msg>Fix RRD length creation attempt of length 0.<commit_after>/*
** Copyright 2011 Merethis
** This file is part of Centreon Broker.
**
** Centreon Broker is free software: you can redistribute it and/or
** modify it under the terms of the GNU General Public License version 2
** as published by the Free Software Foundation.
**
** Centreon Broker is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
** General Public License for more details.
**
** You should have received a copy of the GNU General Public License
** along with Centreon Broker. If not, see
** <http://www.gnu.org/licenses/>.
*/

#include <assert.h>
#include <sstream>
#include <stdlib.h>
#include "com/centreon/broker/exceptions/msg.hh"
#include "com/centreon/broker/logging/logging.hh"
#include "com/centreon/broker/rrd/cached.hh"
#include "com/centreon/broker/rrd/exceptions/open.hh"
#include "com/centreon/broker/rrd/exceptions/update.hh"
#include "com/centreon/broker/rrd/lib.hh"
#include "com/centreon/broker/rrd/output.hh"
#include "com/centreon/broker/storage/metric.hh"
#include "com/centreon/broker/storage/status.hh"

using namespace com::centreon::broker;
using namespace com::centreon::broker::rrd;

/**************************************
*                                     *
*           Private Methods           *
*                                     *
**************************************/

/**
 *  @brief Copy constructor.
 *
 *  Any call to this constructor will result in a call to abort().
 *
 *  @param[in] o Object to copy.
 */
output::output(output const& o) : io::stream(o) {
  assert(false);
  abort();
}

/**
 *  @brief Assignment operator.
 *
 *  Any call to this method will result in a call to abort().
 *
 *  @param[in] o Object to copy.
 *
 *  @return This object.
 */
output& output::operator=(output const& o) {
  (void)o;
  assert(false);
  abort();
  return (*this);
}

/**************************************
*                                     *
*           Public Methods            *
*                                     *
**************************************/

/**
 *  Standard constructor.
 *
 *  @param[in] metrics_path Path in which metrics RRD files should be
 *                          written.
 *  @param[in] status_path  Path in which status RRD files should be
 *                          written.
 */
output::output(QString const& metrics_path, QString const& status_path)
  : _backend(new lib),
    _metrics_path(metrics_path),
    _status_path(status_path) {}

/**
 *  Local socket constructor.
 *
 *  @param[in] metrics_path See standard constructor.
 *  @param[in] status_path  See standard constructor.
 *  @param[in] local        Local socket connection parameters.
 */
output::output(QString const& metrics_path,
               QString const& status_path,
               QString const& local)
  : _metrics_path(metrics_path),
    _status_path(status_path) {
  QScopedPointer<cached> rrdcached(new cached);
  rrdcached->connect_local(local);
  _backend.reset(rrdcached.take());
}

/**
 *  Network socket constructor.
 *
 *  @param[in] metrics_path See standard constructor.
 *  @param[in] status_path  See standard constructor.
 *  @param[in] port         rrdcached listening port.
 */
output::output(QString const& metrics_path,
               QString const& status_path,
               unsigned short port)
  : _metrics_path(metrics_path),
    _status_path(status_path) {
  QScopedPointer<cached> rrdcached(new cached);
  rrdcached->connect_remote("localhost", port);
  _backend.reset(rrdcached.take());
}

/**
 *  Destructor.
 */
output::~output() {}

/**
 *  Read data.
 *
 *  @return Does not return, throw an exception.
 */
QSharedPointer<io::data> output::read() {
  throw (broker::exceptions::msg()
           << "RRD: attempt to read data from an output endpoint");
  return (QSharedPointer<io::data>());
}

/**
 *  Write an event.
 *
 *  @param[in] d Data to write.
 */
void output::write(QSharedPointer<io::data> d) {
  if (d->type() == "com::centreon::broker::storage::metric") {
    // Debug message.
    logging::debug << logging::MEDIUM << "RRD: new metric data";
    QSharedPointer<storage::metric> e(d.staticCast<storage::metric>());

    // Write metrics RRD.
    std::ostringstream oss1;
    oss1 << _metrics_path.toStdString() << "/" << e->metric_id << ".rrd";
    try {
      _backend->open(oss1.str().c_str(), e->name);
    }
    catch (exceptions::open const& b) {
      _backend->open(oss1.str().c_str(),
        e->name,
        e->rrd_len / e->interval + 1,
        0,
        e->interval);
    }
    std::ostringstream oss2;
    oss2 << e->value;
    try {
      _backend->update(e->ctime, oss2.str().c_str());
    }
    catch (exceptions::update const& b) {
      logging::error << logging::MEDIUM << b.what() << " (ignored)";
    }
  }
  else if (d->type() == "com::centreon::broker::storage::status") {
    // Debug message.
    logging::debug << logging::MEDIUM << "RRD: new status data";
    QSharedPointer<storage::status> e(d.staticCast<storage::status>());

    // Write status RRD.
    std::ostringstream oss1;
    oss1 << _status_path.toStdString() << "/" << e->index_id << ".rrd";
    try {
      _backend->open(oss1.str().c_str(), "status");
    }
    catch (exceptions::open const& b) {
      _backend->open(oss1.str().c_str(),
        "status",
        e->rrd_len / e->interval,
        0,
        e->interval);
    }
    std::ostringstream oss2;
    if (e->state == 0)
      oss2 << 100;
    else if (e->state == 1)
      oss2 << 75;
    else if (e->state == 2)
      oss2 << 0;
    try {
      _backend->update(e->ctime, oss2.str().c_str());
    }
    catch (exceptions::update const& b) {
      logging::error << logging::MEDIUM << b.what() << " (ignored)";
    }
  }
  return ;
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef __STOUT_ATTRIBUTES_HPP__
#define __STOUT_ATTRIBUTES_HPP__

#ifdef __has_cpp_attribute
#  define STOUT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
#else
#  define STOUT_HAS_CPP_ATTRIBUTE(x) 0
#endif

// We unconditionally require compiler support for `noreturn`,
// both for legacy reasons and because the presence of this
// attribute might affect code generation.
#if STOUT_HAS_CPP_ATTRIBUTE(noreturn) > 0
#  define STOUT_NORETURN [[noreturn]]
#elif defined(__WINDOWS__)
#  define STOUT_NORETURN __declspec(noreturn)
#else
#  define STOUT_NORETURN __attribute__((noreturn))
#endif

// Non-namespaced version for backwards compatibility.
#define NORETURN STOUT_NORETURN

#if STOUT_HAS_CPP_ATTRIBUTE(nodiscard) > 0
#  define STOUT_NODISCARD [[nodiscard]]
#else
#  define STOUT_NODISCARD
#endif

#endif // __STOUT_ATTRIBUTES_HPP__
<commit_msg>Added support for deprecated attribute to stout.<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef __STOUT_ATTRIBUTES_HPP__
#define __STOUT_ATTRIBUTES_HPP__

#ifdef __has_cpp_attribute
#  define STOUT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
#else
#  define STOUT_HAS_CPP_ATTRIBUTE(x) 0
#endif

// We unconditionally require compiler support for `noreturn`,
// both for legacy reasons and because the presence of this
// attribute might affect code generation.
#if STOUT_HAS_CPP_ATTRIBUTE(noreturn) > 0
#  define STOUT_NORETURN [[noreturn]]
#elif defined(__WINDOWS__)
#  define STOUT_NORETURN __declspec(noreturn)
#else
#  define STOUT_NORETURN __attribute__((noreturn))
#endif

// Non-namespaced version for backwards compatibility.
#define NORETURN STOUT_NORETURN

#if STOUT_HAS_CPP_ATTRIBUTE(nodiscard) > 0
#  define STOUT_NODISCARD [[nodiscard]]
#else
#  define STOUT_NODISCARD
#endif

#if STOUT_HAS_CPP_ATTRIBUTE(deprecated) > 0
#  define STOUT_DEPRECATED [[deprecated]]
#else
#  define STOUT_DEPRECATED
#endif

#endif // __STOUT_ATTRIBUTES_HPP__
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
#include <vector>
#include <iomanip>
#include <cmath>
#include <string>

using namespace std;

double logsumexp(double x1, double x2)
{
	if(x1 > x2)
		return x1 + log(1. + exp(x2 - x1));
	return x2 + log(1. + exp(x1 - x2));
}

int main()
{
	// Temperatures
	double T1 = 0.3; double T2 = 0.3;

	// Normalising constant for prior weights
	double logC = -1E300;

	// Unnormalised posterior (logw + logl)
	double logP = -1E300;

	// For the information
	double logQ = -1E300;

	bool save = false;

	vector<string> directories;
	directories.push_back(string("10.1"));
	directories.push_back(string("10.2"));
	directories.push_back(string("10.3"));
	directories.push_back(string("10.4"));
	directories.push_back(string("10.5"));
	directories.push_back(string("10.6"));

	fstream fout1, fout2;
	if(save)
	{
		fout1.open("logw.txt", ios::out);
		fout2.open("scalars.txt", ios::out);
	}

	int k = 0;
	for(size_t i=0; i<directories.size(); i++)
	{
		string filename1 = directories[i] + string("/logw.txt");
		fstream fin1(filename1.c_str(), ios::in);

		string filename2 = directories[i] + string("/scalars.txt");
		fstream fin2(filename2.c_str(), ios::in);

		double temp1, temp2, temp3;
		while(fin1>>temp1 && fin2>>temp2 && fin2>>temp3)
		{
			if(save)
			{
				fout1<<temp1<<endl;
				fout2<<temp2<<' '<<temp3<<endl;
			}

			logC = logsumexp(logC, temp1);
			logP = logsumexp(logP, temp1 + temp2/T1 + temp3/T2);
			k++;
		}

		fin1.close();
		fin2.close();

		cout<<"# Processed "<<k<<" points."<<endl;
	}

	if(save)
	{
		fout1.close();
		fout2.close();
	}

	cout<<setprecision(8);
	cout<<"# ln(Z) = "<<(logP - logC)<<endl;

	return 0;
}

<commit_msg>Calculate H too<commit_after>#include <iostream>
#include <fstream>
#include <vector>
#include <iomanip>
#include <cmath>
#include <string>

using namespace std;

double logsumexp(double x1, double x2)
{
	if(x1 > x2)
		return x1 + log(1. + exp(x2 - x1));
	return x2 + log(1. + exp(x1 - x2));
}

int main()
{
	// Temperatures
	double T1 = 0.3; double T2 = 0.3;

	// Normalising constant for prior weights
	double logC = -1E300;

	// Prior weights (relative) and 'likelihoods'
	vector<double> logw, logL;

	bool save = false;

	vector<string> directories;
	directories.push_back(string("10.1"));
	directories.push_back(string("10.2"));
	directories.push_back(string("10.3"));
	directories.push_back(string("10.4"));
	directories.push_back(string("10.5"));
	directories.push_back(string("10.6"));

	fstream fout1, fout2;
	if(save)
	{
		fout1.open("logw.txt", ios::out);
		fout2.open("scalars.txt", ios::out);
	}

	int k = 0;
	for(size_t i=0; i<directories.size(); i++)
	{
		string filename1 = directories[i] + string("/logw.txt");
		fstream fin1(filename1.c_str(), ios::in);

		string filename2 = directories[i] + string("/scalars.txt");
		fstream fin2(filename2.c_str(), ios::in);

		double temp1, temp2, temp3;
		while(fin1>>temp1 && fin2>>temp2 && fin2>>temp3)
		{
			if(save)
			{
				fout1<<temp1<<endl;
				fout2<<temp2<<' '<<temp3<<endl;
			}

			logC = logsumexp(logC, temp1);
			logw.push_back(temp1);
			logL.push_back(temp2/T1 + temp3/T2);

			k++;
		}

		fin1.close();
		fin2.close();

		cout<<"# Loaded "<<k<<" points."<<endl;
	}

	if(save)
	{
		fout1.close();
		fout2.close();
	}

	// Calculate log(Z) and H
	double logZ = -1E300;
	for(size_t i=0; i<logL.size(); i++)
		logZ = logsumexp(logZ, (logw[i] - logC) + logL[i]);

	double H = 0.;
	double logP;
	for(size_t i=0; i<logL.size(); i++)
	{
		// Posterior weight
		logP = logw[i] - logC + logL[i] - logZ;
		H += exp(logP)*(logL[i] - logZ);
	}

	cout<<setprecision(8);
	cout<<"# ln(Z) = "<<logZ<<endl;
	cout<<"# H = "<<H<<endl;

	return 0;
}

<|endoftext|>