<commit_before>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "runtime/mem-tracker.h"

#include <boost/algorithm/string/join.hpp>
#include <google/malloc_extension.h>
#include <gutil/strings/substitute.h>

#include "runtime/exec-env.h"
#include "resourcebroker/resource-broker.h"
#include "statestore/query-resource-mgr.h"
#include "util/debug-util.h"
#include "util/mem-info.h"
#include "util/pretty-printer.h"
#include "util/uid-util.h"

using namespace boost;
using namespace std;
using namespace strings;

namespace impala {

const string MemTracker::COUNTER_NAME = "PeakMemoryUsage";
MemTracker::RequestTrackersMap MemTracker::request_to_mem_trackers_;
MemTracker::PoolTrackersMap MemTracker::pool_to_mem_trackers_;
mutex MemTracker::static_mem_trackers_lock_;

AtomicInt<int64_t> MemTracker::released_memory_since_gc_;

// Name for request pool MemTrackers. '$0' is replaced with the pool name.
const string REQUEST_POOL_MEM_TRACKER_LABEL_FORMAT = "RequestPool=$0";

MemTracker::MemTracker(int64_t byte_limit, int64_t rm_reserved_limit, const string& label,
    MemTracker* parent, bool log_usage_if_zero)
  : limit_(byte_limit),
    rm_reserved_limit_(rm_reserved_limit),
    label_(label),
    parent_(parent),
    consumption_(&local_counter_),
    local_counter_(TUnit::BYTES),
    consumption_metric_(NULL),
    auto_unregister_(false),
    enable_logging_(false),
    log_stack_(false),
    log_usage_if_zero_(log_usage_if_zero),
    query_resource_mgr_(NULL),
    num_gcs_metric_(NULL),
    bytes_freed_by_last_gc_metric_(NULL),
    bytes_over_limit_metric_(NULL) {
  if (parent != NULL) parent_->AddChildTracker(this);
  Init();
}

MemTracker::MemTracker(
    RuntimeProfile* profile, int64_t byte_limit, int64_t rm_reserved_limit,
    const std::string& label, MemTracker* parent)
  : limit_(byte_limit),
    rm_reserved_limit_(rm_reserved_limit),
    label_(label),
    parent_(parent),
    consumption_(profile->AddHighWaterMarkCounter(COUNTER_NAME, TUnit::BYTES)),
    local_counter_(TUnit::BYTES),
    consumption_metric_(NULL),
    auto_unregister_(false),
    enable_logging_(false),
    log_stack_(false),
    log_usage_if_zero_(true),
    query_resource_mgr_(NULL),
    num_gcs_metric_(NULL),
    bytes_freed_by_last_gc_metric_(NULL),
    bytes_over_limit_metric_(NULL) {
  if (parent != NULL) parent_->AddChildTracker(this);
  Init();
}

MemTracker::MemTracker(UIntGauge* consumption_metric,
    int64_t byte_limit, int64_t rm_reserved_limit, const string& label)
  : limit_(byte_limit),
    rm_reserved_limit_(rm_reserved_limit),
    label_(label),
    parent_(NULL),
    consumption_(&local_counter_),
    local_counter_(TUnit::BYTES),
    consumption_metric_(consumption_metric),
    auto_unregister_(false),
    enable_logging_(false),
    log_stack_(false),
    log_usage_if_zero_(true),
    query_resource_mgr_(NULL),
    num_gcs_metric_(NULL),
    bytes_freed_by_last_gc_metric_(NULL),
    bytes_over_limit_metric_(NULL) {
  Init();
}

void MemTracker::Init() {
  DCHECK(rm_reserved_limit_ == -1 || limit_ == -1 || rm_reserved_limit_ <= limit_);
  // populate all_trackers_ and limit_trackers_
  MemTracker* tracker = this;
  while (tracker != NULL) {
    all_trackers_.push_back(tracker);
    if (tracker->has_limit()) limit_trackers_.push_back(tracker);
    tracker = tracker->parent_;
  }
  DCHECK_GT(all_trackers_.size(), 0);
  DCHECK_EQ(all_trackers_[0], this);
}

void MemTracker::AddChildTracker(MemTracker* tracker) {
  lock_guard<mutex> l(child_trackers_lock_);
  tracker->child_tracker_it_ = child_trackers_.insert(child_trackers_.end(), tracker);
}

void MemTracker::UnregisterFromParent() {
  DCHECK(parent_ != NULL);
  lock_guard<mutex> l(parent_->child_trackers_lock_);
  parent_->child_trackers_.erase(child_tracker_it_);
  child_tracker_it_ = parent_->child_trackers_.end();
}

MemTracker* MemTracker::GetRequestPoolMemTracker(const string& pool_name,
    MemTracker* parent) {
  DCHECK(!pool_name.empty());
  lock_guard<mutex> l(static_mem_trackers_lock_);
  PoolTrackersMap::iterator it = pool_to_mem_trackers_.find(pool_name);
  if (it != pool_to_mem_trackers_.end()) {
    MemTracker* tracker = it->second;
    DCHECK(pool_name == tracker->pool_name_);
    return tracker;
  } else {
    if (parent == NULL) return NULL;
    // First time this pool_name registered, make a new object.
    MemTracker* tracker = new MemTracker(-1, -1,
          Substitute(REQUEST_POOL_MEM_TRACKER_LABEL_FORMAT, pool_name),
          parent);
    tracker->auto_unregister_ = true;
    tracker->pool_name_ = pool_name;
    pool_to_mem_trackers_[pool_name] = tracker;
    return tracker;
  }
}

shared_ptr<MemTracker> MemTracker::GetQueryMemTracker(
    const TUniqueId& id, int64_t byte_limit, int64_t rm_reserved_limit, MemTracker* parent,
    QueryResourceMgr* res_mgr) {
  if (byte_limit != -1) {
    if (byte_limit > MemInfo::physical_mem()) {
      LOG(WARNING) << "Memory limit "
                   << PrettyPrinter::Print(byte_limit, TUnit::BYTES)
                   << " exceeds physical memory of "
                   << PrettyPrinter::Print(MemInfo::physical_mem(), TUnit::BYTES);
    }
    VLOG_QUERY << "Using query memory limit: "
               << PrettyPrinter::Print(byte_limit, TUnit::BYTES);
  }

  lock_guard<mutex> l(static_mem_trackers_lock_);
  RequestTrackersMap::iterator it = request_to_mem_trackers_.find(id);
  if (it != request_to_mem_trackers_.end()) {
    // Return the existing MemTracker object for this id, converting the weak ptr
    // to a shared ptr.
    shared_ptr<MemTracker> tracker = it->second.lock();
    DCHECK_EQ(tracker->limit_, byte_limit);
    DCHECK(id == tracker->query_id_);
    DCHECK(parent == tracker->parent_);
    return tracker;
  } else {
    // First time this id registered, make a new object.  Give a shared ptr to
    // the caller and put a weak ptr in the map.
    shared_ptr<MemTracker> tracker(new MemTracker(byte_limit, rm_reserved_limit,
        Substitute("Query($0) Limit", lexical_cast<string>(id)), parent));
    tracker->auto_unregister_ = true;
    tracker->query_id_ = id;
    request_to_mem_trackers_[id] = tracker;
    if (res_mgr != NULL) tracker->SetQueryResourceMgr(res_mgr);
    return tracker;
  }
}

MemTracker::~MemTracker() {
  DCHECK_EQ(consumption(), 0) << "Leaked memory in " << this << endl << LogUsage();
  lock_guard<mutex> l(static_mem_trackers_lock_);
  if (auto_unregister_) UnregisterFromParent();
  // Erase the weak ptr reference from the map.
  request_to_mem_trackers_.erase(query_id_);
  // Per-pool trackers should live the entire lifetime of the impalad process, but
  // remove the element from the map in case this changes in the future.
  pool_to_mem_trackers_.erase(pool_name_);
}

void MemTracker::RegisterMetrics(MetricGroup* metrics, const string& prefix) {
  num_gcs_metric_ = metrics->AddCounter(
      Substitute("$0.num-gcs", prefix), 0L, TUnit::UNIT);

  // TODO: Consider a total amount of bytes freed counter
  bytes_freed_by_last_gc_metric_ = metrics->AddGauge<int64_t>(
      Substitute("$0.bytes-freed-by-last-gc", prefix), -1, TUnit::BYTES);

  bytes_over_limit_metric_ = metrics->AddGauge<int64_t>(
      Substitute("$0.bytes-over-limit", prefix), -1, TUnit::BYTES);
}

// Calling this on the query tracker results in output like:
// Query Limit: memory limit exceeded. Limit=100.00 MB Consumption=106.19 MB
//   Fragment 5b45e83bbc2d92bd:d3ff8a7df7a2f491:  Consumption=52.00 KB
//     AGGREGATION_NODE (id=6):  Consumption=44.00 KB
//     EXCHANGE_NODE (id=5):  Consumption=0.00
//     DataStreamMgr:  Consumption=0.00
//   Fragment 5b45e83bbc2d92bd:d3ff8a7df7a2f492:  Consumption=100.00 KB
//     AGGREGATION_NODE (id=2):  Consumption=36.00 KB
//     AGGREGATION_NODE (id=4):  Consumption=40.00 KB
//     EXCHANGE_NODE (id=3):  Consumption=0.00
//     DataStreamMgr:  Consumption=0.00
//     DataStreamSender:  Consumption=16.00 KB
string MemTracker::LogUsage(const string& prefix) const {
  if (!log_usage_if_zero_ && consumption() == 0) return "";

  stringstream ss;
  ss << prefix << label_ << ":";
  if (CheckLimitExceeded()) ss << " memory limit exceeded.";
  if (limit_ > 0) ss << " Limit=" << PrettyPrinter::Print(limit_, TUnit::BYTES);
  ss << " Consumption=" << PrettyPrinter::Print(consumption(), TUnit::BYTES);

  stringstream prefix_ss;
  prefix_ss << prefix << "  ";
  string new_prefix = prefix_ss.str();
  lock_guard<mutex> l(child_trackers_lock_);
  string child_trackers_usage = LogUsage(new_prefix, child_trackers_);
  if (!child_trackers_usage.empty()) ss << "\n" << child_trackers_usage;
  return ss.str();
}

string MemTracker::LogUsage(const string& prefix, const list<MemTracker*>& trackers) {
  vector<string> usage_strings;
  for (list<MemTracker*>::const_iterator it = trackers.begin();
      it != trackers.end(); ++it) {
    string usage_string = (*it)->LogUsage(prefix);
    if (!usage_string.empty()) usage_strings.push_back(usage_string);
  }
  return join(usage_strings, "\n");
}

void MemTracker::LogUpdate(bool is_consume, int64_t bytes) const {
  stringstream ss;
  ss << this << " " << (is_consume ? "Consume: " : "Release: ") << bytes
     << " Consumption: " << consumption() << " Limit: " << limit_;
  if (log_stack_) ss << endl << GetStackTrace();
  LOG(ERROR) << ss.str();
}

bool MemTracker::GcMemory(int64_t max_consumption) {
  if (max_consumption < 0) return true;
  ScopedSpinLock l(&gc_lock_);
  if (consumption_metric_ != NULL) consumption_->Set(consumption_metric_->value());
  uint64_t pre_gc_consumption = consumption();
  // Check if someone gc'd before us
  if (pre_gc_consumption < max_consumption) return false;
  if (num_gcs_metric_ != NULL) num_gcs_metric_->Increment(1);

  // Try to free up some memory
  for (int i = 0; i < gc_functions_.size(); ++i) {
    gc_functions_[i]();
    if (consumption_metric_ != NULL) consumption_->Set(consumption_metric_->value());
    if (consumption() <= max_consumption) break;
  }

  if (bytes_freed_by_last_gc_metric_ != NULL) {
    bytes_freed_by_last_gc_metric_->set_value(pre_gc_consumption - consumption());
  }
  return consumption() > max_consumption;
}

void MemTracker::GcTcmalloc() {
#ifndef ADDRESS_SANITIZER
  released_memory_since_gc_ = 0;
  MallocExtension::instance()->ReleaseFreeMemory();
#else
  // Nothing to do if not using tcmalloc.
#endif
}

bool MemTracker::ExpandRmReservation(int64_t bytes) {
  if (query_resource_mgr_ == NULL || rm_reserved_limit_ == -1) return false;
  // TODO: Make this asynchronous after IO mgr changes to use TryConsume() are done.
  lock_guard<mutex> l(resource_acquisition_lock_);
  int64_t requested = consumption_->current_value() + bytes;
  // Can't exceed the hard limit under any circumstance
  if (requested >= limit_ && limit_ != -1) return false;
  // Test to see if we can satisfy the limit anyhow; maybe a different request was already
  // in flight.
  if (requested < rm_reserved_limit_) return true;

  TResourceBrokerExpansionRequest exp;
  query_resource_mgr_->CreateExpansionRequest(max(1L, bytes / (1024 * 1024)), 0L, &exp);

  TResourceBrokerExpansionResponse response;
  Status status = ExecEnv::GetInstance()->resource_broker()->Expand(exp, &response);
  if (!status.ok()) {
    LOG(INFO) << "Failed to expand memory limit by "
              << PrettyPrinter::Print(bytes, TUnit::BYTES) << ": "
              << status.GetDetail();
    return false;
  }

  DCHECK(response.allocated_resources.size() == 1) << "Got more resources than expected";
  const llama::TAllocatedResource& resource =
      response.allocated_resources.begin()->second;
  DCHECK(resource.v_cpu_cores == 0L) << "Unexpected VCPUs returned by Llama";

  // Finally, check whether the allocation that we got took us over the limits for any of
  // our ancestors.
  int64_t bytes_allocated = resource.memory_mb * 1024L * 1024L;
  BOOST_FOREACH(const MemTracker* tracker, all_trackers_) {
    if (tracker == this) continue;
    if (tracker->consumption_->current_value() + bytes_allocated > tracker->limit_) {
      // Don't adjust our limit; rely on query tear-down to release the resource.
      return false;
    }
  }

  rm_reserved_limit_ += bytes_allocated;
  // Resource broker might give us more than we ask for
  if (limit_ != -1) rm_reserved_limit_ = min(rm_reserved_limit_, limit_);
  return true;
}

}
<commit_msg>Temporarily removing the DCHECK_EQ(consumption(),0) even from mem-tracker destructor<commit_after>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "runtime/mem-tracker.h"

#include <boost/algorithm/string/join.hpp>
#include <google/malloc_extension.h>
#include <gutil/strings/substitute.h>

#include "runtime/exec-env.h"
#include "resourcebroker/resource-broker.h"
#include "statestore/query-resource-mgr.h"
#include "util/debug-util.h"
#include "util/mem-info.h"
#include "util/pretty-printer.h"
#include "util/uid-util.h"

using namespace boost;
using namespace std;
using namespace strings;

namespace impala {

const string MemTracker::COUNTER_NAME = "PeakMemoryUsage";
MemTracker::RequestTrackersMap MemTracker::request_to_mem_trackers_;
MemTracker::PoolTrackersMap MemTracker::pool_to_mem_trackers_;
mutex MemTracker::static_mem_trackers_lock_;

AtomicInt<int64_t> MemTracker::released_memory_since_gc_;

// Name for request pool MemTrackers. '$0' is replaced with the pool name.
const string REQUEST_POOL_MEM_TRACKER_LABEL_FORMAT = "RequestPool=$0";

MemTracker::MemTracker(int64_t byte_limit, int64_t rm_reserved_limit, const string& label,
    MemTracker* parent, bool log_usage_if_zero)
  : limit_(byte_limit),
    rm_reserved_limit_(rm_reserved_limit),
    label_(label),
    parent_(parent),
    consumption_(&local_counter_),
    local_counter_(TUnit::BYTES),
    consumption_metric_(NULL),
    auto_unregister_(false),
    enable_logging_(false),
    log_stack_(false),
    log_usage_if_zero_(log_usage_if_zero),
    query_resource_mgr_(NULL),
    num_gcs_metric_(NULL),
    bytes_freed_by_last_gc_metric_(NULL),
    bytes_over_limit_metric_(NULL) {
  if (parent != NULL) parent_->AddChildTracker(this);
  Init();
}

MemTracker::MemTracker(
    RuntimeProfile* profile, int64_t byte_limit, int64_t rm_reserved_limit,
    const std::string& label, MemTracker* parent)
  : limit_(byte_limit),
    rm_reserved_limit_(rm_reserved_limit),
    label_(label),
    parent_(parent),
    consumption_(profile->AddHighWaterMarkCounter(COUNTER_NAME, TUnit::BYTES)),
    local_counter_(TUnit::BYTES),
    consumption_metric_(NULL),
    auto_unregister_(false),
    enable_logging_(false),
    log_stack_(false),
    log_usage_if_zero_(true),
    query_resource_mgr_(NULL),
    num_gcs_metric_(NULL),
    bytes_freed_by_last_gc_metric_(NULL),
    bytes_over_limit_metric_(NULL) {
  if (parent != NULL) parent_->AddChildTracker(this);
  Init();
}

MemTracker::MemTracker(UIntGauge* consumption_metric,
    int64_t byte_limit, int64_t rm_reserved_limit, const string& label)
  : limit_(byte_limit),
    rm_reserved_limit_(rm_reserved_limit),
    label_(label),
    parent_(NULL),
    consumption_(&local_counter_),
    local_counter_(TUnit::BYTES),
    consumption_metric_(consumption_metric),
    auto_unregister_(false),
    enable_logging_(false),
    log_stack_(false),
    log_usage_if_zero_(true),
    query_resource_mgr_(NULL),
    num_gcs_metric_(NULL),
    bytes_freed_by_last_gc_metric_(NULL),
    bytes_over_limit_metric_(NULL) {
  Init();
}

void MemTracker::Init() {
  DCHECK(rm_reserved_limit_ == -1 || limit_ == -1 || rm_reserved_limit_ <= limit_);
  // populate all_trackers_ and limit_trackers_
  MemTracker* tracker = this;
  while (tracker != NULL) {
    all_trackers_.push_back(tracker);
    if (tracker->has_limit()) limit_trackers_.push_back(tracker);
    tracker = tracker->parent_;
  }
  DCHECK_GT(all_trackers_.size(), 0);
  DCHECK_EQ(all_trackers_[0], this);
}

void MemTracker::AddChildTracker(MemTracker* tracker) {
  lock_guard<mutex> l(child_trackers_lock_);
  tracker->child_tracker_it_ = child_trackers_.insert(child_trackers_.end(), tracker);
}

void MemTracker::UnregisterFromParent() {
  DCHECK(parent_ != NULL);
  lock_guard<mutex> l(parent_->child_trackers_lock_);
  parent_->child_trackers_.erase(child_tracker_it_);
  child_tracker_it_ = parent_->child_trackers_.end();
}

MemTracker* MemTracker::GetRequestPoolMemTracker(const string& pool_name,
    MemTracker* parent) {
  DCHECK(!pool_name.empty());
  lock_guard<mutex> l(static_mem_trackers_lock_);
  PoolTrackersMap::iterator it = pool_to_mem_trackers_.find(pool_name);
  if (it != pool_to_mem_trackers_.end()) {
    MemTracker* tracker = it->second;
    DCHECK(pool_name == tracker->pool_name_);
    return tracker;
  } else {
    if (parent == NULL) return NULL;
    // First time this pool_name registered, make a new object.
    MemTracker* tracker = new MemTracker(-1, -1,
          Substitute(REQUEST_POOL_MEM_TRACKER_LABEL_FORMAT, pool_name),
          parent);
    tracker->auto_unregister_ = true;
    tracker->pool_name_ = pool_name;
    pool_to_mem_trackers_[pool_name] = tracker;
    return tracker;
  }
}

shared_ptr<MemTracker> MemTracker::GetQueryMemTracker(
    const TUniqueId& id, int64_t byte_limit, int64_t rm_reserved_limit, MemTracker* parent,
    QueryResourceMgr* res_mgr) {
  if (byte_limit != -1) {
    if (byte_limit > MemInfo::physical_mem()) {
      LOG(WARNING) << "Memory limit "
                   << PrettyPrinter::Print(byte_limit, TUnit::BYTES)
                   << " exceeds physical memory of "
                   << PrettyPrinter::Print(MemInfo::physical_mem(), TUnit::BYTES);
    }
    VLOG_QUERY << "Using query memory limit: "
               << PrettyPrinter::Print(byte_limit, TUnit::BYTES);
  }

  lock_guard<mutex> l(static_mem_trackers_lock_);
  RequestTrackersMap::iterator it = request_to_mem_trackers_.find(id);
  if (it != request_to_mem_trackers_.end()) {
    // Return the existing MemTracker object for this id, converting the weak ptr
    // to a shared ptr.
    shared_ptr<MemTracker> tracker = it->second.lock();
    DCHECK_EQ(tracker->limit_, byte_limit);
    DCHECK(id == tracker->query_id_);
    DCHECK(parent == tracker->parent_);
    return tracker;
  } else {
    // First time this id registered, make a new object.  Give a shared ptr to
    // the caller and put a weak ptr in the map.
    shared_ptr<MemTracker> tracker(new MemTracker(byte_limit, rm_reserved_limit,
        Substitute("Query($0) Limit", lexical_cast<string>(id)), parent));
    tracker->auto_unregister_ = true;
    tracker->query_id_ = id;
    request_to_mem_trackers_[id] = tracker;
    if (res_mgr != NULL) tracker->SetQueryResourceMgr(res_mgr);
    return tracker;
  }
}

MemTracker::~MemTracker() {
  lock_guard<mutex> l(static_mem_trackers_lock_);
  if (auto_unregister_) UnregisterFromParent();
  // Erase the weak ptr reference from the map.
  request_to_mem_trackers_.erase(query_id_);
  // Per-pool trackers should live the entire lifetime of the impalad process, but
  // remove the element from the map in case this changes in the future.
  pool_to_mem_trackers_.erase(pool_name_);
}

void MemTracker::RegisterMetrics(MetricGroup* metrics, const string& prefix) {
  num_gcs_metric_ = metrics->AddCounter(
      Substitute("$0.num-gcs", prefix), 0L, TUnit::UNIT);

  // TODO: Consider a total amount of bytes freed counter
  bytes_freed_by_last_gc_metric_ = metrics->AddGauge<int64_t>(
      Substitute("$0.bytes-freed-by-last-gc", prefix), -1, TUnit::BYTES);

  bytes_over_limit_metric_ = metrics->AddGauge<int64_t>(
      Substitute("$0.bytes-over-limit", prefix), -1, TUnit::BYTES);
}

// Calling this on the query tracker results in output like:
// Query Limit: memory limit exceeded. Limit=100.00 MB Consumption=106.19 MB
//   Fragment 5b45e83bbc2d92bd:d3ff8a7df7a2f491:  Consumption=52.00 KB
//     AGGREGATION_NODE (id=6):  Consumption=44.00 KB
//     EXCHANGE_NODE (id=5):  Consumption=0.00
//     DataStreamMgr:  Consumption=0.00
//   Fragment 5b45e83bbc2d92bd:d3ff8a7df7a2f492:  Consumption=100.00 KB
//     AGGREGATION_NODE (id=2):  Consumption=36.00 KB
//     AGGREGATION_NODE (id=4):  Consumption=40.00 KB
//     EXCHANGE_NODE (id=3):  Consumption=0.00
//     DataStreamMgr:  Consumption=0.00
//     DataStreamSender:  Consumption=16.00 KB
string MemTracker::LogUsage(const string& prefix) const {
  if (!log_usage_if_zero_ && consumption() == 0) return "";

  stringstream ss;
  ss << prefix << label_ << ":";
  if (CheckLimitExceeded()) ss << " memory limit exceeded.";
  if (limit_ > 0) ss << " Limit=" << PrettyPrinter::Print(limit_, TUnit::BYTES);
  ss << " Consumption=" << PrettyPrinter::Print(consumption(), TUnit::BYTES);

  stringstream prefix_ss;
  prefix_ss << prefix << "  ";
  string new_prefix = prefix_ss.str();
  lock_guard<mutex> l(child_trackers_lock_);
  string child_trackers_usage = LogUsage(new_prefix, child_trackers_);
  if (!child_trackers_usage.empty()) ss << "\n" << child_trackers_usage;
  return ss.str();
}

string MemTracker::LogUsage(const string& prefix, const list<MemTracker*>& trackers) {
  vector<string> usage_strings;
  for (list<MemTracker*>::const_iterator it = trackers.begin();
      it != trackers.end(); ++it) {
    string usage_string = (*it)->LogUsage(prefix);
    if (!usage_string.empty()) usage_strings.push_back(usage_string);
  }
  return join(usage_strings, "\n");
}

void MemTracker::LogUpdate(bool is_consume, int64_t bytes) const {
  stringstream ss;
  ss << this << " " << (is_consume ? "Consume: " : "Release: ") << bytes
     << " Consumption: " << consumption() << " Limit: " << limit_;
  if (log_stack_) ss << endl << GetStackTrace();
  LOG(ERROR) << ss.str();
}

bool MemTracker::GcMemory(int64_t max_consumption) {
  if (max_consumption < 0) return true;
  ScopedSpinLock l(&gc_lock_);
  if (consumption_metric_ != NULL) consumption_->Set(consumption_metric_->value());
  uint64_t pre_gc_consumption = consumption();
  // Check if someone gc'd before us
  if (pre_gc_consumption < max_consumption) return false;
  if (num_gcs_metric_ != NULL) num_gcs_metric_->Increment(1);

  // Try to free up some memory
  for (int i = 0; i < gc_functions_.size(); ++i) {
    gc_functions_[i]();
    if (consumption_metric_ != NULL) consumption_->Set(consumption_metric_->value());
    if (consumption() <= max_consumption) break;
  }

  if (bytes_freed_by_last_gc_metric_ != NULL) {
    bytes_freed_by_last_gc_metric_->set_value(pre_gc_consumption - consumption());
  }
  return consumption() > max_consumption;
}

void MemTracker::GcTcmalloc() {
#ifndef ADDRESS_SANITIZER
  released_memory_since_gc_ = 0;
  MallocExtension::instance()->ReleaseFreeMemory();
#else
  // Nothing to do if not using tcmalloc.
#endif
}

bool MemTracker::ExpandRmReservation(int64_t bytes) {
  if (query_resource_mgr_ == NULL || rm_reserved_limit_ == -1) return false;
  // TODO: Make this asynchronous after IO mgr changes to use TryConsume() are done.
  lock_guard<mutex> l(resource_acquisition_lock_);
  int64_t requested = consumption_->current_value() + bytes;
  // Can't exceed the hard limit under any circumstance
  if (requested >= limit_ && limit_ != -1) return false;
  // Test to see if we can satisfy the limit anyhow; maybe a different request was already
  // in flight.
  if (requested < rm_reserved_limit_) return true;

  TResourceBrokerExpansionRequest exp;
  query_resource_mgr_->CreateExpansionRequest(max(1L, bytes / (1024 * 1024)), 0L, &exp);

  TResourceBrokerExpansionResponse response;
  Status status = ExecEnv::GetInstance()->resource_broker()->Expand(exp, &response);
  if (!status.ok()) {
    LOG(INFO) << "Failed to expand memory limit by "
              << PrettyPrinter::Print(bytes, TUnit::BYTES) << ": "
              << status.GetDetail();
    return false;
  }

  DCHECK(response.allocated_resources.size() == 1) << "Got more resources than expected";
  const llama::TAllocatedResource& resource =
      response.allocated_resources.begin()->second;
  DCHECK(resource.v_cpu_cores == 0L) << "Unexpected VCPUs returned by Llama";

  // Finally, check whether the allocation that we got took us over the limits for any of
  // our ancestors.
  int64_t bytes_allocated = resource.memory_mb * 1024L * 1024L;
  BOOST_FOREACH(const MemTracker* tracker, all_trackers_) {
    if (tracker == this) continue;
    if (tracker->consumption_->current_value() + bytes_allocated > tracker->limit_) {
      // Don't adjust our limit; rely on query tear-down to release the resource.
      return false;
    }
  }

  rm_reserved_limit_ += bytes_allocated;
  // Resource broker might give us more than we ask for
  if (limit_ != -1) rm_reserved_limit_ = min(rm_reserved_limit_, limit_);
  return true;
}

}
<|endoftext|>


<commit_before>/***************************************************************************
  This is a library for the BNO055 orientation sensor

  Designed specifically to work with the Adafruit BNO055 Breakout.

  Pick one up today in the adafruit shop!
  ------> http://www.adafruit.com/products

  These sensors use I2C to communicate, 2 pins are required to interface.

  Adafruit invests time and resources providing this open source code,
  please support Adafruit andopen-source hardware by purchasing products
  from Adafruit!

  Written by KTOWN for Adafruit Industries.

  MIT license, all text above must be included in any redistribution
 ***************************************************************************/

#if ARDUINO >= 100
 #include "Arduino.h"
#else
 #include "WProgram.h"
#endif

#include <math.h>
#include <limits.h>

#include "Adafruit_BNO055.h"

/***************************************************************************
 CONSTRUCTOR
 ***************************************************************************/

/**************************************************************************/
/*!
    @brief  Instantiates a new Adafruit_BNO055 class
*/
/**************************************************************************/
Adafruit_BNO055::Adafruit_BNO055(int32_t sensorID, uint8_t address)
{
  _sensorID = sensorID;
  _address = address;
}

/***************************************************************************
 PUBLIC FUNCTIONS
 ***************************************************************************/

/**************************************************************************/
/*!
    @brief  Sets up the HW
*/
/**************************************************************************/
bool Adafruit_BNO055::begin(adafruit_bno055_opmode_t mode)
{
  /* Enable I2C */
  Wire.begin();

  /* Make sure we have the right device */
  uint8_t id = read8(BNO055_CHIP_ID_ADDR);
  if(id != BNO055_ID)
  {
    delay(1000); // hold on for boot
    if(id != BNO055_ID) {
      return false;  // still not? ok bail
    }
  }

  /* Switch to config mode (just in case since this is the default) */
  setMode(OPERATION_MODE_CONFIG);

  /* Reset */
  write8(BNO055_SYS_TRIGGER_ADDR, 0x20);
  while (read8(BNO055_CHIP_ID_ADDR) != BNO055_ID)
  {
    delay(10);
  }
  delay(50);

  /* Set to normal power mode */
  write8(BNO055_PWR_MODE_ADDR, POWER_MODE_NORMAL);
  delay(10);

  write8(BNO055_PAGE_ID_ADDR, 0);

  /* Set the output units */
  /*
  uint8_t unitsel = (0 << 7) | // Orientation = Android
                    (0 << 4) | // Temperature = Celsius
                    (0 << 2) | // Euler = Degrees
                    (1 << 1) | // Gyro = Rads
                    (0 << 0);  // Accelerometer = m/s^2
  write8(BNO055_UNIT_SEL_ADDR, unitsel);
  */

  write8(BNO055_SYS_TRIGGER_ADDR, 0x0);
  delay(10);
  /* Set the requested operating mode (see section 3.3) */
  setMode(mode);
  delay(20);

  return true;
}

/**************************************************************************/
/*!
    @brief  Puts the chip in the specified operating mode
*/
/**************************************************************************/
void Adafruit_BNO055::setMode(adafruit_bno055_opmode_t mode)
{
  _mode = mode;
  write8(BNO055_OPR_MODE_ADDR, _mode);
  delay(30);
}

/**************************************************************************/
/*!
    @brief  Use the external 32.768KHz crystal
*/
/**************************************************************************/
void Adafruit_BNO055::setExtCrystalUse(boolean usextal)
{
  adafruit_bno055_opmode_t modeback = _mode;

  /* Switch to config mode (just in case since this is the default) */
  setMode(OPERATION_MODE_CONFIG);
  delay(25);
  write8(BNO055_PAGE_ID_ADDR, 0);
  if (usextal) {
    write8(BNO055_SYS_TRIGGER_ADDR, 0x80);
  } else {
    write8(BNO055_SYS_TRIGGER_ADDR, 0x00);
  }
  delay(10);
  /* Set the requested operating mode (see section 3.3) */
  setMode(modeback);
  delay(20);
}

/**************************************************************************/
/*!
    @brief  Gets the latest system status info
*/
/**************************************************************************/
void Adafruit_BNO055::getSystemStatus(uint8_t *system_status, uint8_t *self_test_result, uint8_t *system_error)
{
  adafruit_bno055_opmode_t backupmode = _mode;

  setMode(OPERATION_MODE_CONFIG);
  delay(20);
  write8(BNO055_PAGE_ID_ADDR, 0);

  write8(BNO055_SYS_TRIGGER_ADDR, read8(BNO055_SYS_TRIGGER_ADDR) | 0x1);
  delay(1000);

  /* System Status (see section 4.3.58)
     ---------------------------------
     0 = Idle
     1 = System Error
     2 = Initializing Peripherals
     3 = System Iniitalization
     4 = Executing Self-Test
     5 = Sensor fusio algorithm running
     6 = System running without fusion algorithms */

  if (system_status != 0)
    *system_status    = read8(BNO055_SYS_STAT_ADDR);

  /* Self Test Results (see section )
     --------------------------------
     1 = test passed, 0 = test failed

     Bit 0 = Accelerometer self test
     Bit 1 = Magnetometer self test
     Bit 2 = Gyroscope self test
     Bit 3 = MCU self test

     0x0F = all good! */

  if (self_test_result != 0)
    *self_test_result = read8(BNO055_SELFTEST_RESULT_ADDR);

  /* System Error (see section 4.3.59)
     ---------------------------------
     0 = No error
     1 = Peripheral initialization error
     2 = System initialization error
     3 = Self test result failed
     4 = Register map value out of range
     5 = Register map address out of range
     6 = Register map write error
     7 = BNO low power mode not available for selected operat ion mode
     8 = Accelerometer power mode not available
     9 = Fusion algorithm configuration error
     A = Sensor configuration error */

  if (system_error != 0)
    *system_error     = read8(BNO055_SYS_ERR_ADDR);

  setMode(backupmode);
  delay(20);
}

/**************************************************************************/
/*!
    @brief  Gets the chip revision numbers
*/
/**************************************************************************/
void Adafruit_BNO055::getRevInfo(adafruit_bno055_rev_info_t* info)
{
  uint8_t a, b;

  memset(info, 0, sizeof(adafruit_bno055_rev_info_t));

  /* Check the accelerometer revision */
  info->accel_rev = read8(BNO055_ACCEL_REV_ID_ADDR);

  /* Check the magnetometer revision */
  info->mag_rev   = read8(BNO055_MAG_REV_ID_ADDR);

  /* Check the gyroscope revision */
  info->gyro_rev  = read8(BNO055_GYRO_REV_ID_ADDR);

  /* Check the SW revision */
  info->bl_rev    = read8(BNO055_BL_REV_ID_ADDR);

  a = read8(BNO055_SW_REV_ID_LSB_ADDR);
  b = read8(BNO055_SW_REV_ID_MSB_ADDR);
  info->sw_rev = (((uint16_t)b) << 8) | ((uint16_t)a);
}

/**************************************************************************/
/*!
    @brief  Gets teh temperature in degrees celsius
*/
/**************************************************************************/
int8_t Adafruit_BNO055::getTemp(void)
{
  int8_t temp = (int8_t)(read8(BNO055_TEMP_ADDR));
  return temp;
}

/**************************************************************************/
/*!
    @brief  Gets a vector reading from the specified source
*/
/**************************************************************************/
imu::Vector<3> Adafruit_BNO055::getVector(adafruit_vector_type_t vector_type)
{
  imu::Vector<3> xyz;
  uint8_t buffer[6];
  memset (buffer, 0, 6);

  int16_t x, y, z;
  x = y = z = 0;

  /* Read vector data (6 bytes) */
  readLen((adafruit_bno055_reg_t)vector_type, buffer, 6);

  x = ((int16_t)buffer[0]) | (((int16_t)buffer[1]) << 8);
  y = ((int16_t)buffer[2]) | (((int16_t)buffer[3]) << 8);
  z = ((int16_t)buffer[4]) | (((int16_t)buffer[5]) << 8);

  /* Convert the value to an appropriate range (section 3.6.4) */
  /* and assign the value to the Vector type */
  switch(vector_type)
  {
    case VECTOR_MAGNETOMETER:
      /* 1uT = 16 LSB */
      xyz[0] = ((double)x)/16.0;
      xyz[1] = ((double)y)/16.0;
      xyz[2] = ((double)z)/16.0;
      break;
    case VECTOR_GYROSCOPE:
      /* 1rps = 900 LSB */
      xyz[0] = ((double)x)/900.0;
      xyz[1] = ((double)y)/900.0;
      xyz[2] = ((double)z)/900.0;
      break;
    case VECTOR_EULER:
      /* 1 degree = 16 LSB */
      xyz[0] = ((double)x)/16.0;
      xyz[1] = ((double)y)/16.0;
      xyz[2] = ((double)z)/16.0;
      break;
    case VECTOR_ACCELEROMETER:
    case VECTOR_LINEARACCEL:
    case VECTOR_GRAVITY:
      /* 1m/s^2 = 100 LSB */
      xyz[0] = ((double)x)/100.0;
      xyz[1] = ((double)y)/100.0;
      xyz[2] = ((double)z)/100.0;
      break;
  }

  return xyz;
}

/**************************************************************************/
/*!
    @brief  Gets a quaternion reading from the specified source
*/
/**************************************************************************/
imu::Quaternion Adafruit_BNO055::getQuat(void)
{
  uint8_t buffer[8];
  memset (buffer, 0, 8);

  int16_t x, y, z, w;
  x = y = z = w = 0;

  /* Read quat data (8 bytes) */
  readLen(BNO055_QUATERNION_DATA_W_LSB_ADDR, buffer, 8);
  w = (((uint16_t)buffer[1]) << 8) | ((uint16_t)buffer[0]);
  x = (((uint16_t)buffer[3]) << 8) | ((uint16_t)buffer[2]);
  y = (((uint16_t)buffer[5]) << 8) | ((uint16_t)buffer[4]);
  z = (((uint16_t)buffer[7]) << 8) | ((uint16_t)buffer[6]);

  /* Assign to Quaternion */
  imu::Quaternion quat((double)w, (double)x, (double)y, (double)z);
  return quat;
}

/**************************************************************************/
/*!
    @brief  Provides the sensor_t data for this sensor
*/
/**************************************************************************/
void Adafruit_BNO055::getSensor(sensor_t *sensor)
{
  /* Clear the sensor_t object */
  memset(sensor, 0, sizeof(sensor_t));

  /* Insert the sensor name in the fixed length char array */
  strncpy (sensor->name, "BNO055", sizeof(sensor->name) - 1);
  sensor->name[sizeof(sensor->name)- 1] = 0;
  sensor->version     = 1;
  sensor->sensor_id   = _sensorID;
  sensor->type        = SENSOR_TYPE_ORIENTATION;
  sensor->min_delay   = 0;
  sensor->max_value   = 0.0F;
  sensor->min_value   = 0.0F;
  sensor->resolution  = 0.01F;
}

/**************************************************************************/
/*!
    @brief  Reads the sensor and returns the data as a sensors_event_t
*/
/**************************************************************************/
bool Adafruit_BNO055::getEvent(sensors_event_t *event)
{
  /* Clear the event */
  memset(event, 0, sizeof(sensors_event_t));

  event->version   = sizeof(sensors_event_t);
  event->sensor_id = _sensorID;
  event->type      = SENSOR_TYPE_ORIENTATION;
  event->timestamp = millis();

  /* Get a Euler angle sample for orientation */
  imu::Vector<3> euler = getVector(Adafruit_BNO055::VECTOR_EULER);
  event->orientation.x = euler.x();
  event->orientation.y = euler.y();
  event->orientation.z = euler.z();

  return true;
}

/***************************************************************************
 PRIVATE FUNCTIONS
 ***************************************************************************/

/**************************************************************************/
/*!
    @brief  Writes an 8 bit value over I2C
*/
/**************************************************************************/
bool Adafruit_BNO055::write8(adafruit_bno055_reg_t reg, byte value)
{
  Wire.beginTransmission(_address);
  #if ARDUINO >= 100
    Wire.write((uint8_t)reg);
    Wire.write((uint8_t)value);
  #else
    Wire.send(reg);
    Wire.send(value);
  #endif
  Wire.endTransmission();

  /* ToDo: Check for error! */
  return true;
}

/**************************************************************************/
/*!
    @brief  Reads an 8 bit value over I2C
*/
/**************************************************************************/
byte Adafruit_BNO055::read8(adafruit_bno055_reg_t reg )
{
  byte value = 0;

  Wire.beginTransmission(_address);
  #if ARDUINO >= 100
    Wire.write((uint8_t)reg);
  #else
    Wire.send(reg);
  #endif
  Wire.endTransmission();
  Wire.requestFrom(_address, (byte)1);
  #if ARDUINO >= 100
    value = Wire.read();
  #else
    value = Wire.receive();
  #endif

  return value;
}

/**************************************************************************/
/*!
    @brief  Reads the specified number of bytes over I2C
*/
/**************************************************************************/
bool Adafruit_BNO055::readLen(adafruit_bno055_reg_t reg, byte * buffer, uint8_t len)
{
  Wire.beginTransmission(_address);
  #if ARDUINO >= 100
    Wire.write((uint8_t)reg);
  #else
    Wire.send(reg);
  #endif
  Wire.endTransmission();
  Wire.requestFrom(_address, (byte)len);

  /* Wait until data is available */
  while (Wire.available() < len);

  for (uint8_t i = 0; i < len; i++)
  {
    #if ARDUINO >= 100
      buffer[i] = Wire.read();
    #else
      buffer[i] = Wire.receive();
    #endif
  }

  /* ToDo: Check for errors! */
  return true;
}
<commit_msg>Bug fixes for getQuat() and begin()<commit_after>/***************************************************************************
  This is a library for the BNO055 orientation sensor

  Designed specifically to work with the Adafruit BNO055 Breakout.

  Pick one up today in the adafruit shop!
  ------> http://www.adafruit.com/products

  These sensors use I2C to communicate, 2 pins are required to interface.

  Adafruit invests time and resources providing this open source code,
  please support Adafruit andopen-source hardware by purchasing products
  from Adafruit!

  Written by KTOWN for Adafruit Industries.

  MIT license, all text above must be included in any redistribution
 ***************************************************************************/

#if ARDUINO >= 100
 #include "Arduino.h"
#else
 #include "WProgram.h"
#endif

#include <math.h>
#include <limits.h>

#include "Adafruit_BNO055.h"

/***************************************************************************
 CONSTRUCTOR
 ***************************************************************************/

/**************************************************************************/
/*!
    @brief  Instantiates a new Adafruit_BNO055 class
*/
/**************************************************************************/
Adafruit_BNO055::Adafruit_BNO055(int32_t sensorID, uint8_t address)
{
  _sensorID = sensorID;
  _address = address;
}

/***************************************************************************
 PUBLIC FUNCTIONS
 ***************************************************************************/

/**************************************************************************/
/*!
    @brief  Sets up the HW
*/
/**************************************************************************/
bool Adafruit_BNO055::begin(adafruit_bno055_opmode_t mode)
{
  /* Enable I2C */
  Wire.begin();

  /* Make sure we have the right device */
  uint8_t id = read8(BNO055_CHIP_ID_ADDR);
  if(id != BNO055_ID)
  {
    delay(1000); // hold on for boot
    id = read8(BNO055_CHIP_ID_ADDR);
    if(id != BNO055_ID) {
      return false;  // still not? ok bail
    }
  }

  /* Switch to config mode (just in case since this is the default) */
  setMode(OPERATION_MODE_CONFIG);

  /* Reset */
  write8(BNO055_SYS_TRIGGER_ADDR, 0x20);
  while (read8(BNO055_CHIP_ID_ADDR) != BNO055_ID)
  {
    delay(10);
  }
  delay(50);

  /* Set to normal power mode */
  write8(BNO055_PWR_MODE_ADDR, POWER_MODE_NORMAL);
  delay(10);

  write8(BNO055_PAGE_ID_ADDR, 0);

  /* Set the output units */
  /*
  uint8_t unitsel = (0 << 7) | // Orientation = Android
                    (0 << 4) | // Temperature = Celsius
                    (0 << 2) | // Euler = Degrees
                    (1 << 1) | // Gyro = Rads
                    (0 << 0);  // Accelerometer = m/s^2
  write8(BNO055_UNIT_SEL_ADDR, unitsel);
  */

  write8(BNO055_SYS_TRIGGER_ADDR, 0x0);
  delay(10);
  /* Set the requested operating mode (see section 3.3) */
  setMode(mode);
  delay(20);

  return true;
}

/**************************************************************************/
/*!
    @brief  Puts the chip in the specified operating mode
*/
/**************************************************************************/
void Adafruit_BNO055::setMode(adafruit_bno055_opmode_t mode)
{
  _mode = mode;
  write8(BNO055_OPR_MODE_ADDR, _mode);
  delay(30);
}

/**************************************************************************/
/*!
    @brief  Use the external 32.768KHz crystal
*/
/**************************************************************************/
void Adafruit_BNO055::setExtCrystalUse(boolean usextal)
{
  adafruit_bno055_opmode_t modeback = _mode;

  /* Switch to config mode (just in case since this is the default) */
  setMode(OPERATION_MODE_CONFIG);
  delay(25);
  write8(BNO055_PAGE_ID_ADDR, 0);
  if (usextal) {
    write8(BNO055_SYS_TRIGGER_ADDR, 0x80);
  } else {
    write8(BNO055_SYS_TRIGGER_ADDR, 0x00);
  }
  delay(10);
  /* Set the requested operating mode (see section 3.3) */
  setMode(modeback);
  delay(20);
}

/**************************************************************************/
/*!
    @brief  Gets the latest system status info
*/
/**************************************************************************/
void Adafruit_BNO055::getSystemStatus(uint8_t *system_status, uint8_t *self_test_result, uint8_t *system_error)
{
  adafruit_bno055_opmode_t backupmode = _mode;

  setMode(OPERATION_MODE_CONFIG);
  delay(20);
  write8(BNO055_PAGE_ID_ADDR, 0);

  write8(BNO055_SYS_TRIGGER_ADDR, read8(BNO055_SYS_TRIGGER_ADDR) | 0x1);
  delay(1000);

  /* System Status (see section 4.3.58)
     ---------------------------------
     0 = Idle
     1 = System Error
     2 = Initializing Peripherals
     3 = System Iniitalization
     4 = Executing Self-Test
     5 = Sensor fusio algorithm running
     6 = System running without fusion algorithms */

  if (system_status != 0)
    *system_status    = read8(BNO055_SYS_STAT_ADDR);

  /* Self Test Results (see section )
     --------------------------------
     1 = test passed, 0 = test failed

     Bit 0 = Accelerometer self test
     Bit 1 = Magnetometer self test
     Bit 2 = Gyroscope self test
     Bit 3 = MCU self test

     0x0F = all good! */

  if (self_test_result != 0)
    *self_test_result = read8(BNO055_SELFTEST_RESULT_ADDR);

  /* System Error (see section 4.3.59)
     ---------------------------------
     0 = No error
     1 = Peripheral initialization error
     2 = System initialization error
     3 = Self test result failed
     4 = Register map value out of range
     5 = Register map address out of range
     6 = Register map write error
     7 = BNO low power mode not available for selected operat ion mode
     8 = Accelerometer power mode not available
     9 = Fusion algorithm configuration error
     A = Sensor configuration error */

  if (system_error != 0)
    *system_error     = read8(BNO055_SYS_ERR_ADDR);

  setMode(backupmode);
  delay(20);
}

/**************************************************************************/
/*!
    @brief  Gets the chip revision numbers
*/
/**************************************************************************/
void Adafruit_BNO055::getRevInfo(adafruit_bno055_rev_info_t* info)
{
  uint8_t a, b;

  memset(info, 0, sizeof(adafruit_bno055_rev_info_t));

  /* Check the accelerometer revision */
  info->accel_rev = read8(BNO055_ACCEL_REV_ID_ADDR);

  /* Check the magnetometer revision */
  info->mag_rev   = read8(BNO055_MAG_REV_ID_ADDR);

  /* Check the gyroscope revision */
  info->gyro_rev  = read8(BNO055_GYRO_REV_ID_ADDR);

  /* Check the SW revision */
  info->bl_rev    = read8(BNO055_BL_REV_ID_ADDR);

  a = read8(BNO055_SW_REV_ID_LSB_ADDR);
  b = read8(BNO055_SW_REV_ID_MSB_ADDR);
  info->sw_rev = (((uint16_t)b) << 8) | ((uint16_t)a);
}

/**************************************************************************/
/*!
    @brief  Gets teh temperature in degrees celsius
*/
/**************************************************************************/
int8_t Adafruit_BNO055::getTemp(void)
{
  int8_t temp = (int8_t)(read8(BNO055_TEMP_ADDR));
  return temp;
}

/**************************************************************************/
/*!
    @brief  Gets a vector reading from the specified source
*/
/**************************************************************************/
imu::Vector<3> Adafruit_BNO055::getVector(adafruit_vector_type_t vector_type)
{
  imu::Vector<3> xyz;
  uint8_t buffer[6];
  memset (buffer, 0, 6);

  int16_t x, y, z;
  x = y = z = 0;

  /* Read vector data (6 bytes) */
  readLen((adafruit_bno055_reg_t)vector_type, buffer, 6);

  x = ((int16_t)buffer[0]) | (((int16_t)buffer[1]) << 8);
  y = ((int16_t)buffer[2]) | (((int16_t)buffer[3]) << 8);
  z = ((int16_t)buffer[4]) | (((int16_t)buffer[5]) << 8);

  /* Convert the value to an appropriate range (section 3.6.4) */
  /* and assign the value to the Vector type */
  switch(vector_type)
  {
    case VECTOR_MAGNETOMETER:
      /* 1uT = 16 LSB */
      xyz[0] = ((double)x)/16.0;
      xyz[1] = ((double)y)/16.0;
      xyz[2] = ((double)z)/16.0;
      break;
    case VECTOR_GYROSCOPE:
      /* 1rps = 900 LSB */
      xyz[0] = ((double)x)/900.0;
      xyz[1] = ((double)y)/900.0;
      xyz[2] = ((double)z)/900.0;
      break;
    case VECTOR_EULER:
      /* 1 degree = 16 LSB */
      xyz[0] = ((double)x)/16.0;
      xyz[1] = ((double)y)/16.0;
      xyz[2] = ((double)z)/16.0;
      break;
    case VECTOR_ACCELEROMETER:
    case VECTOR_LINEARACCEL:
    case VECTOR_GRAVITY:
      /* 1m/s^2 = 100 LSB */
      xyz[0] = ((double)x)/100.0;
      xyz[1] = ((double)y)/100.0;
      xyz[2] = ((double)z)/100.0;
      break;
  }

  return xyz;
}

/**************************************************************************/
/*!
    @brief  Gets a quaternion reading from the specified source
*/
/**************************************************************************/
imu::Quaternion Adafruit_BNO055::getQuat(void)
{
  uint8_t buffer[8];
  memset (buffer, 0, 8);

  int16_t x, y, z, w;
  x = y = z = w = 0;

  /* Read quat data (8 bytes) */
  readLen(BNO055_QUATERNION_DATA_W_LSB_ADDR, buffer, 8);
  w = (((uint16_t)buffer[1]) << 8) | ((uint16_t)buffer[0]);
  x = (((uint16_t)buffer[3]) << 8) | ((uint16_t)buffer[2]);
  y = (((uint16_t)buffer[5]) << 8) | ((uint16_t)buffer[4]);
  z = (((uint16_t)buffer[7]) << 8) | ((uint16_t)buffer[6]);

  /* Assign to Quaternion */
  /* See http://ae-bst.resource.bosch.com/media/products/dokumente/bno055/BST_BNO055_DS000_12~1.pdf
     3.6.5.5 Orientation (Quaternion)  */
  const double scale = (1.0 / (1<<14));
  imu::Quaternion quat(scale * w, scale * x, scale * y, scale * z);
  return quat;
}

/**************************************************************************/
/*!
    @brief  Provides the sensor_t data for this sensor
*/
/**************************************************************************/
void Adafruit_BNO055::getSensor(sensor_t *sensor)
{
  /* Clear the sensor_t object */
  memset(sensor, 0, sizeof(sensor_t));

  /* Insert the sensor name in the fixed length char array */
  strncpy (sensor->name, "BNO055", sizeof(sensor->name) - 1);
  sensor->name[sizeof(sensor->name)- 1] = 0;
  sensor->version     = 1;
  sensor->sensor_id   = _sensorID;
  sensor->type        = SENSOR_TYPE_ORIENTATION;
  sensor->min_delay   = 0;
  sensor->max_value   = 0.0F;
  sensor->min_value   = 0.0F;
  sensor->resolution  = 0.01F;
}

/**************************************************************************/
/*!
    @brief  Reads the sensor and returns the data as a sensors_event_t
*/
/**************************************************************************/
bool Adafruit_BNO055::getEvent(sensors_event_t *event)
{
  /* Clear the event */
  memset(event, 0, sizeof(sensors_event_t));

  event->version   = sizeof(sensors_event_t);
  event->sensor_id = _sensorID;
  event->type      = SENSOR_TYPE_ORIENTATION;
  event->timestamp = millis();

  /* Get a Euler angle sample for orientation */
  imu::Vector<3> euler = getVector(Adafruit_BNO055::VECTOR_EULER);
  event->orientation.x = euler.x();
  event->orientation.y = euler.y();
  event->orientation.z = euler.z();

  return true;
}

/***************************************************************************
 PRIVATE FUNCTIONS
 ***************************************************************************/

/**************************************************************************/
/*!
    @brief  Writes an 8 bit value over I2C
*/
/**************************************************************************/
bool Adafruit_BNO055::write8(adafruit_bno055_reg_t reg, byte value)
{
  Wire.beginTransmission(_address);
  #if ARDUINO >= 100
    Wire.write((uint8_t)reg);
    Wire.write((uint8_t)value);
  #else
    Wire.send(reg);
    Wire.send(value);
  #endif
  Wire.endTransmission();

  /* ToDo: Check for error! */
  return true;
}

/**************************************************************************/
/*!
    @brief  Reads an 8 bit value over I2C
*/
/**************************************************************************/
byte Adafruit_BNO055::read8(adafruit_bno055_reg_t reg )
{
  byte value = 0;

  Wire.beginTransmission(_address);
  #if ARDUINO >= 100
    Wire.write((uint8_t)reg);
  #else
    Wire.send(reg);
  #endif
  Wire.endTransmission();
  Wire.requestFrom(_address, (byte)1);
  #if ARDUINO >= 100
    value = Wire.read();
  #else
    value = Wire.receive();
  #endif

  return value;
}

/**************************************************************************/
/*!
    @brief  Reads the specified number of bytes over I2C
*/
/**************************************************************************/
bool Adafruit_BNO055::readLen(adafruit_bno055_reg_t reg, byte * buffer, uint8_t len)
{
  Wire.beginTransmission(_address);
  #if ARDUINO >= 100
    Wire.write((uint8_t)reg);
  #else
    Wire.send(reg);
  #endif
  Wire.endTransmission();
  Wire.requestFrom(_address, (byte)len);

  /* Wait until data is available */
  while (Wire.available() < len);

  for (uint8_t i = 0; i < len; i++)
  {
    #if ARDUINO >= 100
      buffer[i] = Wire.read();
    #else
      buffer[i] = Wire.receive();
    #endif
  }

  /* ToDo: Check for errors! */
  return true;
}
<|endoftext|>


<commit_before>//
// Created by Vadim N. on 16/03/2016.
//

#include "benchutils.h"

using namespace ymir;
using namespace std;

int main(int argc, char* argv[]) {

    if (argc < 2) {
      std::cout << "Please re-run the script with the data folder path supplied." << std::endl;
      return 1;
    }

    std::chrono::system_clock::time_point tp1, tp2;

    std::vector< std::pair<std::string, size_t> > timepoints;

    std::vector<prob_t> logLvec;

    std::string temp_str;

    time_t vj_single_parse, vdj_single_parse,
            vj_single_prob, vj_single_meta,
            vj_single_infer, vdj_single_prob,
            vdj_single_meta, vdj_single_infer;

    time_t vj_good_parse, vdj_good_parse,
            vj_good_prob, vj_good_meta,
            vj_good_infer, vdj_good_prob,
            vdj_good_meta, vdj_good_infer;

    std::string BENCH_DATA_FOLDER = argv[1];

    std::cout << "Data folder:\t[" << BENCH_DATA_FOLDER << "]" << endl;

    //
    // TCR alpha chain repertoire - VJ recombination
    //
    // VDJRecombinationGenes vj_single_genes("Vgene",
    //                                       "/home/vadim/ymir/benchmark/data/trav.txt",
    //                                       "Jgene",
    //                                       "/home/vadim/ymir/benchmark/data/traj.txt");

    string input_alpha_file = "alpha.250k.txt";
    string input_beta_file = "beta.250k.txt";

    CDR3NucParser parser;

    // Cloneset cloneset_vj;
    // YMIR_BENCHMARK("Parsing VJ",
    //                parser.openAndParse(BENCH_DATA_FOLDER + input_alpha_file,
    //                                    &cloneset_vj,
    //                                    vj_single_genes,
    //                                    NUCLEOTIDE,
    //                                    VJ_RECOMB,
    //                                    AlignmentColumnOptions(AlignmentColumnOptions::OVERWRITE, AlignmentColumnOptions::OVERWRITE),
    //                                    VDJAlignerParameters(2)))

    //
    // TCR beta chain repertoire - VDJ recombination
    //
   VDJRecombinationGenes vdj_single_genes("Vgene",
                                   BENCH_DATA_FOLDER + "trbv.txt",
                                   "Jgene",
                                   BENCH_DATA_FOLDER + "trbj.txt",
                                   "Dgene",
                                   BENCH_DATA_FOLDER + "trbd.txt");

   Cloneset cloneset_vdj;
   YMIR_BENCHMARK("Parsing VDJ",
                  parser.openAndParse(BENCH_DATA_FOLDER + input_beta_file,
                                      &cloneset_vdj,
                                      vdj_single_genes,
                                      NUCLEOTIDE,
                                      VDJ_RECOMB,
                                      AlignmentColumnOptions()
                                              .setV(AlignmentColumnOptions::OVERWRITE)
                                              .setD(AlignmentColumnOptions::OVERWRITE)
                                              .setJ(AlignmentColumnOptions::OVERWRITE),
                                      VDJAlignerParameters(3)))

    //
    // VJ MAAG
    //
    // ProbabilisticAssemblingModel vj_single_model(BENCH_DATA_FOLDER + "../../models/hTRA", EMPTY);

    //
    // VDJ MAAG
    //
   ProbabilisticAssemblingModel vdj_single_model(BENCH_DATA_FOLDER + "../../models/hTRB", EMPTY);

    //
    // Inference
    //
    vector<int> vec_sample = {10000, 25000, 50000, 100000, 150000};
     vec_sample = { std::stoi(argv[2]) };
    // int val_sample = cloneset_vdj.noncoding().size();
//    int val_sample = 100;
    vector<int> vec_block = {500, 1000, 2000, 5000, 10000}; //, 2000, 5000, 10000};
//    vec_block = {2000, 5000, 10000};
    vec_block = { std::stoi(argv[3]) };
    vector<double> vec_alpha = {.5, .6, .7, .8, .9};
    // vec_alpha = { std::stof(argv[3]) };
    vector<double> vec_beta =  {.1, .5, 1, 3};
    // vec_beta = { std::stof(argv[4]) };
    vector<double> vec_K =     {1, 2, 3};
    // vec_K = { std::stod(argv[5]) };
    ErrorMode error_mode = COMPUTE_ERRORS;

//    int niter, sample, block;
//    double alpha;
//    niter = 10;
//    int sample = 10000;

//    RUN_EM_INFERENCE(string("vj"), cloneset_vj, vj_single_model, niter, sample, error_mode)
//    RUN_EM_INFERENCE(string("vdj"), cloneset_vdj, vdj_single_model, niter, sample, error_mode)
//    for(auto val_sample: vec_sample) {
//        RUN_EM_INFERENCE(string("vj"), cloneset_vj, vj_single_model, 40, val_sample, error_mode)
//    }

//    ModelParameterVector new_param_vec = vj_single_model.event_probabilities();
//    new_param_vec.fill(1);
//    new_param_vec.normaliseEventFamilies();
//    vj_single_model.updateModelParameterVector(new_param_vec);
//
//    auto rep_nonc = cloneset_vj.noncoding().sample(2500);
//    auto maag_rep = vj_single_model.buildGraphs(rep_nonc, SAVE_METADATA, error_mode, NUCLEOTIDE, true);
//    vector<prob_t> prob_vec(maag_rep.size(), 0);
//
//    vector<bool> good_clonotypes(maag_rep.size(), true);
//    for (size_t i = 0; i < maag_rep.size(); ++i) {
//        if (rep_nonc[i].is_good()) {
//            prob_vec[i] = maag_rep[i].fullProbability();
//            if (std::isnan(prob_vec[i]) || prob_vec[i] == 0) {
//                good_clonotypes[i] = false;
//            }
//        } else {
//            good_clonotypes[i] = false;
//        }
//    }

     for(auto val_sample: vec_sample) {
        for(auto val_block: vec_block) {
            for (auto val_alpha: vec_alpha) {
                for (auto val_beta: vec_beta) {
                    for (auto val_K: vec_K) {
                        // RUN_SG_INFERENCE(string("vj"), cloneset_vj, vj_single_model, 40, val_block, val_alpha, val_beta, val_K, val_sample, error_mode)
                        RUN_SG_INFERENCE(string("vdj"), cloneset_vdj, vdj_single_model, 40, val_block, val_alpha, val_beta, val_K, val_sample, error_mode)
                    }
                }
            }
        }
     }

}
<commit_msg>debug<commit_after>//
// Created by Vadim N. on 16/03/2016.
//

#include "benchutils.h"

using namespace ymir;
using namespace std;

int main(int argc, char* argv[]) {

    if (argc < 2) {
      std::cout << "Please re-run the script with the data folder path supplied." << std::endl;
      return 1;
    }

    std::chrono::system_clock::time_point tp1, tp2;

    std::vector< std::pair<std::string, size_t> > timepoints;

    std::vector<prob_t> logLvec;

    std::string temp_str;

    time_t vj_single_parse, vdj_single_parse,
            vj_single_prob, vj_single_meta,
            vj_single_infer, vdj_single_prob,
            vdj_single_meta, vdj_single_infer;

    time_t vj_good_parse, vdj_good_parse,
            vj_good_prob, vj_good_meta,
            vj_good_infer, vdj_good_prob,
            vdj_good_meta, vdj_good_infer;

    std::string BENCH_DATA_FOLDER = argv[1];

    std::cout << "Data folder:\t[" << BENCH_DATA_FOLDER << "]" << endl;

    //
    // TCR alpha chain repertoire - VJ recombination
    //
    // VDJRecombinationGenes vj_single_genes("Vgene",
    //                                       "/home/vadim/ymir/benchmark/data/trav.txt",
    //                                       "Jgene",
    //                                       "/home/vadim/ymir/benchmark/data/traj.txt");

    string input_alpha_file = "alpha.250k.txt";
    string input_beta_file = "beta.250k.txt";

    CDR3NucParser parser;

    // Cloneset cloneset_vj;
    // YMIR_BENCHMARK("Parsing VJ",
    //                parser.openAndParse(BENCH_DATA_FOLDER + input_alpha_file,
    //                                    &cloneset_vj,
    //                                    vj_single_genes,
    //                                    NUCLEOTIDE,
    //                                    VJ_RECOMB,
    //                                    AlignmentColumnOptions(AlignmentColumnOptions::OVERWRITE, AlignmentColumnOptions::OVERWRITE),
    //                                    VDJAlignerParameters(2)))

    //
    // TCR beta chain repertoire - VDJ recombination
    //
   VDJRecombinationGenes vdj_single_genes("Vgene",
                                   BENCH_DATA_FOLDER + "trbv.txt",
                                   "Jgene",
                                   BENCH_DATA_FOLDER + "trbj.txt",
                                   "Dgene",
                                   BENCH_DATA_FOLDER + "trbd.txt");

   Cloneset cloneset_vdj;
   YMIR_BENCHMARK("Parsing VDJ",
                  parser.openAndParse(BENCH_DATA_FOLDER + input_beta_file,
                                      &cloneset_vdj,
                                      vdj_single_genes,
                                      NUCLEOTIDE,
                                      VDJ_RECOMB,
                                      AlignmentColumnOptions()
                                              .setV(AlignmentColumnOptions::OVERWRITE)
                                              .setD(AlignmentColumnOptions::OVERWRITE)
                                              .setJ(AlignmentColumnOptions::OVERWRITE),
                                      VDJAlignerParameters(3)))

    //
    // VJ MAAG
    //
    // ProbabilisticAssemblingModel vj_single_model(BENCH_DATA_FOLDER + "../../models/hTRA", EMPTY);

    //
    // VDJ MAAG
    //
   ProbabilisticAssemblingModel vdj_single_model(BENCH_DATA_FOLDER + "../../models/hTRB", EMPTY);

    //
    // Inference
    //
    vector<int> vec_sample = {10000, 25000, 50000, 100000, 150000};
     vec_sample = { std::stoi(argv[2]) };
    // int val_sample = cloneset_vdj.noncoding().size();
//    int val_sample = 100;
    vector<int> vec_block = {500, 1000, 2000, 5000, 10000}; //, 2000, 5000, 10000};
//    vec_block = {2000, 5000, 10000};
    vec_block = { std::stoi(argv[3]) };
    vector<double> vec_alpha = {.5, .6, .7, .8, .9};
    // vec_alpha = { std::stof(argv[3]) };
    vector<double> vec_beta =  {.1, .5, 1, 3};
    // vec_beta = { std::stof(argv[4]) };
    vector<double> vec_K =     {1, 2, 3};
    // vec_K = { std::stod(argv[5]) };
    ErrorMode error_mode = COMPUTE_ERRORS;

//    int niter, sample, block;
//    double alpha;
//    niter = 10;
//    int sample = 10000;

//    RUN_EM_INFERENCE(string("vj"), cloneset_vj, vj_single_model, niter, sample, error_mode)
//    RUN_EM_INFERENCE(string("vdj"), cloneset_vdj, vdj_single_model, niter, sample, error_mode)
   for(auto val_sample: vec_sample) {
       RUN_EM_INFERENCE(string("vdj"), cloneset_vdj, vdj_single_model, 40, val_sample, error_mode)
   }

//    ModelParameterVector new_param_vec = vj_single_model.event_probabilities();
//    new_param_vec.fill(1);
//    new_param_vec.normaliseEventFamilies();
//    vj_single_model.updateModelParameterVector(new_param_vec);
//
//    auto rep_nonc = cloneset_vj.noncoding().sample(2500);
//    auto maag_rep = vj_single_model.buildGraphs(rep_nonc, SAVE_METADATA, error_mode, NUCLEOTIDE, true);
//    vector<prob_t> prob_vec(maag_rep.size(), 0);
//
//    vector<bool> good_clonotypes(maag_rep.size(), true);
//    for (size_t i = 0; i < maag_rep.size(); ++i) {
//        if (rep_nonc[i].is_good()) {
//            prob_vec[i] = maag_rep[i].fullProbability();
//            if (std::isnan(prob_vec[i]) || prob_vec[i] == 0) {
//                good_clonotypes[i] = false;
//            }
//        } else {
//            good_clonotypes[i] = false;
//        }
//    }

     for(auto val_sample: vec_sample) {
        for(auto val_block: vec_block) {
            for (auto val_alpha: vec_alpha) {
                for (auto val_beta: vec_beta) {
                    for (auto val_K: vec_K) {
                        // RUN_SG_INFERENCE(string("vj"), cloneset_vj, vj_single_model, 40, val_block, val_alpha, val_beta, val_K, val_sample, error_mode)
                        RUN_SG_INFERENCE(string("vdj"), cloneset_vdj, vdj_single_model, 40, val_block, val_alpha, val_beta, val_K, val_sample, error_mode)
                    }
                }
            }
        }
     }

}
<|endoftext|>


<commit_before>// A simple binary to grep URLs out of text.
//
// Based on the URI class contrib'd to the excellent PEGTL library.
//
// Licenced under the MIT licence as per the original code.
//
// Copyright (c) 2014-2015 Dr. Colin Hirsch and Daniel Frey
// Please see LICENSE for license or visit https://github.com/ColinH/PEGTL/

#include <iostream>
#include <string>
#include <iomanip>
#include <ios>

#include <pegtl.hh>
#include <pegtl/trace.hh>
#include <pegtl/contrib/uri.hh>

using grammar = pegtl::uri::URI;

namespace pegtl {
  namespace uri {
    struct URIState {
      std::string uri;
    };

    // The default base action class.
    template<typename Rule>
    struct URIActions : pegtl::nothing<Rule> {};

    // Action for the URI class.
    template<>
    struct URIActions<URI> {
      static void apply(const pegtl::input& input,
                        URIState& parsed_uri) {
        parsed_uri.uri.assign(input.string());
      }
    };
  }
}

using pegtl::uri::URIActions;
using pegtl::uri::URIState;

// Should be compiled down quite nicely. :)
char brackets[] = {'{', '}', '[', ']', '(', ')', '<', '>', '"', '\''};
bool IsBracket(char c) {
  for (int i = 0; i < sizeof(brackets) / sizeof(char); ++i) {
    if (c == brackets[i]) return true;
  }
  return false;
}

int main(int argc, char* argv[]) {
  ios_base::sync_with_stdio(false);
  cin.tie(nullptr);
  std::string thunk;
  while (std::cin >> thunk) {
    // Trim.
    int start = 0, end = thunk.size();
    for (; start < thunk.size() && IsBracket(thunk[start]); ++start) {}
    for (; end > 0 && IsBracket(thunk[end - 1]); --end) {}

    URIState state;
    bool parse_result = pegtl::parse<grammar, URIActions>(
        thunk.data() + start, thunk.data() + end, "stdin", state);
    if (parse_result) {
      std::cout << state.uri << '\n';
    }
  }
  return 0;
}
<commit_msg>Fix compile bugs. Serves me right for not testing before committing.<commit_after>// A simple binary to grep URLs out of text.
//
// Based on the URI class contrib'd to the excellent PEGTL library.
//
// Licenced under the MIT licence as per the original code.
//
// Copyright (c) 2014-2015 Dr. Colin Hirsch and Daniel Frey
// Please see LICENSE for license or visit https://github.com/ColinH/PEGTL/

#include <iostream>
#include <string>
#include <iomanip>
#include <ios>

#include <pegtl.hh>
#include <pegtl/trace.hh>
#include <pegtl/contrib/uri.hh>

using grammar = pegtl::uri::URI;

namespace pegtl {
  namespace uri {
    struct URIState {
      std::string uri;
    };

    // The default base action class.
    template<typename Rule>
    struct URIActions : pegtl::nothing<Rule> {};

    // Action for the URI class.
    template<>
    struct URIActions<URI> {
      static void apply(const pegtl::input& input,
                        URIState& parsed_uri) {
        parsed_uri.uri.assign(input.string());
      }
    };
  }
}

using pegtl::uri::URIActions;
using pegtl::uri::URIState;

// Should be compiled down quite nicely. :)
char brackets[] = {'{', '}', '[', ']', '(', ')', '<', '>', '"', '\''};
bool IsBracket(char c) {
  for (int i = 0; i < sizeof(brackets) / sizeof(char); ++i) {
    if (c == brackets[i]) return true;
  }
  return false;
}

int main(int argc, char* argv[]) {
  std::ios_base::sync_with_stdio(false);
  std::cin.tie(nullptr);
  std::string thunk;
  while (std::cin >> thunk) {
    // Trim.
    int start = 0, end = thunk.size();
    for (; start < thunk.size() && IsBracket(thunk[start]); ++start) {}
    for (; end > 0 && IsBracket(thunk[end - 1]); --end) {}

    URIState state;
    bool parse_result = pegtl::parse<grammar, URIActions>(
        thunk.data() + start, thunk.data() + end, "stdin", state);
    if (parse_result) {
      std::cout << state.uri << '\n';
    }
  }
  return 0;
}
<|endoftext|>


<commit_before>#include "bench_common.h"
#include <dariadb.h>
#include <storage/capacitor.h>
#include <utils/fs.h>
#include <utils/metrics.h>
#include <utils/thread_manager.h>

#include <boost/program_options.hpp>

namespace po = boost::program_options;

std::atomic_llong append_count{0};
std::atomic_size_t reads_count{0};
bool stop_info = false;
bool stop_readers = false;

class BenchCallback : public dariadb::storage::IReaderClb {
public:
  BenchCallback() { count = 0; }
  void call(const dariadb::Meas &v) { count++; }
  std::atomic<size_t> count;
};

void show_info(dariadb::storage::Engine *storage) {
  clock_t t0 = clock();
  while (true) {
    std::this_thread::sleep_for(std::chrono::milliseconds(100));

    clock_t t1 = clock();
    auto writes_per_sec = append_count.load() / double((t1 - t0) / CLOCKS_PER_SEC);
    auto reads_per_sec = reads_count.load() / double((t1 - t0) / CLOCKS_PER_SEC);
    auto queue_sizes = storage->queue_size();
    std::cout << "\r"
              << " in queue: (p:" << queue_sizes.pages_count
              << " cap:" << queue_sizes.cola_count << " a:" << queue_sizes.aofs_count
              << " T:" << queue_sizes.active_works << ") reads: " << reads_count
              << " speed:" << reads_per_sec << "/sec"
              << " writes: " << append_count << " speed: " << writes_per_sec
              << "/sec progress:"
              << (int64_t(100) * append_count) / dariadb_bench::all_writes
              << "%                ";
    std::cout.flush();
    if (stop_info) {
      std::cout.flush();
      break;
    }
  }
  std::cout << "\n";
}

void show_drop_info(dariadb::storage::Engine *storage) {
  while (true) {
    std::this_thread::sleep_for(std::chrono::milliseconds(100));

    auto queue_sizes = storage->queue_size();
    std::cout << "\r"
              << " in queue: (p:" << queue_sizes.pages_count
              << " cap:" << queue_sizes.cola_count << " a:" << queue_sizes.aofs_count
              << " T:" << queue_sizes.active_works << ")          ";
    std::cout.flush();
    if (stop_info) {
      std::cout.flush();
      break;
    }
  }
  std::cout << "\n";
}

void reader(dariadb::storage::IMeasStorage_ptr ms, dariadb::IdSet all_id_set,
            dariadb::Time from, dariadb::Time to) {
  std::random_device r;
  std::default_random_engine e1(r());
  std::uniform_int_distribution<dariadb::Id> uniform_dist(from, to);
  std::shared_ptr<BenchCallback> clbk{new BenchCallback};

  while (true) {
    clbk->count = 0;
    // auto time_point1 = uniform_dist(e1);
    auto f = from;
    auto t = dariadb::timeutil::current_time();

    auto qi = dariadb::storage::QueryInterval(
        dariadb::IdArray(all_id_set.begin(), all_id_set.end()), 0, f, t);
    ms->foreach (qi, clbk.get());

    reads_count += clbk->count;
    if (stop_readers) {
      break;
    }
  }
}

int main(int argc, char *argv[]) {
  (void)argc;
  (void)argv;
  std::cout << "Performance benchmark" << std::endl;
  std::cout << "Writers count:" << dariadb_bench::total_threads_count << std::endl;

  const std::string storage_path = "perf_benchmark_storage";
  bool readers_enable = false;
  bool metrics_enable = false;
  bool readonly = false;
  bool readall_enabled = false;
  bool dont_clean = false;
  // bool full_flush = false;
  po::options_description desc("Allowed options");
  desc.add_options()("help", "produce help message")("readonly", "readonly mode")(
      "readall", "read all benchmark enable.")
      //("full-flush", "wait end of all async tasks.")
      ("dont-clean", "dont clean storage path before start.")(
          "enable-readers",
          po::value<bool>(&readers_enable)->default_value(readers_enable),
          "enable readers threads")(
          "enable-metrics",
          po::value<bool>(&metrics_enable)->default_value(metrics_enable));

  po::variables_map vm;
  try {
    po::store(po::parse_command_line(argc, argv, desc), vm);
  } catch (std::exception &ex) {
    logger("Error: " << ex.what());
    exit(1);
  }
  po::notify(vm);

  if (vm.count("help")) {
    std::cout << desc << std::endl;
    return 1;
  }

  if (metrics_enable) {
    std::cout << "Enable metrics." << std::endl;
  }

  if (vm.count("readonly")) {
    std::cout << "Readonly mode." << std::endl;
    readonly = true;
  }

  if (vm.count("readall")) {
    std::cout << "Read all benchmark enabled." << std::endl;
    readall_enabled = true;
  }

  if (vm.count("dont-clean")) {
    std::cout << "Dont clean storage." << std::endl;
    dont_clean = true;
  }

  /*if (vm.count("full-flush")) {
          std::cout << "full-flush." << std::endl;
          full_flush = true;
  }*/

  if (readers_enable) {
    std::cout << "Readers enable. count: " << dariadb_bench::total_readers_count
              << std::endl;
  }

  {
    std::cout << "Write..." << std::endl;

    const size_t chunk_size = 512;
    const size_t cap_B = 50;

    // dont_clean = true;
    if (!dont_clean && dariadb::utils::fs::path_exists(storage_path)) {
      if (!readonly) {
        std::cout << " remove " << storage_path << std::endl;
        dariadb::utils::fs::rm(storage_path);
      }
    }

    dariadb::Time start_time = dariadb::timeutil::current_time();
    std::cout << " start time: " << dariadb::timeutil::to_string(start_time) << std::endl;

    dariadb::storage::PageManager::Params page_param(storage_path, chunk_size);
    dariadb::storage::CapacitorManager::Params cap_param(storage_path, cap_B);
	cap_param.store_period = 0; // 1000 * 2;
	cap_param.max_levels = 11;
	cap_param.max_closed_caps = 0; // 5;
	dariadb::storage::AOFManager::Params aof_param(storage_path, 0);
	aof_param.buffer_size = 1000;
	aof_param.max_size = cap_param.measurements_count();

    cap_param.max_levels = 11;
    auto raw_ptr = new dariadb::storage::Engine(aof_param, page_param, cap_param);

    dariadb::storage::IMeasStorage_ptr ms{raw_ptr};

    dariadb::IdSet all_id_set;
    append_count = 0;
    stop_info = false;

    std::thread info_thread(show_info, raw_ptr);

    std::vector<std::thread> writers(dariadb_bench::total_threads_count);
    std::vector<std::thread> readers(dariadb_bench::total_readers_count);

    size_t pos = 0;
    for (size_t i = 1; i < dariadb_bench::total_threads_count + 1; i++) {
      auto id_from = dariadb_bench::get_id_from(pos);
      auto id_to = dariadb_bench::get_id_to(pos);
      for (size_t j = id_from; j < id_to; j++) {
        all_id_set.insert(j);
      }
      if (!readonly) {
        std::thread t{dariadb_bench::thread_writer_rnd_stor, dariadb::Id(pos),
                      dariadb::Time(i), &append_count, raw_ptr};
        writers[pos] = std::move(t);
      }
      pos++;
    }
    if (readers_enable) {
      pos = 0;
      for (size_t i = 1; i < dariadb_bench::total_readers_count + 1; i++) {
        std::thread t{reader, ms, all_id_set, start_time,
                      dariadb::timeutil::current_time()};
        readers[pos++] = std::move(t);
      }
    }

    if (!readonly) {
      pos = 0;
      for (size_t i = 1; i < dariadb_bench::total_threads_count + 1; i++) {
        std::thread t = std::move(writers[pos++]);
        t.join();
      }
    }
    stop_readers = true;
    if (readers_enable) {
      pos = 0;
      for (size_t i = 1; i < dariadb_bench::total_readers_count + 1; i++) {
        std::thread t = std::move(readers[pos++]);
        t.join();
      }
    }

    stop_info = true;
    info_thread.join();
    std::cout << " total id:" << all_id_set.size() << std::endl;

    {
      std::cout << "==> full flush..." << std::endl;
      stop_info = false;
      std::thread flush_info_thread(show_drop_info, raw_ptr);

      auto start = clock();
      raw_ptr->flush();
      // if (full_flush)
      { raw_ptr->wait_all_asyncs(); }
      auto elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
      stop_info = true;
      flush_info_thread.join();
      std::cout << "flush time: " << elapsed << std::endl;
    }

    if (!readonly) {
      size_t ccount = size_t(raw_ptr->queue_size().cola_count*0.5);
      std::cout << "==> drop part caps to " << ccount << "..." << std::endl;
      stop_info = false;
      std::thread flush_info_thread(show_drop_info, raw_ptr);

      auto start = clock();
      raw_ptr->drop_part_caps(ccount);
      raw_ptr->flush();
      raw_ptr->wait_all_asyncs();
      auto elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
      stop_info = true;
      flush_info_thread.join();
      std::cout << "drop time: " << elapsed << std::endl;
    }

    auto queue_sizes = raw_ptr->queue_size();
    std::cout << "\r"
              << " in queue: (p:" << queue_sizes.pages_count
              << " cap:" << queue_sizes.cola_count << " a:" << queue_sizes.aofs_count
              << ")" << append_count << std::endl;

    std::cout << "Active threads: "
              << dariadb::utils::async::ThreadManager::instance()->active_works()
              << std::endl;

    dariadb_bench::readBenchark(all_id_set, ms.get(), 100, start_time,
                                dariadb::timeutil::current_time());

	auto max_time = ms->maxTime();
	std::cout << "==> interval end time: " << dariadb::timeutil::to_string(max_time)
		<< std::endl;

    if (readall_enabled) {
      if (readonly) {
        start_time = dariadb::Time(0);
      }

      std::shared_ptr<BenchCallback> clbk{new BenchCallback()};

      dariadb::storage::QueryInterval qi{
          dariadb::IdArray(all_id_set.begin(), all_id_set.end()), 0, start_time,
          max_time};

      std::cout << "==> foreach all..." << std::endl;

      auto start = clock();

      ms->foreach (qi, clbk.get());

      auto elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
      std::cout << "readed: " << clbk->count << std::endl;
      std::cout << "time: " << elapsed << std::endl;

      std::cout << "==> read all..." << std::endl;

      start = clock();

      auto readed = ms->readInterval(qi);

      elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
      std::cout << "readed: " << readed.size() << std::endl;
      std::cout << "time: " << elapsed << std::endl;

      auto expected = (dariadb_bench::write_per_id_count *
                       dariadb_bench::total_threads_count * dariadb_bench::id_per_thread);

      std::map<dariadb::Id, dariadb::Meas::MeasList> _dict;
      for (auto &v : readed) {
        _dict[v.id].push_back(v);
      }

      if (readed.size() != expected) {
        std::cout << "expected: " << expected << " get:" << clbk->count << std::endl;
		std::cout << " all_writesL " << dariadb_bench::all_writes;
        for (auto &kv : _dict) {
          std::cout << " " << kv.first << " -> " << kv.second.size() << std::endl;
        }
        throw MAKE_EXCEPTION("(clbk->count!=(iteration_count*total_threads_count))");
      }
    }
    std::cout << "stoping storage...\n";
    ms = nullptr;
  }

  if (!(dont_clean || readonly) && (dariadb::utils::fs::path_exists(storage_path))) {
    std::cout << "cleaning...\n";
    dariadb::utils::fs::rm(storage_path);
  }

  if (metrics_enable) {
    std::cout << "metrics:\n"
              << dariadb::utils::metrics::MetricsManager::instance()->to_string()
              << std::endl;
  }
}
<commit_msg>more verbose benchmark.<commit_after>#include "bench_common.h"
#include <dariadb.h>
#include <storage/capacitor.h>
#include <utils/fs.h>
#include <utils/metrics.h>
#include <utils/thread_manager.h>

#include <boost/program_options.hpp>

namespace po = boost::program_options;

std::atomic_llong append_count{0};
std::atomic_size_t reads_count{0};
bool stop_info = false;
bool stop_readers = false;

class BenchCallback : public dariadb::storage::IReaderClb {
public:
  BenchCallback() { count = 0; }
  void call(const dariadb::Meas &v) { count++; }
  std::atomic<size_t> count;
};

void show_info(dariadb::storage::Engine *storage) {
  clock_t t0 = clock();
  while (true) {
    std::this_thread::sleep_for(std::chrono::milliseconds(100));

    clock_t t1 = clock();
    auto writes_per_sec = append_count.load() / double((t1 - t0) / CLOCKS_PER_SEC);
    auto reads_per_sec = reads_count.load() / double((t1 - t0) / CLOCKS_PER_SEC);
    auto queue_sizes = storage->queue_size();
    std::cout << "\r"
              << " in queue: (p:" << queue_sizes.pages_count
              << " cap:" << queue_sizes.cola_count << " a:" << queue_sizes.aofs_count
              << " T:" << queue_sizes.active_works << ") reads: " << reads_count
              << " speed:" << reads_per_sec << "/sec"
              << " writes: " << append_count << " speed: " << writes_per_sec
              << "/sec progress:"
              << (int64_t(100) * append_count) / dariadb_bench::all_writes
              << "%                ";
    std::cout.flush();
    if (stop_info) {
      std::cout.flush();
      break;
    }
  }
  std::cout << "\n";
}

void show_drop_info(dariadb::storage::Engine *storage) {
  while (true) {
    std::this_thread::sleep_for(std::chrono::milliseconds(100));

    auto queue_sizes = storage->queue_size();
    std::cout << "\r"
              << " in queue: (p:" << queue_sizes.pages_count
              << " cap:" << queue_sizes.cola_count << " a:" << queue_sizes.aofs_count
              << " T:" << queue_sizes.active_works << ")          ";
    std::cout.flush();
    if (stop_info) {
      std::cout.flush();
      break;
    }
  }
  std::cout << "\n";
}

void reader(dariadb::storage::IMeasStorage_ptr ms, dariadb::IdSet all_id_set,
            dariadb::Time from, dariadb::Time to) {
  std::random_device r;
  std::default_random_engine e1(r());
  std::uniform_int_distribution<dariadb::Id> uniform_dist(from, to);
  std::shared_ptr<BenchCallback> clbk{new BenchCallback};

  while (true) {
    clbk->count = 0;
    // auto time_point1 = uniform_dist(e1);
    auto f = from;
    auto t = dariadb::timeutil::current_time();

    auto qi = dariadb::storage::QueryInterval(
        dariadb::IdArray(all_id_set.begin(), all_id_set.end()), 0, f, t);
    ms->foreach (qi, clbk.get());

    reads_count += clbk->count;
    if (stop_readers) {
      break;
    }
  }
}

int main(int argc, char *argv[]) {
  (void)argc;
  (void)argv;
  std::cout << "Performance benchmark" << std::endl;
  std::cout << "Writers count:" << dariadb_bench::total_threads_count << std::endl;

  const std::string storage_path = "perf_benchmark_storage";
  bool readers_enable = false;
  bool metrics_enable = false;
  bool readonly = false;
  bool readall_enabled = false;
  bool dont_clean = false;
  // bool full_flush = false;
  po::options_description desc("Allowed options");
  desc.add_options()("help", "produce help message")("readonly", "readonly mode")(
      "readall", "read all benchmark enable.")
      //("full-flush", "wait end of all async tasks.")
      ("dont-clean", "dont clean storage path before start.")(
          "enable-readers",
          po::value<bool>(&readers_enable)->default_value(readers_enable),
          "enable readers threads")(
          "enable-metrics",
          po::value<bool>(&metrics_enable)->default_value(metrics_enable));

  po::variables_map vm;
  try {
    po::store(po::parse_command_line(argc, argv, desc), vm);
  } catch (std::exception &ex) {
    logger("Error: " << ex.what());
    exit(1);
  }
  po::notify(vm);

  if (vm.count("help")) {
    std::cout << desc << std::endl;
    return 1;
  }

  if (metrics_enable) {
    std::cout << "Enable metrics." << std::endl;
  }

  if (vm.count("readonly")) {
    std::cout << "Readonly mode." << std::endl;
    readonly = true;
  }

  if (vm.count("readall")) {
    std::cout << "Read all benchmark enabled." << std::endl;
    readall_enabled = true;
  }

  if (vm.count("dont-clean")) {
    std::cout << "Dont clean storage." << std::endl;
    dont_clean = true;
  }

  /*if (vm.count("full-flush")) {
          std::cout << "full-flush." << std::endl;
          full_flush = true;
  }*/

  if (readers_enable) {
    std::cout << "Readers enable. count: " << dariadb_bench::total_readers_count
              << std::endl;
  }

  {
    std::cout << "Write..." << std::endl;

    const size_t chunk_size = 512;
    const size_t cap_B = 50;

    // dont_clean = true;
    if (!dont_clean && dariadb::utils::fs::path_exists(storage_path)) {
      if (!readonly) {
        std::cout << " remove " << storage_path << std::endl;
        dariadb::utils::fs::rm(storage_path);
      }
    }

    dariadb::Time start_time = dariadb::timeutil::current_time();
    std::cout << " start time: " << dariadb::timeutil::to_string(start_time) << std::endl;

    dariadb::storage::PageManager::Params page_param(storage_path, chunk_size);

	dariadb::storage::CapacitorManager::Params cap_param(storage_path, cap_B);
	cap_param.store_period = 1000 * 60*60;
	cap_param.max_levels = 11;
	cap_param.max_closed_caps = 0; // 5;

	dariadb::storage::AOFManager::Params aof_param(storage_path, 0);
	aof_param.buffer_size = 1000;
	aof_param.max_size = cap_param.measurements_count();
    auto raw_ptr = new dariadb::storage::Engine(aof_param, page_param, cap_param);

    dariadb::storage::IMeasStorage_ptr ms{raw_ptr};

    dariadb::IdSet all_id_set;
    append_count = 0;
    stop_info = false;

    std::thread info_thread(show_info, raw_ptr);

    std::vector<std::thread> writers(dariadb_bench::total_threads_count);
    std::vector<std::thread> readers(dariadb_bench::total_readers_count);

    size_t pos = 0;
	auto writers_start = clock();
    for (size_t i = 1; i < dariadb_bench::total_threads_count + 1; i++) {
      auto id_from = dariadb_bench::get_id_from(pos);
      auto id_to = dariadb_bench::get_id_to(pos);
      for (size_t j = id_from; j < id_to; j++) {
        all_id_set.insert(j);
      }
      if (!readonly) {
        std::thread t{dariadb_bench::thread_writer_rnd_stor, dariadb::Id(pos),
                      dariadb::Time(i), &append_count, raw_ptr};
        writers[pos] = std::move(t);
      }
      pos++;
    }
    if (readers_enable) {
      pos = 0;
      for (size_t i = 1; i < dariadb_bench::total_readers_count + 1; i++) {
        std::thread t{reader, ms, all_id_set, start_time,
                      dariadb::timeutil::current_time()};
        readers[pos++] = std::move(t);
      }
    }

    if (!readonly) {
      pos = 0;
      for (size_t i = 1; i < dariadb_bench::total_threads_count + 1; i++) {
        std::thread t = std::move(writers[pos++]);
        t.join();
      }
    }
	auto writers_elapsed = (((float)clock() - writers_start) / CLOCKS_PER_SEC);
    stop_readers = true;
    if (readers_enable) {
      pos = 0;
      for (size_t i = 1; i < dariadb_bench::total_readers_count + 1; i++) {
        std::thread t = std::move(readers[pos++]);
        t.join();
      }
    }

    stop_info = true;
    info_thread.join();
    std::cout << " total id:" << all_id_set.size() << std::endl;
	std::cout << "write time: " << writers_elapsed << std::endl;
    {
      std::cout << "==> full flush..." << std::endl;
      stop_info = false;
      std::thread flush_info_thread(show_drop_info, raw_ptr);

      auto start = clock();
      raw_ptr->flush();
      // if (full_flush)
      { raw_ptr->wait_all_asyncs(); }
      auto elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
      stop_info = true;
      flush_info_thread.join();
      std::cout << "flush time: " << elapsed << std::endl;
    }

    if (!readonly) {
      size_t ccount = size_t(raw_ptr->queue_size().cola_count*0.5);
      std::cout << "==> drop part caps to " << ccount << "..." << std::endl;
      stop_info = false;
      std::thread flush_info_thread(show_drop_info, raw_ptr);

      auto start = clock();
      raw_ptr->drop_part_caps(ccount);
      raw_ptr->flush();
      raw_ptr->wait_all_asyncs();
      auto elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
      stop_info = true;
      flush_info_thread.join();
      std::cout << "drop time: " << elapsed << std::endl;
    }

    auto queue_sizes = raw_ptr->queue_size();
    std::cout << "\r"
              << " in queue: (p:" << queue_sizes.pages_count
              << " cap:" << queue_sizes.cola_count << " a:" << queue_sizes.aofs_count
              << ")" << std::endl;

    std::cout << "Active threads: "
              << dariadb::utils::async::ThreadManager::instance()->active_works()
              << std::endl;

    dariadb_bench::readBenchark(all_id_set, ms.get(), 100, start_time,
                                dariadb::timeutil::current_time());

	auto max_time = ms->maxTime();
	std::cout << "==> interval end time: " << dariadb::timeutil::to_string(max_time)
		<< std::endl;

    if (readall_enabled) {
      if (readonly) {
        start_time = dariadb::Time(0);
      }

      std::shared_ptr<BenchCallback> clbk{new BenchCallback()};

      dariadb::storage::QueryInterval qi{
          dariadb::IdArray(all_id_set.begin(), all_id_set.end()), 0, start_time,
          max_time};

      std::cout << "==> foreach all..." << std::endl;

      auto start = clock();

      ms->foreach (qi, clbk.get());

      auto elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
      std::cout << "readed: " << clbk->count << std::endl;
      std::cout << "time: " << elapsed << std::endl;

      std::cout << "==> read all..." << std::endl;

      start = clock();

      auto readed = ms->readInterval(qi);

      elapsed = (((float)clock() - start) / CLOCKS_PER_SEC);
      std::cout << "readed: " << readed.size() << std::endl;
      std::cout << "time: " << elapsed << std::endl;

      auto expected = (dariadb_bench::write_per_id_count *
                       dariadb_bench::total_threads_count * dariadb_bench::id_per_thread);

      std::map<dariadb::Id, dariadb::Meas::MeasList> _dict;
      for (auto &v : readed) {
        _dict[v.id].push_back(v);
      }

      if (readed.size() != expected) {
        std::cout << "expected: " << expected << " get:" << clbk->count << std::endl;
		std::cout << " all_writesL " << dariadb_bench::all_writes;
        for (auto &kv : _dict) {
          std::cout << " " << kv.first << " -> " << kv.second.size() << std::endl;
        }
        throw MAKE_EXCEPTION("(clbk->count!=(iteration_count*total_threads_count))");
      }
    }
    std::cout << "stoping storage...\n";
    ms = nullptr;
  }

  if (!(dont_clean || readonly) && (dariadb::utils::fs::path_exists(storage_path))) {
    std::cout << "cleaning...\n";
    dariadb::utils::fs::rm(storage_path);
  }

  if (metrics_enable) {
    std::cout << "metrics:\n"
              << dariadb::utils::metrics::MetricsManager::instance()->to_string()
              << std::endl;
  }
}
<|endoftext|>