<commit_before>#include "mantella_bits/optimisationAlgorithm/hookeJeevesAlgorithm.hpp"

// Mantella
#include "mantella_bits/assert.hpp"
#include "mantella_bits/optimisationProblem.hpp"

namespace mant {
  HookeJeevesAlgorithm::HookeJeevesAlgorithm()
      : OptimisationAlgorithm(),
        stepSize_(0) {
    setNextParametersFunction(
        [this](
            const arma::uword numberOfDimensions_,
            const arma::Mat<double>& parameters_,
            const arma::Row<double>& objectiveValues_,
            const arma::Row<double>& differences_) {
          if (arma::all(differences_ >= 0)) {
            stepSize_ /= stepSizeDecrease_;
          }

          arma::Mat<double> nextParameters(numberOfDimensions_, 2 * numberOfDimensions_);
          for (arma::uword n = 0; n < numberOfDimensions_; ++n) {
            arma::Col<double> nextParameterCandidate = bestParameter_;

            nextParameterCandidate(n) += stepSize_;
            nextParameters.col(2 * n) = nextParameterCandidate;

            nextParameterCandidate(n) -= 2 * stepSize_;
            nextParameters.col(2 * n + 1) = nextParameterCandidate;
          }

          return nextParameters;
        },
        "Hooke-Jeeves algorithm");

    setInitialStepSize(1.0);
    setStepSizeDecrease(0.5);
  }

  void HookeJeevesAlgorithm::initialise(
      const arma::uword numberOfDimensions,
      const arma::Mat<double>& initialParameters) {
    stepSize_ = initialStepSize_;
  }

  void HookeJeevesAlgorithm::optimise(
      OptimisationProblem& optimisationProblem) {
    optimise(optimisationProblem, arma::randu<arma::Col<double>>(optimisationProblem.numberOfDimensions_));
  }

  void HookeJeevesAlgorithm::setInitialStepSize(
      const double initialStepSize) {
    verify(initialStepSize > 0, "HookeJeevesAlgorithm.setInitialStepSize: The initial step size must be strict greater than 0.");

    initialStepSize_ = initialStepSize;
  }

  double HookeJeevesAlgorithm::getInitialStepSize() const {
    return initialStepSize_;
  }

  void HookeJeevesAlgorithm::setStepSizeDecrease(
      const double stepSizeDecrease) {
    verify(stepSizeDecrease > 0 && stepSizeDecrease < 1, "HookeJeevesAlgorithm.setStepSizeDecrease: The step size decrease must be within the interval (0, 1).");

    stepSizeDecrease_ = stepSizeDecrease;
  }

  double HookeJeevesAlgorithm::getStepSizeDecrease() const {
    return stepSizeDecrease_;
  }
}
<commit_msg>hotfix: Fixing the step size decrease in the Hooke-Jeeves algorithm<commit_after>#include "mantella_bits/optimisationAlgorithm/hookeJeevesAlgorithm.hpp"

// Mantella
#include "mantella_bits/assert.hpp"
#include "mantella_bits/optimisationProblem.hpp"

namespace mant {
  HookeJeevesAlgorithm::HookeJeevesAlgorithm()
      : OptimisationAlgorithm(),
        stepSize_(0) {
    setNextParametersFunction(
        [this](
            const arma::uword numberOfDimensions_,
            const arma::Mat<double>& parameters_,
            const arma::Row<double>& objectiveValues_,
            const arma::Row<double>& differences_) {
          if (arma::all(differences_ >= 0)) {
            stepSize_ *= stepSizeDecrease_;
          }

          arma::Mat<double> nextParameters(numberOfDimensions_, 2 * numberOfDimensions_);
          for (arma::uword n = 0; n < numberOfDimensions_; ++n) {
            arma::Col<double> nextParameterCandidate = bestParameter_;

            nextParameterCandidate(n) += stepSize_;
            nextParameters.col(2 * n) = nextParameterCandidate;

            nextParameterCandidate(n) -= 2 * stepSize_;
            nextParameters.col(2 * n + 1) = nextParameterCandidate;
          }

          return nextParameters;
        },
        "Hooke-Jeeves algorithm");

    setInitialStepSize(1.0);
    setStepSizeDecrease(0.5);
  }

  void HookeJeevesAlgorithm::initialise(
      const arma::uword numberOfDimensions,
      const arma::Mat<double>& initialParameters) {
    stepSize_ = initialStepSize_;
  }

  void HookeJeevesAlgorithm::optimise(
      OptimisationProblem& optimisationProblem) {
    optimise(optimisationProblem, arma::randu<arma::Col<double>>(optimisationProblem.numberOfDimensions_));
  }

  void HookeJeevesAlgorithm::setInitialStepSize(
      const double initialStepSize) {
    verify(initialStepSize > 0, "HookeJeevesAlgorithm.setInitialStepSize: The initial step size must be strict greater than 0.");

    initialStepSize_ = initialStepSize;
  }

  double HookeJeevesAlgorithm::getInitialStepSize() const {
    return initialStepSize_;
  }

  void HookeJeevesAlgorithm::setStepSizeDecrease(
      const double stepSizeDecrease) {
    verify(stepSizeDecrease > 0 && stepSizeDecrease < 1, "HookeJeevesAlgorithm.setStepSizeDecrease: The step size decrease must be within the interval (0, 1).");

    stepSizeDecrease_ = stepSizeDecrease;
  }

  double HookeJeevesAlgorithm::getStepSizeDecrease() const {
    return stepSizeDecrease_;
  }
}
<|endoftext|>


<commit_before>#include <unistd.h>
#include <time.h>
#include <string.h>
#include <iostream>

#include "rcu.h"
#include "macros.h"
#include "util.h"
#include "thread.h"
#include "counter.h"
#include "lockguard.h"

using namespace std;
using namespace util;

// avoid some corner cases in the beginning
volatile rcu::epoch_t rcu::global_epoch = 1;
volatile rcu::epoch_t rcu::cleaning_epoch = 0;
rcu::delete_queue rcu::global_queues[2];

volatile bool rcu::gc_thread_started = false;
pthread_t rcu::gc_thread_p;

map<pthread_t, rcu::sync *> rcu::sync_map;

__thread rcu::sync *rcu::tl_sync = NULL;
__thread unsigned int rcu::tl_crit_section_depth = 0;

static event_counter evt_rcu_deletes("rcu_deletes");
static event_counter evt_rcu_frees("rcu_frees");
static event_counter evt_rcu_local_reaps("rcu_local_reaps");

static event_avg_counter evt_avg_gc_reaper_queue_len("avg_gc_reaper_queue_len");
static event_avg_counter evt_avg_rcu_delete_queue_len("avg_rcu_delete_queue_len");
static event_avg_counter evt_avg_rcu_local_delete_queue_len("avg_rcu_local_delete_queue_len");

spinlock &
rcu::rcu_mutex()
{
  static spinlock s_lock;
  return s_lock;
}

rcu::sync *
rcu::register_sync(pthread_t p)
{
  lock_guard<spinlock> l(rcu_mutex());
  map<pthread_t, sync *>::iterator it = sync_map.find(p);
  ALWAYS_ASSERT(it == sync_map.end());
  return (sync_map[p] = new sync(global_epoch));
}

rcu::sync *
rcu::unregister_sync(pthread_t p)
{
  lock_guard<spinlock> l(rcu_mutex());
  map<pthread_t, sync *>::iterator it = sync_map.find(p);
  if (it == sync_map.end())
    return NULL;
  sync * const s = it->second;

  const epoch_t my_cleaning_epoch = cleaning_epoch;
  const epoch_t local_epoch = s->local_epoch;
  const epoch_t local_cleaning_epoch = s->local_epoch - 1;

#ifdef CHECK_INVARIANTS
  const epoch_t my_global_epoch = global_epoch;
  INVARIANT(my_global_epoch == local_epoch ||
            my_global_epoch == (local_epoch + 1));
  INVARIANT(my_global_epoch == (my_cleaning_epoch + 1) ||
            my_global_epoch == (my_cleaning_epoch + 2));
  INVARIANT(my_cleaning_epoch != local_epoch);
#endif

  INVARIANT(local_cleaning_epoch == my_cleaning_epoch); // b/c we are out of RCU loop
  INVARIANT(EnableThreadLocalCleanup ||
            global_queues[local_cleaning_epoch % 2].empty());
  if (EnableThreadLocalCleanup) {
    global_queues[local_cleaning_epoch % 2].insert(
        global_queues[local_cleaning_epoch % 2].end(),
        s->local_queues[local_cleaning_epoch % 2].begin(),
        s->local_queues[local_cleaning_epoch % 2].end());
  }
  global_queues[local_epoch % 2].insert(
      global_queues[local_epoch % 2].end(),
      s->local_queues[local_epoch % 2].begin(),
      s->local_queues[local_epoch % 2].end());
  s->local_queues[0].clear();
  s->local_queues[1].clear();
  sync_map.erase(it);

  INVARIANT(cleaning_epoch == my_cleaning_epoch); // shouldn't change b/c we hold rcu_mutex()
  return s;
}

void
rcu::enable()
{
  if (likely(gc_thread_started))
    return;
  {
    lock_guard<spinlock> l(rcu_mutex());
    if (gc_thread_started)
      return;
    gc_thread_started = true;
  }
  // start gc thread as daemon thread
  pthread_attr_t attr;
  ALWAYS_ASSERT(pthread_attr_init(&attr) == 0);
  ALWAYS_ASSERT(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0);
  ALWAYS_ASSERT(pthread_create(&gc_thread_p, &attr, gc_thread_loop, NULL) == 0);
  ALWAYS_ASSERT(pthread_attr_destroy(&attr) == 0);
}

void
rcu::region_begin()
{
  if (unlikely(!tl_sync)) {
    INVARIANT(!tl_crit_section_depth);
    enable();
    tl_sync = register_sync(pthread_self());
  }
  INVARIANT(tl_sync);
  INVARIANT(gc_thread_started);
  if (!tl_crit_section_depth++) {
    tl_sync->local_critical_mutex.lock();
    tl_sync->local_epoch = global_epoch;
  }
}

void
rcu::free_with_fn(void *p, deleter_t fn)
{
  INVARIANT(tl_sync);
  INVARIANT(tl_crit_section_depth);
  INVARIANT(gc_thread_started);
  tl_sync->local_queues[tl_sync->local_epoch % 2].push_back(delete_entry(p, fn));
  ++evt_rcu_frees;
}

void
rcu::region_end()
{
  INVARIANT(tl_sync);
  INVARIANT(tl_crit_section_depth);
  INVARIANT(gc_thread_started);
  if (!--tl_crit_section_depth) {
    const epoch_t my_global_epoch = global_epoch;
    INVARIANT(tl_sync->local_epoch == my_global_epoch ||
              tl_sync->local_epoch == (my_global_epoch - 1));
    if (EnableThreadLocalCleanup) {
      const epoch_t local_cleaning_epoch  = tl_sync->local_epoch - 1;
      const epoch_t global_cleaning_epoch = cleaning_epoch;
      if (!(local_cleaning_epoch == global_cleaning_epoch ||
            local_cleaning_epoch == global_cleaning_epoch + 1)) {
        cerr << "my_global_epoch      : " << my_global_epoch << endl;
        cerr << "local_cleaning_epoch : " << local_cleaning_epoch << endl;
        cerr << "global_cleaning_epoch: " << global_cleaning_epoch << endl;
        cerr << "cur_global_epoch     : " << global_epoch << endl;
        INVARIANT(false);
      }
      INVARIANT(tl_sync->scratch_queue.empty());
      if (local_cleaning_epoch == global_cleaning_epoch &&
          !tl_sync->local_queues[local_cleaning_epoch % 2].empty()) {
        // reap locally, outside the critical section
        swap(tl_sync->local_queues[local_cleaning_epoch % 2],
             tl_sync->scratch_queue);
        ++evt_rcu_local_reaps;
      }
    }
    tl_sync->local_critical_mutex.unlock();
    delete_queue &q = tl_sync->scratch_queue;
    if (EnableThreadLocalCleanup && !q.empty()) {
      for (rcu::delete_queue::iterator it = q.begin(); it != q.end(); ++it) {
        try {
          it->second(it->first);
        } catch (...) {
          cerr << "rcu::region_end: uncaught exception in free routine" << endl;
        }
      }
      evt_rcu_deletes += q.size();
      evt_avg_rcu_local_delete_queue_len.offer(q.size());
      q.clear();
    }
  }
}

bool
rcu::in_rcu_region()
{
  return tl_crit_section_depth;
}

static const uint64_t rcu_epoch_us = 50 * 1000; /* 50 ms */
static const uint64_t rcu_epoch_ns = rcu_epoch_us * 1000;

class gc_reaper_thread : public ndb_thread {
public:
  gc_reaper_thread()
    : ndb_thread(true, "rcu-reaper")
  {
    queue.reserve(32000);
  }
  virtual void
  run()
  {
    struct timespec t;
    NDB_MEMSET(&t, 0, sizeof(t));
    t.tv_nsec = rcu_epoch_ns;
    rcu::delete_queue stack_queue;
    stack_queue.reserve(rcu::SyncDeleteQueueBufSize);
    for (;;) {
      // see if any elems to process
      {
        lock_guard<spinlock> l(lock);
        stack_queue.swap(queue);
      }
      evt_avg_gc_reaper_queue_len.offer(stack_queue.size());
      if (stack_queue.empty())
        nanosleep(&t, NULL);
      for (rcu::delete_queue::iterator it = stack_queue.begin();
           it != stack_queue.end(); ++it) {
        try {
          it->second(it->first);
        } catch (...) {
          cerr << "rcu-reaper: uncaught exception in free routine" << endl;
        }
      }
      evt_rcu_deletes += stack_queue.size();
      stack_queue.clear();
    }
  }

  void
  reap_and_clear(rcu::delete_queue &local_queue)
  {
    lock_guard<spinlock> l0(lock);
    if (queue.empty()) {
      queue.swap(local_queue);
    } else {
      queue.insert(
          queue.end(),
          local_queue.begin(),
          local_queue.end());
      local_queue.clear();
    }
    INVARIANT(local_queue.empty());
  }

  spinlock lock;
  rcu::delete_queue queue;
};

void *
rcu::gc_thread_loop(void *p)
{
  // runs as daemon thread
  struct timespec t;
  NDB_MEMSET(&t, 0, sizeof(t));
  timer loop_timer;
  static gc_reaper_thread reaper_loops[NGCReapers];
  for (unsigned int i = 0; i < NGCReapers; i++)
    reaper_loops[i].start();
  unsigned int rr = 0;
  for (;;) {

    const uint64_t last_loop_usec = loop_timer.lap();
    const uint64_t delay_time_usec = rcu_epoch_us;
    if (last_loop_usec < delay_time_usec) {
      t.tv_nsec = (delay_time_usec - last_loop_usec) * 1000;
      nanosleep(&t, NULL);
    }

    // increment global epoch
    INVARIANT(cleaning_epoch + 1 == global_epoch);
    COMPILER_MEMORY_FENCE;
    const epoch_t new_cleaning_epoch = global_epoch++;
    INVARIANT(cleaning_epoch + 1 == new_cleaning_epoch);
    __sync_synchronize();

    // now wait for each thread to finish any outstanding critical sections
    // from the previous epoch, and advance it forward to the global epoch
    {
      lock_guard<spinlock> l(rcu_mutex()); // prevents new threads from joining

      // force all threads to advance to new epoch
      for (map<pthread_t, sync *>::iterator it = sync_map.begin();
           it != sync_map.end(); ++it) {
        sync * const s = it->second;
        const epoch_t local_epoch = s->local_epoch;
        if (local_epoch != global_epoch) {
          INVARIANT(local_epoch == new_cleaning_epoch);
          lock_guard<spinlock> l0(s->local_critical_mutex);
          if (s->local_epoch == global_epoch)
            // has moved on, so we cannot safely reap
            // like we do below
            continue;
          INVARIANT(s->local_epoch == new_cleaning_epoch);
          if (EnableThreadLocalCleanup) {
            // if we haven't completely finished thread-local
            // cleanup, then we move the pointers into the
            // gc reapers
            delete_queue &local_queue = s->local_queues[global_epoch % 2];
            if (!local_queue.empty())
              reaper_loops[rr++ % NGCReapers].reap_and_clear(local_queue);
          }
          INVARIANT(s->local_queues[global_epoch % 2].empty());
          s->local_epoch = global_epoch;
        }
      }

      COMPILER_MEMORY_FENCE;
      cleaning_epoch = new_cleaning_epoch;
      __sync_synchronize();

      if (!EnableThreadLocalCleanup) {
        // claim deleted pointers
        for (map<pthread_t, sync *>::iterator it = sync_map.begin();
             it != sync_map.end(); ++it) {
          sync * const s = it->second;
          INVARIANT(s->local_epoch == global_epoch);

          // so we can now claim its deleted pointers from global_epoch - 1
          delete_queue &local_queue = s->local_queues[cleaning_epoch % 2];
          evt_avg_rcu_delete_queue_len.offer(local_queue.size());
          reaper_loops[rr++ % NGCReapers].reap_and_clear(local_queue);
          INVARIANT(local_queue.empty());
        }
      }

      // pull the ones from the global queue
      if (EnableThreadLocalCleanup) {
        delete_queue &q = global_queues[global_epoch % 2];
        if (!q.empty())
          reaper_loops[rr++ % NGCReapers].reap_and_clear(q);
      }
      INVARIANT(global_queues[global_epoch % 2].empty());
      delete_queue &global_queue = global_queues[cleaning_epoch % 2];
      reaper_loops[rr++ % NGCReapers].reap_and_clear(global_queue);
    }
  }
  return NULL;
}

static void rcu_completion_callback(ndb_thread *t)
{
  rcu::sync *s = rcu::unregister_sync(t->pthread_id());
  if (s)
    delete s;
}
NDB_THREAD_REGISTER_COMPLETION_CALLBACK(rcu_completion_callback)
<commit_msg>rcu: more assertions<commit_after>#include <unistd.h>
#include <time.h>
#include <string.h>
#include <iostream>

#include "rcu.h"
#include "macros.h"
#include "util.h"
#include "thread.h"
#include "counter.h"
#include "lockguard.h"

using namespace std;
using namespace util;

// avoid some corner cases in the beginning
volatile rcu::epoch_t rcu::global_epoch = 1;
volatile rcu::epoch_t rcu::cleaning_epoch = 0;
rcu::delete_queue rcu::global_queues[2];

volatile bool rcu::gc_thread_started = false;
pthread_t rcu::gc_thread_p;

map<pthread_t, rcu::sync *> rcu::sync_map;

__thread rcu::sync *rcu::tl_sync = NULL;
__thread unsigned int rcu::tl_crit_section_depth = 0;

static event_counter evt_rcu_deletes("rcu_deletes");
static event_counter evt_rcu_frees("rcu_frees");
static event_counter evt_rcu_local_reaps("rcu_local_reaps");

static event_avg_counter evt_avg_gc_reaper_queue_len("avg_gc_reaper_queue_len");
static event_avg_counter evt_avg_rcu_delete_queue_len("avg_rcu_delete_queue_len");
static event_avg_counter evt_avg_rcu_local_delete_queue_len("avg_rcu_local_delete_queue_len");

spinlock &
rcu::rcu_mutex()
{
  static spinlock s_lock;
  return s_lock;
}

rcu::sync *
rcu::register_sync(pthread_t p)
{
  lock_guard<spinlock> l(rcu_mutex());
  map<pthread_t, sync *>::iterator it = sync_map.find(p);
  ALWAYS_ASSERT(it == sync_map.end());
  return (sync_map[p] = new sync(global_epoch));
}

rcu::sync *
rcu::unregister_sync(pthread_t p)
{
  lock_guard<spinlock> l(rcu_mutex());
  map<pthread_t, sync *>::iterator it = sync_map.find(p);
  if (it == sync_map.end())
    return NULL;
  sync * const s = it->second;

  const epoch_t my_cleaning_epoch = cleaning_epoch;
  const epoch_t local_epoch = s->local_epoch;
  const epoch_t local_cleaning_epoch = s->local_epoch - 1;

#ifdef CHECK_INVARIANTS
  const epoch_t my_global_epoch = global_epoch;
  INVARIANT(my_global_epoch == local_epoch ||
            my_global_epoch == (local_epoch + 1));
  INVARIANT(my_global_epoch == (my_cleaning_epoch + 1) ||
            my_global_epoch == (my_cleaning_epoch + 2));
  INVARIANT(my_cleaning_epoch != local_epoch);
  if (local_cleaning_epoch != my_cleaning_epoch) {
    cerr << "my_cleaning_epoch: " << my_cleaning_epoch << endl;
    cerr << "local_epoch: " << local_epoch << endl;
    cerr << "local_cleaning_epoch: " << local_cleaning_epoch << endl;
    cerr << "my_global_epoch: " << my_global_epoch << endl;
  }
#endif

  INVARIANT(cleaning_epoch == my_cleaning_epoch);
  INVARIANT(EnableThreadLocalCleanup ||
            global_queues[local_cleaning_epoch % 2].empty());
  if (EnableThreadLocalCleanup) {
    global_queues[local_cleaning_epoch % 2].insert(
        global_queues[local_cleaning_epoch % 2].end(),
        s->local_queues[local_cleaning_epoch % 2].begin(),
        s->local_queues[local_cleaning_epoch % 2].end());
  }
  global_queues[local_epoch % 2].insert(
      global_queues[local_epoch % 2].end(),
      s->local_queues[local_epoch % 2].begin(),
      s->local_queues[local_epoch % 2].end());
  s->local_queues[0].clear();
  s->local_queues[1].clear();
  sync_map.erase(it);

  INVARIANT(cleaning_epoch == my_cleaning_epoch); // shouldn't change b/c we hold rcu_mutex()
  return s;
}

void
rcu::enable()
{
  if (likely(gc_thread_started))
    return;
  {
    lock_guard<spinlock> l(rcu_mutex());
    if (gc_thread_started)
      return;
    gc_thread_started = true;
  }
  // start gc thread as daemon thread
  pthread_attr_t attr;
  ALWAYS_ASSERT(pthread_attr_init(&attr) == 0);
  ALWAYS_ASSERT(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0);
  ALWAYS_ASSERT(pthread_create(&gc_thread_p, &attr, gc_thread_loop, NULL) == 0);
  ALWAYS_ASSERT(pthread_attr_destroy(&attr) == 0);
}

void
rcu::region_begin()
{
  if (unlikely(!tl_sync)) {
    INVARIANT(!tl_crit_section_depth);
    enable();
    tl_sync = register_sync(pthread_self());
  }
  INVARIANT(tl_sync);
  INVARIANT(gc_thread_started);
  if (!tl_crit_section_depth++) {
    tl_sync->local_critical_mutex.lock();
    tl_sync->local_epoch = global_epoch;
    INVARIANT(tl_sync->local_epoch != cleaning_epoch);
  }
}

void
rcu::free_with_fn(void *p, deleter_t fn)
{
  INVARIANT(tl_sync);
  INVARIANT(tl_crit_section_depth);
  INVARIANT(gc_thread_started);
  tl_sync->local_queues[tl_sync->local_epoch % 2].push_back(delete_entry(p, fn));
  ++evt_rcu_frees;
#ifdef CHECK_INVARIANTS
  const epoch_t my_global_epoch = global_epoch;
  const epoch_t global_cleaning_epoch = cleaning_epoch;
  const epoch_t local_cleaning_epoch  = tl_sync->local_epoch - 1;
  if (!(local_cleaning_epoch == global_cleaning_epoch ||
        local_cleaning_epoch == (global_cleaning_epoch + 1))) {
    cerr << "my_global_epoch      : " << my_global_epoch << endl;
    cerr << "local_cleaning_epoch : " << local_cleaning_epoch << endl;
    cerr << "global_cleaning_epoch: " << global_cleaning_epoch << endl;
    cerr << "cur_global_epoch     : " << global_epoch << endl;
    INVARIANT(false);
  }
#endif
}

void
rcu::region_end()
{
  INVARIANT(tl_sync);
  INVARIANT(tl_crit_section_depth);
  INVARIANT(gc_thread_started);
  if (!--tl_crit_section_depth) {
    const epoch_t my_global_epoch = global_epoch;
    INVARIANT(tl_sync->local_epoch == my_global_epoch ||
              tl_sync->local_epoch == (my_global_epoch - 1));
    if (EnableThreadLocalCleanup) {
      const epoch_t local_cleaning_epoch  = tl_sync->local_epoch - 1;
      const epoch_t global_cleaning_epoch = cleaning_epoch;
#ifdef CHECK_INVARIANTS
      INVARIANT(tl_sync->local_epoch != global_cleaning_epoch);
      if (!(local_cleaning_epoch == global_cleaning_epoch ||
            local_cleaning_epoch == (global_cleaning_epoch + 1))) {
        cerr << "my_global_epoch      : " << my_global_epoch << endl;
        cerr << "local_cleaning_epoch : " << local_cleaning_epoch << endl;
        cerr << "global_cleaning_epoch: " << global_cleaning_epoch << endl;
        cerr << "cur_global_epoch     : " << global_epoch << endl;
        INVARIANT(false);
      }
#endif
      INVARIANT(tl_sync->scratch_queue.empty());
      if (local_cleaning_epoch == global_cleaning_epoch &&
          !tl_sync->local_queues[local_cleaning_epoch % 2].empty()) {
        // reap locally, outside the critical section
        swap(tl_sync->local_queues[local_cleaning_epoch % 2],
             tl_sync->scratch_queue);
        ++evt_rcu_local_reaps;
      }
    }
    tl_sync->local_critical_mutex.unlock();
    delete_queue &q = tl_sync->scratch_queue;
    if (EnableThreadLocalCleanup && !q.empty()) {
      for (rcu::delete_queue::iterator it = q.begin(); it != q.end(); ++it) {
        try {
          it->second(it->first);
        } catch (...) {
          cerr << "rcu::region_end: uncaught exception in free routine" << endl;
        }
      }
      evt_rcu_deletes += q.size();
      evt_avg_rcu_local_delete_queue_len.offer(q.size());
      q.clear();
    }
  }
}

bool
rcu::in_rcu_region()
{
  return tl_crit_section_depth;
}

static const uint64_t rcu_epoch_us = 50 * 1000; /* 50 ms */
static const uint64_t rcu_epoch_ns = rcu_epoch_us * 1000;

class gc_reaper_thread : public ndb_thread {
public:
  gc_reaper_thread()
    : ndb_thread(true, "rcu-reaper")
  {
    queue.reserve(32000);
  }
  virtual void
  run()
  {
    struct timespec t;
    NDB_MEMSET(&t, 0, sizeof(t));
    t.tv_nsec = rcu_epoch_ns;
    rcu::delete_queue stack_queue;
    stack_queue.reserve(rcu::SyncDeleteQueueBufSize);
    for (;;) {
      // see if any elems to process
      {
        lock_guard<spinlock> l(lock);
        stack_queue.swap(queue);
      }
      evt_avg_gc_reaper_queue_len.offer(stack_queue.size());
      if (stack_queue.empty())
        nanosleep(&t, NULL);
      for (rcu::delete_queue::iterator it = stack_queue.begin();
           it != stack_queue.end(); ++it) {
        try {
          it->second(it->first);
        } catch (...) {
          cerr << "rcu-reaper: uncaught exception in free routine" << endl;
        }
      }
      evt_rcu_deletes += stack_queue.size();
      stack_queue.clear();
    }
  }

  void
  reap_and_clear(rcu::delete_queue &local_queue)
  {
    lock_guard<spinlock> l0(lock);
    if (queue.empty()) {
      queue.swap(local_queue);
    } else {
      queue.insert(
          queue.end(),
          local_queue.begin(),
          local_queue.end());
      local_queue.clear();
    }
    INVARIANT(local_queue.empty());
  }

  spinlock lock;
  rcu::delete_queue queue;
};

void *
rcu::gc_thread_loop(void *p)
{
  // runs as daemon thread
  struct timespec t;
  NDB_MEMSET(&t, 0, sizeof(t));
  timer loop_timer;
  static gc_reaper_thread reaper_loops[NGCReapers];
  for (unsigned int i = 0; i < NGCReapers; i++)
    reaper_loops[i].start();
  unsigned int rr = 0;
  for (;;) {

    const uint64_t last_loop_usec = loop_timer.lap();
    const uint64_t delay_time_usec = rcu_epoch_us;
    if (last_loop_usec < delay_time_usec) {
      t.tv_nsec = (delay_time_usec - last_loop_usec) * 1000;
      nanosleep(&t, NULL);
    }

    // increment global epoch
    INVARIANT(cleaning_epoch + 1 == global_epoch);
    COMPILER_MEMORY_FENCE;
    const epoch_t new_cleaning_epoch = global_epoch++;
    INVARIANT(cleaning_epoch + 1 == new_cleaning_epoch);
    __sync_synchronize();

    // now wait for each thread to finish any outstanding critical sections
    // from the previous epoch, and advance it forward to the global epoch
    {
      lock_guard<spinlock> l(rcu_mutex()); // prevents new threads from joining

      // force all threads to advance to new epoch
      for (map<pthread_t, sync *>::iterator it = sync_map.begin();
           it != sync_map.end(); ++it) {
        sync * const s = it->second;
        const epoch_t local_epoch = s->local_epoch;
        if (local_epoch != global_epoch) {
          INVARIANT(local_epoch == new_cleaning_epoch);
          lock_guard<spinlock> l0(s->local_critical_mutex);
          if (s->local_epoch == global_epoch)
            // has moved on, so we cannot safely reap
            // like we do below
            continue;
          INVARIANT(s->local_epoch == new_cleaning_epoch);
          if (EnableThreadLocalCleanup) {
            // if we haven't completely finished thread-local
            // cleanup, then we move the pointers into the
            // gc reapers
            delete_queue &local_queue = s->local_queues[global_epoch % 2];
            if (!local_queue.empty())
              reaper_loops[rr++ % NGCReapers].reap_and_clear(local_queue);
          }
          INVARIANT(s->local_queues[global_epoch % 2].empty());
          s->local_epoch = global_epoch;
        }
      }

      COMPILER_MEMORY_FENCE;
      cleaning_epoch = new_cleaning_epoch;
      __sync_synchronize();

      if (!EnableThreadLocalCleanup) {
        // claim deleted pointers
        for (map<pthread_t, sync *>::iterator it = sync_map.begin();
             it != sync_map.end(); ++it) {
          sync * const s = it->second;
          INVARIANT(s->local_epoch == global_epoch);

          // so we can now claim its deleted pointers from global_epoch - 1
          delete_queue &local_queue = s->local_queues[cleaning_epoch % 2];
          evt_avg_rcu_delete_queue_len.offer(local_queue.size());
          reaper_loops[rr++ % NGCReapers].reap_and_clear(local_queue);
          INVARIANT(local_queue.empty());
        }
      }

      // pull the ones from the global queue
      if (EnableThreadLocalCleanup) {
        delete_queue &q = global_queues[global_epoch % 2];
        if (!q.empty())
          reaper_loops[rr++ % NGCReapers].reap_and_clear(q);
      }
      INVARIANT(global_queues[global_epoch % 2].empty());
      delete_queue &global_queue = global_queues[cleaning_epoch % 2];
      reaper_loops[rr++ % NGCReapers].reap_and_clear(global_queue);
    }
  }
  return NULL;
}

static void rcu_completion_callback(ndb_thread *t)
{
  rcu::sync *s = rcu::unregister_sync(t->pthread_id());
  if (s)
    delete s;
}
NDB_THREAD_REGISTER_COMPLETION_CALLBACK(rcu_completion_callback)
<|endoftext|>


<commit_before>/*
 * BTreeTest.cpp
 *
 *  Created on: Dec 30, 2013
 *      Author: airekans
 */

#include "BTree.h"
#include <gtest/gtest.h>
#include <memory>

using snippet::algo::BTree;

TEST(BTree, TestCtor)
{
    BTree btree(9);

    ASSERT_EQ(static_cast<unsigned>(0), btree.GetSize());
}

TEST(BTree, TestInsert)
{
    BTree btree(9);

    btree.Insert(1, "a");

    ASSERT_TRUE(btree.Find(1, NULL, NULL));
    ASSERT_EQ(static_cast<unsigned>(1), btree.GetSize());

    btree.Insert(2, "b");
    btree.Dump();

    ASSERT_TRUE(btree.Find(1, NULL, NULL));
    ASSERT_TRUE(btree.Find(2, NULL, NULL));
    ASSERT_EQ(static_cast<unsigned>(2), btree.GetSize());
}

class BTreeValueTest : public ::testing::TestWithParam<int>
{};

TEST_P(BTreeValueTest, TestInsertWithLeave)
{
    const int times = GetParam();
    BTree btree(5);

    for (int i = 0; i < times; ++i)
    {
        btree.Insert(i, "a");
    }

    ASSERT_EQ(static_cast<unsigned>(times), btree.GetSize());
    for (int i = 0; i < times; ++i)
    {
        EXPECT_TRUE(btree.Find(i, NULL, NULL)) << "i: " << i;
    }
}

INSTANTIATE_TEST_CASE_P(TestBTreeInsert,
                        BTreeValueTest,
                        ::testing::Range(3, 21));

TEST(BTree, TestFind)
{
    BTree btree(9);

    btree.Insert(1, "a");
    btree.Insert(2, "b");

    ASSERT_TRUE(btree.Find(1, NULL, NULL));
    ASSERT_TRUE(btree.Find(2, NULL, NULL));
    ASSERT_FALSE(btree.Find(3, NULL, NULL));
}
<commit_msg>Add random test for insert.<commit_after>/*
 * BTreeTest.cpp
 *
 *  Created on: Dec 30, 2013
 *      Author: airekans
 */

#include "BTree.h"
#include <gtest/gtest.h>
#include <memory>
#include <ctime>
#include <cstdlib>
#include <algorithm>
#include <vector>
#include <string>
#include <sstream>

using snippet::algo::BTree;

TEST(BTree, TestCtor)
{
    BTree btree(9);

    ASSERT_EQ(static_cast<unsigned>(0), btree.GetSize());
}

TEST(BTree, TestInsert)
{
    BTree btree(9);

    btree.Insert(1, "a");

    ASSERT_TRUE(btree.Find(1, NULL, NULL));
    ASSERT_EQ(static_cast<unsigned>(1), btree.GetSize());

    btree.Insert(2, "b");
    btree.Dump();

    ASSERT_TRUE(btree.Find(1, NULL, NULL));
    ASSERT_TRUE(btree.Find(2, NULL, NULL));
    ASSERT_EQ(static_cast<unsigned>(2), btree.GetSize());
}

class BTreeValueTest : public ::testing::TestWithParam<int>
{
public:
    static void SetUpTestCase()
    {
        srand(unsigned(time(NULL)));
    }
};

TEST_P(BTreeValueTest, TestInsertWithLeave)
{
    const int times = GetParam();
    BTree btree(5);

    for (int i = 0; i < times; ++i)
    {
        btree.Insert(i, "a");
    }

    ASSERT_EQ(static_cast<unsigned>(times), btree.GetSize());
    for (int i = 0; i < times; ++i)
    {
        EXPECT_TRUE(btree.Find(i, NULL, NULL)) << "i: " << i;
    }
}

TEST_P(BTreeValueTest, TestInsertWithReverseOrder)
{
    const int times = GetParam();
    BTree btree(5);

    for (int i = times; i > 0; --i)
    {
        btree.Insert(i, "a");
    }

    ASSERT_EQ(static_cast<unsigned>(times), btree.GetSize());
    for (int i = times; i > 0; --i)
    {
        EXPECT_TRUE(btree.Find(i, NULL, NULL)) << "i: " << i;
    }
}

namespace {
    template<typename T>
    std::string VectorToString(const std::vector<T>& v)
    {
        if (v.empty())
        {
            return "";
        }

        std::ostringstream oss;
        for (unsigned i = 0; i < v.size() - 1; ++i)
        {
            oss << v[i] << ' ';
        }
        oss << v.back();
        return oss.str();
    }
}

TEST_P(BTreeValueTest, TestInsertWithRandomOrder)
{
    const int times = GetParam();
    BTree btree(5);

    // generate a vector<int> with size times
    std::vector<int> numbers;
    for (int i = 0; i < times; ++i)
    {
        numbers.push_back(i);
    }
    std::random_shuffle(numbers.begin(), numbers.end());

    for (int i = 0; i < times; ++i)
    {
        btree.Insert(numbers[i], "a");
    }

    ASSERT_EQ(static_cast<unsigned>(times), btree.GetSize())
        << VectorToString(numbers);
    for (int i = 0; i < times; ++i)
    {
        EXPECT_TRUE(btree.Find(i, NULL, NULL)) << "i: " << i;
    }
}

INSTANTIATE_TEST_CASE_P(TestBTreeInsert,
                        BTreeValueTest,
                        ::testing::Range(3, 21));

TEST(BTree, TestFind)
{
    BTree btree(9);

    btree.Insert(1, "a");
    btree.Insert(2, "b");

    ASSERT_TRUE(btree.Find(1, NULL, NULL));
    ASSERT_TRUE(btree.Find(2, NULL, NULL));
    ASSERT_FALSE(btree.Find(3, NULL, NULL));
}
<|endoftext|>


<commit_before>/** \file    convert_rss_db_tables.cc
 *  \brief   Transfers data from ub_tools.rss_aggregator to vufind.tuefind_rss_items.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2021 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <set>
#include <unordered_map>
#include <cstdlib>
#include "DbConnection.h"
#include "DbResultSet.h"
#include "DbRow.h"
#include "StlHelpers.h"
#include "StringUtil.h"
#include "util.h"
#include "VuFind.h"

namespace {

struct FeedInfo {
    std::string id_;
    std::set<std::string> subsystem_types_;
public:
    FeedInfo() = default;
    FeedInfo(const FeedInfo &rhs) = default;
    FeedInfo(const std::string &id, const std::set<std::string> &subsystem_types)
        : id_(id), subsystem_types_(subsystem_types) { }
    inline bool isCompatibleWith(const std::string &subsystem_type) const
        { return subsystem_types_.find(subsystem_type) != subsystem_types_.cend(); }
};

bool GetRSSFeedsID(DbConnection * vufind_connection, const std::string &feed_url, FeedInfo * const feed_info) {
    static std::unordered_map<std::string, FeedInfo> urls_to_feed_infos_map;
    const auto url_and_feed_info(urls_to_feed_infos_map.find(feed_url));
    if (url_and_feed_info != urls_to_feed_infos_map.end()) {
        *feed_info = url_and_feed_info->second;
        return true;
    }

    vufind_connection->queryOrDie("SELECT id,subsystem_types FROM tuefind_rss_feeds WHERE feed_url="
                                  + vufind_connection->escapeAndQuoteString(feed_url));
    auto result_set(vufind_connection->getLastResultSet());
    if (result_set.empty()) {
        static std::unordered_set<std::string> known_missing;
        if (known_missing.find(feed_url) == known_missing.end()) {
            LOG_WARNING("found no tuefind_rss_feeds.id for \"" + feed_url + "\"!");
            known_missing.emplace(feed_url);
        }
        return false;
    }

    const auto row(result_set.getNextRow());
    std::vector<std::string> subsystem_types;
    StringUtil::Split(row["subsystem_types"], ',', &subsystem_types);
    *feed_info = FeedInfo(row["id"], StlHelpers::VectorToSet(subsystem_types));
    urls_to_feed_infos_map[feed_url] = *feed_info;
    return true;
}

void CopyItem(DbConnection * const db_writer, const std::string &feed_id, const std::string &item_id,
              const std::string &item_url, const std::string &item_title, const std::string &item_description,
              const std::string &pub_date, const std::string &insertion_time)
{
    db_writer->queryOrDie("INSERT INTO tuefind_rss_items SET rss_feeds_id=" + feed_id + ",item_id='" + item_id
                          + "',item_url=" + db_writer->escapeAndQuoteString(item_url) + ",item_title="
                          + db_writer->escapeAndQuoteString(item_title) + ",item_description="
                          + db_writer->escapeAndQuoteString(item_description) + ",pub_date='"
                          + pub_date + "',insertion_time='" + insertion_time + "'");
}

} // unnamed namespace

int Main(int /*argc*/, char */*argv*/[]) {
    auto db_reader((DbConnection()));
    auto db_writer(VuFind::GetDbConnection());

    db_reader.queryOrDie("SELECT * FROM rss_aggregator");
    auto result_set(db_reader.getLastResultSet());
    DbTransaction transaction(db_writer.get());
    while (const auto row = result_set.getNextRow()) {
        FeedInfo feed_info;
        if (not GetRSSFeedsID(db_writer.get(), row["feed_url"], &feed_info)) {
            transaction.rollback();
            LOG_WARNING("Undid partial conversion w/ a ROLLBACK!");
            return EXIT_FAILURE;
        }

        if (unlikely(not feed_info.isCompatibleWith(row["flavour"])))
            LOG_ERROR("Item w/ item_id \"" + row["item_id"] + " has a flavour \"" + row["flavour"] +
                      "\" which is incompatible with the subsystem_types \""
                      + StlHelpers::ContainerToString(feed_info.subsystem_types_.cbegin(),
                                                      feed_info.subsystem_types_.cend(), ","));
        CopyItem(db_writer.get(), feed_info.id_, row["item_id"], row["item_url"], row["item_title"],
                 row["item_description"], row["pub_date"], row["insertion_time"]);
    }
    transaction.commit();
    db_reader.queryOrDie("DROP TABLE IF EXISTS ub_tools.rss_aggregator");

    return EXIT_SUCCESS;
}
<commit_msg>convert_rss_db_tables.cc: Only migrate tables if VuFind is installed, else just drop rss_aggregator table.<commit_after>/** \file    convert_rss_db_tables.cc
 *  \brief   Transfers data from ub_tools.rss_aggregator to vufind.tuefind_rss_items.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2021 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <set>
#include <unordered_map>
#include <cstdlib>
#include "DbConnection.h"
#include "DbResultSet.h"
#include "DbRow.h"
#include "StlHelpers.h"
#include "StringUtil.h"
#include "util.h"
#include "VuFind.h"

namespace {

struct FeedInfo {
    std::string id_;
    std::set<std::string> subsystem_types_;
public:
    FeedInfo() = default;
    FeedInfo(const FeedInfo &rhs) = default;
    FeedInfo(const std::string &id, const std::set<std::string> &subsystem_types)
        : id_(id), subsystem_types_(subsystem_types) { }
    inline bool isCompatibleWith(const std::string &subsystem_type) const
        { return subsystem_types_.find(subsystem_type) != subsystem_types_.cend(); }
};

bool GetRSSFeedsID(DbConnection * vufind_connection, const std::string &feed_url, FeedInfo * const feed_info) {
    static std::unordered_map<std::string, FeedInfo> urls_to_feed_infos_map;
    const auto url_and_feed_info(urls_to_feed_infos_map.find(feed_url));
    if (url_and_feed_info != urls_to_feed_infos_map.end()) {
        *feed_info = url_and_feed_info->second;
        return true;
    }

    vufind_connection->queryOrDie("SELECT id,subsystem_types FROM tuefind_rss_feeds WHERE feed_url="
                                  + vufind_connection->escapeAndQuoteString(feed_url));
    auto result_set(vufind_connection->getLastResultSet());
    if (result_set.empty()) {
        static std::unordered_set<std::string> known_missing;
        if (known_missing.find(feed_url) == known_missing.end()) {
            LOG_WARNING("found no tuefind_rss_feeds.id for \"" + feed_url + "\"!");
            known_missing.emplace(feed_url);
        }
        return false;
    }

    const auto row(result_set.getNextRow());
    std::vector<std::string> subsystem_types;
    StringUtil::Split(row["subsystem_types"], ',', &subsystem_types);
    *feed_info = FeedInfo(row["id"], StlHelpers::VectorToSet(subsystem_types));
    urls_to_feed_infos_map[feed_url] = *feed_info;
    return true;
}

void CopyItem(DbConnection * const db_writer, const std::string &feed_id, const std::string &item_id,
              const std::string &item_url, const std::string &item_title, const std::string &item_description,
              const std::string &pub_date, const std::string &insertion_time)
{
    db_writer->queryOrDie("INSERT INTO tuefind_rss_items SET rss_feeds_id=" + feed_id + ",item_id='" + item_id
                          + "',item_url=" + db_writer->escapeAndQuoteString(item_url) + ",item_title="
                          + db_writer->escapeAndQuoteString(item_title) + ",item_description="
                          + db_writer->escapeAndQuoteString(item_description) + ",pub_date='"
                          + pub_date + "',insertion_time='" + insertion_time + "'");
}

} // unnamed namespace

int Main(int /*argc*/, char */*argv*/[]) {
    auto db_reader((DbConnection()));

    if (not VuFind::GetTueFindFlavour().empty()) {
        auto db_writer(VuFind::GetDbConnection());
        db_reader.queryOrDie("SELECT * FROM rss_aggregator");
        auto result_set(db_reader.getLastResultSet());
        DbTransaction transaction(db_writer.get());
        while (const auto row = result_set.getNextRow()) {
            FeedInfo feed_info;
            if (not GetRSSFeedsID(db_writer.get(), row["feed_url"], &feed_info)) {
                transaction.rollback();
                LOG_WARNING("Undid partial conversion w/ a ROLLBACK!");
                return EXIT_FAILURE;
            }

            if (unlikely(not feed_info.isCompatibleWith(row["flavour"])))
                LOG_ERROR("Item w/ item_id \"" + row["item_id"] + " has a flavour \"" + row["flavour"] +
                          "\" which is incompatible with the subsystem_types \""
                          + StlHelpers::ContainerToString(feed_info.subsystem_types_.cbegin(),
                                                          feed_info.subsystem_types_.cend(), ","));
            CopyItem(db_writer.get(), feed_info.id_, row["item_id"], row["item_url"], row["item_title"],
                     row["item_description"], row["pub_date"], row["insertion_time"]);
        }
        transaction.commit();
    }
    db_reader.queryOrDie("DROP TABLE IF EXISTS ub_tools.rss_aggregator");

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/* -*- c-basic-offset: 2 -*- */
/*
  Copyright(C) 2015-2017 Kouhei Sutou <kou@clear-code.com>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include "mrn_context_pool.hpp"
#include "mrn_lock.hpp"

#include <time.h>

namespace mrn {
  // for debug
#define MRN_CLASS_NAME "mrn::ContextPool::Impl"

  class ContextPool::Impl {
  public:
    Impl(mysql_mutex_t *mutex,
         long *n_pooling_contexts)
      : mutex_(mutex),
        n_pooling_contexts_(n_pooling_contexts),
        pool_(NULL),
        last_pull_time_(0) {
    }

    ~Impl(void) {
      clear();
    }

    grn_ctx *pull(void) {
      MRN_DBUG_ENTER_METHOD();
      grn_ctx *ctx = NULL;

      {
        time_t now;
        time(&now);

        mrn::Lock lock(mutex_);
        if (pool_) {
          ctx = static_cast<grn_ctx *>(pool_->data);
          list_pop(pool_);
          if ((now - last_pull_time_) >= CLEAR_THREATHOLD_IN_SECONDS) {
            clear_without_lock();
          }
        }
        last_pull_time_ = now;
      }

      if (!ctx) {
        mrn::Lock lock(mutex_);
        ctx = grn_ctx_open(0);
        ++(*n_pooling_contexts_);
      }

      DBUG_RETURN(ctx);
    }

    void release(grn_ctx *ctx) {
      MRN_DBUG_ENTER_METHOD();

      {
        mrn::Lock lock(mutex_);
        pool_ = list_cons(ctx, pool_);
        grn_ctx_use(ctx, NULL);
      }

      DBUG_VOID_RETURN;
    }

    void clear(void) {
      MRN_DBUG_ENTER_METHOD();

      {
        mrn::Lock lock(mutex_);
        clear_without_lock();
      }

      DBUG_VOID_RETURN;
    }

  private:
    static const unsigned int CLEAR_THREATHOLD_IN_SECONDS = 60 * 5;

    mysql_mutex_t *mutex_;
    long *n_pooling_contexts_;
    LIST *pool_;
    time_t last_pull_time_;

    void clear_without_lock(void) {
      MRN_DBUG_ENTER_METHOD();
      while (pool_) {
        grn_ctx *ctx = static_cast<grn_ctx *>(pool_->data);
        grn_ctx_close(ctx);
        list_pop(pool_);
        --(*n_pooling_contexts_);
      }
      DBUG_VOID_RETURN;
    }
  };

// For debug
#undef MRN_CLASS_NAME
#define MRN_CLASS_NAME "mrn::ContextPool"

  ContextPool::ContextPool(mysql_mutex_t *mutex,
                           long *n_pooling_contexts)
    : impl_(new Impl(mutex, n_pooling_contexts)) {
  }

  ContextPool::~ContextPool(void) {
    delete impl_;
  }

  grn_ctx *ContextPool::pull(void) {
    MRN_DBUG_ENTER_METHOD();
    grn_ctx *ctx = impl_->pull();
    DBUG_RETURN(ctx);
  }

  void ContextPool::release(grn_ctx *ctx) {
    MRN_DBUG_ENTER_METHOD();
    impl_->release(ctx);
    DBUG_VOID_RETURN;
  }

  void ContextPool::clear(void) {
    MRN_DBUG_ENTER_METHOD();
    impl_->clear();
    DBUG_VOID_RETURN;
  }
}
<commit_msg>mysql8.0: use std::vector instead of LIST<commit_after>/* -*- c-basic-offset: 2 -*- */
/*
  Copyright(C) 2015-2020 Sutou Kouhei <kou@clear-code.com>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include "mrn_context_pool.hpp"
#include "mrn_lock.hpp"

#include <time.h>
#include <vector>

namespace mrn {
  // for debug
#define MRN_CLASS_NAME "mrn::ContextPool::Impl"

  class ContextPool::Impl {
  public:
    Impl(mysql_mutex_t *mutex,
         long *n_pooling_contexts)
      : mutex_(mutex),
        n_pooling_contexts_(n_pooling_contexts),
        pool_(),
        last_pull_time_(0) {
    }

    ~Impl(void) {
      clear();
    }

    grn_ctx *pull(void) {
      MRN_DBUG_ENTER_METHOD();
      grn_ctx *ctx = NULL;

      {
        time_t now;
        time(&now);

        mrn::Lock lock(mutex_);
        if (!pool_.empty()) {
          ctx = pool_[pool_.size() - 1];
          pool_.pop_back();
          if ((now - last_pull_time_) >= CLEAR_THREATHOLD_IN_SECONDS) {
            clear_without_lock();
          }
        }
        last_pull_time_ = now;
      }

      if (!ctx) {
        mrn::Lock lock(mutex_);
        ctx = grn_ctx_open(0);
        ++(*n_pooling_contexts_);
      }

      DBUG_RETURN(ctx);
    }

    void release(grn_ctx *ctx) {
      MRN_DBUG_ENTER_METHOD();

      {
        mrn::Lock lock(mutex_);
        pool_.push_back(ctx);
        grn_ctx_use(ctx, NULL);
      }

      DBUG_VOID_RETURN;
    }

    void clear(void) {
      MRN_DBUG_ENTER_METHOD();

      {
        mrn::Lock lock(mutex_);
        clear_without_lock();
      }

      DBUG_VOID_RETURN;
    }

  private:
    static const unsigned int CLEAR_THREATHOLD_IN_SECONDS = 60 * 5;

    mysql_mutex_t *mutex_;
    long *n_pooling_contexts_;
    std::vector<grn_ctx *> pool_;
    time_t last_pull_time_;

    void clear_without_lock(void) {
      MRN_DBUG_ENTER_METHOD();
      for (grn_ctx *ctx : pool_) {
        grn_ctx_close(ctx);
        --(*n_pooling_contexts_);
      }
      pool_.clear();
      DBUG_VOID_RETURN;
    }
  };

// For debug
#undef MRN_CLASS_NAME
#define MRN_CLASS_NAME "mrn::ContextPool"

  ContextPool::ContextPool(mysql_mutex_t *mutex,
                           long *n_pooling_contexts)
    : impl_(new Impl(mutex, n_pooling_contexts)) {
  }

  ContextPool::~ContextPool(void) {
    delete impl_;
  }

  grn_ctx *ContextPool::pull(void) {
    MRN_DBUG_ENTER_METHOD();
    grn_ctx *ctx = impl_->pull();
    DBUG_RETURN(ctx);
  }

  void ContextPool::release(grn_ctx *ctx) {
    MRN_DBUG_ENTER_METHOD();
    impl_->release(ctx);
    DBUG_VOID_RETURN;
  }

  void ContextPool::clear(void) {
    MRN_DBUG_ENTER_METHOD();
    impl_->clear();
    DBUG_VOID_RETURN;
  }
}
<|endoftext|>


<commit_before>/*
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

#include "options.hpp"

#include <proton/connection.hpp>
#include <proton/connection_options.hpp>
#include <proton/container.hpp>
#include <proton/message.hpp>
#include <proton/message_id.hpp>
#include <proton/messaging_handler.hpp>
#include <proton/tracker.hpp>
#include <proton/types.hpp>

#include <iostream>
#include <map>

#include "fake_cpp11.hpp"

class simple_send : public proton::messaging_handler {
  private:
    std::string url;
    std::string user;
    std::string password;
    proton::sender sender;
    int sent;
    int confirmed;
    int total;

  public:
    simple_send(const std::string &s, const std::string &u, const std::string &p, int c) :
        url(s), user(u), password(p), sent(0), confirmed(0), total(c) {}

    void on_container_start(proton::container &c) OVERRIDE {
        proton::connection_options co;
        if (!user.empty()) co.user(user);
        if (!password.empty()) co.password(password);
        sender = c.open_sender(url, co);
    }

    void on_sendable(proton::sender &s) OVERRIDE {
        while (s.credit() && sent < total) {
            proton::message msg;
            std::map<std::string, int> m;
            m["sequence"] = sent + 1;

            msg.id(sent + 1);
            msg.body(m);

            s.send(msg);
            sent++;
        }
    }

    void on_tracker_accept(proton::tracker &t) OVERRIDE {
        confirmed++;

        if (confirmed == total) {
            std::cout << "all messages confirmed" << std::endl;
            t.connection().close();
        }
    }

    void on_transport_close(proton::transport &) OVERRIDE {
        sent = confirmed;
    }
};

int main(int argc, char **argv) {
    std::string address("127.0.0.1:5672/examples");
    std::string user;
    std::string password;
    int message_count = 100;
    example::options opts(argc, argv);

    opts.add_value(address, 'a', "address", "connect and send to URL", "URL");
    opts.add_value(message_count, 'm', "messages", "send COUNT messages", "COUNT");
    opts.add_value(user, 'u', "user", "authenticate as USER", "USER");
    opts.add_value(password, 'p', "password", "authenticate with PASSWORD", "PASSWORD");

    try {
        opts.parse();

        simple_send send(address, user, password, message_count);
        proton::container(send).run();

        return 0;
    } catch (const example::bad_option& e) {
        std::cout << opts << std::endl << e.what() << std::endl;
    } catch (const std::exception& e) {
        std::cerr << e.what() << std::endl;
    }

    return 1;
}
<commit_msg>PROTON-1959: [cpp] Update simple_send example to re-send on reconnect<commit_after>/*
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

#include "options.hpp"

#include <proton/connection.hpp>
#include <proton/connection_options.hpp>
#include <proton/container.hpp>
#include <proton/message.hpp>
#include <proton/message_id.hpp>
#include <proton/messaging_handler.hpp>
#include <proton/tracker.hpp>
#include <proton/types.hpp>

#include <iostream>
#include <map>

#include "fake_cpp11.hpp"

class simple_send : public proton::messaging_handler {
  private:
    std::string url;
    std::string user;
    std::string password;
    proton::sender sender;
    int sent;
    int confirmed;
    int total;

  public:
    simple_send(const std::string &s, const std::string &u, const std::string &p, int c) :
        url(s), user(u), password(p), sent(0), confirmed(0), total(c) {}

    void on_container_start(proton::container &c) OVERRIDE {
        proton::connection_options co;
        if (!user.empty()) co.user(user);
        if (!password.empty()) co.password(password);
        sender = c.open_sender(url, co);
    }

    void on_connection_open(proton::connection& c) OVERRIDE {
        if (c.reconnected()) {
            sent = confirmed;   // Re-send unconfirmed messages after a reconnect
        }
    }

    void on_sendable(proton::sender &s) OVERRIDE {
        while (s.credit() && sent < total) {
            proton::message msg;
            std::map<std::string, int> m;
            m["sequence"] = sent + 1;

            msg.id(sent + 1);
            msg.body(m);

            s.send(msg);
            sent++;
        }
    }

    void on_tracker_accept(proton::tracker &t) OVERRIDE {
        confirmed++;

        if (confirmed == total) {
            std::cout << "all messages confirmed" << std::endl;
            t.connection().close();
        }
    }

    void on_transport_close(proton::transport &) OVERRIDE {
        sent = confirmed;
    }
};

int main(int argc, char **argv) {
    std::string address("127.0.0.1:5672/examples");
    std::string user;
    std::string password;
    int message_count = 100;
    example::options opts(argc, argv);

    opts.add_value(address, 'a', "address", "connect and send to URL", "URL");
    opts.add_value(message_count, 'm', "messages", "send COUNT messages", "COUNT");
    opts.add_value(user, 'u', "user", "authenticate as USER", "USER");
    opts.add_value(password, 'p', "password", "authenticate with PASSWORD", "PASSWORD");

    try {
        opts.parse();

        simple_send send(address, user, password, message_count);
        proton::container(send).run();

        return 0;
    } catch (const example::bad_option& e) {
        std::cout << opts << std::endl << e.what() << std::endl;
    } catch (const std::exception& e) {
        std::cerr << e.what() << std::endl;
    }

    return 1;
}
<|endoftext|>


<commit_before>// Copyright 2014 Hazy Research (http://i.stanford.edu/hazy)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef _CYC_SPARSE_SGD_H
#define _CYC_SPARSE_SGD_H

#include "dimmwitted.h"

/**
 * \brief This file shows how to specify the same
 * synthetic model as in app/cyc_dense_sgd.h
 * but store the data as sparse matrix instead
 * of dense matrix.
 *
 * See app/cyc_dense_sgd.h for more detailed
 * comments.
 */
class CYCModelExample_Sparse{
public:
  double * const p;
  int n;

  CYCModelExample_Sparse(int _n):
    n(_n), p(new double[_n]){}

  CYCModelExample_Sparse( const CYCModelExample_Sparse& other ) :
     n(other.n), p(new double[other.n]){
    for(int i=0;i<n;i++){
      p[i] = other.p[i];
    }
  }

};

void f_lr_modelavg(CYCModelExample_Sparse** const p_models, int nreplicas, int ireplica){
  CYCModelExample_Sparse * p_model = p_models[ireplica];
  double sum = 0.0;
  for(int i=0;i<p_model->n;i++){
    sum = 0.0;
    for(int j=0;j<nreplicas;j++){
      sum += p_models[j]->p[i];
    }
    (p_model->p)[i] = sum/nreplicas;
  }
}

double f_lr_loss_sparse(const SparseVector<double>* const ex, CYCModelExample_Sparse* const p_model){

  /*
  double * model = p_model->p;
  double label = ex->p[ex->n-1];
  double dot = 0.0;
  for(int i=0;i<ex->n-1;i++){
    dot += ex->p[i] * model[ex->idxs[i]];
  }
  return  - label * dot + log(exp(dot) + 1.0);
  */
  return 0.0;
}

double f_lr_grad_sparse(const SparseVector<double>* const ex, CYCModelExample_Sparse* const p_model){

  /*
  double * model = p_model->p;
  double label = ex->p[ex->n-1];

  double dot = 0.0;
  for(int i=0;i<ex->n-1;i++){
    dot += ex->p[i] * model[ex->idxs[i]];
  }

  const double d = exp(-dot);
  const double Z = 0.0001 * (-label + 1.0/(1.0+d));

  for(int i=0;i<ex->n-1;i++){
    model[ex->idxs[i]] -= ex->p[i] * Z;
  }
  */
  return 1.0;
}

template<ModelReplType MODELREPL, DataReplType DATAREPL>
double test_cyc_sparse_sgd(){

//  numa_run_on_node(numa_node);
//  numa_set_localalloc();

  int NNUMA_NODE = numa_max_node() + 1;

  // First, create a synthetic data set.
  // Given nexp examples and nfeat features,
  // this data set contains nexp rows, and
  // nfeat + 1 columns, where the last column
  // is the label that we want to train on.
  //
  long nexp = 100; // number of rows
  long nfeat = 10;  // number of features
  long nbatches = 10; // number of batches
  long nepoch = 10;

  long nexp_per_batch = (int)ceil((double)nexp / (double)nbatches);
  long * batch_nexp = new long[nbatches];

  double ** examples_all = new double*[nbatches];
  long ** cols_all = new long*[nbatches];
  long ** rows_all = new long*[nbatches];

  std::vector<long> randperm;
  for (int i=0; i<nfeat; i++){
    randperm.push_back(i);
  }

  for (long ibatch = 0; ibatch < nbatches; ibatch++){
    // Randomly shuffle the features to be assigned to each core in this batch
    std::random_shuffle(randperm.begin(), randperm.end());

    // examples in this batch [batchBegIndex, batchEndIndex)
    long batchBegIndex = ibatch * nexp_per_batch;
    long batchEndIndex = batchBegIndex + nexp_per_batch;
    if (batchEndIndex >= nexp){
      batchEndIndex = nexp - 1;
    }
    batch_nexp[ibatch] = batchEndIndex - batchBegIndex;
    // Number of examples in the batch needs to be a multiple of number of NUMA nodes
    // We'll pad with noops to ensure this later
    double ndata_per_node_double = (double)(batch_nexp[ibatch]) / (double)NNUMA_NODE;
    long batchNumExamples = ceil(ndata_per_node_double) * NNUMA_NODE;
    double * examples = new double [batchNumExamples * (nfeat+1)];  // pointers to each row
    long * rows = new long[batchNumExamples];
    long * cols = new long[batchNumExamples * (nfeat+1)];
    examples_all[ibatch] = examples;
    rows_all[ibatch] = rows;
    cols_all[ibatch] = cols;

    long exampleIndex = 0;
    for(long inuma=0;inuma < NNUMA_NODE; inuma ++){
      numa_run_on_node(inuma);
      numa_set_localalloc();

      // double nodeBegIndex_double = (double) inuma    * ndata_per_node_double;
      double nodeEndIndex_double = (double)(inuma+1) * ndata_per_node_double;

      long ct = 0;

      long nodeExampleIndex = 0;
      for (; (double)exampleIndex < nodeEndIndex_double; exampleIndex++){
        rows[exampleIndex] = ct;
        for(int j=0;j<nfeat;j++){
          examples[ct] = drand48();
          cols[ct] = j;
          ct++;
        }
        examples[ct] = drand48() > 0.8 ? 0 : 1.0; // randomly generate labels
        cols[ct] = nfeat;
        ct++;

        nodeExampleIndex ++;
      }
      for (; nodeExampleIndex < ceil(ndata_per_node_double); exampleIndex++){
        rows[exampleIndex] = ct;
        for(int j=0;j<nfeat;j++){
          examples[ct] = drand48();
          cols[ct] = j;
          ct++;
        }
        examples[ct] = -1; // indicate noop
        cols[ct] = nfeat;
        ct++;

        nodeExampleIndex++;
      }

    }
  }

  // Sanity check of examples assignments
  if (true){
    for (long ibatch = 0; ibatch < nbatches; ibatch++){
      for (long iexample = 0; iexample < batch_nexp[ibatch]; iexample++){
        std::cout << "Batch " << ibatch << ", example " << iexample << std::endl << "\t\t";
        for (long ifeat = 0; ifeat < nfeat+1; ifeat++){
          std::cout << cols_all[ibatch][rows_all[ibatch][iexample] + ifeat] << ":" << examples_all[ibatch][rows_all[ibatch][iexample] + ifeat] << "\t";
        }
        std::cout << std::endl;
      }
    }
  }

  CYCModelExample_Sparse model(nfeat);
  for(int i=0;i<model.n;i++){
    model.p[i] = 0.0;
  }

  // Create DW engines
  SparseDimmWitted<double, CYCModelExample_Sparse, MODELREPL, DATAREPL, DW_ACCESS_ROW> ** dwEngines = new SparseDimmWitted<double, CYCModelExample_Sparse, MODELREPL, DATAREPL, DW_ACCESS_ROW>*[nbatches];
  unsigned int * f_handle_grads = new unsigned int[nbatches];
  unsigned int * f_handle_losss = new unsigned int[nbatches];

  for (long ibatch = 0; ibatch < nbatches; ibatch++){
    // Thrid, create a DenseDimmWitted object because the synthetic data set
    // we created is dense. This object has multiple templates,
    //    - double: the type of the data (type of elements in ``examples'')
    //    - CYCModelExample: the type of the model
    //    - MODELREPL: Model replication strategy
    //    - DATAREPL: Data replication strategy
    //    - DW_ROW: Access method
    //
    SparseDimmWitted<double, CYCModelExample_Sparse, MODELREPL, DATAREPL, DW_ACCESS_ROW> *
      dw = new SparseDimmWitted<double, CYCModelExample_Sparse, MODELREPL, DATAREPL, DW_ACCESS_ROW> (examples_all[ibatch], rows_all[ibatch], cols_all[ibatch], batch_nexp[ibatch], nfeat+1, batch_nexp[ibatch]*(nfeat+1), &model);
    unsigned int f_handle_grad = dw->register_row(f_lr_grad_sparse);
    unsigned int f_handle_loss = dw->register_row(f_lr_loss_sparse);
    dw->register_model_avg(f_handle_grad, f_lr_modelavg);
    dw->register_model_avg(f_handle_loss, f_lr_modelavg);
    dwEngines[ibatch] = dw;
    f_handle_grads[ibatch] = f_handle_grad;
    f_handle_losss[ibatch] = f_handle_loss;

    std::cout << f_handle_grads[ibatch]  << std::endl;
    std::cout << f_handle_losss[ibatch]  << std::endl;

    std::cout << "~~~~~~~~~" << std::endl;
    dw->exec(f_handle_loss);
    std::cout << "~~~~~~~~~" << std::endl;
  }

  double l2sum = 0.0;
  for (long iepoch = 0; iepoch < nepoch; iepoch++){
    // std::cout << "Running epoch " << iepoch << "\n";
    double loss = 0.0;
    for (long ibatch = 0; ibatch < nbatches; ibatch++){
      std::cout << "epoch " << iepoch << ",\tbatch " << ibatch << "\n";
      SparseDimmWitted<double, CYCModelExample_Sparse, MODELREPL, DATAREPL, DW_ACCESS_ROW> * dw = dwEngines[ibatch];
      // dw->exec(f_handle_grads[ibatch]);
      loss = dw->exec(f_handle_losss[ibatch]);
      std::cout << "\tloss = " << loss << std::endl;
      for (long ifeat = 0; ifeat < nfeat; ifeat++){
        std::cout << model.p[ifeat] << ", ";
      }
      std::cout << std::endl;
    }
    loss /= nexp;

    l2sum = 0.0;
    for (int i = 0; i<nfeat; i ++){
      l2sum += model.p[i] * model.p[i];
    }

    std::cout.precision(8);
    std::cout << l2sum << "    loss=" << loss << std::endl;
  }

  // Return the sum of the model. This value should be
  // around 1.3-1.4 for this example.
  //
  return l2sum;

}

/**
 * \brief This is one example of running SGD for logistic regression
 * in DimmWitted. You can find more examples in test/cyc_dense.cc
 * and test/cyc_sparse.cc, and the documented code in
 * app/cyc_dense_sgd.h
 */
//int main(int argc, char** argv){
//  double rs = test_cyc_sparse_sgd<DW_MODELREPL_PERCORE, DW_DATAREPL_SHARDING>();
//  std::cout << "SUM OF MODEL (Should be ~1.3-1.4): " << rs << std::endl;
//  return 0;
//}

#endif

<commit_msg>Cyclades patch<commit_after><|endoftext|>


<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once

#include "hash_map_insert.hpp"
#include "hashtable.hpp"
#include "select.h"

namespace vespalib {

template <typename K, typename V, typename H, typename EQ, typename M>
hash_map<K, V, H, EQ, M>::hash_map(size_t reserveSize) :
    _ht(reserveSize)
{ }

template <typename K, typename V, typename H, typename EQ, typename M>
hash_map<K, V, H, EQ, M>::hash_map(size_t reserveSize, H hasher, EQ equality) :
    _ht(reserveSize, hasher, equality)
{ }

template <typename K, typename V, typename H, typename EQ, typename M>
hash_map<K, V, H, EQ, M>::hash_map(std::initializer_list<value_type> input)
    : _ht(0)
{
    insert(input.begin(), input.end());
}

template <typename K, typename V, typename H, typename EQ, typename M>
hash_map<K, V, H, EQ, M>::~hash_map() noexcept = default;

template <typename K, typename V, typename H, typename EQ, typename M>
void
hash_map<K, V, H, EQ, M>::erase(const K & key) {
    return _ht.erase(key);
}

template <typename K, typename V, typename H, typename EQ, typename M>
void
hash_map<K, V, H, EQ, M>::clear() {
    _ht.clear();
}

template <typename K, typename V, typename H, typename EQ, typename M>
void
hash_map<K, V, H, EQ, M>::resize(size_t newSize) {
    _ht.resize(newSize);
}

template <typename K, typename V, typename H, typename EQ, typename M>
void
hash_map<K, V, H, EQ, M>::swap(hash_map & rhs) {
    _ht.swap(rhs._ht);
}

template <typename K, typename V, typename H, typename EQ, typename M>
size_t
hash_map<K, V, H, EQ, M>::getMemoryConsumption() const {
    return _ht.getMemoryConsumption();
}

template <typename K, typename V, typename H, typename EQ, typename M>
size_t
hash_map<K, V, H, EQ, M>::getMemoryUsed() const
{
    return _ht.getMemoryUsed();
}

}

#define VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, M) \
    template class vespalib::hash_map<K, V, H, E, M>; \
    template class vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>; \
    template vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert_result \
             vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert(std::pair<K,V> &&); \
    template vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert_result \
             vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insertInternal(std::pair<K,V> &&); \

#define VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, E) \
    template class vespalib::Array<vespalib::hash_node<std::pair<K,V>>>; \
    VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::prime_modulator) \
    VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::and_modulator)

#define VESPALIB_HASH_MAP_INSTANTIATE_H(K, V, H) VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, std::equal_to<>)

#define VESPALIB_HASH_MAP_INSTANTIATE(K, V) VESPALIB_HASH_MAP_INSTANTIATE_H(K, V, vespalib::hash<K>)

<commit_msg>Remove continuation token on last line in macro.<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once

#include "hash_map_insert.hpp"
#include "hashtable.hpp"
#include "select.h"

namespace vespalib {

template <typename K, typename V, typename H, typename EQ, typename M>
hash_map<K, V, H, EQ, M>::hash_map(size_t reserveSize) :
    _ht(reserveSize)
{ }

template <typename K, typename V, typename H, typename EQ, typename M>
hash_map<K, V, H, EQ, M>::hash_map(size_t reserveSize, H hasher, EQ equality) :
    _ht(reserveSize, hasher, equality)
{ }

template <typename K, typename V, typename H, typename EQ, typename M>
hash_map<K, V, H, EQ, M>::hash_map(std::initializer_list<value_type> input)
    : _ht(0)
{
    insert(input.begin(), input.end());
}

template <typename K, typename V, typename H, typename EQ, typename M>
hash_map<K, V, H, EQ, M>::~hash_map() noexcept = default;

template <typename K, typename V, typename H, typename EQ, typename M>
void
hash_map<K, V, H, EQ, M>::erase(const K & key) {
    return _ht.erase(key);
}

template <typename K, typename V, typename H, typename EQ, typename M>
void
hash_map<K, V, H, EQ, M>::clear() {
    _ht.clear();
}

template <typename K, typename V, typename H, typename EQ, typename M>
void
hash_map<K, V, H, EQ, M>::resize(size_t newSize) {
    _ht.resize(newSize);
}

template <typename K, typename V, typename H, typename EQ, typename M>
void
hash_map<K, V, H, EQ, M>::swap(hash_map & rhs) {
    _ht.swap(rhs._ht);
}

template <typename K, typename V, typename H, typename EQ, typename M>
size_t
hash_map<K, V, H, EQ, M>::getMemoryConsumption() const {
    return _ht.getMemoryConsumption();
}

template <typename K, typename V, typename H, typename EQ, typename M>
size_t
hash_map<K, V, H, EQ, M>::getMemoryUsed() const
{
    return _ht.getMemoryUsed();
}

}

#define VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, M) \
    template class vespalib::hash_map<K, V, H, E, M>; \
    template class vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>; \
    template vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert_result \
             vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert(std::pair<K,V> &&); \
    template vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert_result \
             vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insertInternal(std::pair<K,V> &&);

#define VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, E) \
    template class vespalib::Array<vespalib::hash_node<std::pair<K,V>>>; \
    VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::prime_modulator) \
    VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::and_modulator)

#define VESPALIB_HASH_MAP_INSTANTIATE_H(K, V, H) VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, std::equal_to<>)

#define VESPALIB_HASH_MAP_INSTANTIATE(K, V) VESPALIB_HASH_MAP_INSTANTIATE_H(K, V, vespalib::hash<K>)

<|endoftext|>


<commit_before>
# include "PDForces_Dipole.hpp"

// -------------------------------------------------------------------------
// Constructor...
// -------------------------------------------------------------------------

Dipole::Dipole(const GetPot& p, vector<double>& rin, vector<double>& vin,
        vector<double>& fin, vector<double>& radin) :
    r(rin), v(vin), f(fin), rad(radin)
{
    rcut = p("PDApp/inter_particle_forces/rcut",4.0);
    rcut2 = rcut*rcut;
    eps = p("PDApp/inter_particle_forces/eps",1.0);
    n = p("PDApp/inter_particle_forces/n",-13.0);
    Ex = p("PDApp/inter_particle_forces/Ex",0.0);
    Ey = p("PDApp/inter_particle_forces/Ey",0.0);
    Ez = p("PDApp/inter_particle_forces/Ez",1.0);
    box[0] = p("Domain/nx",5);
    box[1] = p("Domain/ny",5);
    box[2] = p("Domain/nz",5);
    box[0] *= p("Domain/dx",1.0);
    box[1] *= p("Domain/dy",1.0);
    box[2] *= p("Domain/dz",1.0);
    Emag = sqrt(Ex*Ex + Ey*Ey + Ez*Ez);
    edir[0] = Ex/Emag;
    edir[1] = Ey/Emag;
    edir[2] = Ez/Emag;
}

// -------------------------------------------------------------------------
// Destructor...
// -------------------------------------------------------------------------

Dipole::~Dipole()
{
}

// -------------------------------------------------------------------------
// Function to calculate fij:
// -------------------------------------------------------------------------

void Dipole::fijFunc(int i, int j)
{
    //compute the squared particle distance:
    rij2 = 0.0;
    for (int k=0; k<3; k++)
    {
        rij[k] = r[i*3+k] - r[j*3+k];
        rij[k] -= round(rij[k]/box[k])*box[k];  // <-- pbc's
        rij2 += rij[k]*rij[k];
    }
    // if inside the cutoff radius then calculate ij pair interaction
    if (rij2 <= rcut2)
    {
        rijMag = sqrt(rij2);

        // add soft repulsive force
        radComp = 0.0;
        avgRad = (rad[i] + rad[j])/2.0;
        radComp = (n-1.0)*eps*pow(2.0*avgRad/rijMag),n);

        if (abs(Ex)>0 || abs(Ey)>0 || abs(Ez)>0)
        {
            // add dipole-dipole interaction
            thetaComp = 0.0;
            EdotRij = rij[0]*Ex+rij[1]*Ey+rij[2]*Ez;
            theta = acos(EdotRij/(rijMag*Emag));
            ScaleFactor = pow(avgRad,6.0)*pow(Emag,2.0)/pow(rijMag,4.0);
            radComp -= ScaleFactor*(3*pow(cos(theta),2.0)-1.0);
            thetaComp = -ScaleFactor*sin(2.0*theta);

            // calculate theta hat direction
            cross(edir,rij,normal);
            cross(normal,rij,thetaHat);
            thetaMag = sqrt(pow(thetaHat[0],2.0)+pow(thetaHat[1],2.0)+pow(thetaHat[2],2.0));
            for (int k=0;k<3;k++) {
                if (thetaMag > 0.00001)
                {
                    thetaHat[k]*=1/thetaMag;
                }
                else
                {
                    thetaHat[k]*=0.0;
                }
            }

            // put all force components together
            for (int k=0;k<3;k++)
            {
                f[3*i+k] += radComp*rij[k]/rijMag + thetaComp*thetaHat[k];
                f[3*j+k] -= radComp*rij[k]/rijMag + thetaComp*thetaHat[k];
            }
        }
        else
        {
            for (int k=0;k<3;k++)
            {
                f[3*i+k] += radComp*rij[k]/rijMag;
                f[3*j+k] -= radComp*rij[k]/rijMag;
            }
        }
    }
}

void Dipole::cross(double (&a)[3], double(&b)[3],double (&crs)[3])
{
    crs[0] = a[1]*b[2]-b[1]*a[2];
    crs[1] = a[2]*b[0]-b[2]*a[0];
    crs[2] = a[0]*b[1]-b[0]*a[1];
}
<commit_msg>fix dipole force syntax error.<commit_after>
# include "PDForces_Dipole.hpp"

// -------------------------------------------------------------------------
// Constructor...
// -------------------------------------------------------------------------

Dipole::Dipole(const GetPot& p, vector<double>& rin, vector<double>& vin,
        vector<double>& fin, vector<double>& radin) :
    r(rin), v(vin), f(fin), rad(radin)
{
    rcut = p("PDApp/inter_particle_forces/rcut",4.0);
    rcut2 = rcut*rcut;
    eps = p("PDApp/inter_particle_forces/eps",1.0);
    n = p("PDApp/inter_particle_forces/n",-13.0);
    Ex = p("PDApp/inter_particle_forces/Ex",0.0);
    Ey = p("PDApp/inter_particle_forces/Ey",0.0);
    Ez = p("PDApp/inter_particle_forces/Ez",1.0);
    box[0] = p("Domain/nx",5);
    box[1] = p("Domain/ny",5);
    box[2] = p("Domain/nz",5);
    box[0] *= p("Domain/dx",1.0);
    box[1] *= p("Domain/dy",1.0);
    box[2] *= p("Domain/dz",1.0);
    Emag = sqrt(Ex*Ex + Ey*Ey + Ez*Ez);
    edir[0] = Ex/Emag;
    edir[1] = Ey/Emag;
    edir[2] = Ez/Emag;
}

// -------------------------------------------------------------------------
// Destructor...
// -------------------------------------------------------------------------

Dipole::~Dipole()
{
}

// -------------------------------------------------------------------------
// Function to calculate fij:
// -------------------------------------------------------------------------

void Dipole::fijFunc(int i, int j)
{
    //compute the squared particle distance:
    rij2 = 0.0;
    for (int k=0; k<3; k++)
    {
        rij[k] = r[i*3+k] - r[j*3+k];
        rij[k] -= round(rij[k]/box[k])*box[k];  // <-- pbc's
        rij2 += rij[k]*rij[k];
    }
    // if inside the cutoff radius then calculate ij pair interaction
    if (rij2 <= rcut2)
    {
        rijMag = sqrt(rij2);

        // add soft repulsive force
        radComp = 0.0;
        avgRad = (rad[i] + rad[j])/2.0;
        radComp = (n-1.0)*eps*pow(2.0*avgRad/rijMag,n);

        if (abs(Ex)>0 || abs(Ey)>0 || abs(Ez)>0)
        {
            // add dipole-dipole interaction
            thetaComp = 0.0;
            EdotRij = rij[0]*Ex+rij[1]*Ey+rij[2]*Ez;
            theta = acos(EdotRij/(rijMag*Emag));
            ScaleFactor = pow(avgRad,6.0)*pow(Emag,2.0)/pow(rijMag,4.0);
            radComp -= ScaleFactor*(3*pow(cos(theta),2.0)-1.0);
            thetaComp = -ScaleFactor*sin(2.0*theta);

            // calculate theta hat direction
            cross(edir,rij,normal);
            cross(normal,rij,thetaHat);
            thetaMag = sqrt(pow(thetaHat[0],2.0)+pow(thetaHat[1],2.0)+pow(thetaHat[2],2.0));
            for (int k=0;k<3;k++) {
                if (thetaMag > 0.00001)
                {
                    thetaHat[k]*=1/thetaMag;
                }
                else
                {
                    thetaHat[k]*=0.0;
                }
            }

            // put all force components together
            for (int k=0;k<3;k++)
            {
                f[3*i+k] += radComp*rij[k]/rijMag + thetaComp*thetaHat[k];
                f[3*j+k] -= radComp*rij[k]/rijMag + thetaComp*thetaHat[k];
            }
        }
        else
        {
            for (int k=0;k<3;k++)
            {
                f[3*i+k] += radComp*rij[k]/rijMag;
                f[3*j+k] -= radComp*rij[k]/rijMag;
            }
        }
    }
}

void Dipole::cross(double (&a)[3], double(&b)[3],double (&crs)[3])
{
    crs[0] = a[1]*b[2]-b[1]*a[2];
    crs[1] = a[2]*b[0]-b[2]*a[0];
    crs[2] = a[0]*b[1]-b[0]*a[1];
}
<|endoftext|>