<commit_before>#include "linalg.h"
#include <gsl/gsl_linalg.h>

namespace votca { namespace tools {

void linalg_qrsolve(ub::vector<double> &x, ub::matrix<double> &A, ub::vector<double> &b)
{
    // now A is a tridiagonal system, solve it!
    double* pointer_m = &A(0,0);
    double* pointer_b = &b(0);

    const int N = x.size();

    gsl_matrix_view m
        = gsl_matrix_view_array (pointer_m, N, N);

    gsl_vector_view gb
        = gsl_vector_view_array (pointer_b, N);

    gsl_vector *gx = gsl_vector_alloc (N);
    gsl_vector *tau = gsl_vector_alloc (N);
    gsl_vector *residual = gsl_vector_alloc (N);

    gsl_linalg_QR_decomp (&m.matrix, tau);

    gsl_linalg_QR_lssolve (&m.matrix, tau, &gb.vector, gx, residual);

    for (int i =0 ; i < N; i++) {
        x(i) = gsl_vector_get(gx, i);
    }

    gsl_vector_free (gx);
    gsl_vector_free (tau);
    gsl_vector_free (residual);
}

void linalg_constrained_qrsolve(ub::vector<double> &x, ub::matrix<double> &A, ub::vector<double> &b, ub::matrix<double> &constr)
{
    // Transpose constr:
    constr = trans(constr);

    const int N = b.size();
    const int ngrid = x.size()/2;

    // temporary variables
    ub::matrix<double> Q(2*ngrid, 2*ngrid);       // Q matrix: QR decomposition of trans(B)
    ub::matrix<double> A2(N, ngrid);              // Matrix A2 (see manual)
    ub::matrix<double> Q_k(2*ngrid, 2*ngrid);
    ub::identity_matrix<double> I (2*ngrid);
    ub::vector<double> v(2*ngrid);

    Q = ub::zero_matrix<double>(2*ngrid, 2*ngrid);
    A2 = ub::zero_matrix<double>(N, ngrid);
    Q_k = ub::zero_matrix<double>(2*ngrid, 2*ngrid);
    v = ub::zero_vector<double>(2*ngrid);

    double* pointer_m = & constr(0,0);

    gsl_matrix_view B_t
      = gsl_matrix_view_array (pointer_m, constr.size1(), constr.size2());

    gsl_vector *tau_qr = gsl_vector_alloc (ngrid);

    gsl_linalg_QR_decomp (&B_t.matrix, tau_qr);

    Q = I;

    for (int k = ngrid; k > 0 ; k--) {

        for (int icout = 0; icout < k - 1; icout++) {
             v(icout) = 0;
        }
        v(k - 1) = 1.0;

        for (int icout = k; icout < 2*ngrid; icout++) {
             v(icout) = gsl_matrix_get(&B_t.matrix, icout, k - 1 );
        }

        Q_k = I - gsl_vector_get(tau_qr, k - 1 ) * outer_prod ( v, v );
        Q = prec_prod(Q, Q_k);

    }

    Q = trans(Q);
    gsl_vector_free (tau_qr);

    // Calculate A * Q and store the result in A
    A = prec_prod(A, Q);

    // A = [A1 A2], so A2 is just a block of A
    for (int iraw = 0; iraw < N; iraw++) {
        for (int icol = ngrid; icol < 2*ngrid; icol++) {
            A2(iraw, icol - ngrid) = A(iraw, icol);
        }
    }

    pointer_m = & A2(0,0);

    double* pointer_b = & b(0);

    gsl_matrix_view m
         = gsl_matrix_view_array (pointer_m, N, ngrid);

    gsl_vector_view gsl_b
         = gsl_vector_view_array (pointer_b, N);

    gsl_vector *z = gsl_vector_alloc (ngrid);
    gsl_vector *tau_solve = gsl_vector_alloc (ngrid);  // already done!
    gsl_vector *residual = gsl_vector_alloc (N);

    gsl_linalg_QR_decomp (&m.matrix, tau_solve);
    gsl_linalg_QR_lssolve (&m.matrix, tau_solve, &gsl_b.vector, z, residual);

    // Next two cycles assemble vector from y (which is zero-vector) and z
    // (which we just got by gsl_linalg_QR_lssolve)

    for (int i = 0; i < ngrid; i++ ) {
           x[i] = 0.0;
    }

    for (int i = ngrid; i < 2 * ngrid; i++ ) {
           x[i] = gsl_vector_get(z, i - ngrid);
    }

    // To get the final answer this vector should be multiplied by matrix Q
    // TODO: here i changed the sign, check again! (victor)
    x = -prec_prod( Q, x );

    gsl_vector_free (z);
    gsl_vector_free (tau_solve);
    gsl_vector_free (residual);
}

}}
<commit_msg>qrsolve sizes should be ok now, bit of cleaning<commit_after>#include "linalg.h"
#include <gsl/gsl_linalg.h>
#include <boost/numeric/ublas/matrix_proxy.hpp>

namespace votca { namespace tools {

void linalg_qrsolve(ub::vector<double> &x, ub::matrix<double> &A, ub::vector<double> &b)
{
    gsl_matrix_view m
        = gsl_matrix_view_array (&A(0,0), A.size1(), A.size2());

    gsl_vector_view gb
        = gsl_vector_view_array (&b(0), b.size());

    gsl_vector *gsl_x = gsl_vector_alloc (x.size());
    gsl_vector *tau = gsl_vector_alloc (x.size());
    gsl_vector *residual = gsl_vector_alloc (b.size());

    gsl_linalg_QR_decomp (&m.matrix, tau);

    gsl_linalg_QR_lssolve (&m.matrix, tau, &gb.vector, gsl_x, residual);

    for (int i =0 ; i < x.size(); i++)
        x(i) = gsl_vector_get(gsl_x, i);

    gsl_vector_free (gsl_x);
    gsl_vector_free (tau);
    gsl_vector_free (residual);
}

void linalg_constrained_qrsolve(ub::vector<double> &x, ub::matrix<double> &A, ub::vector<double> &b, ub::matrix<double> &constr)
{
    // Transpose constr:
    constr = trans(constr);

    const int N = b.size();
    const int ngrid = x.size()/2;

    // temporary variables
    ub::matrix<double> Q(2*ngrid, 2*ngrid);       // Q matrix: QR decomposition of trans(B)
    ub::matrix<double> Q_k(2*ngrid, 2*ngrid);
    ub::identity_matrix<double> I (2*ngrid);
    ub::vector<double> v(2*ngrid);

    Q = ub::zero_matrix<double>(2*ngrid, 2*ngrid);
    Q_k = ub::zero_matrix<double>(2*ngrid, 2*ngrid);
    v = ub::zero_vector<double>(2*ngrid);

    double *tmp = & constr(0,0);
    gsl_matrix_view gsl_constr
      = gsl_matrix_view_array (tmp, constr.size1(), constr.size2());

    tmp = &b(0);
    gsl_vector_view gsl_b
         = gsl_vector_view_array (tmp, b.size());

    gsl_vector *tau_qr = gsl_vector_alloc (ngrid);

    gsl_linalg_QR_decomp (&gsl_constr.matrix, tau_qr);

    Q = I;

    for (int k = ngrid; k > 0 ; k--) {

        for (int icout = 0; icout < k - 1; icout++) {
             v(icout) = 0;
        }
        v(k - 1) = 1.0;

        for (int icout = k; icout < 2*ngrid; icout++) {
             v(icout) = gsl_matrix_get(&gsl_constr.matrix, icout, k - 1 );
        }

        Q_k = I - gsl_vector_get(tau_qr, k - 1 ) * outer_prod ( v, v );
        Q = prec_prod(Q, Q_k);

    }

    Q = trans(Q);
    gsl_vector_free (tau_qr);

    // Calculate A * Q and store the result in A
    A = prec_prod(A, Q);

    // A = [A1 A2], so A2 is just a block of A
    ub::matrix<double> A2 = ub::matrix_range<ub::matrix<double> >(A,
            ub::range (0, N), ub::range (ngrid, 2*ngrid)
         );

    tmp = &A2(0,0);
    gsl_matrix_view gsl_A2
         = gsl_matrix_view_array (tmp, A2.size1(), A2.size2());

    gsl_vector *z = gsl_vector_alloc (ngrid);
    gsl_vector *tau_solve = gsl_vector_alloc (ngrid);  // already done!
    gsl_vector *residual = gsl_vector_alloc (N);

    gsl_linalg_QR_decomp (&gsl_A2.matrix, tau_solve);
    gsl_linalg_QR_lssolve (&gsl_A2.matrix, tau_solve, &gsl_b.vector, z, residual);

    // Next two cycles assemble vector from y (which is zero-vector) and z
    // (which we just got by gsl_linalg_QR_lssolve)

    for (int i = 0; i < ngrid; i++ ) {
           x[i] = 0.0;
    }

    for (int i = ngrid; i < 2 * ngrid; i++ ) {
           x[i] = gsl_vector_get(z, i - ngrid);
    }

    // To get the final answer this vector should be multiplied by matrix Q
    // TODO: here i changed the sign, check again! (victor)
    x = -prec_prod( Q, x );

    gsl_vector_free (z);
    gsl_vector_free (tau_solve);
    gsl_vector_free (residual);
}

}}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <assert.h>
#include <fnord-base/exception.h>
#include <fnord-base/inspect.h>
#include <fnord-base/logging.h>
#include <fnord-base/stringutil.h>
#include <fnord-base/uri.h>
#include <fnord-base/wallclock.h>
#include <fnord-rpc/RPC.h>
#include <fnord-json/json.h>
#include <fnord-feeds/RemoteFeedFactory.h>
#include <fnord-feeds/RemoteFeedWriter.h>
#include "ItemRef.h"
#include "JoinedQuery.h"
#include "JoinedItemVisit.h"
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"

using namespace fnord;

namespace cm {

LogJoin::LogJoin(
    LogJoinShard shard,
    bool dry_run,
    bool enable_cache,
    LogJoinTarget* target) :
    shard_(shard),
    dry_run_(dry_run),
    target_(target),
    turbo_(false),
    enable_cache_(enable_cache) {
  addPixelParamID("dw_ab", 1);
  addPixelParamID("l", 2);
  addPixelParamID("u_x", 3);
  addPixelParamID("u_y", 4);
  addPixelParamID("is", 5);
  addPixelParamID("pg", 6);
  addPixelParamID("q_cat1", 7);
  addPixelParamID("q_cat2", 8);
  addPixelParamID("q_cat3", 9);
  addPixelParamID("slrid", 10);
  addPixelParamID("i", 11);
  addPixelParamID("s", 12);
  addPixelParamID("ml", 13);
  addPixelParamID("adm", 14);
  addPixelParamID("lgn", 15);
  addPixelParamID("slr", 16);
  addPixelParamID("lng", 17);
  addPixelParamID("dwnid", 18);
  addPixelParamID("fnm", 19);
  addPixelParamID("qstr~de", 100);
  addPixelParamID("qstr~pl", 101);
  addPixelParamID("qstr~en", 102);
  addPixelParamID("qstr~fr", 103);
  addPixelParamID("qstr~it", 104);
  addPixelParamID("qstr~nl", 105);
  addPixelParamID("qstr~es", 106);
}

size_t LogJoin::numSessions() const {
  return sessions_flush_times_.size();
}

size_t LogJoin::cacheSize() const {
  return session_cache_.size();
}

void LogJoin::insertLogline(
      const std::string& log_line,
      mdb::MDBTransaction* txn) {
  auto c_end = log_line.find("|");
  if (c_end == std::string::npos) {
    RAISEF(kRuntimeError, "invalid logline: $0", log_line);
  }

  auto t_end = log_line.find("|", c_end + 1);
  if (t_end == std::string::npos) {
    RAISEF(kRuntimeError, "invalid logline: $0", log_line);
  }

  auto customer_key = log_line.substr(0, c_end);
  auto body = log_line.substr(t_end + 1);
  auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
  fnord::DateTime time(std::stoul(timestr) * fnord::kMicrosPerSecond);

  insertLogline(customer_key, time, body, txn);
}

void LogJoin::insertLogline(
    const std::string& customer_key,
    const fnord::DateTime& time,
    const std::string& log_line,
    mdb::MDBTransaction* txn) {
  fnord::URI::ParamList params;
  fnord::URI::parseQueryString(log_line, &params);

  stat_loglines_total_.incr(1);

  try {
    /* extract uid (userid) and eid (eventid) */
    std::string c;
    if (!fnord::URI::getParam(params, "c", &c)) {
      RAISE(kParseError, "c param is missing");
    }

    auto c_s = c.find("~");
    if (c_s == std::string::npos) {
      RAISE(kParseError, "c param is invalid");
    }

    std::string uid = c.substr(0, c_s);
    std::string eid = c.substr(c_s + 1, c.length() - c_s - 1);
    if (uid.length() == 0 || eid.length() == 0) {
      RAISE(kParseError, "c param is invalid");
    }

    if (!shard_.testUID(uid)) {
#ifndef NDEBUG
      fnord::logTrace(
          "cm.logjoin",
          "dropping logline with uid=$0 because it does not match my shard",
          uid);
#endif
      return;
    }

    std::string evtype;
    if (!fnord::URI::getParam(params, "e", &evtype) || evtype.length() != 1) {
      RAISE(kParseError, "e param is missing");
    }

    if (evtype.length() != 1) {
      RAISE(kParseError, "e param invalid");
    }

    /* process event */
    switch (evtype[0]) {

      case 'q':
      case 'v':
      case 'c':
      case 'u':
        break;

      default:
        RAISE(kParseError, "invalid e param");

    };

    URI::ParamList stored_params;
    for (const auto& p : params) {
      if (p.first == "c" || p.first == "e" || p.first == "v") {
        continue;
      }

      stored_params.emplace_back(p);
    }

    appendToSession(customer_key, time, uid, eid, evtype, stored_params, txn);
  } catch (...) {
    stat_loglines_invalid_.incr(1);
    throw;
  }
}

void LogJoin::appendToSession(
    const std::string& customer_key,
    const fnord::DateTime& time,
    const std::string& uid,
    const std::string& evid,
    const std::string& evtype,
    const Vector<Pair<String, String>>& logline,
    mdb::MDBTransaction* txn) {

  auto flush_at = time.unixMicros() +
      kSessionIdleTimeoutSeconds * fnord::kMicrosPerSecond;

  bool new_session = sessions_flush_times_.count(uid) == 0;
  sessions_flush_times_.emplace(uid, flush_at);

  util::BinaryMessageWriter buf;
  buf.appendVarUInt(time.unixMicros() / kMicrosPerSecond);
  buf.appendVarUInt(evid.length());
  buf.append(evid.data(), evid.length());
  for (const auto& p : logline) {
    buf.appendVarUInt(getPixelParamID(p.first));
    buf.appendVarUInt(p.second.size());
    buf.append(p.second.data(), p.second.size());
  }

  auto evkey = uid + "~" + evtype;
  txn->insert(evkey.data(), evkey.size(), buf.data(), buf.size());
  txn->update(uid + "~cust", customer_key);
}

void LogJoin::flush(mdb::MDBTransaction* txn, DateTime stream_time_) {
  auto stream_time = stream_time_.unixMicros();

  for (auto iter = sessions_flush_times_.begin();
      iter != sessions_flush_times_.end();) {
    if (iter->second.unixMicros() < stream_time) {
      flushSession(iter->first, stream_time, txn);
      iter = sessions_flush_times_.erase(iter);
    } else {
      ++iter;
    }
  }
}

void LogJoin::flushSession(
    const std::string uid,
    DateTime stream_time,
    mdb::MDBTransaction* txn) {
  auto cursor = txn->getCursor();

  TrackedSession session;
  session.uid = uid;

  Buffer key;
  Buffer value;
  bool eof = false;
  for (int i = 0; ; ++i) {
    if (i == 0) {
      key.append(uid);
      if (!cursor->getFirstOrGreater(&key, &value)) {
        break;
      }
    } else {
      if (!cursor->getNext(&key, &value)) {
        break;
      }
    }

    auto key_str = key.toString();
    if (!StringUtil::beginsWith(key_str, uid)) {
      break;
    }

    if (StringUtil::endsWith(key_str, "~cust")) {
      session.customer_key = value.toString();
    } else {
      auto evtype = key_str.substr(uid.length() + 1);

      util::BinaryMessageReader reader(value.data(), value.size());
      auto time = reader.readVarUInt();
      auto evid_len = reader.readVarUInt();
      auto evid = String((char*) reader.read(evid_len), evid_len);

      try {
        URI::ParamList logline;
        while (reader.remaining() > 0) {
          auto key = getPixelParamName(reader.readVarUInt());
          auto len = reader.readVarUInt();
          logline.emplace_back(key, String((char*) reader.read(len), len));
        }

        session.insertLogline(time, evtype, evid, logline);
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid logline");
        stat_loglines_invalid_.incr(1);
      }
    }

    txn->del(key);
  }

  cursor->close();

  if (session.customer_key.length() == 0) {
    fnord::logError("cm.logjoin", "missing customer key for: $0", uid);
    return;
  }

  try {
    target_->onSession(txn, session);
  } catch (const std::exception& e) {
    fnord::logError("cm.logjoin", e, "LogJoinTarget::onSession crashed");
  }

  stat_joined_sessions_.incr(1);
}

void LogJoin::importTimeoutList(mdb::MDBTransaction* txn) {
  Buffer key;
  Buffer value;
  int n = 0;

  auto cursor = txn->getCursor();

  for (;;) {
    bool eof;
    if (n++ == 0) {
      eof = !cursor->getFirst(&key, &value);
    } else {
      eof = !cursor->getNext(&key, &value);
    }

    if (eof) {
      break;
    }

    if (StringUtil::beginsWith(key.toString(), "__")) {
      continue;
    }

    auto sid = key.toString();
    if (StringUtil::endsWith(sid, "~cust")) {
      continue;
    }

    auto evtype = sid.substr(sid.find("~") + 1);
    sid.erase(sid.end() - evtype.size() - 1, sid.end());

    util::BinaryMessageReader reader(value.data(), value.size());
    auto time = reader.readVarUInt();
    auto ftime = (time + kSessionIdleTimeoutSeconds) * fnord::kMicrosPerSecond;

    auto old_ftime = sessions_flush_times_.find(sid);
    if (old_ftime == sessions_flush_times_.end() ||
        old_ftime->second.unixMicros() < ftime) {
      sessions_flush_times_.emplace(sid, ftime);
    }
  }

  cursor->close();
}

void LogJoin::addPixelParamID(const String& param, uint32_t id) {
  pixel_param_ids_[param] = id;
  pixel_param_names_[id] = param;
}

uint32_t LogJoin::getPixelParamID(const String& param) const {
  auto p = pixel_param_ids_.find(param);

  if (p == pixel_param_ids_.end()) {
    RAISEF(kIndexError, "invalid pixel param: $0", param);
  }

  return p->second;
}

const String& LogJoin::getPixelParamName(uint32_t id) const {
  auto p = pixel_param_names_.find(id);

  if (p == pixel_param_names_.end()) {
    RAISEF(kIndexError, "invalid pixel param: $0", id);
  }

  return p->second;
}

void LogJoin::exportStats(const std::string& prefix) {
  exportStat(
      StringUtil::format("$0/$1", prefix, "loglines_total"),
      &stat_loglines_total_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      StringUtil::format("$0/$1", prefix, "loglines_invalid"),
      &stat_loglines_invalid_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      StringUtil::format("$0/$1", prefix, "joined_sessions"),
      &stat_joined_sessions_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      StringUtil::format("$0/$1", prefix, "joined_queries"),
      &stat_joined_queries_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      StringUtil::format("$0/$1", prefix, "joined_item_visits"),
      &stat_joined_item_visits_,
      fnord::stats::ExportMode::EXPORT_DELTA);
}

void LogJoin::setTurbo(bool turbo) {
  turbo_ = turbo;
}

} // namespace cm
<commit_msg>seconds vs microseconds fix<commit_after>/*
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <assert.h>
#include <fnord-base/exception.h>
#include <fnord-base/inspect.h>
#include <fnord-base/logging.h>
#include <fnord-base/stringutil.h>
#include <fnord-base/uri.h>
#include <fnord-base/wallclock.h>
#include <fnord-rpc/RPC.h>
#include <fnord-json/json.h>
#include <fnord-feeds/RemoteFeedFactory.h>
#include <fnord-feeds/RemoteFeedWriter.h>
#include "ItemRef.h"
#include "JoinedQuery.h"
#include "JoinedItemVisit.h"
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"

using namespace fnord;

namespace cm {

LogJoin::LogJoin(
    LogJoinShard shard,
    bool dry_run,
    bool enable_cache,
    LogJoinTarget* target) :
    shard_(shard),
    dry_run_(dry_run),
    target_(target),
    turbo_(false),
    enable_cache_(enable_cache) {
  addPixelParamID("dw_ab", 1);
  addPixelParamID("l", 2);
  addPixelParamID("u_x", 3);
  addPixelParamID("u_y", 4);
  addPixelParamID("is", 5);
  addPixelParamID("pg", 6);
  addPixelParamID("q_cat1", 7);
  addPixelParamID("q_cat2", 8);
  addPixelParamID("q_cat3", 9);
  addPixelParamID("slrid", 10);
  addPixelParamID("i", 11);
  addPixelParamID("s", 12);
  addPixelParamID("ml", 13);
  addPixelParamID("adm", 14);
  addPixelParamID("lgn", 15);
  addPixelParamID("slr", 16);
  addPixelParamID("lng", 17);
  addPixelParamID("dwnid", 18);
  addPixelParamID("fnm", 19);
  addPixelParamID("qstr~de", 100);
  addPixelParamID("qstr~pl", 101);
  addPixelParamID("qstr~en", 102);
  addPixelParamID("qstr~fr", 103);
  addPixelParamID("qstr~it", 104);
  addPixelParamID("qstr~nl", 105);
  addPixelParamID("qstr~es", 106);
}

size_t LogJoin::numSessions() const {
  return sessions_flush_times_.size();
}

size_t LogJoin::cacheSize() const {
  return session_cache_.size();
}

void LogJoin::insertLogline(
      const std::string& log_line,
      mdb::MDBTransaction* txn) {
  auto c_end = log_line.find("|");
  if (c_end == std::string::npos) {
    RAISEF(kRuntimeError, "invalid logline: $0", log_line);
  }

  auto t_end = log_line.find("|", c_end + 1);
  if (t_end == std::string::npos) {
    RAISEF(kRuntimeError, "invalid logline: $0", log_line);
  }

  auto customer_key = log_line.substr(0, c_end);
  auto body = log_line.substr(t_end + 1);
  auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
  fnord::DateTime time(std::stoul(timestr) * fnord::kMicrosPerSecond);

  insertLogline(customer_key, time, body, txn);
}

void LogJoin::insertLogline(
    const std::string& customer_key,
    const fnord::DateTime& time,
    const std::string& log_line,
    mdb::MDBTransaction* txn) {
  fnord::URI::ParamList params;
  fnord::URI::parseQueryString(log_line, &params);

  stat_loglines_total_.incr(1);

  try {
    /* extract uid (userid) and eid (eventid) */
    std::string c;
    if (!fnord::URI::getParam(params, "c", &c)) {
      RAISE(kParseError, "c param is missing");
    }

    auto c_s = c.find("~");
    if (c_s == std::string::npos) {
      RAISE(kParseError, "c param is invalid");
    }

    std::string uid = c.substr(0, c_s);
    std::string eid = c.substr(c_s + 1, c.length() - c_s - 1);
    if (uid.length() == 0 || eid.length() == 0) {
      RAISE(kParseError, "c param is invalid");
    }

    if (!shard_.testUID(uid)) {
#ifndef NDEBUG
      fnord::logTrace(
          "cm.logjoin",
          "dropping logline with uid=$0 because it does not match my shard",
          uid);
#endif
      return;
    }

    std::string evtype;
    if (!fnord::URI::getParam(params, "e", &evtype) || evtype.length() != 1) {
      RAISE(kParseError, "e param is missing");
    }

    if (evtype.length() != 1) {
      RAISE(kParseError, "e param invalid");
    }

    /* process event */
    switch (evtype[0]) {

      case 'q':
      case 'v':
      case 'c':
      case 'u':
        break;

      default:
        RAISE(kParseError, "invalid e param");

    };

    URI::ParamList stored_params;
    for (const auto& p : params) {
      if (p.first == "c" || p.first == "e" || p.first == "v") {
        continue;
      }

      stored_params.emplace_back(p);
    }

    appendToSession(customer_key, time, uid, eid, evtype, stored_params, txn);
  } catch (...) {
    stat_loglines_invalid_.incr(1);
    throw;
  }
}

void LogJoin::appendToSession(
    const std::string& customer_key,
    const fnord::DateTime& time,
    const std::string& uid,
    const std::string& evid,
    const std::string& evtype,
    const Vector<Pair<String, String>>& logline,
    mdb::MDBTransaction* txn) {

  auto flush_at = time.unixMicros() +
      kSessionIdleTimeoutSeconds * fnord::kMicrosPerSecond;

  bool new_session = sessions_flush_times_.count(uid) == 0;
  sessions_flush_times_.emplace(uid, flush_at);

  util::BinaryMessageWriter buf;
  buf.appendVarUInt(time.unixMicros() / kMicrosPerSecond);
  buf.appendVarUInt(evid.length());
  buf.append(evid.data(), evid.length());
  for (const auto& p : logline) {
    buf.appendVarUInt(getPixelParamID(p.first));
    buf.appendVarUInt(p.second.size());
    buf.append(p.second.data(), p.second.size());
  }

  auto evkey = uid + "~" + evtype;
  txn->insert(evkey.data(), evkey.size(), buf.data(), buf.size());
  txn->update(uid + "~cust", customer_key);
}

void LogJoin::flush(mdb::MDBTransaction* txn, DateTime stream_time_) {
  auto stream_time = stream_time_.unixMicros();

  for (auto iter = sessions_flush_times_.begin();
      iter != sessions_flush_times_.end();) {
    if (iter->second.unixMicros() < stream_time) {
      flushSession(iter->first, stream_time, txn);
      iter = sessions_flush_times_.erase(iter);
    } else {
      ++iter;
    }
  }
}

void LogJoin::flushSession(
    const std::string uid,
    DateTime stream_time,
    mdb::MDBTransaction* txn) {
  auto cursor = txn->getCursor();

  TrackedSession session;
  session.uid = uid;

  Buffer key;
  Buffer value;
  bool eof = false;
  for (int i = 0; ; ++i) {
    if (i == 0) {
      key.append(uid);
      if (!cursor->getFirstOrGreater(&key, &value)) {
        break;
      }
    } else {
      if (!cursor->getNext(&key, &value)) {
        break;
      }
    }

    auto key_str = key.toString();
    if (!StringUtil::beginsWith(key_str, uid)) {
      break;
    }

    if (StringUtil::endsWith(key_str, "~cust")) {
      session.customer_key = value.toString();
    } else {
      auto evtype = key_str.substr(uid.length() + 1);

      util::BinaryMessageReader reader(value.data(), value.size());
      auto time = reader.readVarUInt() * kMicrosPerSecond;
      auto evid_len = reader.readVarUInt();
      auto evid = String((char*) reader.read(evid_len), evid_len);

      try {
        URI::ParamList logline;
        while (reader.remaining() > 0) {
          auto key = getPixelParamName(reader.readVarUInt());
          auto len = reader.readVarUInt();
          logline.emplace_back(key, String((char*) reader.read(len), len));
        }

        session.insertLogline(time, evtype, evid, logline);
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid logline");
        stat_loglines_invalid_.incr(1);
      }
    }

    txn->del(key);
  }

  cursor->close();

  if (session.customer_key.length() == 0) {
    fnord::logError("cm.logjoin", "missing customer key for: $0", uid);
    return;
  }

  try {
    target_->onSession(txn, session);
  } catch (const std::exception& e) {
    fnord::logError("cm.logjoin", e, "LogJoinTarget::onSession crashed");
  }

  stat_joined_sessions_.incr(1);
}

void LogJoin::importTimeoutList(mdb::MDBTransaction* txn) {
  Buffer key;
  Buffer value;
  int n = 0;

  auto cursor = txn->getCursor();

  for (;;) {
    bool eof;
    if (n++ == 0) {
      eof = !cursor->getFirst(&key, &value);
    } else {
      eof = !cursor->getNext(&key, &value);
    }

    if (eof) {
      break;
    }

    if (StringUtil::beginsWith(key.toString(), "__")) {
      continue;
    }

    auto sid = key.toString();
    if (StringUtil::endsWith(sid, "~cust")) {
      continue;
    }

    auto evtype = sid.substr(sid.find("~") + 1);
    sid.erase(sid.end() - evtype.size() - 1, sid.end());

    util::BinaryMessageReader reader(value.data(), value.size());
    auto time = reader.readVarUInt();
    auto ftime = (time + kSessionIdleTimeoutSeconds) * fnord::kMicrosPerSecond;

    auto old_ftime = sessions_flush_times_.find(sid);
    if (old_ftime == sessions_flush_times_.end() ||
        old_ftime->second.unixMicros() < ftime) {
      sessions_flush_times_.emplace(sid, ftime);
    }
  }

  cursor->close();
}

void LogJoin::addPixelParamID(const String& param, uint32_t id) {
  pixel_param_ids_[param] = id;
  pixel_param_names_[id] = param;
}

uint32_t LogJoin::getPixelParamID(const String& param) const {
  auto p = pixel_param_ids_.find(param);

  if (p == pixel_param_ids_.end()) {
    RAISEF(kIndexError, "invalid pixel param: $0", param);
  }

  return p->second;
}

const String& LogJoin::getPixelParamName(uint32_t id) const {
  auto p = pixel_param_names_.find(id);

  if (p == pixel_param_names_.end()) {
    RAISEF(kIndexError, "invalid pixel param: $0", id);
  }

  return p->second;
}

void LogJoin::exportStats(const std::string& prefix) {
  exportStat(
      StringUtil::format("$0/$1", prefix, "loglines_total"),
      &stat_loglines_total_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      StringUtil::format("$0/$1", prefix, "loglines_invalid"),
      &stat_loglines_invalid_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      StringUtil::format("$0/$1", prefix, "joined_sessions"),
      &stat_joined_sessions_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      StringUtil::format("$0/$1", prefix, "joined_queries"),
      &stat_joined_queries_,
      fnord::stats::ExportMode::EXPORT_DELTA);

  exportStat(
      StringUtil::format("$0/$1", prefix, "joined_item_visits"),
      &stat_joined_item_visits_,
      fnord::stats::ExportMode::EXPORT_DELTA);
}

void LogJoin::setTurbo(bool turbo) {
  turbo_ = turbo;
}

} // namespace cm
<|endoftext|>


<commit_before>#include "vtkExodusIIReader.h"
#include "vtkGmshReader.h"
#include "vtkXMLUnstructuredGridReader.h"
#include "vtkXMLPUnstructuredGridReader.h"
#include "vtkMultiBlockDataSet.h"
#include "vtkUnstructuredGrid.h"
#include "vtkXMLUnstructuredGridWriter.h"
#include "vtkXMLMultiBlockDataWriter.h"
#include "vtkGmshWriter.h"
#include "vtkAppendFilter.h"
#include "vtkCellData.h"
#include "vtkIntArray.h"
#include "vtkIdTypeArray.h"

#include <unistd.h>
#include <stdio.h>

#include "mesh_converter.h"

int main(int argc, char *argv[]) {

  char* input_format="exodus";
  char* output_format="gmsh";
  int opt;

  if (argc <3) {
    print_usage();
    return 0;
  }

  while (  (opt = getopt(argc, argv, "i::o::h")) != -1 ) {
    switch ( opt ) {
    case 'i':
      if (optarg) {
	input_format = optarg;
      }
      break;
    case 'o':
      if (optarg) {
	output_format = optarg;
      }
      break;
    case 'h':
      print_usage();
      break;
    case '?':
      cerr << "Unknown option: '" << char(optopt) << "'!" << endl;
      break;
    }
  }

  vtkMultiBlockDataSet* mbdata = NULL;
  vtkUnstructuredGrid* ugdata  = NULL;

  if (strcmp(input_format,"exodus")==0 ) {
    mbdata = read_exodusII(argv[optind++]);
  }  else if (strcmp(input_format,"gmsh")==0 ) {
    ugdata = read_gmsh(argv[optind++]);
  }  else if (strcmp(input_format,"vtu")==0 ) {
    ugdata = read_vtu(argv[optind++]);
  }  else if (strcmp(input_format,"pvtu")==0 ) {
    ugdata = read_pvtu(argv[optind++]);
  }  else {
    std::cout<< "Unrecognised input format: "<< input_format << std::endl;
    return 1;
  }

  int flag;

  if (strcmp(output_format,"gmsh")==0) {
    if (mbdata) {
      ugdata=multiblock_to_unstucturedgrid(mbdata);
    }
    flag=write_gmsh(ugdata,argv[optind]);
  } else if (strcmp(output_format,"vtu")==0) {
    if (mbdata) {
      ugdata=multiblock_to_unstucturedgrid(mbdata);
    }
    flag=write_vtu(ugdata,argv[optind]);
  }  else if (strcmp(output_format,"vtm")==0) {
    if (ugdata) {
      //      mbdata=unstucturedgrid_to_multiblock(ugdata);
    }
    flag=write_vtm(mbdata,argv[optind]);
  } else {
    std::cout<< "Unrecognised output format: "<<output_format<<std::endl;
    return 1;
  }

  if (mbdata){
    mbdata->Delete();
  }
  if (ugdata){
    ugdata->Delete();
  }

  return flag;
}

vtkUnstructuredGrid* multiblock_to_unstucturedgrid(vtkMultiBlockDataSet* data) {
  vtkAppendFilter* appender = vtkAppendFilter::New();
  appender->SetMergePoints(1);

  vtkMultiBlockDataSet* sidesets = vtkMultiBlockDataSet::SafeDownCast(data->GetBlock(4));
  for (int i=0; i<sidesets->GetNumberOfBlocks(); i++) {
    vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::SafeDownCast(sidesets->GetBlock(i));
    vtkIntArray* eids = vtkIntArray::New();
    eids->SetNumberOfValues(ugrid->GetNumberOfCells());
    ugrid->GetCellData()->GetArray("ObjectId")->SetName("PhysicalIds");
    for (int j=0; j<ugrid->GetNumberOfCells(); j++) {
      eids->SetValue(j,i+1);
    }
    eids->SetName("ElementaryEntities");
    ugrid->GetCellData()->AddArray(eids);
    appender->AddInputData(ugrid);
  }

  vtkMultiBlockDataSet* regions = vtkMultiBlockDataSet::SafeDownCast(data->GetBlock(0));
  for (int i=0; i<regions->GetNumberOfBlocks(); i++) {
    vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::SafeDownCast(regions->GetBlock(i));
    ugrid->GetCellData()->GetArray("ObjectId")->SetName("PhysicalIds");
    ugrid->GetCellData()->GetArray("GlobalElementId")->SetName("ElementaryEntities");
    appender->AddInputData(ugrid);
  }

  appender->Update();
  vtkUnstructuredGrid* output = vtkUnstructuredGrid::New();
  output->ShallowCopy(appender->GetOutput());
  appender->Delete();

  return output;
}

vtkMultiBlockDataSet* read_exodusII(char* fname){

  std::cout << "Reading from file: " <<fname<<std::endl;

  vtkExodusIIReader* r = vtkExodusIIReader::New();

  r->SetFileName(fname);
  r->UpdateInformation();
  r->GenerateGlobalNodeIdArrayOn();
  r->GenerateGlobalElementIdArrayOn();
  r->GenerateObjectIdCellArrayOn();
  for (int i=0; i<r->GetNumberOfSideSetArrays(); i++) {
    r->SetSideSetArrayStatus(r->GetSideSetArrayName(i),1);
  }
  r->Update();

  vtkMultiBlockDataSet* data;
  data = r->GetOutput();

  return data;
 }

 vtkUnstructuredGrid* read_gmsh(char* fname) {
   std::cout << "Reading from GMSH file: " <<fname<<std::endl;
   vtkGmshReader* reader= vtkGmshReader::New();
   reader->SetFileName(fname);
   reader->Update();
   vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::New();
   ugrid->ShallowCopy(reader->GetOutput());
   reader->Delete();
   return ugrid;
 }

 vtkUnstructuredGrid* read_vtu(char* fname) {
   std::cout << "Reading from VTK unstructured grid file: " <<fname<<std::endl;
   vtkXMLUnstructuredGridReader* reader= vtkXMLUnstructuredGridReader::New();
   reader->SetFileName(fname);
   reader->Update();
   vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::New();
   ugrid->ShallowCopy(reader->GetOutput());
   reader->Delete();
   return ugrid;
 }

vtkUnstructuredGrid* read_pvtu(char* fname) {
   std::cout << "Reading from VTK parallel unstructured grid file: " <<fname<<std::endl;
   vtkXMLPUnstructuredGridReader* reader= vtkXMLPUnstructuredGridReader::New();
   reader->SetFileName(fname);
   reader->Update();
   vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::New();
   ugrid->ShallowCopy(reader->GetOutput());
   reader->Delete();
   return ugrid;
 }

int print_usage(){
  std::cout << "usage: mesh_converter [-i input_format] [-o output_format] [-h] input_file output_file"<<std::endl;
  return 0;
}

int write_gmsh(vtkUnstructuredGrid* data, char* fname){

  std::cout << "Writing to file: " <<fname<<std::endl;

  vtkGmshWriter* writer = vtkGmshWriter::New();
  writer->SetFileName(fname);
  writer->SetInputData(data);
  writer->Write();
  writer->Delete();
  return 0;
}

int write_vtu(vtkUnstructuredGrid* data, char* fname){

  std::cout << "Writing to file: " <<fname<<std::endl;

  vtkXMLUnstructuredGridWriter* writer = vtkXMLUnstructuredGridWriter::New();
  writer->SetFileName(fname);
  writer->SetInputData(data);
  writer->Write();
  writer->Delete();
  return 0;
}

int write_vtm(vtkMultiBlockDataSet* data, char* fname){

  std::cout << "Writing to file: " <<fname<<std::endl;

  vtkXMLMultiBlockDataWriter* writer = vtkXMLMultiBlockDataWriter::New();
  writer->SetFileName(fname);
  writer->SetInputData(data);
  writer->Write();
  writer->Delete();
  return 0;
}

<commit_msg>Clear that strings warning.<commit_after>#include "vtkExodusIIReader.h"
#include "vtkGmshReader.h"
#include "vtkXMLUnstructuredGridReader.h"
#include "vtkXMLPUnstructuredGridReader.h"
#include "vtkMultiBlockDataSet.h"
#include "vtkUnstructuredGrid.h"
#include "vtkXMLUnstructuredGridWriter.h"
#include "vtkXMLMultiBlockDataWriter.h"
#include "vtkGmshWriter.h"
#include "vtkAppendFilter.h"
#include "vtkCellData.h"
#include "vtkIntArray.h"
#include "vtkIdTypeArray.h"

#include <unistd.h>
#include <stdio.h>
#include <string>

#include "mesh_converter.h"

int main(int argc, char *argv[]) {

  std::string input_format="exodus";
  std::string output_format="gmsh";
  int opt;

  if (argc <3) {
    print_usage();
    return 0;
  }

  while (  (opt = getopt(argc, argv, "i::o::h")) != -1 ) {
    switch ( opt ) {
    case 'i':
      if (optarg) {
	input_format = optarg;
      }
      break;
    case 'o':
      if (optarg) {
	output_format = optarg;
      }
      break;
    case 'h':
      print_usage();
      break;
    case '?':
      cerr << "Unknown option: '" << char(optopt) << "'!" << endl;
      break;
    }
  }

  vtkMultiBlockDataSet* mbdata = NULL;
  vtkUnstructuredGrid* ugdata  = NULL;

  if (input_format.compare("exodus")==0 ) {
    mbdata = read_exodusII(argv[optind++]);
  }  else if (input_format.compare("gmsh")==0 ) {
    ugdata = read_gmsh(argv[optind++]);
  }  else if (input_format.compare("vtu")==0 ) {
    ugdata = read_vtu(argv[optind++]);
  }  else if (input_format.compare("pvtu")==0 ) {
    ugdata = read_pvtu(argv[optind++]);
  }  else {
    std::cout<< "Unrecognised input format: "<< input_format << std::endl;
    return 1;
  }

  int flag;

  if (output_format.compare("gmsh")==0) {
    if (mbdata) {
      ugdata=multiblock_to_unstucturedgrid(mbdata);
    }
    flag=write_gmsh(ugdata,argv[optind]);
  } else if (output_format.compare("vtu")==0) {
    if (mbdata) {
      ugdata=multiblock_to_unstucturedgrid(mbdata);
    }
    flag=write_vtu(ugdata,argv[optind]);
  }  else if (output_format.compare("vtm")==0) {
    if (ugdata) {
      //      mbdata=unstucturedgrid_to_multiblock(ugdata);
    }
    flag=write_vtm(mbdata,argv[optind]);
  } else {
    std::cout<< "Unrecognised output format: "<<output_format<<std::endl;
    return 1;
  }

  if (mbdata){
    mbdata->Delete();
  }
  if (ugdata){
    ugdata->Delete();
  }

  return flag;
}

vtkUnstructuredGrid* multiblock_to_unstucturedgrid(vtkMultiBlockDataSet* data) {
  vtkAppendFilter* appender = vtkAppendFilter::New();
  appender->SetMergePoints(1);

  vtkMultiBlockDataSet* sidesets = vtkMultiBlockDataSet::SafeDownCast(data->GetBlock(4));
  for (int i=0; i<sidesets->GetNumberOfBlocks(); i++) {
    vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::SafeDownCast(sidesets->GetBlock(i));
    vtkIntArray* eids = vtkIntArray::New();
    eids->SetNumberOfValues(ugrid->GetNumberOfCells());
    ugrid->GetCellData()->GetArray("ObjectId")->SetName("PhysicalIds");
    for (int j=0; j<ugrid->GetNumberOfCells(); j++) {
      eids->SetValue(j,i+1);
    }
    eids->SetName("ElementaryEntities");
    ugrid->GetCellData()->AddArray(eids);
    appender->AddInputData(ugrid);
  }

  vtkMultiBlockDataSet* regions = vtkMultiBlockDataSet::SafeDownCast(data->GetBlock(0));
  for (int i=0; i<regions->GetNumberOfBlocks(); i++) {
    vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::SafeDownCast(regions->GetBlock(i));
    ugrid->GetCellData()->GetArray("ObjectId")->SetName("PhysicalIds");
    ugrid->GetCellData()->GetArray("GlobalElementId")->SetName("ElementaryEntities");
    appender->AddInputData(ugrid);
  }

  appender->Update();
  vtkUnstructuredGrid* output = vtkUnstructuredGrid::New();
  output->ShallowCopy(appender->GetOutput());
  appender->Delete();

  return output;
}

vtkMultiBlockDataSet* read_exodusII(char* fname){

  std::cout << "Reading from file: " <<fname<<std::endl;

  vtkExodusIIReader* r = vtkExodusIIReader::New();

  r->SetFileName(fname);
  r->UpdateInformation();
  r->GenerateGlobalNodeIdArrayOn();
  r->GenerateGlobalElementIdArrayOn();
  r->GenerateObjectIdCellArrayOn();
  for (int i=0; i<r->GetNumberOfSideSetArrays(); i++) {
    r->SetSideSetArrayStatus(r->GetSideSetArrayName(i),1);
  }
  r->Update();

  vtkMultiBlockDataSet* data;
  data = r->GetOutput();

  return data;
 }

 vtkUnstructuredGrid* read_gmsh(char* fname) {
   std::cout << "Reading from GMSH file: " <<fname<<std::endl;
   vtkGmshReader* reader= vtkGmshReader::New();
   reader->SetFileName(fname);
   reader->Update();
   vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::New();
   ugrid->ShallowCopy(reader->GetOutput());
   reader->Delete();
   return ugrid;
 }

 vtkUnstructuredGrid* read_vtu(char* fname) {
   std::cout << "Reading from VTK unstructured grid file: " <<fname<<std::endl;
   vtkXMLUnstructuredGridReader* reader= vtkXMLUnstructuredGridReader::New();
   reader->SetFileName(fname);
   reader->Update();
   vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::New();
   ugrid->ShallowCopy(reader->GetOutput());
   reader->Delete();
   return ugrid;
 }

vtkUnstructuredGrid* read_pvtu(char* fname) {
   std::cout << "Reading from VTK parallel unstructured grid file: " <<fname<<std::endl;
   vtkXMLPUnstructuredGridReader* reader= vtkXMLPUnstructuredGridReader::New();
   reader->SetFileName(fname);
   reader->Update();
   vtkUnstructuredGrid* ugrid = vtkUnstructuredGrid::New();
   ugrid->ShallowCopy(reader->GetOutput());
   reader->Delete();
   return ugrid;
 }

int print_usage(){
  std::cout << "usage: mesh_converter [-i input_format] [-o output_format] [-h] input_file output_file"<<std::endl;
  return 0;
}

int write_gmsh(vtkUnstructuredGrid* data, char* fname){

  std::cout << "Writing to file: " <<fname<<std::endl;

  vtkGmshWriter* writer = vtkGmshWriter::New();
  writer->SetFileName(fname);
  writer->SetInputData(data);
  writer->Write();
  writer->Delete();
  return 0;
}

int write_vtu(vtkUnstructuredGrid* data, char* fname){

  std::cout << "Writing to file: " <<fname<<std::endl;

  vtkXMLUnstructuredGridWriter* writer = vtkXMLUnstructuredGridWriter::New();
  writer->SetFileName(fname);
  writer->SetInputData(data);
  writer->Write();
  writer->Delete();
  return 0;
}

int write_vtm(vtkMultiBlockDataSet* data, char* fname){

  std::cout << "Writing to file: " <<fname<<std::endl;

  vtkXMLMultiBlockDataWriter* writer = vtkXMLMultiBlockDataWriter::New();
  writer->SetFileName(fname);
  writer->SetInputData(data);
  writer->Write();
  writer->Delete();
  return 0;
}

<|endoftext|>


<commit_before>#include "trainer.h"
#include "common/timer.h"
#include "common/logger.h"
#include "optimize/opt_gd.hpp"
#include "optimize/opt_cgd.hpp"
#include "optimize/opt_lbfgs.hpp"
#include "sampler.h"
#include "accumulator.h"

namespace ncv
{
        /////////////////////////////////////////////////////////////////////////////////////////

        trainer_state_t::trainer_state_t(size_t n_parameters)
                :       m_params(n_parameters),
                        m_tvalue(std::numeric_limits<scalar_t>::max()),
                        m_terror(std::numeric_limits<scalar_t>::max()),
                        m_vvalue(std::numeric_limits<scalar_t>::max()),
                        m_verror(std::numeric_limits<scalar_t>::max()),
                        m_lambda(std::numeric_limits<scalar_t>::max())
        {
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        bool trainer_state_t::update(const vector_t& params,
                    scalar_t tvalue, scalar_t terror,
                    scalar_t vvalue, scalar_t verror,
                    scalar_t lambda)
        {
                if (verror < m_verror)
                {
                        m_params = params;
                        m_tvalue = tvalue;
                        m_terror = terror;
                        m_vvalue = vvalue;
                        m_verror = verror;
                        m_lambda = lambda;
                        return true;
                }

                else
                {
                        return false;
                }
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        bool trainer_state_t::update(const trainer_state_t& state)
        {
                return update(state.m_params,
                              state.m_tvalue, state.m_terror, state.m_vvalue, state.m_verror,
                              state.m_lambda);
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        bool trainer_t::train(
                const task_t& task, const sampler_t& tsampler, const sampler_t& vsampler, size_t nthreads,
                const loss_t& loss, const string_t& optimizer, size_t iterations, scalar_t epsilon,
                const string_t& regularizer, const model_t& model, trainer_state_t& state)
        {
                // no regularization
                if (regularizer == "none")
                {
                        accumulator_t ldata(model,
                                            accumulator_t::type::value,
                                            accumulator_t::regularizer::none);
                        accumulator_t gdata(model,
                                            accumulator_t::type::vgrad,
                                            accumulator_t::regularizer::none);

                        trainer_t::train(task, tsampler, vsampler, nthreads,
                                         loss, optimizer, iterations, epsilon,
                                         model.params(), ldata, gdata, state);
                }

                // L2-norm regularization
                else if (regularizer == "l2")
                {
                        const scalars_t lambdas = { 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1.0 };

                        // regularize the loss
                        trainer_state_t state(model.psize());
                        for (scalar_t lambda : lambdas)
                        {
                                accumulator_t ldata(model,
                                                    accumulator_t::type::value,
                                                    accumulator_t::regularizer::l2norm, lambda);
                                accumulator_t gdata(model,
                                                    accumulator_t::type::vgrad,
                                                    accumulator_t::regularizer::l2norm, lambda);

                                trainer_t::train(task, tsampler, vsampler, nthreads,
                                                 loss, optimizer, iterations, epsilon,
                                                 model.params(), ldata, gdata, state);
                        }
                }

                // variational regularization
                else if (regularizer == "var")
                {
                        const scalars_t lambdas = { 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1.0 };

                        // regularize the loss
                        trainer_state_t state(model.psize());
                        for (scalar_t lambda : lambdas)
                        {
                                accumulator_t ldata(model,
                                                    accumulator_t::type::value,
                                                    accumulator_t::regularizer::variational, lambda);
                                accumulator_t gdata(model,
                                                    accumulator_t::type::vgrad,
                                                    accumulator_t::regularizer::variational, lambda);

                                trainer_t::train(task, tsampler, vsampler, nthreads,
                                                 loss, optimizer, iterations, epsilon,
                                                 model.params(), ldata, gdata, state);
                        }
                }

                else
                {
                        log_error() << "trainer: invalid regularization method <" << regularizer << ">!";
                        return false;
                }

                // OK
                return true;
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        bool trainer_t::train(
                const task_t& task, const sampler_t& tsampler, const sampler_t& vsampler, size_t nthreads,
                const loss_t& loss, const string_t& optimizer, size_t iterations, scalar_t epsilon,
                const vector_t& x0, accumulator_t& ldata, accumulator_t& gdata, trainer_state_t& state)
        {
                samples_t utsamples = tsampler.get();
                samples_t uvsamples = vsampler.get();

                // construct the optimization problem
                const timer_t timer;

                auto fn_size = [&] ()
                {
                        return ldata.dimensions();
                };

                auto fn_fval = [&] (const vector_t& x)
                {
                        // training samples: loss value
                        ldata.reset(x);
                        ldata.update_mt(task, utsamples, loss, nthreads);
                        const scalar_t tvalue = ldata.value();

                        return tvalue;
                };

                auto fn_fval_grad = [&] (const vector_t& x, vector_t& gx)
                {
                        // stochastic mode: resample training & validation samples
                        if (tsampler.is_random())
                        {
                                utsamples = tsampler.get();
                        }
                        if (vsampler.is_random())
                        {
                                uvsamples = vsampler.get();
                        }

                        // training samples: loss value & gradient
                        gdata.reset(x);
                        gdata.update_mt(task, utsamples, loss, nthreads);
                        const scalar_t tvalue = gdata.value();
                        const scalar_t terror = gdata.error();
                        gx = gdata.vgrad();

                        // validation samples: loss value
                        ldata.reset(x);
                        ldata.update_mt(task, uvsamples, loss, nthreads);
                        const scalar_t vvalue = ldata.value();
                        const scalar_t verror = ldata.error();

                        // update the optimum state
                        state.update(x, tvalue, terror, vvalue, verror, ldata.lambda());

                        return tvalue;
                };

                auto fn_wlog = [] (const string_t& message)
                {
                        log_warning() << message;
                };
                auto fn_elog = [] (const string_t& message)
                {
                        log_error() << message;
                };
                auto fn_ulog = [&] (const opt_state_t& result, const timer_t& timer)
                {
                        log_info() << "[loss = " << result.f
                                   << ", grad = " << result.g.lpNorm<Eigen::Infinity>()
                                   << ", funs = " << result.n_fval_calls() << "/" << result.n_grad_calls()
                                   << ", train* = " << state.m_tvalue << "/" << state.m_terror
                                   << ", valid* = " << state.m_vvalue << "/" << state.m_verror
                                   << ", lambda* = " << ldata.lambda() << "/" << state.m_lambda
                                   << "] done in " << timer.elapsed() << ".";
                };

                // assembly optimization problem & optimize the model
                const opt_problem_t problem(fn_size, fn_fval, fn_fval_grad);

                const opt_opulog_t fn_ulog_ref = std::bind(fn_ulog, _1, std::ref(timer));

                if (text::iequals(optimizer, "lbfgs"))
                {
                        optimize::lbfgs(problem, x0, iterations, epsilon, fn_wlog, fn_elog, fn_ulog_ref);
                }
                else if (text::iequals(optimizer, "cgd"))
                {
                        optimize::cgd_hs(problem, x0, iterations, epsilon, fn_wlog, fn_elog, fn_ulog_ref);
                }
                else if (text::iequals(optimizer, "gd"))
                {
                        optimize::gd(problem, x0, iterations, epsilon, fn_wlog, fn_elog, fn_ulog_ref);
                }
                else
                {
                        log_error() << "trainer: invalid optimization method <" << optimizer << ">!";
                        return false;
                }

                // OK
                return true;
        }

        /////////////////////////////////////////////////////////////////////////////////////////
}

<commit_msg>fix regularization trainer versions<commit_after>#include "trainer.h"
#include "common/timer.h"
#include "common/logger.h"
#include "optimize/opt_gd.hpp"
#include "optimize/opt_cgd.hpp"
#include "optimize/opt_lbfgs.hpp"
#include "sampler.h"
#include "accumulator.h"

namespace ncv
{
        /////////////////////////////////////////////////////////////////////////////////////////

        trainer_state_t::trainer_state_t(size_t n_parameters)
                :       m_params(n_parameters),
                        m_tvalue(std::numeric_limits<scalar_t>::max()),
                        m_terror(std::numeric_limits<scalar_t>::max()),
                        m_vvalue(std::numeric_limits<scalar_t>::max()),
                        m_verror(std::numeric_limits<scalar_t>::max()),
                        m_lambda(std::numeric_limits<scalar_t>::max())
        {
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        bool trainer_state_t::update(const vector_t& params,
                    scalar_t tvalue, scalar_t terror,
                    scalar_t vvalue, scalar_t verror,
                    scalar_t lambda)
        {
                if (verror < m_verror)
                {
                        m_params = params;
                        m_tvalue = tvalue;
                        m_terror = terror;
                        m_vvalue = vvalue;
                        m_verror = verror;
                        m_lambda = lambda;
                        return true;
                }

                else
                {
                        return false;
                }
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        bool trainer_state_t::update(const trainer_state_t& state)
        {
                return update(state.m_params,
                              state.m_tvalue, state.m_terror, state.m_vvalue, state.m_verror,
                              state.m_lambda);
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        bool trainer_t::train(
                const task_t& task, const sampler_t& tsampler, const sampler_t& vsampler, size_t nthreads,
                const loss_t& loss, const string_t& optimizer, size_t iterations, scalar_t epsilon,
                const string_t& regularizer, const model_t& model, trainer_state_t& state)
        {
                // no regularization
                if (regularizer == "none")
                {
                        accumulator_t ldata(model,
                                            accumulator_t::type::value,
                                            accumulator_t::regularizer::none);
                        accumulator_t gdata(model,
                                            accumulator_t::type::vgrad,
                                            accumulator_t::regularizer::none);

                        trainer_t::train(task, tsampler, vsampler, nthreads,
                                         loss, optimizer, iterations, epsilon,
                                         model.params(), ldata, gdata, state);
                }

                // L2-norm regularization
                else if (regularizer == "l2")
                {
                        const scalars_t lambdas = { 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1.0 };

                        // regularize the loss
                        for (scalar_t lambda : lambdas)
                        {
                                accumulator_t ldata(model,
                                                    accumulator_t::type::value,
                                                    accumulator_t::regularizer::l2norm, lambda);
                                accumulator_t gdata(model,
                                                    accumulator_t::type::vgrad,
                                                    accumulator_t::regularizer::l2norm, lambda);

                                trainer_t::train(task, tsampler, vsampler, nthreads,
                                                 loss, optimizer, iterations, epsilon,
                                                 model.params(), ldata, gdata, state);
                        }
                }

                // variational regularization
                else if (regularizer == "var")
                {
                        const scalars_t lambdas = { 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1.0 };

                        // regularize the loss
                        for (scalar_t lambda : lambdas)
                        {
                                accumulator_t ldata(model,
                                                    accumulator_t::type::value,
                                                    accumulator_t::regularizer::variational, lambda);
                                accumulator_t gdata(model,
                                                    accumulator_t::type::vgrad,
                                                    accumulator_t::regularizer::variational, lambda);

                                trainer_t::train(task, tsampler, vsampler, nthreads,
                                                 loss, optimizer, iterations, epsilon,
                                                 model.params(), ldata, gdata, state);
                        }
                }

                else
                {
                        log_error() << "trainer: invalid regularization method <" << regularizer << ">!";
                        return false;
                }

                // OK
                return true;
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        bool trainer_t::train(
                const task_t& task, const sampler_t& tsampler, const sampler_t& vsampler, size_t nthreads,
                const loss_t& loss, const string_t& optimizer, size_t iterations, scalar_t epsilon,
                const vector_t& x0, accumulator_t& ldata, accumulator_t& gdata, trainer_state_t& state)
        {
                samples_t utsamples = tsampler.get();
                samples_t uvsamples = vsampler.get();

                // construct the optimization problem
                const timer_t timer;

                auto fn_size = [&] ()
                {
                        return ldata.dimensions();
                };

                auto fn_fval = [&] (const vector_t& x)
                {
                        // training samples: loss value
                        ldata.reset(x);
                        ldata.update_mt(task, utsamples, loss, nthreads);
                        const scalar_t tvalue = ldata.value();

                        return tvalue;
                };

                auto fn_fval_grad = [&] (const vector_t& x, vector_t& gx)
                {
                        // stochastic mode: resample training & validation samples
                        if (tsampler.is_random())
                        {
                                utsamples = tsampler.get();
                        }
                        if (vsampler.is_random())
                        {
                                uvsamples = vsampler.get();
                        }

                        // training samples: loss value & gradient
                        gdata.reset(x);
                        gdata.update_mt(task, utsamples, loss, nthreads);
                        const scalar_t tvalue = gdata.value();
                        const scalar_t terror = gdata.error();
                        gx = gdata.vgrad();

                        // validation samples: loss value
                        ldata.reset(x);
                        ldata.update_mt(task, uvsamples, loss, nthreads);
                        const scalar_t vvalue = ldata.value();
                        const scalar_t verror = ldata.error();

                        // update the optimum state
                        state.update(x, tvalue, terror, vvalue, verror, ldata.lambda());

                        return tvalue;
                };

                auto fn_wlog = [] (const string_t& message)
                {
                        log_warning() << message;
                };
                auto fn_elog = [] (const string_t& message)
                {
                        log_error() << message;
                };
                auto fn_ulog = [&] (const opt_state_t& result, const timer_t& timer)
                {
                        log_info() << "[loss = " << result.f
                                   << ", grad = " << result.g.lpNorm<Eigen::Infinity>()
                                   << ", funs = " << result.n_fval_calls() << "/" << result.n_grad_calls()
                                   << ", train* = " << state.m_tvalue << "/" << state.m_terror
                                   << ", valid* = " << state.m_vvalue << "/" << state.m_verror
                                   << ", lambda* = " << ldata.lambda() << "/" << state.m_lambda
                                   << "] done in " << timer.elapsed() << ".";
                };

                // assembly optimization problem & optimize the model
                const opt_problem_t problem(fn_size, fn_fval, fn_fval_grad);

                const opt_opulog_t fn_ulog_ref = std::bind(fn_ulog, _1, std::ref(timer));

                if (text::iequals(optimizer, "lbfgs"))
                {
                        optimize::lbfgs(problem, x0, iterations, epsilon, fn_wlog, fn_elog, fn_ulog_ref);
                }
                else if (text::iequals(optimizer, "cgd"))
                {
                        optimize::cgd_hs(problem, x0, iterations, epsilon, fn_wlog, fn_elog, fn_ulog_ref);
                }
                else if (text::iequals(optimizer, "gd"))
                {
                        optimize::gd(problem, x0, iterations, epsilon, fn_wlog, fn_elog, fn_ulog_ref);
                }
                else
                {
                        log_error() << "trainer: invalid optimization method <" << optimizer << ">!";
                        return false;
                }

                // OK
                return true;
        }

        /////////////////////////////////////////////////////////////////////////////////////////
}

<|endoftext|>


<commit_before>#include "braincloud/internal/nix/cURLPinger.h"

#include "curl/curl.h"

#include <chrono>

namespace BrainCloud
{
    IPinger* IPinger::create(BrainCloudClient* pClient)
    {
        return new cURLPinger();
    }

    cURLPinger::cURLPinger() {}

    int cURLPinger::ping(const std::string& url)
    {
        CURL *curl;
        CURLcode res = CURLE_OK;
        curl = curl_easy_init();
        if (!curl)
        {
            return 999;
        }

        curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
        curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 2);
        curl_easy_setopt(curl, CURLOPT_NOBODY, 1);

        auto startTime = std::chrono::high_resolution_clock::now();
        res = curl_easy_perform(curl);
        auto doneTime = std::chrono::high_resolution_clock::now();

        curl_easy_cleanup(curl);

        if (res != CURLE_OK)
        {
            return 999;
        }

        int pingResult = (int)(std::chrono::duration_cast<std::chrono::milliseconds>(doneTime - startTime).count());
        if (pingResult > 999)
        {
            pingResult = 999;
        }

        return pingResult;
    }
}
<commit_msg>Fixed pingregions on linux crashing because of curl<commit_after>#include "braincloud/internal/nix/cURLPinger.h"

#include "curl/curl.h"

#include <chrono>

namespace BrainCloud
{
    IPinger* IPinger::create(BrainCloudClient* pClient)
    {
        return new cURLPinger();
    }

    cURLPinger::cURLPinger() {}

    int cURLPinger::ping(const std::string& url)
    {
        CURL *curl;
        CURLcode res = CURLE_OK;
        curl = curl_easy_init();
        if (!curl)
        {
            return 999;
        }

        curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
        curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 2);
        curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
        curl_easy_setopt(curl, CURLOPT_NOSIGNAL, (long)1);

        auto startTime = std::chrono::high_resolution_clock::now();
        res = curl_easy_perform(curl);
        auto doneTime = std::chrono::high_resolution_clock::now();

        curl_easy_cleanup(curl);

        if (res != CURLE_OK)
        {
            return 999;
        }

        int pingResult = (int)(std::chrono::duration_cast<std::chrono::milliseconds>(doneTime - startTime).count());
        if (pingResult > 999)
        {
            pingResult = 999;
        }

        return pingResult;
    }
}
<|endoftext|>


<commit_before>/**********************************************************************
Copyright (c)2016 Advanced Micro Devices, Inc. All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

?	Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
?	Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or
 other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
********************************************************************/
// to share code with between CPU and GPU

#define MLOPEN
#include <mlopen/mlo_internal.hpp>
#include <mlopen/mlo_utils.hpp>

int mlo_construct_pooling2D::mloConstruct()
{
	int ret = 0;

	if (getDirectcion())
	{

		ret = mloConstructFwd();
	}
	else
	{
		ret = mloConstructBwd();
	}
	return(ret);
}

int mlo_construct_pooling2D::mloConstructFwd()
{
	int ret = 0;

	_grp_tile0 = 8;
	_grp_tile1 = 8;
	int ocl_group_lg2sz0 = static_cast<int>(ceil(log(static_cast<double>(_grp_tile0)) / log(2.)));
	int ocl_group_lg2sz1 = static_cast<int>(ceil(log(static_cast<double>(_grp_tile1)) / log(2.)));;

	_out_pix_tile0 = (_out_width < _grp_tile0 * 2) ? 1 : 2;
	_out_pix_tile1 = (_out_height < _grp_tile1 * 2) ? 1 : 2;

	_comp_options =
		std::string(" -D MLO_POOLING_OP_ID=") + std::to_string(static_cast<long long>(_pooling_method))
		+ std::string(" -D MLO_POOLING_KERNEL_SZ1=") + std::to_string(static_cast<long long>(_kernel_size1))
		+ std::string(" -D MLO_POOLING_PAD1=") + std::to_string(static_cast<long long>(_pad1))
		+ std::string(" -D MLO_POOLING_STRIDE1=") + std::to_string(static_cast<long long>(_kernel_stride1))
		+ std::string(" -D MLO_POOLING_KERNEL_SZ0=") + std::to_string(static_cast<long long>(_kernel_size0))
		+ std::string(" -D MLO_POOLING_PAD0=") + std::to_string(static_cast<long long>(_pad0))
		+ std::string(" -D MLO_POOLING_STRIDE0=") + std::to_string(static_cast<long long>(_kernel_stride0))
		+ std::string(" -D MLO_POOLING_N_OUTPUTS=") + std::to_string(static_cast<long long>(_n_outputs))
		+ std::string(" -D MLO_POOLING_N_CHANNELS=") + std::to_string(static_cast<long long>(_n_inputs))
		+ std::string(" -D MLO_POOLING_N_HORIZ_OUT_PIX=") + std::to_string(static_cast<long long>(_out_pix_tile0))
		+ std::string(" -D MLO_POOLING_N_VERT_OUT_PIX=") + std::to_string(static_cast<long long>(_out_pix_tile1))
		+ std::string(" -D MLO_POOLING_GROUP_SZ0=") + std::to_string(static_cast<long long>(_grp_tile0))
		+ std::string(" -D MLO_POOLING_GROUP_SZ1=") + std::to_string(static_cast<long long>(_grp_tile1))
		+ std::string(" -D MLO_POOLING_GROUP_LG2SZ0=") + std::to_string(static_cast<long long>(ocl_group_lg2sz0))
		+ std::string(" -D MLO_POOLING_GROUP_LG2SZ1=") + std::to_string(static_cast<long long>(ocl_group_lg2sz1))
		+ std::string(" -D MLO_POOLING_BOT_BATCH_STRIDE=") + std::to_string(static_cast<long long>(_in_batch_stride))
		+ std::string(" -D MLO_POOLING_BOT_CHANNEL_STRIDE=") + std::to_string(static_cast<long long>(_in_channel_stride))
		+ std::string(" -D MLO_POOLING_BOT_STRIDE=") + std::to_string(static_cast<long long>(_in_stride))
		+ std::string(" -D MLO_POOLING_TOP_BATCH_STRIDE=") + std::to_string(static_cast<long long>(_out_batch_stride))
		+ std::string(" -D MLO_POOLING_TOP_CHANNEL_STRIDE=") + std::to_string(static_cast<long long>(_out_channel_stride))
		+ std::string(" -D MLO_POOLING_TOP_STRIDE=") + std::to_string(static_cast<long long>(_out_stride))
		+ std::string(" -D MLO_POOLING_BOT_WIDTH=") + std::to_string(static_cast<long long>(_in_width))
		+ std::string(" -D MLO_POOLING_BOT_HEIGHT=") + std::to_string(static_cast<long long>(_in_height))
		+ std::string(" -D MLO_POOLING_TOP_WIDTH=") + std::to_string(static_cast<long long>(_out_width))
		+ std::string(" -D MLO_POOLING_TOP_HEIGHT=") + std::to_string(static_cast<long long>(_out_height))
		+ std::string(_do_backward ? " -D MLO_POOLING_DO_BACKWARD" : "")
		+ getGeneralCompOptions()
		;

	int g_wk_width = ((_out_width + _grp_tile0 * _out_pix_tile0 - 1) / (_grp_tile0 * _out_pix_tile0));
	int g_wk_height = ((_out_height + _grp_tile1 * _out_pix_tile1 - 1) / (_grp_tile1 * _out_pix_tile1));

	_l_wk.clear();
	_l_wk.push_back(_grp_tile0);
	_l_wk.push_back(_grp_tile1);
	_l_wk.push_back(1);

	_g_wk.clear();
	_g_wk.push_back(g_wk_width * _grp_tile0);
	_g_wk.push_back(g_wk_height * _grp_tile1);
	_g_wk.push_back(_n_outputs * _batch_sz);

	_kernel_file = "MLOpenPooling.cl";

	_kernel_name = "mloPooling";

	return(ret);
}

int mlo_construct_pooling2D::mloConstructBwd()
{
	int ret = 0;

	_grp_tile0 = 8;
	_grp_tile1 = 8;

	//_out_pix_tile0 = _kernel_stride0;
	//_out_pix_tile1 = _kernel_stride1;
	_out_pix_tile0 = (_out_width < _grp_tile0 * 2) ? 1 : 2;
	_out_pix_tile1 = (_out_height < _grp_tile1 * 2) ? 1 : 2;

	int ocl_group_lg2sz0 = static_cast<int>(ceil(log(static_cast<double>(_grp_tile0)) / log(2.)));
	int ocl_group_lg2sz1 = static_cast<int>(ceil(log(static_cast<double>(_grp_tile1)) / log(2.)));

	_comp_options =
		std::string(" -D MLO_POOLING_KERNEL_SZ1=") + std::to_string(static_cast<long long>(_kernel_size1))
		+ std::string(" -D MLO_POOLING_PAD1=") + std::to_string(static_cast<long long>(_pad1))
		+ std::string(" -D MLO_POOLING_STRIDE1=") + std::to_string(static_cast<long long>(_kernel_stride1))
		+ std::string(" -D MLO_POOLING_KERNEL_SZ0=") + std::to_string(static_cast<long long>(_kernel_size0))
		+ std::string(" -D MLO_POOLING_PAD0=") + std::to_string(static_cast<long long>(_pad0))
		+ std::string(" -D MLO_POOLING_STRIDE0=") + std::to_string(static_cast<long long>(_kernel_stride0))
		+ std::string(" -D MLO_POOLING_N_OUTPUTS=") + std::to_string(static_cast<long long>(_n_outputs))
		+ std::string(" -D MLO_POOLBWD_N_HORIZ_OUT_PIX=") + std::to_string(static_cast<long long>(_out_pix_tile0))
		+ std::string(" -D MLO_POOLBWD_N_VERT_OUT_PIX=") + std::to_string(static_cast<long long>(_out_pix_tile1))
		+ std::string(" -D MLO_POOLBWD_GROUP_SZ0=") + std::to_string(static_cast<long long>(_grp_tile0))
		+ std::string(" -D MLO_POOLBWD_GROUP_SZ1=") + std::to_string(static_cast<long long>(_grp_tile1))
		+ std::string(" -D MLO_POOLBWD_BOT_WIDTH=") + std::to_string(static_cast<long long>(_in_width))
		+ std::string(" -D MLO_POOLBWD_BOT_HEIGHT=") + std::to_string(static_cast<long long>(_in_height))
		+ std::string(" -D MLO_POOLBWD_TOP_WIDTH=") + std::to_string(static_cast<long long>(_out_width))
		+ std::string(" -D MLO_POOLBWD_TOP_HEIGHT=") + std::to_string(static_cast<long long>(_out_height))
		+ std::string(" -D MLO_POOLBWD_BOTDF_BATCH_STRIDE=") + std::to_string(static_cast<long long>(_in_df_batch_stride))
		+ std::string(" -D MLO_POOLBWD_BOTDF_CHANNEL_STRIDE=") + std::to_string(static_cast<long long>(_in_df_channel_stride))
		+ std::string(" -D MLO_POOLBWD_BOTDF_STRIDE=") + std::to_string(static_cast<long long>(_in_df_stride))
		+ std::string(" -D MLO_POOLBWD_TOPDF_BATCH_STRIDE=") + std::to_string(static_cast<long long>(_out_df_batch_stride))
		+ std::string(" -D MLO_POOLBWD_TOPDF_CHANNEL_STRIDE=") + std::to_string(static_cast<long long>(_out_df_channel_stride))
		+ std::string(" -D MLO_POOLBWD_TOPDF_STRIDE=") + std::to_string(static_cast<long long>(_out_df_stride))

		+ getGeneralCompOptions()
		;

	int g_wk_width = ((_in_width + _grp_tile0 * _out_pix_tile0 - 1) / (_grp_tile0 * _out_pix_tile0));
	int g_wk_height = ((_in_height + _grp_tile1 * _out_pix_tile1 - 1) / (_grp_tile1 * _out_pix_tile1));

	_l_wk.clear();
	_l_wk.push_back(_grp_tile0);
	_l_wk.push_back(_grp_tile1);
	_l_wk.push_back(1);

	_g_wk.clear();
	_g_wk.push_back(g_wk_width * _grp_tile0);
	_g_wk.push_back(g_wk_height * _grp_tile1);
	_g_wk.push_back(_n_inputs * _batch_sz);

	_kernel_file = "MLOpenPoolingBwd.cl";
	if (_pooling_method == MLO_POOLING_OP_MAX)
	{
		_kernel_name = "mloPoolingMaxBwd";
	}
	else if (_pooling_method == MLO_POOLING_OP_AVE)
	{
		_kernel_name = "mloPoolingAveBwd";
	}
	else
	{
		std::cout << "Layer: %s. Error: unknowm method\n";
		ret = -1;
	}
	return(ret);
}
<commit_msg>Remove unused code<commit_after>/**********************************************************************
Copyright (c)2016 Advanced Micro Devices, Inc. All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

?	Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
?	Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or
 other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
********************************************************************/
// to share code with between CPU and GPU

#define MLOPEN
#include <mlopen/mlo_internal.hpp>
#include <mlopen/mlo_utils.hpp>

int mlo_construct_pooling2D::mloConstruct()
{
	int ret = 0;

	if (getDirectcion())
	{

		ret = mloConstructFwd();
	}
	else
	{
		ret = mloConstructBwd();
	}
	return(ret);
}

int mlo_construct_pooling2D::mloConstructFwd()
{
	int ret = 0;

	_grp_tile0 = 8;
	_grp_tile1 = 8;

	_out_pix_tile0 = (_out_width < _grp_tile0 * 2) ? 1 : 2;
	_out_pix_tile1 = (_out_height < _grp_tile1 * 2) ? 1 : 2;

	_comp_options =
		std::string(" -D MLO_POOLING_OP_ID=") + std::to_string(static_cast<long long>(_pooling_method))
		+ std::string(" -D MLO_POOLING_KERNEL_SZ1=") + std::to_string(static_cast<long long>(_kernel_size1))
		+ std::string(" -D MLO_POOLING_PAD1=") + std::to_string(static_cast<long long>(_pad1))
		+ std::string(" -D MLO_POOLING_STRIDE1=") + std::to_string(static_cast<long long>(_kernel_stride1))
		+ std::string(" -D MLO_POOLING_KERNEL_SZ0=") + std::to_string(static_cast<long long>(_kernel_size0))
		+ std::string(" -D MLO_POOLING_PAD0=") + std::to_string(static_cast<long long>(_pad0))
		+ std::string(" -D MLO_POOLING_STRIDE0=") + std::to_string(static_cast<long long>(_kernel_stride0))
		+ std::string(" -D MLO_POOLING_N_OUTPUTS=") + std::to_string(static_cast<long long>(_n_outputs))
		+ std::string(" -D MLO_POOLING_N_CHANNELS=") + std::to_string(static_cast<long long>(_n_inputs))
		+ std::string(" -D MLO_POOLING_N_HORIZ_OUT_PIX=") + std::to_string(static_cast<long long>(_out_pix_tile0))
		+ std::string(" -D MLO_POOLING_N_VERT_OUT_PIX=") + std::to_string(static_cast<long long>(_out_pix_tile1))
		+ std::string(" -D MLO_POOLING_GROUP_SZ0=") + std::to_string(static_cast<long long>(_grp_tile0))
		+ std::string(" -D MLO_POOLING_GROUP_SZ1=") + std::to_string(static_cast<long long>(_grp_tile1))
		+ std::string(" -D MLO_POOLING_BOT_BATCH_STRIDE=") + std::to_string(static_cast<long long>(_in_batch_stride))
		+ std::string(" -D MLO_POOLING_BOT_CHANNEL_STRIDE=") + std::to_string(static_cast<long long>(_in_channel_stride))
		+ std::string(" -D MLO_POOLING_BOT_STRIDE=") + std::to_string(static_cast<long long>(_in_stride))
		+ std::string(" -D MLO_POOLING_TOP_BATCH_STRIDE=") + std::to_string(static_cast<long long>(_out_batch_stride))
		+ std::string(" -D MLO_POOLING_TOP_CHANNEL_STRIDE=") + std::to_string(static_cast<long long>(_out_channel_stride))
		+ std::string(" -D MLO_POOLING_TOP_STRIDE=") + std::to_string(static_cast<long long>(_out_stride))
		+ std::string(" -D MLO_POOLING_BOT_WIDTH=") + std::to_string(static_cast<long long>(_in_width))
		+ std::string(" -D MLO_POOLING_BOT_HEIGHT=") + std::to_string(static_cast<long long>(_in_height))
		+ std::string(" -D MLO_POOLING_TOP_WIDTH=") + std::to_string(static_cast<long long>(_out_width))
		+ std::string(" -D MLO_POOLING_TOP_HEIGHT=") + std::to_string(static_cast<long long>(_out_height))
		+ std::string(_do_backward ? " -D MLO_POOLING_DO_BACKWARD" : "")
		+ getGeneralCompOptions()
		;

	int g_wk_width = ((_out_width + _grp_tile0 * _out_pix_tile0 - 1) / (_grp_tile0 * _out_pix_tile0));
	int g_wk_height = ((_out_height + _grp_tile1 * _out_pix_tile1 - 1) / (_grp_tile1 * _out_pix_tile1));

	_l_wk.clear();
	_l_wk.push_back(_grp_tile0);
	_l_wk.push_back(_grp_tile1);
	_l_wk.push_back(1);

	_g_wk.clear();
	_g_wk.push_back(g_wk_width * _grp_tile0);
	_g_wk.push_back(g_wk_height * _grp_tile1);
	_g_wk.push_back(_n_outputs * _batch_sz);

	_kernel_file = "MLOpenPooling.cl";

	_kernel_name = "mloPooling";

	return(ret);
}

int mlo_construct_pooling2D::mloConstructBwd()
{
	int ret = 0;

	_grp_tile0 = 8;
	_grp_tile1 = 8;

	//_out_pix_tile0 = _kernel_stride0;
	//_out_pix_tile1 = _kernel_stride1;
	_out_pix_tile0 = (_out_width < _grp_tile0 * 2) ? 1 : 2;
	_out_pix_tile1 = (_out_height < _grp_tile1 * 2) ? 1 : 2;

	_comp_options =
		std::string(" -D MLO_POOLING_KERNEL_SZ1=") + std::to_string(static_cast<long long>(_kernel_size1))
		+ std::string(" -D MLO_POOLING_PAD1=") + std::to_string(static_cast<long long>(_pad1))
		+ std::string(" -D MLO_POOLING_STRIDE1=") + std::to_string(static_cast<long long>(_kernel_stride1))
		+ std::string(" -D MLO_POOLING_KERNEL_SZ0=") + std::to_string(static_cast<long long>(_kernel_size0))
		+ std::string(" -D MLO_POOLING_PAD0=") + std::to_string(static_cast<long long>(_pad0))
		+ std::string(" -D MLO_POOLING_STRIDE0=") + std::to_string(static_cast<long long>(_kernel_stride0))
		+ std::string(" -D MLO_POOLING_N_OUTPUTS=") + std::to_string(static_cast<long long>(_n_outputs))
		+ std::string(" -D MLO_POOLBWD_N_HORIZ_OUT_PIX=") + std::to_string(static_cast<long long>(_out_pix_tile0))
		+ std::string(" -D MLO_POOLBWD_N_VERT_OUT_PIX=") + std::to_string(static_cast<long long>(_out_pix_tile1))
		+ std::string(" -D MLO_POOLBWD_GROUP_SZ0=") + std::to_string(static_cast<long long>(_grp_tile0))
		+ std::string(" -D MLO_POOLBWD_GROUP_SZ1=") + std::to_string(static_cast<long long>(_grp_tile1))
		+ std::string(" -D MLO_POOLBWD_BOT_WIDTH=") + std::to_string(static_cast<long long>(_in_width))
		+ std::string(" -D MLO_POOLBWD_BOT_HEIGHT=") + std::to_string(static_cast<long long>(_in_height))
		+ std::string(" -D MLO_POOLBWD_TOP_WIDTH=") + std::to_string(static_cast<long long>(_out_width))
		+ std::string(" -D MLO_POOLBWD_TOP_HEIGHT=") + std::to_string(static_cast<long long>(_out_height))
		+ std::string(" -D MLO_POOLBWD_BOTDF_BATCH_STRIDE=") + std::to_string(static_cast<long long>(_in_df_batch_stride))
		+ std::string(" -D MLO_POOLBWD_BOTDF_CHANNEL_STRIDE=") + std::to_string(static_cast<long long>(_in_df_channel_stride))
		+ std::string(" -D MLO_POOLBWD_BOTDF_STRIDE=") + std::to_string(static_cast<long long>(_in_df_stride))
		+ std::string(" -D MLO_POOLBWD_TOPDF_BATCH_STRIDE=") + std::to_string(static_cast<long long>(_out_df_batch_stride))
		+ std::string(" -D MLO_POOLBWD_TOPDF_CHANNEL_STRIDE=") + std::to_string(static_cast<long long>(_out_df_channel_stride))
		+ std::string(" -D MLO_POOLBWD_TOPDF_STRIDE=") + std::to_string(static_cast<long long>(_out_df_stride))

		+ getGeneralCompOptions()
		;

	int g_wk_width = ((_in_width + _grp_tile0 * _out_pix_tile0 - 1) / (_grp_tile0 * _out_pix_tile0));
	int g_wk_height = ((_in_height + _grp_tile1 * _out_pix_tile1 - 1) / (_grp_tile1 * _out_pix_tile1));

	_l_wk.clear();
	_l_wk.push_back(_grp_tile0);
	_l_wk.push_back(_grp_tile1);
	_l_wk.push_back(1);

	_g_wk.clear();
	_g_wk.push_back(g_wk_width * _grp_tile0);
	_g_wk.push_back(g_wk_height * _grp_tile1);
	_g_wk.push_back(_n_inputs * _batch_sz);

	_kernel_file = "MLOpenPoolingBwd.cl";
	if (_pooling_method == MLO_POOLING_OP_MAX)
	{
		_kernel_name = "mloPoolingMaxBwd";
	}
	else if (_pooling_method == MLO_POOLING_OP_AVE)
	{
		_kernel_name = "mloPoolingAveBwd";
	}
	else
	{
		std::cout << "Layer: %s. Error: unknowm method\n";
		ret = -1;
	}
	return(ret);
}
<|endoftext|>


<commit_before>/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield
 *
 * This library is open source and may be redistributed and/or modified under
 * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
 * (at your option) any later version.  The full license is in LICENSE file
 * included with this distribution, and on the openscenegraph.org website.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * OpenSceneGraph Public License for more details.
*/
#include <stdlib.h>

#include <osg/Referenced>
#include <osg/Notify>
#include <osg/ApplicationUsage>
#include <osg/observer_ptr>

#include <typeinfo>
#include <memory>
#include <set>

#include <OpenThreads/ScopedLock>
#include <OpenThreads/Mutex>

#include <osg/DeleteHandler>

#ifndef OSG_JAVA_BUILD

namespace osg
{

// specialzed smart pointer, used to get round auto_ptr<>'s lack of the destructor reseting itself to 0.
struct DeleteHandlerPointer
{
    DeleteHandlerPointer():
        _ptr(0) {}

    DeleteHandlerPointer(DeleteHandler* ptr):
        _ptr(ptr) {}

    ~DeleteHandlerPointer()
    {
        delete _ptr;
        _ptr = 0;
    }

    inline DeleteHandlerPointer& operator = (DeleteHandler* ptr)
    {
        if (_ptr==ptr) return *this;
        delete _ptr;
        _ptr = ptr;
        return *this;
    }

    void reset(DeleteHandler* ptr)
    {
        if (_ptr==ptr) return;
        delete _ptr;
        _ptr = ptr;
    }

    inline DeleteHandler& operator*()  { return *_ptr; }

    inline const DeleteHandler& operator*() const { return *_ptr; }

    inline DeleteHandler* operator->() { return _ptr; }

    inline const DeleteHandler* operator->() const   { return _ptr; }

    DeleteHandler* get() { return _ptr; }

    const DeleteHandler* get() const { return _ptr; }

    DeleteHandler* _ptr;
};

typedef std::set<Observer*> ObserverSet;

static bool s_useThreadSafeReferenceCounting = getenv("OSG_THREAD_SAFE_REF_UNREF")!=0;
// static std::auto_ptr<DeleteHandler> s_deleteHandler(0);
static DeleteHandlerPointer s_deleteHandler(0);

static ApplicationUsageProxy Referenced_e0(ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_THREAD_SAFE_REF_UNREF","");

void Referenced::setThreadSafeReferenceCounting(bool enableThreadSafeReferenceCounting)
{
    s_useThreadSafeReferenceCounting = enableThreadSafeReferenceCounting;
}

bool Referenced::getThreadSafeReferenceCounting()
{
    return s_useThreadSafeReferenceCounting;
}

void Referenced::setDeleteHandler(DeleteHandler* handler)
{
    s_deleteHandler.reset(handler);
}

DeleteHandler* Referenced::getDeleteHandler()
{
    return s_deleteHandler.get();
}

Referenced::Referenced():
    _refMutex(0),
    _refCount(0),
    _observers(0)
{
    if (s_useThreadSafeReferenceCounting) _refMutex = new OpenThreads::Mutex;
}

Referenced::Referenced(bool threadSafeRefUnref):
    _refMutex(0),
    _refCount(0),
    _observers(0)
{
    if (threadSafeRefUnref) _refMutex = new OpenThreads::Mutex;
}

Referenced::Referenced(const Referenced&):
    _refMutex(0),
    _refCount(0),
    _observers(0)
{
    if (s_useThreadSafeReferenceCounting) _refMutex = new OpenThreads::Mutex;
}

Referenced::~Referenced()
{
    if (_refCount>0)
    {
        notify(WARN)<<"Warning: deleting still referenced object "<<this<<" of type '"<<typeid(this).name()<<"'"<<std::endl;
        notify(WARN)<<"         the final reference count was "<<_refCount<<", memory corruption possible."<<std::endl;
    }

    if (_observers)
    {
        ObserverSet* os = static_cast<ObserverSet*>(_observers);
        for(ObserverSet::iterator itr = os->begin();
            itr != os->end();
            ++itr)
        {
            (*itr)->objectDeleted(this);
        }
        delete os;
        _observers = 0;
    }

    if (_refMutex)
    {
        OpenThreads::Mutex* tmpMutexPtr = _refMutex;
        _refMutex = 0;
        delete tmpMutexPtr;
    }
}

void Referenced::setThreadSafeRefUnref(bool threadSafe)
{
    if (threadSafe)
    {
        if (!_refMutex)
        {
            // we want thread safe ref()/unref() so assing a mutex
            _refMutex = new OpenThreads::Mutex;
        }
    }
    else
    {
        if (_refMutex)
        {
            // we don't want thread safe ref()/unref() so remove any assigned mutex
            OpenThreads::Mutex* tmpMutexPtr = _refMutex;
            _refMutex = 0;
            delete tmpMutexPtr;
        }
    }
}

void Referenced::unref_nodelete() const
{
    if (_refMutex)
    {
        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(*_refMutex);
        --_refCount;
    }
    else
    {
        --_refCount;
    }
}

void Referenced::addObserver(Observer* observer)
{
    if (_refMutex)
    {
        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(*_refMutex);

        if (!_observers) _observers = new ObserverSet;
        if (_observers) static_cast<ObserverSet*>(_observers)->insert(observer);
    }
    else
    {
        if (!_observers) _observers = new ObserverSet;
        if (_observers) static_cast<ObserverSet*>(_observers)->insert(observer);
    }
}

void Referenced::removeObserver(Observer* observer)
{
    if (_refMutex)
    {
        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(*_refMutex);

        if (_observers) static_cast<ObserverSet*>(_observers)->erase(observer);
    }
    else
    {
        if (_observers) static_cast<ObserverSet*>(_observers)->erase(observer);
    }
}

void Referenced::deleteUsingDeleteHandler() const
{
    getDeleteHandler()->requestDelete(this);
}

} // end of namespace osg

#endif //OSG_JAVA_BUILD
<commit_msg>Added debug ENFORCE_THREADSAFE paths into osg::Referenced, these are off by default.<commit_after>/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield
 *
 * This library is open source and may be redistributed and/or modified under
 * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
 * (at your option) any later version.  The full license is in LICENSE file
 * included with this distribution, and on the openscenegraph.org website.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * OpenSceneGraph Public License for more details.
*/
#include <stdlib.h>

#include <osg/Referenced>
#include <osg/Notify>
#include <osg/ApplicationUsage>
#include <osg/observer_ptr>

#include <typeinfo>
#include <memory>
#include <set>

#include <OpenThreads/ScopedLock>
#include <OpenThreads/Mutex>

#include <osg/DeleteHandler>

#ifndef OSG_JAVA_BUILD

namespace osg
{

// specialzed smart pointer, used to get round auto_ptr<>'s lack of the destructor reseting itself to 0.
struct DeleteHandlerPointer
{
    DeleteHandlerPointer():
        _ptr(0) {}

    DeleteHandlerPointer(DeleteHandler* ptr):
        _ptr(ptr) {}

    ~DeleteHandlerPointer()
    {
        delete _ptr;
        _ptr = 0;
    }

    inline DeleteHandlerPointer& operator = (DeleteHandler* ptr)
    {
        if (_ptr==ptr) return *this;
        delete _ptr;
        _ptr = ptr;
        return *this;
    }

    void reset(DeleteHandler* ptr)
    {
        if (_ptr==ptr) return;
        delete _ptr;
        _ptr = ptr;
    }

    inline DeleteHandler& operator*()  { return *_ptr; }

    inline const DeleteHandler& operator*() const { return *_ptr; }

    inline DeleteHandler* operator->() { return _ptr; }

    inline const DeleteHandler* operator->() const   { return _ptr; }

    DeleteHandler* get() { return _ptr; }

    const DeleteHandler* get() const { return _ptr; }

    DeleteHandler* _ptr;
};

typedef std::set<Observer*> ObserverSet;

//#define ENFORCE_THREADSAFE

static bool s_useThreadSafeReferenceCounting = getenv("OSG_THREAD_SAFE_REF_UNREF")!=0;
// static std::auto_ptr<DeleteHandler> s_deleteHandler(0);
static DeleteHandlerPointer s_deleteHandler(0);

static ApplicationUsageProxy Referenced_e0(ApplicationUsage::ENVIRONMENTAL_VARIABLE,"OSG_THREAD_SAFE_REF_UNREF","");

void Referenced::setThreadSafeReferenceCounting(bool enableThreadSafeReferenceCounting)
{
    s_useThreadSafeReferenceCounting = enableThreadSafeReferenceCounting;
}

bool Referenced::getThreadSafeReferenceCounting()
{
    return s_useThreadSafeReferenceCounting;
}

void Referenced::setDeleteHandler(DeleteHandler* handler)
{
    s_deleteHandler.reset(handler);
}

DeleteHandler* Referenced::getDeleteHandler()
{
    return s_deleteHandler.get();
}

Referenced::Referenced():
    _refMutex(0),
    _refCount(0),
    _observers(0)
{
#ifndef ENFORCE_THREADSAFE
    if (s_useThreadSafeReferenceCounting)
#endif
        _refMutex = new OpenThreads::Mutex;
}

Referenced::Referenced(bool threadSafeRefUnref):
    _refMutex(0),
    _refCount(0),
    _observers(0)
{
    // if (!threadSafeRefUnref) osg::notify(osg::NOTICE)<<"Not ThreadSaef "<<std::endl;

#ifndef ENFORCE_THREADSAFE
    if (threadSafeRefUnref)
#endif
        _refMutex = new OpenThreads::Mutex;
}

Referenced::Referenced(const Referenced&):
    _refMutex(0),
    _refCount(0),
    _observers(0)
{
#ifndef ENFORCE_THREADSAFE
    if (s_useThreadSafeReferenceCounting)
#endif
        _refMutex = new OpenThreads::Mutex;
}

Referenced::~Referenced()
{
    if (_refCount>0)
    {
        notify(WARN)<<"Warning: deleting still referenced object "<<this<<" of type '"<<typeid(this).name()<<"'"<<std::endl;
        notify(WARN)<<"         the final reference count was "<<_refCount<<", memory corruption possible."<<std::endl;
    }

    if (_observers)
    {
        ObserverSet* os = static_cast<ObserverSet*>(_observers);
        for(ObserverSet::iterator itr = os->begin();
            itr != os->end();
            ++itr)
        {
            (*itr)->objectDeleted(this);
        }
        delete os;
        _observers = 0;
    }

    if (_refMutex)
    {
        OpenThreads::Mutex* tmpMutexPtr = _refMutex;
        _refMutex = 0;
        delete tmpMutexPtr;
    }
}

void Referenced::setThreadSafeRefUnref(bool threadSafe)
{
    if (threadSafe)
    {
        if (!_refMutex)
        {
            // we want thread safe ref()/unref() so assing a mutex
            _refMutex = new OpenThreads::Mutex;
        }
    }
    else
    {
        if (_refMutex)
        {
            // we don't want thread safe ref()/unref() so remove any assigned mutex
            OpenThreads::Mutex* tmpMutexPtr = _refMutex;
            _refMutex = 0;
            delete tmpMutexPtr;
        }
    }
}

void Referenced::unref_nodelete() const
{
    if (_refMutex)
    {
        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(*_refMutex);
        --_refCount;
    }
    else
    {
        --_refCount;
    }
}

void Referenced::addObserver(Observer* observer)
{
    if (_refMutex)
    {
        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(*_refMutex);

        if (!_observers) _observers = new ObserverSet;
        if (_observers) static_cast<ObserverSet*>(_observers)->insert(observer);
    }
    else
    {
        if (!_observers) _observers = new ObserverSet;
        if (_observers) static_cast<ObserverSet*>(_observers)->insert(observer);
    }
}

void Referenced::removeObserver(Observer* observer)
{
    if (_refMutex)
    {
        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(*_refMutex);

        if (_observers) static_cast<ObserverSet*>(_observers)->erase(observer);
    }
    else
    {
        if (_observers) static_cast<ObserverSet*>(_observers)->erase(observer);
    }
}

void Referenced::deleteUsingDeleteHandler() const
{
    getDeleteHandler()->requestDelete(this);
}

} // end of namespace osg

#endif //OSG_JAVA_BUILD
<|endoftext|>