<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-http/httpconnectionpool.h"
#include "fnord-mdb/MDB.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"
#include "logjoin/LogJoinTarget.h"
#include "logjoin/LogJoinUpload.h"
#include "FeatureIndex.h"
#include "DocStore.h"
#include "IndexChangeRequest.h"
#include "FeatureIndexWriter.h"
#include "ItemRef.h"
#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_logjoin_shutdown;
fnord::thread::EventLoop ev;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  /* shutdown hook */
  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "data dir",
      "<path>");

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "index directory",
      "<path>");

  flags.defineFlag(
      "publish_to",
      fnord::cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "upload target url",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "commit_size",
      "<num>");

  flags.defineFlag(
      "commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "commit_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

   flags.defineFlag(
      "nocache",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no cache",
      "<bool>");

  flags.defineFlag(
      "shard",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* schema... */
  auto joined_sessions_schema = joinedSessionsSchema();

  /* start event loop */
  auto evloop_thread = std::thread([] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);
  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* set up logjoin */
  auto dry_run = !flags.isSet("no_dryrun");
  auto enable_cache = !flags.isSet("nocache");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t commit_size = flags.getInt("commit_size");
  size_t commit_interval = flags.getInt("commit_interval");

  fnord::logInfo(
      "cm.logjoin",
      "Starting cm-logjoin with:\n    dry_run=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    commit_size=$3\n    commit_interval=$9\n"
      "    max_dbsize=$4MB\n" \
      "    shard=$5\n    shard_range=[$6, $7)\n    shard_modulo=$8\n" \
      "    cache=$9",
      dry_run,
      batch_size,
      buffer_size,
      commit_size,
      flags.getInt("db_size"),
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo,
      commit_interval,
      enable_cache);

  fnord::logInfo(
      "cm.logjoin",
      "Using session schema:\n$0",
      joined_sessions_schema.toString());

  /* open session db */
  auto sessdb_path = FileUtil::joinPaths(
      flags.getString("datadir"),
      StringUtil::format("$0/sessions_db", shard.shard_name));

  FileUtil::mkdir_p(sessdb_path);
  auto sessdb = mdb::MDB::open(sessdb_path);
  sessdb->setMaxSize(1000000 * flags.getInt("db_size"));

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(10 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  input_feeds.emplace(
      "tracker_log.feedserver01.nue01.production.fnrd.net",
      URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* setup time backfill */
  feed_reader.setTimeBackfill([] (const feeds::FeedEntry& entry) -> DateTime {
    const auto& log_line = entry.data;

    auto c_end = log_line.find("|");
    if (c_end == std::string::npos) return 0;
    auto t_end = log_line.find("|", c_end + 1);
    if (t_end == std::string::npos) return 0;

    auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
    return std::stoul(timestr) * fnord::kMicrosPerSecond;
  });

  /* open index */
  auto index_path = FileUtil::joinPaths(flags.getString("index"), "db");
  RefPtr<FeatureIndexWriter> index(
      new FeatureIndexWriter(
          index_path,
          "documents-dawanda",
          true));

  /* set up logjoin target */
  fnord::fts::Analyzer analyzer(flags.getString("conf"));
  cm::LogJoinTarget logjoin_target(
      joined_sessions_schema,
      &analyzer,
      index,
      dry_run);

  /* set up logjoin upload */
  cm::LogJoinUpload logjoin_upload(
      sessdb,
      flags.getString("publish_to"),
      &http);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, enable_cache, &logjoin_target);
  logjoin.exportStats("/cm-logjoin/global");
  logjoin.exportStats(StringUtil::format("/cm-logjoin/$0", shard.shard_name));

  fnord::stats::Counter<uint64_t> stat_stream_time_low;
  fnord::stats::Counter<uint64_t> stat_stream_time_high;
  fnord::stats::Counter<uint64_t> stat_active_sessions;
  fnord::stats::Counter<uint64_t> stat_dbsize;

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_low", shard.shard_name),
      &stat_stream_time_low,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_high", shard.shard_name),
      &stat_stream_time_high,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/active_sessions", shard.shard_name),
      &stat_active_sessions,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/dbsize", shard.shard_name),
      &stat_dbsize,
      fnord::stats::ExportMode::EXPORT_VALUE);

  /* upload pending q */
  if (!dry_run) {
    logjoin_upload.upload();
  }

  /* resume from last offset */
  auto txn = sessdb->startTransaction(true);
  try {
    logjoin.importTimeoutList(txn.get());

    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__logjoin_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.logjoin",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.logjoin", "Resuming logjoin...");

  DateTime last_flush;
  uint64_t rate_limit_micros = commit_interval * kMicrosPerSecond;

  for (;;) {
    auto begin = WallClock::unixMicros();
    auto turbo = FileUtil::exists("/tmp/logjoin_turbo.enabled");
    logjoin.setTurbo(turbo);

    feed_reader.fillBuffers();
    auto txn = sessdb->startTransaction();

    int i = 0;
    for (; i < commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        logjoin.insertLogline(entry.get().data, txn.get());
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid log line");
      }
    }

    auto watermarks = feed_reader.watermarks();

    auto etime = WallClock::now().unixMicros() - last_flush.unixMicros();
    if (etime > rate_limit_micros) {
      last_flush = WallClock::now();
      logjoin.flush(txn.get(), watermarks.first);
    }

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__logjoin_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.logjoin",
        "LogJoin comitting...\n    stream_time=<$0 ... $1>\n" \
        "    active_sessions=$2\n    flushed_sessions=$3\n    " \
        "flushed_queries=$4\n    flushed_item_visits=$5\n    turbo=$6\n    " \
        "cache_size=$7$8",
        watermarks.first,
        watermarks.second,
        logjoin.numSessions(),
        logjoin_target.num_sessions,
        logjoin_target.num_queries,
        logjoin_target.num_item_visits,
        turbo,
        logjoin.cacheSize(),
        stream_offsets_str);

    if (dry_run) {
      txn->abort();
    } else {
      txn->commit();

      try {
        logjoin_upload.upload();
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "upload failed");
      }
    }

    if (cm_logjoin_shutdown.load()) {
      break;
    }

    stat_stream_time_low.set(watermarks.first.unixMicros());
    stat_stream_time_high.set(watermarks.second.unixMicros());
    stat_active_sessions.set(logjoin.numSessions());
    stat_dbsize.set(FileUtil::du_c(sessdb_path));

    auto rtime = WallClock::unixMicros() - begin;
    if (rtime < kMicrosPerSecond) {
      begin = WallClock::unixMicros();
      usleep(kMicrosPerSecond - rtime);
    }
  }

  ev.shutdown();
  evloop_thread.join();

  fnord::logInfo("cm.logjoin", "LogJoin exiting...");
  //exit(0); // FIXPAUL

  return 0;
}

<commit_msg>open sessiondb with MDB_NOSYNC<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-http/httpconnectionpool.h"
#include "fnord-mdb/MDB.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "CustomerNamespace.h"
#include "logjoin/LogJoin.h"
#include "logjoin/LogJoinTarget.h"
#include "logjoin/LogJoinUpload.h"
#include "FeatureIndex.h"
#include "DocStore.h"
#include "IndexChangeRequest.h"
#include "FeatureIndexWriter.h"
#include "ItemRef.h"
#include "common.h"
#include "schemas.h"

using namespace cm;
using namespace fnord;

std::atomic<bool> cm_logjoin_shutdown;
fnord::thread::EventLoop ev;

void quit(int n) {
  cm_logjoin_shutdown = true;
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  /* shutdown hook */
  cm_logjoin_shutdown = false;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "data dir",
      "<path>");

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "index directory",
      "<path>");

  flags.defineFlag(
      "publish_to",
      fnord::cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "upload target url",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "commit_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "1024",
      "commit_size",
      "<num>");

  flags.defineFlag(
      "commit_interval",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "5",
      "commit_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

   flags.defineFlag(
      "nocache",
      fnord::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no cache",
      "<bool>");

  flags.defineFlag(
      "shard",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* schema... */
  auto joined_sessions_schema = joinedSessionsSchema();

  /* start event loop */
  auto evloop_thread = std::thread([] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);
  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  fnord::stats::StatsdAgent statsd_agent(
      fnord::net::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * fnord::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* set up logjoin */
  auto dry_run = !flags.isSet("no_dryrun");
  auto enable_cache = !flags.isSet("nocache");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t commit_size = flags.getInt("commit_size");
  size_t commit_interval = flags.getInt("commit_interval");

  fnord::logInfo(
      "cm.logjoin",
      "Starting cm-logjoin with:\n    dry_run=$0\n    batch_size=$1\n" \
      "    buffer_size=$2\n    commit_size=$3\n    commit_interval=$9\n"
      "    max_dbsize=$4MB\n" \
      "    shard=$5\n    shard_range=[$6, $7)\n    shard_modulo=$8\n" \
      "    cache=$9",
      dry_run,
      batch_size,
      buffer_size,
      commit_size,
      flags.getInt("db_size"),
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo,
      commit_interval,
      enable_cache);

  fnord::logInfo(
      "cm.logjoin",
      "Using session schema:\n$0",
      joined_sessions_schema.toString());

  /* open session db */
  auto sessdb = mdb::MDB::open(
      flags.getString("datadir"),
      false,
      1000000 * flags.getInt("db_size"),
      shard.shard_name + ".db",
      shard.shard_name + ".db.lck",
      false);

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);
  feed_reader.setMaxSpread(10 * kMicrosPerSecond);

  HashMap<String, URI> input_feeds;
  input_feeds.emplace(
      "tracker_log.feedserver01.nue01.production.fnrd.net",
      URI("http://s01.nue01.production.fnrd.net:7001/rpc"));
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  /* setup time backfill */
  feed_reader.setTimeBackfill([] (const feeds::FeedEntry& entry) -> DateTime {
    const auto& log_line = entry.data;

    auto c_end = log_line.find("|");
    if (c_end == std::string::npos) return 0;
    auto t_end = log_line.find("|", c_end + 1);
    if (t_end == std::string::npos) return 0;

    auto timestr = log_line.substr(c_end + 1, t_end - c_end - 1);
    return std::stoul(timestr) * fnord::kMicrosPerSecond;
  });

  /* open index */
  auto index_path = FileUtil::joinPaths(flags.getString("index"), "db");
  RefPtr<FeatureIndexWriter> index(
      new FeatureIndexWriter(
          index_path,
          "documents-dawanda",
          true));

  /* set up logjoin target */
  fnord::fts::Analyzer analyzer(flags.getString("conf"));
  cm::LogJoinTarget logjoin_target(
      joined_sessions_schema,
      &analyzer,
      index,
      dry_run);

  /* set up logjoin upload */
  cm::LogJoinUpload logjoin_upload(
      sessdb,
      flags.getString("publish_to"),
      &http);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, enable_cache, &logjoin_target);
  logjoin.exportStats("/cm-logjoin/global");
  logjoin.exportStats(StringUtil::format("/cm-logjoin/$0", shard.shard_name));

  fnord::stats::Counter<uint64_t> stat_stream_time_low;
  fnord::stats::Counter<uint64_t> stat_stream_time_high;
  fnord::stats::Counter<uint64_t> stat_active_sessions;
  fnord::stats::Counter<uint64_t> stat_dbsize;

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_low", shard.shard_name),
      &stat_stream_time_low,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/stream_time_high", shard.shard_name),
      &stat_stream_time_high,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/active_sessions", shard.shard_name),
      &stat_active_sessions,
      fnord::stats::ExportMode::EXPORT_VALUE);

  exportStat(
      StringUtil::format("/cm-logjoin/$0/dbsize", shard.shard_name),
      &stat_dbsize,
      fnord::stats::ExportMode::EXPORT_VALUE);

  /* upload pending q */
  if (!dry_run) {
    logjoin_upload.upload();
  }

  /* resume from last offset */
  auto txn = sessdb->startTransaction(true);
  try {
    logjoin.importTimeoutList(txn.get());

    for (const auto& input_feed : input_feeds) {
      uint64_t offset = 0;

      auto last_offset = txn->get(
          StringUtil::format("__logjoin_offset~$0", input_feed.first));

      if (!last_offset.isEmpty()) {
        offset = std::stoul(last_offset.get().toString());
      }

      fnord::logInfo(
          "cm.logjoin",
          "Adding source feed:\n    feed=$0\n    url=$1\n    offset: $2",
          input_feed.first,
          input_feed.second.toString(),
          offset);

      feed_reader.addSourceFeed(
          input_feed.second,
          input_feed.first,
          offset,
          batch_size,
          buffer_size);
    }

    txn->abort();
  } catch (...) {
    txn->abort();
    throw;
  }

  fnord::logInfo("cm.logjoin", "Resuming logjoin...");

  DateTime last_flush;
  uint64_t rate_limit_micros = commit_interval * kMicrosPerSecond;

  for (;;) {
    auto begin = WallClock::unixMicros();
    auto turbo = FileUtil::exists("/tmp/logjoin_turbo.enabled");
    logjoin.setTurbo(turbo);

    feed_reader.fillBuffers();
    auto txn = sessdb->startTransaction();

    int i = 0;
    for (; i < commit_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      try {
        logjoin.insertLogline(entry.get().data, txn.get());
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "invalid log line");
      }
    }

    auto watermarks = feed_reader.watermarks();

    auto etime = WallClock::now().unixMicros() - last_flush.unixMicros();
    if (etime > rate_limit_micros) {
      last_flush = WallClock::now();
      logjoin.flush(txn.get(), watermarks.first);
    }

    auto stream_offsets = feed_reader.streamOffsets();
    String stream_offsets_str;

    for (const auto& soff : stream_offsets) {
      txn->update(
          StringUtil::format("__logjoin_offset~$0", soff.first),
          StringUtil::toString(soff.second));

      stream_offsets_str +=
          StringUtil::format("\n    offset[$0]=$1", soff.first, soff.second);
    }

    fnord::logInfo(
        "cm.logjoin",
        "LogJoin comitting...\n    stream_time=<$0 ... $1>\n" \
        "    active_sessions=$2\n    flushed_sessions=$3\n    " \
        "flushed_queries=$4\n    flushed_item_visits=$5\n    turbo=$6\n    " \
        "cache_size=$7$8",
        watermarks.first,
        watermarks.second,
        logjoin.numSessions(),
        logjoin_target.num_sessions,
        logjoin_target.num_queries,
        logjoin_target.num_item_visits,
        turbo,
        logjoin.cacheSize(),
        stream_offsets_str);

    if (dry_run) {
      txn->abort();
    } else {
      txn->commit();

      try {
        logjoin_upload.upload();
      } catch (const std::exception& e) {
        fnord::logError("cm.logjoin", e, "upload failed");
      }
    }

    if (cm_logjoin_shutdown.load()) {
      break;
    }

    stat_stream_time_low.set(watermarks.first.unixMicros());
    stat_stream_time_high.set(watermarks.second.unixMicros());
    stat_active_sessions.set(logjoin.numSessions());
    //stat_dbsize.set(FileUtil::du_c(flags.getString("datadir"));

    auto rtime = WallClock::unixMicros() - begin;
    if (rtime < kMicrosPerSecond) {
      begin = WallClock::unixMicros();
      usleep(kMicrosPerSecond - rtime);
    }
  }

  ev.shutdown();
  evloop_thread.join();

  sessdb->sync();
  fnord::logInfo("cm.logjoin", "LogJoin exiting...");
  return 0;
}

<|endoftext|>


<commit_before>// Copyright (c) 2009-2017 The Regents of the University of Michigan
// This file is part of the HOOMD-blue project, released under the BSD 3-Clause License.

// Maintainer: mphoward

#include "hoomd/mpcd/CellList.h"
#include "hoomd/mpcd/CellCommunicator.h"
#include "hoomd/mpcd/CellThermoTypes.h"
#include "hoomd/mpcd/CommunicatorUtilities.h"
#include "hoomd/mpcd/ReductionOperators.h"
#include "hoomd/mpcd/SystemData.h"

#include "hoomd/SnapshotSystemData.h"
#include "hoomd/test/upp11_config.h"

HOOMD_UP_MAIN()

//! Test for correct calculation of MPCD grid dimensions
void cell_communicator_reduce_test(std::shared_ptr<ExecutionConfiguration> exec_conf,
                                   bool mpi_x,
                                   bool mpi_y,
                                   bool mpi_z)
    {
    if (exec_conf->getPartition() != 0) return;

    std::shared_ptr< SnapshotSystemData<Scalar> > snap( new SnapshotSystemData<Scalar>() );
    snap->global_box = BoxDim(5.0);
    snap->particle_data.type_mapping.push_back("A");
    snap->particle_data.resize(0);

    // setup test in mpi
    std::vector<Scalar> fx, fy, fz;
    unsigned int n_req_ranks = 1;
    if (mpi_x)
        {
        n_req_ranks *= 2;
        fx.push_back(0.5);
        }
    if (mpi_y)
        {
        n_req_ranks *= 2;
        fy.push_back(0.45);
        }
    if (mpi_z)
        {
        n_req_ranks *= 2;
        fz.push_back(0.55);
        }
    UP_ASSERT_EQUAL(exec_conf->getNRanks(), n_req_ranks);
    std::shared_ptr<DomainDecomposition> decomposition(new DomainDecomposition(exec_conf,snap->global_box.getL(),fx,fy,fz));
    std::shared_ptr<SystemDefinition> sysdef(new SystemDefinition(snap, exec_conf, decomposition));

    auto mpcd_sys_snap = std::make_shared<mpcd::SystemDataSnapshot>(sysdef);
    auto mpcd_sys = std::make_shared<mpcd::SystemData>(mpcd_sys_snap);
    std::shared_ptr<mpcd::CellList> cl = mpcd_sys->getCellList();
    cl->computeDimensions();

    // Fill in a dummy cell property array, which is just the global index of each cell
    // we use the 1-indexed cell (rather than standard 0) so that we can confirm sums are all done correctly
    const Index3D& ci = cl->getCellIndexer();
    GPUArray<Scalar3> props(ci.getNumElements(), exec_conf);
    GPUArray<Scalar3> ref_props(ci.getNumElements(), exec_conf);
        {
        ArrayHandle<Scalar3> h_props(props, access_location::host, access_mode::overwrite);
        ArrayHandle<Scalar3> h_ref_props(ref_props, access_location::host, access_mode::overwrite);
        for (unsigned int k=0; k < ci.getD(); ++k)
            {
            for (unsigned int j=0; j < ci.getH(); ++j)
                {
                for (unsigned int i=0; i < ci.getW(); ++i)
                    {
                    int3 global_cell = cl->getGlobalCell(make_int3(i,j,k));
                    global_cell.x += 1; global_cell.y += 1; global_cell.z += 1;

                    h_props.data[ci(i,j,k)] = make_scalar3(global_cell.x, global_cell.y, global_cell.z);
                    h_ref_props.data[ci(i,j,k)] = make_scalar3(global_cell.x, global_cell.y, global_cell.z);
                    }
                }
            }
        }

    // on summing, all communicated cells should simply increase by a multiple of the ranks they overlap
    mpcd::CellCommunicator comm(sysdef, cl);
    comm.communicate(props, mpcd::detail::CellEnergyPackOp());
    auto num_comm_cells = cl->getNComm();
        {
        ArrayHandle<Scalar3> h_props(props, access_location::host, access_mode::read);
        ArrayHandle<Scalar3> h_ref_props(ref_props, access_location::host, access_mode::read);
        for (unsigned int k=0; k < ci.getD(); ++k)
            {
            for (unsigned int j=0; j < ci.getH(); ++j)
                {
                for (unsigned int i=0; i < ci.getW(); ++i)
                    {
                    // count the number of ranks this cell overlaps, which gives the multiplier
                    // relative to the reference values
                    unsigned int noverlap = 1;
                    if (i < num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::west)] ||
                        i >= ci.getW() - num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::east)])
                        {
                        noverlap *= 2;
                        }
                    if (j < num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::south)] ||
                        j >= ci.getH() - num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::north)])
                        {
                        noverlap *= 2;
                        }
                    if (k < num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::down)] ||
                        k >= ci.getD() - num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::up)])
                        {
                        noverlap *= 2;
                        }

                    UP_ASSERT_EQUAL(h_props.data[ci(i,j,k)].x, h_ref_props.data[ci(i,j,k)].x * noverlap);
                    UP_ASSERT_EQUAL(h_props.data[ci(i,j,k)].y, h_ref_props.data[ci(i,j,k)].y); // energy packing doesn't touch y element
                    UP_ASSERT_EQUAL(__scalar_as_int(h_props.data[ci(i,j,k)].z), __scalar_as_int(h_ref_props.data[ci(i,j,k)].z) * noverlap);
                    }
                }
            }
        }
    }

//! Test for error handling in overdecomposed systems
void cell_communicator_overdecompose_test(std::shared_ptr<ExecutionConfiguration> exec_conf)
    {
    UP_ASSERT_EQUAL(exec_conf->getNRanks(), 8);

    std::shared_ptr< SnapshotSystemData<Scalar> > snap( new SnapshotSystemData<Scalar>() );
    snap->global_box = BoxDim(6.0);
    snap->particle_data.type_mapping.push_back("A");
    snap->particle_data.resize(0);
    std::shared_ptr<DomainDecomposition> decomposition(new DomainDecomposition(exec_conf,snap->global_box.getL(),2,2,2));
    std::shared_ptr<SystemDefinition> sysdef(new SystemDefinition(snap, exec_conf, decomposition));

    auto mpcd_sys_snap = std::make_shared<mpcd::SystemDataSnapshot>(sysdef);
    auto mpcd_sys = std::make_shared<mpcd::SystemData>(mpcd_sys_snap);
    std::shared_ptr<mpcd::CellList> cl = mpcd_sys->getCellList();
    cl->computeDimensions();

    // Don't really care what's in this array, just want to make sure errors get thrown appropriately
    const Index3D& ci = cl->getCellIndexer();
    GPUArray<Scalar4> props(ci.getNumElements(), exec_conf);
        {
        ArrayHandle<Scalar4> h_props(props, access_location::host, access_mode::overwrite);
        memset(h_props.data, 0, ci.getNumElements() * sizeof(Scalar4));
        }

    mpcd::CellCommunicator comm(sysdef, cl);
    mpcd::detail::CellVelocityPackOp pack_op;

    // initially, reduction should succeed
    comm.communicate(props,pack_op);

    // add a communication cell
    cl->setNExtraCells(1);
    cl->compute(1);

    // should throw an exception since the prop dims don't match the cell dims
    UP_ASSERT_EXCEPTION(std::runtime_error, [&]{ comm.communicate(props, pack_op); });

    // should succeed on resizing of props
    props.resize(cl->getNCells());
    comm.communicate(props, pack_op);

    // add another communication cell, which overdecomposes the system
    cl->setNExtraCells(2);
    cl->compute(2);
    props.resize(cl->getNCells());
    UP_ASSERT_EXCEPTION(std::runtime_error, [&]{ comm.communicate(props, pack_op); });

    // cut the cell size down, which should make it so that the system can be decomposed again
    cl->setCellSize(0.5);
    cl->compute(3);
    props.resize(cl->getNCells());
    comm.communicate(props, pack_op);

    // shrink the box size to the minimum that can be decomposed
    cl->setNExtraCells(0);
    cl->setCellSize(2.0);
    cl->compute(4);
    props.resize(cl->getNCells());
    comm.communicate(props, pack_op);

    // now shrink further to ensure failure
    cl->setCellSize(3.0);
    cl->compute(5);
    props.resize(cl->getNCells());
    UP_ASSERT_EXCEPTION(std::runtime_error, [&]{ comm.communicate(props, pack_op); });
    }

//! dimension test case for MPCD CellList class
UP_TEST( mpcd_cell_communicator )
    {
    // mpi in 1d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::CPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                2));
        cell_communicator_reduce_test(exec_conf, true, false, false);
        cell_communicator_reduce_test(exec_conf, false, true, false);
        cell_communicator_reduce_test(exec_conf, false, false, true);
        }
    // mpi in 2d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::CPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                4));
        cell_communicator_reduce_test(exec_conf, true, true, false);
        cell_communicator_reduce_test(exec_conf, true, false, true);
        cell_communicator_reduce_test(exec_conf, false, true, true);
        }
    // mpi in 3d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::CPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                8));
        cell_communicator_reduce_test(exec_conf, true, true, true);
        }
    }

//! error handling test for overdecomposed boxes
UP_TEST( mpcd_cell_communicator_overdecompose )
    {
    std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::CPU));
    cell_communicator_overdecompose_test(exec_conf);
    }

#ifdef ENABLE_CUDA
//! dimension test case for MPCD CellList class
UP_TEST( mpcd_cell_communicator_gpu )
    {
    // mpi in 1d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::GPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                2));
        cell_communicator_reduce_test(exec_conf, true, false, false);
        cell_communicator_reduce_test(exec_conf, false, true, false);
        cell_communicator_reduce_test(exec_conf, false, false, true);
        }
    // mpi in 2d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::GPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                4));
        cell_communicator_reduce_test(exec_conf, true, true, false);
        cell_communicator_reduce_test(exec_conf, true, false, true);
        cell_communicator_reduce_test(exec_conf, false, true, true);
        }
    // mpi in 3d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::GPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                8));
        cell_communicator_reduce_test(exec_conf, true, true, true);
        }
    }
#endif // ENABLE_CUDA
<commit_msg>Fix typo in failing unit test<commit_after>// Copyright (c) 2009-2017 The Regents of the University of Michigan
// This file is part of the HOOMD-blue project, released under the BSD 3-Clause License.

// Maintainer: mphoward

#include "hoomd/mpcd/CellList.h"
#include "hoomd/mpcd/CellCommunicator.h"
#include "hoomd/mpcd/CellThermoTypes.h"
#include "hoomd/mpcd/CommunicatorUtilities.h"
#include "hoomd/mpcd/ReductionOperators.h"
#include "hoomd/mpcd/SystemData.h"

#include "hoomd/SnapshotSystemData.h"
#include "hoomd/test/upp11_config.h"

HOOMD_UP_MAIN()

//! Test for correct calculation of MPCD grid dimensions
void cell_communicator_reduce_test(std::shared_ptr<ExecutionConfiguration> exec_conf,
                                   bool mpi_x,
                                   bool mpi_y,
                                   bool mpi_z)
    {
    if (exec_conf->getPartition() != 0) return;

    std::shared_ptr< SnapshotSystemData<Scalar> > snap( new SnapshotSystemData<Scalar>() );
    snap->global_box = BoxDim(5.0);
    snap->particle_data.type_mapping.push_back("A");
    snap->particle_data.resize(0);

    // setup test in mpi
    std::vector<Scalar> fx, fy, fz;
    unsigned int n_req_ranks = 1;
    if (mpi_x)
        {
        n_req_ranks *= 2;
        fx.push_back(0.5);
        }
    if (mpi_y)
        {
        n_req_ranks *= 2;
        fy.push_back(0.45);
        }
    if (mpi_z)
        {
        n_req_ranks *= 2;
        fz.push_back(0.55);
        }
    UP_ASSERT_EQUAL(exec_conf->getNRanks(), n_req_ranks);
    std::shared_ptr<DomainDecomposition> decomposition(new DomainDecomposition(exec_conf,snap->global_box.getL(),fx,fy,fz));
    std::shared_ptr<SystemDefinition> sysdef(new SystemDefinition(snap, exec_conf, decomposition));

    auto mpcd_sys_snap = std::make_shared<mpcd::SystemDataSnapshot>(sysdef);
    auto mpcd_sys = std::make_shared<mpcd::SystemData>(mpcd_sys_snap);
    std::shared_ptr<mpcd::CellList> cl = mpcd_sys->getCellList();
    cl->computeDimensions();

    // Fill in a dummy cell property array, which is just the global index of each cell
    // we use the 1-indexed cell (rather than standard 0) so that we can confirm sums are all done correctly
    const Index3D& ci = cl->getCellIndexer();
    GPUArray<Scalar3> props(ci.getNumElements(), exec_conf);
    GPUArray<Scalar3> ref_props(ci.getNumElements(), exec_conf);
        {
        ArrayHandle<Scalar3> h_props(props, access_location::host, access_mode::overwrite);
        ArrayHandle<Scalar3> h_ref_props(ref_props, access_location::host, access_mode::overwrite);
        for (unsigned int k=0; k < ci.getD(); ++k)
            {
            for (unsigned int j=0; j < ci.getH(); ++j)
                {
                for (unsigned int i=0; i < ci.getW(); ++i)
                    {
                    int3 global_cell = cl->getGlobalCell(make_int3(i,j,k));
                    global_cell.x += 1; global_cell.y += 1; global_cell.z += 1;

                    h_props.data[ci(i,j,k)] = make_scalar3(global_cell.x, global_cell.y, __int_as_scalar(global_cell.z));
                    h_ref_props.data[ci(i,j,k)] = make_scalar3(global_cell.x, global_cell.y, __int_as_scalar(global_cell.z));
                    }
                }
            }
        }

    // on summing, all communicated cells should simply increase by a multiple of the ranks they overlap
    mpcd::CellCommunicator comm(sysdef, cl);
    comm.communicate(props, mpcd::detail::CellEnergyPackOp());
    auto num_comm_cells = cl->getNComm();
        {
        ArrayHandle<Scalar3> h_props(props, access_location::host, access_mode::read);
        ArrayHandle<Scalar3> h_ref_props(ref_props, access_location::host, access_mode::read);
        for (unsigned int k=0; k < ci.getD(); ++k)
            {
            for (unsigned int j=0; j < ci.getH(); ++j)
                {
                for (unsigned int i=0; i < ci.getW(); ++i)
                    {
                    // count the number of ranks this cell overlaps, which gives the multiplier
                    // relative to the reference values
                    unsigned int noverlap = 1;
                    if (i < num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::west)] ||
                        i >= ci.getW() - num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::east)])
                        {
                        noverlap *= 2;
                        }
                    if (j < num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::south)] ||
                        j >= ci.getH() - num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::north)])
                        {
                        noverlap *= 2;
                        }
                    if (k < num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::down)] ||
                        k >= ci.getD() - num_comm_cells[static_cast<unsigned int>(mpcd::detail::face::up)])
                        {
                        noverlap *= 2;
                        }

                    UP_ASSERT_EQUAL(h_props.data[ci(i,j,k)].x, h_ref_props.data[ci(i,j,k)].x * noverlap);
                    UP_ASSERT_EQUAL(h_props.data[ci(i,j,k)].y, h_ref_props.data[ci(i,j,k)].y); // energy packing doesn't touch y element
                    UP_ASSERT_EQUAL(__scalar_as_int(h_props.data[ci(i,j,k)].z), __scalar_as_int(h_ref_props.data[ci(i,j,k)].z) * noverlap);
                    }
                }
            }
        }
    }

//! Test for error handling in overdecomposed systems
void cell_communicator_overdecompose_test(std::shared_ptr<ExecutionConfiguration> exec_conf)
    {
    UP_ASSERT_EQUAL(exec_conf->getNRanks(), 8);

    std::shared_ptr< SnapshotSystemData<Scalar> > snap( new SnapshotSystemData<Scalar>() );
    snap->global_box = BoxDim(6.0);
    snap->particle_data.type_mapping.push_back("A");
    snap->particle_data.resize(0);
    std::shared_ptr<DomainDecomposition> decomposition(new DomainDecomposition(exec_conf,snap->global_box.getL(),2,2,2));
    std::shared_ptr<SystemDefinition> sysdef(new SystemDefinition(snap, exec_conf, decomposition));

    auto mpcd_sys_snap = std::make_shared<mpcd::SystemDataSnapshot>(sysdef);
    auto mpcd_sys = std::make_shared<mpcd::SystemData>(mpcd_sys_snap);
    std::shared_ptr<mpcd::CellList> cl = mpcd_sys->getCellList();
    cl->computeDimensions();

    // Don't really care what's in this array, just want to make sure errors get thrown appropriately
    const Index3D& ci = cl->getCellIndexer();
    GPUArray<Scalar4> props(ci.getNumElements(), exec_conf);
        {
        ArrayHandle<Scalar4> h_props(props, access_location::host, access_mode::overwrite);
        memset(h_props.data, 0, ci.getNumElements() * sizeof(Scalar4));
        }

    mpcd::CellCommunicator comm(sysdef, cl);
    mpcd::detail::CellVelocityPackOp pack_op;

    // initially, reduction should succeed
    comm.communicate(props,pack_op);

    // add a communication cell
    cl->setNExtraCells(1);
    cl->compute(1);

    // should throw an exception since the prop dims don't match the cell dims
    UP_ASSERT_EXCEPTION(std::runtime_error, [&]{ comm.communicate(props, pack_op); });

    // should succeed on resizing of props
    props.resize(cl->getNCells());
    comm.communicate(props, pack_op);

    // add another communication cell, which overdecomposes the system
    cl->setNExtraCells(2);
    cl->compute(2);
    props.resize(cl->getNCells());
    UP_ASSERT_EXCEPTION(std::runtime_error, [&]{ comm.communicate(props, pack_op); });

    // cut the cell size down, which should make it so that the system can be decomposed again
    cl->setCellSize(0.5);
    cl->compute(3);
    props.resize(cl->getNCells());
    comm.communicate(props, pack_op);

    // shrink the box size to the minimum that can be decomposed
    cl->setNExtraCells(0);
    cl->setCellSize(2.0);
    cl->compute(4);
    props.resize(cl->getNCells());
    comm.communicate(props, pack_op);

    // now shrink further to ensure failure
    cl->setCellSize(3.0);
    cl->compute(5);
    props.resize(cl->getNCells());
    UP_ASSERT_EXCEPTION(std::runtime_error, [&]{ comm.communicate(props, pack_op); });
    }

//! dimension test case for MPCD CellList class
UP_TEST( mpcd_cell_communicator )
    {
    // mpi in 1d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::CPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                2));
        cell_communicator_reduce_test(exec_conf, true, false, false);
        cell_communicator_reduce_test(exec_conf, false, true, false);
        cell_communicator_reduce_test(exec_conf, false, false, true);
        }
    // mpi in 2d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::CPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                4));
        cell_communicator_reduce_test(exec_conf, true, true, false);
        cell_communicator_reduce_test(exec_conf, true, false, true);
        cell_communicator_reduce_test(exec_conf, false, true, true);
        }
    // mpi in 3d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::CPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                8));
        cell_communicator_reduce_test(exec_conf, true, true, true);
        }
    }

//! error handling test for overdecomposed boxes
UP_TEST( mpcd_cell_communicator_overdecompose )
    {
    std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::CPU));
    cell_communicator_overdecompose_test(exec_conf);
    }

#ifdef ENABLE_CUDA
//! dimension test case for MPCD CellList class
UP_TEST( mpcd_cell_communicator_gpu )
    {
    // mpi in 1d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::GPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                2));
        cell_communicator_reduce_test(exec_conf, true, false, false);
        cell_communicator_reduce_test(exec_conf, false, true, false);
        cell_communicator_reduce_test(exec_conf, false, false, true);
        }
    // mpi in 2d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::GPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                4));
        cell_communicator_reduce_test(exec_conf, true, true, false);
        cell_communicator_reduce_test(exec_conf, true, false, true);
        cell_communicator_reduce_test(exec_conf, false, true, true);
        }
    // mpi in 3d
        {
        std::shared_ptr<ExecutionConfiguration> exec_conf(new ExecutionConfiguration(ExecutionConfiguration::GPU,
                                                                                -1,
                                                                                false,
                                                                                false,
                                                                                std::shared_ptr<Messenger>(),
                                                                                8));
        cell_communicator_reduce_test(exec_conf, true, true, true);
        }
    }
#endif // ENABLE_CUDA
<|endoftext|>


<commit_before>#include "Polymer.h"
#include "consts.h"
#include "Rand.h"
#include<algorithm>
#include<cmath>
#include<iostream>
#include<stdlib.h>
#include<vector>
using namespace consts;
using namespace std;

void Polymer::initiate_monomers_random() {
	double av_velocity = 0.0;
	for (auto& m : monomers) {
		m.velocity = Rand::real_uniform() - 0.5;
		av_velocity += m.velocity;
	}
	av_velocity /= monomers.size();
	for (auto& m : monomers) m.velocity -= av_velocity;
	update_ekin();
	double scale_factor = sqrt(temp()*monomers.size() / ekin*0.5*ref_k);
	for (auto& m : monomers) m.velocity *= scale_factor;
}

void Polymer::initiate_monomers_one() { //erstes Monomer großteil der Energie
	double max_vel = sqrt((monomers.size() - 1)*ref_k*temp() / monomer_mass);
	double speed = -max_vel / (monomers.size() - 1);
	for (auto& m : monomers) m.velocity = speed;
	monomers[0].velocity = max_vel;
}

Polymer::Polymer(unsigned length, double temperature) :
epot(0),
monomer_mass(1. / length),
monomers(std::vector<Monomer>(length, Monomer(0.0, 0.))) {
	temp(temperature);
}

Polymer::~Polymer() {}

double Polymer::temp() { return m_temp; }

void Polymer::temp(double temperature) {
	m_temp = temperature;
	m_feder_konst = monomer_mass * pow(monomers.size() * m_temp * ref_k / ref_hbar, 2);
}

std::ostream & Polymer::print(std::ostream &os) const {
	for (auto& m : monomers) {
		print_m(m, os);
		os << std::endl;
	}
	return os;
}

double Polymer::force(double r) { return -r*m_feder_konst; }

void Polymer::update_forces() {
	for (auto& m : monomers) m.force = 0;
	unsigned int size = monomers.size();
	for (unsigned int i = 1; i <= size; i++) {
		double force_buf = force(monomers[i - 1] - monomers[i % size]);
		monomers[i - 1].force += force_buf; //bin mir da mit dem Vorzeichen nicht sicher!
		monomers[i % size].force -= force_buf;
	}
}

double Polymer::update_ekin(){
	ekin = 0.0;
	for (auto& m : monomers) ekin += m.velocity*m.velocity; //mass = 1
	ekin *= monomer_mass*0.5;
	return ekin;
}

double Polymer::calculate_temp() const {
	double av_velocity = 0, av_energy = 0;
	for (auto& m : monomers) av_velocity += m.velocity;
	av_velocity /= monomers.size();
	for (auto& m : monomers) av_energy += pow(m.velocity - av_velocity, 2);
	av_energy *= monomer_mass / 2 / monomers.size();
	return av_energy / (0.5*ref_k);
}
<commit_msg>1. schleife in update forces in die 2. reingetan.<commit_after>#include "Polymer.h"
#include "consts.h"
#include "Rand.h"
#include<algorithm>
#include<cmath>
#include<iostream>
#include<stdlib.h>
#include<vector>
using namespace consts;
using namespace std;

void Polymer::initiate_monomers_random() {
	double av_velocity = 0.0;
	for (auto& m : monomers) {
		m.velocity = Rand::real_uniform() - 0.5;
		av_velocity += m.velocity;
	}
	av_velocity /= monomers.size();
	for (auto& m : monomers) m.velocity -= av_velocity;
	update_ekin();
	double scale_factor = sqrt(temp()*monomers.size() / ekin*0.5*ref_k);
	for (auto& m : monomers) m.velocity *= scale_factor;
}

void Polymer::initiate_monomers_one() { //erstes Monomer großteil der Energie
	double max_vel = sqrt((monomers.size() - 1)*ref_k*temp() / monomer_mass);
	double speed = -max_vel / (monomers.size() - 1);
	for (auto& m : monomers) m.velocity = speed;
	monomers[0].velocity = max_vel;
}

Polymer::Polymer(unsigned length, double temperature) :
epot(0),
monomer_mass(1. / length),
monomers(std::vector<Monomer>(length, Monomer(0.0, 0.))) {
	temp(temperature);
}

Polymer::~Polymer() {}

double Polymer::temp() { return m_temp; }

void Polymer::temp(double temperature) {
	m_temp = temperature;
	m_feder_konst = monomer_mass * pow(monomers.size() * m_temp * ref_k / ref_hbar, 2);
}

std::ostream & Polymer::print(std::ostream &os) const {
	for (auto& m : monomers) {
		print_m(m, os);
		os << std::endl;
	}
	return os;
}

double Polymer::force(double r) { return -r*m_feder_konst; }

void Polymer::update_forces() {
	unsigned int size = monomers.size();
	for (unsigned int i = 1; i <= size; i++) {
		monomers[i - 1].force = 0;
		double force_buf = force(monomers[i - 1] - monomers[i % size]);
		monomers[i - 1].force += force_buf; //bin mir da mit dem Vorzeichen nicht sicher!
		monomers[i % size].force -= force_buf;
	}
}

double Polymer::update_ekin(){
	ekin = 0.0;
	for (auto& m : monomers) ekin += m.velocity*m.velocity; //mass = 1
	ekin *= monomer_mass*0.5;
	return ekin;
}

double Polymer::calculate_temp() const {
	double av_velocity = 0, av_energy = 0;
	for (auto& m : monomers) av_velocity += m.velocity;
	av_velocity /= monomers.size();
	for (auto& m : monomers) av_energy += pow(m.velocity - av_velocity, 2);
	av_energy *= monomer_mass / 2 / monomers.size();
	return av_energy / (0.5*ref_k);
}
<|endoftext|>


<commit_before><commit_msg>Remove the CHECK() when SQLite is corrupt.<commit_after><|endoftext|>


<commit_before>#include "df_font.h"

#include "df_bitmap.h"
#include "df_common.h"

#if _MSC_VER
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#else
#include "df_bmp.h"
#endif

#include <limits.h>
#include <memory.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>

// ****************************************************************************
// Glyph
// ****************************************************************************

typedef struct
{
    unsigned startX;
    unsigned startY;
    unsigned runLen;
} EncodedRun;

class Glyph
{
public:
    unsigned m_width;
    int m_numRuns;
    EncodedRun *m_pixelRuns;

    Glyph(int w)
    {
        m_width = w;
    }
};

// ****************************************************************************
// Global variables
// ****************************************************************************

DfFont *g_defaultFont = NULL;

static int GetPixel(char *tmpBitmap, int bmpWidth, int x, int y)
{
    return tmpBitmap[y * bmpWidth + x];
}

struct MemBuf
{
    unsigned char *data;
    int len;
    int currentPos;
    int currentByte;
    int hiNibbleNext;

    MemBuf(unsigned *_data, int _len)
    {
        data = (unsigned char *)_data;
        len = _len;
        currentPos = 0;
        currentByte = 0;
        hiNibbleNext = 0;
    }

    int ReadByte()
    {
        if (currentPos >= len) return 0;
        currentByte = data[currentPos];
        currentPos++;
        return currentByte;
    }

    int ReadNibble()
    {
        if (hiNibbleNext) {
            hiNibbleNext = 0;
            return currentByte >> 4;
        }

        ReadByte();
        hiNibbleNext = 1;
        return currentByte & 0xf;
    }
};

// ****************************************************************************
// Public Functions
// ****************************************************************************

DfFont *LoadFontFromMemory(void *_buf, int bufLen)
{
    MemBuf buf((unsigned int *)_buf, bufLen);
    DfFont *tr = new DfFont;
    memset(tr, 0, sizeof(DfFont));

    tr->maxCharWidth = buf.ReadByte();
    tr->charHeight = buf.ReadByte();
    tr->fixedWidth = !buf.ReadByte();

    char glyphWidths[224];
    if (!tr->fixedWidth)
    {
        for (int i = 0; i < 224; i++) glyphWidths[i] = buf.ReadByte();
    }
    else
    {
        for (int i = 0; i < 224; i++) glyphWidths[i] = tr->maxCharWidth;
    }

    // Decode the RLE data into a bitmap with one char per pixel.
    int bmpWidth = tr->maxCharWidth * 16;
    int bmpHeight = tr->charHeight * 14;
    char *tmpBitmap = new char [bmpWidth * bmpHeight];
    {
        int runVal = 1;
        int tmpBitmapOffset = 0;
        while (buf.currentPos < buf.len)
        {
            int runLeft = buf.ReadNibble();

            if (runLeft == 0) // 0 is the escape token.
            {
                runLeft = buf.ReadNibble();
                runLeft += buf.ReadNibble() << 4;
            }

            runVal = 1 - runVal;
            DebugAssert((tmpBitmapOffset + runLeft) < bmpWidth * bmpHeight);
            memset(tmpBitmap + tmpBitmapOffset, runVal, runLeft);
            tmpBitmapOffset += runLeft;
        }

        int pixelsLeft = bmpWidth * bmpHeight - tmpBitmapOffset;
        DebugAssert(pixelsLeft >= 0);
        memset(tmpBitmap + tmpBitmapOffset, 0, pixelsLeft);
    }

    // Undo "up" prediction
    for (int y = 1; y < bmpHeight; y++)
    {
        for (int x = 0; x < bmpWidth; x++)
        {
            char up = tmpBitmap[(y - 1) * bmpWidth + x];
            tmpBitmap[y * bmpWidth + x] ^= up;
        }
    }

    // Allocate enough EncodedRuns to store the worst case for a glyph of this size.
    // Worst case is if the whole glyph is encoded as runs of one pixel. There has
    // to be a gap of one pixel between each run, so there can only be half as many
    // runs as there are pixels.
    unsigned tempRunsSize = tr->charHeight * (tr->maxCharWidth / 2 + 1);
    EncodedRun *tempRuns = new EncodedRun [tempRunsSize];

    // Run-length encode each ASCII character
    for (int i = 32; i < 256; i++)
    {
        // Read back the bitmap and construct a run-length encoding
        memset(tempRuns, 0, tempRunsSize);
        EncodedRun *run = tempRuns;
        for (int y = 0; y < tr->charHeight; y++)
        {
            int x = 0;
            int bmpY = (i - 32) / 16 * tr->charHeight + y;
            while (x < tr->maxCharWidth)
            {
                // Skip blank pixels
                int bmpX = i % 16 * tr->maxCharWidth + x;
                while (GetPixel(tmpBitmap, bmpWidth, bmpX, bmpY) == 0)
                {
                    x++;
                    if (x >= tr->maxCharWidth)
                        break;
                    bmpX++;
                }

                // Have we got to the end of the line?
                if (x >= tr->maxCharWidth)
                    continue;

                run->startX = x;
                run->startY = y;
                run->runLen = 0;

                // Count non-blank pixels
                while (GetPixel(tmpBitmap, bmpWidth, bmpX, bmpY) != 0)
                {
                    x++;
                    run->runLen++;
                    if (x >= tr->maxCharWidth)
                        break;
                    bmpX++;
                }

                run++;
            }
        }

        // Create the glyph to store the encoded runs we've made
        Glyph *glyph = new Glyph(glyphWidths[i - 32]);
        tr->glyphs[i] = glyph;

        // Copy the runs into the glyph
        glyph->m_numRuns = run - tempRuns;
        glyph->m_pixelRuns = new EncodedRun [glyph->m_numRuns];
        memcpy(glyph->m_pixelRuns, tempRuns, glyph->m_numRuns * sizeof(EncodedRun));
    }

    delete [] tempRuns;
    delete [] tmpBitmap;

    return tr;
}

DfFont *LoadFontFromFile(char const *filename, int pixHeight)
{
    FILE *f = fopen(filename, "rb");
    if (!f) return NULL;

    char buf[5];
    fread(buf, 1, 5, f);
    if (memcmp(buf, "dfbf\0", 5) != 0) return NULL;

    unsigned char numFonts = 0;
    fread(&numFonts, 1, 1, f);

    unsigned int fileOffsets[257];
    fread(fileOffsets, 4, numFonts, f);
    fseek(f, 0, SEEK_END);
    fileOffsets[numFonts] = ftell(f);

    for (unsigned i = 0; i < numFonts; i++) {
        fseek(f, fileOffsets[i], SEEK_SET);
        int maxWidth = fgetc(f);
        int pixHeight = fgetc(f);
        if (pixHeight == pixHeight) {
            fseek(f, fileOffsets[i], SEEK_SET);
            int fontBinSize = fileOffsets[i + 1] - fileOffsets[i];
            char *fontBuf = new char[fontBinSize];
            fread(fontBuf, 1, fontBinSize, f);
            DfFont *fnt = LoadFontFromMemory((unsigned int *)fontBuf, fontBinSize / 4);
            delete[] fontBuf;
            return fnt;
        }
    }

    return NULL;
}

static int DrawTextSimpleClipped(DfFont *tr, DfColour col, DfBitmap *bmp, int _x, int y, char const *text, int maxChars)
{
    int x = _x;
    DfColour *startRow = bmp->pixels + y * bmp->width;
    int width = bmp->width;

    if (y + tr->charHeight < bmp->clipTop || y > bmp->clipBottom)
        return 0;

    for (int j = 0; text[j] && j < maxChars; j++)
    {
        if (x > bmp->clipRight)
            break;

        unsigned char c = text[j];
        Glyph *glyph = tr->glyphs[c];

        EncodedRun *rleBuf = glyph->m_pixelRuns;
        for (int i = 0; i < glyph->m_numRuns; i++)
        {
            int y3 = y + rleBuf->startY;
            if (y3 >= bmp->clipTop && y3 < bmp->clipBottom)
            {
                DfColour *thisRow = startRow + rleBuf->startY * width;
                for (unsigned i = 0; i < rleBuf->runLen; i++)
                {
                    int x3 = x + rleBuf->startX + i;
                    if (x3 >= bmp->clipLeft && x3 < bmp->clipRight)
                        thisRow[x3] = col;
                }
            }
            rleBuf++;
        }

        x += glyph->m_width;
    }

    return x - _x;
}

int DrawTextSimpleLen(DfFont *tr, DfColour col, DfBitmap *bmp, int _x, int y, char const *text, int maxChars)
{
    int x = _x;
    int width = bmp->width;

    if (x < bmp->clipLeft || y < bmp->clipTop || (y + tr->charHeight) > bmp->clipBottom)
        return DrawTextSimpleClipped(tr, col, bmp, _x, y, text, maxChars);

    DfColour *startRow = bmp->pixels + y * bmp->width;
    for (int j = 0; text[j] && j < maxChars; j++)
    {
        unsigned char c = text[j];
        if (x + (int)tr->glyphs[c]->m_width > bmp->clipRight)
            break;

        // Copy the glyph onto the stack for better cache performance. This increased the
        // performance from 13.8 to 14.4 million chars per second. Madness.
        Glyph glyph = *tr->glyphs[c];
        EncodedRun *rleBuf = glyph.m_pixelRuns;
        for (int i = 0; i < glyph.m_numRuns; i++)
        {
            DfColour *startPixel = startRow + rleBuf->startY * width + rleBuf->startX + x;
            for (unsigned i = 0; i < rleBuf->runLen; i++)
                startPixel[i] = col;
            rleBuf++;
        }

        x += glyph.m_width;
    }

    return x - _x;
}

int DrawTextSimple(DfFont *f, DfColour col, DfBitmap *bmp, int x, int y, char const *text)
{
    return DrawTextSimpleLen(f, col, bmp, x, y, text, 9999);
}

int DrawTextLeft(DfFont *tr, DfColour c, DfBitmap *bmp, int x, int y, char const *text, ...)
{
    char buf[512];
    va_list ap;
    va_start(ap, text);
    vsprintf(buf, text, ap);
    return DrawTextSimple(tr, c, bmp, x, y, buf);
}

int DrawTextRight(DfFont *tr, DfColour c, DfBitmap *bmp, int x, int y, char const *text, ...)
{
    char buf[512];
    va_list ap;
    va_start(ap, text);
    vsprintf(buf, text, ap);

    int width = GetTextWidth(tr, buf);
    return DrawTextSimple(tr, c, bmp, x - width, y, buf);
}

int DrawTextCentre(DfFont *tr, DfColour c, DfBitmap *bmp, int x, int y, char const *text, ...)
{
    char buf[512];
    va_list ap;
    va_start(ap, text);
    vsprintf(buf, text, ap);

    int width = GetTextWidth(tr, buf);
    return DrawTextSimple(tr, c, bmp, x - width/2, y, buf);
}

int GetTextWidth(DfFont *tr, char const *text, int len)
{
    len = IntMin((int)strlen(text), len);
    if (tr->fixedWidth)
    {
        return len * tr->maxCharWidth;
    }
    else
    {
        int width = 0;
        for (int i = 0; i < len; i++)
            width += tr->glyphs[text[i] & 0xff]->m_width;

        return width;
    }
}
<commit_msg>Fixed a bugs with the new font routines when loading from files.<commit_after>#include "df_font.h"

#include "df_bitmap.h"
#include "df_common.h"

#if _MSC_VER
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#else
#include "df_bmp.h"
#endif

#include <limits.h>
#include <memory.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>

// ****************************************************************************
// Glyph
// ****************************************************************************

typedef struct
{
    unsigned startX;
    unsigned startY;
    unsigned runLen;
} EncodedRun;

class Glyph
{
public:
    unsigned m_width;
    int m_numRuns;
    EncodedRun *m_pixelRuns;

    Glyph(int w)
    {
        m_width = w;
    }
};

// ****************************************************************************
// Global variables
// ****************************************************************************

DfFont *g_defaultFont = NULL;

static int GetPixel(char *tmpBitmap, int bmpWidth, int x, int y)
{
    return tmpBitmap[y * bmpWidth + x];
}

struct MemBuf
{
    unsigned char *data;
    int len;
    int currentPos;
    int currentByte;
    int hiNibbleNext;

    MemBuf(unsigned *_data, int _len)
    {
        data = (unsigned char *)_data;
        len = _len;
        currentPos = 0;
        currentByte = 0;
        hiNibbleNext = 0;
    }

    int ReadByte()
    {
        if (currentPos >= len) return 0;
        currentByte = data[currentPos];
        currentPos++;
        return currentByte;
    }

    int ReadNibble()
    {
        if (hiNibbleNext) {
            hiNibbleNext = 0;
            return currentByte >> 4;
        }

        ReadByte();
        hiNibbleNext = 1;
        return currentByte & 0xf;
    }
};

// ****************************************************************************
// Public Functions
// ****************************************************************************

DfFont *LoadFontFromMemory(void *_buf, int bufLen)
{
    MemBuf buf((unsigned int *)_buf, bufLen);
    DfFont *tr = new DfFont;
    memset(tr, 0, sizeof(DfFont));

    tr->maxCharWidth = buf.ReadByte();
    tr->charHeight = buf.ReadByte();
    tr->fixedWidth = !buf.ReadByte();

    char glyphWidths[224];
    if (!tr->fixedWidth)
    {
        for (int i = 0; i < 224; i++) glyphWidths[i] = buf.ReadByte();
    }
    else
    {
        for (int i = 0; i < 224; i++) glyphWidths[i] = tr->maxCharWidth;
    }

    // Decode the RLE data into a bitmap with one char per pixel.
    int bmpWidth = tr->maxCharWidth * 16;
    int bmpHeight = tr->charHeight * 14;
    char *tmpBitmap = new char [bmpWidth * bmpHeight];
    {
        int runVal = 1;
        int tmpBitmapOffset = 0;
        while (buf.currentPos < buf.len)
        {
            int runLeft = buf.ReadNibble();

            if (runLeft == 0) // 0 is the escape token.
            {
                runLeft = buf.ReadNibble();
                runLeft += buf.ReadNibble() << 4;
            }

            runVal = 1 - runVal;
            DebugAssert((tmpBitmapOffset + runLeft) < bmpWidth * bmpHeight);
            memset(tmpBitmap + tmpBitmapOffset, runVal, runLeft);
            tmpBitmapOffset += runLeft;
        }

        int pixelsLeft = bmpWidth * bmpHeight - tmpBitmapOffset;
        DebugAssert(pixelsLeft >= 0);
        memset(tmpBitmap + tmpBitmapOffset, 0, pixelsLeft);
    }

    // Undo "up" prediction
    for (int y = 1; y < bmpHeight; y++)
    {
        for (int x = 0; x < bmpWidth; x++)
        {
            char up = tmpBitmap[(y - 1) * bmpWidth + x];
            tmpBitmap[y * bmpWidth + x] ^= up;
        }
    }

    // Allocate enough EncodedRuns to store the worst case for a glyph of this size.
    // Worst case is if the whole glyph is encoded as runs of one pixel. There has
    // to be a gap of one pixel between each run, so there can only be half as many
    // runs as there are pixels.
    unsigned tempRunsSize = tr->charHeight * (tr->maxCharWidth / 2 + 1);
    EncodedRun *tempRuns = new EncodedRun [tempRunsSize];

    // Run-length encode each ASCII character
    for (int i = 32; i < 256; i++)
    {
        // Read back the bitmap and construct a run-length encoding
        memset(tempRuns, 0, tempRunsSize);
        EncodedRun *run = tempRuns;
        for (int y = 0; y < tr->charHeight; y++)
        {
            int x = 0;
            int bmpY = (i - 32) / 16 * tr->charHeight + y;
            while (x < tr->maxCharWidth)
            {
                // Skip blank pixels
                int bmpX = i % 16 * tr->maxCharWidth + x;
                while (GetPixel(tmpBitmap, bmpWidth, bmpX, bmpY) == 0)
                {
                    x++;
                    if (x >= tr->maxCharWidth)
                        break;
                    bmpX++;
                }

                // Have we got to the end of the line?
                if (x >= tr->maxCharWidth)
                    continue;

                run->startX = x;
                run->startY = y;
                run->runLen = 0;

                // Count non-blank pixels
                while (GetPixel(tmpBitmap, bmpWidth, bmpX, bmpY) != 0)
                {
                    x++;
                    run->runLen++;
                    if (x >= tr->maxCharWidth)
                        break;
                    bmpX++;
                }

                run++;
            }
        }

        // Create the glyph to store the encoded runs we've made
        Glyph *glyph = new Glyph(glyphWidths[i - 32]);
        tr->glyphs[i] = glyph;

        // Copy the runs into the glyph
        glyph->m_numRuns = run - tempRuns;
        glyph->m_pixelRuns = new EncodedRun [glyph->m_numRuns];
        memcpy(glyph->m_pixelRuns, tempRuns, glyph->m_numRuns * sizeof(EncodedRun));
    }

    delete [] tempRuns;
    delete [] tmpBitmap;

    return tr;
}

DfFont *LoadFontFromFile(char const *filename, int pixHeight)
{
    FILE *f = fopen(filename, "rb");
    if (!f) return NULL;

    char buf[5];
    fread(buf, 1, 5, f);
    if (memcmp(buf, "dfbf\0", 5) != 0) return NULL;

    unsigned char numFonts = 0;
    fread(&numFonts, 1, 1, f);

    unsigned int fileOffsets[257];
    fread(fileOffsets, 4, numFonts, f);
    fseek(f, 0, SEEK_END);
    fileOffsets[numFonts] = ftell(f);

    for (unsigned i = 0; i < numFonts; i++) {
        fseek(f, fileOffsets[i], SEEK_SET);
        int maxWidth = fgetc(f);
        int thisPixHeight = fgetc(f);
        if (thisPixHeight == pixHeight) {
            fseek(f, fileOffsets[i], SEEK_SET);
            int fontBinSize = fileOffsets[i + 1] - fileOffsets[i];
            char *fontBuf = new char[fontBinSize];
            fread(fontBuf, 1, fontBinSize, f);
            DfFont *fnt = LoadFontFromMemory((unsigned int *)fontBuf, fontBinSize);
            delete[] fontBuf;
            return fnt;
        }
    }

    return NULL;
}

static int DrawTextSimpleClipped(DfFont *tr, DfColour col, DfBitmap *bmp, int _x, int y, char const *text, int maxChars)
{
    int x = _x;
    DfColour *startRow = bmp->pixels + y * bmp->width;
    int width = bmp->width;

    if (y + tr->charHeight < bmp->clipTop || y > bmp->clipBottom)
        return 0;

    for (int j = 0; text[j] && j < maxChars; j++)
    {
        if (x > bmp->clipRight)
            break;

        unsigned char c = text[j];
        Glyph *glyph = tr->glyphs[c];

        EncodedRun *rleBuf = glyph->m_pixelRuns;
        for (int i = 0; i < glyph->m_numRuns; i++)
        {
            int y3 = y + rleBuf->startY;
            if (y3 >= bmp->clipTop && y3 < bmp->clipBottom)
            {
                DfColour *thisRow = startRow + rleBuf->startY * width;
                for (unsigned i = 0; i < rleBuf->runLen; i++)
                {
                    int x3 = x + rleBuf->startX + i;
                    if (x3 >= bmp->clipLeft && x3 < bmp->clipRight)
                        thisRow[x3] = col;
                }
            }
            rleBuf++;
        }

        x += glyph->m_width;
    }

    return x - _x;
}

int DrawTextSimpleLen(DfFont *tr, DfColour col, DfBitmap *bmp, int _x, int y, char const *text, int maxChars)
{
    int x = _x;
    int width = bmp->width;

    if (x < bmp->clipLeft || y < bmp->clipTop || (y + tr->charHeight) > bmp->clipBottom)
        return DrawTextSimpleClipped(tr, col, bmp, _x, y, text, maxChars);

    DfColour *startRow = bmp->pixels + y * bmp->width;
    for (int j = 0; text[j] && j < maxChars; j++)
    {
        unsigned char c = text[j];
        if (x + (int)tr->glyphs[c]->m_width > bmp->clipRight)
            break;

        // Copy the glyph onto the stack for better cache performance. This increased the
        // performance from 13.8 to 14.4 million chars per second. Madness.
        Glyph glyph = *tr->glyphs[c];
        EncodedRun *rleBuf = glyph.m_pixelRuns;
        for (int i = 0; i < glyph.m_numRuns; i++)
        {
            DfColour *startPixel = startRow + rleBuf->startY * width + rleBuf->startX + x;
            for (unsigned i = 0; i < rleBuf->runLen; i++)
                startPixel[i] = col;
            rleBuf++;
        }

        x += glyph.m_width;
    }

    return x - _x;
}

int DrawTextSimple(DfFont *f, DfColour col, DfBitmap *bmp, int x, int y, char const *text)
{
    return DrawTextSimpleLen(f, col, bmp, x, y, text, 9999);
}

int DrawTextLeft(DfFont *tr, DfColour c, DfBitmap *bmp, int x, int y, char const *text, ...)
{
    char buf[512];
    va_list ap;
    va_start(ap, text);
    vsprintf(buf, text, ap);
    return DrawTextSimple(tr, c, bmp, x, y, buf);
}

int DrawTextRight(DfFont *tr, DfColour c, DfBitmap *bmp, int x, int y, char const *text, ...)
{
    char buf[512];
    va_list ap;
    va_start(ap, text);
    vsprintf(buf, text, ap);

    int width = GetTextWidth(tr, buf);
    return DrawTextSimple(tr, c, bmp, x - width, y, buf);
}

int DrawTextCentre(DfFont *tr, DfColour c, DfBitmap *bmp, int x, int y, char const *text, ...)
{
    char buf[512];
    va_list ap;
    va_start(ap, text);
    vsprintf(buf, text, ap);

    int width = GetTextWidth(tr, buf);
    return DrawTextSimple(tr, c, bmp, x - width/2, y, buf);
}

int GetTextWidth(DfFont *tr, char const *text, int len)
{
    len = IntMin((int)strlen(text), len);
    if (tr->fixedWidth)
    {
        return len * tr->maxCharWidth;
    }
    else
    {
        int width = 0;
        for (int i = 0; i < len; i++)
            width += tr->glyphs[text[i] & 0xff]->m_width;

        return width;
    }
}
<|endoftext|>


<commit_before>/*
 * This is part of the FL library, a C++ Bayesian filtering library
 * (https://github.com/filtering-library)
 *
 * Copyright (c) 2014 Jan Issac (jan.issac@gmail.com)
 * Copyright (c) 2014 Manuel Wuthrich (manuel.wuthrich@gmail.com)
 *
 * Max-Planck Institute for Intelligent Systems, AMD Lab
 * University of Southern California, CLMC Lab
 *
 * This Source Code Form is subject to the terms of the MIT License (MIT).
 * A copy of the license can be found in the LICENSE file distributed with this
 * source code.
 */

/**
 * \file multiple_of.hpp
 * \date Febuary 2015
 * \author Jan Issac (jan.issac@gmail.com)
 */

#ifndef FL__UTIL__META__OPERATOR__MULTIPLE_OF_HPP
#define FL__UTIL__META__OPERATOR__MULTIPLE_OF_HPP

#include <type_traits>

#include <fl/util/meta/operator/not_adaptive.hpp>
#include <fl/util/meta/operator/forward_adaptive.hpp>

namespace fl
{

/**
 * \ingroup meta
 *
 * Same as CreateTypeSequence, however with a reversed parameter order. This is
 * an attempt to make the use of \c CreateTypeSequence more natural. It also
 * allows dynamic sizes if needed.
 */
template <typename T, int Count>
struct MultipleOf
    //: CreateTypeSequence<Count, typename ForwardAdaptive<T>::Type>
{
    typedef typename ForwardAdaptive<T>::Type Type;

    MultipleOf(const Type& instance, int instance_count = Count)
        : instance(instance),
          count(instance_count)
    { }

    Type instance;
    int count;
};

}

#endif
<commit_msg>Deactivated Adaptive forwarding for the moment<commit_after>/*
 * This is part of the FL library, a C++ Bayesian filtering library
 * (https://github.com/filtering-library)
 *
 * Copyright (c) 2014 Jan Issac (jan.issac@gmail.com)
 * Copyright (c) 2014 Manuel Wuthrich (manuel.wuthrich@gmail.com)
 *
 * Max-Planck Institute for Intelligent Systems, AMD Lab
 * University of Southern California, CLMC Lab
 *
 * This Source Code Form is subject to the terms of the MIT License (MIT).
 * A copy of the license can be found in the LICENSE file distributed with this
 * source code.
 */

/**
 * \file multiple_of.hpp
 * \date Febuary 2015
 * \author Jan Issac (jan.issac@gmail.com)
 */

#ifndef FL__UTIL__META__OPERATOR__MULTIPLE_OF_HPP
#define FL__UTIL__META__OPERATOR__MULTIPLE_OF_HPP

#include <type_traits>

#include <fl/util/meta/operator/not_adaptive.hpp>
#include <fl/util/meta/operator/forward_adaptive.hpp>

namespace fl
{

/**
 * \ingroup meta
 *
 * Same as CreateTypeSequence, however with a reversed parameter order. This is
 * an attempt to make the use of \c CreateTypeSequence more natural. It also
 * allows dynamic sizes if needed.
 */
template <typename T, int Count>
struct MultipleOf
    //: CreateTypeSequence<Count, typename ForwardAdaptive<T>::Type>
{
    //typedef typename ForwardAdaptive<T>::Type Type;
    typedef T Type;

    MultipleOf(const Type& instance, int instance_count = Count)
        : instance(instance),
          count(instance_count)
    { }

    Type instance;
    int count;
};

}

#endif
<|endoftext|>


<commit_before>/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#pragma once

#include <hsa/hsa.h>

#include <algorithm>
#include <cstdint>
#include <istream>
#include <iterator>
#include <string>
#include <utility>
#include <vector>

namespace hip_impl
{
    hsa_isa_t triple_to_hsa_isa(const std::string& triple);

    struct Bundled_code {
        union Header {
            struct {
                std::uint64_t offset;
                std::uint64_t bundle_sz;
                std::uint64_t triple_sz;
            };
            char cbuf[sizeof(offset) + sizeof(bundle_sz) + sizeof(triple_sz)];
        } header;
        std::string triple;
        std::vector<char> blob;
    };

    class Bundled_code_header {
        // DATA - STATICS
        static constexpr const char magic_string_[] =
            "__CLANG_OFFLOAD_BUNDLE__";
        static constexpr auto magic_string_sz_ = sizeof(magic_string_) - 1;

        // DATA
        union Header_ {
            struct {
                char bundler_magic_string_[magic_string_sz_];
                std::uint64_t bundle_cnt_;
            };
            char cbuf_[sizeof(bundler_magic_string_) + sizeof(bundle_cnt_)];
        } header_;
        std::vector<Bundled_code> bundles_;

        // FRIENDS - MANIPULATORS
        template<typename RandomAccessIterator>
        friend
        inline
        bool read(
            RandomAccessIterator f,
            RandomAccessIterator l,
            Bundled_code_header& x)
        {
            if (f == l) return false;

            std::copy_n(f, sizeof(x.header_.cbuf_), x.header_.cbuf_);

            if (valid(x)) {
                x.bundles_.resize(x.header_.bundle_cnt_);

                auto it = f + sizeof(x.header_.cbuf_);
                for (auto&& y : x.bundles_) {
                    std::copy_n(it, sizeof(y.header.cbuf), y.header.cbuf);
                    it += sizeof(y.header.cbuf);

                    y.triple.assign(it, it + y.triple_sz);

                    std::copy_n(
                        f + y.header.offset,
                        y.header.bundle_sz,
                        std::back_inserter(y.blob));

                    it += y.header.triple_sz;
                }

                return true;
            }

            return false;
        }
        friend
        inline
        bool read(const std::vector<char>& blob, Bundled_code_header& x)
        {
            return read(blob.cbegin(), blob.cend(), x);
        }
        friend
        inline
        bool read(std::istream& is, Bundled_code_header& x)
        {
            return read(std::vector<char>{
                std::istreambuf_iterator<char>{is},
                std::istreambuf_iterator<char>{}},
                x);
        }

        // FRIENDS - ACCESSORS
        friend
        inline
        bool valid(const Bundled_code_header& x)
        {
            return std::equal(
                magic_string_,
                magic_string_ + magic_string_sz_,
                x.header_.bundler_magic_string_);
        }
        friend
        inline
        const std::vector<Bundled_code>& bundles(const Bundled_code_header& x)
        {
            return x.bundles_;
        }
    public:
        // CREATORS
        Bundled_code_header() = default;
        template<typename RandomAccessIterator>
        Bundled_code_header(RandomAccessIterator f, RandomAccessIterator l);
        explicit
        Bundled_code_header(const std::vector<char>& blob);
        explicit
        Bundled_code_header(const void* maybe_blob);
        Bundled_code_header(const Bundled_code_header&) = default;
        Bundled_code_header(Bundled_code_header&&) = default;
        ~Bundled_code_header() = default;

        // MANIPULATORS
        Bundled_code_header& operator=(const Bundled_code_header&) = default;
        Bundled_code_header& operator=(Bundled_code_header&&) = default;
    };

    // CREATORS
    template<typename RandomAccessIterator>
    Bundled_code_header::Bundled_code_header(RandomAccessIterator f, RandomAccessIterator l) : Bundled_code_header{}
    {
        read(f, l, *this);
    }
} // Namespace hip_impl.
<commit_msg>Missing merge.<commit_after>/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#pragma once

#include <hsa/hsa.h>

#include <algorithm>
#include <cstdint>
#include <istream>
#include <iterator>
#include <string>
#include <utility>
#include <vector>

namespace hip_impl
{
    hsa_isa_t triple_to_hsa_isa(const std::string& triple);

    struct Bundled_code {
        union Header {
            struct {
                std::uint64_t offset;
                std::uint64_t bundle_sz;
                std::uint64_t triple_sz;
            };
            char cbuf[sizeof(offset) + sizeof(bundle_sz) + sizeof(triple_sz)];
        } header;
        std::string triple;
        std::vector<char> blob;
    };

    class Bundled_code_header {
        // DATA - STATICS
        static constexpr const char magic_string_[] =
            "__CLANG_OFFLOAD_BUNDLE__";
        static constexpr auto magic_string_sz_ = sizeof(magic_string_) - 1;

        // DATA
        union Header_ {
            struct {
                char bundler_magic_string_[magic_string_sz_];
                std::uint64_t bundle_cnt_;
            };
            char cbuf_[sizeof(bundler_magic_string_) + sizeof(bundle_cnt_)];
        } header_;
        std::vector<Bundled_code> bundles_;

        // FRIENDS - MANIPULATORS
        template<typename RandomAccessIterator>
        friend
        inline
        bool read(
            RandomAccessIterator f,
            RandomAccessIterator l,
            Bundled_code_header& x)
        {
            if (f == l) return false;

            std::copy_n(f, sizeof(x.header_.cbuf_), x.header_.cbuf_);

            if (valid(x)) {
                x.bundles_.resize(x.header_.bundle_cnt_);

                auto it = f + sizeof(x.header_.cbuf_);
                for (auto&& y : x.bundles_) {
                    std::copy_n(it, sizeof(y.header.cbuf), y.header.cbuf);
                    it += sizeof(y.header.cbuf);

                    y.triple.assign(it, it + y.header.triple_sz);

                    std::copy_n(
                        f + y.header.offset,
                        y.header.bundle_sz,
                        std::back_inserter(y.blob));

                    it += y.header.triple_sz;
                }

                return true;
            }

            return false;
        }
        friend
        inline
        bool read(const std::vector<char>& blob, Bundled_code_header& x)
        {
            return read(blob.cbegin(), blob.cend(), x);
        }
        friend
        inline
        bool read(std::istream& is, Bundled_code_header& x)
        {
            return read(std::vector<char>{
                std::istreambuf_iterator<char>{is},
                std::istreambuf_iterator<char>{}},
                x);
        }

        // FRIENDS - ACCESSORS
        friend
        inline
        bool valid(const Bundled_code_header& x)
        {
            return std::equal(
                magic_string_,
                magic_string_ + magic_string_sz_,
                x.header_.bundler_magic_string_);
        }
        friend
        inline
        const std::vector<Bundled_code>& bundles(const Bundled_code_header& x)
        {
            return x.bundles_;
        }
    public:
        // CREATORS
        Bundled_code_header() = default;
        template<typename RandomAccessIterator>
        Bundled_code_header(RandomAccessIterator f, RandomAccessIterator l);
        explicit
        Bundled_code_header(const std::vector<char>& blob);
        explicit
        Bundled_code_header(const void* maybe_blob);
        Bundled_code_header(const Bundled_code_header&) = default;
        Bundled_code_header(Bundled_code_header&&) = default;
        ~Bundled_code_header() = default;

        // MANIPULATORS
        Bundled_code_header& operator=(const Bundled_code_header&) = default;
        Bundled_code_header& operator=(Bundled_code_header&&) = default;
    };

    // CREATORS
    template<typename RandomAccessIterator>
    Bundled_code_header::Bundled_code_header(RandomAccessIterator f, RandomAccessIterator l) : Bundled_code_header{}
    {
        read(f, l, *this);
    }
} // Namespace hip_impl.
<|endoftext|>