<commit_before>/**
 * This file is part of the "FnordMetric" project
 *   Copyright (c) 2014 Paul Asmuth, Google Inc.
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <stdlib.h>
#include <unistd.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-mdb/MDB.h"
#include "cm-common/CustomerNamespace.h"
#include "cm-logjoin/LogJoin.h"

using namespace fnord;

struct FeedChunkInfo {
  uint64_t generation;
  uint64_t generation_window;
  uint64_t start_time;
  uint64_t end_time;
  HashMap<String, String> start_offsets;

  template <typename T>
  static void reflect(T* meta) {
    meta->prop(&FeedChunkInfo::generation, 1, "generation", false);
    meta->prop(&FeedChunkInfo::generation_window, 2, "generation_window", false);
    meta->prop(&FeedChunkInfo::start_time, 3, "start_time", false);
    meta->prop(&FeedChunkInfo::end_time, 4, "end_time", false);
    meta->prop(&FeedChunkInfo::start_offsets, 5, "start_offsets", false);
  };
};

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_path",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output file path",
      "<path>");

  flags.defineFlag(
      "output_prefix",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output filename prefix",
      "<prefix>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "generation_window_secs",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "14400",
      "generation_window_secs",
      "<secs>");

  flags.defineFlag(
      "generation_delay_secs",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "3600",
      "generation_delay_secs",
      "<secs>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);

  auto output_path = flags.getString("output_path");
  auto output_prefix = flags.getString("output_prefix");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t generation_window_micros =
      flags.getInt("generation_window_secs") * kMicrosPerSecond;
  size_t generation_delay_micros =
      flags.getInt("generation_delay_secs") * kMicrosPerSecond;

  FileUtil::mkdir_p(output_path);

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);

  /* get source urls */
  Vector<String> uris = flags.getArgv();
  //if (flags.isSet("statefile")) {
  //  auto statefile = FileUtil::read(flags.getString("statefile")).toString();
  //  for (const auto& uri : StringUtil::split(statefile, "\n")) {
  //    if (uri.size() > 0) {
  //      uris.emplace_back(uri);
  //    }
  //  }
  //}

  HashMap<String, String> feed_urls;
  for (const auto& uri_raw : uris) {
    URI uri(uri_raw);
    const auto& params = uri.queryParams();

    std::string feed;
    if (!URI::getParam(params, "feed", &feed)) {
      RAISEF(
          kIllegalArgumentError,
          "feed url missing ?feed query param: $0",
          uri_raw);
    }

    feed_urls.emplace(feed, uri_raw.substr(0, uri_raw.find("?")));

    std::string offset_str;
    uint64_t offset = 0;
    if (URI::getParam(params, "offset", &offset_str)) {
      if (offset_str == "HEAD") {
        offset = std::numeric_limits<uint64_t>::max();
      } else {
        offset = std::stoul(offset_str);
      }
    }

    feed_reader.addSourceFeed(
        uri,
        feed,
        offset,
        batch_size,
        buffer_size);
  }

  uint64_t total_rows = 0;
  uint64_t total_bytes = 0;
  auto start_time = WallClock::now().unixMicros();
  auto last_status_line = start_time;

  DateTime last_iter;
  uint64_t rate_limit_micros = 0.1 * kMicrosPerSecond;

  HashMap<uint64_t, List<feeds::FeedEntry>> generations_;
  HashMap<uint64_t, HashMap<String, String>> generation_offsets_;
  uint64_t max_gen_ = 0;

  for (;;) {
    last_iter = WallClock::now();
    feed_reader.fillBuffers();

    /* read feed and build generations */
    int i = 0;
    for (; i < batch_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      ++total_rows;
      total_bytes += entry.get().data.size();

      auto now = WallClock::now().unixMicros();
      if (now - last_status_line > 10000) {
        Set<uint64_t> active_gens;
        for (const auto& g : generations_) {
          active_gens.emplace(g.first);
        }

        auto runtime = (now - start_time) / 1000000;
        uint64_t bandwidth = total_bytes / (runtime + 1);
        auto str = StringUtil::format(
            "\rrows=$0 bytes=$1B time=$2s bw=$3B/s active_gens=$4"
            "          ",
            total_rows,
            total_bytes,
            runtime,
            bandwidth,
            inspect(active_gens));

        write(0, str.c_str(), str.length());
        fflush(0);
      }

      auto entry_time = entry.get().time.unixMicros();
      if (entry_time == 0) {
        continue;
      }

      uint64_t entry_gen = entry_time / generation_window_micros;

      auto iter = generations_.find(entry_gen);
      if (iter == generations_.end()) {
        if (max_gen_ >= entry_gen) {
          fnord::logWarning(
              "fnord.feedexport",
              "Dropping late data for  generation #$0",
              entry_gen);

          continue;
        }

        fnord::logDebug(
            "fnord.feedexport",
            "Creating new generation #$0",
            entry_gen);

        auto stream_offsets = feed_reader.streamOffsets();
        for (const auto& soff : stream_offsets) {
          generation_offsets_[entry_gen].emplace(
              soff.first,
              StringUtil::toString(soff.second));
        }

        if (entry_gen > max_gen_) {
          max_gen_ = entry_gen;
        }
      }

      generations_[entry_gen].emplace_back(entry.get());
    }

    feed_reader.fillBuffers();

    /* flush generations */
    auto watermarks = feed_reader.watermarks();
    auto stream_time_micros = watermarks.first.unixMicros();

    if (stream_time_micros > generation_delay_micros) {
      stream_time_micros -= generation_delay_micros;
    }

    for (auto iter = generations_.begin(); iter != generations_.end(); ) {
      if (stream_time_micros < ((iter->first + 1) * generation_window_micros)) {
        ++iter;
        continue;
      }

      fnord::logDebug(
          "cm.ctrstatsbuild",
          "Flushing report generation #$0",
          iter->first);

      FeedChunkInfo fci;
      fci.generation = iter->first;
      fci.generation_window = generation_window_micros;
      fci.start_time = iter->first * generation_window_micros;
      fci.end_time = (iter->first + 1) * generation_window_micros;
      fci.start_offsets = generation_offsets_[iter->first];

      auto fci_json = json::toJSONString(fci);

      auto sstable_file_path = FileUtil::joinPaths(
          output_path,
          StringUtil::format("$0.$1.sstable", output_prefix, iter->first));

      auto sstable_writer = sstable::SSTableWriter::create(
          sstable_file_path,
          sstable::IndexProvider{},
          fci_json.c_str(),
          fci_json.length());

      iter = generations_.erase(iter);
      generation_offsets_.erase(iter->first);
    }

    auto etime = WallClock::now().unixMicros() - last_iter.unixMicros();
    if (i < 1 && etime < rate_limit_micros) {
      usleep(rate_limit_micros - etime);
    }
  }

  evloop_thread.join();
  return 0;
}

<commit_msg>display streamtime<commit_after>/**
 * This file is part of the "FnordMetric" project
 *   Copyright (c) 2014 Paul Asmuth, Google Inc.
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <stdlib.h>
#include <unistd.h>
#include "fnord-base/io/filerepository.h"
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/random.h"
#include "fnord-base/thread/eventloop.h"
#include "fnord-base/thread/threadpool.h"
#include "fnord-base/wallclock.h"
#include "fnord-rpc/ServerGroup.h"
#include "fnord-rpc/RPC.h"
#include "fnord-rpc/RPCClient.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-json/json.h"
#include "fnord-json/jsonrpc.h"
#include "fnord-http/httprouter.h"
#include "fnord-http/httpserver.h"
#include "fnord-feeds/FeedService.h"
#include "fnord-feeds/RemoteFeedFactory.h"
#include "fnord-feeds/RemoteFeedReader.h"
#include "fnord-base/stats/statsdagent.h"
#include "fnord-mdb/MDB.h"
#include "cm-common/CustomerNamespace.h"
#include "cm-logjoin/LogJoin.h"

using namespace fnord;

struct FeedChunkInfo {
  uint64_t generation;
  uint64_t generation_window;
  uint64_t start_time;
  uint64_t end_time;
  HashMap<String, String> start_offsets;

  template <typename T>
  static void reflect(T* meta) {
    meta->prop(&FeedChunkInfo::generation, 1, "generation", false);
    meta->prop(&FeedChunkInfo::generation_window, 2, "generation_window", false);
    meta->prop(&FeedChunkInfo::start_time, 3, "start_time", false);
    meta->prop(&FeedChunkInfo::end_time, 4, "end_time", false);
    meta->prop(&FeedChunkInfo::start_offsets, 5, "start_offsets", false);
  };
};

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_path",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output file path",
      "<path>");

  flags.defineFlag(
      "output_prefix",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output filename prefix",
      "<prefix>");

  flags.defineFlag(
      "batch_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "2048",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "generation_window_secs",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "14400",
      "generation_window_secs",
      "<secs>");

  flags.defineFlag(
      "generation_delay_secs",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "3600",
      "generation_delay_secs",
      "<secs>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* start event loop */
  fnord::thread::EventLoop ev;

  auto evloop_thread = std::thread([&ev] {
    ev.run();
  });

  /* set up rpc client */
  HTTPRPCClient rpc_client(&ev);

  auto output_path = flags.getString("output_path");
  auto output_prefix = flags.getString("output_prefix");
  size_t batch_size = flags.getInt("batch_size");
  size_t buffer_size = flags.getInt("buffer_size");
  size_t generation_window_micros =
      flags.getInt("generation_window_secs") * kMicrosPerSecond;
  size_t generation_delay_micros =
      flags.getInt("generation_delay_secs") * kMicrosPerSecond;

  FileUtil::mkdir_p(output_path);

  /* set up input feed reader */
  feeds::RemoteFeedReader feed_reader(&rpc_client);

  /* get source urls */
  Vector<String> uris = flags.getArgv();
  //if (flags.isSet("statefile")) {
  //  auto statefile = FileUtil::read(flags.getString("statefile")).toString();
  //  for (const auto& uri : StringUtil::split(statefile, "\n")) {
  //    if (uri.size() > 0) {
  //      uris.emplace_back(uri);
  //    }
  //  }
  //}

  HashMap<String, String> feed_urls;
  for (const auto& uri_raw : uris) {
    URI uri(uri_raw);
    const auto& params = uri.queryParams();

    std::string feed;
    if (!URI::getParam(params, "feed", &feed)) {
      RAISEF(
          kIllegalArgumentError,
          "feed url missing ?feed query param: $0",
          uri_raw);
    }

    feed_urls.emplace(feed, uri_raw.substr(0, uri_raw.find("?")));

    std::string offset_str;
    uint64_t offset = 0;
    if (URI::getParam(params, "offset", &offset_str)) {
      if (offset_str == "HEAD") {
        offset = std::numeric_limits<uint64_t>::max();
      } else {
        offset = std::stoul(offset_str);
      }
    }

    feed_reader.addSourceFeed(
        uri,
        feed,
        offset,
        batch_size,
        buffer_size);
  }

  uint64_t total_rows = 0;
  uint64_t total_bytes = 0;
  auto start_time = WallClock::now().unixMicros();
  auto last_status_line = start_time;

  DateTime last_iter;
  uint64_t rate_limit_micros = 0.1 * kMicrosPerSecond;

  HashMap<uint64_t, List<feeds::FeedEntry>> generations_;
  HashMap<uint64_t, HashMap<String, String>> generation_offsets_;
  uint64_t max_gen_ = 0;

  for (;;) {
    last_iter = WallClock::now();
    feed_reader.fillBuffers();

    /* read feed and build generations */
    int i = 0;
    for (; i < batch_size; ++i) {
      auto entry = feed_reader.fetchNextEntry();

      if (entry.isEmpty()) {
        break;
      }

      ++total_rows;
      total_bytes += entry.get().data.size();

      auto now = WallClock::now().unixMicros();
      if (now - last_status_line > 10000) {
        Set<uint64_t> active_gens;
        for (const auto& g : generations_) {
          active_gens.emplace(g.first);
        }

        auto watermarks = feed_reader.watermarks();
        auto runtime = (now - start_time) / 1000000;
        uint64_t bandwidth = total_bytes / (runtime + 1);
        auto str = StringUtil::format(
            "\rrows=$0 bytes=$1B time=$2s bw=$3B/s active_gens=$4 streamtime=$5"
            "          ",
            total_rows,
            total_bytes,
            runtime,
            bandwidth,
            inspect(active_gens),
            watermarks.first);

        write(0, str.c_str(), str.length());
        fflush(0);
      }

      auto entry_time = entry.get().time.unixMicros();
      if (entry_time == 0) {
        continue;
      }

      uint64_t entry_gen = entry_time / generation_window_micros;

      auto iter = generations_.find(entry_gen);
      if (iter == generations_.end()) {
        if (max_gen_ >= entry_gen) {
          fnord::logWarning(
              "fnord.feedexport",
              "Dropping late data for  generation #$0",
              entry_gen);

          continue;
        }

        fnord::logDebug(
            "fnord.feedexport",
            "Creating new generation #$0",
            entry_gen);

        auto stream_offsets = feed_reader.streamOffsets();
        for (const auto& soff : stream_offsets) {
          generation_offsets_[entry_gen].emplace(
              soff.first,
              StringUtil::toString(soff.second));
        }

        if (entry_gen > max_gen_) {
          max_gen_ = entry_gen;
        }
      }

      generations_[entry_gen].emplace_back(entry.get());
    }

    feed_reader.fillBuffers();

    /* flush generations */
    auto watermarks = feed_reader.watermarks();
    auto stream_time_micros = watermarks.first.unixMicros();

    if (stream_time_micros > generation_delay_micros) {
      stream_time_micros -= generation_delay_micros;
    }

    for (auto iter = generations_.begin(); iter != generations_.end(); ) {
      if (stream_time_micros < ((iter->first + 1) * generation_window_micros)) {
        ++iter;
        continue;
      }

      fnord::logDebug(
          "cm.ctrstatsbuild",
          "Flushing report generation #$0",
          iter->first);

      FeedChunkInfo fci;
      fci.generation = iter->first;
      fci.generation_window = generation_window_micros;
      fci.start_time = iter->first * generation_window_micros;
      fci.end_time = (iter->first + 1) * generation_window_micros;
      fci.start_offsets = generation_offsets_[iter->first];

      auto fci_json = json::toJSONString(fci);

      auto sstable_file_path = FileUtil::joinPaths(
          output_path,
          StringUtil::format("$0.$1.sstable", output_prefix, iter->first));

      auto sstable_writer = sstable::SSTableWriter::create(
          sstable_file_path,
          sstable::IndexProvider{},
          fci_json.c_str(),
          fci_json.length());

      iter = generations_.erase(iter);
      generation_offsets_.erase(iter->first);
    }

    auto etime = WallClock::now().unixMicros() - last_iter.unixMicros();
    if (i < 1 && etime < rate_limit_micros) {
      usleep(rate_limit_micros - etime);
    }
  }

  evloop_thread.join();
  return 0;
}

<|endoftext|>


<commit_before>//
// Copyright (C) 2006-2017 SIPez LLC.  All rights reserved.
//
//
// $$
//////////////////////////////////////////////////////////////////////////////

#include <os/OsIntTypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>

#include <mp/MpCodecFactory.h>
#include <os/OsTime.h>
#include <os/OsDateTime.h>

#include <sipxunittests.h>

// Setup codec paths..
#include <../test/mp/MpTestCodecPaths.h>

/// Duration of one frame in milliseconds
#define FRAME_MS                 20
/// Maximum length of audio data we expect from decoder (in samples).
#define DECODED_FRAME_MAX_SIZE   3200
/// Maximum size of encoded frame (in bytes).
#define ENCODED_FRAME_MAX_SIZE   1480
/// Number of RTP packets to encode/decode.
#define NUM_PACKETS_TO_TEST      3
/// Maximum number of milliseconds in packet.
#define MAX_PACKET_TIME          20

///  Unit test for testing performance of supported codecs.
class MpCodecsPerformanceTest : public SIPX_UNIT_BASE_CLASS
{
   CPPUNIT_TEST_SUITE(MpCodecsPerformanceTest);
   CPPUNIT_TEST(testCodecsPreformance);
   CPPUNIT_TEST_SUITE_END();

public:

   void setUp()
   {
      // Create pool for data buffers
      mpPool = new MpBufPool(ENCODED_FRAME_MAX_SIZE + MpArrayBuf::getHeaderSize(), 1, "MpCodecsPerformanceTest");
      CPPUNIT_ASSERT(mpPool != NULL);

      // Create pool for buffer headers
      mpHeadersPool = new MpBufPool(sizeof(MpRtpBuf), 1, "MpCodecsPerformanceTestHeaders");
      CPPUNIT_ASSERT(mpHeadersPool != NULL);

      // Set mpHeadersPool as default pool for audio and data pools.
      MpRtpBuf::smpDefaultPool = mpHeadersPool;
   }

   void tearDown()
   {
      if (mpPool != NULL)
      {
         delete mpPool;
      }
      if (mpHeadersPool != NULL)
      {
         delete mpHeadersPool;
      }
   }

   void testCodecsPreformance()
   {
      MpCodecFactory  *pCodecFactory;
      const MppCodecInfoV1_1 **pCodecInfo;
      unsigned         codecInfoNum;

      // Get/create codec factory
      pCodecFactory = MpCodecFactory::getMpCodecFactory();
      CPPUNIT_ASSERT(pCodecFactory != NULL);

      // Load all available codecs
      size_t i;
      for(i = 0; i < sNumCodecPaths; i++)
      {
         printf("MpCodecsPerformanceTest loading codecs from: %s\n", sCodecPaths[i].data());
         pCodecFactory->loadAllDynCodecs(sCodecPaths[i],
                                         CODEC_PLUGINS_FILTER);
      }

      // Get list of loaded codecs
      pCodecFactory->getCodecInfoArray(codecInfoNum, pCodecInfo);
      CPPUNIT_ASSERT(codecInfoNum>0);

      unsigned j;
      for (j=0; j<codecInfoNum; j++)
      {
         const char **pCodecFmtps;
         unsigned     codecFmtpsNum;
         codecFmtpsNum = pCodecInfo[j]->fmtpsNum;
         pCodecFmtps = pCodecInfo[j]->fmtps;
         if(strcmp(pCodecInfo[j]->mimeSubtype, MIME_SUBTYPE_H264) == 0)
         {
             printf("Skipping codec: %s\n", pCodecInfo[j]->mimeSubtype);
         }
         else if (codecFmtpsNum == 0)
         {
            testOneCodecPreformance(pCodecFactory,
                                    pCodecInfo[j]->mimeSubtype,
                                    "",
                                    pCodecInfo[j]->sampleRate,
                                    pCodecInfo[j]->numChannels);
         }
         else
         {
            for (unsigned fmtpNum=0; fmtpNum<codecFmtpsNum; fmtpNum++)
            {
               testOneCodecPreformance(pCodecFactory,
                                       pCodecInfo[j]->mimeSubtype,
                                       pCodecFmtps[fmtpNum],
                                       pCodecInfo[j]->sampleRate,
                                       pCodecInfo[j]->numChannels);
            }
         }
      }

      // Free codec factory
      MpCodecFactory::freeSingletonHandle();
   }

protected:
   MpBufPool *mpPool;         ///< Pool for data buffers
   MpBufPool *mpHeadersPool;  ///< Pool for buffers headers

   void testOneCodecPreformance(MpCodecFactory *pCodecFactory,
                                const UtlString &codecMime,
                                const UtlString &codecFmtp,
                                int sampleRate,
                                int numChannels)
   {
      MpDecoderBase *pDecoder;
      MpEncoderBase *pEncoder;
      MpAudioSample *pOriginal;
      MpAudioSample  pDecoded[DECODED_FRAME_MAX_SIZE];
      int            encodeFrameNum = 0;
      int            maxPacketSamples = (MAX_PACKET_TIME*sampleRate)/1000;
      int            frameSize = (sampleRate*FRAME_MS)/1000;
      int            codecFrameSamples;
      UtlString contextMessage;
      contextMessage.appendFormat("While testing codec: %s fmtp: %s sample rate: %d", codecMime.data(), codecFmtp.data(), sampleRate);

      // Allocate buffer for raw audio data
      pOriginal = new MpAudioSample[frameSize];
      srand(time(NULL));
      for (int k=0; k<frameSize; k++)
      {
         pOriginal[k] = rand()%32000 - 16000;
      }

      // Create and initialize decoder and encoder
      //printf("creating decoder: %s/%d/%d fmpt=\"%s\"\n",
      //       codecMime.data(), sampleRate, numChannels, codecFmtp.data());
      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pCodecFactory->createDecoder(codecMime, codecFmtp,
                                                        sampleRate, numChannels,
                                                        0, pDecoder));

      CPPUNIT_ASSERT(pDecoder);

      OsStatus initDecodeStatus = pDecoder->initDecode();
      CPPUNIT_ASSERT_EQUAL_MESSAGE(contextMessage.data(), OS_SUCCESS, initDecodeStatus);

      // Could not test speed of signaling codec
      if (pDecoder->getInfo() == NULL || pDecoder->getInfo()->isSignalingCodec())
      {
         pDecoder->freeDecode();
         delete pDecoder;
         return;
      }

      printf("codec: %s sample rate: %d frame size: %d\n",
         codecMime.data(), sampleRate, pDecoder->getInfo()->getNumSamplesPerFrame());

      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pCodecFactory->createEncoder(codecMime, codecFmtp,
                                                        sampleRate, numChannels,
                                                        0, pEncoder));

      CPPUNIT_ASSERT(pEncoder);

      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pEncoder->initEncode());

      // Get number of samples we'll pack to get one packet
      if (pEncoder->getInfo()->getCodecType() == CODEC_TYPE_FRAME_BASED)
      {
         codecFrameSamples = pEncoder->getInfo()->getNumSamplesPerFrame();
      }
      else if (pEncoder->getInfo()->getCodecType() == CODEC_TYPE_SAMPLE_BASED)
      {
         codecFrameSamples = frameSize;
      }
      else
      {
         assert(!"Unknown codec type!");
      }

      unsigned char* rtpDataStart = NULL;

      for (int i=0; i<NUM_PACKETS_TO_TEST; i++)
      {
         int            tmpSamplesConsumed;
         int            tmpEncodedSize;
         UtlBoolean     tmpIsPacketReady;
         UtlBoolean     tmpIsPacketSilent;
         MpRtpBufPtr    pRtpPacket = mpPool->getBuffer();
         unsigned char *pRtpPayloadPtr = (unsigned char *)pRtpPacket->getDataWritePtr();
         int            payloadSize = 0;
         int            samplesInPacket = 0;
         OsTime         start;
         OsTime         stop;
         OsTime         diff;

         UtlString loopMessage;
         loopMessage.appendFormat("%s packet: %d", codecMime.data(), i);

         if(rtpDataStart == NULL)
         {
             rtpDataStart = pRtpPayloadPtr;
         }

         // Encode frames until we get tmpSendNow set or reach packet size limit.
         do
         {
            if(pRtpPayloadPtr != rtpDataStart)
            {
               printf("WARNING: rtpDataStart: %p pRtpPayloadPtr: %p\n",
                   rtpDataStart, pRtpPayloadPtr);
            }

            // Encode one frame and measure time it took.
            UtlBoolean setMarkerBit;
            OsStatus result;
            OsDateTime::getCurTime(start);
            result = pEncoder->encode(pOriginal, frameSize, tmpSamplesConsumed,
                                      pRtpPayloadPtr,
                                      ENCODED_FRAME_MAX_SIZE-payloadSize,
                                      tmpEncodedSize, tmpIsPacketReady,
                                      tmpIsPacketSilent,
                                      setMarkerBit);
            OsDateTime::getCurTime(stop);
            CPPUNIT_ASSERT_EQUAL(OS_SUCCESS, result);

            // Print timing in CSV format
            diff = stop - start;
            printf("encode %s/%d/%d %s;%d;%ld.%06ld;%ld.%06ld\n",
                   codecMime.data(), sampleRate, numChannels, codecFmtp.data(),
                   encodeFrameNum,
                   start.seconds(), start.usecs(),
                   diff.seconds(), diff.usecs());

            // Adjust encoding state
            payloadSize += tmpEncodedSize;
            pRtpPayloadPtr += tmpEncodedSize;
            samplesInPacket += tmpSamplesConsumed;
            CPPUNIT_ASSERT_MESSAGE(contextMessage.data(), payloadSize <= ENCODED_FRAME_MAX_SIZE);
            if(payloadSize > ENCODED_FRAME_MAX_SIZE)
            {
                printf("payloadSize: %d ENCODED_FRAME_MAX_SIZE: %d\n",
                    payloadSize, ENCODED_FRAME_MAX_SIZE);
            }
            encodeFrameNum++;
            printf("payloadSize: %d tmpIsPacketReady: %s samplesInPacket: %d codecFrameSamples: %d maxPacketSamples: %d tmpEncodedSize: %d\n",
                payloadSize, tmpIsPacketReady ? "true" : "false", samplesInPacket, codecFrameSamples, maxPacketSamples, tmpEncodedSize);
         } while( (payloadSize == 0)
                ||( (tmpIsPacketReady != TRUE) &&
                    (samplesInPacket+codecFrameSamples <= maxPacketSamples)));

         printf("packet samples: %d\n", samplesInPacket);

         pRtpPacket->setPayloadSize(payloadSize);

         // Decode frame, measure time and verify, that we decoded same number of samples.
         OsDateTime::getCurTime(start);
         tmpSamplesConsumed = pDecoder->decode(pRtpPacket, DECODED_FRAME_MAX_SIZE,
                                               pDecoded);
         OsDateTime::getCurTime(stop);
         CPPUNIT_ASSERT_EQUAL_MESSAGE(loopMessage.data(),
                                      samplesInPacket, tmpSamplesConsumed);

         printf("RTP payload first byte: %x payload size: %d\n",
             *(pRtpPacket->getDataPtr()), pRtpPacket->getPayloadSize());

         // Print timing in TSV format
         diff = stop - start;
         printf("decode %s/%d/%d %s;%d;%ld.%06ld;%ld.%06ld\n",
                codecMime.data(), sampleRate, numChannels, codecFmtp.data(),
                i,
                start.seconds(), start.usecs(),
                diff.seconds(), diff.usecs());
      }

      // Free encoder and decoder
      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pDecoder->freeDecode());
      delete pDecoder;
      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pEncoder->freeEncode());
      delete pEncoder;

      // Free buffer used for raw audio data
      delete[] pOriginal;
   }

};

CPPUNIT_TEST_SUITE_REGISTRATION(MpCodecsPerformanceTest);
<commit_msg>Added some more test cases, changed media frame size to 10 ms (as is typical), fixed a number of issues in unit test and in plugins.  <commit_after>//
// Copyright (C) 2006-2017 SIPez LLC.  All rights reserved.
//
//
// $$
//////////////////////////////////////////////////////////////////////////////

#include <os/OsIntTypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>

#include <mp/MpCodecFactory.h>
#include <mp/NetInTask.h>
#include <os/OsTime.h>
#include <os/OsDateTime.h>

#include <sipxunittests.h>

// Setup codec paths..
#include <../test/mp/MpTestCodecPaths.h>

/// Duration of one frame in milliseconds
#define FRAME_MS                 10
/// Maximum length of audio data we expect from decoder (in samples).
#define DECODED_FRAME_MAX_SIZE   3200
/// Maximum size of encoded frame (in bytes).
#define ENCODED_FRAME_MAX_SIZE   RTP_MTU
/// Number of RTP packets to encode/decode.
#define NUM_PACKETS_TO_TEST      3
/// Maximum number of milliseconds in packet.
#define MAX_PACKET_TIME          20

///  Unit test for testing performance of supported codecs.
class MpCodecsPerformanceTest : public SIPX_UNIT_BASE_CLASS
{
   CPPUNIT_TEST_SUITE(MpCodecsPerformanceTest);
   CPPUNIT_TEST(testCodecsPreformance);
   CPPUNIT_TEST_SUITE_END();

public:

   void setUp()
   {
      // Create pool for data buffers
      mpPool = new MpBufPool(ENCODED_FRAME_MAX_SIZE + MpArrayBuf::getHeaderSize(), 1, "MpCodecsPerformanceTest");
      CPPUNIT_ASSERT(mpPool != NULL);

      // Create pool for buffer headers
      mpHeadersPool = new MpBufPool(sizeof(MpRtpBuf), 1, "MpCodecsPerformanceTestHeaders");
      CPPUNIT_ASSERT(mpHeadersPool != NULL);

      // Set mpHeadersPool as default pool for audio and data pools.
      MpRtpBuf::smpDefaultPool = mpHeadersPool;
   }

   void tearDown()
   {
      if (mpPool != NULL)
      {
         delete mpPool;
      }
      if (mpHeadersPool != NULL)
      {
         delete mpHeadersPool;
      }
   }

   void testCodecsPreformance()
   {
      MpCodecFactory  *pCodecFactory;
      const MppCodecInfoV1_1 **pCodecInfo;
      unsigned         codecInfoNum;

      // Get/create codec factory
      pCodecFactory = MpCodecFactory::getMpCodecFactory();
      CPPUNIT_ASSERT(pCodecFactory != NULL);

      // Load all available codecs
      size_t i;
      for(i = 0; i < sNumCodecPaths; i++)
      {
         printf("MpCodecsPerformanceTest loading codecs from: %s\n", sCodecPaths[i].data());
         pCodecFactory->loadAllDynCodecs(sCodecPaths[i],
                                         CODEC_PLUGINS_FILTER);
      }

      printf("mediaFrame size: %d mSec\n", FRAME_MS);

      // Get list of loaded codecs
      pCodecFactory->getCodecInfoArray(codecInfoNum, pCodecInfo);
      CPPUNIT_ASSERT(codecInfoNum>0);

      unsigned j;
      for (j=0; j<codecInfoNum; j++)
      {
         const char **pCodecFmtps;
         unsigned     codecFmtpsNum;
         codecFmtpsNum = pCodecInfo[j]->fmtpsNum;
         pCodecFmtps = pCodecInfo[j]->fmtps;
         if(strcmp(pCodecInfo[j]->mimeSubtype, MIME_SUBTYPE_H264) == 0)
         {
             printf("Skipping codec: %s\n", pCodecInfo[j]->mimeSubtype);
         }
         else if (codecFmtpsNum == 0)
         {
            testOneCodecPreformance(pCodecFactory,
                                    pCodecInfo[j]->mimeSubtype,
                                    "",
                                    pCodecInfo[j]->sampleRate,
                                    pCodecInfo[j]->numChannels);
         }
         else
         {
            for (unsigned fmtpNum=0; fmtpNum<codecFmtpsNum; fmtpNum++)
            {
               testOneCodecPreformance(pCodecFactory,
                                       pCodecInfo[j]->mimeSubtype,
                                       pCodecFmtps[fmtpNum],
                                       pCodecInfo[j]->sampleRate,
                                       pCodecInfo[j]->numChannels);
            }
         }
      }

      // Free codec factory
      MpCodecFactory::freeSingletonHandle();
   }

protected:
   MpBufPool *mpPool;         ///< Pool for data buffers
   MpBufPool *mpHeadersPool;  ///< Pool for buffers headers

   void testOneCodecPreformance(MpCodecFactory *pCodecFactory,
                                const UtlString &codecMime,
                                const UtlString &codecFmtp,
                                int sampleRate,
                                int numChannels)
   {
      MpDecoderBase *pDecoder;
      MpEncoderBase *pEncoder;
      MpAudioSample *pOriginal;
      MpAudioSample  pDecoded[DECODED_FRAME_MAX_SIZE];
      int            encodeFrameNum = 0;
      int            maxPacketSamples = (MAX_PACKET_TIME*sampleRate)/1000 /
                        (MAX_PACKET_TIME/FRAME_MS) * (MAX_PACKET_TIME/FRAME_MS); // Fix roundoff issues
      int            frameSize = (sampleRate*FRAME_MS)/1000;
      int            codecFrameSamples;
      UtlString contextMessage;
      contextMessage.appendFormat("While testing codec: %s fmtp: %s sample rate: %d", codecMime.data(), codecFmtp.data(), sampleRate);

      // Allocate buffer for raw audio data
      pOriginal = new MpAudioSample[frameSize];
      srand(time(NULL));
      for (int k=0; k<frameSize; k++)
      {
         pOriginal[k] = rand()%32000 - 16000;
      }

      // Create and initialize decoder and encoder
      //printf("creating decoder: %s/%d/%d fmpt=\"%s\"\n",
      //       codecMime.data(), sampleRate, numChannels, codecFmtp.data());
      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pCodecFactory->createDecoder(codecMime, codecFmtp,
                                                        sampleRate, numChannels,
                                                        0, pDecoder));

      CPPUNIT_ASSERT(pDecoder);

      OsStatus initDecodeStatus = pDecoder->initDecode();
      CPPUNIT_ASSERT_EQUAL_MESSAGE(contextMessage.data(), OS_SUCCESS, initDecodeStatus);

      // Could not test speed of signaling codec
      if (pDecoder->getInfo() == NULL || pDecoder->getInfo()->isSignalingCodec())
      {
         pDecoder->freeDecode();
         delete pDecoder;
         return;
      }

      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pCodecFactory->createEncoder(codecMime, codecFmtp,
                                                        sampleRate, numChannels,
                                                        0, pEncoder));

      CPPUNIT_ASSERT(pEncoder);

      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pEncoder->initEncode());

      // Get number of samples we'll pack to get one packet
      if (pEncoder->getInfo()->getCodecType() == CODEC_TYPE_FRAME_BASED)
      {
         codecFrameSamples = pEncoder->getInfo()->getNumSamplesPerFrame();
      }
      else if (pEncoder->getInfo()->getCodecType() == CODEC_TYPE_SAMPLE_BASED)
      {
         codecFrameSamples = frameSize;
      }
      else
      {
         assert(!"Unknown codec type!");
      }

      printf("codec: %s sample rate: %d frame size: %d\n",
         codecMime.data(), sampleRate, codecFrameSamples);

      CPPUNIT_ASSERT_MESSAGE(contextMessage.data(), codecFrameSamples > 0);

      unsigned char* rtpDataStart = NULL;
      int algorithmicDelaySamplesCount = 0;

      for (int i=0; i<NUM_PACKETS_TO_TEST; i++)
      {
         int            tmpSamplesConsumed;
         int            tmpEncodedSize;
         UtlBoolean     tmpIsPacketReady;
         UtlBoolean     tmpIsPacketSilent;
         MpRtpBufPtr    pRtpPacket = mpPool->getBuffer();
         unsigned char *pRtpPayloadPtr = (unsigned char *)pRtpPacket->getDataWritePtr();
         int            payloadSize = 0;
         int            samplesInPacket = 0;
         OsTime         start;
         OsTime         stop;
         OsTime         diff;

         UtlString loopMessage;
         loopMessage.appendFormat("%s packet: %d", codecMime.data(), i);

         if(rtpDataStart == NULL)
         {
             rtpDataStart = pRtpPayloadPtr;
         }

         // Encode frames until we get tmpSendNow set or reach packet size limit.
         do
         {
            if(pRtpPayloadPtr != rtpDataStart)
            {
               printf("WARNING: rtpDataStart: %p pRtpPayloadPtr: %p\n",
                   rtpDataStart, pRtpPayloadPtr);
            }

            // Encode one frame and measure time it took.
            UtlBoolean setMarkerBit;
            OsStatus result;
            OsDateTime::getCurTime(start);
            result = pEncoder->encode(pOriginal, frameSize, tmpSamplesConsumed,
                                      pRtpPayloadPtr,
                                      ENCODED_FRAME_MAX_SIZE-payloadSize,
                                      tmpEncodedSize, tmpIsPacketReady,
                                      tmpIsPacketSilent,
                                      setMarkerBit);
            OsDateTime::getCurTime(stop);
            CPPUNIT_ASSERT_EQUAL(OS_SUCCESS, result);

            // Print timing in CSV format
            diff = stop - start;
            printf("encode %s/%d/%d %s;%d;%ld.%06ld;%ld.%06ld\n",
                   codecMime.data(), sampleRate, numChannels, codecFmtp.data(),
                   encodeFrameNum,
                   start.seconds(), start.usecs(),
                   diff.seconds(), diff.usecs());

            // Adjust encoding state
            payloadSize += tmpEncodedSize;
            pRtpPayloadPtr += tmpEncodedSize;
            samplesInPacket += tmpSamplesConsumed;
            CPPUNIT_ASSERT_MESSAGE(contextMessage.data(), payloadSize <= ENCODED_FRAME_MAX_SIZE);
            if(payloadSize > ENCODED_FRAME_MAX_SIZE)
            {
                printf("payloadSize: %d ENCODED_FRAME_MAX_SIZE: %d\n",
                    payloadSize, ENCODED_FRAME_MAX_SIZE);
            }
            encodeFrameNum++;
            printf("payloadSize: %d tmpIsPacketReady: %s samplesInPacket: %d codecFrameSamples: %d maxPacketSamples: %d tmpEncodedSize: %d\n",
                payloadSize, tmpIsPacketReady ? "true" : "false", samplesInPacket, codecFrameSamples, maxPacketSamples, tmpEncodedSize);
         } while( (payloadSize == 0)
                ||( (tmpIsPacketReady != TRUE) &&
                    (samplesInPacket+codecFrameSamples <= maxPacketSamples)));

         printf("packet samples: %d\n", samplesInPacket);

         pRtpPacket->setPayloadSize(payloadSize);

         // Decode frame, measure time and verify, that we decoded same number of samples.
         OsDateTime::getCurTime(start);
         tmpSamplesConsumed = pDecoder->decode(pRtpPacket, DECODED_FRAME_MAX_SIZE,
                                               pDecoded);
         OsDateTime::getCurTime(stop);
         if(algorithmicDelaySamplesCount == 0 &&
            tmpSamplesConsumed > 0)
         {
             algorithmicDelaySamplesCount = samplesInPacket;

             CPPUNIT_ASSERT_MESSAGE(loopMessage.data(),
                                    algorithmicDelaySamplesCount >=
                                    pEncoder->getInfo()->getAlgorithmicDelay());
             if(algorithmicDelaySamplesCount <
                pEncoder->getInfo()->getAlgorithmicDelay())
             {
                printf("algorithmic delay observed: %d samples codec info: %d samples\n",
                       algorithmicDelaySamplesCount,
                       pEncoder->getInfo()->getAlgorithmicDelay());
             }
             // If we run into the MTU limit, we may not reach the algorithmic delay num samples
             if(pRtpPacket->getPayloadSize() < RTP_MTU ||
                algorithmicDelaySamplesCount >=
                                    codecFrameSamples)
             {
                 CPPUNIT_ASSERT_MESSAGE(loopMessage.data(),
                                        algorithmicDelaySamplesCount >=
                                        codecFrameSamples);
                 if(algorithmicDelaySamplesCount <
                    codecFrameSamples)
                 {
                    printf("algorithmic delay observed: %d codecFrameSamples: %d\n",
                           algorithmicDelaySamplesCount,
                           codecFrameSamples);
                 }
             }
         }
         CPPUNIT_ASSERT_EQUAL_MESSAGE(loopMessage.data(),
                                      samplesInPacket, tmpSamplesConsumed);
         if(pEncoder->getInfo()->getCodecType() == CODEC_TYPE_SAMPLE_BASED)
         {
             if(pRtpPacket->getPayloadSize() < RTP_MTU ||
                tmpSamplesConsumed == maxPacketSamples)
             {
                 CPPUNIT_ASSERT_EQUAL_MESSAGE(loopMessage.data(),
                                              tmpSamplesConsumed,
                                              maxPacketSamples);
             }
             // TODO: need test for case when we fill the RTP packet before we get to maxPacketSamples
             else
             {
                 printf("WARNING: maximum RTP MTU (%d) reached before maxPacketSamples (%d), only %d encoded\n",
                        RTP_MTU,
                        maxPacketSamples,
                        tmpSamplesConsumed);
             }
         }
         else
         {
             CPPUNIT_ASSERT_EQUAL_MESSAGE(loopMessage.data(),
                                          tmpSamplesConsumed,
                                          codecFrameSamples);
         }
         printf("RTP payload first byte: %x payload size: %d\n",
             *(pRtpPacket->getDataPtr()), pRtpPacket->getPayloadSize());

         // Print timing in TSV format
         diff = stop - start;
         printf("decode %s/%d/%d %s;%d;%ld.%06ld;%ld.%06ld\n",
                codecMime.data(), sampleRate, numChannels, codecFmtp.data(),
                i,
                start.seconds(), start.usecs(),
                diff.seconds(), diff.usecs());
      }

      printf("algorithmic-delay %s/%d/%d %s;%d;%f\n",
             codecMime.data(), sampleRate, numChannels, codecFmtp.data(),
             algorithmicDelaySamplesCount,
             ((float)algorithmicDelaySamplesCount) / ((float)sampleRate) * 1000.0);
      printf("packet-samples %s/%d/%d %s;%d;%f\n",
             codecMime.data(), sampleRate, numChannels, codecFmtp.data(),
             codecFrameSamples,
             ((float)codecFrameSamples) / ((float)sampleRate) * 1000.0);
      // Free encoder and decoder
      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pDecoder->freeDecode());
      delete pDecoder;
      CPPUNIT_ASSERT_EQUAL(OS_SUCCESS,
                           pEncoder->freeEncode());
      delete pEncoder;

      // Free buffer used for raw audio data
      delete[] pOriginal;
   }

};

CPPUNIT_TEST_SUITE_REGISTRATION(MpCodecsPerformanceTest);
<|endoftext|>


<commit_before>/** @file
 *
 * The implementation of the logging functions.
 *
 * @author Nicolas Bock <nicolas.bock@freeon.org>
 * @author Matt Challacombe <matt.challacombe@freeon.org>
 */

#include "logger.h"
#include "index.h"

#include <charm++.h>
#include <bitset>
#include <string>
#include <string.h>
#include <sstream>
#include <stdarg.h>
#include <stdio.h>

/** The main logging function. It is usually more convenient to use the macros
 * DEBUG, INFO, and ABORT instead.
 *
 * @param filename The name of the file where the message occurred.
 * @param linenumber The line number in that file.
 * @param function_name The name of the function.
 * @param tag A tag.
 * @param format The format string. See printf() for details.
 */
void logger (const char *const filename,
    const int linenumber,
    const char *const function_name,
    const char *const tag,
    const char *const format, ...)
{
  const int format_length = 2000;
  const int string_length = 5000;

  va_list ap;
  char new_format[format_length];
  char output_string[string_length];

  if(strlen(tag) > 0)
  {
    snprintf(new_format, format_length, "[%s:%d (%s) PE:%d %s] %s", filename,
        linenumber, function_name, CkMyPe(), tag, format);
  }

  else
  {
    snprintf(new_format, format_length, "[%s:%d (%s) PE:%d] %s", filename,
        linenumber, function_name, CkMyPe(), format);
  }

  va_start(ap, format);
  vsnprintf(output_string, string_length, new_format, ap);
  va_end(ap);
  CkPrintf(output_string);
}

/** Print a dense matrix.
 *
 * @param N The matrix size.
 * @param A The matrix.
 * @param format The format string for a message that is printed in front of
 * the matrix. See printf() for details.
 */
void printDense (int N, double *A, const char *const format, ...)
{
  std::ostringstream o;
  o.setf(std::ios::scientific);

  va_list ap;
  const int message_length = 2000;
  char message[message_length];

  va_start(ap, format);
  vsnprintf(message, message_length, format, ap);
  va_end(ap);

  o << message << std::endl;
  if(N <= 32)
  {
    for(int i = 0; i < N; i++) {
      for(int j = 0; j < N; j++)
      {
        o << " " << A[BLOCK_INDEX(i, j, 0, 0, N)];
      }
      o << std::endl;
    }
    CkPrintf(o.str().c_str());
  }

  else
  {
    INFO("matrix size too large for printing\n");
  }
}

/** Convert an integer to a binary as a string.
 *
 * @param i The integer.
 *
 * @return The string represenation.
 */
std::string toBinary (unsigned int i)
{
  std::string bitString = std::bitset<8*sizeof(unsigned int)>(i).to_string();
  while(bitString[0] == '0')
  {
    bitString.erase(0, 1);
  }
  return bitString;
}

/** Convert some number of bytes into a human readable form. For instance, n =
 * 1024 would result in "1 kB".
 *
 * @param n The number of bytes to convert.
 *
 * @return The string representation of the human readable form.
 */
std::string humanReadableSize (unsigned long n)
{
  std::ostringstream o;
  double size = 0;
  const int numberUnits = 5;
  std::string unit[numberUnits] = { "B", "kiB", "MiB", "GiB", "TiB" };

  for(int i = 0; i < numberUnits; i++)
  {
    size /= 1024.;

    if(n/1024 > 0)
    {
      size += (n%1024);
      n /= 1024;
    }

    else
    {
      size += n;
      o << size << " " << unit[i];
      return o.str();
    }
  }
  ABORT("can not convert this number\n");
}
<commit_msg>Small fix in documentation.<commit_after>/** @file
 *
 * The implementation of the logging functions.
 *
 * @author Nicolas Bock <nicolas.bock@freeon.org>
 * @author Matt Challacombe <matt.challacombe@freeon.org>
 */

#include "logger.h"
#include "index.h"

#include <charm++.h>
#include <bitset>
#include <string>
#include <string.h>
#include <sstream>
#include <stdarg.h>
#include <stdio.h>

/** The main logging function. It is usually more convenient to use the macros
 * DEBUG, INFO, and ABORT instead.
 *
 * @param filename The name of the file where the message occurred.
 * @param linenumber The line number in that file.
 * @param function_name The name of the function.
 * @param tag A tag.
 * @param format The format string. See printf() for details.
 */
void logger (const char *const filename,
    const int linenumber,
    const char *const function_name,
    const char *const tag,
    const char *const format, ...)
{
  const int format_length = 2000;
  const int string_length = 5000;

  va_list ap;
  char new_format[format_length];
  char output_string[string_length];

  if(strlen(tag) > 0)
  {
    snprintf(new_format, format_length, "[%s:%d (%s) PE:%d %s] %s", filename,
        linenumber, function_name, CkMyPe(), tag, format);
  }

  else
  {
    snprintf(new_format, format_length, "[%s:%d (%s) PE:%d] %s", filename,
        linenumber, function_name, CkMyPe(), format);
  }

  va_start(ap, format);
  vsnprintf(output_string, string_length, new_format, ap);
  va_end(ap);
  CkPrintf(output_string);
}

/** Print a dense matrix.
 *
 * @param N The matrix size.
 * @param A The matrix.
 * @param format The format string for a message that is printed in front of
 * the matrix. See printf() for details.
 */
void printDense (int N, double *A, const char *const format, ...)
{
  std::ostringstream o;
  o.setf(std::ios::scientific);

  va_list ap;
  const int message_length = 2000;
  char message[message_length];

  va_start(ap, format);
  vsnprintf(message, message_length, format, ap);
  va_end(ap);

  o << message << std::endl;
  if(N <= 32)
  {
    for(int i = 0; i < N; i++) {
      for(int j = 0; j < N; j++)
      {
        o << " " << A[BLOCK_INDEX(i, j, 0, 0, N)];
      }
      o << std::endl;
    }
    CkPrintf(o.str().c_str());
  }

  else
  {
    INFO("matrix size too large for printing\n");
  }
}

/** Convert an integer to a binary as a string.
 *
 * @param i The integer.
 *
 * @return The string represenation.
 */
std::string toBinary (unsigned int i)
{
  std::string bitString = std::bitset<8*sizeof(unsigned int)>(i).to_string();
  while(bitString[0] == '0')
  {
    bitString.erase(0, 1);
  }
  return bitString;
}

/** Convert some number of bytes into a human readable form. For instance, n =
 * 1024 would result in "1 kiB".
 *
 * @param n The number of bytes to convert.
 *
 * @return The string representation of the human readable form.
 */
std::string humanReadableSize (unsigned long n)
{
  std::ostringstream o;
  double size = 0;
  const int numberUnits = 5;
  std::string unit[numberUnits] = { "B", "kiB", "MiB", "GiB", "TiB" };

  for(int i = 0; i < numberUnits; i++)
  {
    size /= 1024.;

    if(n/1024 > 0)
    {
      size += (n%1024);
      n /= 1024;
    }

    else
    {
      size += n;
      o << size << " " << unit[i];
      return o.str();
    }
  }
  ABORT("can not convert this number\n");
}
<|endoftext|>


<commit_before>#include "cp_task.h"
#include <regex>

#define BOOST_FILESYSTEM_NO_DEPRECATED
#define BOOST_NO_CXX11_SCOPED_ENUMS
#include <boost/filesystem.hpp>

namespace fs = boost::filesystem;

namespace {

void replace_substring(std::string &data, const std::string &from, const std::string &to)
{
	std::size_t pos = data.find(from);
	while(pos != std::string::npos) {
		data.replace(pos, from.size(), to);
		pos = data.find(from, pos + to.size());
	}
}

void escape_regex(std::string &regex)
{
	replace_substring(regex, "\\", "\\\\");
	replace_substring(regex, "^", "\\^");
	replace_substring(regex, ".", "\\.");
	replace_substring(regex, "$", "\\$");
	replace_substring(regex, "|", "\\|");
	replace_substring(regex, "(", "\\(");
	replace_substring(regex, ")", "\\)");
	replace_substring(regex, "[", "\\[");
	replace_substring(regex, "]", "\\]");
	replace_substring(regex, "*", "\\*");
	replace_substring(regex, "+", "\\+");
	replace_substring(regex, "?", "\\?");
	replace_substring(regex, "/", "\\/");
}

std::regex wildcards_regex(std::string wildcard_pattern)
{
	// Escape all regex special chars
	escape_regex(wildcard_pattern);

	// Convert chars '*?' back to their regex equivalents
	replace_substring(wildcard_pattern, "\\?", ".");
	replace_substring(wildcard_pattern, "\\*", ".*");

	return std::regex(wildcard_pattern);
}

} // namespace

cp_task::cp_task(std::size_t id, std::shared_ptr<task_metadata> task_meta) : task_base(id, task_meta)
{
	if (task_meta_->cmd_args.size() != 2) {
		throw task_exception(
			"Wrong number of arguments. Required: 2, Actual: " + std::to_string(task_meta_->cmd_args.size()));
	}
}

std::shared_ptr<task_results> cp_task::run()
{
	std::shared_ptr<task_results> result(new task_results());

	// Cannot use this normal variant due to bug in boost which
	// causes calling abort() instead of throwing an exception
	// try {
	// 	fs::copy(task_meta_->cmd_args[0], task_meta_->cmd_args[1]);
	// } catch (fs::filesystem_error &e) {
	// 	result->status = task_status::FAILED;
	// 	result->error_message = std::string("Cannot copy files. Error: ") + e.what();
	// }

	// Instead, following version seems to work fine

	// parse the input path and separate the last part
	fs::path input(task_meta_->cmd_args[0]);
	auto filename_matcher = input.filename().string();
	auto base_dir = input.remove_filename();
	auto pattern = wildcards_regex(filename_matcher);

	// parse output path and check if it is existing directory or not
	fs::path output(task_meta_->cmd_args[1]);
	bool output_is_dir = fs::is_directory(output);

	if (!fs::exists(base_dir)) {
		result->status = task_status::FAILED;
		result->error_message = "Source directory '" + base_dir.string() + "' does not exist";
		return result;
	}

	// go through all items in base directory and copy matching items to result location
	// TODO: replace to range loop when boost is upgraded
	fs::directory_iterator end_itr;
	for (fs::directory_iterator item(base_dir); item != end_itr; ++item) {
		if (regex_match(item->path().filename().string(), pattern)) {
			auto target = output_is_dir ? (output / item->path().filename()) : output;
			boost::system::error_code error_code;
			fs::copy(item->path(), target, error_code);
			if (error_code.value() != boost::system::errc::success) {
				result->status = task_status::FAILED;
				result->error_message = std::string("Cannot copy files. Error: ") + error_code.message();
				break;
			}
		}
	}

	return result;
}
<commit_msg>Copy task should be able to copy directories<commit_after>#include "cp_task.h"
#include <regex>
#include "helpers/filesystem.h"

#define BOOST_FILESYSTEM_NO_DEPRECATED
#define BOOST_NO_CXX11_SCOPED_ENUMS
#include <boost/filesystem.hpp>

namespace fs = boost::filesystem;

namespace {

void replace_substring(std::string &data, const std::string &from, const std::string &to)
{
	std::size_t pos = data.find(from);
	while(pos != std::string::npos) {
		data.replace(pos, from.size(), to);
		pos = data.find(from, pos + to.size());
	}
}

void escape_regex(std::string &regex)
{
	replace_substring(regex, "\\", "\\\\");
	replace_substring(regex, "^", "\\^");
	replace_substring(regex, ".", "\\.");
	replace_substring(regex, "$", "\\$");
	replace_substring(regex, "|", "\\|");
	replace_substring(regex, "(", "\\(");
	replace_substring(regex, ")", "\\)");
	replace_substring(regex, "[", "\\[");
	replace_substring(regex, "]", "\\]");
	replace_substring(regex, "*", "\\*");
	replace_substring(regex, "+", "\\+");
	replace_substring(regex, "?", "\\?");
	replace_substring(regex, "/", "\\/");
}

std::regex wildcards_regex(std::string wildcard_pattern)
{
	// Escape all regex special chars
	escape_regex(wildcard_pattern);

	// Convert chars '*?' back to their regex equivalents
	replace_substring(wildcard_pattern, "\\?", ".");
	replace_substring(wildcard_pattern, "\\*", ".*");

	return std::regex(wildcard_pattern);
}

} // namespace

cp_task::cp_task(std::size_t id, std::shared_ptr<task_metadata> task_meta) : task_base(id, task_meta)
{
	if (task_meta_->cmd_args.size() != 2) {
		throw task_exception(
			"Wrong number of arguments. Required: 2, Actual: " + std::to_string(task_meta_->cmd_args.size()));
	}
}

std::shared_ptr<task_results> cp_task::run()
{
	std::shared_ptr<task_results> result(new task_results());

	// Cannot use this normal variant due to bug in boost which
	// causes calling abort() instead of throwing an exception
	// try {
	// 	fs::copy(task_meta_->cmd_args[0], task_meta_->cmd_args[1]);
	// } catch (fs::filesystem_error &e) {
	// 	result->status = task_status::FAILED;
	// 	result->error_message = std::string("Cannot copy files. Error: ") + e.what();
	// }

	// Instead, following version seems to work fine

	// parse the input path and separate the last part
	fs::path input(task_meta_->cmd_args[0]);
	auto filename_matcher = input.filename().string();
	auto base_dir = input.remove_filename();
	auto pattern = wildcards_regex(filename_matcher);

	// parse output path and check if it is existing directory or not
	fs::path output(task_meta_->cmd_args[1]);
	bool output_is_dir = fs::is_directory(output);

	if (!fs::exists(base_dir)) {
		result->status = task_status::FAILED;
		result->error_message = "Source directory '" + base_dir.string() + "' does not exist";
		return result;
	}

	// go through all items in base directory and copy matching items to result location
	// TODO: replace to range loop when boost is upgraded
	fs::directory_iterator end_itr;
	for (fs::directory_iterator item(base_dir); item != end_itr; ++item) {
		if (regex_match(item->path().filename().string(), pattern)) {
			auto target = output_is_dir ? (output / item->path().filename()) : output;
			if (fs::is_directory(item->path())) {
				helpers::copy_directory(item->path(), target);
			} else {
				boost::system::error_code error_code;
				fs::copy(item->path(), target, error_code);
				if (error_code.value() != boost::system::errc::success) {
					result->status = task_status::FAILED;
					result->error_message = std::string("Cannot copy files. Error: ") + error_code.message();
					break;
				}
			}
		}
	}

	return result;
}
<|endoftext|>


<commit_before>// $Id$

// The libMesh Finite Element Library.
// Copyright (C) 2002-2008 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner

// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

// C++ includes
#include "libmesh_config.h"

#ifdef LIBMESH_HAVE_PETSC

// Local includes
#include "petsc_matrix.h"
#include "dof_map.h"
#include "dense_matrix.h"
#include "petsc_vector.h"

namespace libMesh
{

//-----------------------------------------------------------------------
// PetscMatrix members
template <typename T>
void PetscMatrix<T>::init (const unsigned int m,
			   const unsigned int n,
			   const unsigned int m_l,
			   const unsigned int n_l,
			   const unsigned int nnz,
			   const unsigned int noz)
{
  // We allow 0x0 matrices now
  //if ((m==0) || (n==0))
  //  return;

  // Clear initialized matrices
  if (this->initialized())
    this->clear();

  this->_is_initialized = true;

  int ierr     = 0;
  int m_global = static_cast<int>(m);
  int n_global = static_cast<int>(n);
  int m_local  = static_cast<int>(m_l);
  int n_local  = static_cast<int>(n_l);
  int n_nz     = static_cast<int>(nnz);
  int n_oz     = static_cast<int>(noz);

  // create a sequential matrix on one processor
  if (libMesh::n_processors() == 1)
    {
      libmesh_assert ((m_l == m) && (n_l == n));

      // Create matrix.  Revisit later to do preallocation and make more efficient
      ierr = MatCreateSeqAIJ (libMesh::COMM_WORLD, m_global, n_global,
			      n_nz, PETSC_NULL, &_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatSetFromOptions (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  else
    {
      parallel_only();

      ierr = MatCreateMPIAIJ (libMesh::COMM_WORLD, m_local, n_local, m_global, n_global,
			      n_nz, PETSC_NULL, n_oz, PETSC_NULL, &_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatSetFromOptions (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  this->zero ();
}

template <typename T>
void PetscMatrix<T>::init ()
{
  libmesh_assert (this->_dof_map != NULL);

  // Clear initialized matrices
  if (this->initialized())
    this->clear();

  this->_is_initialized = true;

  const unsigned int m   = this->_dof_map->n_dofs();
  const unsigned int n   = m;
  const unsigned int n_l = this->_dof_map->n_dofs_on_processor(libMesh::processor_id());
  const unsigned int m_l = n_l;

  const std::vector<unsigned int>& n_nz = this->_dof_map->get_n_nz();
  const std::vector<unsigned int>& n_oz = this->_dof_map->get_n_oz();

  // Make sure the sparsity pattern isn't empty unless the matrix is 0x0
  libmesh_assert (n_nz.size() == n_l);
  libmesh_assert (n_oz.size() == n_l);

  // We allow 0x0 matrices now
  //if (m==0)
  //  return;

  int ierr     = 0;
  int m_global = static_cast<int>(m);
  int n_global = static_cast<int>(n);
  int m_local  = static_cast<int>(m_l);
  int n_local  = static_cast<int>(n_l);

  // create a sequential matrix on one processor
  if (libMesh::n_processors() == 1)
    {
      libmesh_assert ((m_l == m) && (n_l == n));
      if (n_nz.empty())
        ierr = MatCreateSeqAIJ (libMesh::COMM_WORLD, n_global, n_global,
			        PETSC_NULL, (int*) PETSC_NULL, &_mat);
      else
        ierr = MatCreateSeqAIJ (libMesh::COMM_WORLD, n_global, n_global,
			        PETSC_NULL, (int*) &n_nz[0], &_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatSetFromOptions (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  else
    {
      parallel_only();

      if (n_nz.empty())
        ierr = MatCreateMPIAIJ (libMesh::COMM_WORLD,
			        m_local, n_local,
			        m_global, n_global,
			        PETSC_NULL, (int*) PETSC_NULL,
			        PETSC_NULL, (int*) PETSC_NULL, &_mat);
      else
        ierr = MatCreateMPIAIJ (libMesh::COMM_WORLD,
			        m_local, n_local,
			        m_global, n_global,
			        PETSC_NULL, (int*) &n_nz[0],
			        PETSC_NULL, (int*) &n_oz[0], &_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatSetFromOptions (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  this->zero();
}

template <typename T>
void PetscMatrix<T>::zero ()
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  int ierr=0;

  ierr = MatZeroEntries(_mat);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::zero_rows (std::vector<int> & rows, T diag_value)
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  int ierr=0;

  if(!rows.empty())
    ierr = MatZeroRows(_mat, rows.size(), &rows[0], diag_value);
  else
    ierr = MatZeroRows(_mat, 0, PETSC_NULL, diag_value);

  CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::clear ()
{
  int ierr=0;

  if ((this->initialized()) && (this->_destroy_mat_on_exit))
    {
      semiparallel_only();

      ierr = MatDestroy (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      this->_is_initialized = false;
    }
}

template <typename T>
Real PetscMatrix<T>::l1_norm () const
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  int ierr=0;
  PetscReal petsc_value;
  Real value;

  libmesh_assert (this->closed());

  ierr = MatNorm(_mat, NORM_1, &petsc_value);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);

  value = static_cast<Real>(petsc_value);

  return value;
}

template <typename T>
Real PetscMatrix<T>::linfty_norm () const
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  int ierr=0;
  PetscReal petsc_value;
  Real value;

  libmesh_assert (this->closed());

  ierr = MatNorm(_mat, NORM_INFINITY, &petsc_value);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);

  value = static_cast<Real>(petsc_value);

  return value;
}

template <typename T>
void PetscMatrix<T>::print_matlab (const std::string name) const
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  // libmesh_assert (this->closed());
  this->close();

  int ierr=0;
  PetscViewer petsc_viewer;

  ierr = PetscViewerCreate (libMesh::COMM_WORLD,
			    &petsc_viewer);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);

  /**
   * Create an ASCII file containing the matrix
   * if a filename was provided.
   */
  if (name != "NULL")
    {
      ierr = PetscViewerASCIIOpen( libMesh::COMM_WORLD,
				   name.c_str(),
				   &petsc_viewer);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = PetscViewerSetFormat (petsc_viewer,
				   PETSC_VIEWER_ASCII_MATLAB);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatView (_mat, petsc_viewer);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  /**
   * Otherwise the matrix will be dumped to the screen.
   */
  else
    {
      ierr = PetscViewerSetFormat (PETSC_VIEWER_STDOUT_WORLD,
				   PETSC_VIEWER_ASCII_MATLAB);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatView (_mat, PETSC_VIEWER_STDOUT_WORLD);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  /**
   * Destroy the viewer.
   */
  ierr = PetscViewerDestroy (petsc_viewer);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::add_matrix(const DenseMatrix<T>& dm,
				const std::vector<unsigned int>& rows,
				const std::vector<unsigned int>& cols)
{
  libmesh_assert (this->initialized());

  const unsigned int m = dm.m();
  const unsigned int n = dm.n();

  libmesh_assert (rows.size() == m);
  libmesh_assert (cols.size() == n);

  int ierr=0;

  // These casts are required for PETSc <= 2.1.5
  ierr = MatSetValues(_mat,
		      m, (int*) &rows[0],
		      n, (int*) &cols[0],
		      (PetscScalar*) &dm.get_values()[0],
		      ADD_VALUES);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::_get_submatrix(SparseMatrix<T>& submatrix,
				    const std::vector<unsigned int> &rows,
				    const std::vector<unsigned int> &cols,
				    const bool reuse_submatrix) const
{
  // Can only extract submatrices from closed matrices
  this->close();

  // Make sure the SparseMatrix passed in is really a PetscMatrix
  PetscMatrix<T>* petsc_submatrix = libmesh_cast_ptr<PetscMatrix<T>*>(&submatrix);

  // If we're not reusing submatrix and submatrix is already initialized
  // then we need to clear it, otherwise we get a memory leak.
  if( !reuse_submatrix && submatrix.initialized() )
    submatrix.clear();

  // Construct row and column index sets.
  int ierr=0;
  IS isrow, iscol;

  ierr = ISCreateGeneral(libMesh::COMM_WORLD,
			 rows.size(),
			 (int*) &rows[0],
			 &isrow); CHKERRABORT(libMesh::COMM_WORLD,ierr);

  ierr = ISCreateGeneral(libMesh::COMM_WORLD,
			 cols.size(),
			 (int*) &cols[0],
			 &iscol); CHKERRABORT(libMesh::COMM_WORLD,ierr);

  // Extract submatrix
#if !PETSC_VERSION_LESS_THAN(3,0,1) || !PETSC_VERSION_RELEASE
  ierr = MatGetSubMatrix(_mat,
			 isrow,
			 iscol,
			 (reuse_submatrix ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX),
			 &(petsc_submatrix->_mat));  CHKERRABORT(libMesh::COMM_WORLD,ierr);
#else
  ierr = MatGetSubMatrix(_mat,
			 isrow,
			 iscol,
			 PETSC_DECIDE,
			 (reuse_submatrix ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX),
			 &(petsc_submatrix->_mat));  CHKERRABORT(libMesh::COMM_WORLD,ierr);
#endif

  // Specify that the new submatrix is initialized and close it.
  petsc_submatrix->_is_initialized = true;
  petsc_submatrix->close();

  // Clean up PETSc data structures
  ierr = ISDestroy(isrow); CHKERRABORT(libMesh::COMM_WORLD,ierr);
  ierr = ISDestroy(iscol); CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::get_diagonal (NumericVector<T>& dest) const
{
  // Make sure the NumericVector passed in is really a PetscVector
  PetscVector<T>& petsc_dest = libmesh_cast_ref<PetscVector<T>&>(dest);

  // Call PETSc function.

#if PETSC_VERSION_LESS_THAN(2,3,1)

  libMesh::out << "This method has been developed with PETSc 2.3.1.  "
	        << "No one has made it backwards compatible with older "
	        << "versions of PETSc so far; however, it might work "
	        << "without any change with some older version." << std::endl;
  libmesh_error();

#else

  // Needs a const_cast since PETSc does not work with const.
  int ierr =
    MatGetDiagonal(const_cast<PetscMatrix<T>*>(this)->mat(),petsc_dest.vec()); CHKERRABORT(libMesh::COMM_WORLD,ierr);

#endif

}

template <typename T>
void PetscMatrix<T>::get_transpose (SparseMatrix<T>& dest) const
{
  // Make sure the SparseMatrix passed in is really a PetscMatrix
  PetscMatrix<T>& petsc_dest = libmesh_cast_ref<PetscMatrix<T>&>(dest);

  // If we aren't reusing the matrix then need to clear dest,
  // otherwise we get a memory leak
  if(&petsc_dest != this)
    dest.clear();

  int ierr;
#if PETSC_VERSION_LESS_THAN(3,0,0)
  if (&petsc_dest == this)
    ierr = MatTranspose(_mat,PETSC_NULL);
  else
    ierr = MatTranspose(_mat,&petsc_dest._mat);
  CHKERRABORT(libMesh::COMM_WORLD,ierr);
#else
  // FIXME - we can probably use MAT_REUSE_MATRIX in more situations
  if (&petsc_dest == this)
    ierr = MatTranspose(_mat,MAT_REUSE_MATRIX,&petsc_dest._mat);
  else
    ierr = MatTranspose(_mat,MAT_INITIAL_MATRIX,&petsc_dest._mat);
  CHKERRABORT(libMesh::COMM_WORLD,ierr);
#endif

  // Specify that the transposed matrix is initialized and close it.
  petsc_dest._is_initialized = true;
  petsc_dest.close();
}

//------------------------------------------------------------------
// Explicit instantiations
template class PetscMatrix<Number>;

} // namespace libMesh

#endif // #ifdef LIBMESH_HAVE_PETSC
<commit_msg>Fixed another rectangular PetscMatrix "bug" - this one is impossible to trigger as-is but might crop up if we change that constructor later.<commit_after>// $Id$

// The libMesh Finite Element Library.
// Copyright (C) 2002-2008 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner

// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

// C++ includes
#include "libmesh_config.h"

#ifdef LIBMESH_HAVE_PETSC

// Local includes
#include "petsc_matrix.h"
#include "dof_map.h"
#include "dense_matrix.h"
#include "petsc_vector.h"

namespace libMesh
{

//-----------------------------------------------------------------------
// PetscMatrix members
template <typename T>
void PetscMatrix<T>::init (const unsigned int m,
			   const unsigned int n,
			   const unsigned int m_l,
			   const unsigned int n_l,
			   const unsigned int nnz,
			   const unsigned int noz)
{
  // We allow 0x0 matrices now
  //if ((m==0) || (n==0))
  //  return;

  // Clear initialized matrices
  if (this->initialized())
    this->clear();

  this->_is_initialized = true;

  int ierr     = 0;
  int m_global = static_cast<int>(m);
  int n_global = static_cast<int>(n);
  int m_local  = static_cast<int>(m_l);
  int n_local  = static_cast<int>(n_l);
  int n_nz     = static_cast<int>(nnz);
  int n_oz     = static_cast<int>(noz);

  // create a sequential matrix on one processor
  if (libMesh::n_processors() == 1)
    {
      libmesh_assert ((m_l == m) && (n_l == n));

      // Create matrix.  Revisit later to do preallocation and make more efficient
      ierr = MatCreateSeqAIJ (libMesh::COMM_WORLD, m_global, n_global,
			      n_nz, PETSC_NULL, &_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatSetFromOptions (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  else
    {
      parallel_only();

      ierr = MatCreateMPIAIJ (libMesh::COMM_WORLD, m_local, n_local, m_global, n_global,
			      n_nz, PETSC_NULL, n_oz, PETSC_NULL, &_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatSetFromOptions (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  this->zero ();
}

template <typename T>
void PetscMatrix<T>::init ()
{
  libmesh_assert (this->_dof_map != NULL);

  // Clear initialized matrices
  if (this->initialized())
    this->clear();

  this->_is_initialized = true;

  const unsigned int m   = this->_dof_map->n_dofs();
  const unsigned int n   = m;
  const unsigned int n_l = this->_dof_map->n_dofs_on_processor(libMesh::processor_id());
  const unsigned int m_l = n_l;

  const std::vector<unsigned int>& n_nz = this->_dof_map->get_n_nz();
  const std::vector<unsigned int>& n_oz = this->_dof_map->get_n_oz();

  // Make sure the sparsity pattern isn't empty unless the matrix is 0x0
  libmesh_assert (n_nz.size() == n_l);
  libmesh_assert (n_oz.size() == n_l);

  // We allow 0x0 matrices now
  //if (m==0)
  //  return;

  int ierr     = 0;
  int m_global = static_cast<int>(m);
  int n_global = static_cast<int>(n);
  int m_local  = static_cast<int>(m_l);
  int n_local  = static_cast<int>(n_l);

  // create a sequential matrix on one processor
  if (libMesh::n_processors() == 1)
    {
      libmesh_assert ((m_l == m) && (n_l == n));
      if (n_nz.empty())
        ierr = MatCreateSeqAIJ (libMesh::COMM_WORLD, m_global, n_global,
			        PETSC_NULL, (int*) PETSC_NULL, &_mat);
      else
        ierr = MatCreateSeqAIJ (libMesh::COMM_WORLD, m_global, n_global,
			        PETSC_NULL, (int*) &n_nz[0], &_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatSetFromOptions (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  else
    {
      parallel_only();

      if (n_nz.empty())
        ierr = MatCreateMPIAIJ (libMesh::COMM_WORLD,
			        m_local, n_local,
			        m_global, n_global,
			        PETSC_NULL, (int*) PETSC_NULL,
			        PETSC_NULL, (int*) PETSC_NULL, &_mat);
      else
        ierr = MatCreateMPIAIJ (libMesh::COMM_WORLD,
			        m_local, n_local,
			        m_global, n_global,
			        PETSC_NULL, (int*) &n_nz[0],
			        PETSC_NULL, (int*) &n_oz[0], &_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatSetFromOptions (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  this->zero();
}

template <typename T>
void PetscMatrix<T>::zero ()
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  int ierr=0;

  ierr = MatZeroEntries(_mat);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::zero_rows (std::vector<int> & rows, T diag_value)
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  int ierr=0;

  if(!rows.empty())
    ierr = MatZeroRows(_mat, rows.size(), &rows[0], diag_value);
  else
    ierr = MatZeroRows(_mat, 0, PETSC_NULL, diag_value);

  CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::clear ()
{
  int ierr=0;

  if ((this->initialized()) && (this->_destroy_mat_on_exit))
    {
      semiparallel_only();

      ierr = MatDestroy (_mat);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      this->_is_initialized = false;
    }
}

template <typename T>
Real PetscMatrix<T>::l1_norm () const
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  int ierr=0;
  PetscReal petsc_value;
  Real value;

  libmesh_assert (this->closed());

  ierr = MatNorm(_mat, NORM_1, &petsc_value);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);

  value = static_cast<Real>(petsc_value);

  return value;
}

template <typename T>
Real PetscMatrix<T>::linfty_norm () const
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  int ierr=0;
  PetscReal petsc_value;
  Real value;

  libmesh_assert (this->closed());

  ierr = MatNorm(_mat, NORM_INFINITY, &petsc_value);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);

  value = static_cast<Real>(petsc_value);

  return value;
}

template <typename T>
void PetscMatrix<T>::print_matlab (const std::string name) const
{
  libmesh_assert (this->initialized());

  semiparallel_only();

  // libmesh_assert (this->closed());
  this->close();

  int ierr=0;
  PetscViewer petsc_viewer;

  ierr = PetscViewerCreate (libMesh::COMM_WORLD,
			    &petsc_viewer);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);

  /**
   * Create an ASCII file containing the matrix
   * if a filename was provided.
   */
  if (name != "NULL")
    {
      ierr = PetscViewerASCIIOpen( libMesh::COMM_WORLD,
				   name.c_str(),
				   &petsc_viewer);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = PetscViewerSetFormat (petsc_viewer,
				   PETSC_VIEWER_ASCII_MATLAB);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatView (_mat, petsc_viewer);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  /**
   * Otherwise the matrix will be dumped to the screen.
   */
  else
    {
      ierr = PetscViewerSetFormat (PETSC_VIEWER_STDOUT_WORLD,
				   PETSC_VIEWER_ASCII_MATLAB);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);

      ierr = MatView (_mat, PETSC_VIEWER_STDOUT_WORLD);
             CHKERRABORT(libMesh::COMM_WORLD,ierr);
    }

  /**
   * Destroy the viewer.
   */
  ierr = PetscViewerDestroy (petsc_viewer);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::add_matrix(const DenseMatrix<T>& dm,
				const std::vector<unsigned int>& rows,
				const std::vector<unsigned int>& cols)
{
  libmesh_assert (this->initialized());

  const unsigned int m = dm.m();
  const unsigned int n = dm.n();

  libmesh_assert (rows.size() == m);
  libmesh_assert (cols.size() == n);

  int ierr=0;

  // These casts are required for PETSc <= 2.1.5
  ierr = MatSetValues(_mat,
		      m, (int*) &rows[0],
		      n, (int*) &cols[0],
		      (PetscScalar*) &dm.get_values()[0],
		      ADD_VALUES);
         CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::_get_submatrix(SparseMatrix<T>& submatrix,
				    const std::vector<unsigned int> &rows,
				    const std::vector<unsigned int> &cols,
				    const bool reuse_submatrix) const
{
  // Can only extract submatrices from closed matrices
  this->close();

  // Make sure the SparseMatrix passed in is really a PetscMatrix
  PetscMatrix<T>* petsc_submatrix = libmesh_cast_ptr<PetscMatrix<T>*>(&submatrix);

  // If we're not reusing submatrix and submatrix is already initialized
  // then we need to clear it, otherwise we get a memory leak.
  if( !reuse_submatrix && submatrix.initialized() )
    submatrix.clear();

  // Construct row and column index sets.
  int ierr=0;
  IS isrow, iscol;

  ierr = ISCreateGeneral(libMesh::COMM_WORLD,
			 rows.size(),
			 (int*) &rows[0],
			 &isrow); CHKERRABORT(libMesh::COMM_WORLD,ierr);

  ierr = ISCreateGeneral(libMesh::COMM_WORLD,
			 cols.size(),
			 (int*) &cols[0],
			 &iscol); CHKERRABORT(libMesh::COMM_WORLD,ierr);

  // Extract submatrix
#if !PETSC_VERSION_LESS_THAN(3,0,1) || !PETSC_VERSION_RELEASE
  ierr = MatGetSubMatrix(_mat,
			 isrow,
			 iscol,
			 (reuse_submatrix ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX),
			 &(petsc_submatrix->_mat));  CHKERRABORT(libMesh::COMM_WORLD,ierr);
#else
  ierr = MatGetSubMatrix(_mat,
			 isrow,
			 iscol,
			 PETSC_DECIDE,
			 (reuse_submatrix ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX),
			 &(petsc_submatrix->_mat));  CHKERRABORT(libMesh::COMM_WORLD,ierr);
#endif

  // Specify that the new submatrix is initialized and close it.
  petsc_submatrix->_is_initialized = true;
  petsc_submatrix->close();

  // Clean up PETSc data structures
  ierr = ISDestroy(isrow); CHKERRABORT(libMesh::COMM_WORLD,ierr);
  ierr = ISDestroy(iscol); CHKERRABORT(libMesh::COMM_WORLD,ierr);
}

template <typename T>
void PetscMatrix<T>::get_diagonal (NumericVector<T>& dest) const
{
  // Make sure the NumericVector passed in is really a PetscVector
  PetscVector<T>& petsc_dest = libmesh_cast_ref<PetscVector<T>&>(dest);

  // Call PETSc function.

#if PETSC_VERSION_LESS_THAN(2,3,1)

  libMesh::out << "This method has been developed with PETSc 2.3.1.  "
	        << "No one has made it backwards compatible with older "
	        << "versions of PETSc so far; however, it might work "
	        << "without any change with some older version." << std::endl;
  libmesh_error();

#else

  // Needs a const_cast since PETSc does not work with const.
  int ierr =
    MatGetDiagonal(const_cast<PetscMatrix<T>*>(this)->mat(),petsc_dest.vec()); CHKERRABORT(libMesh::COMM_WORLD,ierr);

#endif

}

template <typename T>
void PetscMatrix<T>::get_transpose (SparseMatrix<T>& dest) const
{
  // Make sure the SparseMatrix passed in is really a PetscMatrix
  PetscMatrix<T>& petsc_dest = libmesh_cast_ref<PetscMatrix<T>&>(dest);

  // If we aren't reusing the matrix then need to clear dest,
  // otherwise we get a memory leak
  if(&petsc_dest != this)
    dest.clear();

  int ierr;
#if PETSC_VERSION_LESS_THAN(3,0,0)
  if (&petsc_dest == this)
    ierr = MatTranspose(_mat,PETSC_NULL);
  else
    ierr = MatTranspose(_mat,&petsc_dest._mat);
  CHKERRABORT(libMesh::COMM_WORLD,ierr);
#else
  // FIXME - we can probably use MAT_REUSE_MATRIX in more situations
  if (&petsc_dest == this)
    ierr = MatTranspose(_mat,MAT_REUSE_MATRIX,&petsc_dest._mat);
  else
    ierr = MatTranspose(_mat,MAT_INITIAL_MATRIX,&petsc_dest._mat);
  CHKERRABORT(libMesh::COMM_WORLD,ierr);
#endif

  // Specify that the transposed matrix is initialized and close it.
  petsc_dest._is_initialized = true;
  petsc_dest.close();
}

//------------------------------------------------------------------
// Explicit instantiations
template class PetscMatrix<Number>;

} // namespace libMesh

#endif // #ifdef LIBMESH_HAVE_PETSC
<|endoftext|>


<commit_before>#include <sirius.h>

using namespace sirius;

extern "C" void randomize_on_gpu(double* ptr, size_t size);

void test()
{
    int N = 500;
    int K = 30000;
    mdarray<double_complex, 2> A(N, K);
    mdarray<double_complex, 2> B(K, N);
    mdarray<double_complex, 2> C(N, N);

    for (int j = 0; j < K; j++)
    {
        for (int i = 0; i < N; i++)
        {
            A(i, j) = 1.0;
            B(j, i) = 1.0;
        }
    }
    blas<cpu>::gemm(0, 0, N, N, K, A.at<cpu>(), A.ld(), B.at<cpu>(), B.ld(), C.at<cpu>(), C.ld());
    Platform::comm_world().allreduce(C.ptr(), (int)C.size());

    std::cout << C(0, 0) << " " << C(N - 1, N - 1) << std::endl;

    A.allocate_on_device();
    A.copy_to_device();
    B.allocate_on_device();
    B.copy_to_device();
    C.allocate_on_device();

    blas<gpu>::gemm(0, 0, N, N, K, A.at<gpu>(), A.ld(), B.at<gpu>(), B.ld(), C.at<gpu>(), C.ld());
    Platform::comm_world().allreduce(C.at<gpu>(), (int)C.size());
    C.copy_to_host();

    std::cout << C(0, 0) << " " << C(N - 1, N - 1) << std::endl;
}

void test1()
{
    mdarray<double_complex, 1> A(1000000);
    A.allocate_on_device();
    randomize_on_gpu((double*)A.at<gpu>(), A.size() * 2);

    A.copy_to_host();
    Platform::comm_world().allreduce(A.at<cpu>(), (int)A.size());

    mdarray<double_complex, 1> A_ref(1000000);
    A >> A_ref;

    Platform::comm_world().allreduce(A.at<gpu>(), (int)A.size());
    A.copy_to_host();

    for (int i = 0; i < 1000000; i++)
    {
        double d = std::abs(A(i) - A_ref(i));
        if (d > 1e-8) INFO << "i=" << i << " diff=" << d << std::endl;
    }
}

int main(int argn, char** argv)
{
    Platform::initialize(1);
    test();
    test1();
    Platform::finalize();
}
<commit_msg>better test of gpudirect<commit_after>#include <sirius.h>

using namespace sirius;

extern "C" void randomize_on_gpu(double* ptr, size_t size);

void test1()
{
    int N = 1000;
    int K = 10000;
    mdarray<double_complex, 2> A(N, K);
    mdarray<double_complex, 2> B(K, N);
    mdarray<double_complex, 2> C(N, N);

    for (int j = 0; j < K; j++)
    {
        for (int i = 0; i < N; i++)
        {
            A(i, j) = double_complex(1, 1);
            B(j, i) = double_complex(1, 1);
        }
    }
    A.allocate_on_device();
    A.copy_to_device();
    B.allocate_on_device();
    B.copy_to_device();
    C.allocate_on_device();

    blas<gpu>::gemm(0, 0, N, N, K, A.at<gpu>(), A.ld(), B.at<gpu>(), B.ld(), C.at<gpu>(), C.ld());
    Platform::comm_world().allreduce(C.at<gpu>(), (int)C.size());
    C.copy_to_host();

    int nerr = 0;
    for (int i = 0; i < N; i++)
    {
        for (int j = 0; j < N; j++)
        {
            if (std::abs(C(i, j) - double_complex(0, 2 * K * Platform::comm_world().size())) > 1e-12) nerr++;
        }
    }

    if (nerr)
    {
        printf("test1, number of errors: %i\n", nerr);
    }
    else
    {
        printf("test1 passed!\n");
    }
}

void test2()
{
    int N = 1000000;

    mdarray<double_complex, 1> A(N);
    A.allocate_on_device();
    for (int i = 0; i < N; i++) A(i) = double_complex(1, 1);

    A.copy_to_device();
    Platform::comm_world().allreduce(A.at<gpu>(), (int)A.size());
    A.copy_to_host();

    int nerr = 0;
    for (int i = 0; i < N; i++)
    {
        double d = std::abs(A(i) - double_complex(Platform::comm_world().size(), Platform::comm_world().size()));
        if (d > 1e-12) nerr++;
    }

    if (nerr)
    {
        printf("test2, number of errors: %i\n", nerr);
    }
    else
    {
        printf("test2 passed!\n");
    }
}

void test3()
{
    mdarray<double_complex, 1> A(1000000);
    A.allocate_on_device();
    randomize_on_gpu((double*)A.at<gpu>(), A.size() * 2);

    A.copy_to_host();
    Platform::comm_world().allreduce(A.at<cpu>(), (int)A.size());

    mdarray<double_complex, 1> A_ref(1000000);
    A >> A_ref;

    Platform::comm_world().allreduce(A.at<gpu>(), (int)A.size());
    A.copy_to_host();

    for (int i = 0; i < 1000000; i++)
    {
        double d = std::abs(A(i) - A_ref(i));
        if (d > 1e-8) INFO << "i=" << i << " diff=" << d << std::endl;
    }
}

int main(int argn, char** argv)
{
    Platform::initialize(1);
    test1();
    test2();
    test3();
    test1();
    test2();
    test3();
    Platform::finalize();
}
<|endoftext|>


<commit_before>#include "apps_starter.h"

std::vector<job> apps_vect;			// Определение вектора запущенных программ
struct termios hos_tmode;			// настройки терминала для hos

void sighandler(int signo)
{
	if (signo == SIGTSTP) {

	}

	if (signo == SIGINT) {

	}
}

void init_signals()
{
	signal(SIGINT, &sighandler);
	signal(SIGTSTP, &sighandler);
	signal(SIGTTIN, SIG_IGN);
	signal(SIGTTOU, SIG_IGN);
}

int app_start(int number_of_app, char** argv) {
	std::string name_app	= configurator(APPS_FILE, str(number_of_app) + "_app_launcher", "", false);
	std::string path_to_dir	= configurator(APPS_FILE, str(number_of_app) + "_app_path", "", false);
	erase();
	endwin();

	int		status;
	pid_t	chpid	= fork();

	job j = {
		.name = name_app,
		.pid = chpid
	};

	j.running = true;

	apps_vect.insert(apps_vect.end(), j);

	if (chpid == 0) {
		signal(SIGTTIN, SIG_IGN);
		signal(SIGTTOU, SIG_IGN);
		tcsetpgrp(STDIN_FILENO, getpid());
		chdir(path_to_dir.c_str());
		setpgid(getpid(), getpid());			 	// Создаём группу процессов
		if (execl(name_app.c_str(), "", NULL) == -1) 	// parent process
			exit(0);
	} else {
		waitpid(chpid, &status,WUNTRACED);
		tcsetpgrp(STDIN_FILENO, getpid());
		tcgetattr(STDIN_FILENO, &j.tmode);
		tcsetattr(STDIN_FILENO, TCSADRAIN, &hos_tmode);
		init_display();
		init_color();
	}
	return 0;
}<commit_msg>fg interface<commit_after>#include "apps_starter.h"

std::vector<job> apps_vect;			// Определение вектора запущенных программ
struct termios hos_tmode;			// настройки терминала для hos

void sighandler(int signo)
{
	if (signo == SIGTSTP) {

	}

	if (signo == SIGINT) {

	}
}

void fg_job(pid_t pid)
{

}

void init_signals()
{
	signal(SIGINT, &sighandler);
	signal(SIGTSTP, &sighandler);
	signal(SIGTTIN, SIG_IGN);
	signal(SIGTTOU, SIG_IGN);
}

int app_start(int number_of_app, char** argv) {
	std::string name_app	= configurator(APPS_FILE, str(number_of_app) + "_app_launcher", "", false);
	std::string path_to_dir	= configurator(APPS_FILE, str(number_of_app) + "_app_path", "", false);
	erase();
	endwin();

	int		status;
	pid_t	chpid	= fork();

	job j = {
		.name = name_app,
		.pid = chpid
	};

	j.running = true;

	// Помещаем процесс в список запущенных процессов
	apps_vect.insert(apps_vect.end(), j);

	if (chpid == 0) {
		signal(SIGTTIN, SIG_IGN);
		signal(SIGTTOU, SIG_IGN);
		tcsetpgrp(STDIN_FILENO, getpid());
		chdir(path_to_dir.c_str());
		setpgid(getpid(), getpid());			 	// Создаём группу процессов
		if (execl(name_app.c_str(), "", NULL) == -1) 	// parent process
			exit(0);
	} else {
		waitpid(chpid, &status,WUNTRACED);
		tcsetpgrp(STDIN_FILENO, getpid());
		tcgetattr(STDIN_FILENO, &j.tmode);
		tcsetattr(STDIN_FILENO, TCSADRAIN, &hos_tmode);

		if (WIFSTOPPED(status)) {	/* Если процесс был остановлен во время выполнения */
			apps_vect.back().running = false;
		}

		init_display();
		init_color();
	}
	return 0;
}<|endoftext|>


<commit_before><commit_msg>odb: make corner IO cells not auto placeable<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2009-2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <pubkey.h>
#include <script/interpreter.h>
#include <streams.h>
#include <util/memory.h>
#include <version.h>

#include <test/fuzz/fuzz.h>

/** Flags that are not forbidden by an assert */
static bool IsValidFlagCombination(unsigned flags);

void initialize()
{
    static const auto verify_handle = MakeUnique<ECCVerifyHandle>();
}

void test_one_input(const std::vector<uint8_t>& buffer)
{
    CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION);
    try {
        int nVersion;
        ds >> nVersion;
        ds.SetVersion(nVersion);
    } catch (const std::ios_base::failure&) {
        return;
    }

    try {
        const CTransaction tx(deserialize, ds);

        unsigned int verify_flags;
        ds >> verify_flags;

        if (!IsValidFlagCombination(verify_flags)) return;

        unsigned int fuzzed_flags;
        ds >> fuzzed_flags;

        std::vector<CTxOut> spent_outputs;
        for (unsigned i = 0; i < tx.vin.size(); ++i) {
            CTxOut prevout;
            ds >> prevout;
            if (!MoneyRange(prevout.nValue)) {
                // prevouts should be consensus-valid
                prevout.nValue = 1;
            }
            spent_outputs.push_back(prevout);
        }
        PrecomputedTransactionData txdata;
        txdata.Init(tx, std::move(spent_outputs));

        for (unsigned i = 0; i < tx.vin.size(); ++i) {
            const CTxOut& prevout = txdata.m_spent_outputs.at(i);
            const TransactionSignatureChecker checker{&tx, i, prevout.nValue, txdata};

            ScriptError serror;
            const bool ret = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, verify_flags, checker, &serror);
            assert(ret == (serror == SCRIPT_ERR_OK));

            // Verify that removing flags from a passing test or adding flags to a failing test does not change the result
            if (ret) {
                verify_flags &= ~fuzzed_flags;
            } else {
                verify_flags |= fuzzed_flags;
            }
            if (!IsValidFlagCombination(verify_flags)) return;

            ScriptError serror_fuzzed;
            const bool ret_fuzzed = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, verify_flags, checker, &serror_fuzzed);
            assert(ret_fuzzed == (serror_fuzzed == SCRIPT_ERR_OK));

            assert(ret_fuzzed == ret);
        }
    } catch (const std::ios_base::failure&) {
        return;
    }
}

static bool IsValidFlagCombination(unsigned flags)
{
    if (flags & SCRIPT_VERIFY_CLEANSTACK && ~flags & SCRIPT_VERIFY_P2SH) return false;
    return true;
}
<commit_msg>fuzz: revert PrecomputedTransactionData backport introduced in dash#4435<commit_after>// Copyright (c) 2009-2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <pubkey.h>
#include <script/interpreter.h>
#include <streams.h>
#include <util/memory.h>
#include <version.h>

#include <test/fuzz/fuzz.h>

/** Flags that are not forbidden by an assert */
static bool IsValidFlagCombination(unsigned flags);

void initialize()
{
    static const auto verify_handle = MakeUnique<ECCVerifyHandle>();
}

void test_one_input(const std::vector<uint8_t>& buffer)
{
    CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION);
    try {
        int nVersion;
        ds >> nVersion;
        ds.SetVersion(nVersion);
    } catch (const std::ios_base::failure&) {
        return;
    }

    try {
        const CTransaction tx(deserialize, ds);
        const PrecomputedTransactionData txdata(tx);

        unsigned int verify_flags;
        ds >> verify_flags;

        if (!IsValidFlagCombination(verify_flags)) return;

        unsigned int fuzzed_flags;
        ds >> fuzzed_flags;

        for (unsigned i = 0; i < tx.vin.size(); ++i) {
            CTxOut prevout;
            ds >> prevout;
            if (!MoneyRange(prevout.nValue)) {
                // prevouts should be consensus-valid
                prevout.nValue = 1;
            }

            const TransactionSignatureChecker checker{&tx, i, prevout.nValue, txdata};

            ScriptError serror;
            const bool ret = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, verify_flags, checker, &serror);
            assert(ret == (serror == SCRIPT_ERR_OK));

            // Verify that removing flags from a passing test or adding flags to a failing test does not change the result
            if (ret) {
                verify_flags &= ~fuzzed_flags;
            } else {
                verify_flags |= fuzzed_flags;
            }
            if (!IsValidFlagCombination(verify_flags)) return;

            ScriptError serror_fuzzed;
            const bool ret_fuzzed = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, verify_flags, checker, &serror_fuzzed);
            assert(ret_fuzzed == (serror_fuzzed == SCRIPT_ERR_OK));

            assert(ret_fuzzed == ret);
        }
    } catch (const std::ios_base::failure&) {
        return;
    }
}

static bool IsValidFlagCombination(unsigned flags)
{
    if (flags & SCRIPT_VERIFY_CLEANSTACK && ~flags & SCRIPT_VERIFY_P2SH) return false;
    return true;
}
<|endoftext|>


<commit_before>// test base bloom filter
#include "catch.hpp"

#include "BaseBloomFilter.hpp"

// define main

TEST_CASE( "testing BaseBloomFilter", "[bae_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	REQUIRE(bf.get_bit_count() == 1024 * 1024 * 32 * 10);
	REQUIRE(bf.get_hash_count() == 2);
	CHECK(bf.get_inserted_count() == 0);
}<commit_msg>added a test for BaseBloomFitler<commit_after>// test base bloom filter
#include "catch.hpp"

#include "BaseBloomFilter.hpp"

// define main

TEST_CASE( "testing BaseBloomFilter", "[base_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	REQUIRE(bf.get_bit_count() == 1024 * 1024 * 32 * 10);
	REQUIRE(bf.get_hash_count() == 2);
	CHECK(bf.get_inserted_count() == 0);
}

TEST_CASE( "testing add BaseBloomFilter", "[base_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	size_t expected_size = 1024 * 1024 * 32 * 10;

	bf.add(1);
	REQUIRE(bf.get_bit_count() == expected_size);
	REQUIRE(bf.get_hash_count() == 2);
	REQUIRE(bf.get_inserted_count() == 1);
}

TEST_CASE( "testing contains BaseBloomFilter", "[base_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	size_t expected_size = 1024 * 1024 * 32 * 10;

	bf.add(1);
	REQUIRE(bf.get_bit_count() == expected_size);
	REQUIRE(bf.contains(1));
	// should not expect this collision since the BF so far is empty (except
	// for 1 bit)
	REQUIRE(!bf.contains(0) );
}

TEST_CASE( "testing populate BaseBloomFilter", "[base_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	size_t expected_size = 1024 * 1024 * 32 * 10;
	unordered_set<kmer_t> kmers;
	for (int i = 0; i < 1000; i++)
		kmers.emplace(i);

	bf.populate(kmers);
	REQUIRE(bf.get_bit_count() == expected_size);
	for (const auto & item : kmers)
		REQUIRE(bf.contains(item));
	REQUIRE(bf.get_inserted_count() == kmers.size());
}<|endoftext|>