<commit_before>#include <iostream>
#include <thread>
#include <future>
#include <unordered_map>

#include <boost/utility/string_ref.hpp>
#include <boost/optional.hpp>

#include "restc-cpp/restc-cpp.h"

using namespace std;

namespace restc_cpp {

#define BYTES_AVAILABLE (read_buffer_->size() - bytes_used)

class ReplyImpl : public Reply {
public:
    using buffer_t = std::array<char, 1024 * 16>;

    ReplyImpl(Connection::ptr_t& connection,
              Context& ctx,
              RestClient& owner)
    : connection_{connection}, ctx_{ctx}, owner_{owner}
    {
    }

    boost::optional< string > GetHeader(const string& name) override {
        boost::optional< string > rval;

        auto it = headers_.find(name);
        if (it != headers_.end()) {
            rval = it->second;
        }

        return rval;
    }

    void StartReceiveFromServer() override {
        static const std::string content_len_name{"Content-Length"};

        read_buffer_ = make_unique<buffer_t>();

        // Get the header part of the message into header_
        data_bytes_received_ = ReadHeaderAndMayBeSomeMore();

        ParseHeaders();

        // Check if we have content-length, get content

        // Check for errors
        if (GetResponseCode() >= 400) {
            throw runtime_error("The request failed");
        }

        // TODO: Handle redirects

        auto cl = GetHeader(content_len_name);
        if (cl) {
            content_length_ = stoi(*cl);
        }

        // TODO: Handle chunked responses

        body_bytes_received_ = body_.size();
        CheckIfWeAreDone();
    }

    int GetResponseCode() const { return status_code_; }

    boost::asio::const_buffers_1 GetSomeData() {
        if (!body_.empty()) {

            auto rval = boost::asio::const_buffers_1{body_.data(), body_.size()};
            body_.clear();
            return rval;
        }

        if (have_received_all_data_)
            return  boost::asio::const_buffers_1{nullptr, 0};

        // Determine how much data we should read
        size_t want_bytes = 0;

        if (content_length_) {
            const auto bytes_left = *content_length_ - body_bytes_received_;
            want_bytes = std::min(bytes_left, read_buffer_->size());
        } else {
            // TODO: Implement chunked mode
            assert("Chunked response not yet implemented");
        }

        assert(want_bytes);

        const size_t received = connection_->GetSocket().AsyncReadSome(
            {read_buffer_->data(), want_bytes}, ctx_.GetYield());

        if (received == 0) {
            throw runtime_error("Got 0 bytes from server");
        }

        data_bytes_received_ += received;
        body_bytes_received_ += received;

        CheckIfWeAreDone();

        return {read_buffer_->data(), received};
    }

    string GetBodyAsString() {
        std::string buffer;

        while(MoreDataToRead()) {
            auto data = GetSomeData();
            buffer.append(boost::asio::buffer_cast<const char*>(data),
                          boost::asio::buffer_size(data));
        }

        return buffer;
    }

    bool MoreDataToRead() override {
        return !body_.empty() || !have_received_all_data_;
    }

private:
    void CheckIfWeAreDone() {
        if (!have_received_all_data_
            && content_length_
            && (*content_length_ == body_bytes_received_)) {

            have_received_all_data_ = true;
            owner_.LogDebug("Have received all data in the current request.");
        }
    }

    void ParseHeaders() {
        static const string crlf{"\r\n"};
        static const string expected_protocol{"HTTP/1.1"};

        owner_.LogDebug(header_.to_string());

        auto remaining = header_;

        {
            auto pos = remaining.find(crlf);
            if (pos == remaining.npos) {
                throw runtime_error(
                    "Invalid header - malformed response line - no CRLF");
            }
            status_line_ = boost::string_ref(remaining.data(), pos);

            pos = status_line_.find(' ');
            if (pos == status_line_.npos) {
                throw runtime_error(
                    "Invalid header - malformed response line");
            }

            const auto protocol = boost::string_ref(status_line_.data(), pos);

            if (strncasecmp(protocol.data(),
                expected_protocol.c_str(),
                            expected_protocol.size())) {

                throw runtime_error("Invalid header - Unexpected protocol");
            }

            if (status_line_.size() < (pos + 3)) {
                throw runtime_error(
                    "Invalid header - malformed response line - status code");
            }

            auto code = boost::string_ref(status_line_.data() + pos + 1, 3);

            if (status_line_.size() > (pos + 5)) {
                status_code_ = stoi(code.to_string());
                status_message_ = boost::string_ref(
                    status_line_.data() + pos + 5,
                    status_line_.size() - pos - 5);
            }
        }

        while(true) {
            // Get Next Line
            auto start_of_line = remaining.find(crlf);
            if (start_of_line == remaining.npos) {
                throw runtime_error("Invalid header - missing CRLF");
            }
            start_of_line += 2;
            remaining = boost::string_ref(remaining.data() + start_of_line,
                                          remaining.size() - start_of_line);
            auto end_of_line = remaining.find(crlf);
            if (end_of_line == remaining.npos) {
                // We are done.
                return;
            }

            auto line = boost::string_ref(remaining.data(), end_of_line);

            // Get Name
            boost::string_ref name;
            std::string value;
            enum class State { PARSE_NAME, PARSE_DELIM, PARSE_VALUE };
            State state = State::PARSE_NAME;
            for(auto ch = line.cbegin() ; ch != line.cend(); ++ch) {
                if (!value.empty())
                    break; // Done

                if (*ch == ' ' || *ch == ':') {
                    switch(state) {
                        case State::PARSE_NAME:
                            name = boost::string_ref(line.data(), ch - line.begin());
                            state = State::PARSE_DELIM;
                            break;
                        case State::PARSE_DELIM:
                            break; // Skip
                        case State::PARSE_VALUE:
                            break; // Consume
                    }
                } else {
                    switch(state) {
                        case State::PARSE_NAME:
                            break; // Consume
                        case State::PARSE_DELIM:
                            state = State::PARSE_VALUE;
                            value.assign(ch, line.cend() - ch);
                            break;
                        case State::PARSE_VALUE:
                            break;
                    } // switch
                } // else *ch == ' ' || *ch == ':'
            } // for(auto ch : line)

            // Is the line wrapped? If so, append the value.
            while ((remaining.size() > 2) && (remaining.at(2) == ' ')) {
                assert(remaining.at(0) == '\r');
                assert(remaining.at(1) == '\n');

                // Wrapped line.
                remaining = boost::string_ref(remaining.data() + 2,
                                                remaining.size() - 2);
                end_of_line = remaining.find(crlf);
                if (end_of_line == remaining.npos) {
                    throw runtime_error("Invalid header - missing CRLF");
                }

                value.append(remaining.data() + 1, end_of_line - 1);

                remaining = boost::string_ref(remaining.data() + end_of_line,
                                                remaining.size() - end_of_line);
            } // while remaining.size() > 2

            if (!name.empty()) {
                headers_[name.to_string()] = value;
            }
        }
    }

    size_t ReadHeaderAndMayBeSomeMore() {
        size_t bytes_used = 0;
        static string end_of_header{"\r\n\r\n"};

        while(true) {
            if (BYTES_AVAILABLE < 1) {
                throw runtime_error("Header is too long - out of buffer space");
            }

            const size_t received = connection_->GetSocket().AsyncReadSome(
                {read_buffer_->data() + bytes_used, BYTES_AVAILABLE}, ctx_.GetYield());

            if (received == 0) {
                throw runtime_error("Received 0 bytes on read.");
            }

            bytes_used += received;

            buffer_ = boost::string_ref(read_buffer_->data(), bytes_used);
            auto pos = buffer_.find(end_of_header);
            if (pos != buffer_.npos) {
                header_ = buffer_ = boost::string_ref(read_buffer_->data(), pos);
                body_ = boost::string_ref(header_.end() + 4,
                                          bytes_used - 4 - header_.size());
                return bytes_used;
            }
        }
    }

    Connection::ptr_t connection_;
    Context& ctx_;
    RestClient& owner_;
    std::vector<char> memory_buffer_;
    boost::string_ref buffer_;
    boost::string_ref status_line_;;
    boost::string_ref header_;
    boost::string_ref body_;
    int status_code_ = 0;
    boost::string_ref status_message_;
    map<string, string, ciLessLibC> headers_;
    bool have_received_all_data_ = false;
    std::unique_ptr<buffer_t> read_buffer_;
    boost::optional<size_t> content_length_;
    size_t data_bytes_received_ = 0;
    size_t body_bytes_received_ = 0;
};

std::unique_ptr<Reply>
Reply::Create(Connection::ptr_t& connection,
       Context& ctx,
       RestClient& owner) {

    return make_unique<ReplyImpl>(connection, ctx, owner);
}

} // restc_cpp

<commit_msg>Fixed some clang warnings<commit_after>#include <iostream>
#include <thread>
#include <future>
#include <unordered_map>

#include <boost/utility/string_ref.hpp>
#include <boost/optional.hpp>

#include "restc-cpp/restc-cpp.h"

using namespace std;

namespace restc_cpp {

#define BYTES_AVAILABLE (read_buffer_->size() - bytes_used)

class ReplyImpl : public Reply {
public:
    using buffer_t = std::array<char, 1024 * 16>;

    ReplyImpl(Connection::ptr_t& connection,
              Context& ctx,
              RestClient& owner)
    : connection_{connection}, ctx_{ctx}, owner_{owner}
    {
    }

    boost::optional< string > GetHeader(const string& name) override {
        boost::optional< string > rval;

        auto it = headers_.find(name);
        if (it != headers_.end()) {
            rval = it->second;
        }

        return rval;
    }

    void StartReceiveFromServer() override {
        static const std::string content_len_name{"Content-Length"};

        read_buffer_ = make_unique<buffer_t>();

        // Get the header part of the message into header_
        data_bytes_received_ = ReadHeaderAndMayBeSomeMore();

        ParseHeaders();

        // Check if we have content-length, get content

        // Check for errors
        if (GetResponseCode() >= 400) {
            throw runtime_error("The request failed");
        }

        // TODO: Handle redirects

        auto cl = GetHeader(content_len_name);
        if (cl) {
            content_length_ = stoi(*cl);
        }

        // TODO: Handle chunked responses

        body_bytes_received_ = body_.size();
        CheckIfWeAreDone();
    }

    int GetResponseCode() const override { return status_code_; }

    boost::asio::const_buffers_1 GetSomeData() override {
        if (!body_.empty()) {

            auto rval = boost::asio::const_buffers_1{body_.data(), body_.size()};
            body_.clear();
            return rval;
        }

        if (have_received_all_data_)
            return  boost::asio::const_buffers_1{nullptr, 0};

        // Determine how much data we should read
        size_t want_bytes = 0;

        if (content_length_) {
            const auto bytes_left = *content_length_ - body_bytes_received_;
            want_bytes = std::min(bytes_left, read_buffer_->size());
        } else {
            // TODO: Implement chunked mode
            assert("Chunked response not yet implemented");
        }

        assert(want_bytes);

        const size_t received = connection_->GetSocket().AsyncReadSome(
            {read_buffer_->data(), want_bytes}, ctx_.GetYield());

        if (received == 0) {
            throw runtime_error("Got 0 bytes from server");
        }

        data_bytes_received_ += received;
        body_bytes_received_ += received;

        CheckIfWeAreDone();

        return {read_buffer_->data(), received};
    }

    string GetBodyAsString() override {
        std::string buffer;

        while(MoreDataToRead()) {
            auto data = GetSomeData();
            buffer.append(boost::asio::buffer_cast<const char*>(data),
                          boost::asio::buffer_size(data));
        }

        return buffer;
    }

    bool MoreDataToRead() override {
        return !body_.empty() || !have_received_all_data_;
    }

private:
    void CheckIfWeAreDone() {
        if (!have_received_all_data_
            && content_length_
            && (*content_length_ == body_bytes_received_)) {

            have_received_all_data_ = true;
            owner_.LogDebug("Have received all data in the current request.");
        }
    }

    void ParseHeaders() {
        static const string crlf{"\r\n"};
        static const string expected_protocol{"HTTP/1.1"};

        owner_.LogDebug(header_.to_string());

        auto remaining = header_;

        {
            auto pos = remaining.find(crlf);
            if (pos == remaining.npos) {
                throw runtime_error(
                    "Invalid header - malformed response line - no CRLF");
            }
            status_line_ = boost::string_ref(remaining.data(), pos);

            pos = status_line_.find(' ');
            if (pos == status_line_.npos) {
                throw runtime_error(
                    "Invalid header - malformed response line");
            }

            const auto protocol = boost::string_ref(status_line_.data(), pos);

            if (strncasecmp(protocol.data(),
                expected_protocol.c_str(),
                            expected_protocol.size())) {

                throw runtime_error("Invalid header - Unexpected protocol");
            }

            if (status_line_.size() < (pos + 3)) {
                throw runtime_error(
                    "Invalid header - malformed response line - status code");
            }

            auto code = boost::string_ref(status_line_.data() + pos + 1, 3);

            if (status_line_.size() > (pos + 5)) {
                status_code_ = stoi(code.to_string());
                status_message_ = boost::string_ref(
                    status_line_.data() + pos + 5,
                    status_line_.size() - pos - 5);
            }
        }

        while(true) {
            // Get Next Line
            auto start_of_line = remaining.find(crlf);
            if (start_of_line == remaining.npos) {
                throw runtime_error("Invalid header - missing CRLF");
            }
            start_of_line += 2;
            remaining = boost::string_ref(remaining.data() + start_of_line,
                                          remaining.size() - start_of_line);
            auto end_of_line = remaining.find(crlf);
            if (end_of_line == remaining.npos) {
                // We are done.
                return;
            }

            auto line = boost::string_ref(remaining.data(), end_of_line);

            // Get Name
            boost::string_ref name;
            std::string value;
            enum class State { PARSE_NAME, PARSE_DELIM, PARSE_VALUE };
            State state = State::PARSE_NAME;
            for(auto ch = line.cbegin() ; ch != line.cend(); ++ch) {
                if (!value.empty())
                    break; // Done

                if (*ch == ' ' || *ch == ':') {
                    switch(state) {
                        case State::PARSE_NAME:
                            name = boost::string_ref(line.data(), ch - line.begin());
                            state = State::PARSE_DELIM;
                            break;
                        case State::PARSE_DELIM:
                            break; // Skip
                        case State::PARSE_VALUE:
                            break; // Consume
                    }
                } else {
                    switch(state) {
                        case State::PARSE_NAME:
                            break; // Consume
                        case State::PARSE_DELIM:
                            state = State::PARSE_VALUE;
                            value.assign(ch, line.cend() - ch);
                            break;
                        case State::PARSE_VALUE:
                            break;
                    } // switch
                } // else *ch == ' ' || *ch == ':'
            } // for(auto ch : line)

            // Is the line wrapped? If so, append the value.
            while ((remaining.size() > 2) && (remaining.at(2) == ' ')) {
                assert(remaining.at(0) == '\r');
                assert(remaining.at(1) == '\n');

                // Wrapped line.
                remaining = boost::string_ref(remaining.data() + 2,
                                                remaining.size() - 2);
                end_of_line = remaining.find(crlf);
                if (end_of_line == remaining.npos) {
                    throw runtime_error("Invalid header - missing CRLF");
                }

                value.append(remaining.data() + 1, end_of_line - 1);

                remaining = boost::string_ref(remaining.data() + end_of_line,
                                                remaining.size() - end_of_line);
            } // while remaining.size() > 2

            if (!name.empty()) {
                headers_[name.to_string()] = value;
            }
        }
    }

    size_t ReadHeaderAndMayBeSomeMore() {
        size_t bytes_used = 0;
        static string end_of_header{"\r\n\r\n"};

        while(true) {
            if (BYTES_AVAILABLE < 1) {
                throw runtime_error("Header is too long - out of buffer space");
            }

            const size_t received = connection_->GetSocket().AsyncReadSome(
                {read_buffer_->data() + bytes_used, BYTES_AVAILABLE}, ctx_.GetYield());

            if (received == 0) {
                throw runtime_error("Received 0 bytes on read.");
            }

            bytes_used += received;

            buffer_ = boost::string_ref(read_buffer_->data(), bytes_used);
            auto pos = buffer_.find(end_of_header);
            if (pos != buffer_.npos) {
                header_ = buffer_ = boost::string_ref(read_buffer_->data(), pos);
                body_ = boost::string_ref(header_.end() + 4,
                                          bytes_used - 4 - header_.size());
                return bytes_used;
            }
        }
    }

    Connection::ptr_t connection_;
    Context& ctx_;
    RestClient& owner_;
    std::vector<char> memory_buffer_;
    boost::string_ref buffer_;
    boost::string_ref status_line_;;
    boost::string_ref header_;
    boost::string_ref body_;
    int status_code_ = 0;
    boost::string_ref status_message_;
    map<string, string, ciLessLibC> headers_;
    bool have_received_all_data_ = false;
    std::unique_ptr<buffer_t> read_buffer_;
    boost::optional<size_t> content_length_;
    size_t data_bytes_received_ = 0;
    size_t body_bytes_received_ = 0;
};

std::unique_ptr<Reply>
Reply::Create(Connection::ptr_t& connection,
       Context& ctx,
       RestClient& owner) {

    return make_unique<ReplyImpl>(connection, ctx, owner);
}

} // restc_cpp

<|endoftext|>


<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <signal.h>
#include <unistd.h>
#include <evhttp.h>
#include <event2/event.h>
#include <event2/http.h>
#include <event2/bufferevent.h>
#include "util/config.h"
#include "util/log.h"

#define MAX_BIND_PORTS 1

void chunk_cb(struct evhttp_request *req, void *arg){
	static int num = 0;
	if(++num % 1000 == 1){
		char buf[1024];
		int s = evbuffer_remove(req->input_buffer, &buf, sizeof(buf) - 1);
		buf[s] = '\0';
		printf("< %s", buf);
	}
}

void http_request_done(struct evhttp_request *req, void *arg){
	static int num = 0;
	if(++num % 1000 == 1){
		log_debug("request done %d", num);
	}
}

int main(int argc, char **argv){
	if(argc < 3){
		printf("Usage: %s ip port\n", argv[0]);
		exit(0);
	}
	const char *host = argv[1];
	int port = atoi(argv[2]);

	srand(time(NULL));
	signal(SIGPIPE, SIG_IGN);

	struct event_base *base = event_base_new();
	if(!base){
		fprintf(stderr, "event_base_new() error!\n");
		exit(0);
	}

	struct evhttp_connection *conn;
	struct evhttp_request *req;

	int num = 0;
	while(1){
		if(num % 100000 == 0){
			printf("press Enter to continue: ");
			getchar();
		}
		if(num % 1000 == 1){
			log_debug("sub: %d", num);
		}
		conn = evhttp_connection_base_new(base, NULL, host, port + num % MAX_BIND_PORTS);
		req = evhttp_request_new(http_request_done, NULL);
		evhttp_request_set_chunked_cb(req, chunk_cb);

		char buf[128];
		snprintf(buf, sizeof(buf), "/sub?cname=%d", rand());
		evhttp_make_request(conn, req, EVHTTP_REQ_GET, buf);
		evhttp_connection_set_timeout(req->evcon, 864000);
		event_base_loop(base, EVLOOP_NONBLOCK);

		num ++;
		usleep(1 * 1000);
	}

	event_base_dispatch(base);

	log_debug("quit");
	return 0;
}

<commit_msg>update tools/benchmark.cpp<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <signal.h>
#include <unistd.h>
#include <evhttp.h>
#include <event2/event.h>
#include <event2/http.h>
#include <event2/bufferevent.h>
#include "util/log.h"

#define MAX_BIND_PORTS 1

void chunk_cb(struct evhttp_request *req, void *arg){
	static int num = 0;
	if(++num % 1000 == 1){
		char buf[1024];
		int s = evbuffer_remove(req->input_buffer, &buf, sizeof(buf) - 1);
		buf[s] = '\0';
		printf("< %s", buf);
	}
}

void http_request_done(struct evhttp_request *req, void *arg){
	static int num = 0;
	if(++num % 1000 == 1){
		log_debug("request done %d", num);
	}
}

int main(int argc, char **argv){
	if(argc < 3){
		printf("Usage: %s ip port\n", argv[0]);
		exit(0);
	}
	const char *host = argv[1];
	int port = atoi(argv[2]);

	srand(time(NULL));
	signal(SIGPIPE, SIG_IGN);

	struct event_base *base = event_base_new();
	if(!base){
		fprintf(stderr, "event_base_new() error!\n");
		exit(0);
	}

	struct evhttp_connection *conn;
	struct evhttp_request *req;

	int num = 0;
	while(1){
		if(num % 100000 == 0){
			printf("press Enter to continue: ");
			getchar();
		}
		if(num % 1000 == 1){
			log_debug("sub: %d", num);
		}
		conn = evhttp_connection_base_new(base, NULL, host, port + num % MAX_BIND_PORTS);
		req = evhttp_request_new(http_request_done, NULL);
		evhttp_request_set_chunked_cb(req, chunk_cb);

		char buf[128];
		snprintf(buf, sizeof(buf), "/sub?cname=%d", rand());
		evhttp_make_request(conn, req, EVHTTP_REQ_GET, buf);
		evhttp_connection_set_timeout(req->evcon, 864000);
		event_base_loop(base, EVLOOP_NONBLOCK);

		num ++;
		usleep(1 * 1000);
	}

	event_base_dispatch(base);

	log_debug("quit");
	return 0;
}

<|endoftext|>


<commit_before><commit_msg>Added new executors, also added helper macros to acquire bytecode parameters<commit_after><|endoftext|>


<commit_before>#include "ShapefileLoader.h"

#include "PlanarGraph.h"
#include "ArcData2D.h"

#include <Table.h>
#include <DBase3File.h>

#include <cassert>
#include <shapefil.h>
#include <iostream>
#include <cmath>
#include <sstream>

using namespace std;

ShapefileLoader::ShapefileLoader() : FileTypeHandler("ESRI Shapefile", false) {
  addExtension("shp");
}

std::shared_ptr<Graph>
ShapefileLoader::openGraph(const char * filename, const std::shared_ptr<NodeArray> & initial_nodes) {
  SHPHandle shp = SHPOpen(filename, "rb");
  if (!shp) {
    cerr << "failed to open shapefile\n";
    return 0;
  }

  double minb[4], maxb[4];
  int shape_count, global_shape_type;
  SHPGetInfo(shp, &shape_count, &global_shape_type, minb, maxb);

  std::shared_ptr<Graph> graph;

  map<string, int> nodes;
  map<string, pair<glm::dvec2, set<string> > > node_edges;
  map<string, int> waiting_faces;

  cerr << "loading shapefile (" << shape_count << ")\n";

  bool has_polygons = false;

  for (int i = 0; i < shape_count; i++) {
    SHPObject * o = SHPReadObject(shp, i);
    assert(o);

    switch (o->nSHPType) {
    case SHPT_POINT:
    case SHPT_POINTZ:
    case SHPT_POINTM:
      if (!graph.get()) {
	graph = std::make_shared<PlanarGraph>();
	graph->setNodeArray(initial_nodes);
	graph->getNodeArray().setHasSpatialData(true);
	graph->setNodeVisibility(false);
	graph->setEdgeVisibility(false);
	graph->setFaceVisibility(false);
      }
      assert(o->nVertices == 1);
      {
	int face_id = graph->addFace();
	double x = o->padfX[0], y = o->padfY[0], z = o->padfZ[0];
	int node_id = createNode2D(*graph, nodes, x, y);
	graph->addEdge(node_id, node_id, face_id, 0.0f);
	auto & fd = graph->getFaceAttributes(face_id);
	fd.centroid = glm::vec2(x, y);
	fd.mbr = glm::vec3(x, y, 0.0f);
      }
      break;
    case SHPT_MULTIPOINT:
    case SHPT_MULTIPOINTZ:
    case SHPT_MULTIPOINTM:
      assert(0);
#if 0
      if (!graph.get()) {
	graph = std::make_shared<PlanarGraph>();
	graph->setNodeArray(initial_nodes);
	graph->getNodeArray().setHasSpatialData(true);
      }
      for (int j = 0; j < o->nVertices; j++) {
	double x = o->padfX[j], y = o->padfY[j], z = o->padfZ[j];
	int node_id = createNode2D(*graph, nodes, x, y);
	graph->addEdge(node_id, node_id, -1, 0.0f);
      }
#endif
      break;
    case SHPT_ARC: // = polyline
    case SHPT_ARCZ:
    case SHPT_ARCM:
      if (!graph.get()) {
	graph = std::make_shared<PlanarGraph>();
	graph->setNodeArray(initial_nodes);
	graph->getNodeArray().setHasSpatialData(true);
	graph->getNodeArray().setHasArcData(true);
	graph->setNodeVisibility(false);
	graph->setEdgeVisibility(true);
	graph->setFaceVisibility(false);
      }
      {
	int hyperedge_id = graph->addFace();
	for (int j = 0; j < o->nParts; j++) {
	  int start = o->nParts > 1 ? o->panPartStart[j] : 0;
	  int end = o->nParts > 1 && j + 1 < o->nParts ? o->panPartStart[j + 1] : o->nVertices;
	  ArcData2D arc;
	  for (int k0 = start; k0 < end; k0++) {
	    double x = o->padfX[k0], y = o->padfY[k0], z = o->padfZ[k0];
	    arc.data.push_back(glm::dvec2(x, y));
	  }

	  int arc_id = graph->getNodeArray().addArcGeometry(arc);
	  pair<int, int> n = createNodesForArc(arc, *graph, nodes);
	  assert(arc_id);
	  assert(arc_id >= 1 && arc_id <= graph->getNodeArray().getArcGeometry().size());
	  int edge_id = graph->addEdge(n.first, n.second, hyperedge_id, 1.0f, arc_id);
	  assert(graph->getEdgeAttributes(edge_id).arc == arc_id);
	}
      }
      break;
    case SHPT_POLYGON:
    case SHPT_POLYGONZ:
    case SHPT_POLYGONM:
      if (!graph.get()) {
	cerr << "creating planar graph for shapefile\n";
	graph = std::make_shared<PlanarGraph>();
	graph->setNodeArray(initial_nodes);
	graph->getNodeArray().setHasSpatialData(true);
	graph->getNodeArray().setHasArcData(true);
	graph->setNodeVisibility(false);
	graph->setEdgeVisibility(false);
	graph->setFaceVisibility(true);
      }
      has_polygons = true;
      for (int j = 0; j < o->nParts; j++) {
	int start = o->nParts > 1 ? o->panPartStart[j] : 0;
	int end = o->nParts > 1 && j + 1 < o->nParts ? o->panPartStart[j + 1] : o->nVertices;
	int n = end - start;
	for (int k0 = 0; k0 + 1 < n; k0++) {
	  int k1 = start + (k0 % n), k2 = start + ((k0 + 1) % n);
	  glm::dvec2 v1(o->padfX[k1], o->padfY[k1]); //, o->padfZ[k]);
	  glm::dvec2 v2(o->padfX[k2], o->padfY[k2]); // o->padfZ[k + 1]);
	  ostringstream key1, key2;
	  key1 << v1.x << "/" << v1.y;
	  key2 << v2.x << "/" << v2.y;
	  pair<glm::dvec2, set<string> > & ne1 = node_edges[key1.str()];
	  pair<glm::dvec2, set<string> > & ne2 = node_edges[key2.str()];
	  ne1.first = v1;
	  ne2.first = v2;
	  ne1.second.insert(key2.str());
	  ne2.second.insert(key1.str());
	}
      }
      break;
    case SHPT_MULTIPATCH:
      cerr << "multipatches not supported\n";
      assert(0);
      break;
    default:
      cerr << "unhandled type\n";
      assert(0);
    }
    SHPDestroyObject(o);
  }

  if (has_polygons) {
    unsigned int connected_face_arcs = 0;
    cerr << "creating nodes (" << node_edges.size() << ")\n";
    for (map<string, pair<glm::dvec2, set<string> > >::iterator it = node_edges.begin(); it != node_edges.end(); it++) {
      if (it->second.second.size() >= 3) {
	createNode2D(*graph, nodes, it->second.first.x, it->second.first.y);
      }
    }
    cerr << "creating faces\n";
    for (int i = 0; i < shape_count; i++) {
      SHPObject * o = SHPReadObject(shp, i);
      assert(o);
      assert(o->nSHPType == SHPT_POLYGON || o->nSHPType == SHPT_POLYGONZ || o->nSHPType == SHPT_POLYGONM);
      // int region_id = graph->addRegion();
      int face_id = graph->addFace();
      double area = 0, centroid_x = 0, centroid_y = 0;
      assert(o->nParts >= 1); // could be zero to
      for (int j = 0; j < o->nParts; j++) {
	// int face_id = graph->addFace(region_id);
	// cerr << "handling polygon " << i << ", part " << j << endl;
	int start = o->panPartStart[j];
	if (j == 0 && start > 0) {
	  cerr << "invalid start " << start << endl;
	  start = 0;
	}
	int end = j + 1 < o->nParts ? o->panPartStart[j + 1] : o->nVertices;
	list<glm::dvec2> input;
	for (int k = start; k < end; k++) {
	  double x = o->padfX[k], y = o->padfY[k], z = o->padfZ[k];
	  input.push_back(glm::dvec2(x, y));
	  if (k + 1 < end) {
	    double a = o->padfX[k] * o->padfY[k + 1] - o->padfY[k] * o->padfX[k + 1];
	    area -= a;
	    centroid_x -= (o->padfX[k] + o->padfX[k + 1]) * a;
	    centroid_y -= (o->padfY[k] + o->padfY[k + 1]) * a;
	  }
	}
	if (input.size() > 1 &&
	    input.front().x == input.back().x &&
	    input.front().y == input.back().y) {
	  input.pop_back();
	}
	// cerr << "rolling vertices (" << input.size() << ")\n";
	bool node_found = false;
	for (int step = 0; step < input.size(); step++) {
	  ostringstream key1;
	  key1 << input.front().x << "/" << input.front().y;
	  if (nodes.find(key1.str()) != nodes.end()) {
	    node_found = true;
	    break;
	  } else {
	    input.push_back(input.front());
	    input.pop_front();
	  }
	}
	input.push_back(input.front());
	assert(input.size() >= 4);
	if (!node_found) { // island
	  createNode2D(*graph, nodes, input.front().x, input.front().y);
	}
	// ArcData2D arc;
	// for (list<glm::dvec3>::iterator it = input.begin(); it != input.end(); it++) {
	//   arc.data.push_back(*it);
	// }
	// pair<int, int> n = createNodesForArc(arc, graph, nodes);
	// assert(n.first == n.second);
	// int edge_id = graph->addEdge(n.first, n.second, face_id);
	// geometries.setArc(edge_id, arc);
	vector<ArcData2D> arcs;
	vector<int> face_nodes;
	for (auto & v : input) {
	  ostringstream key1;
	  key1 << v.x << "/" << v.y;
	  map<string, int>::iterator it2 = nodes.find(key1.str());
	  if (it2 != nodes.end()) {
	    if (!arcs.empty()) {
	      arcs.back().data.push_back(v);
	    }
	    face_nodes.push_back(it2->second);
	    arcs.push_back(ArcData2D());
	  }
	  assert(!arcs.empty());
	  arcs.back().data.push_back(v);
	}
	assert(face_nodes.front() == face_nodes.back());
	for (int l = 0; l + 1 < face_nodes.size(); l++) {
	  int node1 = face_nodes[l], node2 = face_nodes[l + 1];
	  if (node1 == node2) {
	    // cerr << "same nodes, size = " << face_nodes.size() << endl;
	  }

	  int arc_id = 0, pair_edge = -1;
	  ostringstream key1;
	  key1 << node2 << "/" << node1;
	  map<string, int>::iterator it = waiting_faces.find(key1.str());
	  if (it != waiting_faces.end()) {
	    pair_edge = it->second;
	    auto & ed = graph->getEdgeAttributes(pair_edge);
	    assert(ed.arc >= 1 && ed.arc <= graph->getNodeArray().getArcGeometry().size());
	    arc_id = -ed.arc;
	    waiting_faces.erase(it);
	    connected_face_arcs++;
	  } else {
	    arc_id = graph->getNodeArray().addArcGeometry(arcs[l]);
	    assert(arc_id >= 1 && arc_id <= graph->getNodeArray().getArcGeometry().size());
	  }

	  assert(arc_id);

	  int edge_id = graph->addEdge(node1, node2, face_id, 1.0f, arc_id);
	  if (pair_edge != -1) {
	    graph->connectEdgePair(edge_id, pair_edge);
	  } else {
	    graph->updateMBR(edge_id);
	    ostringstream key2;
	    key2 << node1 << "/" << node2;
	    waiting_faces[key2.str()] = edge_id;
	  }
	  assert(graph->getEdgeAttributes(edge_id).arc == arc_id);
	}
      }

      area *= 0.5;
      centroid_x /= (6.0*area);
      centroid_y /= (6.0*area);
      graph->getFaceAttributes(face_id).centroid = glm::vec2(centroid_x, centroid_y);

      SHPDestroyObject(o);
    }
    cerr << "ARCS: connected face arcs = " << connected_face_arcs << ", unconnected = " << waiting_faces.size() << endl;
  }

  SHPClose(shp);

  assert(graph.get());

  auto dbf = std::make_shared<table::DBase3File>(filename);

  if (dbf->getRecordCount() != shape_count) {
    cerr << "bad number of records\n";
  }

  table::Table & table = graph->getFaceData();
  for (auto & col : dbf->getColumns()) {
    table.addColumn(col);
  }

  graph->updateAppearance();

  return graph;
}
<commit_msg>rename method call<commit_after>#include "ShapefileLoader.h"

#include "PlanarGraph.h"
#include "ArcData2D.h"

#include <Table.h>
#include <DBase3File.h>

#include <cassert>
#include <shapefil.h>
#include <iostream>
#include <cmath>
#include <sstream>

using namespace std;

ShapefileLoader::ShapefileLoader() : FileTypeHandler("ESRI Shapefile", false) {
  addExtension("shp");
}

std::shared_ptr<Graph>
ShapefileLoader::openGraph(const char * filename, const std::shared_ptr<NodeArray> & initial_nodes) {
  SHPHandle shp = SHPOpen(filename, "rb");
  if (!shp) {
    cerr << "failed to open shapefile\n";
    return 0;
  }

  double minb[4], maxb[4];
  int shape_count, global_shape_type;
  SHPGetInfo(shp, &shape_count, &global_shape_type, minb, maxb);

  std::shared_ptr<Graph> graph;

  map<string, int> nodes;
  map<string, pair<glm::dvec2, set<string> > > node_edges;
  map<string, int> waiting_faces;

  cerr << "loading shapefile (" << shape_count << ")\n";

  bool has_polygons = false;

  for (int i = 0; i < shape_count; i++) {
    SHPObject * o = SHPReadObject(shp, i);
    assert(o);

    switch (o->nSHPType) {
    case SHPT_POINT:
    case SHPT_POINTZ:
    case SHPT_POINTM:
      if (!graph.get()) {
	graph = std::make_shared<PlanarGraph>();
	graph->setNodeArray(initial_nodes);
	graph->getNodeArray().setHasSpatialData(true);
	graph->setNodeVisibility(false);
	graph->setEdgeVisibility(false);
	graph->setFaceVisibility(false);
      }
      assert(o->nVertices == 1);
      {
	int face_id = graph->addFace();
	double x = o->padfX[0], y = o->padfY[0], z = o->padfZ[0];
	int node_id = createNode2D(*graph, nodes, x, y);
	graph->addEdge(node_id, node_id, face_id, 0.0f);
	auto & fd = graph->getFaceAttributes(face_id);
	fd.centroid = glm::vec2(x, y);
	fd.mbr = glm::vec3(x, y, 0.0f);
      }
      break;
    case SHPT_MULTIPOINT:
    case SHPT_MULTIPOINTZ:
    case SHPT_MULTIPOINTM:
      assert(0);
#if 0
      if (!graph.get()) {
	graph = std::make_shared<PlanarGraph>();
	graph->setNodeArray(initial_nodes);
	graph->getNodeArray().setHasSpatialData(true);
      }
      for (int j = 0; j < o->nVertices; j++) {
	double x = o->padfX[j], y = o->padfY[j], z = o->padfZ[j];
	int node_id = createNode2D(*graph, nodes, x, y);
	graph->addEdge(node_id, node_id, -1, 0.0f);
      }
#endif
      break;
    case SHPT_ARC: // = polyline
    case SHPT_ARCZ:
    case SHPT_ARCM:
      if (!graph.get()) {
	graph = std::make_shared<PlanarGraph>();
	graph->setNodeArray(initial_nodes);
	graph->getNodeArray().setHasSpatialData(true);
	graph->getNodeArray().setHasArcData(true);
	graph->setNodeVisibility(false);
	graph->setEdgeVisibility(true);
	graph->setFaceVisibility(false);
      }
      {
	int hyperedge_id = graph->addFace();
	for (int j = 0; j < o->nParts; j++) {
	  int start = o->nParts > 1 ? o->panPartStart[j] : 0;
	  int end = o->nParts > 1 && j + 1 < o->nParts ? o->panPartStart[j + 1] : o->nVertices;
	  ArcData2D arc;
	  for (int k0 = start; k0 < end; k0++) {
	    double x = o->padfX[k0], y = o->padfY[k0], z = o->padfZ[k0];
	    arc.data.push_back(glm::dvec2(x, y));
	  }

	  int arc_id = graph->getNodeArray().addArcGeometry(arc);
	  pair<int, int> n = createNodesForArc(arc, *graph, nodes);
	  assert(arc_id);
	  assert(arc_id >= 1 && arc_id <= graph->getNodeArray().getArcGeometry().size());
	  int edge_id = graph->addEdge(n.first, n.second, hyperedge_id, 1.0f, arc_id);
	  assert(graph->getEdgeAttributes(edge_id).arc == arc_id);
	}
      }
      break;
    case SHPT_POLYGON:
    case SHPT_POLYGONZ:
    case SHPT_POLYGONM:
      if (!graph.get()) {
	cerr << "creating planar graph for shapefile\n";
	graph = std::make_shared<PlanarGraph>();
	graph->setNodeArray(initial_nodes);
	graph->getNodeArray().setHasSpatialData(true);
	graph->getNodeArray().setHasArcData(true);
	graph->setNodeVisibility(false);
	graph->setEdgeVisibility(false);
	graph->setFaceVisibility(true);
      }
      has_polygons = true;
      for (int j = 0; j < o->nParts; j++) {
	int start = o->nParts > 1 ? o->panPartStart[j] : 0;
	int end = o->nParts > 1 && j + 1 < o->nParts ? o->panPartStart[j + 1] : o->nVertices;
	int n = end - start;
	for (int k0 = 0; k0 + 1 < n; k0++) {
	  int k1 = start + (k0 % n), k2 = start + ((k0 + 1) % n);
	  glm::dvec2 v1(o->padfX[k1], o->padfY[k1]); //, o->padfZ[k]);
	  glm::dvec2 v2(o->padfX[k2], o->padfY[k2]); // o->padfZ[k + 1]);
	  ostringstream key1, key2;
	  key1 << v1.x << "/" << v1.y;
	  key2 << v2.x << "/" << v2.y;
	  pair<glm::dvec2, set<string> > & ne1 = node_edges[key1.str()];
	  pair<glm::dvec2, set<string> > & ne2 = node_edges[key2.str()];
	  ne1.first = v1;
	  ne2.first = v2;
	  ne1.second.insert(key2.str());
	  ne2.second.insert(key1.str());
	}
      }
      break;
    case SHPT_MULTIPATCH:
      cerr << "multipatches not supported\n";
      assert(0);
      break;
    default:
      cerr << "unhandled type\n";
      assert(0);
    }
    SHPDestroyObject(o);
  }

  if (has_polygons) {
    unsigned int connected_face_arcs = 0;
    cerr << "creating nodes (" << node_edges.size() << ")\n";
    for (map<string, pair<glm::dvec2, set<string> > >::iterator it = node_edges.begin(); it != node_edges.end(); it++) {
      if (it->second.second.size() >= 3) {
	createNode2D(*graph, nodes, it->second.first.x, it->second.first.y);
      }
    }
    cerr << "creating faces\n";
    for (int i = 0; i < shape_count; i++) {
      SHPObject * o = SHPReadObject(shp, i);
      assert(o);
      assert(o->nSHPType == SHPT_POLYGON || o->nSHPType == SHPT_POLYGONZ || o->nSHPType == SHPT_POLYGONM);
      // int region_id = graph->addRegion();
      int face_id = graph->addFace();
      double area = 0, centroid_x = 0, centroid_y = 0;
      assert(o->nParts >= 1); // could be zero to
      for (int j = 0; j < o->nParts; j++) {
	// int face_id = graph->addFace(region_id);
	// cerr << "handling polygon " << i << ", part " << j << endl;
	int start = o->panPartStart[j];
	if (j == 0 && start > 0) {
	  cerr << "invalid start " << start << endl;
	  start = 0;
	}
	int end = j + 1 < o->nParts ? o->panPartStart[j + 1] : o->nVertices;
	list<glm::dvec2> input;
	for (int k = start; k < end; k++) {
	  double x = o->padfX[k], y = o->padfY[k], z = o->padfZ[k];
	  input.push_back(glm::dvec2(x, y));
	  if (k + 1 < end) {
	    double a = o->padfX[k] * o->padfY[k + 1] - o->padfY[k] * o->padfX[k + 1];
	    area -= a;
	    centroid_x -= (o->padfX[k] + o->padfX[k + 1]) * a;
	    centroid_y -= (o->padfY[k] + o->padfY[k + 1]) * a;
	  }
	}
	if (input.size() > 1 &&
	    input.front().x == input.back().x &&
	    input.front().y == input.back().y) {
	  input.pop_back();
	}
	// cerr << "rolling vertices (" << input.size() << ")\n";
	bool node_found = false;
	for (int step = 0; step < input.size(); step++) {
	  ostringstream key1;
	  key1 << input.front().x << "/" << input.front().y;
	  if (nodes.find(key1.str()) != nodes.end()) {
	    node_found = true;
	    break;
	  } else {
	    input.push_back(input.front());
	    input.pop_front();
	  }
	}
	input.push_back(input.front());
	assert(input.size() >= 4);
	if (!node_found) { // island
	  createNode2D(*graph, nodes, input.front().x, input.front().y);
	}
	// ArcData2D arc;
	// for (list<glm::dvec3>::iterator it = input.begin(); it != input.end(); it++) {
	//   arc.data.push_back(*it);
	// }
	// pair<int, int> n = createNodesForArc(arc, graph, nodes);
	// assert(n.first == n.second);
	// int edge_id = graph->addEdge(n.first, n.second, face_id);
	// geometries.setArc(edge_id, arc);
	vector<ArcData2D> arcs;
	vector<int> face_nodes;
	for (auto & v : input) {
	  ostringstream key1;
	  key1 << v.x << "/" << v.y;
	  map<string, int>::iterator it2 = nodes.find(key1.str());
	  if (it2 != nodes.end()) {
	    if (!arcs.empty()) {
	      arcs.back().data.push_back(v);
	    }
	    face_nodes.push_back(it2->second);
	    arcs.push_back(ArcData2D());
	  }
	  assert(!arcs.empty());
	  arcs.back().data.push_back(v);
	}
	assert(face_nodes.front() == face_nodes.back());
	for (int l = 0; l + 1 < face_nodes.size(); l++) {
	  int node1 = face_nodes[l], node2 = face_nodes[l + 1];
	  if (node1 == node2) {
	    // cerr << "same nodes, size = " << face_nodes.size() << endl;
	  }

	  int arc_id = 0, pair_edge = -1;
	  ostringstream key1;
	  key1 << node2 << "/" << node1;
	  map<string, int>::iterator it = waiting_faces.find(key1.str());
	  if (it != waiting_faces.end()) {
	    pair_edge = it->second;
	    auto & ed = graph->getEdgeAttributes(pair_edge);
	    assert(ed.arc >= 1 && ed.arc <= graph->getNodeArray().getArcGeometry().size());
	    arc_id = -ed.arc;
	    waiting_faces.erase(it);
	    connected_face_arcs++;
	  } else {
	    arc_id = graph->getNodeArray().addArcGeometry(arcs[l]);
	    assert(arc_id >= 1 && arc_id <= graph->getNodeArray().getArcGeometry().size());
	  }

	  assert(arc_id);

	  int edge_id = graph->addEdge(node1, node2, face_id, 1.0f, arc_id);
	  if (pair_edge != -1) {
	    graph->connectEdgePair(edge_id, pair_edge);
	  } else {
	    graph->updateMBR(edge_id);
	    ostringstream key2;
	    key2 << node1 << "/" << node2;
	    waiting_faces[key2.str()] = edge_id;
	  }
	  assert(graph->getEdgeAttributes(edge_id).arc == arc_id);
	}
      }

      area *= 0.5;
      centroid_x /= (6.0*area);
      centroid_y /= (6.0*area);
      graph->getFaceAttributes(face_id).centroid = glm::vec2(centroid_x, centroid_y);

      SHPDestroyObject(o);
    }
    cerr << "ARCS: connected face arcs = " << connected_face_arcs << ", unconnected = " << waiting_faces.size() << endl;
  }

  SHPClose(shp);

  assert(graph.get());

  auto dbf = std::make_shared<table::DBase3File>(filename);

  if (dbf->getRecordCount() != shape_count) {
    cerr << "bad number of records\n";
  }

  table::Table & table = graph->getFaceData();
  for (auto & col : dbf->getColumns()) {
    table.addColumn(col);
  }

  graph->updateFaceAppearance();

  return graph;
}
<|endoftext|>


<commit_before><commit_msg>inc/processes.hpp: Added explicit initialization of Process member variables<commit_after><|endoftext|>


<commit_before>/*  cbr
 *  StreamTxElement.hpp
 *
 *  Copyright (c) 2009, Ewen Cheslack-Postava
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name of cbr nor the names of its contributors may
 *    be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _CBR_STREAM_TX_ELEMENT_HPP_
#define _CBR_STREAM_TX_ELEMENT_HPP_

#include "Utility.hpp"
#include "RouterElement.hpp"
#include <sirikata/network/Stream.hpp>
#include <sirikata/network/IOStrandImpl.hpp>
#include "Statistics.hpp"

namespace CBR {

/** A Sirikata Network Stream transmit element.  This element pulls
 *  packets from upstream and ships them over a user provided network
 *  stream.  Note that this contains a small, 1 packet buffer in order
 *  to avoid dropping packets it actively requests (i.e. the last packet
 *  requested is not going to fit, otherwise we'd request another
 *  packet.
 */
template<typename PacketType>
class StreamTxElement : public DownstreamElementFixed<PacketType, 1> {
public:
    // Note: This isn't a great solution, but until we can set *only* the ReadySendCallback for the stream,
    // we need to provide the code that gives us the stream and creates us a callback to register on our
    // behalf
    typedef std::tr1::function<void()> ReadySendCallback;
private:
    class Impl;
    typedef std::tr1::shared_ptr<Impl> ImplPtr;

    class Impl : public std::tr1::enable_shared_from_this<Impl> {
    public:
        /** Create a stream transmit element which outputs to the specified stream,
         *  works in the specified strand, and is polled at the specified maximum rate.
         *  \param strm the stream to transmit over
         *  \param strand the strand to run requests in
         *  \param max_rate the maximum rate (minimum duration between polls) for
         *         polling for new packets to send
         */
        Impl(StreamTxElement* parent, Context* ctx, Sirikata::Network::Stream* strm, IOStrand* strand, const Duration& max_rate, Trace::MessagePath tag)
         : mParent(parent),
           mContext(ctx),
           mStream(strm),
           mStrand(strand),
           mMaxRate(max_rate),
           mTag(tag),
           mWaitingPacket(NULL),
           mSerialized(NULL),
           mShutdown( false ),
           mTimer( IOTimer::create(strand->service()) ),
           mIterationProfiler(NULL),
           mWorkProfiler(NULL)
        {
            mIterationProfiler = mContext->profiler->addStage("StreamTx Iteration");
            mWorkProfiler = mContext->profiler->addStage("StreamTx Work");
        }

        ~Impl() {
            delete mIterationProfiler;
            delete mWorkProfiler;

            if (mWaitingPacket != NULL)
                delete mWaitingPacket;
            if (mSerialized != NULL)
                delete mSerialized;
        }

        void start() {
            mTimer->setCallback( readySendCallback() );
            mIterationProfiler->started();
            startPoll(this->shared_from_this());
        }

        void shutdown() {
            mShutdown = true;
        }

        ReadySendCallback readySendCallback() {
            return mStrand->wrap( std::tr1::bind(&Impl::doIter, this, this->shared_from_this()) );
        }

    private:
        bool trySend(PacketType* next_pkt) {
            std::string* data = next_pkt->serialize();

            // Try to push to the network
            bool success = mStream->send(
                Sirikata::MemoryReference(&((*data)[0]), data->size()),
                Sirikata::Network::ReliableOrdered
            );

            // If the packet can't fit on the queue, store it and indicate we want
            // a callback when its ready again.
            if (!success) {
                mWaitingPacket = next_pkt;
                mSerialized = data;
                return false;
            }

            if (next_pkt->source_port() == OBJECT_PORT_PING && next_pkt->dest_port() == OBJECT_PORT_PING) {
                TIMESTAMP(next_pkt, mTag);
            }

            // Otherwise the packet was handled and we can clean up and continue
            delete next_pkt;
            delete data;
            return true;
        }

        void doIter(ImplPtr me) {
            mIterationProfiler->finished();
            mWorkProfiler->started();

            pullInput(me);

            mIterationProfiler->started();
            mWorkProfiler->finished();
        }

        // Attempts to pull input. Continues as long as a) input is available and
        // b) the stream continues to accept the input.
        void pullInput(ImplPtr me) {
            if (mShutdown)
                return;

            bool sent = true;

            if (mWaitingPacket) {
                sent = trySend(mWaitingPacket);
                if (!sent)
                    return;
            }

            mWaitingPacket = NULL;
            mSerialized = NULL;

            while(sent) {
                PacketType* next_pkt = mParent->input(0).pull();

                // If we run out of input, start polling and exit
                if (next_pkt == NULL) {
                    startPoll(me);
                    return;
                }

                sent = trySend(next_pkt);

                // If we couldn't send it, store it for next time and wait for a ready callback
                if (!sent) {
                    // FIXME currently we just turn on polling again because we don't
                    // have an easy way to set the stream ready callback.
                    mStream->requestReadySendCallback();
                }
            }
        }

        void startPoll(ImplPtr me) {
            if (mShutdown)
                return;

            mTimer->wait(mMaxRate);
        }

        StreamTxElement* mParent; // This is the one element which may not always be valid
        Context* mContext;
        Sirikata::Network::Stream* mStream;
        IOStrand* mStrand;
        Duration mMaxRate;
        Trace::MessagePath mTag;
        PacketType* mWaitingPacket;
        std::string* mSerialized;
        bool mShutdown;
        IOTimerPtr mTimer;
        TimeProfiler::Stage* mIterationProfiler;
        TimeProfiler::Stage* mWorkProfiler;
    };
public:
    /** Create a stream transmit element which outputs to the specified stream,
     *  , and is polled at the specified maximum rate.
     *  \param strm the stream to transmit over
     *  \param strand the strand to run requests in
     *  \param max_rate the maximum rate (minimum duration between polls) for
     *         polling for new packets to send
     *  \param tag the tag to timestamp with
     */
    StreamTxElement(Context* ctx, Sirikata::Network::Stream* strm, IOStrand* strand, const Duration& max_rate, Trace::MessagePath tag)
     : mImpl( new Impl(this, ctx, strm, strand, max_rate, tag) )
    {
    }

    ~StreamTxElement() {
        mImpl->shutdown();
    }

    virtual bool push(uint32 port, PacketType* pkt) {
        assert(false);
        return false;
    }

    /** Start the polling process for this element.  Until this is called, the element will
     *  not be active.  This allows you time to connect to other elements before processing
     *  begins.
     */
    void run() {
        mImpl->start();
    }

    void shutdown() {
        mImpl->shutdown();
    }

    ReadySendCallback readySendCallback() {
        return mImpl->readySendCallback();
    }
private:
    ImplPtr mImpl;
}; // class StreamTxElement

} // namespace CBR

#endif //_CBR_STREAM_TX_ELEMENT_HPP_
<commit_msg>If timer rate on StreamTxElement is zero, just use post.<commit_after>/*  cbr
 *  StreamTxElement.hpp
 *
 *  Copyright (c) 2009, Ewen Cheslack-Postava
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name of cbr nor the names of its contributors may
 *    be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _CBR_STREAM_TX_ELEMENT_HPP_
#define _CBR_STREAM_TX_ELEMENT_HPP_

#include "Utility.hpp"
#include "RouterElement.hpp"
#include <sirikata/network/Stream.hpp>
#include <sirikata/network/IOStrandImpl.hpp>
#include "Statistics.hpp"

namespace CBR {

/** A Sirikata Network Stream transmit element.  This element pulls
 *  packets from upstream and ships them over a user provided network
 *  stream.  Note that this contains a small, 1 packet buffer in order
 *  to avoid dropping packets it actively requests (i.e. the last packet
 *  requested is not going to fit, otherwise we'd request another
 *  packet.
 */
template<typename PacketType>
class StreamTxElement : public DownstreamElementFixed<PacketType, 1> {
public:
    // Note: This isn't a great solution, but until we can set *only* the ReadySendCallback for the stream,
    // we need to provide the code that gives us the stream and creates us a callback to register on our
    // behalf
    typedef std::tr1::function<void()> ReadySendCallback;
private:
    class Impl;
    typedef std::tr1::shared_ptr<Impl> ImplPtr;

    class Impl : public std::tr1::enable_shared_from_this<Impl> {
    public:
        /** Create a stream transmit element which outputs to the specified stream,
         *  works in the specified strand, and is polled at the specified maximum rate.
         *  \param strm the stream to transmit over
         *  \param strand the strand to run requests in
         *  \param max_rate the maximum rate (minimum duration between polls) for
         *         polling for new packets to send
         */
        Impl(StreamTxElement* parent, Context* ctx, Sirikata::Network::Stream* strm, IOStrand* strand, const Duration& max_rate, Trace::MessagePath tag)
         : mParent(parent),
           mContext(ctx),
           mStream(strm),
           mStrand(strand),
           mMaxRate(max_rate),
           mTag(tag),
           mWaitingPacket(NULL),
           mSerialized(NULL),
           mShutdown( false ),
           mTimer( IOTimer::create(strand->service()) ),
           mIterationProfiler(NULL),
           mWorkProfiler(NULL)
        {
            mIterationProfiler = mContext->profiler->addStage("StreamTx Iteration");
            mWorkProfiler = mContext->profiler->addStage("StreamTx Work");
        }

        ~Impl() {
            delete mIterationProfiler;
            delete mWorkProfiler;

            if (mWaitingPacket != NULL)
                delete mWaitingPacket;
            if (mSerialized != NULL)
                delete mSerialized;
        }

        void start() {
            mTimer->setCallback( readySendCallback() );
            mIterationProfiler->started();
            startPoll(this->shared_from_this());
        }

        void shutdown() {
            mShutdown = true;
        }

        ReadySendCallback readySendCallback() {
            return mStrand->wrap( std::tr1::bind(&Impl::doIter, this, this->shared_from_this()) );
        }

    private:
        bool trySend(PacketType* next_pkt) {
            std::string* data = next_pkt->serialize();

            // Try to push to the network
            bool success = mStream->send(
                Sirikata::MemoryReference(&((*data)[0]), data->size()),
                Sirikata::Network::ReliableOrdered
            );

            // If the packet can't fit on the queue, store it and indicate we want
            // a callback when its ready again.
            if (!success) {
                mWaitingPacket = next_pkt;
                mSerialized = data;
                return false;
            }

            if (next_pkt->source_port() == OBJECT_PORT_PING && next_pkt->dest_port() == OBJECT_PORT_PING) {
                TIMESTAMP(next_pkt, mTag);
            }

            // Otherwise the packet was handled and we can clean up and continue
            delete next_pkt;
            delete data;
            return true;
        }

        void doIter(ImplPtr me) {
            mIterationProfiler->finished();
            mWorkProfiler->started();

            pullInput(me);

            mIterationProfiler->started();
            mWorkProfiler->finished();
        }

        // Attempts to pull input. Continues as long as a) input is available and
        // b) the stream continues to accept the input.
        void pullInput(ImplPtr me) {
            if (mShutdown)
                return;

            bool sent = true;

            if (mWaitingPacket) {
                sent = trySend(mWaitingPacket);
                if (!sent)
                    return;
            }

            mWaitingPacket = NULL;
            mSerialized = NULL;

            while(sent) {
                PacketType* next_pkt = mParent->input(0).pull();

                // If we run out of input, start polling and exit
                if (next_pkt == NULL) {
                    startPoll(me);
                    return;
                }

                sent = trySend(next_pkt);

                // If we couldn't send it, store it for next time and wait for a ready callback
                if (!sent) {
                    // FIXME currently we just turn on polling again because we don't
                    // have an easy way to set the stream ready callback.
                    mStream->requestReadySendCallback();
                }
            }
        }

        void startPoll(ImplPtr me) {
            if (mShutdown)
                return;

            if (mMaxRate == Duration::zero())
                mStrand->post(readySendCallback());
            else
                mTimer->wait(mMaxRate);
        }

        StreamTxElement* mParent; // This is the one element which may not always be valid
        Context* mContext;
        Sirikata::Network::Stream* mStream;
        IOStrand* mStrand;
        Duration mMaxRate;
        Trace::MessagePath mTag;
        PacketType* mWaitingPacket;
        std::string* mSerialized;
        bool mShutdown;
        IOTimerPtr mTimer;
        TimeProfiler::Stage* mIterationProfiler;
        TimeProfiler::Stage* mWorkProfiler;
    };
public:
    /** Create a stream transmit element which outputs to the specified stream,
     *  , and is polled at the specified maximum rate.
     *  \param strm the stream to transmit over
     *  \param strand the strand to run requests in
     *  \param max_rate the maximum rate (minimum duration between polls) for
     *         polling for new packets to send
     *  \param tag the tag to timestamp with
     */
    StreamTxElement(Context* ctx, Sirikata::Network::Stream* strm, IOStrand* strand, const Duration& max_rate, Trace::MessagePath tag)
     : mImpl( new Impl(this, ctx, strm, strand, max_rate, tag) )
    {
    }

    ~StreamTxElement() {
        mImpl->shutdown();
    }

    virtual bool push(uint32 port, PacketType* pkt) {
        assert(false);
        return false;
    }

    /** Start the polling process for this element.  Until this is called, the element will
     *  not be active.  This allows you time to connect to other elements before processing
     *  begins.
     */
    void run() {
        mImpl->start();
    }

    void shutdown() {
        mImpl->shutdown();
    }

    ReadySendCallback readySendCallback() {
        return mImpl->readySendCallback();
    }
private:
    ImplPtr mImpl;
}; // class StreamTxElement

} // namespace CBR

#endif //_CBR_STREAM_TX_ELEMENT_HPP_
<|endoftext|>


<commit_before>// Copyright 2017 Netherlands Institute for Radio Astronomy (ASTRON)
// Copyright 2017 Netherlands eScience Center
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <iostream>
#include <string>
#include <vector>
#include <exception>
#include <fstream>
#include <iomanip>
#include <limits>
#include <algorithm>

#include <configuration.hpp>

#include <ArgumentList.hpp>
#include <Observation.hpp>
#include <InitializeOpenCL.hpp>
#include <Kernel.hpp>
#include <SNR.hpp>
#include <utils.hpp>
#include <Timer.hpp>
#include <Stats.hpp>

void initializeDeviceMemoryD(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * outputSNR_d, const uint64_t outputSNR_size, cl::Buffer * outputSample_d, const uint64_t outputSample_size);

int main(int argc, char * argv[]) {
  bool reInit = true;
  bool DMsSamples = false;
  bool bestMode = false;
  unsigned int padding = 0;
  unsigned int nrIterations = 0;
  unsigned int clPlatformID = 0;
  unsigned int clDeviceID = 0;
  unsigned int minThreads = 0;
  unsigned int maxItems = 0;
  unsigned int maxThreads = 0;
  double bestGBs = 0.0;
  AstroData::Observation observation;
  SNR::snrConf conf;
  SNR::snrConf bestConf;
  cl::Event event;

  try {
    isa::utils::ArgumentList args(argc, argv);
    DMsSamples = args.getSwitch("-dms_samples");
    bool samplesDMs = args.getSwitch("-samples_dms");
    if ( (DMsSamples && samplesDMs) || (!DMsSamples && !samplesDMs) ) {
      std::cerr << "-dms_samples and -samples_dms are mutually exclusive." << std::endl;
      return 1;
    }
    nrIterations = args.getSwitchArgument< unsigned int >("-iterations");
    clPlatformID = args.getSwitchArgument< unsigned int >("-opencl_platform");
    clDeviceID = args.getSwitchArgument< unsigned int >("-opencl_device");
    bestMode = args.getSwitch("-best");
    padding = args.getSwitchArgument< unsigned int >("-padding");
    minThreads = args.getSwitchArgument< unsigned int >("-min_threads");
    maxItems = args.getSwitchArgument< unsigned int >("-max_items");
    maxThreads = args.getSwitchArgument< unsigned int >("-max_threads");
    conf.setSubbandDedispersion(args.getSwitch("-subband"));
    observation.setNrSynthesizedBeams(args.getSwitchArgument< unsigned int >("-beams"));
    observation.setNrSamplesPerBatch(args.getSwitchArgument< unsigned int >("-samples"));
    if ( conf.getSubbandDedispersion() ) {
      observation.setDMRange(args.getSwitchArgument< unsigned int >("-subbanding_dms"), 0.0f, 0.0f, true);
    } else {
      observation.setDMRange(1, 0.0f, 0.0f, true);
    }
    observation.setDMRange(args.getSwitchArgument< unsigned int >("-dms"), 0.0, 0.0);
  } catch ( isa::utils::EmptyCommandLine & err ) {
    std::cerr << argv[0] << " [-best] [-dms_samples | -samples_dms] -iterations ... -opencl_platform ... -opencl_device ... -padding ... -min_threads ... -max_threads ... -max_items ... [-subband] -beams ... -dms ... -samples ..." << std::endl;
    std::cerr << "\t -subband : -subbanding_dms ..." << std::endl;
    return 1;
  } catch ( std::exception & err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }

  // Initialize OpenCL
  cl::Context clContext;
  std::vector< cl::Platform > * clPlatforms = new std::vector< cl::Platform >();
  std::vector< cl::Device > * clDevices = new std::vector< cl::Device >();
  std::vector< std::vector< cl::CommandQueue > > * clQueues = 0;

  // Allocate memory
  std::vector< inputDataType > input;
  cl::Buffer input_d, outputSNR_d, outputSample_d;

  if ( DMsSamples ) {
    input.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(true) * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, padding / sizeof(inputDataType)));
  } else {
    input.resize(observation.getNrSynthesizedBeams() * observation.getNrSamplesPerBatch() * observation.getNrDMs(true) * observation.getNrDMs(false, padding / sizeof(inputDataType)));
  }

  srand(time(0));
  for ( unsigned int beam = 0; beam < observation.getNrSynthesizedBeams(); beam++ ) {
    if ( DMsSamples ) {
      for ( unsigned int subbandDM = 0; subbandDM < observation.getNrDMs(true); subbandDM++ ) {
        for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm++ ) {
          for ( unsigned int sample = 0; sample < observation.getNrSamplesPerBatch(); sample++ ) {
            input[(beam * observation.getNrDMs(true) * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, padding / sizeof(inputDataType))) + (subbandDM * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, padding / sizeof(inputDataType))) + (dm * observation.getNrSamplesPerBatch(false, padding / sizeof(inputDataType))) + sample] = static_cast< inputDataType >(rand() % 10);
          }
        }
      }
    } else {
      for ( unsigned int sample = 0; sample < observation.getNrSamplesPerBatch(); sample++ ) {
        for ( unsigned int subbandDM = 0; subbandDM < observation.getNrDMs(true); subbandDM++ ) {
          for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm++ ) {
            input[(beam * observation.getNrSamplesPerBatch() * observation.getNrDMs(true) * observation.getNrDMs(false, padding / sizeof(inputDataType))) + (sample * observation.getNrDMs(true) * observation.getNrDMs(false, padding / sizeof(inputDataType))) + (subbandDM * observation.getNrDMs(false, padding / sizeof(inputDataType))) + dm] = static_cast< inputDataType >(rand() % 10);
          }
        }
      }
    }
  }

  if ( !bestMode ) {
    std::cout << std::fixed << std::endl;
    std::cout << "# nrBeams nrDMs nrSamples *configuration* GB/s time stdDeviation COV" << std::endl << std::endl;
  }

  for ( unsigned int threads = minThreads; threads <= maxThreads; ) {
    conf.setNrThreadsD0(threads);
    if ( DMsSamples ) {
      threads *= 2;
    } else {
      threads++;
    }
    for ( unsigned int itemsPerThread = 1; itemsPerThread <= maxItems; itemsPerThread++ ) {
      if ( DMsSamples ) {
        if ( ((itemsPerThread * 5) + 8) > maxItems ) {
          break;
        }
        if ( (observation.getNrSamplesPerBatch() % itemsPerThread) != 0 ) {
          continue;
        }
      } else {
        if ( ((itemsPerThread * 5) + 3) > maxItems ) {
          break;
        }
        if ( observation.getNrDMs() % ( itemsPerThread * conf.getNrThreadsD0()) != 0 ) {
          continue;
        }
      }
      conf.setNrItemsD0(itemsPerThread);

      // Generate kernel
      double gbs = isa::utils::giga((observation.getNrSynthesizedBeams() * static_cast< uint64_t >(observation.getNrDMs(true) * observation.getNrDMs()) * observation.getNrSamplesPerBatch() * sizeof(inputDataType)) + (observation.getNrSynthesizedBeams() * static_cast< uint64_t >(observation.getNrDMs(true) * observation.getNrDMs()) * sizeof(float)) + (observation.getNrSynthesizedBeams() * static_cast< uint64_t >(observation.getNrDMs(true) * observation.getNrDMs()) * sizeof(unsigned int)));
      cl::Kernel * kernel;
      isa::utils::Timer timer;
      std::string * code;
      if ( DMsSamples ) {
        code = SNR::getSNRDMsSamplesOpenCL< inputDataType >(conf, inputDataName, observation, observation.getNrSamplesPerBatch(), padding);
      } else {
        code = SNR::getSNRSamplesDMsOpenCL< inputDataType >(conf, inputDataName, observation, observation.getNrSamplesPerBatch(), padding);
      }

      if ( reInit ) {
        delete clQueues;
        clQueues = new std::vector< std::vector< cl::CommandQueue > >();
        isa::OpenCL::initializeOpenCL(clPlatformID, 1, clPlatforms, &clContext, clDevices, clQueues);
        try {
          initializeDeviceMemoryD(clContext, &(clQueues->at(clDeviceID)[0]), &input, &input_d, &outputSNR_d, observation.getNrSynthesizedBeams() * isa::utils::pad(observation.getNrDMs(true) * observation.getNrDMs(), padding / sizeof(float)) * sizeof(float), &outputSample_d, observation.getNrSynthesizedBeams() * isa::utils::pad(observation.getNrDMs(true) * observation.getNrDMs(), padding / sizeof(unsigned int)) * sizeof(unsigned int));
        } catch ( cl::Error & err ) {
          return -1;
        }
        reInit = false;
      }
      try {
        if ( DMsSamples ) {
          kernel = isa::OpenCL::compile("snrDMsSamples" + std::to_string(observation.getNrSamplesPerBatch()), *code, "-cl-mad-enable -Werror", clContext, clDevices->at(clDeviceID));
        } else {
          kernel = isa::OpenCL::compile("snrSamplesDMs" + std::to_string(observation.getNrDMs(true) * observation.getNrDMs()), *code, "-cl-mad-enable -Werror", clContext, clDevices->at(clDeviceID));
        }
      } catch ( isa::OpenCL::OpenCLError & err ) {
        std::cerr << err.what() << std::endl;
        delete code;
        break;
      }
      delete code;

      cl::NDRange global, local;
      if ( DMsSamples ) {
        global = cl::NDRange(conf.getNrThreadsD0(), observation.getNrDMs(true) * observation.getNrDMs(), observation.getNrSynthesizedBeams());
        local = cl::NDRange(conf.getNrThreadsD0(), 1, 1);
      } else {
        global = cl::NDRange((observation.getNrDMs(true) * observation.getNrDMs()) / conf.getNrItemsD0(), observation.getNrSynthesizedBeams());
        local = cl::NDRange(conf.getNrThreadsD0(), 1);
      }

      kernel->setArg(0, input_d);
      kernel->setArg(1, outputSNR_d);
      kernel->setArg(2, outputSample_d);

      try {
        // Warm-up run
        clQueues->at(clDeviceID)[0].finish();
        clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &event);
        event.wait();
        // Tuning runs
        for ( unsigned int iteration = 0; iteration < nrIterations; iteration++ ) {
          timer.start();
          clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &event);
          event.wait();
          timer.stop();
        }
      } catch ( cl::Error & err ) {
        std::cerr << "OpenCL error kernel execution (";
        std::cerr << conf.print();
        std::cerr << "): " << std::to_string(err.err()) << "." << std::endl;
        delete kernel;
        if ( err.err() == -4 || err.err() == -61 ) {
          return -1;
        }
        reInit = true;
        break;
      }
      delete kernel;

      if ( (gbs / timer.getAverageTime()) > bestGBs ) {
        bestGBs = gbs / timer.getAverageTime();
        bestConf = conf;
      }
      if ( !bestMode ) {
        std::cout << observation.getNrSynthesizedBeams() << " " << observation.getNrDMs(true) * observation.getNrDMs() << " " << observation.getNrSamplesPerBatch() << " ";
        std::cout << conf.print() << " ";
        std::cout << std::setprecision(3);
        std::cout << gbs / timer.getAverageTime() << " ";
        std::cout << std::setprecision(6);
        std::cout << timer.getAverageTime() << " " << timer.getStandardDeviation() << " " << timer.getCoefficientOfVariation() << std::endl;
      }
    }
  }

  if ( bestMode ) {
    std::cout << observation.getNrDMs(true) * observation.getNrDMs() << " " << observation.getNrSamplesPerBatch() << " " << bestConf.print() << std::endl;
  } else {
    std::cout << std::endl;
  }

  return 0;
}

void initializeDeviceMemoryD(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * outputSNR_d, const uint64_t outputSNR_size, cl::Buffer * outputSample_d, const uint64_t outputSample_size) {
  try {
    *input_d = cl::Buffer(clContext, CL_MEM_READ_WRITE, input->size() * sizeof(inputDataType), 0, 0);
    *outputSNR_d = cl::Buffer(clContext, CL_MEM_WRITE_ONLY, outputSNR_size, 0, 0);
    *outputSample_d = cl::Buffer(clContext, CL_MEM_WRITE_ONLY, outputSample_size, 0, 0);
    clQueue->enqueueWriteBuffer(*input_d, CL_FALSE, 0, input->size() * sizeof(inputDataType), reinterpret_cast< void * >(input->data()));
    clQueue->finish();
  } catch ( cl::Error & err ) {
    std::cerr << "OpenCL error: " << std::to_string(err.err()) << "." << std::endl;
    throw;
  }
}

<commit_msg>Variable name change.<commit_after>// Copyright 2017 Netherlands Institute for Radio Astronomy (ASTRON)
// Copyright 2017 Netherlands eScience Center
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <iostream>
#include <string>
#include <vector>
#include <exception>
#include <fstream>
#include <iomanip>
#include <limits>
#include <algorithm>

#include <configuration.hpp>

#include <ArgumentList.hpp>
#include <Observation.hpp>
#include <InitializeOpenCL.hpp>
#include <Kernel.hpp>
#include <SNR.hpp>
#include <utils.hpp>
#include <Timer.hpp>
#include <Stats.hpp>

void initializeDeviceMemoryD(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * outputSNR_d, const uint64_t outputSNR_size, cl::Buffer * outputSample_d, const uint64_t outputSample_size);

int main(int argc, char * argv[]) {
  bool reinitializeDeviceMemory = true;
  bool DMsSamples = false;
  bool bestMode = false;
  unsigned int padding = 0;
  unsigned int nrIterations = 0;
  unsigned int clPlatformID = 0;
  unsigned int clDeviceID = 0;
  unsigned int minThreads = 0;
  unsigned int maxItems = 0;
  unsigned int maxThreads = 0;
  double bestGBs = 0.0;
  AstroData::Observation observation;
  SNR::snrConf conf;
  SNR::snrConf bestConf;
  cl::Event event;

  try {
    isa::utils::ArgumentList args(argc, argv);
    DMsSamples = args.getSwitch("-dms_samples");
    bool samplesDMs = args.getSwitch("-samples_dms");
    if ( (DMsSamples && samplesDMs) || (!DMsSamples && !samplesDMs) ) {
      std::cerr << "-dms_samples and -samples_dms are mutually exclusive." << std::endl;
      return 1;
    }
    nrIterations = args.getSwitchArgument< unsigned int >("-iterations");
    clPlatformID = args.getSwitchArgument< unsigned int >("-opencl_platform");
    clDeviceID = args.getSwitchArgument< unsigned int >("-opencl_device");
    bestMode = args.getSwitch("-best");
    padding = args.getSwitchArgument< unsigned int >("-padding");
    minThreads = args.getSwitchArgument< unsigned int >("-min_threads");
    maxItems = args.getSwitchArgument< unsigned int >("-max_items");
    maxThreads = args.getSwitchArgument< unsigned int >("-max_threads");
    conf.setSubbandDedispersion(args.getSwitch("-subband"));
    observation.setNrSynthesizedBeams(args.getSwitchArgument< unsigned int >("-beams"));
    observation.setNrSamplesPerBatch(args.getSwitchArgument< unsigned int >("-samples"));
    if ( conf.getSubbandDedispersion() ) {
      observation.setDMRange(args.getSwitchArgument< unsigned int >("-subbanding_dms"), 0.0f, 0.0f, true);
    } else {
      observation.setDMRange(1, 0.0f, 0.0f, true);
    }
    observation.setDMRange(args.getSwitchArgument< unsigned int >("-dms"), 0.0, 0.0);
  } catch ( isa::utils::EmptyCommandLine & err ) {
    std::cerr << argv[0] << " [-best] [-dms_samples | -samples_dms] -iterations ... -opencl_platform ... -opencl_device ... -padding ... -min_threads ... -max_threads ... -max_items ... [-subband] -beams ... -dms ... -samples ..." << std::endl;
    std::cerr << "\t -subband : -subbanding_dms ..." << std::endl;
    return 1;
  } catch ( std::exception & err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }

  // Initialize OpenCL
  cl::Context clContext;
  std::vector< cl::Platform > * clPlatforms = new std::vector< cl::Platform >();
  std::vector< cl::Device > * clDevices = new std::vector< cl::Device >();
  std::vector< std::vector< cl::CommandQueue > > * clQueues = 0;

  // Allocate memory
  std::vector< inputDataType > input;
  cl::Buffer input_d, outputSNR_d, outputSample_d;

  if ( DMsSamples ) {
    input.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(true) * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, padding / sizeof(inputDataType)));
  } else {
    input.resize(observation.getNrSynthesizedBeams() * observation.getNrSamplesPerBatch() * observation.getNrDMs(true) * observation.getNrDMs(false, padding / sizeof(inputDataType)));
  }

  srand(time(0));
  for ( unsigned int beam = 0; beam < observation.getNrSynthesizedBeams(); beam++ ) {
    if ( DMsSamples ) {
      for ( unsigned int subbandDM = 0; subbandDM < observation.getNrDMs(true); subbandDM++ ) {
        for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm++ ) {
          for ( unsigned int sample = 0; sample < observation.getNrSamplesPerBatch(); sample++ ) {
            input[(beam * observation.getNrDMs(true) * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, padding / sizeof(inputDataType))) + (subbandDM * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, padding / sizeof(inputDataType))) + (dm * observation.getNrSamplesPerBatch(false, padding / sizeof(inputDataType))) + sample] = static_cast< inputDataType >(rand() % 10);
          }
        }
      }
    } else {
      for ( unsigned int sample = 0; sample < observation.getNrSamplesPerBatch(); sample++ ) {
        for ( unsigned int subbandDM = 0; subbandDM < observation.getNrDMs(true); subbandDM++ ) {
          for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm++ ) {
            input[(beam * observation.getNrSamplesPerBatch() * observation.getNrDMs(true) * observation.getNrDMs(false, padding / sizeof(inputDataType))) + (sample * observation.getNrDMs(true) * observation.getNrDMs(false, padding / sizeof(inputDataType))) + (subbandDM * observation.getNrDMs(false, padding / sizeof(inputDataType))) + dm] = static_cast< inputDataType >(rand() % 10);
          }
        }
      }
    }
  }

  if ( !bestMode ) {
    std::cout << std::fixed << std::endl;
    std::cout << "# nrBeams nrDMs nrSamples *configuration* GB/s time stdDeviation COV" << std::endl << std::endl;
  }

  for ( unsigned int threads = minThreads; threads <= maxThreads; ) {
    conf.setNrThreadsD0(threads);
    if ( DMsSamples ) {
      threads *= 2;
    } else {
      threads++;
    }
    for ( unsigned int itemsPerThread = 1; itemsPerThread <= maxItems; itemsPerThread++ ) {
      if ( DMsSamples ) {
        if ( ((itemsPerThread * 5) + 8) > maxItems ) {
          break;
        }
        if ( (observation.getNrSamplesPerBatch() % itemsPerThread) != 0 ) {
          continue;
        }
      } else {
        if ( ((itemsPerThread * 5) + 3) > maxItems ) {
          break;
        }
        if ( observation.getNrDMs() % ( itemsPerThread * conf.getNrThreadsD0()) != 0 ) {
          continue;
        }
      }
      conf.setNrItemsD0(itemsPerThread);

      // Generate kernel
      double gbs = isa::utils::giga((observation.getNrSynthesizedBeams() * static_cast< uint64_t >(observation.getNrDMs(true) * observation.getNrDMs()) * observation.getNrSamplesPerBatch() * sizeof(inputDataType)) + (observation.getNrSynthesizedBeams() * static_cast< uint64_t >(observation.getNrDMs(true) * observation.getNrDMs()) * sizeof(float)) + (observation.getNrSynthesizedBeams() * static_cast< uint64_t >(observation.getNrDMs(true) * observation.getNrDMs()) * sizeof(unsigned int)));
      cl::Kernel * kernel;
      isa::utils::Timer timer;
      std::string * code;
      if ( DMsSamples ) {
        code = SNR::getSNRDMsSamplesOpenCL< inputDataType >(conf, inputDataName, observation, observation.getNrSamplesPerBatch(), padding);
      } else {
        code = SNR::getSNRSamplesDMsOpenCL< inputDataType >(conf, inputDataName, observation, observation.getNrSamplesPerBatch(), padding);
      }

      if ( reinitializeDeviceMemory ) {
        delete clQueues;
        clQueues = new std::vector< std::vector< cl::CommandQueue > >();
        isa::OpenCL::initializeOpenCL(clPlatformID, 1, clPlatforms, &clContext, clDevices, clQueues);
        try {
          initializeDeviceMemoryD(clContext, &(clQueues->at(clDeviceID)[0]), &input, &input_d, &outputSNR_d, observation.getNrSynthesizedBeams() * isa::utils::pad(observation.getNrDMs(true) * observation.getNrDMs(), padding / sizeof(float)) * sizeof(float), &outputSample_d, observation.getNrSynthesizedBeams() * isa::utils::pad(observation.getNrDMs(true) * observation.getNrDMs(), padding / sizeof(unsigned int)) * sizeof(unsigned int));
        } catch ( cl::Error & err ) {
          return -1;
        }
        reinitializeDeviceMemory = false;
      }
      try {
        if ( DMsSamples ) {
          kernel = isa::OpenCL::compile("snrDMsSamples" + std::to_string(observation.getNrSamplesPerBatch()), *code, "-cl-mad-enable -Werror", clContext, clDevices->at(clDeviceID));
        } else {
          kernel = isa::OpenCL::compile("snrSamplesDMs" + std::to_string(observation.getNrDMs(true) * observation.getNrDMs()), *code, "-cl-mad-enable -Werror", clContext, clDevices->at(clDeviceID));
        }
      } catch ( isa::OpenCL::OpenCLError & err ) {
        std::cerr << err.what() << std::endl;
        delete code;
        break;
      }
      delete code;

      cl::NDRange global, local;
      if ( DMsSamples ) {
        global = cl::NDRange(conf.getNrThreadsD0(), observation.getNrDMs(true) * observation.getNrDMs(), observation.getNrSynthesizedBeams());
        local = cl::NDRange(conf.getNrThreadsD0(), 1, 1);
      } else {
        global = cl::NDRange((observation.getNrDMs(true) * observation.getNrDMs()) / conf.getNrItemsD0(), observation.getNrSynthesizedBeams());
        local = cl::NDRange(conf.getNrThreadsD0(), 1);
      }

      kernel->setArg(0, input_d);
      kernel->setArg(1, outputSNR_d);
      kernel->setArg(2, outputSample_d);

      try {
        // Warm-up run
        clQueues->at(clDeviceID)[0].finish();
        clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &event);
        event.wait();
        // Tuning runs
        for ( unsigned int iteration = 0; iteration < nrIterations; iteration++ ) {
          timer.start();
          clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &event);
          event.wait();
          timer.stop();
        }
      } catch ( cl::Error & err ) {
        std::cerr << "OpenCL error kernel execution (";
        std::cerr << conf.print();
        std::cerr << "): " << std::to_string(err.err()) << "." << std::endl;
        delete kernel;
        if ( err.err() == -4 || err.err() == -61 ) {
          return -1;
        }
        reinitializeDeviceMemory = true;
        break;
      }
      delete kernel;

      if ( (gbs / timer.getAverageTime()) > bestGBs ) {
        bestGBs = gbs / timer.getAverageTime();
        bestConf = conf;
      }
      if ( !bestMode ) {
        std::cout << observation.getNrSynthesizedBeams() << " " << observation.getNrDMs(true) * observation.getNrDMs() << " " << observation.getNrSamplesPerBatch() << " ";
        std::cout << conf.print() << " ";
        std::cout << std::setprecision(3);
        std::cout << gbs / timer.getAverageTime() << " ";
        std::cout << std::setprecision(6);
        std::cout << timer.getAverageTime() << " " << timer.getStandardDeviation() << " " << timer.getCoefficientOfVariation() << std::endl;
      }
    }
  }

  if ( bestMode ) {
    std::cout << observation.getNrDMs(true) * observation.getNrDMs() << " " << observation.getNrSamplesPerBatch() << " " << bestConf.print() << std::endl;
  } else {
    std::cout << std::endl;
  }

  return 0;
}

void initializeDeviceMemoryD(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * outputSNR_d, const uint64_t outputSNR_size, cl::Buffer * outputSample_d, const uint64_t outputSample_size) {
  try {
    *input_d = cl::Buffer(clContext, CL_MEM_READ_WRITE, input->size() * sizeof(inputDataType), 0, 0);
    *outputSNR_d = cl::Buffer(clContext, CL_MEM_WRITE_ONLY, outputSNR_size, 0, 0);
    *outputSample_d = cl::Buffer(clContext, CL_MEM_WRITE_ONLY, outputSample_size, 0, 0);
    clQueue->enqueueWriteBuffer(*input_d, CL_FALSE, 0, input->size() * sizeof(inputDataType), reinterpret_cast< void * >(input->data()));
    clQueue->finish();
  } catch ( cl::Error & err ) {
    std::cerr << "OpenCL error: " << std::to_string(err.err()) << "." << std::endl;
    throw;
  }
}

<|endoftext|>


<commit_before>#include "styledoutputpaneplaceholder.h"

#include <QtCore/QChildEvent>
#include <QtCore/QFile>
#include <QtGui/QTabWidget>
#include <QtGui/QStackedWidget>
#include <QDebug>

StyledOutputpanePlaceHolder::StyledOutputpanePlaceHolder(Core::IMode *mode, QSplitter *parent) : Core::OutputPanePlaceHolder(mode, parent)
{
    QFile file(":/qmldesigner/outputpane-style.css");
    file.open(QFile::ReadOnly);
    QFile file2(":/qmldesigner/scrollbar.css");
    file2.open(QFile::ReadOnly);
    m_customStylesheet = file.readAll() + file2.readAll();
    file.close();
    file2.close();

}

void StyledOutputpanePlaceHolder::childEvent(QChildEvent *event)
{
    Core::OutputPanePlaceHolder::childEvent(event);

    if (event->type() == QEvent::ChildAdded) {
        QWidget *child = qobject_cast<QWidget*>(event->child());
        if (child) {
            QList<QTabWidget*> widgets = child->findChildren<QTabWidget*>();
            if (!widgets.isEmpty()) {
                QPalette paletteP = palette();
                paletteP.setBrush(QPalette::All, QPalette::Light, Qt::red);
                paletteP.setBrush(QPalette::All, QPalette::Shadow, Qt::red);
                paletteP.setBrush(QPalette::All, QPalette::Dark, Qt::red);
                paletteP.setBrush(QPalette::All, QPalette::Window, Qt::red);
                setPalette(paletteP);
                widgets.first()->parentWidget()->ensurePolished();
                widgets.first()->parentWidget()->setStyleSheet(m_customStylesheet);
            }

        }
    } else if (event->type() == QEvent::ChildRemoved) {
        QWidget *child = qobject_cast<QWidget*>(event->child());
        if (child) {
            QList<QTabWidget*> widgets = child->findChildren<QTabWidget*>();
            if (!widgets.isEmpty())
                widgets.first()->parentWidget()->setStyleSheet(QString());

        }
    }
}
<commit_msg>code cosmetics<commit_after>#include "styledoutputpaneplaceholder.h"

#include <QtCore/QChildEvent>
#include <QtCore/QFile>
#include <QtGui/QTabWidget>
#include <QtGui/QStackedWidget>
#include <QDebug>

StyledOutputpanePlaceHolder::StyledOutputpanePlaceHolder(Core::IMode *mode, QSplitter *parent) : Core::OutputPanePlaceHolder(mode, parent)
{
    QFile file(":/qmldesigner/outputpane-style.css");
    file.open(QFile::ReadOnly);
    QFile file2(":/qmldesigner/scrollbar.css");
    file2.open(QFile::ReadOnly);
    m_customStylesheet = file.readAll() + file2.readAll();
    file.close();
    file2.close();
}

void StyledOutputpanePlaceHolder::childEvent(QChildEvent *event)
{
    Core::OutputPanePlaceHolder::childEvent(event);

    if (event->type() == QEvent::ChildAdded) {
        QWidget *child = qobject_cast<QWidget*>(event->child());
        if (child) {
            QList<QTabWidget*> widgets = child->findChildren<QTabWidget*>();
            if (!widgets.isEmpty()) {
                widgets.first()->parentWidget()->ensurePolished();
                widgets.first()->parentWidget()->setStyleSheet(m_customStylesheet);
            }

        }
    } else if (event->type() == QEvent::ChildRemoved) {
        QWidget *child = qobject_cast<QWidget*>(event->child());
        if (child) {
            QList<QTabWidget*> widgets = child->findChildren<QTabWidget*>();
            if (!widgets.isEmpty())
                widgets.first()->parentWidget()->setStyleSheet(QString());
        }
    }
}
<|endoftext|>