<commit_before>#include "catch.hpp"
#include <iostream>
#include "Database_Creator.hpp"
#include "Database_Sorter.hpp"

person readPerson(std::string file_name, size_t index) {
	person result;
	std::ifstream file(file_name);
	for (size_t i = 0; i < index + 1; i++) {
		file >> result;
	}
	file.close();
	return result;
}
bool isALessThanB(char A, char B, bool is_capital) {
	std::string alphabet;
	if (is_capital) {
		alphabet = " АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ";
	}
	else {
		alphabet = " абвгдеёжзийклмнопрстуфхцчшщъыьэюя";
	}
	size_t A_i;
	size_t B_i;
	for (size_t i = 0; i < alphabet.size(); i++) {
		if (alphabet[i] == A) {
			A_i = i;
		}
		if (alphabet[i] == B) {
			B_i = i;
		}
	}
	return A_i < B_i;
}
bool isANotMoreThanB(person A, person B) {
	for (size_t i = 0; i < A.surname.length(); i++) {
		char A_char = A.surname[i];
		char B_char = (i < B.surname.length()) ? B.surname[i] : ' ';
		if (isALessThanB(A.surname[i], B.surname[i], i == 0)) {
			return true;
		}
		else {
			if (A.surname[i] != B.surname[i]) {
				return false;
			}
		}
	}
	return true;
}

SCENARIO("Sort", "[s]") {
	//setlocale(LC_ALL, "ru_RU.utf8");
	std::ifstream ru("F:\\1\\russian_letters.txt");
	for (size_t i = 0; i < 6; i++) {
		std::string str;
		ru >> str;
		russian_letters[i] = str[0];
	}
	ru.close();
	size_t n_persons = 2001;
	size_t RAM_amount = 10000;
	std::string names_file_name = "names.txt";
	std::string surnames_file_name = "surnames.txt";
	std::string database_file_name = "database.txt";
	std::string output_file_name = "sorted_database.txt";
	{
		Database_Creator database_creator(database_file_name, names_file_name, surnames_file_name, n_persons);
		Database_Sorter database_sorter(database_file_name, output_file_name, RAM_amount);
		size_t start = clock();
		database_sorter.sortDatabase();
		size_t result = clock() - start;
		std::cout << result << std::endl;
		system("pause");
	}
	for (size_t i = 0; i < 20; i++) {
		size_t person1_i = rand() % n_persons;
		size_t person2_i = person1_i + rand() % (n_persons - person1_i);
		person person1 = readPerson(output_file_name, person1_i);
		person person2 = readPerson(output_file_name, person2_i);
		REQUIRE(isANotMoreThanB(person1, person2));
	}
}
<commit_msg>Update init.cpp<commit_after>#include "catch.hpp"
#include <iostream>
#include "Database_Creator.hpp"
#include "Database_Sorter.hpp"

person readPerson(std::string file_name, size_t index) {
	person result;
	std::ifstream file(file_name);
	for (size_t i = 0; i < index + 1; i++) {
		file >> result;
	}
	file.close();
	return result;
}
bool isALessThanB(char A, char B, bool is_capital) {
	std::string alphabet;
	if (is_capital) {
		alphabet = " АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ";
	}
	else {
		alphabet = " абвгдеёжзийклмнопрстуфхцчшщъыьэюя";
	}
	size_t A_i;
	size_t B_i;
	for (size_t i = 0; i < alphabet.size(); i++) {
		if (alphabet[i] == A) {
			A_i = i;
		}
		if (alphabet[i] == B) {
			B_i = i;
		}
	}
	return A_i < B_i;
}
bool isANotMoreThanB(person A, person B) {
	for (size_t i = 0; i < A.surname.length(); i++) {
		char A_char = A.surname[i];
		char B_char = (i < B.surname.length()) ? B.surname[i] : ' ';
		if (isALessThanB(A.surname[i], B.surname[i], i == 0)) {
			return true;
		}
		else {
			if (A.surname[i] != B.surname[i]) {
				return false;
			}
		}
	}
	return true;
}

SCENARIO("Sort", "[s]") {
	//setlocale(LC_ALL, "ru_RU.utf8");
	std::ifstream ru(russian_letters.txt");
	for (size_t i = 0; i < 6; i++) {
		std::string str;
		ru >> str;
		russian_letters[i] = str[0];
	}
	ru.close();
	size_t n_persons = 2001;
	size_t RAM_amount = 10000;
	std::string names_file_name = "names.txt";
	std::string surnames_file_name = "surnames.txt";
	std::string database_file_name = "database.txt";
	std::string output_file_name = "sorted_database.txt";
	{
		Database_Creator database_creator(database_file_name, names_file_name, surnames_file_name, n_persons);
		Database_Sorter database_sorter(database_file_name, output_file_name, RAM_amount);
		size_t start = clock();
		database_sorter.sortDatabase();
		size_t result = clock() - start;
		std::cout << result << std::endl;
		system("pause");
	}
	for (size_t i = 0; i < 20; i++) {
		size_t person1_i = rand() % n_persons;
		size_t person2_i = person1_i + rand() % (n_persons - person1_i);
		person person1 = readPerson(output_file_name, person1_i);
		person person2 = readPerson(output_file_name, person2_i);
		REQUIRE(isANotMoreThanB(person1, person2));
	}
}
<|endoftext|>


<commit_before>/* $Id$ */

// Helper macros for creating chains

TChain* CreateESDChain(const char* aDataDir = "ESDfiles.txt", Int_t aRuns = 20, Int_t offset = 0)
{
  // creates chain of files in a given directory or file containing a list.
  // In case of directory the structure is expected as:
  // <aDataDir>/<dir0>/AliESDs.root
  // <aDataDir>/<dir1>/AliESDs.root
  // ...

  if (!aDataDir)
    return 0;

  Long_t id, size, flags, modtime;
  if (gSystem->GetPathInfo(aDataDir, &id, &size, &flags, &modtime))
  {
    printf("%s not found.\n", aDataDir);
    return 0;
  }

  TChain* chain = new TChain("esdTree");
  TChain* chaingAlice = 0;

  if (flags & 2)
  {
    TString execDir(gSystem->pwd());
    TSystemDirectory* baseDir = new TSystemDirectory(".", aDataDir);
    TList* dirList            = baseDir->GetListOfFiles();
    Int_t nDirs               = dirList->GetEntries();
    gSystem->cd(execDir);

    Int_t count = 0;

    for (Int_t iDir=0; iDir<nDirs; ++iDir)
    {
      TSystemFile* presentDir = (TSystemFile*) dirList->At(iDir);
      if (!presentDir || !presentDir->IsDirectory() || strcmp(presentDir->GetName(), ".") == 0 || strcmp(presentDir->GetName(), "..") == 0)
        continue;

      if (offset > 0)
      {
        --offset;
        continue;
      }

      if (count++ == aRuns)
        break;

      TString presentDirName(aDataDir);
      presentDirName += "/";
      presentDirName += presentDir->GetName();

      chain->Add(presentDirName + "/AliESDs.root/esdTree");
    }
  }
  else
  {
    // Open the input stream
    ifstream in;
    in.open(aDataDir);

    Int_t count = 0;

    // Read the input list of files and add them to the chain
    TString esdfile;
    while(in.good()) {
      in >> esdfile;
      if (!esdfile.Contains("root")) continue; // protection

      if (offset > 0)
      {
        --offset;
        continue;
      }

      if (count++ == aRuns)
        break;

        // add esd file
      chain->Add(esdfile);
    }

    in.close();
  }

  return chain;
}
<commit_msg>added function that writes a lookedup chain back to disk<commit_after>/* $Id$ */

// Helper macros for creating chains

TChain* CreateESDChain(const char* aDataDir = "ESDfiles.txt", Int_t aRuns = 20, Int_t offset = 0)
{
  // creates chain of files in a given directory or file containing a list.
  // In case of directory the structure is expected as:
  // <aDataDir>/<dir0>/AliESDs.root
  // <aDataDir>/<dir1>/AliESDs.root
  // ...

  if (!aDataDir)
    return 0;

  Long_t id, size, flags, modtime;
  if (gSystem->GetPathInfo(aDataDir, &id, &size, &flags, &modtime))
  {
    printf("%s not found.\n", aDataDir);
    return 0;
  }

  TChain* chain = new TChain("esdTree");
  TChain* chaingAlice = 0;

  if (flags & 2)
  {
    TString execDir(gSystem->pwd());
    TSystemDirectory* baseDir = new TSystemDirectory(".", aDataDir);
    TList* dirList            = baseDir->GetListOfFiles();
    Int_t nDirs               = dirList->GetEntries();
    gSystem->cd(execDir);

    Int_t count = 0;

    for (Int_t iDir=0; iDir<nDirs; ++iDir)
    {
      TSystemFile* presentDir = (TSystemFile*) dirList->At(iDir);
      if (!presentDir || !presentDir->IsDirectory() || strcmp(presentDir->GetName(), ".") == 0 || strcmp(presentDir->GetName(), "..") == 0)
        continue;

      if (offset > 0)
      {
        --offset;
        continue;
      }

      if (count++ == aRuns)
        break;

      TString presentDirName(aDataDir);
      presentDirName += "/";
      presentDirName += presentDir->GetName();

      chain->Add(presentDirName + "/AliESDs.root/esdTree");
    }
  }
  else
  {
    // Open the input stream
    ifstream in;
    in.open(aDataDir);

    Int_t count = 0;

    // Read the input list of files and add them to the chain
    TString esdfile;
    while(in.good()) {
      in >> esdfile;
      if (!esdfile.Contains("root")) continue; // protection

      if (offset > 0)
      {
        --offset;
        continue;
      }

      if (count++ == aRuns)
        break;

        // add esd file
      chain->Add(esdfile);
    }

    in.close();
  }

  return chain;
}

void LookupWrite(TChain* chain, const char* target)
{
  // looks up the chain and writes the remaining files to the text file target

  chain->Lookup();

  TObjArray* list = chain->GetListOfFiles();
  TIterator* iter = list->MakeIterator();
  TObject* obj = 0;

  ofstream outfile;
  outfile.open(target);

  while ((obj = iter->Next()))
    outfile << obj->GetTitle() << "#AliESDs.root" << endl;

  outfile.close();

  delete iter;
}
<|endoftext|>


<commit_before>#include <BinaryTree.hpp>
#include <catch.hpp>

SCENARIO("default constructor")
{
	Tree tree;
	REQUIRE(tree.x_() == 0);
	REQUIRE(tree.left_() == NULL);
	REQUIRE(tree.right_() == NULL);
}

SCENARIO("constructor with params")
{

}
SCENARIO("copy constructor")
{

}

SCENARIO("add")
{

}

SCENARIO("search")
{

}

SCENARIO("fIn")
{

}

SCENARIO("fOut")
{

}

SCENARIO("Out")
{

}
<commit_msg>Update init.cpp<commit_after>#include <BinaryTree.hpp>
#include <catch.hpp>

SCENARIO("default constructor")
{
	Tree node;
	REQUIRE(node.x_() == 0);
	REQUIRE(node.left_() == NULL);
	REQUIRE(node.right_() == NULL);
}

SCENARIO("add")
{
	Tree tree1, tree2;
	Node* newEl;
	tree1.fIn("Tree.txt");
	std::ifstream("newEl.txt") >> newEl->x;
	tree1.add(newEl);
	tree2.fIn("Tree+newEl.txt");
	REQUIRE(tree1 == tree2);
}

SCENARIO("search")
{
	Tree tree;
	Node* node;
	tree.fIn("Tree.txt");
	node.fIn("Node.txt");
	REQUIRE(tree.search(7) == node);
}

SCENARIO("fIn")
{
	Tree tree;
	tree.fIn("Tree.txt");
	REQUIRE(tree->x_ == 5);
}

SCENARIO("fOut")
{
	Tree tree1, tree2;
	tree1.fIn("Tree.txt");
	tree2.fIn("TreeOut.txt");
	tree1.fOut("TreeOut.txt");
	tree1.fIn("TreeOut.txt");
	REQUIRE(tree1 == tree2);
}
<|endoftext|>


<commit_before>#include "clustering/administration/main/import.hpp"

#include "arch/io/network.hpp"
#include "clustering/administration/admin_tracker.hpp"
#include "clustering/administration/auto_reconnect.hpp"
#include "clustering/administration/issues/local.hpp"
#include "clustering/administration/logger.hpp"
#include "clustering/administration/main/initial_join.hpp"
#include "clustering/administration/main/json_import.hpp"
#include "clustering/administration/main/watchable_fields.hpp"
#include "clustering/administration/metadata.hpp"
#include "clustering/administration/network_logger.hpp"
#include "clustering/administration/perfmon_collection_repo.hpp"
#include "clustering/administration/proc_stats.hpp"
#include "clustering/administration/sys_stats.hpp"
#include "extproc/pool.hpp"
#include "http/json.hpp"
#include "rpc/connectivity/multiplexer.hpp"
#include "rpc/directory/read_manager.hpp"
#include "rpc/directory/write_manager.hpp"
#include "rpc/semilattice/semilattice_manager.hpp"
#include "rpc/semilattice/view/field.hpp"
#include "utils.hpp"

bool do_json_importation(machine_id_t machine_id,
                         const boost::shared_ptr<semilattice_readwrite_view_t<databases_semilattice_metadata_t> > &databases,
                         const boost::shared_ptr<semilattice_read_view_t<datacenters_semilattice_metadata_t> > &datacenters,
                         const boost::shared_ptr<semilattice_readwrite_view_t<cow_ptr_t<namespaces_semilattice_metadata_t<rdb_protocol_t> > > > &namespaces,
                         namespace_repo_t<rdb_protocol_t> *repo, json_importer_t *importer,
                         json_import_target_t target,
                         signal_t *interruptor);

bool run_json_import(extproc::spawner_t::info_t *spawner_info, std::set<peer_address_t> joins, int ports_port, int ports_client_port, json_import_target_t target, json_importer_t *importer, signal_t *stop_cond) {

    guarantee(spawner_info);
    extproc::pool_group_t extproc_pool_group(spawner_info, extproc::pool_group_t::DEFAULTS);

    machine_id_t machine_id = generate_uuid();

    local_issue_tracker_t local_issue_tracker;
    thread_pool_log_writer_t log_writer(&local_issue_tracker);

    connectivity_cluster_t connectivity_cluster;
    message_multiplexer_t message_multiplexer(&connectivity_cluster);
    message_multiplexer_t::client_t mailbox_manager_client(&message_multiplexer, 'M');
    mailbox_manager_t mailbox_manager(&mailbox_manager_client);
    message_multiplexer_t::client_t::run_t mailbox_manager_client_run(&mailbox_manager_client, &mailbox_manager);

    message_multiplexer_t::client_t semilattice_manager_client(&message_multiplexer, 'S');
    semilattice_manager_t<cluster_semilattice_metadata_t> semilattice_manager_cluster(&semilattice_manager_client, cluster_semilattice_metadata_t());
    message_multiplexer_t::client_t::run_t semilattice_manager_client_run(&semilattice_manager_client, &semilattice_manager_cluster);

    log_server_t log_server(&mailbox_manager, &log_writer);

    stat_manager_t stat_manager(&mailbox_manager);

    metadata_change_handler_t<cluster_semilattice_metadata_t> metadata_change_handler(&mailbox_manager, semilattice_manager_cluster.get_root_view());

    watchable_variable_t<cluster_directory_metadata_t> our_root_directory_variable(
        cluster_directory_metadata_t(
            machine_id,
            get_ips(),
            stat_manager.get_address(),
            metadata_change_handler.get_request_mailbox_address(),
            log_server.get_business_card(),
            PROXY_PEER));

    message_multiplexer_t::client_t directory_manager_client(&message_multiplexer, 'D');
    // TODO(sam): Are we going to use the write manager at all?
    directory_write_manager_t<cluster_directory_metadata_t> directory_write_manager(&directory_manager_client, our_root_directory_variable.get_watchable());
    directory_read_manager_t<cluster_directory_metadata_t> directory_read_manager(connectivity_cluster.get_connectivity_service());
    message_multiplexer_t::client_t::run_t directory_manager_client_run(&directory_manager_client, &directory_read_manager);

    network_logger_t network_logger(
        connectivity_cluster.get_me(),
        directory_read_manager.get_root_view(),
        metadata_field(&cluster_semilattice_metadata_t::machines, semilattice_manager_cluster.get_root_view()));

    message_multiplexer_t::run_t message_multiplexer_run(&message_multiplexer);
    connectivity_cluster_t::run_t connectivity_cluster_run(&connectivity_cluster, ports_port, &message_multiplexer_run, ports_client_port);

    if (0 == ports_port) {
        ports_port = connectivity_cluster_run.get_port();
    } else {
        guarantee(ports_port == connectivity_cluster_run.get_port());
    }
    logINF("Listening for intracluster traffic on port %d.\n", ports_port);

    auto_reconnector_t auto_reconnector(
        &connectivity_cluster,
        &connectivity_cluster_run,
        directory_read_manager.get_root_view()->subview(
            field_getter_t<machine_id_t, cluster_directory_metadata_t>(&cluster_directory_metadata_t::machine_id)),
        metadata_field(&cluster_semilattice_metadata_t::machines, semilattice_manager_cluster.get_root_view()));

    // Skipped field_copier_t, for fun.

    admin_tracker_t admin_tracker(
        semilattice_manager_cluster.get_root_view(), directory_read_manager.get_root_view());

    perfmon_collection_t proc_stats_collection;
    perfmon_membership_t proc_stats_membership(&get_global_perfmon_collection(), &proc_stats_collection, "proc");

    proc_stats_collector_t proc_stats_collector(&proc_stats_collection);

    perfmon_collection_t sys_stats_collection;
    perfmon_membership_t sys_stats_membership(&get_global_perfmon_collection(), &sys_stats_collection, "sys");

    const char *bs_filepath = "";
    sys_stats_collector_t sys_stats_collector(bs_filepath, &sys_stats_collection);

    scoped_ptr_t<initial_joiner_t> initial_joiner;
    if (!joins.empty()) {
        initial_joiner.init(new initial_joiner_t(&connectivity_cluster, &connectivity_cluster_run, joins));
        try {
            wait_interruptible(initial_joiner->get_ready_signal(), stop_cond);
        } catch (interrupted_exc_t) {
            return false;
        }
    }

    perfmon_collection_repo_t perfmon_repo(&get_global_perfmon_collection());

    // Namespace repos

    // (rdb only)

    rdb_protocol_t::context_t rdb_ctx(&extproc_pool_group,
                                      NULL,
                                      semilattice_manager_cluster.get_root_view(),
                                      &directory_read_manager,
                                      machine_id);

    namespace_repo_t<rdb_protocol_t> rdb_namespace_repo(&mailbox_manager,
        directory_read_manager.get_root_view()->subview(
            field_getter_t<namespaces_directory_metadata_t<rdb_protocol_t>, cluster_directory_metadata_t>(&cluster_directory_metadata_t::rdb_namespaces)),
        &rdb_ctx);

    //This is an annoying chicken and egg problem here
    rdb_ctx.ns_repo = &rdb_namespace_repo;

    // TODO: Handle interrupted exceptions?
    return do_json_importation(machine_id,
                               metadata_field(&cluster_semilattice_metadata_t::databases, semilattice_manager_cluster.get_root_view()),
                               metadata_field(&cluster_semilattice_metadata_t::datacenters, semilattice_manager_cluster.get_root_view()),
                               metadata_field(&cluster_semilattice_metadata_t::rdb_namespaces, semilattice_manager_cluster.get_root_view()),
                               &rdb_namespace_repo, importer, target, stop_cond);
}

bool get_or_create_namespace(UNUSED const boost::shared_ptr<semilattice_read_view_t<datacenters_semilattice_metadata_t> > &datacenters,
                             const boost::shared_ptr<semilattice_readwrite_view_t<cow_ptr_t<namespaces_semilattice_metadata_t<rdb_protocol_t> > > > &namespaces,
                             database_id_t db_id,
                             UNUSED boost::optional<std::string> datacenter_name,
                             std::string table_name,
                             boost::optional<std::string> maybe_primary_key,
                             namespace_id_t *namespace_out,
                             std::string *primary_key_out) {
    namespaces_semilattice_metadata_t<rdb_protocol_t> ns = *namespaces->get();
    metadata_searcher_t<namespace_semilattice_metadata_t<rdb_protocol_t> > searcher(&ns.namespaces);
    metadata_search_status_t error;
    std::map<namespace_id_t, deletable_t<namespace_semilattice_metadata_t<rdb_protocol_t> > >::iterator it = searcher.find_uniq(namespace_predicate_t(table_name, db_id), &error);

    if (error == METADATA_SUCCESS) {
        std::string existing_pk = it->second.get().primary_key.get();
        if (maybe_primary_key) {
            if (existing_pk != *maybe_primary_key) {
                *namespace_out = namespace_id_t();
                primary_key_out->clear();
                return false;
            } else {
                *primary_key_out = *maybe_primary_key;
            }
        } else {
            *primary_key_out = existing_pk;
        }

        *namespace_out = it->first;
        return true;
    } else if (error == METADATA_ERR_NONE) {

        // TODO(sam) implement

        *namespace_out = namespace_id_t();
        primary_key_out->clear();
        return false;
    } else {
        *namespace_out = namespace_id_t();
        primary_key_out->clear();
        return false;
    }
}

bool get_or_create_database(machine_id_t us, const boost::shared_ptr<semilattice_readwrite_view_t<databases_semilattice_metadata_t> > &databases, std::string db_name, database_id_t *db_out) {
    std::map<database_id_t, deletable_t<database_semilattice_metadata_t> > dbmap = databases->get().databases;
    metadata_searcher_t<database_semilattice_metadata_t> searcher(&dbmap);

    metadata_search_status_t error;
    std::map<database_id_t, deletable_t<database_semilattice_metadata_t> >::iterator it = searcher.find_uniq(db_name, &error);

    if (error == METADATA_SUCCESS) {
        *db_out = it->first;
        return true;
    } else if (error == METADATA_ERR_NONE) {
        databases_semilattice_metadata_t dbs;
        database_semilattice_metadata_t db;
        db.name = vclock_t<std::string>(db_name, us);
        database_id_t db_id = generate_uuid();
        dbs.databases.insert(std::make_pair(db_id, db));

        databases->join(dbs);

        *db_out = db_id;
        return true;
    } else {
        *db_out = database_id_t();
        return false;
    }
}

bool do_json_importation(machine_id_t machine_id,
                         const boost::shared_ptr<semilattice_readwrite_view_t<databases_semilattice_metadata_t> > &databases,
                         const boost::shared_ptr<semilattice_read_view_t<datacenters_semilattice_metadata_t> > &datacenters,
                         const boost::shared_ptr<semilattice_readwrite_view_t<cow_ptr_t<namespaces_semilattice_metadata_t<rdb_protocol_t> > > > &namespaces,
                         namespace_repo_t<rdb_protocol_t> *repo, json_importer_t *importer,
                         json_import_target_t target,
                         signal_t *interruptor) {
    std::string db_name = target.db_name;
    boost::optional<std::string> datacenter_name = target.datacenter_name;
    std::string table_name = target.table_name;
    boost::optional<std::string> maybe_primary_key = target.primary_key;

    database_id_t db_id;
    if (!get_or_create_database(machine_id, databases, db_name, &db_id)) {
        debugf("could not get or create database named '%s'\n", db_name.c_str());
        return false;
    }

    namespace_id_t namespace_id;
    std::string primary_key;
    if (!get_or_create_namespace(datacenters, namespaces, db_id, datacenter_name, table_name, maybe_primary_key, &namespace_id, &primary_key)) {
        debugf("could not get or create namespace named '%s' (in db '%s')\n", table_name.c_str(), uuid_to_str(db_id).c_str());
        return false;
    }

    // TODO(sam): What if construction fails?  An exception is thrown?
    namespace_repo_t<rdb_protocol_t>::access_t access(repo, namespace_id, interruptor);

    namespace_interface_t<rdb_protocol_t> *ni = access.get_namespace_if();

    order_source_t order_source;

    for (scoped_cJSON_t json; importer->get_json(&json); json.reset(NULL)) {
        debugf("json: %s\n", json.Print().c_str());

        cJSON *pkey_value = json.GetObjectItem(primary_key.c_str());
        if (!pkey_value) {
            debugf("pkey_value NULL\n");
            // TODO(sam): We want to silently ignore?
            continue;
        }

        if (pkey_value->type != cJSON_String && pkey_value->type != cJSON_Number) {
            debugf("pkey_value->type is bad\n");
            // TODO(sam): We want to silently ignore?
            continue;
        }

        std::string internal_key = cJSON_print_lexicographic(pkey_value);

        if (internal_key.size() > MAX_KEY_SIZE) {
            debugf("key size is overlarge\n");
            // TODO(sam): We want to silently ignore?
            continue;
        }

        store_key_t key(internal_key);

        boost::shared_ptr<scoped_cJSON_t> json_copy_fml(new scoped_cJSON_t(json.DeepCopy()));

        rdb_protocol_t::point_write_t point_write(key, json_copy_fml, true);
        rdb_protocol_t::write_t rdb_write;
        rdb_write.write = point_write;
        rdb_protocol_t::write_response_t response;
        ni->write(rdb_write, &response, order_source.check_in("do_json_importation"), interruptor);

        if (!boost::get<rdb_protocol_t::point_write_response_t>(&response.response)) {
            debugf("did not get a point write response\n");
        } else {
            rdb_protocol_t::point_write_response_t *resp = boost::get<rdb_protocol_t::point_write_response_t>(&response.response);
            debugf("point write response: %s\n", resp->result == STORED ? "STORED" : resp->result == DUPLICATE ? "DUPLICATE" : "???");
        }

        // We don't care about the response.
    }

    debugf("do_json_importation ... returning bogus success!\n");
    return true;
}
<commit_msg>Simplified branching logic in get_or_create_namespace.<commit_after>#include "clustering/administration/main/import.hpp"

#include "arch/io/network.hpp"
#include "clustering/administration/admin_tracker.hpp"
#include "clustering/administration/auto_reconnect.hpp"
#include "clustering/administration/issues/local.hpp"
#include "clustering/administration/logger.hpp"
#include "clustering/administration/main/initial_join.hpp"
#include "clustering/administration/main/json_import.hpp"
#include "clustering/administration/main/watchable_fields.hpp"
#include "clustering/administration/metadata.hpp"
#include "clustering/administration/network_logger.hpp"
#include "clustering/administration/perfmon_collection_repo.hpp"
#include "clustering/administration/proc_stats.hpp"
#include "clustering/administration/sys_stats.hpp"
#include "extproc/pool.hpp"
#include "http/json.hpp"
#include "rpc/connectivity/multiplexer.hpp"
#include "rpc/directory/read_manager.hpp"
#include "rpc/directory/write_manager.hpp"
#include "rpc/semilattice/semilattice_manager.hpp"
#include "rpc/semilattice/view/field.hpp"
#include "utils.hpp"

bool do_json_importation(machine_id_t machine_id,
                         const boost::shared_ptr<semilattice_readwrite_view_t<databases_semilattice_metadata_t> > &databases,
                         const boost::shared_ptr<semilattice_read_view_t<datacenters_semilattice_metadata_t> > &datacenters,
                         const boost::shared_ptr<semilattice_readwrite_view_t<cow_ptr_t<namespaces_semilattice_metadata_t<rdb_protocol_t> > > > &namespaces,
                         namespace_repo_t<rdb_protocol_t> *repo, json_importer_t *importer,
                         json_import_target_t target,
                         signal_t *interruptor);

bool run_json_import(extproc::spawner_t::info_t *spawner_info, std::set<peer_address_t> joins, int ports_port, int ports_client_port, json_import_target_t target, json_importer_t *importer, signal_t *stop_cond) {

    guarantee(spawner_info);
    extproc::pool_group_t extproc_pool_group(spawner_info, extproc::pool_group_t::DEFAULTS);

    machine_id_t machine_id = generate_uuid();

    local_issue_tracker_t local_issue_tracker;
    thread_pool_log_writer_t log_writer(&local_issue_tracker);

    connectivity_cluster_t connectivity_cluster;
    message_multiplexer_t message_multiplexer(&connectivity_cluster);
    message_multiplexer_t::client_t mailbox_manager_client(&message_multiplexer, 'M');
    mailbox_manager_t mailbox_manager(&mailbox_manager_client);
    message_multiplexer_t::client_t::run_t mailbox_manager_client_run(&mailbox_manager_client, &mailbox_manager);

    message_multiplexer_t::client_t semilattice_manager_client(&message_multiplexer, 'S');
    semilattice_manager_t<cluster_semilattice_metadata_t> semilattice_manager_cluster(&semilattice_manager_client, cluster_semilattice_metadata_t());
    message_multiplexer_t::client_t::run_t semilattice_manager_client_run(&semilattice_manager_client, &semilattice_manager_cluster);

    log_server_t log_server(&mailbox_manager, &log_writer);

    stat_manager_t stat_manager(&mailbox_manager);

    metadata_change_handler_t<cluster_semilattice_metadata_t> metadata_change_handler(&mailbox_manager, semilattice_manager_cluster.get_root_view());

    watchable_variable_t<cluster_directory_metadata_t> our_root_directory_variable(
        cluster_directory_metadata_t(
            machine_id,
            get_ips(),
            stat_manager.get_address(),
            metadata_change_handler.get_request_mailbox_address(),
            log_server.get_business_card(),
            PROXY_PEER));

    message_multiplexer_t::client_t directory_manager_client(&message_multiplexer, 'D');
    // TODO(sam): Are we going to use the write manager at all?
    directory_write_manager_t<cluster_directory_metadata_t> directory_write_manager(&directory_manager_client, our_root_directory_variable.get_watchable());
    directory_read_manager_t<cluster_directory_metadata_t> directory_read_manager(connectivity_cluster.get_connectivity_service());
    message_multiplexer_t::client_t::run_t directory_manager_client_run(&directory_manager_client, &directory_read_manager);

    network_logger_t network_logger(
        connectivity_cluster.get_me(),
        directory_read_manager.get_root_view(),
        metadata_field(&cluster_semilattice_metadata_t::machines, semilattice_manager_cluster.get_root_view()));

    message_multiplexer_t::run_t message_multiplexer_run(&message_multiplexer);
    connectivity_cluster_t::run_t connectivity_cluster_run(&connectivity_cluster, ports_port, &message_multiplexer_run, ports_client_port);

    if (0 == ports_port) {
        ports_port = connectivity_cluster_run.get_port();
    } else {
        guarantee(ports_port == connectivity_cluster_run.get_port());
    }
    logINF("Listening for intracluster traffic on port %d.\n", ports_port);

    auto_reconnector_t auto_reconnector(
        &connectivity_cluster,
        &connectivity_cluster_run,
        directory_read_manager.get_root_view()->subview(
            field_getter_t<machine_id_t, cluster_directory_metadata_t>(&cluster_directory_metadata_t::machine_id)),
        metadata_field(&cluster_semilattice_metadata_t::machines, semilattice_manager_cluster.get_root_view()));

    // Skipped field_copier_t, for fun.

    admin_tracker_t admin_tracker(
        semilattice_manager_cluster.get_root_view(), directory_read_manager.get_root_view());

    perfmon_collection_t proc_stats_collection;
    perfmon_membership_t proc_stats_membership(&get_global_perfmon_collection(), &proc_stats_collection, "proc");

    proc_stats_collector_t proc_stats_collector(&proc_stats_collection);

    perfmon_collection_t sys_stats_collection;
    perfmon_membership_t sys_stats_membership(&get_global_perfmon_collection(), &sys_stats_collection, "sys");

    const char *bs_filepath = "";
    sys_stats_collector_t sys_stats_collector(bs_filepath, &sys_stats_collection);

    scoped_ptr_t<initial_joiner_t> initial_joiner;
    if (!joins.empty()) {
        initial_joiner.init(new initial_joiner_t(&connectivity_cluster, &connectivity_cluster_run, joins));
        try {
            wait_interruptible(initial_joiner->get_ready_signal(), stop_cond);
        } catch (interrupted_exc_t) {
            return false;
        }
    }

    perfmon_collection_repo_t perfmon_repo(&get_global_perfmon_collection());

    // Namespace repos

    // (rdb only)

    rdb_protocol_t::context_t rdb_ctx(&extproc_pool_group,
                                      NULL,
                                      semilattice_manager_cluster.get_root_view(),
                                      &directory_read_manager,
                                      machine_id);

    namespace_repo_t<rdb_protocol_t> rdb_namespace_repo(&mailbox_manager,
        directory_read_manager.get_root_view()->subview(
            field_getter_t<namespaces_directory_metadata_t<rdb_protocol_t>, cluster_directory_metadata_t>(&cluster_directory_metadata_t::rdb_namespaces)),
        &rdb_ctx);

    //This is an annoying chicken and egg problem here
    rdb_ctx.ns_repo = &rdb_namespace_repo;

    // TODO: Handle interrupted exceptions?
    return do_json_importation(machine_id,
                               metadata_field(&cluster_semilattice_metadata_t::databases, semilattice_manager_cluster.get_root_view()),
                               metadata_field(&cluster_semilattice_metadata_t::datacenters, semilattice_manager_cluster.get_root_view()),
                               metadata_field(&cluster_semilattice_metadata_t::rdb_namespaces, semilattice_manager_cluster.get_root_view()),
                               &rdb_namespace_repo, importer, target, stop_cond);
}

bool get_or_create_namespace(UNUSED const boost::shared_ptr<semilattice_read_view_t<datacenters_semilattice_metadata_t> > &datacenters,
                             const boost::shared_ptr<semilattice_readwrite_view_t<cow_ptr_t<namespaces_semilattice_metadata_t<rdb_protocol_t> > > > &namespaces,
                             database_id_t db_id,
                             UNUSED boost::optional<std::string> datacenter_name,
                             std::string table_name,
                             boost::optional<std::string> maybe_primary_key,
                             namespace_id_t *namespace_out,
                             std::string *primary_key_out) {
    namespaces_semilattice_metadata_t<rdb_protocol_t> ns = *namespaces->get();
    metadata_searcher_t<namespace_semilattice_metadata_t<rdb_protocol_t> > searcher(&ns.namespaces);
    metadata_search_status_t error;
    std::map<namespace_id_t, deletable_t<namespace_semilattice_metadata_t<rdb_protocol_t> > >::iterator it = searcher.find_uniq(namespace_predicate_t(table_name, db_id), &error);

    if (error == METADATA_SUCCESS) {
        std::string existing_pk = it->second.get().primary_key.get();
        if (maybe_primary_key) {
            if (existing_pk != *maybe_primary_key) {
                *namespace_out = namespace_id_t();
                primary_key_out->clear();
                return false;
            }
        }
        *primary_key_out = existing_pk;
        *namespace_out = it->first;
        return true;
    } else if (error == METADATA_ERR_NONE) {
        //

        // TODO(sam) implement

        *namespace_out = namespace_id_t();
        primary_key_out->clear();
        return false;
    } else {
        *namespace_out = namespace_id_t();
        primary_key_out->clear();
        return false;
    }
}

bool get_or_create_database(machine_id_t us, const boost::shared_ptr<semilattice_readwrite_view_t<databases_semilattice_metadata_t> > &databases, std::string db_name, database_id_t *db_out) {
    std::map<database_id_t, deletable_t<database_semilattice_metadata_t> > dbmap = databases->get().databases;
    metadata_searcher_t<database_semilattice_metadata_t> searcher(&dbmap);

    metadata_search_status_t error;
    std::map<database_id_t, deletable_t<database_semilattice_metadata_t> >::iterator it = searcher.find_uniq(db_name, &error);

    if (error == METADATA_SUCCESS) {
        *db_out = it->first;
        return true;
    } else if (error == METADATA_ERR_NONE) {
        databases_semilattice_metadata_t dbs;
        database_semilattice_metadata_t db;
        db.name = vclock_t<std::string>(db_name, us);
        database_id_t db_id = generate_uuid();
        dbs.databases.insert(std::make_pair(db_id, db));

        databases->join(dbs);

        *db_out = db_id;
        return true;
    } else {
        *db_out = database_id_t();
        return false;
    }
}

bool do_json_importation(machine_id_t machine_id,
                         const boost::shared_ptr<semilattice_readwrite_view_t<databases_semilattice_metadata_t> > &databases,
                         const boost::shared_ptr<semilattice_read_view_t<datacenters_semilattice_metadata_t> > &datacenters,
                         const boost::shared_ptr<semilattice_readwrite_view_t<cow_ptr_t<namespaces_semilattice_metadata_t<rdb_protocol_t> > > > &namespaces,
                         namespace_repo_t<rdb_protocol_t> *repo, json_importer_t *importer,
                         json_import_target_t target,
                         signal_t *interruptor) {
    std::string db_name = target.db_name;
    boost::optional<std::string> datacenter_name = target.datacenter_name;
    std::string table_name = target.table_name;
    boost::optional<std::string> maybe_primary_key = target.primary_key;

    database_id_t db_id;
    if (!get_or_create_database(machine_id, databases, db_name, &db_id)) {
        debugf("could not get or create database named '%s'\n", db_name.c_str());
        return false;
    }

    namespace_id_t namespace_id;
    std::string primary_key;
    if (!get_or_create_namespace(datacenters, namespaces, db_id, datacenter_name, table_name, maybe_primary_key, &namespace_id, &primary_key)) {
        debugf("could not get or create namespace named '%s' (in db '%s')\n", table_name.c_str(), uuid_to_str(db_id).c_str());
        return false;
    }

    // TODO(sam): What if construction fails?  An exception is thrown?
    namespace_repo_t<rdb_protocol_t>::access_t access(repo, namespace_id, interruptor);

    namespace_interface_t<rdb_protocol_t> *ni = access.get_namespace_if();

    order_source_t order_source;

    for (scoped_cJSON_t json; importer->get_json(&json); json.reset(NULL)) {
        debugf("json: %s\n", json.Print().c_str());

        cJSON *pkey_value = json.GetObjectItem(primary_key.c_str());
        if (!pkey_value) {
            debugf("pkey_value NULL\n");
            // TODO(sam): We want to silently ignore?
            continue;
        }

        if (pkey_value->type != cJSON_String && pkey_value->type != cJSON_Number) {
            debugf("pkey_value->type is bad\n");
            // TODO(sam): We want to silently ignore?
            continue;
        }

        std::string internal_key = cJSON_print_lexicographic(pkey_value);

        if (internal_key.size() > MAX_KEY_SIZE) {
            debugf("key size is overlarge\n");
            // TODO(sam): We want to silently ignore?
            continue;
        }

        store_key_t key(internal_key);

        boost::shared_ptr<scoped_cJSON_t> json_copy_fml(new scoped_cJSON_t(json.DeepCopy()));

        rdb_protocol_t::point_write_t point_write(key, json_copy_fml, true);
        rdb_protocol_t::write_t rdb_write;
        rdb_write.write = point_write;
        rdb_protocol_t::write_response_t response;
        ni->write(rdb_write, &response, order_source.check_in("do_json_importation"), interruptor);

        if (!boost::get<rdb_protocol_t::point_write_response_t>(&response.response)) {
            debugf("did not get a point write response\n");
        } else {
            rdb_protocol_t::point_write_response_t *resp = boost::get<rdb_protocol_t::point_write_response_t>(&response.response);
            debugf("point write response: %s\n", resp->result == STORED ? "STORED" : resp->result == DUPLICATE ? "DUPLICATE" : "???");
        }

        // We don't care about the response.
    }

    debugf("do_json_importation ... returning bogus success!\n");
    return true;
}
<|endoftext|>


<commit_before>#include "clustering/reactor/reactor.hpp"

#include "clustering/immediate_consistency/branch/backfiller.hpp"
#include "clustering/immediate_consistency/branch/multistore.hpp"
#include "clustering/immediate_consistency/branch/replier.hpp"
#include "concurrency/cross_thread_signal.hpp"

/* Returns true if every peer listed as a primary for this shard in the
 * blueprint has activity primary_t and every peer listed as a secondary has
 * activity secondary_up_to_date_t. */
template <class protocol_t>
bool reactor_t<protocol_t>::is_safe_for_us_to_be_nothing(const std::map<peer_id_t, boost::optional<directory_echo_wrapper_t<reactor_business_card_t<protocol_t> > > > &_reactor_directory, const blueprint_t<protocol_t> &blueprint,
                                                         const typename protocol_t::region_t &region)
{
    typedef reactor_business_card_t<protocol_t> rb_t;

    /* Iterator through the peers the blueprint claims we should be able to
     * see. */
    for (typename blueprint_t<protocol_t>::role_map_t::const_iterator p_it =  blueprint.peers_roles.begin();
                                                                      p_it != blueprint.peers_roles.end();
                                                                      p_it++) {
        typename std::map<peer_id_t, boost::optional<directory_echo_wrapper_t<reactor_business_card_t<protocol_t> > > >::const_iterator bcard_it = _reactor_directory.find(p_it->first);
        if (bcard_it == _reactor_directory.end() || !bcard_it->second) {
            //The peer is down or has no reactor
            return false;
        }

        typename blueprint_t<protocol_t>::region_to_role_map_t::const_iterator r_it = p_it->second.find(region);
        rassert(r_it != p_it->second.end(), "Invalid blueprint issued, different peers have different sharding schemes.\n");

        /* Whether or not we found a directory entry for this peer */
        bool found = false;
        for (typename rb_t::activity_map_t::const_iterator it =  (*bcard_it->second).internal.activities.begin();
                                                           it != (*bcard_it->second).internal.activities.end();
                                                           it++) {
            if (it->second.first == region) {
                if (r_it->second == blueprint_role_primary) {
                    if (!boost::get<typename rb_t::primary_t>(&it->second.second)) {
                        return false;
                    }
                } else if (r_it->second == blueprint_role_secondary) {
                    if (!boost::get<typename rb_t::secondary_up_to_date_t>(&it->second.second)) {
                        return false;
                    }
                }
                found = true;
                break;
            }
        }

        if (!found) {
            return false;
        }
    }

    return true;
}

template<class protocol_t>
void reactor_t<protocol_t>::be_nothing(typename protocol_t::region_t region,
        store_view_t<protocol_t> *svs, const clone_ptr_t<watchable_t<blueprint_t<protocol_t> > > &blueprint,
        signal_t *interruptor) THROWS_NOTHING {
    try {
        directory_entry_t directory_entry(this, region);

        {
            cross_thread_signal_t ct_interruptor(interruptor, svs->home_thread());
            on_thread_t th(svs->home_thread());

            order_source_t order_source;  // TODO: order_token_t::ignore

            /* We offer backfills while waiting for it to be safe to shutdown
             * in case another peer needs a copy of the data */
            backfiller_t<protocol_t> backfiller(mailbox_manager, branch_history_manager, svs);

            /* Tell the other peers that we are looking to shutdown and
             * offering backfilling until we do. */
            scoped_ptr_t<fifo_enforcer_sink_t::exit_read_t> read_token;
            svs->new_read_token(&read_token);
            region_map_t<protocol_t, binary_blob_t> metainfo_blob;
            svs->do_get_metainfo(order_source.check_in("be_nothing").with_read_mode(), &read_token, &ct_interruptor, &metainfo_blob);
            typename reactor_business_card_t<protocol_t>::nothing_when_safe_t
                activity(to_version_range_map(metainfo_blob), backfiller.get_business_card());

            on_thread_t th2(this->home_thread());
            directory_echo_version_t version_to_wait_on = directory_entry.set(activity);

            /* Make sure everyone sees that we're trying to erase our data,
             * it's important to do this to avoid the following race condition:
             *
             * Peer 1 and Peer 2 both are secondaries.
             * Peer 1 gets a blueprint saying its role is nothing and peer 2's is secondary,
             * Peer 2 gets a blueprint saying its role is nothing and peer 1's is secondary,
             *
             * since each one sees the other is secondary they both think it's
             * safe to shutdown and thus destroy their data leading to data
             * loss.
             *
             * The below line makes sure that they will sync their new roles
             * with one another before they begin destroying data.
             *
             * This makes it possible for either to proceed with deleting the
             * data, but never both, it's also possible that neither proceeds
             * which is okay as well.
             */
            wait_for_directory_acks(version_to_wait_on, interruptor);

            /* Make sure we don't go down and delete the data on our machine
             * before every who needs a copy has it. */
            run_until_satisfied_2(
                reactor_directory,
                blueprint,
                boost::bind(&reactor_t<protocol_t>::is_safe_for_us_to_be_nothing, this, _1, _2, region),
                interruptor);
        }

        /* We now know that it's safe to shutdown so we tell the other peers
         * that we are beginning the process of erasing data. */
        directory_entry.set(typename reactor_business_card_t<protocol_t>::nothing_when_done_erasing_t());

        /* This actually erases the data. */
        {
            scoped_ptr_t<fifo_enforcer_sink_t::exit_write_t> write_token;
            svs->new_write_token(&write_token);

            svs->reset_data(region, region_map_t<protocol_t, binary_blob_t>(region, binary_blob_t(version_range_t(version_t::zero()))), &write_token, interruptor);
        }

        /* Tell the other peers that we are officially nothing for this region,
         * end of story. */
        directory_entry.set(typename reactor_business_card_t<protocol_t>::nothing_t());

        interruptor->wait_lazily_unordered();
    } catch (interrupted_exc_t) {
        /* ignore */
    }
}

#include "mock/dummy_protocol.hpp"
#include "memcached/protocol.hpp"
#include "rdb_protocol/protocol.hpp"

template class reactor_t<mock::dummy_protocol_t>;
template class reactor_t<memcached_protocol_t>;
template class reactor_t<rdb_protocol_t>;
<commit_msg>Spurious whitespace changes in reactor_be_nothing.cc and the like.<commit_after>#include "clustering/reactor/reactor.hpp"

#include "clustering/immediate_consistency/branch/backfiller.hpp"
#include "clustering/immediate_consistency/branch/multistore.hpp"
#include "clustering/immediate_consistency/branch/replier.hpp"
#include "concurrency/cross_thread_signal.hpp"

/* Returns true if every peer listed as a primary for this shard in the
 * blueprint has activity primary_t and every peer listed as a secondary has
 * activity secondary_up_to_date_t. */
template <class protocol_t>
bool reactor_t<protocol_t>::is_safe_for_us_to_be_nothing(const std::map<peer_id_t, boost::optional<directory_echo_wrapper_t<reactor_business_card_t<protocol_t> > > > &_reactor_directory, const blueprint_t<protocol_t> &blueprint,
                                                         const typename protocol_t::region_t &region) {
    /* Iterator through the peers the blueprint claims we should be able to
     * see. */
    for (typename std::map<peer_id_t, std::map<typename protocol_t::region_t, blueprint_role_t> >::const_iterator p_it = blueprint.peers_roles.begin();
         p_it != blueprint.peers_roles.end();
         ++p_it) {
        typename std::map<peer_id_t, boost::optional<directory_echo_wrapper_t<reactor_business_card_t<protocol_t> > > >::const_iterator bcard_it = _reactor_directory.find(p_it->first);
        if (bcard_it == _reactor_directory.end() || !bcard_it->second) {
            //The peer is down or has no reactor
            return false;
        }

        typename std::map<typename protocol_t::region_t, blueprint_role_t>::const_iterator r_it = p_it->second.find(region);
        rassert(r_it != p_it->second.end(), "Invalid blueprint issued, different peers have different sharding schemes.\n");

        /* Whether or not we found a directory entry for this peer */
        bool found = false;
        for (typename reactor_business_card_t<protocol_t>::activity_map_t::const_iterator it = (*bcard_it->second).internal.activities.begin();
             it != bcard_it->second->internal.activities.end();
             ++it) {
            if (it->second.first == region) {
                if (r_it->second == blueprint_role_primary) {
                    if (!boost::get<typename reactor_business_card_t<protocol_t>::primary_t>(&it->second.second)) {
                        return false;
                    }
                } else if (r_it->second == blueprint_role_secondary) {
                    if (!boost::get<typename reactor_business_card_t<protocol_t>::secondary_up_to_date_t>(&it->second.second)) {
                        return false;
                    }
                }
                found = true;
                break;
            }
        }

        if (!found) {
            return false;
        }
    }

    return true;
}

template<class protocol_t>
void reactor_t<protocol_t>::be_nothing(typename protocol_t::region_t region,
        store_view_t<protocol_t> *svs, const clone_ptr_t<watchable_t<blueprint_t<protocol_t> > > &blueprint,
        signal_t *interruptor) THROWS_NOTHING {
    try {
        directory_entry_t directory_entry(this, region);

        {
            cross_thread_signal_t ct_interruptor(interruptor, svs->home_thread());
            on_thread_t th(svs->home_thread());

            order_source_t order_source;  // TODO: order_token_t::ignore

            /* We offer backfills while waiting for it to be safe to shutdown
             * in case another peer needs a copy of the data */
            backfiller_t<protocol_t> backfiller(mailbox_manager, branch_history_manager, svs);

            /* Tell the other peers that we are looking to shutdown and
             * offering backfilling until we do. */
            scoped_ptr_t<fifo_enforcer_sink_t::exit_read_t> read_token;
            svs->new_read_token(&read_token);
            region_map_t<protocol_t, binary_blob_t> metainfo_blob;
            svs->do_get_metainfo(order_source.check_in("be_nothing").with_read_mode(), &read_token, &ct_interruptor, &metainfo_blob);
            typename reactor_business_card_t<protocol_t>::nothing_when_safe_t
                activity(to_version_range_map(metainfo_blob), backfiller.get_business_card());

            on_thread_t th2(this->home_thread());
            directory_echo_version_t version_to_wait_on = directory_entry.set(activity);

            /* Make sure everyone sees that we're trying to erase our data,
             * it's important to do this to avoid the following race condition:
             *
             * Peer 1 and Peer 2 both are secondaries.
             * Peer 1 gets a blueprint saying its role is nothing and peer 2's is secondary,
             * Peer 2 gets a blueprint saying its role is nothing and peer 1's is secondary,
             *
             * since each one sees the other is secondary they both think it's
             * safe to shutdown and thus destroy their data leading to data
             * loss.
             *
             * The below line makes sure that they will sync their new roles
             * with one another before they begin destroying data.
             *
             * This makes it possible for either to proceed with deleting the
             * data, but never both, it's also possible that neither proceeds
             * which is okay as well.
             */
            wait_for_directory_acks(version_to_wait_on, interruptor);

            /* Make sure we don't go down and delete the data on our machine
             * before every who needs a copy has it. */
            run_until_satisfied_2(
                reactor_directory,
                blueprint,
                boost::bind(&reactor_t<protocol_t>::is_safe_for_us_to_be_nothing, this, _1, _2, region),
                interruptor);
        }

        /* We now know that it's safe to shutdown so we tell the other peers
         * that we are beginning the process of erasing data. */
        directory_entry.set(typename reactor_business_card_t<protocol_t>::nothing_when_done_erasing_t());

        /* This actually erases the data. */
        {
            scoped_ptr_t<fifo_enforcer_sink_t::exit_write_t> write_token;
            svs->new_write_token(&write_token);

            svs->reset_data(region, region_map_t<protocol_t, binary_blob_t>(region, binary_blob_t(version_range_t(version_t::zero()))), &write_token, interruptor);
        }

        /* Tell the other peers that we are officially nothing for this region,
         * end of story. */
        directory_entry.set(typename reactor_business_card_t<protocol_t>::nothing_t());

        interruptor->wait_lazily_unordered();
    } catch (interrupted_exc_t) {
        /* ignore */
    }
}

#include "mock/dummy_protocol.hpp"
#include "memcached/protocol.hpp"
#include "rdb_protocol/protocol.hpp"

template class reactor_t<mock::dummy_protocol_t>;
template class reactor_t<memcached_protocol_t>;
template class reactor_t<rdb_protocol_t>;
<|endoftext|>


<commit_before>//
//  Z80.cpp
//  Clock Signal
//
//  Created by Thomas Harte on 25/04/2021.
//  Copyright © 2021 Thomas Harte. All rights reserved.
//

#include "Z80.hpp"

#include "../FileHolder.hpp"

#include "../../Analyser/Static/ZXSpectrum/Target.hpp"
#include "../../Machines/Sinclair/ZXSpectrum/State.hpp"

using namespace Storage::State;

namespace {

std::vector<uint8_t> read_memory(Storage::FileHolder &file, size_t size, bool is_compressed) {
	if(!is_compressed) {
		return file.read(size);
	}

	std::vector<uint8_t> result(size);
	size_t cursor = 0;

	uint8_t incoming[2] = { file.get8(), file.get8()};
	while(true) {
		if(incoming[0] == 0xed && incoming[1] == 0xed) {
			const uint8_t count = file.get8();
			const uint8_t value = file.get8();

			memset(&result[cursor], value, count);
			cursor += count;
			if(cursor == size) break;
			incoming[0] = file.get8();
			incoming[1] = file.get8();
		} else {
			result[cursor] = incoming[0];
			++cursor;
			if(cursor == size) break;
			incoming[0] = incoming[1];
			incoming[1] = file.get8();
		}
	}

	return result;
}

}

std::unique_ptr<Analyser::Static::Target> Z80::load(const std::string &file_name) {
	FileHolder file(file_name);

	// Construct a target with a Spectrum state.
	using Target = Analyser::Static::ZXSpectrum::Target;
	auto result = std::make_unique<Target>();
	auto *const state = new Sinclair::ZXSpectrum::State();
	result->state = std::unique_ptr<Reflection::Struct>(state);

	// Read version 1 header.
	state->z80.registers.a = file.get8();
	state->z80.registers.flags = file.get8();
	state->z80.registers.bc = file.get16le();
	state->z80.registers.hl = file.get16le();
	state->z80.registers.program_counter = file.get16le();
	state->z80.registers.stack_pointer = file.get16le();
	state->z80.registers.ir = file.get16be();	// Stored I then R.

	// Bit 7 of R is stored separately; likely this relates to an
	// optimisation in the Z80 emulator that for some reason was
	// exported into its file format.
	const uint8_t raw_misc = file.get8();
	const uint8_t misc = (raw_misc == 0xff) ? 1 : raw_misc;
	state->z80.registers.ir = uint16_t((state->z80.registers.ir & ~0x80) | ((misc&1) << 7));

	state->z80.registers.de = file.get16le();
	state->z80.registers.bc_dash = file.get16le();
	state->z80.registers.de_dash = file.get16le();
	state->z80.registers.hl_dash = file.get16le();
	state->z80.registers.af_dash = file.get16be();	// Stored A' then F'.
	state->z80.registers.iy = file.get16le();
	state->z80.registers.ix = file.get16le();
	state->z80.registers.iff1 = bool(file.get8());
	state->z80.registers.iff2 = bool(file.get8());

	// Ignored from the next byte:
	//
	//	bit 2 = 1 	=> issue 2 emulation
	//	bit 3 = 1	=> double interrupt frequency (?)
	//	bit 4–5		=> video synchronisation (to do with emulation hackery?)
	//	bit 6–7		=> joystick type
	state->z80.registers.interrupt_mode = file.get8() & 3;

	// If the program counter is non-0 then this is a version 1 snapshot,
	// which means it's definitely a 48k image.
	if(state->z80.registers.program_counter) {
		result->model = Target::Model::FortyEightK;
		state->ram = read_memory(file, 48*1024, misc & 0x20);
		return result;
	}

	// This was a version 1 or 2 snapshot, so keep going...
	const uint16_t bonus_header_size = file.get16le();
	if(bonus_header_size != 23 && bonus_header_size != 54 && bonus_header_size != 55) {
		return nullptr;
	}

	state->z80.registers.program_counter = file.get16le();
	switch(file.get8()) {
		default: return nullptr;
		case 0:		result->model = Target::Model::FortyEightK;		break;
		case 3:		result->model = Target::Model::OneTwoEightK;	break;
		case 7:
		case 8:		result->model = Target::Model::Plus3;			break;
		case 12:	result->model = Target::Model::Plus2;			break;
		case 13:	result->model = Target::Model::Plus2a;			break;
	}

	const uint8_t last7ffd = file.get8();	(void)last7ffd;	// TODO

	file.seek(1, SEEK_CUR);
	if(file.get8() & 0x80) {
		// The 'hardware modify' bit, which inexplicably does this:
		switch(result->model) {
			default: break;
			case Target::Model::FortyEightK:	result->model = Target::Model::SixteenK;	break;
			case Target::Model::OneTwoEightK:	result->model = Target::Model::Plus2;		break;
			case Target::Model::Plus3:			result->model = Target::Model::Plus2a;		break;
		}
	}

	const uint8_t lastfffd = file.get8();	(void)lastfffd;	// TODO
	file.seek(16, SEEK_CUR);	// Sound chip registers: TODO.

	if(bonus_header_size != 23) {
		// More Z80, the emulator, lack of encapsulation to deal with here.
		const uint16_t low_t_state = file.get16le();
		const uint16_t high_t_state = file.get8();
		int time_since_interrupt;
		switch(result->model) {
			case Target::Model::SixteenK:
			case Target::Model::FortyEightK:
				time_since_interrupt = (17471 - low_t_state) + (high_t_state * 17472);
			break;

			default:
				time_since_interrupt = (17726 - low_t_state) + (high_t_state * 17727);
			break;
		}
		// TODO: map time_since_interrupt to time_into_frame, somehow.

		// Skip: Spectator flag, MGT, Multiface and other ROM flags.
		file.seek(5, SEEK_CUR);

		// Skip: highly Z80-the-emulator-specific stuff about user-defined joystick.
		file.seek(20, SEEK_CUR);

		// Skip: Disciple/Plus D stuff.
		file.seek(3, SEEK_CUR);

		if(bonus_header_size == 55) {
			const uint8_t last1ffd = file.get8();	(void)last1ffd;	// TODO
		}
	}

	// Grab RAM.
	state->ram.resize(128 * 1024);
	while(true) {
		const uint16_t block_size = file.get16le();
		const uint8_t page = file.get8();
		const auto location = file.tell();
		if(file.eof()) break;

		switch(page) {
			default: break;
		}

		file.seek(location + block_size, SEEK_SET);
	}

	return result;
}
<commit_msg>Adds RAM deserialisation.<commit_after>//
//  Z80.cpp
//  Clock Signal
//
//  Created by Thomas Harte on 25/04/2021.
//  Copyright © 2021 Thomas Harte. All rights reserved.
//

#include "Z80.hpp"

#include "../FileHolder.hpp"

#include "../../Analyser/Static/ZXSpectrum/Target.hpp"
#include "../../Machines/Sinclair/ZXSpectrum/State.hpp"

using namespace Storage::State;

namespace {

std::vector<uint8_t> read_memory(Storage::FileHolder &file, size_t size, bool is_compressed) {
	if(!is_compressed) {
		return file.read(size);
	}

	std::vector<uint8_t> result(size);
	size_t cursor = 0;

	uint8_t incoming[2] = { file.get8(), file.get8() };
	while(true) {
		if(incoming[0] == 0xed && incoming[1] == 0xed) {
			const uint8_t count = file.get8();
			const uint8_t value = file.get8();

			memset(&result[cursor], value, count);
			cursor += count;
			if(cursor == size) break;
			incoming[0] = file.get8();
			incoming[1] = file.get8();
		} else {
			result[cursor] = incoming[0];
			++cursor;
			if(cursor == size) break;
			incoming[0] = incoming[1];
			incoming[1] = file.get8();
		}
	}

	return result;
}

}

std::unique_ptr<Analyser::Static::Target> Z80::load(const std::string &file_name) {
	FileHolder file(file_name);

	// Construct a target with a Spectrum state.
	using Target = Analyser::Static::ZXSpectrum::Target;
	auto result = std::make_unique<Target>();
	auto *const state = new Sinclair::ZXSpectrum::State();
	result->state = std::unique_ptr<Reflection::Struct>(state);

	// Read version 1 header.
	state->z80.registers.a = file.get8();
	state->z80.registers.flags = file.get8();
	state->z80.registers.bc = file.get16le();
	state->z80.registers.hl = file.get16le();
	state->z80.registers.program_counter = file.get16le();
	state->z80.registers.stack_pointer = file.get16le();
	state->z80.registers.ir = file.get16be();	// Stored I then R.

	// Bit 7 of R is stored separately; likely this relates to an
	// optimisation in the Z80 emulator that for some reason was
	// exported into its file format.
	const uint8_t raw_misc = file.get8();
	const uint8_t misc = (raw_misc == 0xff) ? 1 : raw_misc;
	state->z80.registers.ir = uint16_t((state->z80.registers.ir & ~0x80) | ((misc&1) << 7));

	state->z80.registers.de = file.get16le();
	state->z80.registers.bc_dash = file.get16le();
	state->z80.registers.de_dash = file.get16le();
	state->z80.registers.hl_dash = file.get16le();
	state->z80.registers.af_dash = file.get16be();	// Stored A' then F'.
	state->z80.registers.iy = file.get16le();
	state->z80.registers.ix = file.get16le();
	state->z80.registers.iff1 = bool(file.get8());
	state->z80.registers.iff2 = bool(file.get8());

	// Ignored from the next byte:
	//
	//	bit 2 = 1 	=> issue 2 emulation
	//	bit 3 = 1	=> double interrupt frequency (?)
	//	bit 4–5		=> video synchronisation (to do with emulation hackery?)
	//	bit 6–7		=> joystick type
	state->z80.registers.interrupt_mode = file.get8() & 3;

	// If the program counter is non-0 then this is a version 1 snapshot,
	// which means it's definitely a 48k image.
	if(state->z80.registers.program_counter) {
		result->model = Target::Model::FortyEightK;
		state->ram = read_memory(file, 48*1024, misc & 0x20);
		return result;
	}

	// This was a version 1 or 2 snapshot, so keep going...
	const uint16_t bonus_header_size = file.get16le();
	if(bonus_header_size != 23 && bonus_header_size != 54 && bonus_header_size != 55) {
		return nullptr;
	}

	state->z80.registers.program_counter = file.get16le();
	switch(file.get8()) {
		default: return nullptr;
		case 0:		result->model = Target::Model::FortyEightK;		break;
		case 3:		result->model = Target::Model::OneTwoEightK;	break;
		case 7:
		case 8:		result->model = Target::Model::Plus3;			break;
		case 12:	result->model = Target::Model::Plus2;			break;
		case 13:	result->model = Target::Model::Plus2a;			break;
	}

	const uint8_t last7ffd = file.get8();	(void)last7ffd;	// TODO

	file.seek(1, SEEK_CUR);
	if(file.get8() & 0x80) {
		// The 'hardware modify' bit, which inexplicably does this:
		switch(result->model) {
			default: break;
			case Target::Model::FortyEightK:	result->model = Target::Model::SixteenK;	break;
			case Target::Model::OneTwoEightK:	result->model = Target::Model::Plus2;		break;
			case Target::Model::Plus3:			result->model = Target::Model::Plus2a;		break;
		}
	}

	const uint8_t lastfffd = file.get8();	(void)lastfffd;	// TODO
	file.seek(16, SEEK_CUR);	// Sound chip registers: TODO.

	if(bonus_header_size != 23) {
		// More Z80, the emulator, lack of encapsulation to deal with here.
		const uint16_t low_t_state = file.get16le();
		const uint16_t high_t_state = file.get8();
		int time_since_interrupt;
		switch(result->model) {
			case Target::Model::SixteenK:
			case Target::Model::FortyEightK:
				time_since_interrupt = (17471 - low_t_state) + (high_t_state * 17472);
			break;

			default:
				time_since_interrupt = (17726 - low_t_state) + (high_t_state * 17727);
			break;
		}
		// TODO: map time_since_interrupt to time_into_frame, somehow.

		// Skip: Spectator flag, MGT, Multiface and other ROM flags.
		file.seek(5, SEEK_CUR);

		// Skip: highly Z80-the-emulator-specific stuff about user-defined joystick.
		file.seek(20, SEEK_CUR);

		// Skip: Disciple/Plus D stuff.
		file.seek(3, SEEK_CUR);

		if(bonus_header_size == 55) {
			const uint8_t last1ffd = file.get8();	(void)last1ffd;	// TODO
		}
	}

	// Grab RAM.
	switch(result->model) {
		case Target::Model::SixteenK:		state->ram.resize(16 * 1024);	break;
		case Target::Model::FortyEightK:	state->ram.resize(48 * 1024);	break;
		default:							state->ram.resize(128 * 1024);	break;
	}

	while(true) {
		const uint16_t block_size = file.get16le();
		const uint8_t page = file.get8();
		const auto location = file.tell();
		if(file.eof()) break;

		const auto data = read_memory(file, 16384, block_size != 0xffff);

		if(result->model == Target::Model::SixteenK || result->model == Target::Model::FortyEightK) {
			switch(page) {
				default: break;
				case 4:	memcpy(&state->ram[0x4000], data.data(), 16384);	break;
				case 5:	memcpy(&state->ram[0x8000], data.data(), 16384);	break;
				case 8:	memcpy(&state->ram[0x0000], data.data(), 16384);	break;
			}
		} else {
			if(page >= 3 && page <= 10) {
				memcpy(&state->ram[(page - 3) * 0x4000], data.data(), 16384);
			}
		}

		file.seek(location + block_size, SEEK_SET);
	}

	return result;
}
<|endoftext|>


<commit_before>/*M///////////////////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2015, Youssef Kashef
// Copyright (c) 2015, ELM Library Project
// 3-clause BSD License
//
//M*/
/** @file test out Mat_ visitors,
 * some fixtures may be scattered somewhat awkwardly in this file due to conditional support (e.g. PCL support)
 */
#include "elm/core/visitors/visitorsparsemat1f.h"

#include "elm/core/exception.h"
#include "elm/core/pcl/cloud_.h"
#include "elm/core/pcl/vertices.h"
#include "elm/ts/ts.h"

using namespace std;
using namespace cv;
using namespace elm;

namespace {

/**
 * @brief test class around VisitorMat1f
 */
class VisitorSparseMat1fTest : public ::testing::Test
{
protected:
    VisitorSparseMat1f to_;    ///< test object
};

TEST_F(VisitorSparseMat1fTest, EmptySparseMat1f)
{
    EXPECT_EQ(0, to_(SparseMat1f()).size());
    EXPECT_THROW(to_(SparseMat1f(0, 0)), std::exception);
}

TEST_F(VisitorSparseMat1fTest, EmptyMat)
{
    EXPECT_EQ(0, to_(Mat1f()).size());
    EXPECT_EQ(0, to_(Mat1f(0, 0)).size());
    EXPECT_EQ(0, to_(Mat1f(1, 0)).size());
    EXPECT_EQ(0, to_(Mat1f(0, 1)).size());
}

TEST_F(VisitorSparseMat1fTest, FromSparseMat1f)
{
    int N=5;

    while(N--) {

        const int MAT_ROWS = abs(randu<int>()) % 10 + 1;
        const int MAT_COLS = abs(randu<int>()) % 100 + 1;

        Mat1f tmp(MAT_ROWS, MAT_COLS);
        randn(tmp, 0.f, 100.f);
        SparseMat1f m(tmp);

        Mat1f m2;
        to_(m).convertTo(m2, CV_32FC1);

        EXPECT_MAT_EQ(m, m2) << "Matrices are not equal";
    }
}

TEST_F(VisitorSparseMat1fTest, FromDenseMat1f)
{
    int N=5;

    while(N--) {

        const int MAT_ROWS = abs(randu<int>()) % 10 + 1;
        const int MAT_COLS = abs(randu<int>()) % 100 + 1;

        Mat1f m(MAT_ROWS, MAT_COLS);
        randn(m, 0.f, 100.f);

        Mat1f m2;
        to_(m).convertTo(m2, CV_32FC1);

        EXPECT_MAT_EQ(m, m2) << "Matrices are not equal";
        EXPECT_NE(m.data, m2.data);
    }
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_empty)
{
    EXPECT_EQ(0, to_(VecMat1f()).size());
    EXPECT_EQ(0, to_(VecMat1f(0)).size());\
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_empty_elements)
{
    EXPECT_EQ(0, to_(VecMat1f(1)).size());
    EXPECT_EQ(0, to_(VecMat1f(1, Mat1f())).size());
    EXPECT_EQ(0, to_(VecMat1f(3, Mat1f())).size());
    EXPECT_EQ(0, to_(VecMat1f(3, Mat1f(1, 0))).size());
    EXPECT_EQ(0, to_(VecMat1f(3, Mat1f(0, 1))).size());
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_row_mats)
{
    const int N=5;
    VecMat1f v;
    for(int i=0; i<N; i++) {

        v.push_back(Mat1f(1, 3, static_cast<float>(i)));

        SparseMat1f sparse = to_(v);
        Mat1f m;
        sparse.convertTo(m, CV_32FC1);

        EXPECT_EQ(3, m.cols);
        EXPECT_EQ(i+1, m.rows);

        for(int r=0; r<m.rows; r++) {

            for(int c=0; c<m.cols; c++) {

                EXPECT_FLOAT_EQ(m(r, c), v[r](c));
            }
        }
    }
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_rows_concat)
{
    VecMat1f v;
    v.push_back(Mat1f(2, 3, 1.f));
    v.push_back(Mat1f(3, 3, 2.f));

    SparseMat1f sparse = to_(v);
    Mat1f m;
    sparse.convertTo(m, CV_32FC1);

    EXPECT_MAT_EQ(m.rowRange(0, 2), v[0]);
    EXPECT_MAT_EQ(m.rowRange(2, m.rows), v[1]);
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_variable_size)
{
    {
        VecMat1f v;
        v.push_back(Mat1f(1, 5, 1.f));
        v.push_back(Mat1f(1, 2, 2.f));

        EXPECT_THROW(to_(v), ExceptionBadDims);
    }
    {
        VecMat1f v;
        v.push_back(Mat1f(1, 5, 1.f));
        v.push_back(Mat1f(1, 2, 0.f));

        EXPECT_THROW(to_(v), ExceptionBadDims);
    }
    {
        VecMat1f v;
        v.push_back(Mat1f(1, 2, 1.f));
        v.push_back(Mat1f(1, 5, 2.f));

        EXPECT_THROW(to_(v), ExceptionBadDims);
    }
    {
        VecMat1f v;
        v.push_back(Mat1f(1, 2, 0.f));
        v.push_back(Mat1f(1, 5, 2.f));

        EXPECT_THROW(to_(v), ExceptionBadDims);
    }
}

TEST_F(VisitorSparseMat1fTest, Reset)
{
    EXPECT_NO_THROW(to_.Reset());

    Mat1f m(4, 3);
    randn(m, 0.f, 100.f);

    Mat1f m2;
    to_(m).convertTo(m2, CV_32FC1);
    EXPECT_NO_THROW(to_.Reset());
}

#ifdef __WITH_PCL // unit tests that require PCL support (e.g point cloud)

TEST_F(VisitorSparseMat1fTest, EmptyCLoud)
{
    CloudXYZPtr cld(new CloudXYZ);
    EXPECT_EQ(0, to_(cld).size());
}

TEST_F(VisitorSparseMat1fTest, FromCloud)
{
    Mat1f m(4, 3);
    randn(m, 0.f, 100.f);
    CloudXYZPtr cloud = Mat2PointCloud_<pcl::PointXYZ>(m);

    Mat1f m2;
    to_(cloud).convertTo(m2, CV_32FC1);

    hconcat(m, Mat1f(m.rows, 1, 1), m);

    EXPECT_MAT_EQ(m, m2);
    EXPECT_NE(m.data, m2.data) << "Expecting deep copy. If intentionally optimized to be a shared copy, please update test.";
}

TEST_F(VisitorSparseMat1fTest, Reset_with_cloud)
{
    EXPECT_NO_THROW(to_.Reset());

    Mat1f m(4, 3);
    randn(m, 0.f, 100.f);

    Mat1f m2;
    to_(m).convertTo(m2, CV_32FC1);
    EXPECT_NO_THROW(to_.Reset());

    CloudXYZPtr cloud = Mat2PointCloud_<pcl::PointXYZ>(m);
    to_(cloud);
    EXPECT_NO_THROW(to_.Reset());

    to_(m);
    EXPECT_NO_THROW(to_.Reset());
}

// fixtures for VecVertices

TEST_F(VisitorSparseMat1fTest, Empty_VecVertices)
{
    EXPECT_EQ(0, to_(VecVertices()).size());
}

TEST_F(VisitorSparseMat1fTest, Empty_VecVertices_Size)
{
    EXPECT_MAT_DIMS_EQ(to_(VecVertices()), Size2i(0, 0));
}

TEST_F(VisitorSparseMat1fTest, From_VecVertices)
{
    Mat1f m(4, 3);
    for(size_t i=0; i<m.total(); i++) {

        m(i) = static_cast<float>(randu<uint32_t>() % 256);
    }
    VecVertices vv = Mat2VecVertices(m.clone());

    Mat1f m2;
    to_(vv).convertTo(m2, CV_32FC1);

    EXPECT_MAT_EQ(m, m2);
    EXPECT_NE(m.data, m2.data) << "Expecting deep copy. If intentionally optimized to be a shared copy, please update test.";
}

TEST_F(VisitorSparseMat1fTest, Reset_with_VecVertices)
{
    EXPECT_NO_THROW(to_.Reset());

    Mat1f m(4, 3);
    randn(m, 0.f, 100.f);

    SparseMat1f m2 = to_(m);
    EXPECT_NO_THROW(to_.Reset());

    CloudXYZPtr cloud = Mat2PointCloud_<pcl::PointXYZ >(m);
    m2 = to_(cloud);
    EXPECT_NO_THROW(to_.Reset());

    m2 = to_(m);
    EXPECT_NO_THROW(to_.Reset());
}

#endif // __WITH_PCL

} // annonymous namespace for visitors' test fixtures
<commit_msg>increase coverage for sparse mat visitor<commit_after>/*M///////////////////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2015, Youssef Kashef
// Copyright (c) 2015, ELM Library Project
// 3-clause BSD License
//
//M*/
/** @file test out Mat_ visitors,
 * some fixtures may be scattered somewhat awkwardly in this file due to conditional support (e.g. PCL support)
 */
#include "elm/core/visitors/visitorsparsemat1f.h"

#include "elm/core/exception.h"
#include "elm/core/pcl/cloud_.h"
#include "elm/core/pcl/vertices.h"
#include "elm/ts/pcl_point_typed_tests.h"
#include "elm/ts/ts.h"

using namespace std;
using namespace cv;
using namespace elm;

namespace {

/**
 * @brief test class around VisitorMat1f
 */
class VisitorSparseMat1fTest : public ::testing::Test
{
protected:
    VisitorSparseMat1f to_;    ///< test object
};

TEST_F(VisitorSparseMat1fTest, EmptySparseMat1f)
{
    EXPECT_EQ(0, to_(SparseMat1f()).size());
    EXPECT_THROW(to_(SparseMat1f(0, 0)), std::exception);
}

TEST_F(VisitorSparseMat1fTest, EmptyMat1f)
{
    EXPECT_EQ(0, to_(Mat1f()).size());
    EXPECT_EQ(0, to_(Mat1f(0, 0)).size());
    EXPECT_EQ(0, to_(Mat1f(1, 0)).size());
    EXPECT_EQ(0, to_(Mat1f(0, 1)).size());
}

TEST_F(VisitorSparseMat1fTest, EmptyMat)
{
    EXPECT_THROW(to_(Mat()), ExceptionBadDims);

    EXPECT_THROW(to_(Mat(0, 0, CV_32FC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 0, CV_32FC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 0, CV_32FC3)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_32FC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_32FC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_32FC3)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_32FC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_32FC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_32FC3)), ExceptionBadDims);

    EXPECT_THROW(to_(Mat(0, 0, CV_8UC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 0, CV_8UC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 0, CV_8UC3)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_8UC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_8UC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_8UC3)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_8UC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_8UC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_8UC3)), ExceptionBadDims);

    EXPECT_THROW(to_(Mat(0, 0, CV_32SC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 0, CV_32SC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 0, CV_32SC3)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_32SC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_32SC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_32SC3)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_32SC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_32SC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_32SC3)), ExceptionBadDims);

    EXPECT_THROW(to_(Mat(0, 0, CV_8UC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 0, CV_8UC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 0, CV_8UC3)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_8UC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_8UC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(0, 1, CV_8UC3)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_8UC1)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_8UC2)), ExceptionBadDims);
    EXPECT_THROW(to_(Mat(1, 0, CV_8UC3)), ExceptionBadDims);
}

TEST_F(VisitorSparseMat1fTest, FromSparseMat1f)
{
    int N=5;

    while(N--) {

        const int MAT_ROWS = abs(randu<int>()) % 10 + 1;
        const int MAT_COLS = abs(randu<int>()) % 100 + 1;

        Mat1f tmp(MAT_ROWS, MAT_COLS);
        randn(tmp, 0.f, 100.f);
        SparseMat1f m(tmp);

        Mat1f m2;
        to_(m).convertTo(m2, CV_32FC1);

        EXPECT_MAT_EQ(m, m2) << "Matrices are not equal";
    }
}

TEST_F(VisitorSparseMat1fTest, FromDenseMat1f)
{
    int N=5;

    while(N--) {

        const int MAT_ROWS = abs(randu<int>()) % 10 + 1;
        const int MAT_COLS = abs(randu<int>()) % 100 + 1;

        Mat1f m(MAT_ROWS, MAT_COLS);
        randn(m, 0.f, 100.f);

        {
            Mat1f m2;
            to_(m).convertTo(m2, CV_32FC1);

            EXPECT_MAT_EQ(m, m2) << "Matrices are not equal";
            EXPECT_NE(m.data, m2.data);
        }
        {
            Mat m1;
            m.convertTo(m1, CV_64FC1);
            Mat1f m2;
            to_(m).convertTo(m2, CV_32FC1);

            EXPECT_MAT_EQ(m, m2) << "Matrices are not equal";
            EXPECT_NE(m.data, m2.data);
        }
        {
            Mat m1;
            m.convertTo(m1, CV_8UC1);
            Mat1f m2;
            to_(m).convertTo(m2, CV_32FC1);

            EXPECT_MAT_EQ(m, m2) << "Matrices are not equal";
            EXPECT_NE(m.data, m2.data);
        }
        {
            Mat m1;
            m.convertTo(m1, CV_32SC1);
            Mat1f m2;
            to_(m).convertTo(m2, CV_32FC1);

            EXPECT_MAT_EQ(m, m2) << "Matrices are not equal";
            EXPECT_NE(m.data, m2.data);
        }
    }
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_empty)
{
    EXPECT_EQ(0, to_(VecMat1f()).size());
    EXPECT_EQ(0, to_(VecMat1f(0)).size());\
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_empty_elements)
{
    EXPECT_EQ(0, to_(VecMat1f(1)).size());
    EXPECT_EQ(0, to_(VecMat1f(1, Mat1f())).size());
    EXPECT_EQ(0, to_(VecMat1f(3, Mat1f())).size());
    EXPECT_EQ(0, to_(VecMat1f(3, Mat1f(1, 0))).size());
    EXPECT_EQ(0, to_(VecMat1f(3, Mat1f(0, 1))).size());
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_row_mats)
{
    const int N=5;
    VecMat1f v;
    for(int i=0; i<N; i++) {

        v.push_back(Mat1f(1, 3, static_cast<float>(i)));

        SparseMat1f sparse = to_(v);
        Mat1f m;
        sparse.convertTo(m, CV_32FC1);

        EXPECT_EQ(3, m.cols);
        EXPECT_EQ(i+1, m.rows);

        for(int r=0; r<m.rows; r++) {

            for(int c=0; c<m.cols; c++) {

                EXPECT_FLOAT_EQ(m(r, c), v[r](c));
            }
        }
    }
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_rows_concat)
{
    VecMat1f v;
    v.push_back(Mat1f(2, 3, 1.f));
    v.push_back(Mat1f(3, 3, 2.f));

    SparseMat1f sparse = to_(v);
    Mat1f m;
    sparse.convertTo(m, CV_32FC1);

    EXPECT_MAT_EQ(m.rowRange(0, 2), v[0]);
    EXPECT_MAT_EQ(m.rowRange(2, m.rows), v[1]);
}

TEST_F(VisitorSparseMat1fTest, FromVecMat1f_variable_size)
{
    {
        VecMat1f v;
        v.push_back(Mat1f(1, 5, 1.f));
        v.push_back(Mat1f(1, 2, 2.f));

        EXPECT_THROW(to_(v), ExceptionBadDims);
    }
    {
        VecMat1f v;
        v.push_back(Mat1f(1, 5, 1.f));
        v.push_back(Mat1f(1, 2, 0.f));

        EXPECT_THROW(to_(v), ExceptionBadDims);
    }
    {
        VecMat1f v;
        v.push_back(Mat1f(1, 2, 1.f));
        v.push_back(Mat1f(1, 5, 2.f));

        EXPECT_THROW(to_(v), ExceptionBadDims);
    }
    {
        VecMat1f v;
        v.push_back(Mat1f(1, 2, 0.f));
        v.push_back(Mat1f(1, 5, 2.f));

        EXPECT_THROW(to_(v), ExceptionBadDims);
    }
}

TEST_F(VisitorSparseMat1fTest, Reset)
{
    EXPECT_NO_THROW(to_.Reset());

    Mat1f m(4, 3);
    randn(m, 0.f, 100.f);

    Mat1f m2;
    to_(m).convertTo(m2, CV_32FC1);
    EXPECT_NO_THROW(to_.Reset());
}

#ifdef __WITH_PCL // unit tests that require PCL support (e.g point cloud)

// fixtures for VecVertices

TEST_F(VisitorSparseMat1fTest, Empty_VecVertices)
{
    EXPECT_EQ(0, to_(VecVertices()).size());
}

TEST_F(VisitorSparseMat1fTest, Empty_VecVertices_Size)
{
    EXPECT_MAT_DIMS_EQ(to_(VecVertices()), Size2i(0, 0));
}

TEST_F(VisitorSparseMat1fTest, From_VecVertices)
{
    Mat1f m(4, 3);
    for(size_t i=0; i<m.total(); i++) {

        m(i) = static_cast<float>(randu<uint32_t>() % 256);
    }
    VecVertices vv = Mat2VecVertices(m.clone());

    Mat1f m2;
    to_(vv).convertTo(m2, CV_32FC1);

    EXPECT_MAT_EQ(m, m2);
    EXPECT_NE(m.data, m2.data) << "Expecting deep copy. If intentionally optimized to be a shared copy, please update test.";
}

TEST_F(VisitorSparseMat1fTest, Reset_with_VecVertices)
{
    EXPECT_NO_THROW(to_.Reset());

    Mat1f m(4, 3);
    randn(m, 0.f, 100.f);

    SparseMat1f m2 = to_(m);
    EXPECT_NO_THROW(to_.Reset());

    CloudXYZPtr cloud = Mat2PointCloud_<pcl::PointXYZ >(m);
    m2 = to_(cloud);
    EXPECT_NO_THROW(to_.Reset());

    m2 = to_(m);
    EXPECT_NO_THROW(to_.Reset());
}

template <class TPoint>
class VisitorSparseMat1fCloud_TypedTest : public VisitorSparseMat1fTest
{
protected:
    virtual void SetUp() {

        size_t field_count_sz = elm::ts::ExpectedPointAttr_<TPoint>::field_count;
        field_count_ = static_cast<int>(field_count_sz);

        size_t nb_floats_sz = elm::ts::ExpectedPointAttr_<TPoint>::nb_floats;
        nb_floats_ = static_cast<int>(nb_floats_sz);
    }

    // members
    int field_count_;
    int nb_floats_;
};

TYPED_TEST_CASE(VisitorSparseMat1fCloud_TypedTest, PCLPointTypes);

TYPED_TEST(VisitorSparseMat1fCloud_TypedTest, EmptyCLoud) {

    typename pcl::PointCloud<TypeParam >::Ptr cld(new pcl::PointCloud<TypeParam >);
    EXPECT_EQ(0, to_(cld).size());
}

TYPED_TEST(VisitorSparseMat1fCloud_TypedTest, FromCloud) {

    Mat1f m(4, this->field_count_);
    randn(m, 0.f, 100.f);
    typename pcl::PointCloud<TypeParam >::Ptr cloud = Mat2PointCloud_<TypeParam >(m);

    Mat1f m2;
    to_(cloud).convertTo(m2, CV_32FC1);

    Mat1f m_ref = PointCloud2Mat_<TypeParam> (cloud);

    EXPECT_MAT_EQ(m_ref, m2);
    EXPECT_NE(m.data, m2.data) << "Expecting deep copy. If intentionally optimized to be a shared copy, please update test.";
}

TYPED_TEST(VisitorSparseMat1fCloud_TypedTest, Reset_with_cloud) {

    EXPECT_NO_THROW(this->to_.Reset());

    Mat1f m(4, this->field_count_);
    randn(m, 0.f, 100.f);

    Mat1f m2;
    this->to_(m).convertTo(m2, CV_32FC1);
    EXPECT_NO_THROW(this->to_.Reset());

    typename pcl::PointCloud<TypeParam >::Ptr cloud = Mat2PointCloud_<TypeParam >(m);
    this->to_(cloud);
    EXPECT_NO_THROW(this->to_.Reset());

    this->to_(m);
    EXPECT_NO_THROW(this->to_.Reset());
}

#endif // __WITH_PCL

template <class T>
class VisitorSparseMat1fPOD_TypedTest : public VisitorSparseMat1fTest
{
protected:
};

typedef ::testing::Types<float, int, uchar> PODTypes;
TYPED_TEST_CASE(VisitorSparseMat1fPOD_TypedTest, PODTypes);

TYPED_TEST(VisitorSparseMat1fPOD_TypedTest, FromPOD)
{
    int N=10;

    while(N--) {

        TypeParam v = randu<TypeParam>();

        SparseMat1f m = this->to_(v);
        EXPECT_EQ(2, m.dims());
        EXPECT_EQ(1, m.size(0));
        EXPECT_EQ(1, m.size(1));
        EXPECT_EQ(m.size(0), m.size(1));

        Mat1f m2;
        m.convertTo(m2, CV_32FC1);
        EXPECT_EQ(static_cast<float>(v), m2(0));
    }
}

} // annonymous namespace for visitors' test fixtures
<|endoftext|>