<commit_before>#include "kyheader.h"
#include "DataSetVOC.h"

DataSetVOC::DataSetVOC(CStr &_wkDir)
{
	wkDir = _wkDir;
	resDir = wkDir + "Results/";
	localDir = wkDir + "Local/";
	imgPathW = wkDir + "JPEGImages/%s.jpg";
	annoPathW = wkDir + "Annotations/%s.yml";
	CmFile::MkDir(resDir);
	CmFile::MkDir(localDir);

    trainSet = CmFile::loadStrList(wkDir + "ImageSets/Main/train.txt");
    testSet = CmFile::loadStrList(wkDir + "ImageSets/Main/test.txt");
	classNames = CmFile::loadStrList(wkDir + "ImageSets/Main/class.txt");

	// testSet.insert(testSet.end(), trainSet.begin(), trainSet.end());
	// testSet.resize(min(1000, (int)testSet.size()));

	trainNum = trainSet.size();
	testNum = testSet.size();
}

Vec4i getMaskRange(CMat &mask1u, int ext = 0)
{
	int maxX = INT_MIN, maxY = INT_MIN, minX = INT_MAX, minY = INT_MAX, rows = mask1u.rows, cols = mask1u.cols;
	for (int r = 0; r < rows; r++)	{
		const byte* data = mask1u.ptr<byte>(r);
		for (int c = 0; c < cols; c++)
			if (data[c] > 10) {
				maxX = max(maxX, c);
				minX = min(minX, c);
				maxY = max(maxY, r);
				minY = min(minY, r);
			}
	}

	maxX = maxX + ext + 1 < cols ? maxX + ext + 1 : cols;
	maxY = maxY + ext + 1 < rows ? maxY + ext + 1 : rows;
	minX = minX - ext > 0 ? minX - ext : 0;
	minY = minY - ext > 0 ? minY - ext : 0;

	return Vec4i(minX + 1, minY + 1, maxX, maxY); // Rect(minX, minY, maxX - minX, maxY - minY);
}

DataSetVOC::~DataSetVOC(void)
{
}

void DataSetVOC::loadAnnotations()
{
	gtTrainBoxes.resize(trainNum);
	gtTrainClsIdx.resize(trainNum);
	for (int i = 0; i < trainNum; i++)
		if (!loadBBoxes(trainSet[i], gtTrainBoxes[i], gtTrainClsIdx[i]))
			return;

	gtTestBoxes.resize(testNum);
	gtTestClsIdx.resize(testNum);
	for (int i = 0; i < testNum; i++)
		if(!loadBBoxes(testSet[i], gtTestBoxes[i], gtTestClsIdx[i]))
			return;
	printf("Load annotations finished\n");
}

void DataSetVOC::loadDataGenericOverCls()
{
	vecS allSet = trainSet;
	allSet.insert(allSet.end(), testSet.begin(), testSet.end());
	int imgN = (int)allSet.size();
	trainSet.clear(), testSet.clear();
	trainSet.reserve(imgN), testSet.reserve(imgN);
	vector<vector<Vec4i>> gtBoxes(imgN);
	vector<vecI> gtClsIdx(imgN);
	for (int i = 0; i < imgN; i++){
		if (!loadBBoxes(allSet[i], gtBoxes[i], gtClsIdx[i]))
			return;
		vector<Vec4i> trainBoxes, testBoxes;
		vecI trainIdx, testIdx;
		for (size_t j = 0; j < gtBoxes[i].size(); j++)
			if (gtClsIdx[i][j] < 6){
				trainBoxes.push_back(gtBoxes[i][j]);
				trainIdx.push_back(gtClsIdx[i][j]);
			}
			else{
				testBoxes.push_back(gtBoxes[i][j]);
				testIdx.push_back(gtClsIdx[i][j]);
			}
		if (trainBoxes.size()){
			trainSet.push_back(allSet[i]);
			gtTrainBoxes.push_back(trainBoxes);
			gtTrainClsIdx.push_back(trainIdx);
		}
		else{
			testSet.push_back(allSet[i]);
			gtTestBoxes.push_back(testBoxes);
			gtTestClsIdx.push_back(testIdx);
		}
	}
	trainNum = trainSet.size();
	testNum = testSet.size();
	printf("Load annotations (generic over classes) finished\n");
}

void DataSetVOC::loadBox(const FileNode &fn, vector<Vec4i> &boxes, vecI &clsIdx){
	string isDifficult;
	fn["difficult"]>>isDifficult;
	if (isDifficult == "1")
		return;

	string strXmin, strYmin, strXmax, strYmax;
	fn["bndbox"]["xmin"] >> strXmin;
	fn["bndbox"]["ymin"] >> strYmin;
	fn["bndbox"]["xmax"] >> strXmax;
	fn["bndbox"]["ymax"] >> strYmax;
	boxes.push_back(Vec4i(atoi(_S(strXmin)), atoi(_S(strYmin)), atoi(_S(strXmax)), atoi(_S(strYmax))));

	string clsName;
	fn["name"]>>clsName;
	clsIdx.push_back(findFromList(clsName, classNames));
	CV_Assert_(clsIdx[clsIdx.size() - 1] >= 0, ("Invalidate class name\n"));
}

bool DataSetVOC::loadBBoxes(CStr &nameNE, vector<Vec4i> &boxes, vecI &clsIdx)
{
	string fName = format(_S(annoPathW), _S(nameNE));
	FileStorage fs(fName, FileStorage::READ);
	FileNode fn = fs["annotation"]["object"];
	boxes.clear();
	clsIdx.clear();
	if (fn.isSeq()){
        for (FileNodeIterator it = fn.begin(), it_end = fn.end(); it != it_end; it++){
            loadBox(*it, boxes, clsIdx);
        }
	}
	else
		loadBox(fn, boxes, clsIdx);
	return true;
}

// Needs to call yml.m in this solution before running this function.
bool DataSetVOC::cvt2OpenCVYml(CStr &annoDir)
{
	vecS namesNE;
	int imgNum = CmFile::GetNamesNE(annoDir + "*.yaml", namesNE);
	printf("Converting annotations to OpenCV yml format:\n");
	for (int i = 0; i < imgNum; i++){
		printf("%d/%d %s.yaml\r", i, imgNum, _S(namesNE[i]));
		string fPath = annoDir + namesNE[i];
		cvt2OpenCVYml(fPath + ".yaml", fPath + ".yml");
	}
	return true;
}

// Needs to call yml.m in this solution before running this function.
bool DataSetVOC::cvt2OpenCVYml(CStr &yamlName, CStr &ymlName)
{
	ifstream f(yamlName);
	FILE *fO = fopen(_S(ymlName), "w");
	if (!f.is_open() && fO == NULL)
		return false;
	fprintf(fO, "%s\n", "%YAML:1.0\n");
	string line;

	int addIdent = 0;
	while(getline(f, line)){
		if (line.substr(0, 12) == "  filename: ")
			line = "  filename: \"" + line.substr(12) + "\"";
		int tmp = line.find_first_of('-');
		if (tmp != string::npos){
			bool allSpace = true;
			for (int k = 0; k < tmp; k++)
				if (line[k] != ' ')
					allSpace = false;
			if (allSpace)
				addIdent = tmp;
		}
		for (int k = 0; k < addIdent; k++)
			fprintf(fO, " ");
		fprintf(fO, "%s\n", _S(line));
	}
	fclose(fO);

	FileStorage fs(ymlName, FileStorage::READ);
	string tmp;
	fs["annotation"]["folder"]>>tmp;
	return true;
}

// Get training and testing for demonstrating the generative of the objectness over classes
void DataSetVOC::getTrainTest()
{
	const int TRAIN_CLS_NUM = 6;
	string trainCls[TRAIN_CLS_NUM] = {"bird", "car", "cat", "cow", "dog", "sheep"};

}

void DataSetVOC::getXmlStrVOC(CStr &fName, string &buf)
{
	ifstream fin(fName);
	string strLine;
	buf.clear();
	buf.reserve(100000);
	buf += "<?xml version=\"1.0\"?>\n<opencv_storage>\n";
	while (getline(fin, strLine) && strLine.size())	{
		int startP = strLine.find_first_of(">") + 1;
		int endP = strLine.find_last_of("<");
		if (endP > startP){
			string val = keepXmlChar(strLine.substr(startP, endP - startP));
			if (val.size() < endP - startP)
				strLine = strLine.substr(0, startP) + val + strLine.substr(endP);
		}
		buf += strLine + "\n";
	}
	buf += "</opencv_storage>\n";
	//FileStorage fs(buf, FileStorage::READ + FileStorage::MEMORY);
	ofstream fout("D:/t.xml");
	fout<< buf;
}
<commit_msg>Dataset updates<commit_after>#include "kyheader.h"
#include "DataSetVOC.h"

DataSetVOC::DataSetVOC(CStr &_wkDir)
{
	wkDir = _wkDir;
	resDir = wkDir + "Results/";
	localDir = wkDir + "Local/";
	imgPathW = wkDir + "JPEGImages/%s.jpg";
	annoPathW = wkDir + "Annotations/%s.yml";
	CmFile::MkDir(resDir);
	CmFile::MkDir(localDir);

    trainSet = CmFile::loadStrList(wkDir + "ImageSets/Main/train.txt");
    testSet = CmFile::loadStrList(wkDir + "ImageSets/Main/test.txt");
	classNames = CmFile::loadStrList(wkDir + "ImageSets/Main/class.txt");

	// testSet.insert(testSet.end(), trainSet.begin(), trainSet.end());
	// testSet.resize(min(1000, (int)testSet.size()));

	trainNum = trainSet.size();
	testNum = testSet.size();
}

Vec4i getMaskRange(CMat &mask1u, int ext = 0)
{
	int maxX = INT_MIN, maxY = INT_MIN, minX = INT_MAX, minY = INT_MAX, rows = mask1u.rows, cols = mask1u.cols;
	for (int r = 0; r < rows; r++)	{
		const byte* data = mask1u.ptr<byte>(r);
		for (int c = 0; c < cols; c++)
			if (data[c] > 10) {
				maxX = max(maxX, c);
				minX = min(minX, c);
				maxY = max(maxY, r);
				minY = min(minY, r);
			}
	}

	maxX = maxX + ext + 1 < cols ? maxX + ext + 1 : cols;
	maxY = maxY + ext + 1 < rows ? maxY + ext + 1 : rows;
	minX = minX - ext > 0 ? minX - ext : 0;
	minY = minY - ext > 0 ? minY - ext : 0;

	return Vec4i(minX + 1, minY + 1, maxX, maxY); // Rect(minX, minY, maxX - minX, maxY - minY);
}

DataSetVOC::~DataSetVOC(void)
{
}

void DataSetVOC::loadAnnotations()
{
	gtTrainBoxes.resize(trainNum);
	gtTrainClsIdx.resize(trainNum);
	for (int i = 0; i < trainNum; i++)
		if (!loadBBoxes(trainSet[i], gtTrainBoxes[i], gtTrainClsIdx[i]))
			return;

	gtTestBoxes.resize(testNum);
	gtTestClsIdx.resize(testNum);
	for (int i = 0; i < testNum; i++)
		if(!loadBBoxes(testSet[i], gtTestBoxes[i], gtTestClsIdx[i]))
			return;
	printf("Load annotations finished\n");
}

void DataSetVOC::loadDataGenericOverCls()
{
	vecS allSet = trainSet;
	allSet.insert(allSet.end(), testSet.begin(), testSet.end());
	int imgN = (int)allSet.size();
	trainSet.clear(), testSet.clear();
	trainSet.reserve(imgN), testSet.reserve(imgN);
	vector<vector<Vec4i>> gtBoxes(imgN);
	vector<vecI> gtClsIdx(imgN);
	for (int i = 0; i < imgN; i++){
		if (!loadBBoxes(allSet[i], gtBoxes[i], gtClsIdx[i]))
			return;
		vector<Vec4i> trainBoxes, testBoxes;
		vecI trainIdx, testIdx;
		for (size_t j = 0; j < gtBoxes[i].size(); j++)
			if (gtClsIdx[i][j] < 6){
				trainBoxes.push_back(gtBoxes[i][j]);
				trainIdx.push_back(gtClsIdx[i][j]);
			}
			else{
				testBoxes.push_back(gtBoxes[i][j]);
				testIdx.push_back(gtClsIdx[i][j]);
			}
		if (trainBoxes.size()){
			trainSet.push_back(allSet[i]);
			gtTrainBoxes.push_back(trainBoxes);
			gtTrainClsIdx.push_back(trainIdx);
		}
		else{
			testSet.push_back(allSet[i]);
			gtTestBoxes.push_back(testBoxes);
			gtTestClsIdx.push_back(testIdx);
		}
	}
	trainNum = trainSet.size();
	testNum = testSet.size();
	printf("Load annotations (generic over classes) finished\n");
}

void DataSetVOC::loadBox(const FileNode &fn, vector<Vec4i> &boxes, vecI &clsIdx){
	string isDifficult;
	fn["difficult"]>>isDifficult;
	if (isDifficult == "1")
		return;

	string strXmin, strYmin, strXmax, strYmax;
	fn["bndbox"]["xmin"] >> strXmin;
	fn["bndbox"]["ymin"] >> strYmin;
	fn["bndbox"]["xmax"] >> strXmax;
	fn["bndbox"]["ymax"] >> strYmax;
	boxes.push_back(Vec4i(atoi(_S(strXmin)), atoi(_S(strYmin)), atoi(_S(strXmax)), atoi(_S(strYmax))));

	string clsName;
	fn["name"]>>clsName;
	printf("Class name: %s", clsName.c_str());
	clsIdx.push_back(findFromList(clsName, classNames));
	CV_Assert_(clsIdx[clsIdx.size() - 1] >= 0, ("Invalidate class name\n"));
}

bool DataSetVOC::loadBBoxes(CStr &nameNE, vector<Vec4i> &boxes, vecI &clsIdx)
{
	string fName = format(_S(annoPathW), _S(nameNE));
	FileStorage fs(fName, FileStorage::READ);
	FileNode fn = fs["annotation"]["object"];
	boxes.clear();
	clsIdx.clear();
	if (fn.isSeq()){
        for (FileNodeIterator it = fn.begin(), it_end = fn.end(); it != it_end; it++){
            loadBox(*it, boxes, clsIdx);
        }
	}
	else
		loadBox(fn, boxes, clsIdx);
	return true;
}

// Needs to call yml.m in this solution before running this function.
bool DataSetVOC::cvt2OpenCVYml(CStr &annoDir)
{
	vecS namesNE;
	int imgNum = CmFile::GetNamesNE(annoDir + "*.yaml", namesNE);
	printf("Converting annotations to OpenCV yml format:\n");
	for (int i = 0; i < imgNum; i++){
		printf("%d/%d %s.yaml\r", i, imgNum, _S(namesNE[i]));
		string fPath = annoDir + namesNE[i];
		cvt2OpenCVYml(fPath + ".yaml", fPath + ".yml");
	}
	return true;
}

// Needs to call yml.m in this solution before running this function.
bool DataSetVOC::cvt2OpenCVYml(CStr &yamlName, CStr &ymlName)
{
	ifstream f(yamlName);
	FILE *fO = fopen(_S(ymlName), "w");
	if (!f.is_open() && fO == NULL)
		return false;
	fprintf(fO, "%s\n", "%YAML:1.0\n");
	string line;

	int addIdent = 0;
	while(getline(f, line)){
		if (line.substr(0, 12) == "  filename: ")
			line = "  filename: \"" + line.substr(12) + "\"";
		int tmp = line.find_first_of('-');
		if (tmp != string::npos){
			bool allSpace = true;
			for (int k = 0; k < tmp; k++)
				if (line[k] != ' ')
					allSpace = false;
			if (allSpace)
				addIdent = tmp;
		}
		for (int k = 0; k < addIdent; k++)
			fprintf(fO, " ");
		fprintf(fO, "%s\n", _S(line));
	}
	fclose(fO);

	FileStorage fs(ymlName, FileStorage::READ);
	string tmp;
	fs["annotation"]["folder"]>>tmp;
	return true;
}

// Get training and testing for demonstrating the generative of the objectness over classes
void DataSetVOC::getTrainTest()
{
	const int TRAIN_CLS_NUM = 6;
	string trainCls[TRAIN_CLS_NUM] = {"bird", "car", "cat", "cow", "dog", "sheep"};

}

void DataSetVOC::getXmlStrVOC(CStr &fName, string &buf)
{
	ifstream fin(fName);
	string strLine;
	buf.clear();
	buf.reserve(100000);
	buf += "<?xml version=\"1.0\"?>\n<opencv_storage>\n";
	while (getline(fin, strLine) && strLine.size())	{
		int startP = strLine.find_first_of(">") + 1;
		int endP = strLine.find_last_of("<");
		if (endP > startP){
			string val = keepXmlChar(strLine.substr(startP, endP - startP));
			if (val.size() < endP - startP)
				strLine = strLine.substr(0, startP) + val + strLine.substr(endP);
		}
		buf += strLine + "\n";
	}
	buf += "</opencv_storage>\n";
	//FileStorage fs(buf, FileStorage::READ + FileStorage::MEMORY);
	ofstream fout("D:/t.xml");
	fout<< buf;
}
<|endoftext|>


<commit_before>// Copyright 2010-2014 RethinkDB, all rights reserved.
#ifndef CONTAINERS_COUNTED_HPP_
#define CONTAINERS_COUNTED_HPP_

#include <stddef.h>
#include <stdint.h>

#include <utility>
#include <functional>

#include "containers/scoped.hpp"
#include "errors.hpp"
#include "threading.hpp"

// Yes, this is a clone of boost::intrusive_ptr.  This will probably
// not be the case in the future.

// Now it supports .unique(), and in order to use it, your type needs
// to provide an counted_use_count implementation.

template <class T>
class counted_t {
public:
    template <class U>
    friend class counted_t;

    counted_t() : p_(NULL) { }
    explicit counted_t(T *p) : p_(p) {
        if (p_ != nullptr) { counted_add_ref(p_); }
    }

    explicit counted_t(scoped_ptr_t<T> &&p) : p_(p.release()) {
        if (p_ != nullptr) { counted_add_ref(p_); }
    }

    counted_t(const counted_t &copyee) : p_(copyee.p_) {
        if (p_ != nullptr) { counted_add_ref(p_); }
    }

    counted_t(counted_t &&movee) noexcept : p_(std::move(movee.p_)) {
        movee.p_ = nullptr;
    }

    template <class U>
    counted_t(counted_t<U> &&movee) noexcept : p_(std::move(movee.p_)) {
        movee.p_ = nullptr;
    }

// Avoid a spurious warning when building on Saucy. See issue #1674
#if defined(__GNUC__) && (100 * __GNUC__ + __GNUC_MINOR__ >= 408)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif

    ~counted_t() {
        if (p_) { counted_release(p_); }
    }

#if defined(__GNUC__) && (100 * __GNUC__ + __GNUC_MINOR__ >= 408)
#pragma GCC diagnostic pop
#endif

    void swap(counted_t &other) noexcept {
        T *tmp = p_;
        p_ = other.p_;
        other.p_ = tmp;
    }

    counted_t &operator=(counted_t &&other) noexcept {
        counted_t tmp(std::move(other));
        swap(tmp);
        return *this;
    }

    template <class U>
    counted_t &operator=(counted_t<U> &&other) noexcept {
        counted_t tmp(std::move(other));
        swap(tmp);
        return *this;
    }

    counted_t &operator=(const counted_t &other) {
        counted_t tmp(other);
        swap(tmp);
        return *this;
    }

    void reset() {
        counted_t tmp;
        swap(tmp);
    }

    void reset(T *other) {
        counted_t tmp(other);
        swap(tmp);
    }

    T *operator->() const {
        return p_;
    }

    T &operator*() const {
        return *p_;
    }

    T *get() const {
        return p_;
    }

    bool has() const {
        return p_ != NULL;
    }

    bool unique() const {
        return counted_use_count(p_) == 1;
    }

    explicit operator bool() const {
        return p_ != NULL;
    }

private:
    T *p_;
};

template <class T, class... Args>
counted_t<T> make_counted(Args &&... args) {
    return counted_t<T>(new T(std::forward<Args>(args)...));
}

template <class T, class Alloc, class... Args>
counted_t<T> make_counted(std::allocator_arg_t, const Alloc &a, Args &&... args) {
    typedef std::allocator_traits<Alloc> traits;
    T* result = traits::allocate(a, 1);
    traits::construct(a, result, std::forward<Args>(args)...);
    counted_set_deleter(result, [a](T* region) { traits::deallocate(region, 1); });
    return counted_t<T>(result);
}

template <class> class single_threaded_countable_t;

template <class T>
inline void counted_add_ref(const single_threaded_countable_t<T> *p);
template <class T>
inline void counted_release(const single_threaded_countable_t<T> *p);
template <class T>
inline void counted_set_deleter(const single_threaded_countable_t<T> *p,
                                std::function<void(single_threaded_countable_t<T>*)> &&d);
template <class T>
inline intptr_t counted_use_count(const single_threaded_countable_t<T> *p);

template <class T>
class single_threaded_countable_t : private home_thread_mixin_debug_only_t {
public:
    single_threaded_countable_t() : refcount_(0) { }

protected:
    counted_t<T> counted_from_this() {
        assert_thread();
        rassert(refcount_ > 0);
        return counted_t<T>(static_cast<T *>(this));
    }

    counted_t<const T> counted_from_this() const {
        assert_thread();
        rassert(refcount_ > 0);
        return counted_t<const T>(static_cast<const T *>(this));
    }

    ~single_threaded_countable_t() {
        assert_thread();
        rassert(refcount_ == 0);
    }

private:
    friend void counted_add_ref<T>(const single_threaded_countable_t<T> *p);
    friend void counted_release<T>(const single_threaded_countable_t<T> *p);
    friend void
    counted_set_deleter<T>(const single_threaded_countable_t<T> *p,
                           std::function<void(single_threaded_countable_t<T>*)> &&d);
    friend intptr_t counted_use_count<T>(const single_threaded_countable_t<T> *p);

    mutable intptr_t refcount_;
    mutable std::function<void(single_threaded_countable_t<T>*)> deleter;
    DISABLE_COPYING(single_threaded_countable_t);
};

template <class T>
inline void counted_add_ref(const single_threaded_countable_t<T> *p) {
    p->assert_thread();
    p->refcount_ += 1;
    rassert(p->refcount_ > 0);
}

template <class T>
inline void counted_release(const single_threaded_countable_t<T> *p) {
    p->assert_thread();
    rassert(p->refcount_ > 0);
    --p->refcount_;
    if (p->refcount_ == 0) {
        T *realptr = static_cast<T *>(const_cast<single_threaded_countable_t<T> *>(p));
        if (p->deleter) {
            p->deleter(realptr);
        } else {
            delete realptr;
        }
    }
}

template <class T>
inline void
counted_set_deleter(const single_threaded_countable_t<T> *p,
                    std::function<void(single_threaded_countable_t<T>*)> &&d) {
    p->deleter = std::move(d);
}

template <class T>
inline intptr_t counted_use_count(const single_threaded_countable_t<T> *p) {
    p->assert_thread();
    return p->refcount_;
}

template <class> class slow_atomic_countable_t;

template <class T>
inline void counted_add_ref(const slow_atomic_countable_t<T> *p);
template <class T>
inline void counted_release(const slow_atomic_countable_t<T> *p);
template <class T>
inline void counted_set_deleter(const slow_atomic_countable_t<T> *p,
                                std::function<void(slow_atomic_countable_t<T>*)> &&d);
template <class T>
inline intptr_t counted_use_count(const slow_atomic_countable_t<T> *p);

template <class T>
class slow_atomic_countable_t {
public:
    slow_atomic_countable_t() : refcount_(0) { }

protected:
    ~slow_atomic_countable_t() {
        rassert(refcount_ == 0);
    }

    counted_t<T> counted_from_this() {
        rassert(counted_use_count(this) > 0);
        return counted_t<T>(static_cast<T *>(this));
    }

    counted_t<const T> counted_from_this() const {
        rassert(counted_use_count(this) > 0);
        return counted_t<const T>(static_cast<const T *>(this));
    }

private:
    friend void counted_add_ref<T>(const slow_atomic_countable_t<T> *p);
    friend void counted_release<T>(const slow_atomic_countable_t<T> *p);
    friend void
    counted_set_deleter<T>(const slow_atomic_countable_t<T> *p,
                           std::function<void(slow_atomic_countable_t<T>*)> &&d);
    friend intptr_t counted_use_count<T>(const slow_atomic_countable_t<T> *p);

    mutable intptr_t refcount_;
    mutable std::function<void(slow_atomic_countable_t<T>*)> deleter;
    DISABLE_COPYING(slow_atomic_countable_t);
};

template <class T>
inline void counted_add_ref(const slow_atomic_countable_t<T> *p) {
    DEBUG_VAR intptr_t res = __sync_add_and_fetch(&p->refcount_, 1);
    rassert(res > 0);
}

template <class T>
inline void counted_release(const slow_atomic_countable_t<T> *p) {
    intptr_t res = __sync_sub_and_fetch(&p->refcount_, 1);
    rassert(res >= 0);
    if (res == 0) {
        T *realptr = static_cast<T *>(const_cast<slow_atomic_countable_t<T> *>(p));
        if (p->deleter) {
            p->deleter(realptr);
        } else {
            delete realptr;
        }
    }
}

template <class T>
inline void counted_set_deleter(const slow_atomic_countable_t<T> *p,
                                std::function<void(slow_atomic_countable_t<T>*)> &&d) {
    p->deleter = std::move(d);
}

template <class T>
inline intptr_t counted_use_count(const slow_atomic_countable_t<T> *p) {
    // Finally a practical use for volatile.
    intptr_t tmp = static_cast<const volatile intptr_t&>(p->refcount_);
    rassert(tmp > 0);
    return tmp;
}

// A noncopyable reference to a reference-counted object.
template <class T>
class movable_t {
public:
    explicit movable_t(const counted_t<T> &copyee) : ptr_(copyee) { }
    movable_t(movable_t &&movee) noexcept : ptr_(std::move(movee.ptr_)) { }
    movable_t &operator=(movable_t &&movee) noexcept {
        ptr_ = std::move(movee.ptr_);
        return *this;
    }

    void reset() { ptr_.reset(); }

    T *get() const { return ptr_.get(); }
    T *operator->() const { return ptr_.get(); }
    T &operator*() const { return *ptr_; }

    bool has() const { return ptr_.has(); }

private:
    counted_t<T> ptr_;
    DISABLE_COPYING(movable_t);
};

// Extends an arbitrary object with a slow_atomic_countable_t
template<class T>
class countable_wrapper_t : public T,
                            public slow_atomic_countable_t<countable_wrapper_t<T> > {
public:
    template <class... Args>
    explicit countable_wrapper_t(Args &&... args)
        : T(std::forward<Args>(args)...) { }
};

#endif  // CONTAINERS_COUNTED_HPP_
<commit_msg>Make this exception safe.<commit_after>// Copyright 2010-2014 RethinkDB, all rights reserved.
#ifndef CONTAINERS_COUNTED_HPP_
#define CONTAINERS_COUNTED_HPP_

#include <stddef.h>
#include <stdint.h>

#include <utility>
#include <functional>

#include "containers/scoped.hpp"
#include "errors.hpp"
#include "threading.hpp"

// Yes, this is a clone of boost::intrusive_ptr.  This will probably
// not be the case in the future.

// Now it supports .unique(), and in order to use it, your type needs
// to provide an counted_use_count implementation.

template <class T>
class counted_t {
public:
    template <class U>
    friend class counted_t;

    counted_t() : p_(NULL) { }
    explicit counted_t(T *p) : p_(p) {
        if (p_ != nullptr) { counted_add_ref(p_); }
    }

    explicit counted_t(scoped_ptr_t<T> &&p) : p_(p.release()) {
        if (p_ != nullptr) { counted_add_ref(p_); }
    }

    counted_t(const counted_t &copyee) : p_(copyee.p_) {
        if (p_ != nullptr) { counted_add_ref(p_); }
    }

    counted_t(counted_t &&movee) noexcept : p_(std::move(movee.p_)) {
        movee.p_ = nullptr;
    }

    template <class U>
    counted_t(counted_t<U> &&movee) noexcept : p_(std::move(movee.p_)) {
        movee.p_ = nullptr;
    }

// Avoid a spurious warning when building on Saucy. See issue #1674
#if defined(__GNUC__) && (100 * __GNUC__ + __GNUC_MINOR__ >= 408)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif

    ~counted_t() {
        if (p_) { counted_release(p_); }
    }

#if defined(__GNUC__) && (100 * __GNUC__ + __GNUC_MINOR__ >= 408)
#pragma GCC diagnostic pop
#endif

    void swap(counted_t &other) noexcept {
        T *tmp = p_;
        p_ = other.p_;
        other.p_ = tmp;
    }

    counted_t &operator=(counted_t &&other) noexcept {
        counted_t tmp(std::move(other));
        swap(tmp);
        return *this;
    }

    template <class U>
    counted_t &operator=(counted_t<U> &&other) noexcept {
        counted_t tmp(std::move(other));
        swap(tmp);
        return *this;
    }

    counted_t &operator=(const counted_t &other) {
        counted_t tmp(other);
        swap(tmp);
        return *this;
    }

    void reset() {
        counted_t tmp;
        swap(tmp);
    }

    void reset(T *other) {
        counted_t tmp(other);
        swap(tmp);
    }

    T *operator->() const {
        return p_;
    }

    T &operator*() const {
        return *p_;
    }

    T *get() const {
        return p_;
    }

    bool has() const {
        return p_ != NULL;
    }

    bool unique() const {
        return counted_use_count(p_) == 1;
    }

    explicit operator bool() const {
        return p_ != NULL;
    }

private:
    T *p_;
};

template <class T, class... Args>
counted_t<T> make_counted(Args &&... args) {
    return counted_t<T>(new T(std::forward<Args>(args)...));
}

template <class T, class Alloc, class... Args>
counted_t<T> make_counted(std::allocator_arg_t, const Alloc &a, Args &&... args) {
    typedef std::allocator_traits<Alloc> traits;
    T* result = traits::allocate(a, 1);
    try {
        traits::construct(a, result, std::forward<Args>(args)...);
        counted_set_deleter(result, [a](T* region) { traits::deallocate(region, 1); });
        return counted_t<T>(result);
    } catch (...) {
        traits::deallocate(result, 1);
        throw;
    }
}

template <class> class single_threaded_countable_t;

template <class T>
inline void counted_add_ref(const single_threaded_countable_t<T> *p);
template <class T>
inline void counted_release(const single_threaded_countable_t<T> *p);
template <class T>
inline void counted_set_deleter(const single_threaded_countable_t<T> *p,
                                std::function<void(single_threaded_countable_t<T>*)> &&d);
template <class T>
inline intptr_t counted_use_count(const single_threaded_countable_t<T> *p);

template <class T>
class single_threaded_countable_t : private home_thread_mixin_debug_only_t {
public:
    single_threaded_countable_t() : refcount_(0) { }

protected:
    counted_t<T> counted_from_this() {
        assert_thread();
        rassert(refcount_ > 0);
        return counted_t<T>(static_cast<T *>(this));
    }

    counted_t<const T> counted_from_this() const {
        assert_thread();
        rassert(refcount_ > 0);
        return counted_t<const T>(static_cast<const T *>(this));
    }

    ~single_threaded_countable_t() {
        assert_thread();
        rassert(refcount_ == 0);
    }

private:
    friend void counted_add_ref<T>(const single_threaded_countable_t<T> *p);
    friend void counted_release<T>(const single_threaded_countable_t<T> *p);
    friend void
    counted_set_deleter<T>(const single_threaded_countable_t<T> *p,
                           std::function<void(single_threaded_countable_t<T>*)> &&d);
    friend intptr_t counted_use_count<T>(const single_threaded_countable_t<T> *p);

    mutable intptr_t refcount_;
    mutable std::function<void(single_threaded_countable_t<T>*)> deleter;
    DISABLE_COPYING(single_threaded_countable_t);
};

template <class T>
inline void counted_add_ref(const single_threaded_countable_t<T> *p) {
    p->assert_thread();
    p->refcount_ += 1;
    rassert(p->refcount_ > 0);
}

template <class T>
inline void counted_release(const single_threaded_countable_t<T> *p) {
    p->assert_thread();
    rassert(p->refcount_ > 0);
    --p->refcount_;
    if (p->refcount_ == 0) {
        T *realptr = static_cast<T *>(const_cast<single_threaded_countable_t<T> *>(p));
        if (p->deleter) {
            p->deleter(realptr);
        } else {
            delete realptr;
        }
    }
}

template <class T>
inline void
counted_set_deleter(const single_threaded_countable_t<T> *p,
                    std::function<void(single_threaded_countable_t<T>*)> &&d) {
    p->deleter = std::move(d);
}

template <class T>
inline intptr_t counted_use_count(const single_threaded_countable_t<T> *p) {
    p->assert_thread();
    return p->refcount_;
}

template <class> class slow_atomic_countable_t;

template <class T>
inline void counted_add_ref(const slow_atomic_countable_t<T> *p);
template <class T>
inline void counted_release(const slow_atomic_countable_t<T> *p);
template <class T>
inline void counted_set_deleter(const slow_atomic_countable_t<T> *p,
                                std::function<void(slow_atomic_countable_t<T>*)> &&d);
template <class T>
inline intptr_t counted_use_count(const slow_atomic_countable_t<T> *p);

template <class T>
class slow_atomic_countable_t {
public:
    slow_atomic_countable_t() : refcount_(0) { }

protected:
    ~slow_atomic_countable_t() {
        rassert(refcount_ == 0);
    }

    counted_t<T> counted_from_this() {
        rassert(counted_use_count(this) > 0);
        return counted_t<T>(static_cast<T *>(this));
    }

    counted_t<const T> counted_from_this() const {
        rassert(counted_use_count(this) > 0);
        return counted_t<const T>(static_cast<const T *>(this));
    }

private:
    friend void counted_add_ref<T>(const slow_atomic_countable_t<T> *p);
    friend void counted_release<T>(const slow_atomic_countable_t<T> *p);
    friend void
    counted_set_deleter<T>(const slow_atomic_countable_t<T> *p,
                           std::function<void(slow_atomic_countable_t<T>*)> &&d);
    friend intptr_t counted_use_count<T>(const slow_atomic_countable_t<T> *p);

    mutable intptr_t refcount_;
    mutable std::function<void(slow_atomic_countable_t<T>*)> deleter;
    DISABLE_COPYING(slow_atomic_countable_t);
};

template <class T>
inline void counted_add_ref(const slow_atomic_countable_t<T> *p) {
    DEBUG_VAR intptr_t res = __sync_add_and_fetch(&p->refcount_, 1);
    rassert(res > 0);
}

template <class T>
inline void counted_release(const slow_atomic_countable_t<T> *p) {
    intptr_t res = __sync_sub_and_fetch(&p->refcount_, 1);
    rassert(res >= 0);
    if (res == 0) {
        T *realptr = static_cast<T *>(const_cast<slow_atomic_countable_t<T> *>(p));
        if (p->deleter) {
            p->deleter(realptr);
        } else {
            delete realptr;
        }
    }
}

template <class T>
inline void counted_set_deleter(const slow_atomic_countable_t<T> *p,
                                std::function<void(slow_atomic_countable_t<T>*)> &&d) {
    p->deleter = std::move(d);
}

template <class T>
inline intptr_t counted_use_count(const slow_atomic_countable_t<T> *p) {
    // Finally a practical use for volatile.
    intptr_t tmp = static_cast<const volatile intptr_t&>(p->refcount_);
    rassert(tmp > 0);
    return tmp;
}

// A noncopyable reference to a reference-counted object.
template <class T>
class movable_t {
public:
    explicit movable_t(const counted_t<T> &copyee) : ptr_(copyee) { }
    movable_t(movable_t &&movee) noexcept : ptr_(std::move(movee.ptr_)) { }
    movable_t &operator=(movable_t &&movee) noexcept {
        ptr_ = std::move(movee.ptr_);
        return *this;
    }

    void reset() { ptr_.reset(); }

    T *get() const { return ptr_.get(); }
    T *operator->() const { return ptr_.get(); }
    T &operator*() const { return *ptr_; }

    bool has() const { return ptr_.has(); }

private:
    counted_t<T> ptr_;
    DISABLE_COPYING(movable_t);
};

// Extends an arbitrary object with a slow_atomic_countable_t
template<class T>
class countable_wrapper_t : public T,
                            public slow_atomic_countable_t<countable_wrapper_t<T> > {
public:
    template <class... Args>
    explicit countable_wrapper_t(Args &&... args)
        : T(std::forward<Args>(args)...) { }
};

#endif  // CONTAINERS_COUNTED_HPP_
<|endoftext|>


<commit_before>#include "config.h"
#include "vm.hpp"
#include "on_stack.hpp"
#include "objectmemory.hpp"
#include "call_frame.hpp"

#include "builtin/array.hpp"
#include "builtin/class.hpp"
#include "builtin/module.hpp"
#include "builtin/thread.hpp"
#include "builtin/nativemethod.hpp"

#include "capi/handle.hpp"

#include "gc/finalize.hpp"

#include "dtrace/dtrace.h"

namespace rubinius {
  FinalizerHandler::iterator::iterator(FinalizerHandler* fh)
    : handler_(fh)
    , current_list_(NULL)
  {
    if(handler_->live_list_->empty()) {
      if(handler_->lists_->empty()) {
        current_ = handler_->live_list_->begin();
        end_ = handler_->live_list_->end();
        return;
      } else {
        current_list_ = handler_->lists_->front();
      }
    } else {
      current_list_ = handler_->live_list_;
    }

    current_ = current_list_->begin();

    if(handler_->lists_->empty()) {
      end_ = current_list_->end();
    } else {
      lists_iterator_ = handler_->lists_->begin();
      end_ = handler_->lists_->back()->end();
    }
  }

  void FinalizerHandler::iterator::next(bool live) {
    if(current_list_ == handler_->live_list_) {
      if(!live) current_->queued();

      if(++current_ == current_list_->end()) {
        if(!handler_->lists_->empty()) {
          current_list_ = *lists_iterator_;
          current_ = current_list_->begin();
        }
      }
    } else {
      if(++current_ == end_) {
        return;
      } else if(current_ == current_list_->end()) {
        if(++lists_iterator_ != handler_->lists_->end()) {
          current_list_ = *lists_iterator_;
          current_ = current_list_->begin();
        }
      }
    }
  }

  bool FinalizerHandler::iterator::end() {
    return current_ == end_;
  }

  Object* finalizer_handler_tramp(STATE) {
    state->shared().finalizer_handler()->perform(state);
    return cNil;
  }

  FinalizerHandler::FinalizerHandler(STATE)
    : AuxiliaryThread()
    , shared_(state->shared())
    , self_(NULL)
    , thread_(state)
    , lists_(NULL)
    , live_list_(NULL)
    , process_list_(NULL)
    , iterator_(NULL)
    , process_item_kind_(eRuby)
    , paused_(false)
    , exit_(false)
    , finishing_(false)
  {
    shared_.auxiliary_threads()->register_thread(this);
    shared_.set_finalizer_handler(this);

    lists_ = new FinalizeObjectsList();
    live_list_ = new FinalizeObjects();

    live_guard_.init();
    worker_lock_.init();
    worker_cond_.init();
    pause_cond_.init();

    supervisor_lock_.init();
    supervisor_cond_.init();
  }

  FinalizerHandler::~FinalizerHandler() {
    shared_.auxiliary_threads()->unregister_thread(this);

    if(iterator_) delete iterator_;
    if(live_list_) delete live_list_;

    for(FinalizeObjectsList::iterator i = lists_->begin(); i != lists_->end(); ++i) {
      delete *i;
    }

    if(lists_) delete lists_;
  }

  void FinalizerHandler::start_thread(STATE) {
    SYNC(state);
    if(self_) return;
    utilities::thread::Mutex::LockGuard lg(worker_lock_);
    self_ = state->shared().new_vm();
    paused_ = false;
    exit_ = false;
    thread_.set(Thread::create(state, self_, G(thread), finalizer_handler_tramp, false, true));
    run(state);
  }

  void FinalizerHandler::stop_thread(STATE) {
    SYNC(state);
    if(!self_) return;

    pthread_t os = self_->os_thread();
    {
      utilities::thread::Mutex::LockGuard lg(worker_lock_);
      // Thread might have already been stopped
      exit_ = true;
      worker_signal();
    }

    void* return_value;
    pthread_join(os, &return_value);
    self_ = NULL;
  }

  void FinalizerHandler::shutdown(STATE) {
    // We do nothing now because we have to finish processing all remaining
    // live objects once everything shuts down.
  }

  void FinalizerHandler::before_exec(STATE) {
    stop_thread(state);
  }

  void FinalizerHandler::after_exec(STATE) {
    start_thread(state);
  }

  void FinalizerHandler::before_fork(STATE) {
    utilities::thread::Mutex::LockGuard lg(worker_lock_);
    while(!paused_ && self_->run_state() == ManagedThread::eRunning) {
      pause_cond_.wait(worker_lock_);
    }
  }

  void FinalizerHandler::after_fork_parent(STATE) {
    utilities::thread::Mutex::LockGuard lg(worker_lock_);
    pause_cond_.signal();
  }

  void FinalizerHandler::after_fork_child(STATE) {
    live_guard_.init();
    worker_lock_.init();
    worker_cond_.init();
    pause_cond_.init();
    supervisor_lock_.init();
    supervisor_cond_.init();

    if(self_) {
      VM::discard(state, self_);
      self_ = NULL;
    }

    start_thread(state);
  }

  void FinalizerHandler::run(STATE) {
    int error = thread_.get()->fork_attached(state);
    if(error) rubinius::bug("Unable to start finalizer handler thread");
  }

  void FinalizerHandler::perform(STATE) {
    GCTokenImpl gct;
    const char* thread_name = "rbx.finalizer";
    self_->set_name(thread_name);

    RUBINIUS_THREAD_START(thread_name, state->vm()->thread_id(), 1);

    state->vm()->thread->hard_unlock(state, gct, 0);

    while(!exit_) {
      state->vm()->set_call_frame(0);

      if(!process_list_) first_process_item();

      if(!process_list_) {
        {
          utilities::thread::Mutex::LockGuard lg(worker_lock_);

          if(finishing_) supervisor_signal();

          // exit_ might have been set in the mean while after
          // we grabbed the worker_lock
          if(exit_) break;
          state->gc_independent(gct, 0);
          paused_ = true;
          pause_cond_.signal();
          worker_wait();
          if(exit_) break;
        }
        state->gc_dependent(gct, 0);
        {
          utilities::thread::Mutex::LockGuard lg(worker_lock_);
          paused_ = false;
          if(exit_) break;
        }

        continue;
      }

      finalize(state);
      next_process_item();
    }
    RUBINIUS_THREAD_STOP(thread_name, state->vm()->thread_id(), 1);
  }

  void FinalizerHandler::finalize(STATE) {

    switch(process_item_kind_) {
    case eRuby: {
      if(process_item_->ruby_finalizer) {
        CallFrame* call_frame = 0;
        // Rubinius specific code. If the finalizer is cTrue, then send the
        // object the __finalize__ message.
        if(process_item_->ruby_finalizer == cTrue) {
          process_item_->object->send(state, call_frame, state->symbol("__finalize__"));
        } else {
          Array* ary = Array::create(state, 1);
          ary->set(state, 0, process_item_->object->id(state));
          process_item_->ruby_finalizer->send(state, call_frame, G(sym_call), ary);
        }
      }

      process_item_->status = FinalizeObject::eRubyFinalized;
      break;
    }
    case eNative:
      if(process_item_->finalizer) {

        NativeMethodEnvironment* env = NativeMethodEnvironment::get();
        NativeMethodFrame nmf(0, 0);
        CallFrame* call_frame = ALLOCA_CALLFRAME(0);
        call_frame->previous = 0;
        call_frame->constant_scope_ = 0;
        call_frame->dispatch_data = (void*)&nmf;
        call_frame->compiled_code = 0;
        call_frame->flags = CallFrame::cNativeMethod;
        call_frame->optional_jit_data = 0;
        call_frame->top_scope_ = 0;
        call_frame->scope = 0;
        call_frame->arguments = 0;

        env->set_current_call_frame(0);
        env->set_current_native_frame(&nmf);

        // Register the CallFrame, because we might GC below this.
        state->set_call_frame(call_frame);

        nmf.setup(Qnil, Qnil, Qnil, Qnil);

        (*process_item_->finalizer)(state, process_item_->object);

        state->set_call_frame(0);
        env->set_current_call_frame(0);
        env->set_current_native_frame(0);
      }
      process_item_->status = FinalizeObject::eNativeFinalized;
      break;
    case eRelease:
      // Unhook any handle used by fi->object so that we don't accidentally
      // try and mark it later (after we've finalized it)
      if(capi::Handle* handle = process_item_->object->handle(state)) {
        handle->forget_object();
        process_item_->object->clear_handle(state);
      }

      // If the object was remembered, unremember it.
      if(process_item_->object->remembered_p()) {
        state->memory()->unremember_object(process_item_->object);
      }

      process_item_->status = FinalizeObject::eReleased;

      break;
    }
  }

  void FinalizerHandler::first_process_item() {
    if(!process_list_ && !lists_->empty()) {
      process_list_ = lists_->back();
      process_item_ = process_list_->begin();
    }
  }

  void FinalizerHandler::next_process_item() {
    if(++process_item_ == process_list_->end()) {
      switch(process_item_kind_) {
      case eRuby:
        process_item_ = process_list_->begin();
        process_item_kind_ = eNative;
        break;
      case eNative:
        process_item_ = process_list_->begin();
        process_item_kind_ = eRelease;
        break;
      case eRelease:
        delete process_list_;
        process_list_ = NULL;
        process_item_kind_ = eRuby;
        lists_->pop_back();
        break;
      }
    }
  }

  void FinalizerHandler::finish(STATE, GCToken gct) {
    if(!self_) {
      if(process_list_ || !lists_->empty() || !live_list_->empty()) {
        rubinius::bug("FinalizerHandler worker thread dead during halt");
      } else {
        return;
      }
    }

    finishing_ = true;

    while(true) {
      {
        StopTheWorld stw(state, gct, 0);

        if(!process_list_) {
          if(live_list_->empty() && lists_->empty()) break;

          // Everything is garbage when halting so keep adding live objects to
          // finalize queue until done.
          if(!live_list_->empty()) {
            for(FinalizeObjects::iterator i = live_list_->begin();
                i != live_list_->end();
                ++i)
            {
              i->queued();
            }

            queue_objects();
          }

          first_process_item();
          if(!process_list_) break;
        }
      }

      worker_signal();

      {
        utilities::thread::Mutex::LockGuard lg(supervisor_lock_);

        GCIndependent indy(state, 0);
        if(process_list_) supervisor_wait();
      }
    }

    if(!lists_->empty() || !live_list_->empty() || process_list_ != NULL)
      rubinius::bug("FinalizerHandler exiting with pending finalizers");

    stop_thread(state);
  }

  void FinalizerHandler::record(Object* obj, FinalizerFunction func) {
    utilities::thread::Mutex::LockGuard lg(live_guard_);

    FinalizeObject fi;
    fi.object = obj;
    fi.status = FinalizeObject::eLive;
    fi.finalizer = func;

    // Makes a copy of fi.
    live_list_->push_front(fi);
  }

  void FinalizerHandler::set_ruby_finalizer(Object* obj, Object* finalizer) {
    utilities::thread::Mutex::LockGuard lg(live_guard_);

    // Ignore Ruby finalizers created when finishing running finalizers.
    if(finishing_) return;

    // Check if the object is already in the finalizer list.
    for(FinalizeObjects::iterator i = live_list_->begin();
        i != live_list_->end();
        ++i)
    {
      if(i->object == obj) {
        if(finalizer->nil_p()) {
          live_list_->erase(i);
        } else {
          i->ruby_finalizer = finalizer;
        }
        return;
      }
    }

    // Adding a nil finalizer is only used to delete an existing finalizer,
    // which we apparently don't have if we get here.
    if(finalizer->nil_p()) {
      return;
    }

    // Ok, create it.

    FinalizeObject fi;
    fi.object = obj;
    fi.status = FinalizeObject::eLive;

    // Rubinius specific API. If the finalizer is the object, we're going to send
    // the object __finalize__. We mark that the user wants this by putting cTrue
    // as the ruby_finalizer.
    if(obj == finalizer) {
      fi.ruby_finalizer = cTrue;
    } else {
      fi.ruby_finalizer = finalizer;
    }

    // Makes a copy of fi.
    live_list_->push_front(fi);
  }

  void FinalizerHandler::queue_objects() {
    FinalizeObjects* dead_list = new FinalizeObjects();

    for(FinalizeObjects::iterator i = live_list_->begin();
        i != live_list_->end();
        /* advance is handled in the loop */)
    {
      if(i->queued_p()) {
        dead_list->push_front(*i);
        i = live_list_->erase(i);
      } else {
        ++i;
      }
    }

    if(!dead_list->empty()) {
      lists_->push_front(dead_list);
    } else {
      delete dead_list;
    }
  }

  void FinalizerHandler::start_collection(STATE) {
    if(process_item_kind_ == eRelease) {
      while(process_list_) {
        finalize(state);
        next_process_item();
      }
    }
  }

  void FinalizerHandler::finish_collection(STATE) {
    queue_objects();

    if(iterator_) {
      delete iterator_;
      iterator_ = NULL;
    }

    worker_signal();
  }

  void FinalizerHandler::supervisor_signal() {
    supervisor_cond_.signal();
  }

  void FinalizerHandler::supervisor_wait() {
    supervisor_cond_.wait(supervisor_lock_);
  }

  void FinalizerHandler::worker_signal() {
    worker_cond_.signal();
  }

  void FinalizerHandler::worker_wait() {
    worker_cond_.wait(worker_lock_);
  }

  FinalizerHandler::iterator& FinalizerHandler::begin() {
    if(iterator_) delete iterator_;
    iterator_ = new iterator(this);
    return *iterator_;
  }
}
<commit_msg>Update current finalizer if a GC is triggered<commit_after>#include "config.h"
#include "vm.hpp"
#include "on_stack.hpp"
#include "objectmemory.hpp"
#include "call_frame.hpp"

#include "builtin/array.hpp"
#include "builtin/class.hpp"
#include "builtin/module.hpp"
#include "builtin/thread.hpp"
#include "builtin/nativemethod.hpp"

#include "capi/handle.hpp"

#include "gc/finalize.hpp"

#include "dtrace/dtrace.h"

namespace rubinius {
  FinalizerHandler::iterator::iterator(FinalizerHandler* fh)
    : handler_(fh)
    , current_list_(NULL)
  {
    if(handler_->live_list_->empty()) {
      if(handler_->lists_->empty()) {
        current_ = handler_->live_list_->begin();
        end_ = handler_->live_list_->end();
        return;
      } else {
        current_list_ = handler_->lists_->front();
      }
    } else {
      current_list_ = handler_->live_list_;
    }

    current_ = current_list_->begin();

    if(handler_->lists_->empty()) {
      end_ = current_list_->end();
    } else {
      lists_iterator_ = handler_->lists_->begin();
      end_ = handler_->lists_->back()->end();
    }
  }

  void FinalizerHandler::iterator::next(bool live) {
    if(current_list_ == handler_->live_list_) {
      if(!live) current_->queued();

      if(++current_ == current_list_->end()) {
        if(!handler_->lists_->empty()) {
          current_list_ = *lists_iterator_;
          current_ = current_list_->begin();
        }
      }
    } else {
      if(++current_ == end_) {
        return;
      } else if(current_ == current_list_->end()) {
        if(++lists_iterator_ != handler_->lists_->end()) {
          current_list_ = *lists_iterator_;
          current_ = current_list_->begin();
        }
      }
    }
  }

  bool FinalizerHandler::iterator::end() {
    return current_ == end_;
  }

  Object* finalizer_handler_tramp(STATE) {
    state->shared().finalizer_handler()->perform(state);
    return cNil;
  }

  FinalizerHandler::FinalizerHandler(STATE)
    : AuxiliaryThread()
    , shared_(state->shared())
    , self_(NULL)
    , thread_(state)
    , lists_(NULL)
    , live_list_(NULL)
    , process_list_(NULL)
    , iterator_(NULL)
    , process_item_kind_(eRuby)
    , paused_(false)
    , exit_(false)
    , finishing_(false)
  {
    shared_.auxiliary_threads()->register_thread(this);
    shared_.set_finalizer_handler(this);

    lists_ = new FinalizeObjectsList();
    live_list_ = new FinalizeObjects();

    live_guard_.init();
    worker_lock_.init();
    worker_cond_.init();
    pause_cond_.init();

    supervisor_lock_.init();
    supervisor_cond_.init();
  }

  FinalizerHandler::~FinalizerHandler() {
    shared_.auxiliary_threads()->unregister_thread(this);

    if(iterator_) delete iterator_;
    if(live_list_) delete live_list_;

    for(FinalizeObjectsList::iterator i = lists_->begin(); i != lists_->end(); ++i) {
      delete *i;
    }

    if(lists_) delete lists_;
  }

  void FinalizerHandler::start_thread(STATE) {
    SYNC(state);
    if(self_) return;
    utilities::thread::Mutex::LockGuard lg(worker_lock_);
    self_ = state->shared().new_vm();
    paused_ = false;
    exit_ = false;
    thread_.set(Thread::create(state, self_, G(thread), finalizer_handler_tramp, false, true));
    run(state);
  }

  void FinalizerHandler::stop_thread(STATE) {
    SYNC(state);
    if(!self_) return;

    pthread_t os = self_->os_thread();
    {
      utilities::thread::Mutex::LockGuard lg(worker_lock_);
      // Thread might have already been stopped
      exit_ = true;
      worker_signal();
    }

    void* return_value;
    pthread_join(os, &return_value);
    self_ = NULL;
  }

  void FinalizerHandler::shutdown(STATE) {
    // We do nothing now because we have to finish processing all remaining
    // live objects once everything shuts down.
  }

  void FinalizerHandler::before_exec(STATE) {
    stop_thread(state);
  }

  void FinalizerHandler::after_exec(STATE) {
    start_thread(state);
  }

  void FinalizerHandler::before_fork(STATE) {
    utilities::thread::Mutex::LockGuard lg(worker_lock_);
    while(!paused_ && self_->run_state() == ManagedThread::eRunning) {
      pause_cond_.wait(worker_lock_);
    }
  }

  void FinalizerHandler::after_fork_parent(STATE) {
    utilities::thread::Mutex::LockGuard lg(worker_lock_);
    pause_cond_.signal();
  }

  void FinalizerHandler::after_fork_child(STATE) {
    live_guard_.init();
    worker_lock_.init();
    worker_cond_.init();
    pause_cond_.init();
    supervisor_lock_.init();
    supervisor_cond_.init();

    if(self_) {
      VM::discard(state, self_);
      self_ = NULL;
    }

    start_thread(state);
  }

  void FinalizerHandler::run(STATE) {
    int error = thread_.get()->fork_attached(state);
    if(error) rubinius::bug("Unable to start finalizer handler thread");
  }

  void FinalizerHandler::perform(STATE) {
    GCTokenImpl gct;
    const char* thread_name = "rbx.finalizer";
    self_->set_name(thread_name);

    RUBINIUS_THREAD_START(thread_name, state->vm()->thread_id(), 1);

    state->vm()->thread->hard_unlock(state, gct, 0);

    while(!exit_) {
      state->vm()->set_call_frame(0);

      if(!process_list_) first_process_item();

      if(!process_list_) {
        {
          utilities::thread::Mutex::LockGuard lg(worker_lock_);

          if(finishing_) supervisor_signal();

          // exit_ might have been set in the mean while after
          // we grabbed the worker_lock
          if(exit_) break;
          state->gc_independent(gct, 0);
          paused_ = true;
          pause_cond_.signal();
          worker_wait();
          if(exit_) break;
        }
        state->gc_dependent(gct, 0);
        {
          utilities::thread::Mutex::LockGuard lg(worker_lock_);
          paused_ = false;
          if(exit_) break;
        }

        continue;
      }

      finalize(state);
      next_process_item();
    }
    RUBINIUS_THREAD_STOP(thread_name, state->vm()->thread_id(), 1);
  }

  void FinalizerHandler::finalize(STATE) {
    Object* obj = process_item_->object;
    Object* fin = process_item_->ruby_finalizer;
    OnStack<2> os(state, obj, fin);
    switch(process_item_kind_) {
    case eRuby: {
      if(process_item_->ruby_finalizer) {
        CallFrame* call_frame = 0;
        // Rubinius specific code. If the finalizer is cTrue, then send the
        // object the __finalize__ message.
        if(process_item_->ruby_finalizer == cTrue) {
          process_item_->object->send(state, call_frame, state->symbol("__finalize__"));
        } else {
          Array* ary = Array::create(state, 1);
          ary->set(state, 0, process_item_->object->id(state));
          process_item_->ruby_finalizer->send(state, call_frame, G(sym_call), ary);
        }
      }

      process_item_->status = FinalizeObject::eRubyFinalized;
      break;
    }
    case eNative:
      if(process_item_->finalizer) {

        NativeMethodEnvironment* env = NativeMethodEnvironment::get();
        NativeMethodFrame nmf(0, 0);
        CallFrame* call_frame = ALLOCA_CALLFRAME(0);
        call_frame->previous = 0;
        call_frame->constant_scope_ = 0;
        call_frame->dispatch_data = (void*)&nmf;
        call_frame->compiled_code = 0;
        call_frame->flags = CallFrame::cNativeMethod;
        call_frame->optional_jit_data = 0;
        call_frame->top_scope_ = 0;
        call_frame->scope = 0;
        call_frame->arguments = 0;

        env->set_current_call_frame(0);
        env->set_current_native_frame(&nmf);

        // Register the CallFrame, because we might GC below this.
        state->set_call_frame(call_frame);

        nmf.setup(Qnil, Qnil, Qnil, Qnil);

        (*process_item_->finalizer)(state, process_item_->object);

        state->set_call_frame(0);
        env->set_current_call_frame(0);
        env->set_current_native_frame(0);
      }
      process_item_->status = FinalizeObject::eNativeFinalized;
      break;
    case eRelease:
      // Unhook any handle used by fi->object so that we don't accidentally
      // try and mark it later (after we've finalized it)
      if(capi::Handle* handle = process_item_->object->handle(state)) {
        handle->forget_object();
        process_item_->object->clear_handle(state);
      }

      // If the object was remembered, unremember it.
      if(process_item_->object->remembered_p()) {
        state->memory()->unremember_object(process_item_->object);
      }

      process_item_->status = FinalizeObject::eReleased;

      break;
    }
    process_item_->object = obj;
    process_item_->ruby_finalizer = fin;
  }

  void FinalizerHandler::first_process_item() {
    if(!process_list_ && !lists_->empty()) {
      process_list_ = lists_->back();
      process_item_ = process_list_->begin();
    }
  }

  void FinalizerHandler::next_process_item() {
    if(++process_item_ == process_list_->end()) {
      switch(process_item_kind_) {
      case eRuby:
        process_item_ = process_list_->begin();
        process_item_kind_ = eNative;
        break;
      case eNative:
        process_item_ = process_list_->begin();
        process_item_kind_ = eRelease;
        break;
      case eRelease:
        delete process_list_;
        process_list_ = NULL;
        process_item_kind_ = eRuby;
        lists_->pop_back();
        break;
      }
    }
  }

  void FinalizerHandler::finish(STATE, GCToken gct) {
    if(!self_) {
      if(process_list_ || !lists_->empty() || !live_list_->empty()) {
        rubinius::bug("FinalizerHandler worker thread dead during halt");
      } else {
        return;
      }
    }

    finishing_ = true;

    while(true) {
      {
        StopTheWorld stw(state, gct, 0);

        if(!process_list_) {
          if(live_list_->empty() && lists_->empty()) break;

          // Everything is garbage when halting so keep adding live objects to
          // finalize queue until done.
          if(!live_list_->empty()) {
            for(FinalizeObjects::iterator i = live_list_->begin();
                i != live_list_->end();
                ++i)
            {
              i->queued();
            }

            queue_objects();
          }

          first_process_item();
          if(!process_list_) break;
        }
      }

      worker_signal();

      {
        utilities::thread::Mutex::LockGuard lg(supervisor_lock_);

        GCIndependent indy(state, 0);
        if(process_list_) supervisor_wait();
      }
    }

    if(!lists_->empty() || !live_list_->empty() || process_list_ != NULL)
      rubinius::bug("FinalizerHandler exiting with pending finalizers");

    stop_thread(state);
  }

  void FinalizerHandler::record(Object* obj, FinalizerFunction func) {
    utilities::thread::Mutex::LockGuard lg(live_guard_);

    FinalizeObject fi;
    fi.object = obj;
    fi.status = FinalizeObject::eLive;
    fi.finalizer = func;

    // Makes a copy of fi.
    live_list_->push_front(fi);
  }

  void FinalizerHandler::set_ruby_finalizer(Object* obj, Object* finalizer) {
    utilities::thread::Mutex::LockGuard lg(live_guard_);

    // Ignore Ruby finalizers created when finishing running finalizers.
    if(finishing_) return;

    // Check if the object is already in the finalizer list.
    for(FinalizeObjects::iterator i = live_list_->begin();
        i != live_list_->end();
        ++i)
    {
      if(i->object == obj) {
        if(finalizer->nil_p()) {
          live_list_->erase(i);
        } else {
          i->ruby_finalizer = finalizer;
        }
        return;
      }
    }

    // Adding a nil finalizer is only used to delete an existing finalizer,
    // which we apparently don't have if we get here.
    if(finalizer->nil_p()) {
      return;
    }

    // Ok, create it.

    FinalizeObject fi;
    fi.object = obj;
    fi.status = FinalizeObject::eLive;

    // Rubinius specific API. If the finalizer is the object, we're going to send
    // the object __finalize__. We mark that the user wants this by putting cTrue
    // as the ruby_finalizer.
    if(obj == finalizer) {
      fi.ruby_finalizer = cTrue;
    } else {
      fi.ruby_finalizer = finalizer;
    }

    // Makes a copy of fi.
    live_list_->push_front(fi);
  }

  void FinalizerHandler::queue_objects() {
    FinalizeObjects* dead_list = new FinalizeObjects();

    for(FinalizeObjects::iterator i = live_list_->begin();
        i != live_list_->end();
        /* advance is handled in the loop */)
    {
      if(i->queued_p()) {
        dead_list->push_front(*i);
        i = live_list_->erase(i);
      } else {
        ++i;
      }
    }

    if(!dead_list->empty()) {
      lists_->push_front(dead_list);
    } else {
      delete dead_list;
    }
  }

  void FinalizerHandler::start_collection(STATE) {
    if(process_item_kind_ == eRelease) {
      while(process_list_) {
        finalize(state);
        next_process_item();
      }
    }
  }

  void FinalizerHandler::finish_collection(STATE) {
    queue_objects();

    if(iterator_) {
      delete iterator_;
      iterator_ = NULL;
    }

    worker_signal();
  }

  void FinalizerHandler::supervisor_signal() {
    supervisor_cond_.signal();
  }

  void FinalizerHandler::supervisor_wait() {
    supervisor_cond_.wait(supervisor_lock_);
  }

  void FinalizerHandler::worker_signal() {
    worker_cond_.signal();
  }

  void FinalizerHandler::worker_wait() {
    worker_cond_.wait(worker_lock_);
  }

  FinalizerHandler::iterator& FinalizerHandler::begin() {
    if(iterator_) delete iterator_;
    iterator_ = new iterator(this);
    return *iterator_;
  }
}
<|endoftext|>


<commit_before>#pragma once
#include <functional>
#include <vector>
#include <memory>
#include <unordered_map>

#include "mutex_lock.hpp"
#include "queue.hpp"

namespace aim {

        constexpr std::size_t EVENTS_SIZE_STEP = 1000u;

        using EventArguments_t = std::function<void(void)>;

        enum class EventType : std::size_t {
                InputEvent,
                ControlEvent
        };

        /*! \brief Basic class for events aggregator.
         * Can collect only one type of events. Lives in separate thread.
         */
        struct IEventsAggregator {
                /*! \brief Main aggregator procedure.
                 */
                virtual void exec() = 0;
        };

        struct EventsDispenser {
                EventsDispenser()
                {
                }

                ~EventsDispenser() {

                }

                /*! \brief Register event aggregator.
                 * \param type Event type group.
                 * \param value Event aggregator.
                 */
                void registerAggregator(const EventType type, std::unique_ptr<IEventsAggregator> value) {
                        eventsAggregators[type] = std::move(value);
                }

                void finish() {
                        finised = !finised;
                }

                void operator()();

        private:
                bool finised = false;

                std::unordered_map<EventType, MutexLock<Queue_t<std::any>> eventsBus;
                std::unordered_map<EventType, std::unique_ptr<IEventsAggregator>> eventsAggregators;
                std::unordered_map<EventType, std::vector<int>> subscribers; // TODO
        };
}<commit_msg>Try to fix build<commit_after>#pragma once
#include <functional>
#include <vector>
#include <memory>
#include <unordered_map>

#include "mutex_lock.hpp"
#include "queue.hpp"

namespace aim {

        constexpr std::size_t EVENTS_SIZE_STEP = 1000u;

        using EventArguments_t = std::function<void(void)>;

        enum class EventType : std::size_t {
                InputEvent,
                ControlEvent
        };

        /*! \brief Basic class for events aggregator.
         * Can collect only one type of events. Lives in separate thread.
         */
        struct IEventsAggregator {
                /*! \brief Main aggregator procedure.
                 */
                virtual void exec() = 0;
        };

        struct EventsDispenser {
                EventsDispenser()
                {
                }

                ~EventsDispenser() {

                }

                /*! \brief Register event aggregator.
                 * \param type Event type group.
                 * \param value Event aggregator.
                 */
                void registerAggregator(const EventType type, std::unique_ptr<IEventsAggregator> value) {
                        eventsAggregators[type] = std::move(value);
                }

                void finish() {
                        finised = !finised;
                }

                void operator()();

        private:
                bool finised = false;

                //std::unordered_map<EventType, MutexLock<Queue_t<std::any>> eventsBus;
                //std::unordered_map<EventType, std::unique_ptr<IEventsAggregator>> eventsAggregators;
                //std::unordered_map<EventType, std::vector<int>> subscribers; // TODO
        };
}<|endoftext|>


<commit_before>/**
@file traits.hpp
@brief Non-standard traits and trait operators.

@author Tim Howard
@copyright 2013 Tim Howard under the MIT license;
see @ref index or the accompanying LICENSE file for full text.
*/

#ifndef TRAIT_WRANGLER_TRAITS_HPP_
#define TRAIT_WRANGLER_TRAITS_HPP_

#include "./config.hpp"
#include "./detail/traits.hpp"

#include <type_traits>

namespace trait_wrangler {

/**
	@addtogroup traits
	@{
*/

/**
	@name Trait operators
	@{
*/

/**
	Logical AND of traits.

	@param ...TraitP Traits.
*/
template<class... TraitP>
struct op_and
	: public ::trait_wrangler::detail::op_and<TraitP...>
{};

/**

*/
template<template<typename> class... TplP>
struct op_and_t final {
	/**
		Trait template adapter.

		@tparam T Type to apply to @a TplP.
	*/
	template<typename T>
	struct type
		: public ::trait_wrangler::detail::op_and_t<T, TplP...>
	{};
};

/**
	Logical OR of traits.

	@param ...TraitP Traits.
*/
template<class... TraitP>
struct op_or
	: public ::trait_wrangler::detail::op_or<TraitP...>
{};

/**

*/
template<template<typename> class... TplP>
struct op_or_t final {
	/**
		Trait template adapter.

		@tparam T Type to apply to @a TplP.
	*/
	template<typename T>
	struct type
		: public ::trait_wrangler::detail::op_or_t<T, TplP...>
	{};
};

/**
	Logical NOT of a trait.

	@tparam Trait Trait to negate.
*/
template<class Trait>
struct op_not
	: public std::integral_constant<bool,
		!Trait::value
	>
{};

/**
	Logical NOT of a trait template.

	@tparam TraitTpl Trait template to negate.
*/
template<template<typename> class TraitTpl>
struct op_not_t final {
	/**
		Trait template adapter.

		@tparam T Type to apply to @a TraitTpl.
	*/
	template<typename T>
	struct type
		: public std::integral_constant<bool,
			!TraitTpl<T>::value
		>
	{};
};

/** @} */ // end of name-group Trait operators

/**
	@name Traits
	@{
*/

/**
	Whether a type is either copy constructible or copy assignable.

	@tparam T Type to test.
*/
template<typename T>
struct is_copyable
	: public std::integral_constant<bool,
		std::is_copy_constructible<T>::value ||
		std::is_copy_assignable   <T>::value
	>
{};

/**
	Whether a type is either move constructible or move assignable.

	@tparam T Type to test.
*/
template<typename T>
struct is_moveable
	: public std::integral_constant<bool,
		std::is_move_constructible<T>::value ||
		std::is_move_assignable   <T>::value
	>
{};

/** @} */ // end of name-group Traits

/** @} */ // end of doc-group traits

} // namespace trait_wrangler

#endif // TRAIT_WRANGLER_TRAITS_HPP_
<commit_msg>traits: added missing docs for logical trait-template operators.<commit_after>/**
@file traits.hpp
@brief Non-standard traits and trait operators.

@author Tim Howard
@copyright 2013 Tim Howard under the MIT license;
see @ref index or the accompanying LICENSE file for full text.
*/

#ifndef TRAIT_WRANGLER_TRAITS_HPP_
#define TRAIT_WRANGLER_TRAITS_HPP_

#include "./config.hpp"
#include "./detail/traits.hpp"

#include <type_traits>

namespace trait_wrangler {

/**
	@addtogroup traits
	@{
*/

/**
	@name Trait operators
	@{
*/

/**
	Logical AND of traits.

	@param ...TraitP Traits.
*/
template<class... TraitP>
struct op_and
	: public ::trait_wrangler::detail::op_and<TraitP...>
{};

/**
	Logical AND of trait templates.

	@tparam TplP Trait templates.
*/
template<template<typename> class... TplP>
struct op_and_t final {
	/**
		Trait template adapter.

		@tparam T Type to apply to @a TplP.
	*/
	template<typename T>
	struct type
		: public ::trait_wrangler::detail::op_and_t<T, TplP...>
	{};
};

/**
	Logical OR of traits.

	@param ...TraitP Traits.
*/
template<class... TraitP>
struct op_or
	: public ::trait_wrangler::detail::op_or<TraitP...>
{};

/**
	Logical OR of trait templates.

	@tparam TplP Trait templates.
*/
template<template<typename> class... TplP>
struct op_or_t final {
	/**
		Trait template adapter.

		@tparam T Type to apply to @a TplP.
	*/
	template<typename T>
	struct type
		: public ::trait_wrangler::detail::op_or_t<T, TplP...>
	{};
};

/**
	Logical NOT of a trait.

	@tparam Trait Trait to negate.
*/
template<class Trait>
struct op_not
	: public std::integral_constant<bool,
		!Trait::value
	>
{};

/**
	Logical NOT of a trait template.

	@tparam TraitTpl Trait template to negate.
*/
template<template<typename> class TraitTpl>
struct op_not_t final {
	/**
		Trait template adapter.

		@tparam T Type to apply to @a TraitTpl.
	*/
	template<typename T>
	struct type
		: public std::integral_constant<bool,
			!TraitTpl<T>::value
		>
	{};
};

/** @} */ // end of name-group Trait operators

/**
	@name Traits
	@{
*/

/**
	Whether a type is either copy constructible or copy assignable.

	@tparam T Type to test.
*/
template<typename T>
struct is_copyable
	: public std::integral_constant<bool,
		std::is_copy_constructible<T>::value ||
		std::is_copy_assignable   <T>::value
	>
{};

/**
	Whether a type is either move constructible or move assignable.

	@tparam T Type to test.
*/
template<typename T>
struct is_moveable
	: public std::integral_constant<bool,
		std::is_move_constructible<T>::value ||
		std::is_move_assignable   <T>::value
	>
{};

/** @} */ // end of name-group Traits

/** @} */ // end of doc-group traits

} // namespace trait_wrangler

#endif // TRAIT_WRANGLER_TRAITS_HPP_
<|endoftext|>


<commit_before>/*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
 *
 *  DataDifferential Utility Library
 *
 *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *
 *      * Redistributions of source code must retain the above copyright
 *  notice, this list of conditions and the following disclaimer.
 *
 *      * Redistributions in binary form must reproduce the above
 *  copyright notice, this list of conditions and the following disclaimer
 *  in the documentation and/or other materials provided with the
 *  distribution.
 *
 *      * The names of its contributors may not be used to endorse or
 *  promote products derived from this software without specific prior
 *  written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <config.h>

#include "util/logfile.hpp"

#include <cerrno>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fcntl.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <sys/stat.h>
#include <sys/types.h>

namespace datadifferential {
namespace util {

Logfile::Logfile(const std::string &arg) :
  _filename(arg)
{
  time_t tmp= time(NULL);
  _log_file << "shutdown: " << ctime(&tmp) << std::endl;
}

Logfile::~Logfile()
{
  if (not _filename.empty())
  {
    _log_file.close();
    if (access(_filename.c_str(), F_OK) == -1)
    { }
    else if (unlink(_filename.c_str()) == -1)
    { }
  }
}

bool Logfile::open()
{
  if (_filename.empty())
  {
    _log_file.open("/dev/stderr");
    return true;
  }

  _log_file.open(_filename.c_str());
  if (not _log_file.good())
  {
    return false;
  }

  time_t tmp= time(NULL);
  _log_file << "startup: " << ctime(&tmp) << std::endl;

  return true;
}

} /* namespace util */
} /* namespace datadifferential */
<commit_msg>Additional fix for include files for Fedora17<commit_after>/*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
 *
 *  DataDifferential Utility Library
 *
 *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *
 *      * Redistributions of source code must retain the above copyright
 *  notice, this list of conditions and the following disclaimer.
 *
 *      * Redistributions in binary form must reproduce the above
 *  copyright notice, this list of conditions and the following disclaimer
 *  in the documentation and/or other materials provided with the
 *  distribution.
 *
 *      * The names of its contributors may not be used to endorse or
 *  promote products derived from this software without specific prior
 *  written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <config.h>

#include "util/logfile.hpp"

#include <cerrno>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fcntl.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

namespace datadifferential {
namespace util {

Logfile::Logfile(const std::string &arg) :
  _filename(arg)
{
  time_t tmp= time(NULL);
  _log_file << "shutdown: " << ctime(&tmp) << std::endl;
}

Logfile::~Logfile()
{
  if (not _filename.empty())
  {
    _log_file.close();
    if (access(_filename.c_str(), F_OK) == -1)
    { }
    else if (unlink(_filename.c_str()) == -1)
    { }
  }
}

bool Logfile::open()
{
  if (_filename.empty())
  {
    _log_file.open("/dev/stderr");
    return true;
  }

  _log_file.open(_filename.c_str());
  if (not _log_file.good())
  {
    return false;
  }

  time_t tmp= time(NULL);
  _log_file << "startup: " << ctime(&tmp) << std::endl;

  return true;
}

} /* namespace util */
} /* namespace datadifferential */
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2014-2015 Pavel Dolgov
 *
 * See the LICENSE file for terms of use.
 */

#ifndef CHECKER_HPP_
#define CHECKER_HPP_

#include "game_desk.hpp"
#include "points.hpp"

class Checker {
public:
    bool checkStep(const GameDesk& desk,
                   const Points& points) const;

    bool checkIndex(const Points& points) const;
};

#endif

<commit_msg>Checker::checkIndex() must be private!<commit_after>/*
 * Copyright (C) 2014-2015 Pavel Dolgov
 *
 * See the LICENSE file for terms of use.
 */

#ifndef CHECKER_HPP_
#define CHECKER_HPP_

#include "game_desk.hpp"
#include "points.hpp"

class Checker {
public:
    bool checkStep(const GameDesk& desk,
                   const Points& points) const;

private:
    bool checkIndex(const Points& points) const;
};

#endif

<|endoftext|>


<commit_before>#include <assert.h>

// SFS includes
#include <refcnt.h>
#include <err.h>
#include <crypt.h>

#include "modlogger.h"
#define idatrace modlogger ("ida")
#include "ida.h"

// Auto-generated field inverses file
#include "ida-field.C"

// If you change this, it must be prime and then you will
// have to be very careful about how well the operations below
// execute in 32-bit variables.   Basically, the code is not
// really flexible enough to allow you to arbitrarily change
// this constant.  It may be safer to lower it than raise it,
// but why would you want to do that?
// Also, you'll need to change the constant in ida-genfield.py.
const u_long Ida::field = 65537;

const char Ida::magic = 0xef;

inline u_long
Ida::DIV (u_long a, u_long b)
{
  return MUL (a, INV(b));
}

inline u_long
Ida::MUL (u_long a, u_long b)
{
  // If you change field, better think about this.
  assert (field == 65537);
  assert (sizeof (u_long) >= 4);

  // field - 1 is -1, it's square is 1.  In the case
  // where field is p = 65537, only -1 will overflow an unsigned
  // 32-bit number.
  if (a == field - 1 && b == field - 1)
    return 1;

  return (a * b % field);
}

inline u_long
Ida::SUB (u_long a, u_long b)
{
  return ((a - b) + field) % field;
}

inline u_long
Ida::ADD (u_long a, u_long b)
{
  return (a + b) % field;
}

inline u_long
Ida::INV (u_long a)
{
  return inv[a];
}

u_long
Ida::optimal_dfrag (u_long len, u_long mtu)
{
  // Calculate best packing for fragments to minimize packet count.
  // Estimate first assuming no per-fragment overhead.
  u_long m = (len + (mtu - 1)) / mtu;
  if (m == 0) m = 1; // Handle special case if len == 0
  // Check to see that a fragment would really fit, with overhead
  u_long nomfragsize = (len / m) + (4 + m)*2;
  // If it doesn't fit (i.e. would need to chunk), might as well parallelize.
  if (nomfragsize > mtu)
    m++;
  return m;
}

/*
 * Generate one piece.
 * Clears out and replaces it with the following:
 *  Total # of FIELD elements (including this one).
 *  Length of decoded block
 *  M
 *  M-element matrix row.
 *  N/m
 *  N/m data elements.
 * While we operate with u_longs, each element will really be 2 bytes.
 */
void
Ida::gen_frag_ (int m, const str &in, vec<u_long> &out)
{
  size_t rawlen = in.len ();
  size_t fraglen = (rawlen + 2*m - 1) / (2*m);

  size_t total = 3 + m + 1 + fraglen;
  out.clear ();
  out.setsize (total);
  size_t outp = 0;

  out[outp++] = total;
  out[outp++] = rawlen;
  out[outp++] = m;

  /* generate our matrix row; these are the 'a's */
  vec<u_long> v;
  v.setsize (m);
  for (int k = 0; k < m; k++) {
    u_long vi = 0;
    while (vi == 0)
      vi = random_getword () % field;
    out[outp++] = vi;
    v[k] = vi;
  }
#if 0
  for (int k = 0; k < m; k++) {
    warnx ("%8lu", v[k]);
  }
  warnx << "\n";
#endif /* 0 */
  out[outp++] = fraglen;

  u_long cik, inp;
  for (size_t k = 0; k < fraglen; k++) {
    cik = 0;
    inp = k*2*m;
    for (int j = 0; j < m; j++) {
      // Encode two bytes at a time.
      u_long bi = (inp > rawlen) ? 0 : (u_char) in[inp++];
      bi <<= 8;
      bi |= (inp > rawlen) ? 0 : (u_char) in[inp++];
      assert (bi == (bi & 0x0000ffff));

      cik = ADD(cik, MUL(bi, v[j]));
      assert (cik < field);
    }
    out[outp++] = cik;
  }
}

/*
 * Packs in into a string and returns it.
 */
str
Ida::pack (vec<u_long> &in)
{
  int i, n;

  n = in[0]; /* over length */

  strbuf out;
  // This encoding takes advantage of the fact that the 32-bit in[]
  // values rarely have anything in the high 16 bits.
  for (i = 0; i < n; i++) {
    register u_long c = in[i];
    // The magic character and high-bit nums must be encoded carefully.
    // NOTE: if field == 65537, the highest order byte should always be zero.
    assert (!(c & 0xff000000));
    char c1 = (char) (c >> 8) & 0xff;
    char c2 = (char) c & 0xff;
    if (c & 0xffff0000) {
      out.fmt ("%c%c%c%c",
	       magic,
	       (char) (c >> 16) & 0xff,
	       c1,
	       c2);
    } else {
      if (c1 == magic) {
	out.fmt ("%c%c%c%c", magic, 0, c1, c2);
      } else {
	out.fmt ("%c%c", c1, c2);
      }
    }
  }
  // XXX there is a copy here, but I'm not sure we can really get
  //     rid of it the way that str's and strbuf's are structured.
  //     The other option would be to use a char array but then
  //     callers would have to worry about delete'ing the memory.
  return out;
}

str
Ida::gen_frag (int m, const str &in)
{
  assert (m);
  vec<u_long> tmp;
  gen_frag_ (m, in, tmp);
  // This copy is cheap; only refcounting involved.
  return pack (tmp);
}

// =========

inline u_long
Ida::unpackone (const str &in, int &inp)
{
  // XXX handle cases where input string is too short
  u_long x;
  if (in[inp] != magic)
    x = ((u_char) in[inp++] << 8) | (u_char) in[inp++];
  else {
    inp++;
    x = (((u_char) in[inp++] << 16) |
	 ((u_char) in[inp++] <<  8) |
	 ((u_char) in[inp++]));
  }
  assert(!(x & 0xff000000));
  return x;
}

bool
Ida::unpack (const str &in, vec<u_long> &out)
{
  int inp = 0;
  u_long n = unpackone (in, inp);

  inp = 0;
  out.setsize (n);
  for (u_long i = 0; i < n; i++) {
    out[i] = unpackone (in, inp);
  }

  return true;
}

/*
 * Decodes the fragments in frags and returns recoded block in out.
 */
bool
Ida::reconstruct (const vec<str> &frags, strbuf &out)
{
  if (frags.size () == 0) {
    idatrace << "no fragments!\n";
    return false;
  }
  int inp = 0;

#if 0
  // Select a random combination of the supplied fragments.
  vec<ptrdiff_t> o;
  o.setsize (frags.size ());
#endif /* 0 */

  u_long len = unpackone (frags[0], inp);
  u_long rawlen = unpackone (frags[0], inp);
  u_long m = unpackone (frags[0], inp);
  if (len < m + 4) {
    idatrace << "fragment 0 too short; coded length " << frags[0].len () << "\n";
    return false;
  }
  if (frags.size () < m) {
    //idatrace << "not enough fragments (" << frags.size () << "/" << m << ").\n";
    return false;
  }

  u_long blocksize = 0;

  vec<vec<u_long> > dfrags; // data section of fragments
  vec<vec<u_long> > a;      // encoding matrix for fragments

  // Sanity check the fragments and extract the encoding matrix.
  // Use a customized unpack to avoid some memory copies.
  for (size_t i = 0; i < m; i++) {
    int inp = 0;
    str in = frags[i];
    len = unpackone (in, inp);
    if (len < m + 4) {
      idatrace << "fragment " << i << " length " << len
	       << " too short; want at least " << m + 4 << "; coded length "
	       << in.len () << ".\n";
      return false;
    }
    u_long myrawlen = unpackone (in, inp);
    if (myrawlen != rawlen) {
      idatrace << "fragment " << i << " rawlen = " << myrawlen
	       << " not consistent; expected rawlen = " << rawlen << "\n";
      return false;
    }
    u_long mym = unpackone (in, inp);
    if (mym != m) {
      idatrace << "fragment " << i << " m = " << mym
	       << " not consistent; expected m = " << m << "\n";
      return false;
    }

    // Extract row of encode matrix
    vec<u_long> arow;
    for (size_t j = 0; j < m; j++) {
      arow.push_back (unpackone (in, inp));
    }

    u_long myblocksize = unpackone (in, inp);
    if (!blocksize)
      blocksize = myblocksize;
    if (blocksize != myblocksize) {
      idatrace << "fragment " << i << " block length " << myblocksize
	       << " not consistent; expected " << blocksize << "\n";
      idatrace << hexdump (in.cstr (), in.len ()) << "\n";
      return false;
    }
    len -= m + 4;
    assert (len == blocksize);

    // Extract encoded block
    vec<u_long> drow;
    while (len > 0) {
      drow.push_back (unpackone (in, inp));
      len--;
    }

    // If all good, push things onto our to-be-decoded matrix.
    a.push_back (arow);
    dfrags.push_back (drow);
  }

  // Calculate the decode matrix
  vec<vec<u_long> > a_inv;
  bool ok = minvert (m, m, a, a_inv);
  if (!ok) {
    idatrace << "couldn't invert matrix.\n";
    return false;
  }

  // Produce the decoded block.
  vec<u_long> c;
  c.setsize (m);
  char *outb = out.tosuio ()->getspace (blocksize * 2 * m);
  size_t outp = 0;
  for (size_t inp = 0; inp < blocksize; inp++) {
    for (size_t i = 0; i < m; i++) {
      c[i] = dfrags[i][inp];
    }
    for (size_t i = 0; i < m; i++) {
      u_long b = 0;
      for (size_t j = 0; j < m; j++)
	b = ADD (b, MUL(c[j], a_inv[i][j]));
      assert (!(b & 0xffff0000));
      outb[outp++] = (char) (b >> 8) & 0xff;
      outb[outp++] = (char) (b       & 0xff);
    }
  }
  out.tosuio ()->print (outb, rawlen);
  return true;
}

bool
Ida::minvert (int rows, int cols,
	      vec<vec<u_long> > &from,
	      vec<vec<u_long> > &to)
{
  int r, c, r1;
  u_long x, y;

  to.setsize (0);
  to.reserve (rows);
  for (r = 0; r < rows; r++) {
    vec<u_long> row;
    row.reserve (cols);
    for (c = 0; c < cols; c++)
      row.push_back (from[r][c]);
    for (c = 0; c < cols; c++)
      row.push_back (r == c);
    to.push_back (row);
  }

  // Row reduction
  for (r = 0; r < rows; r++){
    x = to[r][r];

    // No pivot? [Swap rows??? XXX]
    if (x == 0) {
      for (r = 0; r < rows; r++) {
	strbuf s;
	for (c = 0; c < cols; c++)
	  s.fmt ("%8lu,", from[r][c]);
	idatrace << "from: " << s << "\n";
      }
      for (r = 0; r < rows; r++) {
	strbuf s;
	for (c = 0; c < 2*cols; c++)
	  s.fmt ("%8lu,", to[r][c]);
	idatrace << "to:   " << s << "\n";
      }
      return false;
    }

    for (c = 0; c < cols*2; c++)
      to[r][c] = DIV(to[r][c], x);
    for (r1 = 0; r1 < rows; r1++){
      if (r1 == r)
	continue;
      y = DIV(to[r1][r], to[r][r]);
      for (c = 0; c < cols*2; c++) {
	to[r1][c] = SUB(to[r1][c], MUL(y, to[r][c]));
      }
    }
  }

#if 0
  for (r = 0; r < rows; r++) {
    for (c = 0; c < 2*cols; c++)
      warnx ("%8lu,", to[r][c]);
    warnx ("\n");
  }
#endif /* 0 */

  for (r = 0; r < rows; r++) {
    to[r].popn_front (cols);
  }
  return true;
}

#if 0
str
Ida::frag_get_header (const str &in)
{
  // A lot of trouble just to get the true end of the header.
  int inp = 0;
  (void) unpackone (in, inp);
  (void) unpackone (in, inp);
  u_long m = unpackone (in, inp);

  for (size_t j = 0; j < m; j++) {
    (void) unpackone (in, inp);
  }
  (void) unpackone (in, inp);
  return substr (in, 0, inp);
}
#endif /* 0 */
<commit_msg>"randomly" shuffle fragments before re-construction. the first fragment is chosen to set the parameters for the re-construction attempt. skip any fragment whose parameters do not agree with the first fragment. if there are still enough fragments left, re-construct the block.<commit_after>#include <assert.h>

// SFS includes
#include <refcnt.h>
#include <err.h>
#include <crypt.h>

#include "modlogger.h"
#define idatrace modlogger ("ida")
#include "ida.h"

// Auto-generated field inverses file
#include "ida-field.C"

// If you change this, it must be prime and then you will
// have to be very careful about how well the operations below
// execute in 32-bit variables.   Basically, the code is not
// really flexible enough to allow you to arbitrarily change
// this constant.  It may be safer to lower it than raise it,
// but why would you want to do that?
// Also, you'll need to change the constant in ida-genfield.py.
const u_long Ida::field = 65537;

const char Ida::magic = 0xef;

inline u_long
Ida::DIV (u_long a, u_long b)
{
  return MUL (a, INV(b));
}

inline u_long
Ida::MUL (u_long a, u_long b)
{
  // If you change field, better think about this.
  assert (field == 65537);
  assert (sizeof (u_long) >= 4);

  // field - 1 is -1, it's square is 1.  In the case
  // where field is p = 65537, only -1 will overflow an unsigned
  // 32-bit number.
  if (a == field - 1 && b == field - 1)
    return 1;

  return (a * b % field);
}

inline u_long
Ida::SUB (u_long a, u_long b)
{
  return ((a - b) + field) % field;
}

inline u_long
Ida::ADD (u_long a, u_long b)
{
  return (a + b) % field;
}

inline u_long
Ida::INV (u_long a)
{
  return inv[a];
}

u_long
Ida::optimal_dfrag (u_long len, u_long mtu)
{
  // Calculate best packing for fragments to minimize packet count.
  // Estimate first assuming no per-fragment overhead.
  u_long m = (len + (mtu - 1)) / mtu;
  if (m == 0) m = 1; // Handle special case if len == 0
  // Check to see that a fragment would really fit, with overhead
  u_long nomfragsize = (len / m) + (4 + m)*2;
  // If it doesn't fit (i.e. would need to chunk), might as well parallelize.
  if (nomfragsize > mtu)
    m++;
  return m;
}

/*
 * Generate one piece.
 * Clears out and replaces it with the following:
 *  Total # of FIELD elements (including this one).
 *  Length of decoded block
 *  M
 *  M-element matrix row.
 *  N/m
 *  N/m data elements.
 * While we operate with u_longs, each element will really be 2 bytes.
 */
void
Ida::gen_frag_ (int m, const str &in, vec<u_long> &out)
{
  size_t rawlen = in.len ();
  size_t fraglen = (rawlen + 2*m - 1) / (2*m);

  size_t total = 3 + m + 1 + fraglen;
  out.clear ();
  out.setsize (total);
  size_t outp = 0;

  out[outp++] = total;
  out[outp++] = rawlen;
  out[outp++] = m;

  /* generate our matrix row; these are the 'a's */
  vec<u_long> v;
  v.setsize (m);
  for (int k = 0; k < m; k++) {
    u_long vi = 0;
    while (vi == 0)
      vi = random_getword () % field;
    out[outp++] = vi;
    v[k] = vi;
  }
#if 0
  for (int k = 0; k < m; k++) {
    warnx ("%8lu", v[k]);
  }
  warnx << "\n";
#endif /* 0 */
  out[outp++] = fraglen;

  u_long cik, inp;
  for (size_t k = 0; k < fraglen; k++) {
    cik = 0;
    inp = k*2*m;
    for (int j = 0; j < m; j++) {
      // Encode two bytes at a time.
      u_long bi = (inp > rawlen) ? 0 : (u_char) in[inp++];
      bi <<= 8;
      bi |= (inp > rawlen) ? 0 : (u_char) in[inp++];
      assert (bi == (bi & 0x0000ffff));

      cik = ADD(cik, MUL(bi, v[j]));
      assert (cik < field);
    }
    out[outp++] = cik;
  }
}

/*
 * Packs in into a string and returns it.
 */
str
Ida::pack (vec<u_long> &in)
{
  int i, n;

  n = in[0]; /* over length */

  strbuf out;
  // This encoding takes advantage of the fact that the 32-bit in[]
  // values rarely have anything in the high 16 bits.
  for (i = 0; i < n; i++) {
    register u_long c = in[i];
    // The magic character and high-bit nums must be encoded carefully.
    // NOTE: if field == 65537, the highest order byte should always be zero.
    assert (!(c & 0xff000000));
    char c1 = (char) (c >> 8) & 0xff;
    char c2 = (char) c & 0xff;
    if (c & 0xffff0000) {
      out.fmt ("%c%c%c%c",
	       magic,
	       (char) (c >> 16) & 0xff,
	       c1,
	       c2);
    } else {
      if (c1 == magic) {
	out.fmt ("%c%c%c%c", magic, 0, c1, c2);
      } else {
	out.fmt ("%c%c", c1, c2);
      }
    }
  }
  // XXX there is a copy here, but I'm not sure we can really get
  //     rid of it the way that str's and strbuf's are structured.
  //     The other option would be to use a char array but then
  //     callers would have to worry about delete'ing the memory.
  return out;
}

str
Ida::gen_frag (int m, const str &in)
{
  assert (m);
  vec<u_long> tmp;
  gen_frag_ (m, in, tmp);
  // This copy is cheap; only refcounting involved.
  return pack (tmp);
}

// =========

inline u_long
Ida::unpackone (const str &in, int &inp)
{
  // XXX handle cases where input string is too short
  u_long x;
  if (in[inp] != magic)
    x = ((u_char) in[inp++] << 8) | (u_char) in[inp++];
  else {
    inp++;
    x = (((u_char) in[inp++] << 16) |
	 ((u_char) in[inp++] <<  8) |
	 ((u_char) in[inp++]));
  }
  assert(!(x & 0xff000000));
  return x;
}

bool
Ida::unpack (const str &in, vec<u_long> &out)
{
  int inp = 0;
  u_long n = unpackone (in, inp);

  inp = 0;
  out.setsize (n);
  for (u_long i = 0; i < n; i++) {
    out[i] = unpackone (in, inp);
  }

  return true;
}

static void
drop (vec<str> &frags, unsigned i)
{
  for (unsigned x = i; x+1 < frags.size (); x++) {
    frags [x] = frags [x+1];
  }
  frags.pop_back ();
}

static void
random_shuffle (vec<str> &frags)
{
  for (unsigned i=0; i<frags.size (); i++) {
    int r = rand () % (frags.size ());
    if ((unsigned)r == i) continue;
    str t = frags [i];
    frags [i] = frags [r];
    frags [r] = t;
  }
}

/*
 * Decodes the fragments in frags and returns recoded block in out.
 */
bool
Ida::reconstruct (const vec<str> &f, strbuf &out)
{
  if (f.size () == 0) {
    idatrace << "no fragments!\n";
    return false;
  }
  int inp = 0;

  vec<str> frags = f;
  random_shuffle (frags);

  u_long len = unpackone (frags[0], inp);
  u_long rawlen = unpackone (frags[0], inp);
  u_long m = unpackone (frags[0], inp);
  if (len < m + 4) {
    idatrace
      << "fragment 0 too short; coded length " << frags[0].len () << "\n";
    return false;
  }
  if (frags.size () < m) {
#if 0
    idatrace
      << "not enough fragments (" << frags.size () << "/" << m << ").\n";
#endif
    return false;
  }

  u_long blocksize = 0;

  vec<vec<u_long> > dfrags; // data section of fragments
  vec<vec<u_long> > a;      // encoding matrix for fragments

  // Sanity check the fragments and extract the encoding matrix.
  // Use a customized unpack to avoid some memory copies.
  for (size_t i = 0; i < m; i++) {
    int inp = 0;
    str in = frags[i];
    len = unpackone (in, inp);
    if (len < m + 4) {
      idatrace << "fragment " << i << " length " << len
	       << " too short; want at least " << m + 4 << "; coded length "
	       << in.len () << ".\n";
      drop (frags, i);
      continue;
    }
    u_long myrawlen = unpackone (in, inp);
    if (myrawlen != rawlen) {
      idatrace << "fragment " << i << " rawlen = " << myrawlen
	       << " not consistent; expected rawlen = " << rawlen << "\n";
      drop (frags, i);
      continue;
    }
    u_long mym = unpackone (in, inp);
    if (mym != m) {
      idatrace << "fragment " << i << " m = " << mym
	       << " not consistent; expected m = " << m << "\n";
      drop (frags, i);
      continue;
    }

    // Extract row of encode matrix
    vec<u_long> arow;
    for (size_t j = 0; j < m; j++) {
      arow.push_back (unpackone (in, inp));
    }

    u_long myblocksize = unpackone (in, inp);
    if (!blocksize)
      blocksize = myblocksize;
    if (blocksize != myblocksize) {
      idatrace << "fragment " << i << " block length " << myblocksize
	       << " not consistent; expected " << blocksize << "\n";
      idatrace << hexdump (in.cstr (), in.len ()) << "\n";
      drop (frags, i);
      continue;
    }
    len -= m + 4;
    assert (len == blocksize);

    // Extract encoded block
    vec<u_long> drow;
    while (len > 0) {
      drow.push_back (unpackone (in, inp));
      len--;
    }

    // If all good, push things onto our to-be-decoded matrix.
    a.push_back (arow);
    dfrags.push_back (drow);
  }

  if (frags.size () < m) {
    idatrace << "not enough fragments left\n";
    return false;
  }

  // Calculate the decode matrix
  vec<vec<u_long> > a_inv;
  bool ok = minvert (m, m, a, a_inv);
  if (!ok) {
    idatrace << "couldn't invert matrix.\n";
    return false;
  }

  // Produce the decoded block.
  vec<u_long> c;
  c.setsize (m);
  char *outb = out.tosuio ()->getspace (blocksize * 2 * m);
  size_t outp = 0;
  for (size_t inp = 0; inp < blocksize; inp++) {
    for (size_t i = 0; i < m; i++) {
      c[i] = dfrags[i][inp];
    }
    for (size_t i = 0; i < m; i++) {
      u_long b = 0;
      for (size_t j = 0; j < m; j++)
	b = ADD (b, MUL(c[j], a_inv[i][j]));
      assert (!(b & 0xffff0000));
      outb[outp++] = (char) (b >> 8) & 0xff;
      outb[outp++] = (char) (b       & 0xff);
    }
  }
  out.tosuio ()->print (outb, rawlen);
  return true;
}

bool
Ida::minvert (int rows, int cols,
	      vec<vec<u_long> > &from,
	      vec<vec<u_long> > &to)
{
  int r, c, r1;
  u_long x, y;

  to.setsize (0);
  to.reserve (rows);
  for (r = 0; r < rows; r++) {
    vec<u_long> row;
    row.reserve (cols);
    for (c = 0; c < cols; c++)
      row.push_back (from[r][c]);
    for (c = 0; c < cols; c++)
      row.push_back (r == c);
    to.push_back (row);
  }

  // Row reduction
  for (r = 0; r < rows; r++){
    x = to[r][r];

    // No pivot? [Swap rows??? XXX]
    if (x == 0) {
      for (r = 0; r < rows; r++) {
	strbuf s;
	for (c = 0; c < cols; c++)
	  s.fmt ("%8lu,", from[r][c]);
	idatrace << "from: " << s << "\n";
      }
      for (r = 0; r < rows; r++) {
	strbuf s;
	for (c = 0; c < 2*cols; c++)
	  s.fmt ("%8lu,", to[r][c]);
	idatrace << "to:   " << s << "\n";
      }
      return false;
    }

    for (c = 0; c < cols*2; c++)
      to[r][c] = DIV(to[r][c], x);
    for (r1 = 0; r1 < rows; r1++){
      if (r1 == r)
	continue;
      y = DIV(to[r1][r], to[r][r]);
      for (c = 0; c < cols*2; c++) {
	to[r1][c] = SUB(to[r1][c], MUL(y, to[r][c]));
      }
    }
  }

#if 0
  for (r = 0; r < rows; r++) {
    for (c = 0; c < 2*cols; c++)
      warnx ("%8lu,", to[r][c]);
    warnx ("\n");
  }
#endif /* 0 */

  for (r = 0; r < rows; r++) {
    to[r].popn_front (cols);
  }
  return true;
}

#if 0
str
Ida::frag_get_header (const str &in)
{
  // A lot of trouble just to get the true end of the header.
  int inp = 0;
  (void) unpackone (in, inp);
  (void) unpackone (in, inp);
  u_long m = unpackone (in, inp);

  for (size_t j = 0; j < m; j++) {
    (void) unpackone (in, inp);
  }
  (void) unpackone (in, inp);
  return substr (in, 0, inp);
}
#endif /* 0 */
<|endoftext|>