<commit_before>#include <Windows.h>
#include <cstdio>
#include <cstring>

using namespace std;

enum DisplaySelection {
	DS_MONITOR = 1,
	DS_HDMI = 2,
	DS_TOGGLE = 3,
};

/* This array must match the above enum */
char *DisplaySelectionString[] = {
	"Monitor",
	"HDMI",
	"Toggle",
};

#define MAX_PATH_ELEMENTS 10
#define MAX_MODE_ELEMENTS 10

int main(int argc, char *argv[])
{
	LONG retval;
	DISPLAYCONFIG_PATH_INFO pathArray[MAX_PATH_ELEMENTS];
	DISPLAYCONFIG_MODE_INFO modeArray[MAX_MODE_ELEMENTS];
	UINT32 numPathElements = sizeof(pathArray)/sizeof(pathArray[0]);
	UINT32 numModeElements = sizeof(modeArray)/sizeof(modeArray[0]);
	UINT32 hdmi_idx = MAX_PATH_ELEMENTS;
	UINT32 dvi_idx = MAX_PATH_ELEMENTS;
	DisplaySelection current_selection = DS_TOGGLE;

	retval = QueryDisplayConfig(QDC_ALL_PATHS, &numPathElements, pathArray, &numModeElements, modeArray, NULL);

	if (ERROR_SUCCESS != retval) {
		printf("Query failure - unable to set new path!!\n");
		return 2;
	}

	/* Find the target paths we care about */
	for (unsigned int path = 0; path < numPathElements; path++) {
		if (pathArray[path].sourceInfo.modeInfoIdx >= 0
			&& pathArray[path].sourceInfo.modeInfoIdx < numModeElements) {

			if ((pathArray[path].targetInfo.outputTechnology == DISPLAYCONFIG_OUTPUT_TECHNOLOGY_HDMI)
				&& (pathArray[path].targetInfo.targetAvailable == TRUE)) {
				hdmi_idx = path;
				if (DISPLAYCONFIG_PATH_ACTIVE & pathArray[path].flags)
					current_selection = DS_HDMI;
			}
			else if ((pathArray[path].targetInfo.outputTechnology == DISPLAYCONFIG_OUTPUT_TECHNOLOGY_DVI)
				&& (pathArray[path].targetInfo.targetAvailable == TRUE)) {
				dvi_idx = path;
				if (DISPLAYCONFIG_PATH_ACTIVE & pathArray[path].flags)
					current_selection = DS_MONITOR;
			}
		}
	}
	if (current_selection == DS_TOGGLE) {
		printf("Unable to determine current video output -- exiting\n");
		return 3;
	}

	/* No argument means toggle output */
	unsigned int input = DS_TOGGLE;
	DISPLAYCONFIG_PATH_INFO desiredSettings;

	/* Try to get the desired output setting */
	if (argc > 1) {
		input = atoi(argv[1]);
	}

	/* input should match the enum above. indexed into the string array as (input - 1) */
	if ((input - 1) < (sizeof(DisplaySelectionString) / sizeof(DisplaySelectionString[0]))) {
		printf("Device to be selected is the %s selection.\n", DisplaySelectionString[input - 1]);
		if ((DS_MONITOR == input && current_selection == DS_HDMI) || (DS_TOGGLE == input && DS_HDMI == current_selection)) {
			if (dvi_idx < MAX_PATH_ELEMENTS)
				desiredSettings = pathArray[dvi_idx];
			else {
				printf("Attempting to switch to DVI, but unable to find config element -- exiting\n");
				return 4;
			}
		}
		else if ((DS_HDMI == input && current_selection == DS_MONITOR) || (DS_TOGGLE == input && DS_MONITOR == current_selection))
		{
			if (hdmi_idx < MAX_PATH_ELEMENTS)
				desiredSettings = pathArray[hdmi_idx];
			else {
				printf("Attempting to switch to HDMI, but unable to find config element -- exiting\n");
				return 5;
			}
		}
		else
		{
			printf("Output already set to the desired selection.\n");
			return 0;
		}
	}
	else
	{
		printf("Unknown display selection: %d\n\n", input);
		printf("Usage:\n");
		printf("\tDisplayController_Config.exe [new_output]\n");
		for (int i = 0; i < sizeof(DisplaySelectionString) / sizeof(DisplaySelectionString[0]); i++) {
			printf("\t\t%d: %s\n", i + 1, DisplaySelectionString[i]);
		}
		printf("\n\tNo parameter means Toggle\n");
		return 1;
	}

	/* If we get here we will attempt to set the path -- let SetDisplayConfig pick the mode info */
	desiredSettings.flags = DISPLAYCONFIG_PATH_ACTIVE;
	desiredSettings.targetInfo.modeInfoIdx = DISPLAYCONFIG_PATH_MODE_IDX_INVALID;
	desiredSettings.sourceInfo.modeInfoIdx = DISPLAYCONFIG_PATH_MODE_IDX_INVALID;
	retval = SetDisplayConfig(1, &desiredSettings, 0, NULL, SDC_APPLY | SDC_TOPOLOGY_SUPPLIED | SDC_ALLOW_PATH_ORDER_CHANGES);

	if (ERROR_SUCCESS != retval) {
		printf("Unable to set -- trying again with SetDisplayConfig creating mode info\n");
		retval = SetDisplayConfig(1, &desiredSettings, 0, NULL, SDC_TOPOLOGY_SUPPLIED | SDC_ALLOW_CHANGES |
													SDC_ALLOW_PATH_ORDER_CHANGES |SDC_USE_SUPPLIED_DISPLAY_CONFIG);
		if (ERROR_SUCCESS != retval) {
			printf("Unable to set the new display config!!\n");
			return 3;
		}
	}

	return 0;
}<commit_msg>Increase path elements buffer<commit_after>#include <Windows.h>
#include <cstdio>
#include <cstring>

using namespace std;

enum DisplaySelection {
	DS_MONITOR = 1,
	DS_HDMI = 2,
	DS_TOGGLE = 3,
};

/* This array must match the above enum */
char *DisplaySelectionString[] = {
	"Monitor",
	"HDMI",
	"Toggle",
};

#define MAX_PATH_ELEMENTS 128
#define MAX_MODE_ELEMENTS 8

int main(int argc, char *argv[])
{
	LONG retval;
	DISPLAYCONFIG_PATH_INFO pathArray[MAX_PATH_ELEMENTS];
	DISPLAYCONFIG_MODE_INFO modeArray[MAX_MODE_ELEMENTS];
	UINT32 numPathElements = sizeof(pathArray)/sizeof(pathArray[0]);
	UINT32 numModeElements = sizeof(modeArray)/sizeof(modeArray[0]);
	UINT32 hdmi_idx = MAX_PATH_ELEMENTS;
	UINT32 dvi_idx = MAX_PATH_ELEMENTS;
	DisplaySelection current_selection = DS_TOGGLE;

	retval = QueryDisplayConfig(QDC_ALL_PATHS, &numPathElements, pathArray, &numModeElements, modeArray, NULL);

	if (ERROR_SUCCESS != retval) {
		printf("Query failure - unable to set new path!!\n");
		return 2;
	}

	/* Find the target paths we care about */
	for (unsigned int path = 0; path < numPathElements; path++) {
		if (pathArray[path].sourceInfo.modeInfoIdx >= 0
			&& pathArray[path].sourceInfo.modeInfoIdx < numModeElements) {

			if ((pathArray[path].targetInfo.outputTechnology == DISPLAYCONFIG_OUTPUT_TECHNOLOGY_HDMI)
				&& (pathArray[path].targetInfo.targetAvailable == TRUE)) {
				hdmi_idx = path;
				if (DISPLAYCONFIG_PATH_ACTIVE & pathArray[path].flags)
					current_selection = DS_HDMI;
			}
			else if ((pathArray[path].targetInfo.outputTechnology == DISPLAYCONFIG_OUTPUT_TECHNOLOGY_DVI)
				&& (pathArray[path].targetInfo.targetAvailable == TRUE)) {
				dvi_idx = path;
				if (DISPLAYCONFIG_PATH_ACTIVE & pathArray[path].flags)
					current_selection = DS_MONITOR;
			}
		}
	}
	if (current_selection == DS_TOGGLE) {
		printf("Unable to determine current video output -- exiting\n");
		return 3;
	}

	/* No argument means toggle output */
	unsigned int input = DS_TOGGLE;
	DISPLAYCONFIG_PATH_INFO desiredSettings;

	/* Try to get the desired output setting */
	if (argc > 1) {
		input = atoi(argv[1]);
	}

	/* input should match the enum above. indexed into the string array as (input - 1) */
	if ((input - 1) < (sizeof(DisplaySelectionString) / sizeof(DisplaySelectionString[0]))) {
		printf("Device to be selected is the %s selection.\n", DisplaySelectionString[input - 1]);
		if ((DS_MONITOR == input && current_selection == DS_HDMI) || (DS_TOGGLE == input && DS_HDMI == current_selection)) {
			if (dvi_idx < MAX_PATH_ELEMENTS)
				desiredSettings = pathArray[dvi_idx];
			else {
				printf("Attempting to switch to DVI, but unable to find config element -- exiting\n");
				return 4;
			}
		}
		else if ((DS_HDMI == input && current_selection == DS_MONITOR) || (DS_TOGGLE == input && DS_MONITOR == current_selection))
		{
			if (hdmi_idx < MAX_PATH_ELEMENTS)
				desiredSettings = pathArray[hdmi_idx];
			else {
				printf("Attempting to switch to HDMI, but unable to find config element -- exiting\n");
				return 5;
			}
		}
		else
		{
			printf("Output already set to the desired selection.\n");
			return 0;
		}
	}
	else
	{
		printf("Unknown display selection: %d\n\n", input);
		printf("Usage:\n");
		printf("\tDisplayController_Config.exe [new_output]\n");
		for (int i = 0; i < sizeof(DisplaySelectionString) / sizeof(DisplaySelectionString[0]); i++) {
			printf("\t\t%d: %s\n", i + 1, DisplaySelectionString[i]);
		}
		printf("\n\tNo parameter means Toggle\n");
		return 1;
	}

	/* If we get here we will attempt to set the path -- let SetDisplayConfig pick the mode info */
	desiredSettings.flags = DISPLAYCONFIG_PATH_ACTIVE;
	desiredSettings.targetInfo.modeInfoIdx = DISPLAYCONFIG_PATH_MODE_IDX_INVALID;
	desiredSettings.sourceInfo.modeInfoIdx = DISPLAYCONFIG_PATH_MODE_IDX_INVALID;
	retval = SetDisplayConfig(1, &desiredSettings, 0, NULL, SDC_APPLY | SDC_TOPOLOGY_SUPPLIED | SDC_ALLOW_PATH_ORDER_CHANGES);

	if (ERROR_SUCCESS != retval) {
		printf("Unable to set -- trying again with SetDisplayConfig creating mode info\n");
		retval = SetDisplayConfig(1, &desiredSettings, 0, NULL, SDC_TOPOLOGY_SUPPLIED | SDC_ALLOW_CHANGES |
													SDC_ALLOW_PATH_ORDER_CHANGES |SDC_USE_SUPPLIED_DISPLAY_CONFIG);
		if (ERROR_SUCCESS != retval) {
			printf("Unable to set the new display config!!\n");
			return 3;
		}
	}

	return 0;
}<|endoftext|>


<commit_before>// [WriteFile Name=GenerateGeodatabase, Category=Features]
// [Legal]
// Copyright 2016 Esri.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// [Legal]

#include "GenerateGeodatabase.h"

#include "Map.h"
#include "MapQuickView.h"
#include "FeatureLayer.h"
#include "Basemap.h"
#include "SpatialReference.h"
#include "ServiceFeatureTable.h"
#include "ArcGISTiledLayer.h"
#include "Envelope.h"
#include "GenerateGeodatabaseParameters.h"
#include "GeodatabaseSyncTask.h"
#include "GeometryEngine.h"
#include "GenerateLayerOption.h"
#include "GeodatabaseFeatureTable.h"
#include <QUrl>
#include <QQmlProperty>

using namespace Esri::ArcGISRuntime;

GenerateGeodatabase::GenerateGeodatabase(QQuickItem* parent) :
    QQuickItem(parent),
    m_map(nullptr),
    m_mapView(nullptr),
    m_syncTask(nullptr),
    m_dataPath(""),
    m_featureServiceUrl("http://sampleserver6.arcgisonline.com/arcgis/rest/services/Sync/WildfireSync/FeatureServer/"),
    m_serviceIds(QStringList() << "0" << "1" << "2")
{
}

GenerateGeodatabase::~GenerateGeodatabase()
{
}

void GenerateGeodatabase::componentComplete()
{
    QQuickItem::componentComplete();

    // find QML MapView component
    m_mapView = findChild<MapQuickView*>("mapView");
    m_mapView->setWrapAroundMode(WrapAroundMode::Disabled);

    // Create a map using a local tile package
    m_dataPath = QQmlProperty::read(m_mapView, "dataPath").toString();
    TileCache* tileCache = new TileCache(m_dataPath + "tpk/SanFrancisco.tpk", this);
    ArcGISTiledLayer* tiledLayer = new ArcGISTiledLayer(tileCache, this);
    Basemap* basemap = new Basemap(tiledLayer, this);
    m_map = new Map(basemap, this);

    // Set map to map view
    m_mapView->setMap(m_map);

    // add feature layers to map
    addFeatureLayers(m_featureServiceUrl, m_serviceIds);

    // create the GeodatabaseSyncTask
    m_syncTask = new GeodatabaseSyncTask(QUrl(m_featureServiceUrl), this);
}

void GenerateGeodatabase::addFeatureLayers(QString serviceUrl, QStringList serviceIds)
{
    foreach (auto id, serviceIds)
    {
        ServiceFeatureTable* serviceFeatureTable = new ServiceFeatureTable(QUrl(serviceUrl + id));
        FeatureLayer* featureLayer = new FeatureLayer(serviceFeatureTable, this);
        m_map->operationalLayers()->append(featureLayer);
    }
}

GenerateGeodatabaseParameters GenerateGeodatabase::getUpdatedParameters(Envelope gdbExtent)
{
    // create the parameters
    GenerateGeodatabaseParameters params;
    params.setReturnAttachments(false);
    params.setOutSpatialReference(SpatialReference::webMercator());
    params.setExtent(gdbExtent);

    // set the layer options for all of the service IDs
    QList<GenerateLayerOption> layerOptions;
    foreach (auto id, m_serviceIds)
    {
        GenerateLayerOption generateLayerOption(id.toInt());
        layerOptions << generateLayerOption;
    }
    params.setLayerOptions(layerOptions);

    return params;
}

void GenerateGeodatabase::generateGeodatabaseFromCorners(double xCorner1, double yCorner1, double xCorner2, double yCorner2)
{
    // create an envelope from the QML rectangle corners
    auto corner1 = m_mapView->screenToLocation(xCorner1, yCorner1);
    auto corner2 = m_mapView->screenToLocation(xCorner2, yCorner2);
    auto extent = Envelope(corner1, corner2);
    auto geodatabaseExtent = GeometryEngine::project(extent, SpatialReference::webMercator());

    // get the updated parameters
    auto params = getUpdatedParameters(geodatabaseExtent);

    // execute the task and obtain the job
    auto generateJob = m_syncTask->generateGeodatabase(params, m_dataPath + "Wildfire.geodatabase");

    // connect to the job's status changed signal
    connect(generateJob, &GenerateGeodatabaseJob::jobStatusChanged, [this, generateJob]()
    {
        // connect to the job's status changed signal to know once it is done
        switch (generateJob->jobStatus()) {
        case JobStatus::Failed:
            emit updateStatus("Generate failed");
            emit hideWindow(5000, false);
            break;
        case JobStatus::NotStarted:
            emit updateStatus("Job not started");
            break;
        case JobStatus::Paused:
            emit updateStatus("Job paused");
            break;
        case JobStatus::Started:
            emit updateStatus("In progress...");
            break;
        case JobStatus::Succeeded:
            emit updateStatus("Complete");
            emit hideWindow(1500, true);
            addOfflineData(generateJob->result());
            break;
        default:
            break;
        }
    });

    // start the generate job
    generateJob->start();
}

void GenerateGeodatabase::addOfflineData(Geodatabase* gdb)
{
    // remove the original online feature layers
    m_map->operationalLayers()->clear();

    // load the geodatabase
    connect(gdb, &Geodatabase::doneLoading, [this, gdb](Error)
    {
        // create a feature layer from each feature table, and add to the map
        foreach (auto featureTable, gdb->geodatabaseFeatureTables())
        {
            FeatureLayer* featureLayer = new FeatureLayer(featureTable, this);
            m_map->operationalLayers()->append(featureLayer);
        }

        // unregister geodatabase since there will be no edits uploaded
        m_syncTask->unregisterGeodatabase(gdb);
    });
    gdb->load();
}
<commit_msg>Update GenerateGeodatabase.cpp<commit_after>// [WriteFile Name=GenerateGeodatabase, Category=Features]
// [Legal]
// Copyright 2016 Esri.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// [Legal]

#include "GenerateGeodatabase.h"

#include "Map.h"
#include "MapQuickView.h"
#include "FeatureLayer.h"
#include "Basemap.h"
#include "SpatialReference.h"
#include "ServiceFeatureTable.h"
#include "ArcGISTiledLayer.h"
#include "Envelope.h"
#include "GenerateGeodatabaseParameters.h"
#include "GeodatabaseSyncTask.h"
#include "GeometryEngine.h"
#include "GenerateLayerOption.h"
#include "GeodatabaseFeatureTable.h"
#include <QUrl>
#include <QQmlProperty>

using namespace Esri::ArcGISRuntime;

GenerateGeodatabase::GenerateGeodatabase(QQuickItem* parent) :
    QQuickItem(parent),
    m_map(nullptr),
    m_mapView(nullptr),
    m_syncTask(nullptr),
    m_dataPath(""),
    m_featureServiceUrl("http://sampleserver6.arcgisonline.com/arcgis/rest/services/Sync/WildfireSync/FeatureServer/"),
    m_serviceIds(QStringList() << "0" << "1" << "2")
{
}

GenerateGeodatabase::~GenerateGeodatabase()
{
}

void GenerateGeodatabase::componentComplete()
{
    QQuickItem::componentComplete();

    // find QML MapView component
    m_mapView = findChild<MapQuickView*>("mapView");
    m_mapView->setWrapAroundMode(WrapAroundMode::Disabled);

    // Create a map using a local tile package
    m_dataPath = QQmlProperty::read(m_mapView, "dataPath").toString();
    TileCache* tileCache = new TileCache(m_dataPath + "tpk/SanFrancisco.tpk", this);
    ArcGISTiledLayer* tiledLayer = new ArcGISTiledLayer(tileCache, this);
    Basemap* basemap = new Basemap(tiledLayer, this);
    m_map = new Map(basemap, this);

    // Set map to map view
    m_mapView->setMap(m_map);

    // add feature layers to map
    addFeatureLayers(m_featureServiceUrl, m_serviceIds);

    // create the GeodatabaseSyncTask
    m_syncTask = new GeodatabaseSyncTask(QUrl(m_featureServiceUrl), this);
}

void GenerateGeodatabase::addFeatureLayers(QString serviceUrl, QStringList serviceIds)
{
    foreach (auto id, serviceIds)
    {
        ServiceFeatureTable* serviceFeatureTable = new ServiceFeatureTable(QUrl(serviceUrl + id));
        FeatureLayer* featureLayer = new FeatureLayer(serviceFeatureTable, this);
        m_map->operationalLayers()->append(featureLayer);
    }
}

GenerateGeodatabaseParameters GenerateGeodatabase::getUpdatedParameters(Envelope gdbExtent)
{
    // create the parameters
    GenerateGeodatabaseParameters params;
    params.setReturnAttachments(false);
    params.setOutSpatialReference(SpatialReference::webMercator());
    params.setExtent(gdbExtent);

    // set the layer options for all of the service IDs
    QList<GenerateLayerOption> layerOptions;
    foreach (auto id, m_serviceIds)
    {
        GenerateLayerOption generateLayerOption(id.toInt());
        layerOptions << generateLayerOption;
    }
    params.setLayerOptions(layerOptions);

    return params;
}

void GenerateGeodatabase::generateGeodatabaseFromCorners(double xCorner1, double yCorner1, double xCorner2, double yCorner2)
{
    // create an envelope from the QML rectangle corners
    auto corner1 = m_mapView->screenToLocation(xCorner1, yCorner1);
    auto corner2 = m_mapView->screenToLocation(xCorner2, yCorner2);
    auto extent = Envelope(corner1, corner2);
    auto geodatabaseExtent = GeometryEngine::project(extent, SpatialReference::webMercator());

    // get the updated parameters
    auto params = getUpdatedParameters(geodatabaseExtent);

    // execute the task and obtain the job
    auto generateJob = m_syncTask->generateGeodatabase(params, m_dataPath + "WildfireCpp.geodatabase");

    // connect to the job's status changed signal
    connect(generateJob, &GenerateGeodatabaseJob::jobStatusChanged, [this, generateJob]()
    {
        // connect to the job's status changed signal to know once it is done
        switch (generateJob->jobStatus()) {
        case JobStatus::Failed:
            emit updateStatus("Generate failed");
            emit hideWindow(5000, false);
            break;
        case JobStatus::NotStarted:
            emit updateStatus("Job not started");
            break;
        case JobStatus::Paused:
            emit updateStatus("Job paused");
            break;
        case JobStatus::Started:
            emit updateStatus("In progress...");
            break;
        case JobStatus::Succeeded:
            emit updateStatus("Complete");
            emit hideWindow(1500, true);
            addOfflineData(generateJob->result());
            break;
        default:
            break;
        }
    });

    // start the generate job
    generateJob->start();
}

void GenerateGeodatabase::addOfflineData(Geodatabase* gdb)
{
    // remove the original online feature layers
    m_map->operationalLayers()->clear();

    // load the geodatabase
    connect(gdb, &Geodatabase::doneLoading, [this, gdb](Error)
    {
        // create a feature layer from each feature table, and add to the map
        foreach (auto featureTable, gdb->geodatabaseFeatureTables())
        {
            FeatureLayer* featureLayer = new FeatureLayer(featureTable, this);
            m_map->operationalLayers()->append(featureLayer);
        }

        // unregister geodatabase since there will be no edits uploaded
        m_syncTask->unregisterGeodatabase(gdb);
    });
    gdb->load();
}
<|endoftext|>


<commit_before>#include "inverseIndexStorageUnorderedMap.h"

InverseIndexStorageUnorderedMap::InverseIndexStorageUnorderedMap(size_t pSizeOfInverseIndex, size_t pMaxBinSize) {
	mSignatureStorage = new vector__umapVector(pSizeOfInverseIndex);
	mMaxBinSize = pMaxBinSize;
	mKeys = new vvsize_t(pSizeOfInverseIndex, vsize_t());
	mValues = new vvsize_t(pSizeOfInverseIndex, vsize_t());
}
InverseIndexStorageUnorderedMap::~InverseIndexStorageUnorderedMap() {

}
size_t InverseIndexStorageUnorderedMap::size() {
	return mSignatureStorage->size();
}
vsize_t* InverseIndexStorageUnorderedMap::getElement(size_t pVectorId, size_t pHashValue) {
	auto iterator = (*mSignatureStorage)[pVectorId].find(pHashValue);
	if (iterator != (*mSignatureStorage)[pVectorId].end()) {
		return &(iterator->second);
	}
	return new vsize_t(0);

	// auto itHashValue_InstanceVector = mInverseIndexUmapVector->operator[](j).find((*signature)[j]);
    //             // if for hash function h_i() the given hash values is already stored
    //             if (itHashValue_InstanceVector != mInverseIndexUmapVector->operator[](j).end()) {
    //                 // insert the instance id if not too many collisions (maxBinSize)
    //                 if (itHashValue_InstanceVector->second.size()
}
void InverseIndexStorageUnorderedMap::insert(size_t pVectorId, size_t pHashValue, size_t pInstance) {
	auto itHashValue_InstanceVector = (*mSignatureStorage)[pVectorId].find(pHashValue);
	// if for hash function h_i() the given hash values is already stored
	if (itHashValue_InstanceVector != (*mSignatureStorage)[pVectorId].end()) {
		// insert the instance id if not too many collisions (maxBinSize)
		if (itHashValue_InstanceVector->second.size() < mMaxBinSize) {
			// insert only if there wasn't any collisions in the past
			if (itHashValue_InstanceVector->second.size() > 0) {
				itHashValue_InstanceVector->second.push_back(pInstance);
			}
		} else {
			// too many collisions: delete stored ids. empty vector is interpreted as an error code
			// for too many collisions
			itHashValue_InstanceVector->second.clear();
		}
	} else {
		// given hash value for the specific hash function was not avaible: insert new hash value
		vsize_t instanceIdVector;
		instanceIdVector.push_back(pInstance);
		(*mSignatureStorage)[pVectorId][pHashValue] = instanceIdVector;
	}
}

// void InverseIndexStorageUnorderedMap::create() {

// }<commit_msg>Added debugging line<commit_after>#include "inverseIndexStorageUnorderedMap.h"

InverseIndexStorageUnorderedMap::InverseIndexStorageUnorderedMap(size_t pSizeOfInverseIndex, size_t pMaxBinSize) {
	mSignatureStorage = new vector__umapVector(pSizeOfInverseIndex);
	mMaxBinSize = pMaxBinSize;
	mKeys = new vvsize_t(pSizeOfInverseIndex, vsize_t());
	mValues = new vvsize_t(pSizeOfInverseIndex, vsize_t());
}
InverseIndexStorageUnorderedMap::~InverseIndexStorageUnorderedMap() {

}
size_t InverseIndexStorageUnorderedMap::size() {
	return mSignatureStorage->size();
}
vsize_t* InverseIndexStorageUnorderedMap::getElement(size_t pVectorId, size_t pHashValue) {
	auto iterator = (*mSignatureStorage)[pVectorId].find(pHashValue);
	if (iterator != (*mSignatureStorage)[pVectorId].end()) {
		return &(iterator->second);
	}
	return new vsize_t(0);

	// auto itHashValue_InstanceVector = mInverseIndexUmapVector->operator[](j).find((*signature)[j]);
    //             // if for hash function h_i() the given hash values is already stored
    //             if (itHashValue_InstanceVector != mInverseIndexUmapVector->operator[](j).end()) {
    //                 // insert the instance id if not too many collisions (maxBinSize)
    //                 if (itHashValue_InstanceVector->second.size()
}
void InverseIndexStorageUnorderedMap::insert(size_t pVectorId, size_t pHashValue, size_t pInstance) {
	// std::cout << "insert" << std::endl;
	auto itHashValue_InstanceVector = (*mSignatureStorage)[pVectorId].find(pHashValue);
	// if for hash function h_i() the given hash values is already stored
	if (itHashValue_InstanceVector != (*mSignatureStorage)[pVectorId].end()) {
		// insert the instance id if not too many collisions (maxBinSize)
		if (itHashValue_InstanceVector->second.size() < mMaxBinSize) {
			// insert only if there wasn't any collisions in the past
			if (itHashValue_InstanceVector->second.size() > 0) {
				itHashValue_InstanceVector->second.push_back(pInstance);
			}
		} else {
			// too many collisions: delete stored ids. empty vector is interpreted as an error code
			// for too many collisions
			itHashValue_InstanceVector->second.clear();
		}
	} else {
		// given hash value for the specific hash function was not avaible: insert new hash value
		vsize_t instanceIdVector;
		instanceIdVector.push_back(pInstance);
		(*mSignatureStorage)[pVectorId][pHashValue] = instanceIdVector;
	}
}

// void InverseIndexStorageUnorderedMap::create() {

// }<|endoftext|>


<commit_before>// Copyright (c) 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "git_command.h"

int GitCommand::parseFileSystem(pp::VarDictionary message, std::string name,
    pp::FileSystem& system) {
  pp::Var var_filesystem = message.Get(name);

  if (!var_filesystem.is_resource()) {
    //TODO(grv): return error code;
     return 1;
  }

  pp::Resource resource_filesystem = var_filesystem.AsResource();
  fileSystem = pp::FileSystem(resource_filesystem);
  return 0;
}

int GitCommand::parseArgs() {

  int error = 0;

  if ((error = parseFileSystem(_args, kFileSystem, fileSystem))) {

  }

  if ((error = parseString(_args, kFullPath,  fullPath))) {

  }

  if ((error = parseString(_args, kUrl,  url))) {

  }
  return 0;
}

int GitClone::runCommand() {
  // mount the folder as a filesystem.
  ChromefsInit();

  git_threads_init();

  std::string message = "clone successful";

  if (!url.length()) {
    git_repository_open(&repo, "/chromefs");
    message = "repository load successful";
  } else {
    git_clone(&repo, url.c_str(), "/chromefs", NULL);
  }

  const git_error *a = giterr_last();

  if (a != NULL) {
    printf("giterror: %s\n", a->message);
  }

  pp::VarDictionary arg;
  arg.Set(kMessage, message);

  pp::VarDictionary response;
  response.Set(kRegarding, subject);
  response.Set(kArg, arg);
  response.Set(kName, kResult);

  _gitSalt->PostMessage(response);
  return 0;
}

void GitClone::ChromefsInit() {
  int32_t r = (int32_t) fileSystem.pp_resource();
  char fs_resource[100] = "filesystem_resource=";
  sprintf(&fs_resource[20], "%d", r);
  mount(fullPath.c_str(),                     /* source */
      "/chromefs",                            /* target */
      "html5fs",                              /* filesystemtype */
      0,                                      /* mountflags */
      fs_resource);                           /* data */
}

int GitCommit::runCommand() {
  //TODO(grv): implement.
  char message[100];
  sprintf(message, "%s", subject.c_str());
  _gitSalt->PostMessage(pp::Var(message));
  printf("GitCommit: to be implemented");
  return 0;
}

int GitCurrentBranch::parseArgs() {
  return 0;
}

int GitCurrentBranch::runCommand() {

  git_reference* ref = NULL;
  char *branch = NULL;
  int r= git_repository_head(&ref, repo);
  if (r == 0) {
    git_branch_name((const char**)&branch, ref);
  }

  const git_error *a = giterr_last();

  if (a != NULL) {
    printf("giterror: %s\n", a->message);
  }

  pp::VarDictionary arg;
  arg.Set(kBranch, branch);

  pp::VarDictionary response;
  response.Set(kRegarding, subject);
  response.Set(kArg, arg);
  response.Set(kName, kResult);

  _gitSalt->PostMessage(response);
  git_reference_free(ref);
  return 0;
}

int GitGetBranches::parseArgs() {
  int error = 0;
  if ((error = parseInt(_args, kFlags,  &flags))) {
  }
  return 0;
}

int GitGetBranches::runCommand() {

  git_branch_iterator* iter = NULL;

  git_branch_t type = (git_branch_t) flags;

  int r = git_branch_iterator_new(&iter, repo, type);

  pp::VarDictionary arg;
  pp::VarArray branches;
  int index = 0;

  char* branch = NULL;

  if (r == 0) {
    while (r == 0) {
      git_reference* ref;
      r = git_branch_next(&ref, &type, iter);
      if (r == 0) {
        git_branch_name((const char**)&branch, ref);
        branches.Set(index, branch);
        index++;
        git_reference_free(ref);
      }
    }
  }

  arg.Set(kBranches, branches);

  pp::VarDictionary response;
  response.Set(kRegarding, subject);
  response.Set(kArg, arg);
  response.Set(kName, kResult);

  _gitSalt->PostMessage(response);
  git_branch_iterator_free(iter);
  return 0;
}

int StatusCb(const char* path, unsigned int status, void* payload) {
  pp::VarDictionary* statuses = (pp::VarDictionary*) payload;
  statuses->Set(path, (int)status);
  return 0;
}

int GitStatus::runCommand() {

  git_status_cb cb = StatusCb;

  pp::VarDictionary statuses;
  int r = git_status_foreach(repo, cb, &statuses);

  const git_error *a = giterr_last();

  if (a != NULL) {
    printf("giterror: %s\n", a->message);
  }

  pp::VarDictionary arg;
  arg.Set(kStatuses, statuses);

  pp::VarDictionary response;
  response.Set(kRegarding, subject);
  response.Set(kArg, arg);
  response.Set(kName, kResult);

  _gitSalt->PostMessage(response);
  return 0;
}
<commit_msg>Fix a warning.<commit_after>// Copyright (c) 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "git_command.h"

int GitCommand::parseFileSystem(pp::VarDictionary message, std::string name,
    pp::FileSystem& system) {
  pp::Var var_filesystem = message.Get(name);

  if (!var_filesystem.is_resource()) {
    //TODO(grv): return error code;
     return 1;
  }

  pp::Resource resource_filesystem = var_filesystem.AsResource();
  fileSystem = pp::FileSystem(resource_filesystem);
  return 0;
}

int GitCommand::parseArgs() {

  int error = 0;

  if ((error = parseFileSystem(_args, kFileSystem, fileSystem))) {

  }

  if ((error = parseString(_args, kFullPath,  fullPath))) {

  }

  if ((error = parseString(_args, kUrl,  url))) {

  }
  return 0;
}

int GitClone::runCommand() {
  // mount the folder as a filesystem.
  ChromefsInit();

  git_threads_init();

  std::string message = "clone successful";

  if (!url.length()) {
    git_repository_open(&repo, "/chromefs");
    message = "repository load successful";
  } else {
    git_clone(&repo, url.c_str(), "/chromefs", NULL);
  }

  const git_error *a = giterr_last();

  if (a != NULL) {
    printf("giterror: %s\n", a->message);
  }

  pp::VarDictionary arg;
  arg.Set(kMessage, message);

  pp::VarDictionary response;
  response.Set(kRegarding, subject);
  response.Set(kArg, arg);
  response.Set(kName, kResult);

  _gitSalt->PostMessage(response);
  return 0;
}

void GitClone::ChromefsInit() {
  int32_t r = (int32_t) fileSystem.pp_resource();
  char fs_resource[100] = "filesystem_resource=";
  sprintf(&fs_resource[20], "%d", r);
  mount(fullPath.c_str(),                     /* source */
      "/chromefs",                            /* target */
      "html5fs",                              /* filesystemtype */
      0,                                      /* mountflags */
      fs_resource);                           /* data */
}

int GitCommit::runCommand() {
  //TODO(grv): implement.
  char message[100];
  sprintf(message, "%s", subject.c_str());
  _gitSalt->PostMessage(pp::Var(message));
  printf("GitCommit: to be implemented");
  return 0;
}

int GitCurrentBranch::parseArgs() {
  return 0;
}

int GitCurrentBranch::runCommand() {

  git_reference* ref = NULL;
  char *branch = NULL;
  int r= git_repository_head(&ref, repo);
  if (r == 0) {
    git_branch_name((const char**)&branch, ref);
  }

  const git_error *a = giterr_last();

  if (a != NULL) {
    printf("giterror: %s\n", a->message);
  }

  pp::VarDictionary arg;
  arg.Set(kBranch, branch);

  pp::VarDictionary response;
  response.Set(kRegarding, subject);
  response.Set(kArg, arg);
  response.Set(kName, kResult);

  _gitSalt->PostMessage(response);
  git_reference_free(ref);
  return 0;
}

int GitGetBranches::parseArgs() {
  int error = 0;
  if ((error = parseInt(_args, kFlags,  &flags))) {
  }
  return 0;
}

int GitGetBranches::runCommand() {

  git_branch_iterator* iter = NULL;

  git_branch_t type = (git_branch_t) flags;

  int r = git_branch_iterator_new(&iter, repo, type);

  pp::VarDictionary arg;
  pp::VarArray branches;
  int index = 0;

  char* branch = NULL;

  if (r == 0) {
    while (r == 0) {
      git_reference* ref;
      r = git_branch_next(&ref, &type, iter);
      if (r == 0) {
        git_branch_name((const char**)&branch, ref);
        branches.Set(index, branch);
        index++;
        git_reference_free(ref);
      }
    }
  }

  arg.Set(kBranches, branches);

  pp::VarDictionary response;
  response.Set(kRegarding, subject);
  response.Set(kArg, arg);
  response.Set(kName, kResult);

  _gitSalt->PostMessage(response);
  git_branch_iterator_free(iter);
  return 0;
}

int StatusCb(const char* path, unsigned int status, void* payload) {
  pp::VarDictionary* statuses = (pp::VarDictionary*) payload;
  statuses->Set(path, (int)status);
  return 0;
}

int GitStatus::runCommand() {

  git_status_cb cb = StatusCb;

  pp::VarDictionary statuses;
  git_status_foreach(repo, cb, &statuses);

  const git_error *a = giterr_last();

  if (a != NULL) {
    printf("giterror: %s\n", a->message);
  }

  pp::VarDictionary arg;
  arg.Set(kStatuses, statuses);

  pp::VarDictionary response;
  response.Set(kRegarding, subject);
  response.Set(kArg, arg);
  response.Set(kName, kResult);

  _gitSalt->PostMessage(response);
  return 0;
}
<|endoftext|>


<commit_before>#ifndef STAN_MATH_PRIM_FUN_INV_CLOGLOG_HPP
#define STAN_MATH_PRIM_FUN_INV_CLOGLOG_HPP

#include <stan/math/prim/meta.hpp>
#include <stan/math/prim/fun/exp.hpp>
#include <cmath>

namespace stan {
namespace math {

/**
 * The inverse complementary log-log function.
 *
 * The function is defined by
 *
 * inv_cloglog(x) = 1 - exp(-exp(x)).
 *
 * This function can be used to implement the inverse link
 * function for complementary-log-log regression.
 *
 *
   \f[
   \mbox{inv\_cloglog}(y) =
   \begin{cases}
     \mbox{cloglog}^{-1}(y) & \mbox{if } -\infty\leq y \leq \infty \\[6pt]
     \textrm{NaN} & \mbox{if } y = \textrm{NaN}
   \end{cases}
   \f]

   \f[
   \frac{\partial\, \mbox{inv\_cloglog}(y)}{\partial y} =
   \begin{cases}
     \frac{\partial\, \mbox{cloglog}^{-1}(y)}{\partial y} & \mbox{if }
 -\infty\leq y\leq \infty \\[6pt] \textrm{NaN} & \mbox{if } y = \textrm{NaN}
   \end{cases}
   \f]

   \f[
   \mbox{cloglog}^{-1}(y) = 1 - \exp \left( - \exp(y) \right)
   \f]

   \f[
   \frac{\partial \, \mbox{cloglog}^{-1}(y)}{\partial y} = \exp(y-\exp(y))
   \f]
 *
 * @param x Argument.
 * @return Inverse complementary log-log of the argument.
 */
inline double inv_cloglog(double x) {
  using std::exp;
  return 1 - exp(-exp(x));
}

/**
 * Structure to wrap inv_cloglog() so that it can be vectorized.
 *
 * @tparam T type of variable
 * @param x variable
 * @return 1 - exp(-exp(x)).
 */
struct inv_cloglog_fun {
  template <typename T>
  static inline T fun(const T& x) {
    return inv_cloglog(x);
  }
};

/**
 * Vectorized version of inv_cloglog().
 *
 * @tparam Container type of container
 * @param x container
 * @return 1 - exp(-exp()) applied to each value in x.
 */
template <typename Container,
          require_not_container_st<std::is_arithmetic, Container>* = nullptr>
inline auto inv_cloglog(const Container& x) {
  return apply_scalar_unary<inv_cloglog_fun, Container>::apply(x);
}

/**
 * Version of inv_cloglog() that accepts std::vectors, Eigen Matrix/Array
 * objects or expressions, and containers of these.
 *
 * @tparam Container Type of x
 * @param x Container
 * @return 1 - exp(-exp()) applied to each value in x.
 */
template <typename Container,
          require_container_st<std::is_arithmetic, Container>* = nullptr>
inline auto inv_cloglog(const Container& x) {
  return apply_vector_unary<Container>::apply(
      x, [](const auto& v) { return 1 - (-v.array().exp()).exp(); });
}

}  // namespace math
}  // namespace stan

#endif
<commit_msg>Fixed merge conflicts<commit_after>#ifndef STAN_MATH_PRIM_FUN_INV_CLOGLOG_HPP
#define STAN_MATH_PRIM_FUN_INV_CLOGLOG_HPP

#include <stan/math/prim/meta.hpp>
#include <stan/math/prim/fun/exp.hpp>
#include <cmath>

namespace stan {
namespace math {

/**
 * The inverse complementary log-log function.
 *
 * The function is defined by
 *
 * inv_cloglog(x) = 1 - exp(-exp(x)).
 *
 * This function can be used to implement the inverse link
 * function for complementary-log-log regression.
 *
 *
   \f[
   \mbox{inv\_cloglog}(y) =
   \begin{cases}
     \mbox{cloglog}^{-1}(y) & \mbox{if } -\infty\leq y \leq \infty \\[6pt]
     \textrm{NaN} & \mbox{if } y = \textrm{NaN}
   \end{cases}
   \f]
   \f[
   \frac{\partial\, \mbox{inv\_cloglog}(y)}{\partial y} =
   \begin{cases}
     \frac{\partial\, \mbox{cloglog}^{-1}(y)}{\partial y} & \mbox{if }
 -\infty\leq y\leq \infty \\[6pt] \textrm{NaN} & \mbox{if } y = \textrm{NaN}
   \end{cases}
   \f]
   \f[
   \mbox{cloglog}^{-1}(y) = 1 - \exp \left( - \exp(y) \right)
   \f]
   \f[
   \frac{\partial \, \mbox{cloglog}^{-1}(y)}{\partial y} = \exp(y-\exp(y))
   \f]
 *
 * @param x Argument.
 * @return Inverse complementary log-log of the argument.
 */
inline double inv_cloglog(double x) {
  using std::exp;
  return 1 - exp(-exp(x));
}

/**
 * Structure to wrap inv_cloglog() so that it can be vectorized.
 *
 * @tparam T type of variable
 * @param x variable
 * @return 1 - exp(-exp(x)).
 */
struct inv_cloglog_fun {
  template <typename T>
  static inline T fun(const T& x) {
    return inv_cloglog(x);
  }
};

/**
 * Vectorized version of inv_cloglog().
 *
 * @tparam Container type of container
 * @param x container
 * @return 1 - exp(-exp()) applied to each value in x.
 */
template <typename Container,
          require_not_container_st<std::is_arithmetic, Container>* = nullptr>
inline auto inv_cloglog(const Container& x) {
  return apply_scalar_unary<inv_cloglog_fun, Container>::apply(x);
}

/**
 * Version of inv_cloglog() that accepts std::vectors, Eigen Matrix/Array
 * objects or expressions, and containers of these.
 *
 * @tparam Container Type of x
 * @param x Container
 * @return 1 - exp(-exp()) applied to each value in x.
 */
template <typename Container,
          require_container_st<std::is_arithmetic, Container>* = nullptr>
inline auto inv_cloglog(const Container& x) {
  return apply_vector_unary<Container>::apply(
      x, [](const auto& v) { return 1 - (-v.array().exp()).exp(); });
}

}  // namespace math
}  // namespace stan

#endif
<|endoftext|>


<commit_before>#include <iostream>
#include <string>
using namespace std::string_literals;

#include "acmacs-base/argc-argv.hh"
#include "acmacs-base/read-file.hh"
#include "acmacs-base/quicklook.hh"
// #include "acmacs-base/enumerate.hh"
#include "acmacs-chart/ace.hh"
#include "acmacs-map-draw/draw.hh"

#include "settings.hh"
#include "mod-applicator.hh"

// ----------------------------------------------------------------------

constexpr const char* sUsage = " [options] <chart.ace> [<map.pdf>]\n";

int main(int argc, char* const argv[])
{
    try {
        argc_argv args(argc, argv, {
                {"-h", false},
                {"--help", false},
                {"-s", ""},
                {"--settings", ""},
                {"--projection", 0L},
                {"--seqdb", ""},
                {"--hidb-dir", ""},
                {"--locdb", ""},
                {"-v", false},
                {"--verbose", false},
            });
        if (args["-h"] || args["--help"] || args.number_of_arguments() < 1 || args.number_of_arguments() > 2) {
            throw std::runtime_error("Usage: "s + args.program() + sUsage + args.usage_options());
        }
        const bool verbose = args["-v"] || args["--verbose"];

        auto settings = default_settings();
        std::unique_ptr<Chart> chart{import_chart(args[0], verbose ? report_time::Yes : report_time::No)};

        ChartDraw chart_draw(*chart, args["--projection"]);
        chart_draw.prepare();

          // auto mods = rjson::parse_string(R"(["all_grey", {"N": "clades", "seqdb_file": "/Users/eu/AD/data/seqdb.json.xz", "report": false}])");
          // auto mods = rjson::parse_string(R"([{"N": "clades", "seqdb_file": "/Users/eu/AD/data/seqdb.json.xz", "report": false}])");
        auto mods = rjson::parse_string(R"(["all_red"])");
        apply_mods(chart_draw, mods, settings);

        chart_draw.calculate_viewport();
        const auto temp_file = acmacs_base::TempFile(".pdf");
        chart_draw.draw(temp_file, 800, report_time::Yes);
        acmacs::quicklook(temp_file, 2);
        return 0;
    }
    catch (std::exception& err) {
        std::cerr << "ERROR: " << err.what() << '\n';
        return 1;
    }
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>map-draw --init-settings<commit_after>#include <iostream>
#include <string>
using namespace std::string_literals;

#include "acmacs-base/argc-argv.hh"
#include "acmacs-base/read-file.hh"
#include "acmacs-base/quicklook.hh"
// #include "acmacs-base/enumerate.hh"
#include "acmacs-chart/ace.hh"
#include "acmacs-map-draw/draw.hh"

#include "settings.hh"
#include "mod-applicator.hh"

// ----------------------------------------------------------------------

constexpr const char* sUsage = " [options] <chart.ace> [<map.pdf>]\n";

int main(int argc, char* const argv[])
{
    try {
        argc_argv args(argc, argv, {
                {"-h", false},
                {"--help", false},
                {"-s", ""},
                {"--settings", ""},
                {"--init-settings", ""},
                {"--projection", 0L},
                {"--seqdb", ""},
                {"--hidb-dir", ""},
                {"--locdb", ""},
                {"-v", false},
                {"--verbose", false},
            });
        if (args["-h"] || args["--help"] || args.number_of_arguments() < 1 || args.number_of_arguments() > 2) {
            throw std::runtime_error("Usage: "s + args.program() + sUsage + args.usage_options());
        }
        const bool verbose = args["-v"] || args["--verbose"];

        auto settings = default_settings();
        std::unique_ptr<Chart> chart{import_chart(args[0], verbose ? report_time::Yes : report_time::No)};

        ChartDraw chart_draw(*chart, args["--projection"]);
        chart_draw.prepare();

          // auto mods = rjson::parse_string(R"(["all_grey", {"N": "clades", "seqdb_file": "/Users/eu/AD/data/seqdb.json.xz", "report": false}])");
          // auto mods = rjson::parse_string(R"([{"N": "clades", "seqdb_file": "/Users/eu/AD/data/seqdb.json.xz", "report": false}])");
        auto mods = rjson::parse_string(R"(["all_red"])");
        apply_mods(chart_draw, mods, settings);

        chart_draw.calculate_viewport();
        const auto temp_file = acmacs_base::TempFile(".pdf");
        chart_draw.draw(temp_file, 800, report_time::Yes);

        if (const std::string save_settings = args["--init-settings"]; !save_settings.empty())
            acmacs_base::write_file(save_settings, settings.to_json_pp());

        acmacs::quicklook(temp_file, 2);
        return 0;
    }
    catch (std::exception& err) {
        std::cerr << "ERROR: " << err.what() << '\n';
        return 1;
    }
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>


<commit_before>#ifndef ALEPH_GEOMETRY_COVER_TREE_HH__
#define ALEPH_GEOMETRY_COVER_TREE_HH__

#include <iterator>
#include <limits>
#include <memory>
#include <ostream>
#include <queue>
#include <stdexcept>
#include <vector>

// FIXME: remove after debugging
#include <iostream>

#include <cassert>
#include <cmath>

namespace aleph
{

namespace geometry
{

/**
  @class CoverTree
  @brief Generic cover tree data structure

  This class models a cover tree data structure, as described in the
  paper "Cover trees for nearest neighbor" by Beygelzimer et al.; an
  implementation is given at:

    http://hunch.net/~jl/projects/cover_tree/cover_tree.html

  This implementation attempts to be as generic as possible. It uses
  the simplified description of the cover tree, as given by Izbicki,
  Shelton in "Faster Cover Trees".
*/

template <class Point, class Metric> class CoverTree
{
public:

  /**
    Covering constant of the cover tree. It might make sense to change
    this later on in order to improve performance. Some papers set the
    constant to `1.3`.
  */

  constexpr static const double coveringConstant = 2.0;

  class Node
  {
  public:

    /** Creates a new node that stores a point */
    Node( const Point& point, unsigned level )
      : _point( point )
      , _level( level )
    {
      assert( level >= 1 );
    }

    /** Calculates current covering distance of the node */
    double coveringDistance() const noexcept
    {
      return std::pow( coveringConstant, static_cast<double>( _level ) );
    }

    void addChild( const Point& p )
    {
      _children.push_back( std::unique_ptr<Node>( new Node( p, _level - 1 ) ) );
    }

    void insert( const Point& p )
    {
      auto d = Metric()( _point, p );

      std::cerr << __PRETTY_FUNCTION__ << ": Distance from point to root = " << d << "\n";

      if( d > this->coveringDistance() )
      {
        std::cerr << __PRETTY_FUNCTION__ << ": Distance is bigger than covering distance; need to raise level of tree\n";

        throw std::runtime_error( "Not yet implemented" );
      }

      return insert_( p );
    }

    /**
      Auxiliary function for performing the recursive insertion of a new
      node into the tree.
    */

    void insert_( const Point& p )
    {
      for( auto&& child : _children )
      {
        auto d = Metric()( child->_point, p );
        if( d <= child->coveringDistance() )
        {
          std::cerr << __PRETTY_FUNCTION__ << ": Recursive enumeration of the tree\n";

          // We found a node in which the new point can be inserted
          // *without* violating the covering invariant.
          child->insert_( p );
          return;
        }
      }

      // Add the new point as a child of the current root node. This
      // might require updating levels.

      if( _children.empty() )
        _level += 1;

      this->addChild( p );
    }

    Point    _point; //< The point stored in the node
    unsigned _level; //< The level of the node (>= 1)

    /**
      All children of the node. Their order depends on the insertion
      order into the data set.
    */

    std::vector< std::unique_ptr<Node> > _children;
  };

  /**
    Inserts a new point into the cover tree. If the tree is empty,
    the new point will become the root of the tree. Else, it shall
    be inserted according to the covering invariant.
  */

  void insert( const Point& p )
  {
    if( !_root )
      _root = std::unique_ptr<Node>( new Node(p,1) );
    else
      _root->insert( p );
  }

  // Pretty-printing function for the tree; this is only meant for
  // debugging purposes and could conceivably be implemented using
  // `std::ostream`.
  void print( std::ostream& o )
  {
    std::queue<const Node*> nodes;
    nodes.push( _root.get() );

    while( !nodes.empty() )
    {
      {
        auto&& node = nodes.front();

        o << node->_level << ": "
          << node->_point
          << "\n";

        for( auto&& child : node->_children )
          nodes.push( child.get() );
      }

      nodes.pop();
    }
  }

private:

  /**
    Updates the levels of a cover tree. This function ensures that all
    information is being represented correctly. It is not mentioned in
    the original paper but appears to be necessary.
  */

  void updateLevels()
  {
    // Forward pass ----------------------------------------------------
    //
    // Determine depth of the tree. This is required in order to assign
    // levels correctly later on.

    unsigned depth = 1;

    std::queue<const Node*> nodes;
    nodes.push( _root.get() );

    while( !nodes.empty() )
    {
      auto n = nodes.size();

      for( decltype(n) i = 0; i < n; i++ )
      {
        auto node = nodes.front();
        nodes.pop();

        for( auto&& child : node->_children )
          nodes.push( child.get() );
      }

      ++level;
    }

    std::cerr << __PRETTY_FUNCTION__ << ": Maximum depth = " << level << "\n";
  }

  /**
    Auxiliary function for calculating the distance between a point and
    a given set of nodes.
  */

  double distanceToSet( const Point& p, const std::vector<Node*>& Q )
  {
    Metric metric;

    double d = std::numeric_limits<double>::max();
    for( auto&& q : Q )
      d = std::min( d, double( metric(p, q->_point) ) );

    return d;
  }

  bool insert( const Point& p, const std::vector<Node*>& Qi, unsigned i )
  {
    std::cerr << "Inserting point " << p << " at level " << i << "\n";

    Metric metric;

    // Contains all nodes that satisfy the distance criterion for this
    // level.
    std::vector<Node*> Qj;

    double d = std::numeric_limits<double>::max();
    for( auto&& Q : Qi )
    {
      for( auto&& child : Q->_children )
      {
        double distance = double( metric( p, child->_point ) );
        if( unsigned( std::log2( distance ) ) <= i )
        {
          d = std::min( d, distance );
          Qj.push_back( child.get() );
        }
      }
    }

    d = std::log2( d );

    if( d > i )
      return false;
    else
    {
      auto parentFound = insert( p, Qj, i-1 );
      auto distance    = unsigned( std::log2( distanceToSet(p, Qi) ) );

      if( !parentFound && distance < i )
        return true;
      else
        return false;
    }
  }

  /** Root pointer of the tree */
  std::unique_ptr<Node> _root;
};

} // namespace geometry

} // namespace aleph

#endif
<commit_msg>Updating levels after node insertion<commit_after>#ifndef ALEPH_GEOMETRY_COVER_TREE_HH__
#define ALEPH_GEOMETRY_COVER_TREE_HH__

#include <iterator>
#include <limits>
#include <memory>
#include <ostream>
#include <queue>
#include <stdexcept>
#include <vector>

// FIXME: remove after debugging
#include <iostream>

#include <cassert>
#include <cmath>

namespace aleph
{

namespace geometry
{

/**
  @class CoverTree
  @brief Generic cover tree data structure

  This class models a cover tree data structure, as described in the
  paper "Cover trees for nearest neighbor" by Beygelzimer et al.; an
  implementation is given at:

    http://hunch.net/~jl/projects/cover_tree/cover_tree.html

  This implementation attempts to be as generic as possible. It uses
  the simplified description of the cover tree, as given by Izbicki,
  Shelton in "Faster Cover Trees".
*/

template <class Point, class Metric> class CoverTree
{
public:

  /**
    Covering constant of the cover tree. It might make sense to change
    this later on in order to improve performance. Some papers set the
    constant to `1.3`.
  */

  constexpr static const double coveringConstant = 2.0;

  class Node
  {
  public:

    /** Creates a new node that stores a point */
    Node( const Point& point, unsigned level )
      : _point( point )
      , _level( level )
    {
      assert( _level >= 1 );
    }

    /** Calculates current covering distance of the node */
    double coveringDistance() const noexcept
    {
      return std::pow( coveringConstant, static_cast<double>( _level ) );
    }

    void addChild( const Point& p )
    {
      _children.push_back( std::unique_ptr<Node>( new Node( p, _level - 1 ) ) );
    }

    void insert( const Point& p )
    {
      auto d = Metric()( _point, p );

      std::cerr << __PRETTY_FUNCTION__ << ": Distance from point to root = " << d << "\n";

      if( d > this->coveringDistance() )
      {
        std::cerr << __PRETTY_FUNCTION__ << ": Distance is bigger than covering distance; need to raise level of tree\n";

        throw std::runtime_error( "Not yet implemented" );
      }

      return insert_( p );
    }

    /**
      Auxiliary function for performing the recursive insertion of a new
      node into the tree.
    */

    void insert_( const Point& p )
    {
      for( auto&& child : _children )
      {
        auto d = Metric()( child->_point, p );
        if( d <= child->coveringDistance() )
        {
          std::cerr << __PRETTY_FUNCTION__ << ": Recursive enumeration of the tree\n";

          // We found a node in which the new point can be inserted
          // *without* violating the covering invariant.
          child->insert_( p );
          return;
        }
      }

      // Add the new point as a child of the current root node. This
      // might require updating levels.

      if( _children.empty() )
        _level += 1;

      this->addChild( p );
    }

    Point    _point; //< The point stored in the node
    unsigned _level; //< The level of the node (>= 1)

    /**
      All children of the node. Their order depends on the insertion
      order into the data set.
    */

    std::vector< std::unique_ptr<Node> > _children;
  };

  /**
    Inserts a new point into the cover tree. If the tree is empty,
    the new point will become the root of the tree. Else, it shall
    be inserted according to the covering invariant.
  */

  void insert( const Point& p )
  {
    if( !_root )
      _root = std::unique_ptr<Node>( new Node(p,1) );
    else
      _root->insert( p );

    this->updateLevels();
  }

  // Pretty-printing function for the tree; this is only meant for
  // debugging purposes and could conceivably be implemented using
  // `std::ostream`.
  void print( std::ostream& o )
  {
    std::queue<const Node*> nodes;
    nodes.push( _root.get() );

    while( !nodes.empty() )
    {
      {
        auto&& node = nodes.front();

        o << node->_level << ": "
          << node->_point
          << "\n";

        for( auto&& child : node->_children )
          nodes.push( child.get() );
      }

      nodes.pop();
    }
  }

private:

  /**
    Updates the levels of a cover tree. This function ensures that all
    information is being represented correctly. It is not mentioned in
    the original paper but appears to be necessary.
  */

  void updateLevels()
  {
    // Forward pass ----------------------------------------------------
    //
    // Determine depth of the tree. This is required in order to assign
    // levels correctly later on.

    unsigned depth = 0;

    std::queue<const Node*> nodes;
    nodes.push( _root.get() );

    while( !nodes.empty() )
    {
      auto n = nodes.size();

      for( decltype(n) i = 0; i < n; i++ )
      {
        auto node = nodes.front();
        nodes.pop();

        for( auto&& child : node->_children )
          nodes.push( child.get() );
      }

      ++depth;
    }

    assert( nodes.empty() );

    // Backward pass ---------------------------------------------------
    //
    // Set the level of the root node and update child levels. Note that
    // all nodes will be traversed because this function is dumb.

    _root->_level = depth;

    nodes.push( _root.get() );

    while( !nodes.empty() )
    {
      {
        auto&& parent = nodes.front();

        for( auto&& child : parent->_children )
        {
          assert( parent->_level >= 1 );

          child->_level = parent->_level - 1;
          nodes.push( child.get() );
        }
      }

      nodes.pop();
    }
  }

  /**
    Auxiliary function for calculating the distance between a point and
    a given set of nodes.
  */

  double distanceToSet( const Point& p, const std::vector<Node*>& Q )
  {
    Metric metric;

    double d = std::numeric_limits<double>::max();
    for( auto&& q : Q )
      d = std::min( d, double( metric(p, q->_point) ) );

    return d;
  }

  bool insert( const Point& p, const std::vector<Node*>& Qi, unsigned i )
  {
    std::cerr << "Inserting point " << p << " at level " << i << "\n";

    Metric metric;

    // Contains all nodes that satisfy the distance criterion for this
    // level.
    std::vector<Node*> Qj;

    double d = std::numeric_limits<double>::max();
    for( auto&& Q : Qi )
    {
      for( auto&& child : Q->_children )
      {
        double distance = double( metric( p, child->_point ) );
        if( unsigned( std::log2( distance ) ) <= i )
        {
          d = std::min( d, distance );
          Qj.push_back( child.get() );
        }
      }
    }

    d = std::log2( d );

    if( d > i )
      return false;
    else
    {
      auto parentFound = insert( p, Qj, i-1 );
      auto distance    = unsigned( std::log2( distanceToSet(p, Qi) ) );

      if( !parentFound && distance < i )
        return true;
      else
        return false;
    }
  }

  /** Root pointer of the tree */
  std::unique_ptr<Node> _root;
};

} // namespace geometry

} // namespace aleph

#endif
<|endoftext|>


<commit_before>
#include "config.h"
#include "include/types.h"

//#define MDS_CACHE_SIZE        4*10000   -> <20mb
//#define MDS_CACHE_SIZE        80000         62mb

#define AVG_PER_INODE_SIZE    450
#define MDS_CACHE_MB_TO_INODES(x) ((x)*1000000/AVG_PER_INODE_SIZE)

//#define MDS_CACHE_SIZE       MDS_CACHE_MB_TO_INODES( 50 )
//#define MDS_CACHE_SIZE 1500000
#define MDS_CACHE_SIZE 150000

// hack hack hack ugly FIXME
#include "common/Mutex.h"
long buffer_total_alloc = 0;
Mutex bufferlock;

FileLayout g_OSD_FileLayout( 1<<20, 1, 1<<20, 2 );  // stripe over 1M objects, 2x replication
//FileLayout g_OSD_FileLayout( 1<<17, 4, 1<<20 );   // 128k stripes over sets of 4

// ??
FileLayout g_OSD_MDDirLayout( 1<<14, 1<<2, 1<<19, 3 );

// stripe mds log over 128 byte bits (see mds_log_pad_entry below to match!)
FileLayout g_OSD_MDLogLayout( 1<<7, 32, 1<<20, 3 );  // new (good?) way
//FileLayout g_OSD_MDLogLayout( 57, 32, 1<<20 );  // pathological case to test striping buffer mapping
//FileLayout g_OSD_MDLogLayout( 1<<20, 1, 1<<20 );  // old way

md_config_t g_conf = {
  num_mds: 1,
  num_osd: 4,
  num_client: 1,

  mkfs: false,

  // profiling and debugging
  log: true,
  log_interval: 1,
  log_name: (char*)0,

  log_messages: true,
  log_pins: true,

  fake_clock: false,
  fakemessenger_serialize: true,
  fake_osdmap_expand: 0,
  fake_osd_sync: false,

  debug: 1,
  debug_mds_balancer: 1,
  debug_mds_log: 1,
  debug_buffer: 0,
  debug_filer: 0,
  debug_client: 0,
  debug_osd: 0,
  debug_ebofs: 1,
  debug_bdev: 1,         // block device

  // --- client ---
  client_cache_size: 300,
  client_cache_mid: .5,
  client_cache_stat_ttl: 10, // seconds until cached stat results become invalid
  client_use_random_mds:  false,

  client_sync_writes: 0,

  client_bcache: 1,
  client_bcache_alloc_minsize: 1<<10, // 1KB
  client_bcache_alloc_maxsize: 1<<18, // 256KB
  client_bcache_ttl: 30, // seconds until dirty buffers are written to disk
  client_bcache_size: 2<<30, // 2GB
  //client_bcache_size: 5<<20, // 5MB
  client_bcache_lowater: 60, // % of size
  client_bcache_hiwater: 80, // % of size
  client_bcache_align: 1<<10, // 1KB splice alignment

  client_trace: 0,
  fuse_direct_io: 0,

  // --- mds ---
  mds_cache_size: MDS_CACHE_SIZE,
  mds_cache_mid: .7,

  mds_log: true,
  mds_log_max_len:  MDS_CACHE_SIZE / 3,
  mds_log_max_trimming: 256,
  mds_log_read_inc: 1024,//1<<20,
  mds_log_pad_entry: 64,
  mds_log_before_reply: true,
  mds_log_flush_on_shutdown: true,

  mds_bal_replicate_threshold: 8000,
  mds_bal_unreplicate_threshold: 1000,
  mds_bal_interval: 30,           // seconds
  mds_bal_idle_threshold: .1,
  mds_bal_max: -1,
  mds_bal_max_until: -1,

  mds_commit_on_shutdown: true,

  mds_verify_export_dirauth: true,

  // --- osd ---
  osd_pg_bits: 8,
  osd_max_rep: 4,
  osd_fsync: true,
  osd_writesync: false,
  osd_maxthreads: 1,   // 0 == no threading!
  osd_mkfs: false,

  osd_fakestore_syncthreads: 4,

  // --- ebofs ---
  ebofs: 0,
  ebofs_commit_interval: 2,    // seconds.  0 = no timeout (for debugging/tracing)
  ebofs_oc_size:      1000,
  ebofs_cc_size:      1000,
  ebofs_bc_size:      (15 *256),  // 4k blocks, or *256 for MB
  ebofs_bc_max_dirty: (10 *256),  // before write() will block

  ebofs_abp_zero: false,
  ebofs_abp_max_alloc: 4096*32,

  // --- block device ---
  bdev_el_fw_max_ms: 1000,      // restart elevator at least once every 1000 ms
  bdev_el_bw_max_ms: 300,       // restart elevator at least once every 1000 ms
  bdev_el_bidir: true,          // bidirectional elevator?

  // --- fakeclient (mds regression testing) (ancient history) ---
  num_fakeclient: 100,
  fakeclient_requests: 100,
  fakeclient_deterministic: false,

  fakeclient_op_statfs:     false,

  // loosely based on Roselli workload paper numbers
  fakeclient_op_stat:     610,
  fakeclient_op_lstat:      false,
  fakeclient_op_utime:    0,
  fakeclient_op_chmod:    1,
  fakeclient_op_chown:    1,

  fakeclient_op_readdir:  2,
  fakeclient_op_mknod:    30,
  fakeclient_op_link:     false,
  fakeclient_op_unlink:   20,
  fakeclient_op_rename:   0,//40,

  fakeclient_op_mkdir:    10,
  fakeclient_op_rmdir:    20,
  fakeclient_op_symlink:  20,

  fakeclient_op_openrd:   200,
  fakeclient_op_openwr:   0,
  fakeclient_op_openwrc:  0,
  fakeclient_op_read:       false,  // osd!
  fakeclient_op_write:      false,  // osd!
  fakeclient_op_truncate:   false,
  fakeclient_op_fsync:      false,
  fakeclient_op_close:    200
};

#include <stdlib.h>
#include <string.h>
#include <iostream>
using namespace std;

void argv_to_vec(int argc, char **argv,
				 vector<char*>& args)
{
  for (int i=1; i<argc; i++)
	args.push_back(argv[i]);
}

void vec_to_argv(vector<char*>& args,
				 int& argc, char **&argv)
{
  argv = (char**)malloc(sizeof(char*) * argc);
  argc = 1;
  argv[0] = "asdf";

  for (unsigned i=0; i<args.size(); i++)
	argv[argc++] = args[i];
}

void parse_config_options(vector<char*>& args)
{
  vector<char*> nargs;

  for (unsigned i=0; i<args.size(); i++) {
	if (strcmp(args[i], "--nummds") == 0)
	  g_conf.num_mds = atoi(args[++i]);
	else if (strcmp(args[i], "--numclient") == 0)
	  g_conf.num_client = atoi(args[++i]);
	else if (strcmp(args[i], "--numosd") == 0)
	  g_conf.num_osd = atoi(args[++i]);

	else if (strcmp(args[i], "--mkfs") == 0)
	  g_conf.osd_mkfs = g_conf.mkfs = 1; //atoi(args[++i]);

	else if (strcmp(args[i], "--fake_osdmap_expand") == 0)
	  g_conf.fake_osdmap_expand = atoi(args[++i]);
	else if (strcmp(args[i], "--fake_osd_sync") == 0)
	  g_conf.fake_osd_sync = atoi(args[++i]);

	else if (strcmp(args[i], "--debug") == 0)
	  g_conf.debug = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_mds_balancer") == 0)
	  g_conf.debug_mds_balancer = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_mds_log") == 0)
	  g_conf.debug_mds_log = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_buffer") == 0)
	  g_conf.debug_buffer = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_filer") == 0)
	  g_conf.debug_filer = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_client") == 0)
	  g_conf.debug_client = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_osd") == 0)
	  g_conf.debug_osd = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_ebofs") == 0)
	  g_conf.debug_ebofs = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_bdev") == 0)
	  g_conf.debug_bdev = atoi(args[++i]);

	else if (strcmp(args[i], "--log") == 0)
	  g_conf.log = atoi(args[++i]);
	else if (strcmp(args[i], "--log_name") == 0)
	  g_conf.log_name = args[++i];

	else if (strcmp(args[i], "--fakemessenger_serialize") == 0)
	  g_conf.fakemessenger_serialize = atoi(args[++i]);

	else if (strcmp(args[i], "--mds_cache_size") == 0)
	  g_conf.mds_cache_size = atoi(args[++i]);

	else if (strcmp(args[i], "--mds_log") == 0)
	  g_conf.mds_log = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_before_reply") == 0)
	  g_conf.mds_log_before_reply = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_max_len") == 0)
	  g_conf.mds_log_max_len = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_read_inc") == 0)
	  g_conf.mds_log_read_inc = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_max_trimming") == 0)
	  g_conf.mds_log_max_trimming = atoi(args[++i]);

	else if (strcmp(args[i], "--mds_commit_on_shutdown") == 0)
	  g_conf.mds_commit_on_shutdown = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_flush_on_shutdown") == 0)
	  g_conf.mds_log_flush_on_shutdown = atoi(args[++i]);

	else if (strcmp(args[i], "--mds_bal_interval") == 0)
	  g_conf.mds_bal_interval = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_bal_rep") == 0)
	  g_conf.mds_bal_replicate_threshold = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_bal_unrep") == 0)
	  g_conf.mds_bal_unreplicate_threshold = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_bal_max") == 0)
	  g_conf.mds_bal_max = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_bal_max_until") == 0)
	  g_conf.mds_bal_max_until = atoi(args[++i]);

	else if (strcmp(args[i], "--client_cache_size") == 0)
	  g_conf.client_cache_size = atoi(args[++i]);
	else if (strcmp(args[i], "--client_cache_stat_ttl") == 0)
	  g_conf.client_cache_stat_ttl = atoi(args[++i]);
	else if (strcmp(args[i], "--client_trace") == 0)
	  g_conf.client_trace = atoi(args[++i]);
	else if (strcmp(args[i], "--fuse_direct_io") == 0)
	  g_conf.fuse_direct_io = atoi(args[++i]);

	else if (strcmp(args[i], "--client_sync_writes") == 0)
	  g_conf.client_sync_writes = atoi(args[++i]);
	else if (strcmp(args[i], "--client_bcache") == 0)
	  g_conf.client_bcache = atoi(args[++i]);
	else if (strcmp(args[i], "--client_bcache_ttl") == 0)
	  g_conf.client_bcache_ttl = atoi(args[++i]);

	else if (strcmp(args[i], "--ebofs") == 0)
	  g_conf.ebofs = 1;
	else if (strcmp(args[i], "--fakestore") == 0)
	  g_conf.ebofs = 0;

	else if (strcmp(args[i], "--osd_mkfs") == 0)
	  g_conf.osd_mkfs = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_pg_bits") == 0)
	  g_conf.osd_pg_bits = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_max_rep") == 0)
	  g_conf.osd_max_rep = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_fsync") == 0)
	  g_conf.osd_fsync = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_writesync") == 0)
	  g_conf.osd_writesync = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_maxthreads") == 0)
	  g_conf.osd_maxthreads = atoi(args[++i]);

	else {
	  nargs.push_back(args[i]);
	}
  }

  args = nargs;
}
<commit_msg>*** empty log message ***<commit_after>
#include "config.h"
#include "include/types.h"

//#define MDS_CACHE_SIZE        4*10000   -> <20mb
//#define MDS_CACHE_SIZE        80000         62mb

#define AVG_PER_INODE_SIZE    450
#define MDS_CACHE_MB_TO_INODES(x) ((x)*1000000/AVG_PER_INODE_SIZE)

//#define MDS_CACHE_SIZE       MDS_CACHE_MB_TO_INODES( 50 )
//#define MDS_CACHE_SIZE 1500000
#define MDS_CACHE_SIZE 150000

// hack hack hack ugly FIXME
#include "common/Mutex.h"
long buffer_total_alloc = 0;
Mutex bufferlock;

FileLayout g_OSD_FileLayout( 1<<20, 1, 1<<20, 2 );  // stripe over 1M objects, 2x replication
//FileLayout g_OSD_FileLayout( 1<<17, 4, 1<<20 );   // 128k stripes over sets of 4

// ??
FileLayout g_OSD_MDDirLayout( 1<<14, 1<<2, 1<<19, 3 );

// stripe mds log over 128 byte bits (see mds_log_pad_entry below to match!)
FileLayout g_OSD_MDLogLayout( 1<<7, 32, 1<<20, 3 );  // new (good?) way
//FileLayout g_OSD_MDLogLayout( 57, 32, 1<<20 );  // pathological case to test striping buffer mapping
//FileLayout g_OSD_MDLogLayout( 1<<20, 1, 1<<20 );  // old way

md_config_t g_conf = {
  num_mds: 1,
  num_osd: 4,
  num_client: 1,

  mkfs: false,

  // profiling and debugging
  log: true,
  log_interval: 1,
  log_name: (char*)0,

  log_messages: true,
  log_pins: true,

  fake_clock: false,
  fakemessenger_serialize: true,
  fake_osdmap_expand: 0,
  fake_osd_sync: false,

  debug: 1,
  debug_mds_balancer: 1,
  debug_mds_log: 1,
  debug_buffer: 0,
  debug_filer: 0,
  debug_client: 0,
  debug_osd: 0,
  debug_ebofs: 1,
  debug_bdev: 1,         // block device

  // --- client ---
  client_cache_size: 300,
  client_cache_mid: .5,
  client_cache_stat_ttl: 10, // seconds until cached stat results become invalid
  client_use_random_mds:  false,

  client_sync_writes: 0,

  client_bcache: 1,
  client_bcache_alloc_minsize: 1<<10, // 1KB
  client_bcache_alloc_maxsize: 1<<18, // 256KB
  client_bcache_ttl: 30, // seconds until dirty buffers are written to disk
  client_bcache_size: 2<<30, // 2GB
  //client_bcache_size: 5<<20, // 5MB
  client_bcache_lowater: 60, // % of size
  client_bcache_hiwater: 80, // % of size
  client_bcache_align: 1<<10, // 1KB splice alignment

  client_trace: 0,
  fuse_direct_io: 0,

  // --- mds ---
  mds_cache_size: MDS_CACHE_SIZE,
  mds_cache_mid: .7,

  mds_log: true,
  mds_log_max_len:  MDS_CACHE_SIZE / 3,
  mds_log_max_trimming: 256,
  mds_log_read_inc: 1024,//1<<20,
  mds_log_pad_entry: 64,
  mds_log_before_reply: true,
  mds_log_flush_on_shutdown: true,

  mds_bal_replicate_threshold: 8000,
  mds_bal_unreplicate_threshold: 1000,
  mds_bal_interval: 30,           // seconds
  mds_bal_idle_threshold: .1,
  mds_bal_max: -1,
  mds_bal_max_until: -1,

  mds_commit_on_shutdown: true,

  mds_verify_export_dirauth: true,

  // --- osd ---
  osd_pg_bits: 8,
  osd_max_rep: 4,
  osd_fsync: true,
  osd_writesync: false,
  osd_maxthreads: 10,   // 0 == no threading!
  osd_mkfs: false,

  osd_fakestore_syncthreads: 4,

  // --- ebofs ---
  ebofs: 0,
  ebofs_commit_interval: 2,    // seconds.  0 = no timeout (for debugging/tracing)
  ebofs_oc_size:      1000,
  ebofs_cc_size:      1000,
  ebofs_bc_size:      (15 *256),  // 4k blocks, or *256 for MB
  ebofs_bc_max_dirty: (10 *256),  // before write() will block

  ebofs_abp_zero: false,
  ebofs_abp_max_alloc: 4096*32,

  // --- block device ---
  bdev_el_fw_max_ms: 1000,      // restart elevator at least once every 1000 ms
  bdev_el_bw_max_ms: 300,       // restart elevator at least once every 1000 ms
  bdev_el_bidir: true,          // bidirectional elevator?

  // --- fakeclient (mds regression testing) (ancient history) ---
  num_fakeclient: 100,
  fakeclient_requests: 100,
  fakeclient_deterministic: false,

  fakeclient_op_statfs:     false,

  // loosely based on Roselli workload paper numbers
  fakeclient_op_stat:     610,
  fakeclient_op_lstat:      false,
  fakeclient_op_utime:    0,
  fakeclient_op_chmod:    1,
  fakeclient_op_chown:    1,

  fakeclient_op_readdir:  2,
  fakeclient_op_mknod:    30,
  fakeclient_op_link:     false,
  fakeclient_op_unlink:   20,
  fakeclient_op_rename:   0,//40,

  fakeclient_op_mkdir:    10,
  fakeclient_op_rmdir:    20,
  fakeclient_op_symlink:  20,

  fakeclient_op_openrd:   200,
  fakeclient_op_openwr:   0,
  fakeclient_op_openwrc:  0,
  fakeclient_op_read:       false,  // osd!
  fakeclient_op_write:      false,  // osd!
  fakeclient_op_truncate:   false,
  fakeclient_op_fsync:      false,
  fakeclient_op_close:    200
};

#include <stdlib.h>
#include <string.h>
#include <iostream>
using namespace std;

void argv_to_vec(int argc, char **argv,
				 vector<char*>& args)
{
  for (int i=1; i<argc; i++)
	args.push_back(argv[i]);
}

void vec_to_argv(vector<char*>& args,
				 int& argc, char **&argv)
{
  argv = (char**)malloc(sizeof(char*) * argc);
  argc = 1;
  argv[0] = "asdf";

  for (unsigned i=0; i<args.size(); i++)
	argv[argc++] = args[i];
}

void parse_config_options(vector<char*>& args)
{
  vector<char*> nargs;

  for (unsigned i=0; i<args.size(); i++) {
	if (strcmp(args[i], "--nummds") == 0)
	  g_conf.num_mds = atoi(args[++i]);
	else if (strcmp(args[i], "--numclient") == 0)
	  g_conf.num_client = atoi(args[++i]);
	else if (strcmp(args[i], "--numosd") == 0)
	  g_conf.num_osd = atoi(args[++i]);

	else if (strcmp(args[i], "--mkfs") == 0)
	  g_conf.osd_mkfs = g_conf.mkfs = 1; //atoi(args[++i]);

	else if (strcmp(args[i], "--fake_osdmap_expand") == 0)
	  g_conf.fake_osdmap_expand = atoi(args[++i]);
	else if (strcmp(args[i], "--fake_osd_sync") == 0)
	  g_conf.fake_osd_sync = atoi(args[++i]);

	else if (strcmp(args[i], "--debug") == 0)
	  g_conf.debug = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_mds_balancer") == 0)
	  g_conf.debug_mds_balancer = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_mds_log") == 0)
	  g_conf.debug_mds_log = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_buffer") == 0)
	  g_conf.debug_buffer = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_filer") == 0)
	  g_conf.debug_filer = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_client") == 0)
	  g_conf.debug_client = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_osd") == 0)
	  g_conf.debug_osd = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_ebofs") == 0)
	  g_conf.debug_ebofs = atoi(args[++i]);
	else if (strcmp(args[i], "--debug_bdev") == 0)
	  g_conf.debug_bdev = atoi(args[++i]);

	else if (strcmp(args[i], "--log") == 0)
	  g_conf.log = atoi(args[++i]);
	else if (strcmp(args[i], "--log_name") == 0)
	  g_conf.log_name = args[++i];

	else if (strcmp(args[i], "--fakemessenger_serialize") == 0)
	  g_conf.fakemessenger_serialize = atoi(args[++i]);

	else if (strcmp(args[i], "--mds_cache_size") == 0)
	  g_conf.mds_cache_size = atoi(args[++i]);

	else if (strcmp(args[i], "--mds_log") == 0)
	  g_conf.mds_log = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_before_reply") == 0)
	  g_conf.mds_log_before_reply = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_max_len") == 0)
	  g_conf.mds_log_max_len = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_read_inc") == 0)
	  g_conf.mds_log_read_inc = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_max_trimming") == 0)
	  g_conf.mds_log_max_trimming = atoi(args[++i]);

	else if (strcmp(args[i], "--mds_commit_on_shutdown") == 0)
	  g_conf.mds_commit_on_shutdown = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_log_flush_on_shutdown") == 0)
	  g_conf.mds_log_flush_on_shutdown = atoi(args[++i]);

	else if (strcmp(args[i], "--mds_bal_interval") == 0)
	  g_conf.mds_bal_interval = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_bal_rep") == 0)
	  g_conf.mds_bal_replicate_threshold = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_bal_unrep") == 0)
	  g_conf.mds_bal_unreplicate_threshold = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_bal_max") == 0)
	  g_conf.mds_bal_max = atoi(args[++i]);
	else if (strcmp(args[i], "--mds_bal_max_until") == 0)
	  g_conf.mds_bal_max_until = atoi(args[++i]);

	else if (strcmp(args[i], "--client_cache_size") == 0)
	  g_conf.client_cache_size = atoi(args[++i]);
	else if (strcmp(args[i], "--client_cache_stat_ttl") == 0)
	  g_conf.client_cache_stat_ttl = atoi(args[++i]);
	else if (strcmp(args[i], "--client_trace") == 0)
	  g_conf.client_trace = atoi(args[++i]);
	else if (strcmp(args[i], "--fuse_direct_io") == 0)
	  g_conf.fuse_direct_io = atoi(args[++i]);

	else if (strcmp(args[i], "--client_sync_writes") == 0)
	  g_conf.client_sync_writes = atoi(args[++i]);
	else if (strcmp(args[i], "--client_bcache") == 0)
	  g_conf.client_bcache = atoi(args[++i]);
	else if (strcmp(args[i], "--client_bcache_ttl") == 0)
	  g_conf.client_bcache_ttl = atoi(args[++i]);

	else if (strcmp(args[i], "--ebofs") == 0)
	  g_conf.ebofs = 1;
	else if (strcmp(args[i], "--fakestore") == 0)
	  g_conf.ebofs = 0;

	else if (strcmp(args[i], "--osd_mkfs") == 0)
	  g_conf.osd_mkfs = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_pg_bits") == 0)
	  g_conf.osd_pg_bits = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_max_rep") == 0)
	  g_conf.osd_max_rep = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_fsync") == 0)
	  g_conf.osd_fsync = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_writesync") == 0)
	  g_conf.osd_writesync = atoi(args[++i]);
	else if (strcmp(args[i], "--osd_maxthreads") == 0)
	  g_conf.osd_maxthreads = atoi(args[++i]);

	else {
	  nargs.push_back(args[i]);
	}
  }

  args = nargs;
}
<|endoftext|>


<commit_before>// xiva (acronym for HTTP Extended EVent Automata) is a simple HTTP server.
// Copyright (C) 2009 Yandex <highpower@yandex.ru>

// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

#ifndef XIVA_DETAILS_REQUEST_CHECKER_HPP_INCLUDED
#define XIVA_DETAILS_REQUEST_CHECKER_HPP_INCLUDED

#include <utility>
#include <iterator>
#include <algorithm>

#include <boost/type_traits.hpp>
#include <boost/static_assert.hpp>

#include "details/asio.hpp"
#include "details/functors.hpp"
#include "details/line_reader.hpp"
#include "details/range.hpp"
#include "details/string_utils.hpp"
#include "details/http_constants.hpp"
#include "details/iterator_checker.hpp"
#include "details/request_helper.hpp"

namespace xiva { namespace details {

struct http_request_checker {
	template <typename Iter> std::pair<Iter, bool> operator () (Iter first, Iter second) const;
};

struct request_policy_checker {
	template <typename Iter> std::pair<Iter, bool> operator () (Iter first, Iter second) const;
};

template <typename Iter> inline std::pair<Iter, bool>
http_request_checker::operator () (Iter first, Iter last) const {

	typedef typename std::iterator_traits<Iter>::value_type char_type;
	BOOST_STATIC_ASSERT(sizeof(char_type) == sizeof(char));

	range<Iter> line, end_line;
	line_reader<Iter> reader(first, last);
	if (reader.read_line(line, end_line) && !end_line.empty()) {

		int read_body_ahead = 0;
		request_helper<Iter> helper;

		helper.parse_request_line(line);
		while (reader.read_line(line, end_line)) {
			int res = helper.check_read_body_ahead(line);
			if (res > 0) {
				read_body_ahead = res;
			}
			Iter break_ptr;
			if (helper.find_headers_break(end_line, break_ptr)) {
				if (!read_body_ahead) {
					return std::make_pair(end_line.begin(), true);
				}
				if (std::distance(break_ptr, last) >= read_body_ahead) {
					return std::make_pair(last, true);
				}
			}
		}
	}

	return std::make_pair(last, false);
}

template <typename Iter> inline std::pair<Iter, bool>
request_policy_checker::operator () (Iter first, Iter last) const {

	typedef std::reverse_iterator<Iter> iterator_type;
	typedef typename std::iterator_traits<Iter>::value_type char_type;
	BOOST_STATIC_ASSERT(sizeof(char_type) == sizeof(char));

	iterator_checker<Iter> iter_check;
	(void) iter_check;

	iterator_type begin(last), end(first);
	if (static_cast<char_type>('<') == *first) {
		return std::make_pair(last, (static_cast<char_type>(0) == *begin));
	}
	return std::make_pair(last, false);
}

class request_checker {

public:
	explicit request_checker(int max_size);
	template <typename Iter> std::pair<Iter, bool> operator () (Iter first, Iter second) const;

private:
	http_request_checker http_checker_;
	request_policy_checker policy_checker_;
	int max_size_;
};

request_checker::request_checker(int max_size) : max_size_(max_size)
{
}

template <typename Iter> inline std::pair<Iter, bool>
request_checker::operator () (Iter first, Iter last) const {

	typedef typename std::iterator_traits<Iter>::difference_type diff_type;
	diff_type size = std::distance(first, last);

	enum { min_size = sizeof("get / http/1.1\n\n") - 1 };
	// sizeof ("<policy-file-request/>\0") - 1 > min_size

	if (size < min_size) {
		return std::make_pair(last, false);
	}
	if (size > (diff_type)max_size_) {
		return std::make_pair(last, true); // will be handled in connection_impl
	}
	try {
		std::pair<Iter, bool> result = policy_checker_(first, last);
		return (result.second) ? result : http_checker_(first, last);
	}
	catch (...) {
		return std::make_pair(last, true); // will be handled in request_impl
	}
}

}} // namespaces

XIVA_BEGIN_ASIO_NAMESPACE

template <>
struct is_match_condition<xiva::details::request_checker> : public boost::true_type {};

XIVA_END_ASIO_NAMESPACE

#endif // XIVA_DETAILS_REQUEST_CHECKER_HPP_INCLUDED
<commit_msg>fixed checking big requests (size above 512b)<commit_after>// xiva (acronym for HTTP Extended EVent Automata) is a simple HTTP server.
// Copyright (C) 2009 Yandex <highpower@yandex.ru>

// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

#ifndef XIVA_DETAILS_REQUEST_CHECKER_HPP_INCLUDED
#define XIVA_DETAILS_REQUEST_CHECKER_HPP_INCLUDED

#include <utility>
#include <iterator>
#include <algorithm>

#include <boost/type_traits.hpp>
#include <boost/static_assert.hpp>

#include "details/asio.hpp"
#include "details/functors.hpp"
#include "details/line_reader.hpp"
#include "details/range.hpp"
#include "details/string_utils.hpp"
#include "details/http_constants.hpp"
#include "details/iterator_checker.hpp"
#include "details/request_helper.hpp"

namespace xiva { namespace details {

struct http_request_checker {
	template <typename Iter> std::pair<Iter, bool> operator () (Iter first, Iter second) const;
};

struct request_policy_checker {
	template <typename Iter> std::pair<Iter, bool> operator () (Iter first, Iter second) const;
};

template <typename Iter> inline std::pair<Iter, bool>
http_request_checker::operator () (Iter first, Iter last) const {

	typedef typename std::iterator_traits<Iter>::value_type char_type;
	BOOST_STATIC_ASSERT(sizeof(char_type) == sizeof(char));

	range<Iter> line, end_line;
	line_reader<Iter> reader(first, last);
	if (reader.read_line(line, end_line) && !end_line.empty()) {

		int read_body_ahead = 0;
		request_helper<Iter> helper;

		helper.parse_request_line(line);
		while (reader.read_line(line, end_line)) {
			int res = helper.check_read_body_ahead(line);
			if (res > 0) {
				read_body_ahead = res;
			}
			Iter break_ptr;
			if (helper.find_headers_break(end_line, break_ptr)) {
				if (!read_body_ahead) {
					return std::make_pair(end_line.begin(), true);  // ok, without body
				}
				if (std::distance(break_ptr, last) >= read_body_ahead) {
					return std::make_pair(last, true); // ok, with ahead body
				}
			}
		}
	}

	return std::make_pair(first, false); // continue, cannot find end of headers or ahead body
}

template <typename Iter> inline std::pair<Iter, bool>
request_policy_checker::operator () (Iter first, Iter last) const {

	typedef std::reverse_iterator<Iter> iterator_type;
	typedef typename std::iterator_traits<Iter>::value_type char_type;
	BOOST_STATIC_ASSERT(sizeof(char_type) == sizeof(char));

	iterator_checker<Iter> iter_check;
	(void) iter_check;

	iterator_type begin(last), end(first);
	if (static_cast<char_type>('<') == *first) {
		return std::make_pair(last, (static_cast<char_type>(0) == *begin));
	}
	return std::make_pair(last, false);
}

class request_checker {

public:
	explicit request_checker(int max_size);
	template <typename Iter> std::pair<Iter, bool> operator () (Iter first, Iter second) const;

private:
	http_request_checker http_checker_;
	request_policy_checker policy_checker_;
	int max_size_;
};

request_checker::request_checker(int max_size) : max_size_(max_size)
{
}

template <typename Iter> inline std::pair<Iter, bool>
request_checker::operator () (Iter first, Iter last) const {

	typedef typename std::iterator_traits<Iter>::difference_type diff_type;
	diff_type size = std::distance(first, last);

	enum { min_size = sizeof("get / http/1.1\n\n") - 1 };
	// sizeof ("<policy-file-request/>\0") - 1 > min_size

	if (size < min_size) {
		return std::make_pair(first, false); // continue, too little data
	}
	if (size > (diff_type)max_size_) {
		return std::make_pair(last, true); // will be handled in connection_impl
	}
	try {
		std::pair<Iter, bool> result = policy_checker_(first, last);
		return (result.second) ? result : http_checker_(first, last);
	}
	catch (...) {
		return std::make_pair(last, true); // will be handled in request_impl
	}
}

}} // namespaces

XIVA_BEGIN_ASIO_NAMESPACE

template <>
struct is_match_condition<xiva::details::request_checker> : public boost::true_type {};

XIVA_END_ASIO_NAMESPACE

#endif // XIVA_DETAILS_REQUEST_CHECKER_HPP_INCLUDED
<|endoftext|>


<commit_before>#include <cmdstan/command.hpp>
#include <test/test-models/proper.hpp>
#include <stan/math/prim/core/init_threadpool_tbb.hpp>
#include <tbb/task_scheduler_init.h>
#include <gtest/gtest.h>

TEST(StanUiCommand, threadpool_init) {
  tbb::task_scheduler_init &scheduler_init = stan::math::init_threadpool_tbb();
  EXPECT_TRUE(scheduler_init.is_active());
  scheduler_init.terminate();
  EXPECT_FALSE(scheduler_init.is_active());
}
<commit_msg>use auto for threadpool init so we can update tbb<commit_after>#include <cmdstan/command.hpp>
#include <test/test-models/proper.hpp>
#include <stan/math/prim/core/init_threadpool_tbb.hpp>
#include <tbb/task_scheduler_init.h>
#include <gtest/gtest.h>

TEST(StanUiCommand, threadpool_init) {
  auto& scheduler_init = stan::math::init_threadpool_tbb();
  EXPECT_TRUE(scheduler_init.is_active());
  scheduler_init.terminate();
  EXPECT_FALSE(scheduler_init.is_active());
}
<|endoftext|>