<commit_before>/*
 * FogLAMP plugin filter class
 *
 * Copyright (c) 2018 Dianomic Systems
 *
 * Released under the Apache 2.0 Licence
 *
 * Author: Amandeep Singh Arora
 */

#include <filter_pipeline.h>
#include <config_handler.h>
#include <service_handler.h>
#include "rapidjson/writer.h"
#include "rapidjson/stringbuffer.h"

#define JSON_CONFIG_FILTER_ELEM "filter"
#define JSON_CONFIG_PIPELINE_ELEM "pipeline"

using namespace std;

/**
 * FilterPipeline class constructor
 *
 * This class abstracts the filter pipeline interface
 *
 * @param mgtClient	Management client handle
 * @param storage	Storage client handle
 * @param serviceName	Name of the service to which this pipeline applies
 */
FilterPipeline::FilterPipeline(ManagementClient* mgtClient, StorageClient& storage, string serviceName) :
			mgtClient(mgtClient), storage(storage), serviceName(serviceName)
{
}

/**
 * FilterPipeline destructor
 */
FilterPipeline::~FilterPipeline()
{
}

/**
 * Load the specified filter plugin
 *
 * @param filterName	The filter plugin to load
 * @return		Plugin handle on success, NULL otherwise
 *
 */
PLUGIN_HANDLE FilterPipeline::loadFilterPlugin(const string& filterName)
{
	if (filterName.empty())
	{
		Logger::getLogger()->error("Unable to fetch filter plugin '%s' from configuration.",
			filterName.c_str());
		// Failure
		return NULL;
	}
	Logger::getLogger()->info("Loading filter plugin '%s'.", filterName.c_str());

	PluginManager* manager = PluginManager::getInstance();
	PLUGIN_HANDLE handle;
	if ((handle = manager->loadPlugin(filterName, PLUGIN_TYPE_FILTER)) != NULL)
	{
		// Suceess
		Logger::getLogger()->info("Loaded filter plugin '%s'.", filterName.c_str());
	}
	return handle;
}

/**
 * Load all filter plugins in the pipeline
 *
 * @param categoryName	Configuration category name
 * @return		True if filters are loaded (or no filters at all)
 *			False otherwise
 */
bool FilterPipeline::loadFilters(const string& categoryName)
{
	vector<string> children;	// The Child categories of 'Filters'
	try
	{
		// Get the category with values and defaults
		ConfigCategory config = mgtClient->getCategory(categoryName);
		string filter = config.getValue(JSON_CONFIG_FILTER_ELEM);
		Logger::getLogger()->info("FilterPipeline::loadFilters(): categoryName=%s, filters=%s", categoryName.c_str(), filter.c_str());
		if (!filter.empty())
		{
			std::vector<pair<string, PLUGIN_HANDLE>> filterInfo;

			// Remove \" and leading/trailing "
			// TODO: improve/change this
			filter.erase(remove(filter.begin(), filter.end(), '\\' ), filter.end());
			size_t i;
			while (! (i = filter.find('"')) || (i = filter.rfind('"')) == static_cast<unsigned char>(filter.size() - 1))
			{
				filter.erase(i, 1);
			}

			//Parse JSON object for filters
			Document theFilters;
			theFilters.Parse(filter.c_str());
			// The "pipeline" property must be an array
			if (theFilters.HasParseError() ||
				!theFilters.HasMember(JSON_CONFIG_PIPELINE_ELEM) ||
				!theFilters[JSON_CONFIG_PIPELINE_ELEM].IsArray())
			{
				string errMsg("loadFilters: can not parse JSON '");
				errMsg += string(JSON_CONFIG_FILTER_ELEM) + "' property";
				Logger::getLogger()->fatal(errMsg.c_str());
				throw runtime_error(errMsg);
			}
			else
			{
				const Value& filterList = theFilters[JSON_CONFIG_PIPELINE_ELEM];
				if (!filterList.Size())
				{
					// Empty array, just return true
					return true;
				}

				// Prepare printable list of filters
				StringBuffer buffer;
				Writer<StringBuffer> writer(buffer);
				filterList.Accept(writer);
				string printableList(buffer.GetString());

				string logMsg("loadFilters: found filter(s) ");
				logMsg += printableList + " for plugin '";
				logMsg += categoryName + "'";

				Logger::getLogger()->info(logMsg.c_str());

				// Try loading all filter plugins: abort on any error
				for (Value::ConstValueIterator itr = filterList.Begin(); itr != filterList.End(); ++itr)
				{
					// Get "plugin" item fromn filterCategoryName
					string filterCategoryName = itr->GetString();
					ConfigCategory filterDetails = mgtClient->getCategory(filterCategoryName);
					if (!filterDetails.itemExists("plugin"))
					{
						string errMsg("loadFilters: 'plugin' item not found ");
						errMsg += "in " + filterCategoryName + " category";
						Logger::getLogger()->fatal(errMsg.c_str());
						throw runtime_error(errMsg);
					}
					string filterName = filterDetails.getValue("plugin");
					PLUGIN_HANDLE filterHandle;
					// Load filter plugin only: we don't call any plugin method right now
					filterHandle = loadFilterPlugin(filterName);
					if (!filterHandle)
					{
						string errMsg("Cannot load filter plugin '" + filterName + "'");
						Logger::getLogger()->fatal(errMsg.c_str());
						throw runtime_error(errMsg);
					}
					else
					{
						// Save filter handler: key is filterCategoryName
						filterInfo.push_back(pair<string,PLUGIN_HANDLE>
								     (filterCategoryName, filterHandle));
					}
				}

				// We have kept filter default config in the filterInfo map
				// Handle configuration for each filter
				PluginManager *pluginManager = PluginManager::getInstance();
				for (vector<pair<string, PLUGIN_HANDLE>>::iterator itr = filterInfo.begin();
				     itr != filterInfo.end();
				     ++itr)
				{
					// Get plugin default configuration
					string filterConfig = pluginManager->getInfo(itr->second)->config;

					// Create/Update default filter category items
					DefaultConfigCategory filterDefConfig(categoryName + "_" + itr->first, filterConfig);
					string filterDescription = "Configuration of '" + itr->first;
					filterDescription += "' filter for plugin '" + categoryName + "'";
					filterDefConfig.setDescription(filterDescription);

					if (!mgtClient->addCategory(filterDefConfig, true))
					{
						string errMsg("Cannot create/update '" + \
							      categoryName + "' filter category");
						Logger::getLogger()->fatal(errMsg.c_str());
						throw runtime_error(errMsg);
					}
					children.push_back(categoryName + "_" + itr->first);

					// Instantiate the FilterPlugin class
					// in order to call plugin entry points
					FilterPlugin* currentFilter = new FilterPlugin(itr->first,
										       itr->second);

					// Add filter to filters vector
					m_filters.push_back(currentFilter);
				}
			}
		}

		/*
		 * Put all the new catregories in the Filter category parent
		 * Create an empty South category if one doesn't exist
		 */
		string parentName = categoryName + " Filters";
		DefaultConfigCategory filterConfig(parentName, string("{}"));
		filterConfig.setDescription("Filters for " + categoryName);
		mgtClient->addCategory(filterConfig, true);
		mgtClient->addChildCategories(parentName, children);
		vector<string> children1;
		children1.push_back(parentName);
		mgtClient->addChildCategories(categoryName, children1);
		return true;
	}
	catch (ConfigItemNotFound* e)
	{
		delete e;
		Logger::getLogger()->info("loadFilters: no filters configured for '" + categoryName + "'");
		return true;
	}
	catch (exception& e)
	{
		Logger::getLogger()->fatal("loadFilters: failed to handle '" + categoryName + "' filters.");
		return false;
	}
	catch (...)
	{
		Logger::getLogger()->fatal("loadFilters: generic exception while loading '" + categoryName + "' filters.");
		return false;
	}
}

/**
 * Set the filter pipeline
 *
 * This method calls the method "plugin_init" for all loadad filters.
 * Up-to-date filter configurations and Ingest filtering methods
 * are passed to "plugin_init"
 *
 * @param passToOnwardFilter	Ptr to function that passes data to next filter
 * @param useFilteredData	Ptr to function that gets final filtered data
 * @param ingest		The ingest class handle
 * @return 		True on success,
 *			False otherwise.
 * @thown		Any caught exception
 */
bool FilterPipeline::setupFiltersPipeline(void *passToOnwardFilter, void *useFilteredData, void *ingest)
{
	bool initErrors = false;
	string errMsg = "'plugin_init' failed for filter '";
	for (auto it = m_filters.begin(); it != m_filters.end(); ++it)
	{
		string filterCategoryName =  serviceName + "_" + (*it)->getName();
		ConfigCategory updatedCfg;
		vector<string> children;

		try
		{
			Logger::getLogger()->info("Load plugin categoryName %s", filterCategoryName.c_str());
			// Fetch up to date filter configuration
			updatedCfg = mgtClient->getCategory(filterCategoryName);

			// Add filter category name under service/process config name
			children.push_back(filterCategoryName);
			mgtClient->addChildCategories(serviceName, children);

			ConfigHandler *configHandler = ConfigHandler::getInstance(mgtClient);
			configHandler->registerCategory((ServiceHandler *)ingest, filterCategoryName);

			m_filterCategories[filterCategoryName] = (*it);
		}
		// TODO catch specific exceptions
		catch (...)
		{
			throw;
		}

		// Iterate the load filters set in the Ingest class m_filters member
		if ((it + 1) != m_filters.end())
		{
			// Set next filter pointer as OUTPUT_HANDLE
			if (!(*it)->init(updatedCfg,
					(OUTPUT_HANDLE *)(*(it + 1)),
					filterReadingSetFn(passToOnwardFilter)))
			{
				errMsg += (*it)->getName() + "'";
				initErrors = true;
				break;
			}
		}
		else
		{
			// Set the Ingest class pointer as OUTPUT_HANDLE
			if (!(*it)->init(updatedCfg,
					 (OUTPUT_HANDLE *)(ingest),
					 filterReadingSetFn(useFilteredData)))
			{
				errMsg += (*it)->getName() + "'";
				initErrors = true;
				break;
			}
		}

		if ((*it)->persistData())
		{
			// Plugin support SP_PERSIST_DATA
			// Instantiate the PluginData class
			(*it)->m_plugin_data = new PluginData(&storage);
			// Load plugin data from storage layer
			string pluginStoredData = (*it)->m_plugin_data->loadStoredData(serviceName + (*it)->getName());
			//call 'plugin_start' with plugin data: startData()
			(*it)->startData(pluginStoredData);
		}
		else
		{
			// We don't call simple plugin_start for filters right now
		}
	}

	if (initErrors)
	{
		// Failure
		Logger::getLogger()->fatal("%s error: %s", __FUNCTION__, errMsg.c_str());
		return false;
	}

	//Success
	return true;
}

/**
 * Cleanup all the loaded filters
 *
 * Call "plugin_shutdown" method and free the FilterPlugin object
 *
 * @param categoryName		Configuration category name
 *
 */
void FilterPipeline::cleanupFilters(const string& categoryName)
{
	// Cleanup filters
	for (auto it = m_filters.begin(); it != m_filters.end(); ++it)
	{
		FilterPlugin* filter = *it;
		//string filterCategoryName =  categoryName + "_" + filter->getName();
		//mgtClient->unregisterCategory(filterCategoryName);
		//Logger::getLogger()->info("FilterPipeline::cleanupFilters(): unregistered category %s", filterCategoryName.c_str());

		// If plugin has SP_PERSIST_DATA option:
		if (filter->m_plugin_data)
	 	{
			// 1- call shutdownSaveData and get up-to-date plugin data.
			string saveData = filter->shutdownSaveData();
			// 2- store returned data: key is service/task categoryName + pluginName
			string key(categoryName + filter->getName());
			if (!filter->m_plugin_data->persistPluginData(key, saveData))
			{
				Logger::getLogger()->error("Filter plugin %s has failed to save data [%s] for key %s",
							   filter->getName().c_str(),
							   saveData.c_str(),
							   key.c_str());
			}
		}
		else
		{
			// Call filter plugin shutdown
			(*it)->shutdown();
		}

		// Free filter
		delete filter;
	}
}

/**
 * Configuration change for one of the filters. Lookup the category name and
 * find the plugin to call. Call the reconfigure method of that plugin with
 * the new configuration.
 *
 * @param category	The name of the configuration category
 * @param newConfig	The new category contents
 */
void FilterPipeline::configChange(const string& category, const string& newConfig)
{
	auto it = m_filterCategories.find(category);
	if (it != m_filterCategories.end())
	{
		it->second->reconfigure(newConfig);
	}
}

<commit_msg>FOGK-3146: Cleanup filters pipeline fix: use reverse order (#1643)<commit_after>/*
 * FogLAMP plugin filter class
 *
 * Copyright (c) 2018 Dianomic Systems
 *
 * Released under the Apache 2.0 Licence
 *
 * Author: Amandeep Singh Arora
 */

#include <filter_pipeline.h>
#include <config_handler.h>
#include <service_handler.h>
#include "rapidjson/writer.h"
#include "rapidjson/stringbuffer.h"

#define JSON_CONFIG_FILTER_ELEM "filter"
#define JSON_CONFIG_PIPELINE_ELEM "pipeline"

using namespace std;

/**
 * FilterPipeline class constructor
 *
 * This class abstracts the filter pipeline interface
 *
 * @param mgtClient	Management client handle
 * @param storage	Storage client handle
 * @param serviceName	Name of the service to which this pipeline applies
 */
FilterPipeline::FilterPipeline(ManagementClient* mgtClient, StorageClient& storage, string serviceName) :
			mgtClient(mgtClient), storage(storage), serviceName(serviceName)
{
}

/**
 * FilterPipeline destructor
 */
FilterPipeline::~FilterPipeline()
{
}

/**
 * Load the specified filter plugin
 *
 * @param filterName	The filter plugin to load
 * @return		Plugin handle on success, NULL otherwise
 *
 */
PLUGIN_HANDLE FilterPipeline::loadFilterPlugin(const string& filterName)
{
	if (filterName.empty())
	{
		Logger::getLogger()->error("Unable to fetch filter plugin '%s' from configuration.",
			filterName.c_str());
		// Failure
		return NULL;
	}
	Logger::getLogger()->info("Loading filter plugin '%s'.", filterName.c_str());

	PluginManager* manager = PluginManager::getInstance();
	PLUGIN_HANDLE handle;
	if ((handle = manager->loadPlugin(filterName, PLUGIN_TYPE_FILTER)) != NULL)
	{
		// Suceess
		Logger::getLogger()->info("Loaded filter plugin '%s'.", filterName.c_str());
	}
	return handle;
}

/**
 * Load all filter plugins in the pipeline
 *
 * @param categoryName	Configuration category name
 * @return		True if filters are loaded (or no filters at all)
 *			False otherwise
 */
bool FilterPipeline::loadFilters(const string& categoryName)
{
	vector<string> children;	// The Child categories of 'Filters'
	try
	{
		// Get the category with values and defaults
		ConfigCategory config = mgtClient->getCategory(categoryName);
		string filter = config.getValue(JSON_CONFIG_FILTER_ELEM);
		Logger::getLogger()->info("FilterPipeline::loadFilters(): categoryName=%s, filters=%s", categoryName.c_str(), filter.c_str());
		if (!filter.empty())
		{
			std::vector<pair<string, PLUGIN_HANDLE>> filterInfo;

			// Remove \" and leading/trailing "
			// TODO: improve/change this
			filter.erase(remove(filter.begin(), filter.end(), '\\' ), filter.end());
			size_t i;
			while (! (i = filter.find('"')) || (i = filter.rfind('"')) == static_cast<unsigned char>(filter.size() - 1))
			{
				filter.erase(i, 1);
			}

			//Parse JSON object for filters
			Document theFilters;
			theFilters.Parse(filter.c_str());
			// The "pipeline" property must be an array
			if (theFilters.HasParseError() ||
				!theFilters.HasMember(JSON_CONFIG_PIPELINE_ELEM) ||
				!theFilters[JSON_CONFIG_PIPELINE_ELEM].IsArray())
			{
				string errMsg("loadFilters: can not parse JSON '");
				errMsg += string(JSON_CONFIG_FILTER_ELEM) + "' property";
				Logger::getLogger()->fatal(errMsg.c_str());
				throw runtime_error(errMsg);
			}
			else
			{
				const Value& filterList = theFilters[JSON_CONFIG_PIPELINE_ELEM];
				if (!filterList.Size())
				{
					// Empty array, just return true
					return true;
				}

				// Prepare printable list of filters
				StringBuffer buffer;
				Writer<StringBuffer> writer(buffer);
				filterList.Accept(writer);
				string printableList(buffer.GetString());

				string logMsg("loadFilters: found filter(s) ");
				logMsg += printableList + " for plugin '";
				logMsg += categoryName + "'";

				Logger::getLogger()->info(logMsg.c_str());

				// Try loading all filter plugins: abort on any error
				for (Value::ConstValueIterator itr = filterList.Begin(); itr != filterList.End(); ++itr)
				{
					// Get "plugin" item fromn filterCategoryName
					string filterCategoryName = itr->GetString();
					ConfigCategory filterDetails = mgtClient->getCategory(filterCategoryName);
					if (!filterDetails.itemExists("plugin"))
					{
						string errMsg("loadFilters: 'plugin' item not found ");
						errMsg += "in " + filterCategoryName + " category";
						Logger::getLogger()->fatal(errMsg.c_str());
						throw runtime_error(errMsg);
					}
					string filterName = filterDetails.getValue("plugin");
					PLUGIN_HANDLE filterHandle;
					// Load filter plugin only: we don't call any plugin method right now
					filterHandle = loadFilterPlugin(filterName);
					if (!filterHandle)
					{
						string errMsg("Cannot load filter plugin '" + filterName + "'");
						Logger::getLogger()->fatal(errMsg.c_str());
						throw runtime_error(errMsg);
					}
					else
					{
						// Save filter handler: key is filterCategoryName
						filterInfo.push_back(pair<string,PLUGIN_HANDLE>
								     (filterCategoryName, filterHandle));
					}
				}

				// We have kept filter default config in the filterInfo map
				// Handle configuration for each filter
				PluginManager *pluginManager = PluginManager::getInstance();
				for (vector<pair<string, PLUGIN_HANDLE>>::iterator itr = filterInfo.begin();
				     itr != filterInfo.end();
				     ++itr)
				{
					// Get plugin default configuration
					string filterConfig = pluginManager->getInfo(itr->second)->config;

					// Create/Update default filter category items
					DefaultConfigCategory filterDefConfig(categoryName + "_" + itr->first, filterConfig);
					string filterDescription = "Configuration of '" + itr->first;
					filterDescription += "' filter for plugin '" + categoryName + "'";
					filterDefConfig.setDescription(filterDescription);

					if (!mgtClient->addCategory(filterDefConfig, true))
					{
						string errMsg("Cannot create/update '" + \
							      categoryName + "' filter category");
						Logger::getLogger()->fatal(errMsg.c_str());
						throw runtime_error(errMsg);
					}
					children.push_back(categoryName + "_" + itr->first);

					// Instantiate the FilterPlugin class
					// in order to call plugin entry points
					FilterPlugin* currentFilter = new FilterPlugin(itr->first,
										       itr->second);

					// Add filter to filters vector
					m_filters.push_back(currentFilter);
				}
			}
		}

		/*
		 * Put all the new catregories in the Filter category parent
		 * Create an empty South category if one doesn't exist
		 */
		string parentName = categoryName + " Filters";
		DefaultConfigCategory filterConfig(parentName, string("{}"));
		filterConfig.setDescription("Filters for " + categoryName);
		mgtClient->addCategory(filterConfig, true);
		mgtClient->addChildCategories(parentName, children);
		vector<string> children1;
		children1.push_back(parentName);
		mgtClient->addChildCategories(categoryName, children1);
		return true;
	}
	catch (ConfigItemNotFound* e)
	{
		delete e;
		Logger::getLogger()->info("loadFilters: no filters configured for '" + categoryName + "'");
		return true;
	}
	catch (exception& e)
	{
		Logger::getLogger()->fatal("loadFilters: failed to handle '" + categoryName + "' filters.");
		return false;
	}
	catch (...)
	{
		Logger::getLogger()->fatal("loadFilters: generic exception while loading '" + categoryName + "' filters.");
		return false;
	}
}

/**
 * Set the filter pipeline
 *
 * This method calls the method "plugin_init" for all loadad filters.
 * Up-to-date filter configurations and Ingest filtering methods
 * are passed to "plugin_init"
 *
 * @param passToOnwardFilter	Ptr to function that passes data to next filter
 * @param useFilteredData	Ptr to function that gets final filtered data
 * @param ingest		The ingest class handle
 * @return 		True on success,
 *			False otherwise.
 * @thown		Any caught exception
 */
bool FilterPipeline::setupFiltersPipeline(void *passToOnwardFilter, void *useFilteredData, void *ingest)
{
	bool initErrors = false;
	string errMsg = "'plugin_init' failed for filter '";
	for (auto it = m_filters.begin(); it != m_filters.end(); ++it)
	{
		string filterCategoryName =  serviceName + "_" + (*it)->getName();
		ConfigCategory updatedCfg;
		vector<string> children;

		try
		{
			Logger::getLogger()->info("Load plugin categoryName %s", filterCategoryName.c_str());
			// Fetch up to date filter configuration
			updatedCfg = mgtClient->getCategory(filterCategoryName);

			// Add filter category name under service/process config name
			children.push_back(filterCategoryName);
			mgtClient->addChildCategories(serviceName, children);

			ConfigHandler *configHandler = ConfigHandler::getInstance(mgtClient);
			configHandler->registerCategory((ServiceHandler *)ingest, filterCategoryName);

			m_filterCategories[filterCategoryName] = (*it);
		}
		// TODO catch specific exceptions
		catch (...)
		{
			throw;
		}

		// Iterate the load filters set in the Ingest class m_filters member
		if ((it + 1) != m_filters.end())
		{
			// Set next filter pointer as OUTPUT_HANDLE
			if (!(*it)->init(updatedCfg,
					(OUTPUT_HANDLE *)(*(it + 1)),
					filterReadingSetFn(passToOnwardFilter)))
			{
				errMsg += (*it)->getName() + "'";
				initErrors = true;
				break;
			}
		}
		else
		{
			// Set the Ingest class pointer as OUTPUT_HANDLE
			if (!(*it)->init(updatedCfg,
					 (OUTPUT_HANDLE *)(ingest),
					 filterReadingSetFn(useFilteredData)))
			{
				errMsg += (*it)->getName() + "'";
				initErrors = true;
				break;
			}
		}

		if ((*it)->persistData())
		{
			// Plugin support SP_PERSIST_DATA
			// Instantiate the PluginData class
			(*it)->m_plugin_data = new PluginData(&storage);
			// Load plugin data from storage layer
			string pluginStoredData = (*it)->m_plugin_data->loadStoredData(serviceName + (*it)->getName());
			//call 'plugin_start' with plugin data: startData()
			(*it)->startData(pluginStoredData);
		}
		else
		{
			// We don't call simple plugin_start for filters right now
		}
	}

	if (initErrors)
	{
		// Failure
		Logger::getLogger()->fatal("%s error: %s", __FUNCTION__, errMsg.c_str());
		return false;
	}

	//Success
	return true;
}

/**
 * Cleanup all the loaded filters
 *
 * Call "plugin_shutdown" method and free the FilterPlugin object
 *
 * @param categoryName		Configuration category name
 *
 */
void FilterPipeline::cleanupFilters(const string& categoryName)
{
	// Cleanup filters, in reverse order
	for (auto it = m_filters.rbegin(); it != m_filters.rend(); ++it)
	{
		FilterPlugin* filter = *it;
		//string filterCategoryName =  categoryName + "_" + filter->getName();
		//mgtClient->unregisterCategory(filterCategoryName);
		//Logger::getLogger()->info("FilterPipeline::cleanupFilters(): unregistered category %s", filterCategoryName.c_str());

		// If plugin has SP_PERSIST_DATA option:
		if (filter->m_plugin_data)
	 	{
			// 1- call shutdownSaveData and get up-to-date plugin data.
			string saveData = filter->shutdownSaveData();
			// 2- store returned data: key is service/task categoryName + pluginName
			string key(categoryName + filter->getName());
			if (!filter->m_plugin_data->persistPluginData(key, saveData))
			{
				Logger::getLogger()->error("Filter plugin %s has failed to save data [%s] for key %s",
							   filter->getName().c_str(),
							   saveData.c_str(),
							   key.c_str());
			}
		}
		else
		{
			// Call filter plugin shutdown
			(*it)->shutdown();
		}

		// Free filter
		delete filter;
	}
}

/**
 * Configuration change for one of the filters. Lookup the category name and
 * find the plugin to call. Call the reconfigure method of that plugin with
 * the new configuration.
 *
 * @param category	The name of the configuration category
 * @param newConfig	The new category contents
 */
void FilterPipeline::configChange(const string& category, const string& newConfig)
{
	auto it = m_filterCategories.find(category);
	if (it != m_filterCategories.end())
	{
		it->second->reconfigure(newConfig);
	}
}

<|endoftext|>


<commit_before>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "otbGenericRSResampleImageFilter.h"
#include "otbVectorImage.h"

#include "otbImageFileReader.h"
#include "otbStreamingImageFileWriter.h"

#include <ogr_spatialref.h>

// Default value
#include "itkPixelBuilder.h"

// Extract ROI
#include "otbMultiChannelExtractROI.h"

// Images definition
const unsigned int Dimension = 2;
typedef double                                      PixelType;
typedef otb::VectorImage<PixelType, Dimension>      ImageType;
typedef ImageType::SizeType                         SizeType;

typedef otb::GenericRSResampleImageFilter<ImageType,
                                          ImageType> ImageResamplerType;
typedef ImageResamplerType::OriginType              OriginType;
typedef ImageResamplerType::SpacingType             SpacingType;

typedef otb::ImageFileReader<ImageType>             ReaderType;
typedef otb::StreamingImageFileWriter<ImageType>    WriterType;

int otbGenericRSResampleImageFilterNew(int argc, char* argv[])
{
  // SmartPointer instanciation
  ImageResamplerType::Pointer resampler = ImageResamplerType::New();
  std::cout << resampler << std::endl;
  return EXIT_SUCCESS;
}

int otbGenericRSResampleImageFilter(int argc, char* argv[])
{

  // SmartPointer instanciation
  ImageResamplerType::Pointer resampler = ImageResamplerType::New();

  const char * infname = argv[1];
  const char * outfname = argv[6];
  unsigned int isize    = atoi(argv[2]);
  double iGridSpacing    = atof(argv[3]);
  int    useInRpc          = atoi(argv[4]);
  int    useOutRpc          = atoi(argv[5]);

  ReaderType::Pointer         reader    = ReaderType::New();

  // Read the input image
  reader->SetFileName(infname);
  reader->UpdateOutputInformation();

  // Fill the output size with the user selection
  SizeType      size;
  size.Fill(isize);

  // Set the origin & the spacing of the output
  OriginType  origin;
  origin[0] = 367340;
  origin[1] = 4.83467e+06;

  SpacingType  spacing;
  spacing[0] = 0.6;
  spacing[1] = -0.6;

  // Build the ouput projection ref : UTM ref
  OGRSpatialReference    oSRS;
  oSRS.SetProjCS("UTM");
  oSRS.SetUTM(31, true);
  char * utmRef = NULL;
  oSRS.exportToWkt(&utmRef);

  // Deformation Field spacing
  SpacingType  gridSpacing;
  gridSpacing[0] = iGridSpacing;
  gridSpacing[1] = -iGridSpacing;

  // Default value builder
  ImageType::PixelType defaultValue;
  itk::PixelBuilder<ImageType::PixelType>::Zero(defaultValue,
                                                reader->GetOutput()->GetNumberOfComponentsPerPixel());

  // Set the Resampler Parameters
  resampler->SetInput(reader->GetOutput());
  resampler->SetDeformationFieldSpacing(gridSpacing);
  resampler->SetOutputOrigin(origin);
  resampler->SetOutputSize(size);
  resampler->SetOutputSpacing(spacing);
  resampler->SetOutputProjectionRef(utmRef);
  resampler->SetEdgePaddingValue(defaultValue);
  if (useInRpc)
    {
    resampler->SetInputRpcGridSize(20);
    resampler->EstimateInputRpcModelOn();
    }

  if (useOutRpc)
    {
    resampler->SetOutputRpcGridSize(20);
    resampler->EstimateOutputRpcModelOn();
    }

  // Write the resampled image
  WriterType::Pointer writer= WriterType::New();
  writer->SetNumberOfDivisionsTiledStreaming(4);
  writer->SetFileName(outfname);
  writer->SetInput(resampler->GetOutput());
  writer->Update();

  std::cout << resampler << std::endl;

  return EXIT_SUCCESS;
}

int otbGenericRSResampleImageFilterFromMap(int argc, char* argv[])
{
  typedef otb::MultiChannelExtractROI<PixelType, PixelType>  ExtractROIType;

  // SmartPointer instanciation
  ExtractROIType::Pointer extractor = ExtractROIType::New();
  ImageResamplerType::Pointer resampler = ImageResamplerType::New();

  const char * infname   = argv[1];
  const char * outfname  = argv[4];
  double iGridSpacing    = atof(argv[2]);
  int    useInRpc        = atoi(argv[3]);

  // Reader Instanciation
  ReaderType::Pointer         reader    = ReaderType::New();
  reader->SetFileName(infname);
  reader->UpdateOutputInformation();

  SpacingType  spacing;
  spacing[0] =  2.5;
  spacing[1] = -2.5;

  // Deformation Field spacing
  SpacingType  gridSpacing;
  gridSpacing[0] = iGridSpacing;
  gridSpacing[1] = -iGridSpacing;

  // Default value builder
  ImageType::PixelType defaultValue;
  itk::PixelBuilder<ImageType::PixelType>::Zero(defaultValue,
                                                reader->GetOutput()->GetNumberOfComponentsPerPixel());

  // Extract a roi centered on the input center
  ImageType::RegionType roi;
  ImageType::IndexType  roiIndex;
  SizeType              roiSize;

  // Fill the size
  roiSize.Fill(500);

  // Fill the start index
  roiIndex[0] = (unsigned int)((reader->GetOutput()->GetLargestPossibleRegion().GetSize()[0] - roiSize[0]) /2);
  roiIndex[1] = (unsigned int)((reader->GetOutput()->GetLargestPossibleRegion().GetSize()[1] - roiSize[1]) /2);

  roi.SetIndex(roiIndex);
  roi.SetSize(roiSize);

  extractor->SetExtractionRegion(roi);
  extractor->SetInput(reader->GetOutput());
  extractor->UpdateOutputInformation();

  // Set the Resampler Parameters
  resampler->SetInput(extractor->GetOutput());
  resampler->SetDeformationFieldSpacing(gridSpacing);
  resampler->SetOutputParametersFromMap("UTM", spacing);

  if (useInRpc)
    {
    resampler->SetInputRpcGridSize(20);
    resampler->EstimateInputRpcModelOn();
    }

  // Write the resampled image
  typedef otb::StreamingImageFileWriter<ImageType>    WriterType;
  WriterType::Pointer writer= WriterType::New();
  writer->SetAutomaticTiledStreaming();
  writer->SetFileName(outfname);
  writer->SetInput(resampler->GetOutput());
  writer->Update();

  std::cout << resampler << std::endl;

  return EXIT_SUCCESS;
}

<commit_msg>ENH: reduce the size of the output image to be able to test the WordView sensor<commit_after>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "otbGenericRSResampleImageFilter.h"
#include "otbVectorImage.h"

#include "otbImageFileReader.h"
#include "otbStreamingImageFileWriter.h"

#include <ogr_spatialref.h>

// Default value
#include "itkPixelBuilder.h"

// Extract ROI
#include "otbMultiChannelExtractROI.h"

// Images definition
const unsigned int Dimension = 2;
typedef double                                      PixelType;
typedef otb::VectorImage<PixelType, Dimension>      ImageType;
typedef ImageType::SizeType                         SizeType;

typedef otb::GenericRSResampleImageFilter<ImageType,
                                          ImageType> ImageResamplerType;
typedef ImageResamplerType::OriginType              OriginType;
typedef ImageResamplerType::SpacingType             SpacingType;

typedef otb::ImageFileReader<ImageType>             ReaderType;
typedef otb::StreamingImageFileWriter<ImageType>    WriterType;

int otbGenericRSResampleImageFilterNew(int argc, char* argv[])
{
  // SmartPointer instanciation
  ImageResamplerType::Pointer resampler = ImageResamplerType::New();
  std::cout << resampler << std::endl;
  return EXIT_SUCCESS;
}

int otbGenericRSResampleImageFilter(int argc, char* argv[])
{

  // SmartPointer instanciation
  ImageResamplerType::Pointer resampler = ImageResamplerType::New();

  const char * infname = argv[1];
  const char * outfname = argv[6];
  unsigned int isize    = atoi(argv[2]);
  double iGridSpacing    = atof(argv[3]);
  int    useInRpc          = atoi(argv[4]);
  int    useOutRpc          = atoi(argv[5]);

  ReaderType::Pointer         reader    = ReaderType::New();

  // Read the input image
  reader->SetFileName(infname);
  reader->UpdateOutputInformation();

  // Fill the output size with the user selection
  SizeType      size;
  size.Fill(isize);

  // Set the origin & the spacing of the output
  OriginType  origin;
  origin[0] = 367340;
  origin[1] = 4.83467e+06;

  SpacingType  spacing;
  spacing[0] = 0.6;
  spacing[1] = -0.6;

  // Build the ouput projection ref : UTM ref
  OGRSpatialReference    oSRS;
  oSRS.SetProjCS("UTM");
  oSRS.SetUTM(31, true);
  char * utmRef = NULL;
  oSRS.exportToWkt(&utmRef);

  // Deformation Field spacing
  SpacingType  gridSpacing;
  gridSpacing[0] = iGridSpacing;
  gridSpacing[1] = -iGridSpacing;

  // Default value builder
  ImageType::PixelType defaultValue;
  itk::PixelBuilder<ImageType::PixelType>::Zero(defaultValue,
                                                reader->GetOutput()->GetNumberOfComponentsPerPixel());

  // Set the Resampler Parameters
  resampler->SetInput(reader->GetOutput());
  resampler->SetDeformationFieldSpacing(gridSpacing);
  resampler->SetOutputOrigin(origin);
  resampler->SetOutputSize(size);
  resampler->SetOutputSpacing(spacing);
  resampler->SetOutputProjectionRef(utmRef);
  resampler->SetEdgePaddingValue(defaultValue);
  if (useInRpc)
    {
    resampler->SetInputRpcGridSize(20);
    resampler->EstimateInputRpcModelOn();
    }

  if (useOutRpc)
    {
    resampler->SetOutputRpcGridSize(20);
    resampler->EstimateOutputRpcModelOn();
    }

  // Write the resampled image
  WriterType::Pointer writer= WriterType::New();
  writer->SetNumberOfDivisionsTiledStreaming(4);
  writer->SetFileName(outfname);
  writer->SetInput(resampler->GetOutput());
  writer->Update();

  std::cout << resampler << std::endl;

  return EXIT_SUCCESS;
}

int otbGenericRSResampleImageFilterFromMap(int argc, char* argv[])
{
  typedef otb::MultiChannelExtractROI<PixelType, PixelType>  ExtractROIType;

  // SmartPointer instanciation
  ExtractROIType::Pointer extractor = ExtractROIType::New();
  ImageResamplerType::Pointer resampler = ImageResamplerType::New();

  const char * infname   = argv[1];
  const char * outfname  = argv[4];
  double iGridSpacing    = atof(argv[2]);
  int    useInRpc        = atoi(argv[3]);

  // Reader Instanciation
  ReaderType::Pointer         reader    = ReaderType::New();
  reader->SetFileName(infname);
  reader->UpdateOutputInformation();

  SpacingType  spacing;
  spacing[0] =  2.5;
  spacing[1] = -2.5;

  // Deformation Field spacing
  SpacingType  gridSpacing;
  gridSpacing[0] = iGridSpacing;
  gridSpacing[1] = -iGridSpacing;

  // Default value builder
  ImageType::PixelType defaultValue;
  itk::PixelBuilder<ImageType::PixelType>::Zero(defaultValue,
                                                reader->GetOutput()->GetNumberOfComponentsPerPixel());

  // Extract a roi centered on the input center
  ImageType::RegionType roi;
  ImageType::IndexType  roiIndex;
  SizeType              roiSize;

  // Fill the size
  roiSize.Fill(250);

  // Fill the start index
  roiIndex[0] = (unsigned int)((reader->GetOutput()->GetLargestPossibleRegion().GetSize()[0] - roiSize[0]) /2);
  roiIndex[1] = (unsigned int)((reader->GetOutput()->GetLargestPossibleRegion().GetSize()[1] - roiSize[1]) /2);

  roi.SetIndex(roiIndex);
  roi.SetSize(roiSize);

  extractor->SetExtractionRegion(roi);
  extractor->SetInput(reader->GetOutput());
  extractor->UpdateOutputInformation();

  // Set the Resampler Parameters
  resampler->SetInput(extractor->GetOutput());
  resampler->SetDeformationFieldSpacing(gridSpacing);
  resampler->SetOutputParametersFromMap("UTM", spacing);

  if (useInRpc)
    {
    resampler->SetInputRpcGridSize(20);
    resampler->EstimateInputRpcModelOn();
    }

  // Write the resampled image
  typedef otb::StreamingImageFileWriter<ImageType>    WriterType;
  WriterType::Pointer writer= WriterType::New();
  writer->SetAutomaticTiledStreaming();
  writer->SetFileName(outfname);
  writer->SetInput(resampler->GetOutput());
  writer->Update();

  std::cout << resampler << std::endl;

  return EXIT_SUCCESS;
}

<|endoftext|>


<commit_before>/*
 * Copyright (C) 2005-2017 Centre National d'Etudes Spatiales (CNES)
 *
 * This file is part of Orfeo Toolbox
 *
 *     https://www.orfeo-toolbox.org/
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <time.h>

#include "otbWrapperApplication.h"
#include "otbWrapperApplicationFactory.h"

#include "otbStreamingStatisticsMapFromLabelImageFilter.h"
#include "otbLabelImageSmallRegionMergingFilter.h"
#include "itkChangeLabelImageFilter.h"

namespace otb
{
namespace Wrapper
{
class SmallRegionsMerging : public Application
{
public:
  typedef SmallRegionsMerging Self;
  typedef Application Superclass;
  typedef itk::SmartPointer<Self> Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;

  typedef FloatVectorImageType              ImageType;
  typedef ImageType::InternalPixelType      ImagePixelType;
  typedef UInt32ImageType                   LabelImageType;
  typedef LabelImageType::InternalPixelType LabelImagePixelType;

  //typedef otb::StreamingStatisticsImageFilter<LabelImageType> StatisticsImageFilterType;
  typedef otb::StreamingStatisticsMapFromLabelImageFilter<ImageType, LabelImageType> StatisticsMapFromLabelImageFilterType;

  typedef otb::LabelImageSmallRegionMergingFilter<LabelImageType> LabelImageSmallRegionMergingFilterType;

  typedef itk::ChangeLabelImageFilter<LabelImageType,LabelImageType> ChangeLabelImageFilterType;

  itkNewMacro(Self);
  itkTypeMacro(Merging, otb::Application);

private:
  ChangeLabelImageFilterType::Pointer m_ChangeLabelFilter;

  void DoInit() override
  {
    SetName("SmallRegionsMerging");
    SetDescription("This application merges small regions of a segmentation result to connected region.");

    SetDocName("Small Region Merging");
    SetDocLongDescription("Given a segmentation result and the original image, it will"
                          " merge segments whose size in pixels is lower than minsize parameter"
                          " with the adjacent segments with the adjacent segment with closest"
                          " radiometry and acceptable size.\n\n"
                          "Small segments will be processed by increasing size: first all segments"
                          " for which area is equal to 1 pixel will be merged with adjacent"
                          " segments, then all segments of area equal to 2 pixels will be processed,"
                          " until segments of area minsize."
                          " \n\n");
    SetDocLimitations("This application is more efficient if the labels are contiguous, starting from 0.");
    SetDocAuthors("OTB-Team");
    SetDocSeeAlso( "Segmentation");
    AddDocTag(Tags::Segmentation);

    AddParameter(ParameterType_InputImage,  "in", "Input image");
    SetParameterDescription( "in", "The input image, containing initial spectral signatures corresponding to the segmented image (inseg)." );
    AddParameter(ParameterType_InputImage,  "inseg", "Segmented image");
    SetParameterDescription( "inseg", "Segmented image where each pixel value is the unique integer label of the segment it belongs to." );

    AddParameter(ParameterType_OutputImage, "out", "Output Image");
    SetParameterDescription( "out", "The output image. The output image is the segmented image where the minimal segments have been merged." );
    SetDefaultOutputPixelType("out",ImagePixelType_uint32);

    AddParameter(ParameterType_Int, "minsize", "Minimum Segment Size");
    SetParameterDescription("minsize", "Minimum Segment Size. If, after the segmentation, a segment is of size strictly lower than this criterion, the segment is merged with the segment that has the closest sepctral signature.");
    SetDefaultParameterInt("minsize", 50);
    SetMinimumParameterIntValue("minsize", 1);
    MandatoryOff("minsize");

    AddRAMParameter();

    // Doc example parameter settings
    SetDocExampleParameterValue("in","smooth.tif");
    SetDocExampleParameterValue("inseg","segmentation.tif");
    SetDocExampleParameterValue("out","merged.tif");
    SetDocExampleParameterValue("minsize","50");

    SetOfficialDocLink();
  }

  void DoUpdateParameters() override
  {
  }

  void DoExecute() override
  {
    clock_t tic = clock();

    unsigned int minSize     = GetParameterInt("minsize");

    //Acquisition of the input image dimensions
    ImageType::Pointer imageIn = GetParameterImage("in");

    LabelImageType::Pointer labelIn = GetParameterUInt32Image("inseg");

    // Compute statistics for each segment
    auto labelStatsFilter = StatisticsMapFromLabelImageFilterType::New();
    labelStatsFilter->SetInput(imageIn);
    labelStatsFilter->SetInputLabelImage(labelIn);
    AddProcess(labelStatsFilter->GetStreamer() , "Computing stats on input image ...");
    labelStatsFilter->Update();

    // Convert Map to Unordered map

    auto  labelPopulationMap = labelStatsFilter->GetLabelPopulationMap();
    std::unordered_map< unsigned int,double> labelPopulation;
    for (auto population : labelPopulationMap)
      {
      labelPopulation[population.first]=population.second;
      }
    auto  meanValueMap = labelStatsFilter->GetMeanValueMap();
    std::unordered_map< unsigned int, itk::VariableLengthVector<double> > meanValues;
    for (const auto & mean : meanValueMap)
      {
      meanValues[mean.first] = mean.second;
      }

    // Compute the LUT from the original label image to the merged output label image.
    auto regionMergingFilter = LabelImageSmallRegionMergingFilterType::New();
    regionMergingFilter->SetInput( labelIn );
    regionMergingFilter->SetLabelPopulation( labelPopulation );
    regionMergingFilter->SetLabelStatistic( meanValues );

    regionMergingFilter->SetMinSize( minSize);

    AddProcess(regionMergingFilter, "Computing LUT ...");
    regionMergingFilter->Update();

    // Relabelling using the LUT
    auto changeLabelFilter = ChangeLabelImageFilterType::New();
    changeLabelFilter->SetInput(labelIn);

    const auto & LUT = regionMergingFilter->GetLUT();

    for (auto label : LUT)
    {
      if (label.first != label.second)
      {
        changeLabelFilter->SetChange(label.first, label.second);
      }
    }
    SetParameterOutputImage("out", changeLabelFilter->GetOutput());
    RegisterPipeline();
    clock_t toc = clock();
    otbAppLogINFO(<<"Elapsed time: "<<(double)(toc - tic) / CLOCKS_PER_SEC<<" seconds");
  }

};
}
}

OTB_APPLICATION_EXPORT(otb::Wrapper::SmallRegionsMerging)

<commit_msg>DOC : Long description should not end with a newline<commit_after>/*
 * Copyright (C) 2005-2017 Centre National d'Etudes Spatiales (CNES)
 *
 * This file is part of Orfeo Toolbox
 *
 *     https://www.orfeo-toolbox.org/
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <time.h>

#include "otbWrapperApplication.h"
#include "otbWrapperApplicationFactory.h"

#include "otbStreamingStatisticsMapFromLabelImageFilter.h"
#include "otbLabelImageSmallRegionMergingFilter.h"
#include "itkChangeLabelImageFilter.h"

namespace otb
{
namespace Wrapper
{
class SmallRegionsMerging : public Application
{
public:
  typedef SmallRegionsMerging Self;
  typedef Application Superclass;
  typedef itk::SmartPointer<Self> Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;

  typedef FloatVectorImageType              ImageType;
  typedef ImageType::InternalPixelType      ImagePixelType;
  typedef UInt32ImageType                   LabelImageType;
  typedef LabelImageType::InternalPixelType LabelImagePixelType;

  //typedef otb::StreamingStatisticsImageFilter<LabelImageType> StatisticsImageFilterType;
  typedef otb::StreamingStatisticsMapFromLabelImageFilter<ImageType, LabelImageType> StatisticsMapFromLabelImageFilterType;

  typedef otb::LabelImageSmallRegionMergingFilter<LabelImageType> LabelImageSmallRegionMergingFilterType;

  typedef itk::ChangeLabelImageFilter<LabelImageType,LabelImageType> ChangeLabelImageFilterType;

  itkNewMacro(Self);
  itkTypeMacro(Merging, otb::Application);

private:
  ChangeLabelImageFilterType::Pointer m_ChangeLabelFilter;

  void DoInit() override
  {
    SetName("SmallRegionsMerging");
    SetDescription("This application merges small regions of a segmentation result to connected region.");

    SetDocName("Small Region Merging");
    SetDocLongDescription("Given a segmentation result and the original image, it will"
                          " merge segments whose size in pixels is lower than minsize parameter"
                          " with the adjacent segments with the adjacent segment with closest"
                          " radiometry and acceptable size.\n\n"
                          "Small segments will be processed by increasing size: first all segments"
                          " for which area is equal to 1 pixel will be merged with adjacent"
                          " segments, then all segments of area equal to 2 pixels will be processed,"
                          " until segments of area minsize.");

    SetDocLimitations("This application is more efficient if the labels are contiguous, starting from 0.");
    SetDocAuthors("OTB-Team");
    SetDocSeeAlso( "Segmentation");
    AddDocTag(Tags::Segmentation);

    AddParameter(ParameterType_InputImage,  "in", "Input image");
    SetParameterDescription( "in", "The input image, containing initial spectral signatures corresponding to the segmented image (inseg)." );
    AddParameter(ParameterType_InputImage,  "inseg", "Segmented image");
    SetParameterDescription( "inseg", "Segmented image where each pixel value is the unique integer label of the segment it belongs to." );

    AddParameter(ParameterType_OutputImage, "out", "Output Image");
    SetParameterDescription( "out", "The output image. The output image is the segmented image where the minimal segments have been merged." );
    SetDefaultOutputPixelType("out",ImagePixelType_uint32);

    AddParameter(ParameterType_Int, "minsize", "Minimum Segment Size");
    SetParameterDescription("minsize", "Minimum Segment Size. If, after the segmentation, a segment is of size strictly lower than this criterion, the segment is merged with the segment that has the closest sepctral signature.");
    SetDefaultParameterInt("minsize", 50);
    SetMinimumParameterIntValue("minsize", 1);
    MandatoryOff("minsize");

    AddRAMParameter();

    // Doc example parameter settings
    SetDocExampleParameterValue("in","smooth.tif");
    SetDocExampleParameterValue("inseg","segmentation.tif");
    SetDocExampleParameterValue("out","merged.tif");
    SetDocExampleParameterValue("minsize","50");

    SetOfficialDocLink();
  }

  void DoUpdateParameters() override
  {
  }

  void DoExecute() override
  {
    clock_t tic = clock();

    unsigned int minSize     = GetParameterInt("minsize");

    //Acquisition of the input image dimensions
    ImageType::Pointer imageIn = GetParameterImage("in");

    LabelImageType::Pointer labelIn = GetParameterUInt32Image("inseg");

    // Compute statistics for each segment
    auto labelStatsFilter = StatisticsMapFromLabelImageFilterType::New();
    labelStatsFilter->SetInput(imageIn);
    labelStatsFilter->SetInputLabelImage(labelIn);
    AddProcess(labelStatsFilter->GetStreamer() , "Computing stats on input image ...");
    labelStatsFilter->Update();

    // Convert Map to Unordered map

    auto  labelPopulationMap = labelStatsFilter->GetLabelPopulationMap();
    std::unordered_map< unsigned int,double> labelPopulation;
    for (auto population : labelPopulationMap)
      {
      labelPopulation[population.first]=population.second;
      }
    auto  meanValueMap = labelStatsFilter->GetMeanValueMap();
    std::unordered_map< unsigned int, itk::VariableLengthVector<double> > meanValues;
    for (const auto & mean : meanValueMap)
      {
      meanValues[mean.first] = mean.second;
      }

    // Compute the LUT from the original label image to the merged output label image.
    auto regionMergingFilter = LabelImageSmallRegionMergingFilterType::New();
    regionMergingFilter->SetInput( labelIn );
    regionMergingFilter->SetLabelPopulation( labelPopulation );
    regionMergingFilter->SetLabelStatistic( meanValues );

    regionMergingFilter->SetMinSize( minSize);

    AddProcess(regionMergingFilter, "Computing LUT ...");
    regionMergingFilter->Update();

    // Relabelling using the LUT
    auto changeLabelFilter = ChangeLabelImageFilterType::New();
    changeLabelFilter->SetInput(labelIn);

    const auto & LUT = regionMergingFilter->GetLUT();

    for (auto label : LUT)
    {
      if (label.first != label.second)
      {
        changeLabelFilter->SetChange(label.first, label.second);
      }
    }
    SetParameterOutputImage("out", changeLabelFilter->GetOutput());
    RegisterPipeline();
    clock_t toc = clock();
    otbAppLogINFO(<<"Elapsed time: "<<(double)(toc - tic) / CLOCKS_PER_SEC<<" seconds");
  }

};
}
}

OTB_APPLICATION_EXPORT(otb::Wrapper::SmallRegionsMerging)

<|endoftext|>


<commit_before>/*
 * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 * Written by Disa Mhembere (disa@jhu.edu)
 *
 * This file is part of FlashMatrix.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY CURRENT_KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "kmeans.h"
#define OMP_MAX_THREADS 32
#define KM_TEST 0

namespace {
bool updated = false;
static unsigned NEV;
static size_t K;
static unsigned NUM_ROWS;
static omp_lock_t writelock;

static struct timeval start, end;

/**
  * \brief Print an arry of some length `len`.
  *	 \param len The length of the array.
  */
template <typename T>
static void print_arr(T* arr, unsigned len) {
    printf("[ ");
    for (unsigned i = 0; i < len; i++) {
        std::cout << arr[i] << " ";
    }
    printf("]\n");
}

/*
 * \Internal
 * \brief Simple helper used to print a vector.
 * \param v The vector to print.
 */
template <typename T>
static void print_vector(typename std::vector<T>& v)
{
	std::cout << "[";

	typename std::vector<T>::iterator itr = v.begin();
	for (; itr != v.end(); itr++) {
		std::cout << " "<< *itr;
	}
	std::cout <<  " ]\n";
}

/**
 * \brief Get the squared distance given two values.
 *	\param arg1 the first value.
 *	\param arg2 the second value.
 *  \return the squared distance.
 */
static double dist_sq(double arg1, double arg2)
{
	double diff = arg1 - arg2;
	return diff*diff;
}

/**
  * \brief This initializes clusters by randomly choosing sample
  *		membership in a cluster.
  * See: http://en.wikipedia.org/wiki/K-means_clustering#Initialization_methods
  *	\param cluster_assignments Which cluster each sample falls into.
  */
static void random_partition_init(unsigned* cluster_assignments)
{
	BOOST_LOG_TRIVIAL(info) << "Random init start";

// #pragma omp parallel for firstprivate(cluster_assignments, K) shared(cluster_assignments)
	for (unsigned vid = 0; vid < NUM_ROWS; vid++) {
		size_t assigned_cluster = random() % K; // 0...K
		cluster_assignments[vid] = assigned_cluster;
	}

	// NOTE: M-Step is called in compute func to update cluster counts & centers
#if 0
	printf("After rand paritions cluster_asgns: "); print_arr(cluster_assignments, NUM_ROWS);
#endif
	BOOST_LOG_TRIVIAL(info) << "Random init end\n";
}

/**
  * \brief Forgy init takes `K` random samples from the matrix
  *		and uses them as cluster centers.
  * \param matrix the flattened matrix who's rows are being clustered.
  * \param clusters The cluster centers (means) flattened matrix.
  */
static void forgy_init(const double* matrix, double* clusters) {

	BOOST_LOG_TRIVIAL(info) << "Forgy init start";

	for (unsigned clust_idx = 0; clust_idx < K; clust_idx++) { // 0...K
		unsigned rand_idx = random() % (NUM_ROWS - 1); // 0...(n-1)
		memcpy(&clusters[clust_idx*NEV], &matrix[rand_idx*NEV], sizeof(clusters[0])*NEV);
	}
	BOOST_LOG_TRIVIAL(info) << "Forgy init end";
}

/**
 * \brief A parallel version of the kmeans++ initialization alg.
 */
static void kmeanspp_init()
{
	BOOST_LOG_TRIVIAL(fatal) << "kmeanspp not yet implemented";
	exit(-1);
}

/*\Internal
* \brief print a col wise matrix of type double / double.
* Used for testing only.
* \param matrix The col wise matrix.
* \param rows The number of rows in the mat
* \param cols The number of cols in the mat
*/
template <typename T>
static void print_mat(T* matrix, const unsigned rows, const unsigned cols) {
	for (unsigned row = 0; row < rows; row++) {
		std::cout << "[";
		for (unsigned col = 0; col < cols; col++) {
			std::cout << " " << matrix[row*cols + col];
		}
		std::cout <<  " ]\n";
	}
}

/**
  * \brief Update the cluster assignments while recomputing distance matrix.
  * \param matrix The flattened matrix who's rows are being clustered.
  * \param clusters The cluster centers (means) flattened matrix.
  *	\param cluster_assignments Which cluster each sample falls into.
  */
static void E_step(const double* matrix, double* clusters,
		unsigned* cluster_assignments, unsigned* cluster_assignment_counts)
{

    gettimeofday(&start, NULL);

	// Create per thread vectors
	std::vector<std::vector<unsigned>> pt_cl_as_cnt; // K * OMP_MAX_THREADS
	std::vector<std::vector<double>> pt_cl; // K * nev * OMP_MAX_THREADS

	pt_cl_as_cnt.resize(OMP_MAX_THREADS);
	pt_cl.resize(OMP_MAX_THREADS);

	for (int i = 0; i < OMP_MAX_THREADS; i++) {
		pt_cl_as_cnt[i].resize(K); // C++ default is 0 value in all
		pt_cl[i].resize(K*NEV);
	}

#pragma omp parallel for firstprivate(matrix, clusters) shared(pt_cl_as_cnt, cluster_assignments)
    for (unsigned row = 0; row < NUM_ROWS; row++) {
        size_t asgnd_clust = std::numeric_limits<size_t>::max();
        double best = std::numeric_limits<double>::max();
        for (size_t clust_idx = 0; clust_idx < K; clust_idx++) {
            double sum = 0;
            for (unsigned col = 0; col < NEV; col++) {
                sum += dist_sq(matrix[(row*NEV) + col], clusters[clust_idx*NEV + col]);
            }
            if (sum < best) {
                best = sum;
                asgnd_clust = clust_idx;
            }
        }

		if (asgnd_clust != cluster_assignments[row]) {
			// We must avoid the possible race condition on the `updated` variable
			// Cheap because it only happens a max of once per iteration.
			if (!updated) {
				omp_set_lock(&writelock);
				updated = true;
				omp_unset_lock(&writelock);
			}
		}
        cluster_assignments[row] = asgnd_clust;

		pt_cl_as_cnt[omp_get_thread_num()][asgnd_clust]++; // Add to local copy
		// Accumulate for local copies
		for (unsigned col = 0; col < NEV; col++) {
			pt_cl[omp_get_thread_num()][asgnd_clust*NEV + col] =
				pt_cl[omp_get_thread_num()][asgnd_clust*NEV + col] + matrix[row*NEV + col];
		}
    }

	BOOST_LOG_TRIVIAL(info) << "Clearing cluster assignment counts";
	memset(cluster_assignment_counts, 0, sizeof(cluster_assignment_counts[0])*K);

	// Serial aggreate of OMP_MAX_THREADS vectors
	for (int i = 0; i < OMP_MAX_THREADS; i++) {
		// Counts
		for (unsigned j = 0; j < K; j++) {
			cluster_assignment_counts[j] += pt_cl_as_cnt[i][j];
		}

		// Summation for cluster centers
#pragma omp parallel for firstprivate(pt_cl) shared(clusters)
		for (unsigned row = 0; row < K; row++) { /* ClusterID */
			for (unsigned col = 0; col < NEV; col++) { /* Features */
				clusters[row*NEV+col] = pt_cl[i][row*NEV + col] + clusters[row*NEV + col];
			}
		}
	}

    gettimeofday(&end, NULL);
    BOOST_LOG_TRIVIAL(info) << "E-step time taken = " << time_diff(start, end) << " sec";

#if KM_TEST
	printf("Cluster assignment counts: "); print_arr(cluster_assignment_counts, K);
    printf("Cluster assignments:\n"); print_arr(cluster_assignments, NUM_ROWS);
#endif
}

/**
 * \brief Update the cluster means
 * \param matrix The matrix who's rows are being clustered.
 * \param clusters The cluster centers (means).
 * \param cluster_assignment_counts How many members each cluster has.
 * \param cluster_assignments Which cluster each sample falls into.
 */
static void M_step(const double* matrix, double* clusters, unsigned*
		cluster_assignment_counts, const unsigned* cluster_assignments, bool init=false)
{
	BOOST_LOG_TRIVIAL(info) << "M_step start";

	if (init) {
		BOOST_LOG_TRIVIAL(info) << "Clearing cluster centers ...";
		memset(clusters, 0, sizeof(clusters[0])*K*NEV);

		BOOST_LOG_TRIVIAL(info) << "Clearing cluster assignment counts";
		memset(cluster_assignment_counts, 0, sizeof(cluster_assignment_counts[0])*K);

		for (unsigned vid = 0; vid < NUM_ROWS; vid++) {

			unsigned asgnd_clust = cluster_assignments[vid];
			cluster_assignment_counts[asgnd_clust]++;

			for (unsigned col = 0; col < NEV; col++) {
				clusters[asgnd_clust*NEV + col] =
					clusters[asgnd_clust*NEV + col] + matrix[vid*NEV + col];
			}
		}
	}

	// Take the mean of all added means
	BOOST_LOG_TRIVIAL(info) << " Div by in place ...";
	for (unsigned clust_idx = 0; clust_idx < K; clust_idx++) {
		if (cluster_assignment_counts[clust_idx] > 0) { // Avoid div by 0
// #pragma omp parallel for firstprivate(cluster_assignment_counts, NEV) shared(clusters)
			for (unsigned col = 0; col < NEV; col++) {
				clusters[clust_idx*NEV + col] =
					clusters[clust_idx*NEV + col] / cluster_assignment_counts[clust_idx];
			}
		}
	}

# if KM_TEST
	printf("Cluster centers: \n"); print_mat(clusters, K, NEV);
#endif
}
}

namespace fg
{
unsigned compute_kmeans(const double* matrix, double* clusters,
		unsigned* cluster_assignments, unsigned* cluster_assignment_counts,
		const unsigned num_rows, const unsigned nev, const size_t k, const unsigned MAX_ITERS,
		const std::string init)
{
	NEV = nev;
	K = k;
	NUM_ROWS = num_rows;

	if (K > NUM_ROWS || K < 2 || K == (unsigned)-1) {
		BOOST_LOG_TRIVIAL(fatal)
			<< "'k' must be between 2 and the number of rows in the matrix";
		exit(-1);
	}

	if (init.compare("random") != 0 && init.compare("kmeanspp") != 0 &&
			init.compare("forgy") != 0) {
		BOOST_LOG_TRIVIAL(fatal)
			<< "[ERROR]: param init must be one of: 'random', 'kmeanspp'.It is '"
			<< init << "'";
		exit(-1);
	}

#ifdef PROFILER
	ProfilerStart(PROF_FILE_LOC);
#endif

	/*** Begin VarInit of data structures ***/
	memset(cluster_assignments, -1,
			sizeof(cluster_assignments[0])*NUM_ROWS);
	memset(cluster_assignment_counts, 0, sizeof(cluster_assignment_counts[0])*K);
	omp_init_lock(&writelock);

	/*** End VarInit ***/

	if (init == "random") {
		BOOST_LOG_TRIVIAL(info) << "Init is random_partition \n";
		random_partition_init(cluster_assignments);
		// We must now update cluster centers before we begin
		M_step(matrix, clusters, cluster_assignment_counts, cluster_assignments, true);
	}
	if (init == "forgy") {
		BOOST_LOG_TRIVIAL(info) << "Init is forgy";
		forgy_init(matrix, clusters);
	}
	else {
		if (init == "kmeanspp") {
			kmeanspp_init(); // TODO: kmeanspp
		}
	}

	BOOST_LOG_TRIVIAL(info) << "Matrix K-means starting ...";

	bool converged = false;

	std::string str_iters =  MAX_ITERS == std::numeric_limits<unsigned>::max() ?
	   	"until convergence ...":
		std::to_string(MAX_ITERS) + " iterations ...";
	BOOST_LOG_TRIVIAL(info) << "Computing " << str_iters;
	unsigned iter = 1;
	while (iter < MAX_ITERS) {
		// Hold cluster assignment counter

		BOOST_LOG_TRIVIAL(info) << "\nE-step Iteration " << iter <<
			" . Computing cluster assignments ...";
		E_step(matrix, clusters, cluster_assignments, cluster_assignment_counts);

		if (!updated) {
			converged = true;
			break;
		} else { updated = false; }

		BOOST_LOG_TRIVIAL(info) << "M-step Updating cluster means ...";
		M_step(matrix, clusters, cluster_assignment_counts, cluster_assignments);

		iter++;
	}

	omp_destroy_lock(&writelock);

#ifdef PROFILER
	ProfilerStop();
#endif
	BOOST_LOG_TRIVIAL(info) << "\n******************************************\n";

	if (converged) {
		BOOST_LOG_TRIVIAL(info) <<
			"K-means converged in " << iter << " iterations";
	} else {
		BOOST_LOG_TRIVIAL(warning) << "[Warning]: K-means failed to converge in "
			<< iter << " iterations";
	}
	BOOST_LOG_TRIVIAL(info) << "\n******************************************\n";

	return iter;
}
}
<commit_msg>[Matrix]: Using libcommon omp method to determine the number of threads on the machine<commit_after>/*
 * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 * Written by Disa Mhembere (disa@jhu.edu)
 *
 * This file is part of FlashMatrix.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY CURRENT_KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "kmeans.h"
#define KM_TEST 0

namespace {
bool updated = false;
static unsigned NEV;
static size_t K;
static unsigned NUM_ROWS;
static omp_lock_t writelock;
short OMP_MAX_THREADS;

static struct timeval start, end;

/**
  * \brief Print an arry of some length `len`.
  *	 \param len The length of the array.
  */
template <typename T>
static void print_arr(T* arr, unsigned len) {
    printf("[ ");
    for (unsigned i = 0; i < len; i++) {
        std::cout << arr[i] << " ";
    }
    printf("]\n");
}

/*
 * \Internal
 * \brief Simple helper used to print a vector.
 * \param v The vector to print.
 */
template <typename T>
static void print_vector(typename std::vector<T>& v)
{
	std::cout << "[";

	typename std::vector<T>::iterator itr = v.begin();
	for (; itr != v.end(); itr++) {
		std::cout << " "<< *itr;
	}
	std::cout <<  " ]\n";
}

/**
 * \brief Get the squared distance given two values.
 *	\param arg1 the first value.
 *	\param arg2 the second value.
 *  \return the squared distance.
 */
static double dist_sq(double arg1, double arg2)
{
	double diff = arg1 - arg2;
	return diff*diff;
}

/**
  * \brief This initializes clusters by randomly choosing sample
  *		membership in a cluster.
  * See: http://en.wikipedia.org/wiki/K-means_clustering#Initialization_methods
  *	\param cluster_assignments Which cluster each sample falls into.
  */
static void random_partition_init(unsigned* cluster_assignments)
{
	BOOST_LOG_TRIVIAL(info) << "Random init start";

// #pragma omp parallel for firstprivate(cluster_assignments, K) shared(cluster_assignments)
	for (unsigned vid = 0; vid < NUM_ROWS; vid++) {
		size_t assigned_cluster = random() % K; // 0...K
		cluster_assignments[vid] = assigned_cluster;
	}

	// NOTE: M-Step is called in compute func to update cluster counts & centers
#if 0
	printf("After rand paritions cluster_asgns: "); print_arr(cluster_assignments, NUM_ROWS);
#endif
	BOOST_LOG_TRIVIAL(info) << "Random init end\n";
}

/**
  * \brief Forgy init takes `K` random samples from the matrix
  *		and uses them as cluster centers.
  * \param matrix the flattened matrix who's rows are being clustered.
  * \param clusters The cluster centers (means) flattened matrix.
  */
static void forgy_init(const double* matrix, double* clusters) {

	BOOST_LOG_TRIVIAL(info) << "Forgy init start";

	for (unsigned clust_idx = 0; clust_idx < K; clust_idx++) { // 0...K
		unsigned rand_idx = random() % (NUM_ROWS - 1); // 0...(n-1)
		memcpy(&clusters[clust_idx*NEV], &matrix[rand_idx*NEV], sizeof(clusters[0])*NEV);
	}
	BOOST_LOG_TRIVIAL(info) << "Forgy init end";
}

/**
 * \brief A parallel version of the kmeans++ initialization alg.
 */
static void kmeanspp_init()
{
	BOOST_LOG_TRIVIAL(fatal) << "kmeanspp not yet implemented";
	exit(-1);
}

/*\Internal
* \brief print a col wise matrix of type double / double.
* Used for testing only.
* \param matrix The col wise matrix.
* \param rows The number of rows in the mat
* \param cols The number of cols in the mat
*/
template <typename T>
static void print_mat(T* matrix, const unsigned rows, const unsigned cols) {
	for (unsigned row = 0; row < rows; row++) {
		std::cout << "[";
		for (unsigned col = 0; col < cols; col++) {
			std::cout << " " << matrix[row*cols + col];
		}
		std::cout <<  " ]\n";
	}
}

/**
  * \brief Update the cluster assignments while recomputing distance matrix.
  * \param matrix The flattened matrix who's rows are being clustered.
  * \param clusters The cluster centers (means) flattened matrix.
  *	\param cluster_assignments Which cluster each sample falls into.
  */
static void E_step(const double* matrix, double* clusters,
		unsigned* cluster_assignments, unsigned* cluster_assignment_counts)
{

    gettimeofday(&start, NULL);

	// Create per thread vectors
	std::vector<std::vector<unsigned>> pt_cl_as_cnt; // K * OMP_MAX_THREADS
	std::vector<std::vector<double>> pt_cl; // K * nev * OMP_MAX_THREADS

	pt_cl_as_cnt.resize(OMP_MAX_THREADS);
	pt_cl.resize(OMP_MAX_THREADS);

	for (int i = 0; i < OMP_MAX_THREADS; i++) {
		pt_cl_as_cnt[i].resize(K); // C++ default is 0 value in all
		pt_cl[i].resize(K*NEV);
	}

#pragma omp parallel for firstprivate(matrix, clusters) shared(pt_cl_as_cnt, cluster_assignments)
    for (unsigned row = 0; row < NUM_ROWS; row++) {
        size_t asgnd_clust = std::numeric_limits<size_t>::max();
        double best = std::numeric_limits<double>::max();
        for (size_t clust_idx = 0; clust_idx < K; clust_idx++) {
            double sum = 0;
            for (unsigned col = 0; col < NEV; col++) {
                sum += dist_sq(matrix[(row*NEV) + col], clusters[clust_idx*NEV + col]);
            }
            if (sum < best) {
                best = sum;
                asgnd_clust = clust_idx;
            }
        }

		if (asgnd_clust != cluster_assignments[row]) {
			// We must avoid the possible race condition on the `updated` variable
			// Cheap because it only happens a max of once per iteration.
			if (!updated) {
				omp_set_lock(&writelock);
				updated = true;
				omp_unset_lock(&writelock);
			}
		}
        cluster_assignments[row] = asgnd_clust;

		pt_cl_as_cnt[omp_get_thread_num()][asgnd_clust]++; // Add to local copy
		// Accumulate for local copies
		for (unsigned col = 0; col < NEV; col++) {
			pt_cl[omp_get_thread_num()][asgnd_clust*NEV + col] =
				pt_cl[omp_get_thread_num()][asgnd_clust*NEV + col] + matrix[row*NEV + col];
		}
    }

	BOOST_LOG_TRIVIAL(info) << "Clearing cluster assignment counts";
	memset(cluster_assignment_counts, 0, sizeof(cluster_assignment_counts[0])*K);

	// Serial aggreate of OMP_MAX_THREADS vectors
	for (int i = 0; i < OMP_MAX_THREADS; i++) {
		// Counts
		for (unsigned j = 0; j < K; j++) {
			cluster_assignment_counts[j] += pt_cl_as_cnt[i][j];
		}

		// Summation for cluster centers
#pragma omp parallel for firstprivate(pt_cl) shared(clusters)
		for (unsigned row = 0; row < K; row++) { /* ClusterID */
			for (unsigned col = 0; col < NEV; col++) { /* Features */
				clusters[row*NEV+col] = pt_cl[i][row*NEV + col] + clusters[row*NEV + col];
			}
		}
	}

    gettimeofday(&end, NULL);
    BOOST_LOG_TRIVIAL(info) << "E-step time taken = " << time_diff(start, end) << " sec";

#if KM_TEST
	printf("Cluster assignment counts: "); print_arr(cluster_assignment_counts, K);
    printf("Cluster assignments:\n"); print_arr(cluster_assignments, NUM_ROWS);
#endif
}

/**
 * \brief Update the cluster means
 * \param matrix The matrix who's rows are being clustered.
 * \param clusters The cluster centers (means).
 * \param cluster_assignment_counts How many members each cluster has.
 * \param cluster_assignments Which cluster each sample falls into.
 */
static void M_step(const double* matrix, double* clusters, unsigned*
		cluster_assignment_counts, const unsigned* cluster_assignments, bool init=false)
{
	BOOST_LOG_TRIVIAL(info) << "M_step start";

	if (init) {
		BOOST_LOG_TRIVIAL(info) << "Clearing cluster centers ...";
		memset(clusters, 0, sizeof(clusters[0])*K*NEV);

		BOOST_LOG_TRIVIAL(info) << "Clearing cluster assignment counts";
		memset(cluster_assignment_counts, 0, sizeof(cluster_assignment_counts[0])*K);

		for (unsigned vid = 0; vid < NUM_ROWS; vid++) {

			unsigned asgnd_clust = cluster_assignments[vid];
			cluster_assignment_counts[asgnd_clust]++;

			for (unsigned col = 0; col < NEV; col++) {
				clusters[asgnd_clust*NEV + col] =
					clusters[asgnd_clust*NEV + col] + matrix[vid*NEV + col];
			}
		}
	}

	// Take the mean of all added means
	BOOST_LOG_TRIVIAL(info) << " Div by in place ...";
	for (unsigned clust_idx = 0; clust_idx < K; clust_idx++) {
		if (cluster_assignment_counts[clust_idx] > 0) { // Avoid div by 0
// #pragma omp parallel for firstprivate(cluster_assignment_counts, NEV) shared(clusters)
			for (unsigned col = 0; col < NEV; col++) {
				clusters[clust_idx*NEV + col] =
					clusters[clust_idx*NEV + col] / cluster_assignment_counts[clust_idx];
			}
		}
	}

# if KM_TEST
	printf("Cluster centers: \n"); print_mat(clusters, K, NEV);
#endif
}
}

namespace fg
{
unsigned compute_kmeans(const double* matrix, double* clusters,
		unsigned* cluster_assignments, unsigned* cluster_assignment_counts,
		const unsigned num_rows, const unsigned nev, const size_t k, const unsigned MAX_ITERS,
		const std::string init)
{
	NEV = nev;
	K = k;
	NUM_ROWS = num_rows;
	OMP_MAX_THREADS = get_num_omp_threads();

	if (K > NUM_ROWS || K < 2 || K == (unsigned)-1) {
		BOOST_LOG_TRIVIAL(fatal)
			<< "'k' must be between 2 and the number of rows in the matrix";
		exit(-1);
	}

	if (init.compare("random") != 0 && init.compare("kmeanspp") != 0 &&
			init.compare("forgy") != 0) {
		BOOST_LOG_TRIVIAL(fatal)
			<< "[ERROR]: param init must be one of: 'random', 'kmeanspp'.It is '"
			<< init << "'";
		exit(-1);
	}

#ifdef PROFILER
	ProfilerStart(PROF_FILE_LOC);
#endif

	/*** Begin VarInit of data structures ***/
	memset(cluster_assignments, -1,
			sizeof(cluster_assignments[0])*NUM_ROWS);
	memset(cluster_assignment_counts, 0, sizeof(cluster_assignment_counts[0])*K);
	omp_init_lock(&writelock);

	/*** End VarInit ***/

	if (init == "random") {
		BOOST_LOG_TRIVIAL(info) << "Init is random_partition \n";
		random_partition_init(cluster_assignments);
		// We must now update cluster centers before we begin
		M_step(matrix, clusters, cluster_assignment_counts, cluster_assignments, true);
	}
	if (init == "forgy") {
		BOOST_LOG_TRIVIAL(info) << "Init is forgy";
		forgy_init(matrix, clusters);
	}
	else {
		if (init == "kmeanspp") {
			kmeanspp_init(); // TODO: kmeanspp
		}
	}

	BOOST_LOG_TRIVIAL(info) << "Matrix K-means starting ...";

	bool converged = false;

	std::string str_iters =  MAX_ITERS == std::numeric_limits<unsigned>::max() ?
	   	"until convergence ...":
		std::to_string(MAX_ITERS) + " iterations ...";
	BOOST_LOG_TRIVIAL(info) << "Computing " << str_iters;
	unsigned iter = 1;
	while (iter < MAX_ITERS) {
		// Hold cluster assignment counter

		BOOST_LOG_TRIVIAL(info) << "\nE-step Iteration " << iter <<
			" . Computing cluster assignments ...";
		E_step(matrix, clusters, cluster_assignments, cluster_assignment_counts);

		if (!updated) {
			converged = true;
			break;
		} else { updated = false; }

		BOOST_LOG_TRIVIAL(info) << "M-step Updating cluster means ...";
		M_step(matrix, clusters, cluster_assignment_counts, cluster_assignments);

		iter++;
	}

	omp_destroy_lock(&writelock);

#ifdef PROFILER
	ProfilerStop();
#endif
	BOOST_LOG_TRIVIAL(info) << "\n******************************************\n";

	if (converged) {
		BOOST_LOG_TRIVIAL(info) <<
			"K-means converged in " << iter << " iterations";
	} else {
		BOOST_LOG_TRIVIAL(warning) << "[Warning]: K-means failed to converge in "
			<< iter << " iterations";
	}
	BOOST_LOG_TRIVIAL(info) << "\n******************************************\n";

	return iter;
}
}
<|endoftext|>


<commit_before>#include <stdlib.h>
#include <wchar.h>
#include "XSData.h"
#include "Tab.h"
#include "Parser.h"
#include "Scanner.h"

namespace CocoXml{
    struct options_s enum_options[]={
        {"UNKNOWN_NAMESPACE", UNKNOWN_NAMESPACE},
        {"END_UNKNOWN_NAMESPACE", END_UNKNOWN_NAMESPACE},
        {"UNKNOWN_TAG", UNKNOWN_TAG},
        {"END_UNKNOWN_TAG", END_UNKNOWN_TAG},
        {"UNKNOWN_ATTR_NAMESPACE", UNKNOWN_ATTR_NAMESPACE},
        {"UNKNOWN_ATTR", UNKNOWN_ATTR},
        {"UNKNOWN_PROCESSING_INSTRUCTION", UNKNOWN_PROCESSING_INSTRUCTION},
        // For the nodes in common status.
        {"TEXT", TEXT},
        {"CDATA", CDATA},
        {"COMMENT", COMMENT},
        {"WHITESPACE", WHITESPACE},
        // For the nodes in Unknown namespaces.
        {"UNS_TEXT", UNS_TEXT},
        {"UNS_CDATA", UNS_CDATA},
        {"UNS_COMMENT", UNS_COMMENT},
        {"UNS_WHITESPACE", UNS_WHITESPACE},
        // For the nodes in Unknown tags.
        {"UT_TEXT", UT_TEXT},
        {"UT_CDATA", UT_CDATA},
        {"UT_COMMENT", UT_COMMENT},
        {"UT_WHITESPACE", UT_WHITESPACE},
        {NULL, -1}
    };
    map<const wchar_t*, int, wcharCmp> optionsMap;

    XmlLangDefinition::XmlLangDefinition(Tab *tab, Errors *errors){
        int i;
        this->tab = tab;
        this->errors = errors;

        for (i=0; enum_options[i].value>=0; i++){
            optionsMap.insert(make_pair(coco_string_create(enum_options[i].name), enum_options[i].value));
        }
        useVector.resize(optionsMap.size());
    }

    void XmlLangDefinition::AddOption(const wchar_t* optname, int line){
        int optval; Symbol* sym;

        map<const wchar_t*, int>::iterator it = optionsMap.find(optname);
        if(it != optionsMap.end()){
            /* find it */
            optval = it->second;
        }else{
            wchar_t fmt[200];
            coco_swprintf(fmt, 200, L"Unsupported option '%ls' encountered. Line(%d)\n", optname, line);
            for(it=optionsMap.begin();it!=optionsMap.end();it++){
                wprintf(L"Options:%ls,%d\n", it->first, it->second);
            }
            /* unsupported option */
            errors->Exception(fmt);
            return;
        }
        useVector[optval] = true;

        sym = tab->FindSym(optname);
        if (sym == NULL) tab->NewSym(Node::t, optname, line);
    }
    int XmlLangDefinition::addUniqueToken(const wchar_t* tokenName, const wchar_t* typeName, int line){
        Symbol *sym = tab->FindSym(tokenName);
        if (sym != NULL){
            wprintf(L"typeName '%ls' declared twice\n", tokenName);
            errors->count ++;
        }
        sym = tab->NewSym(Node::t, tokenName, line);
        return sym->n;
    }
    void XmlLangDefinition::AddTag(const wchar_t* tagname, wchar_t* tokenname, int line){
        TagInfo *tinfo = new TagInfo();
        wchar_t tmp[200];

        tinfo->startToken = addUniqueToken(tokenname, L"Tag", line);

        coco_swprintf(tmp, 200, L"END_%ls", tokenname);
        tinfo->endToken = addUniqueToken(tmp, L"Tag", line);

        map<const wchar_t*, TagInfo*>::iterator it = Tags.find(tagname);
        if (it != Tags.end()){
            wprintf(L"Tag '%ls' declared twice\n", tagname);
            errors->count ++;
        }
        Tags.insert(make_pair(tagname, tinfo));
    }
    void XmlLangDefinition::AddAttr(const wchar_t* attrname, const wchar_t* tokenname, int line){
        int value = addUniqueToken(tokenname, L"Attribute", line);
        map<const wchar_t*, int>::iterator it = Attrs.find(attrname);
        if (it!=Attrs.end()){
            wprintf(L"Attribute '%ls' declared twice\n", attrname);
            errors->count ++;
        }
        Attrs.insert(make_pair(attrname, value));
    }
    void XmlLangDefinition::AddProcessingInstruction(const wchar_t* pinstruction, const wchar_t* tokenname, int line){
        int value = addUniqueToken(tokenname, L"Processing instruction", line);
        map<const wchar_t*, int>::iterator it = PInstructions.find(pinstruction);
        if (it!=PInstructions.end()){
            wprintf(L"Processing Instruction '%ls' declared twice\n", pinstruction);
            errors->count ++;
        }
        PInstructions.insert(make_pair(pinstruction, value));
    }
    void XmlLangDefinition::Write(FILE* gen){
        int option;
        for (option = 0; option < useVector.size(); ++option)
            if (useVector[option]){
                const char *optionname = enum_options[option].name;
                map<const wchar_t*, int>::iterator iter=optionsMap.find(coco_string_create(optionname));
                if(iter!=optionsMap.end()){
                    fwprintf(gen, L"\tcurXLDef.useVector[%d] = true; // %ls", option, iter->first);
                }else{
                    errors->Exception(L"Why come here? it MUST have some error in somwhere?\n");
                }
            }
        map<const wchar_t*, TagInfo*>::iterator iter;
        for (iter=Tags.begin(); iter!=Tags.end(); ++iter)
            fwprintf(gen, L"\tcurXLDef.AddTag(%ls, %d, %d);",
                     iter->first, iter->second->startToken, iter->second->endToken);
        map<const wchar_t*, int>::iterator iter1;
        for (iter1=Attrs.begin(); iter1!=Attrs.end(); ++iter1)
            fwprintf(gen, L"\tcurXLDef.AddAttr(%ls, %d);", iter1->first, iter1->second);
        map<const wchar_t*, int>::iterator iter2;
        for (iter2=PInstructions.begin(); iter2!=PInstructions.end(); ++iter2)
            fwprintf(gen, L"\tcurXLDef.AddProcessInstruction(%ls, %d);", iter2->first, iter2->second);
    }
    XmlScannerData::XmlScannerData(Parser *parser){
        this->tab = parser->tab;
        this->errors = parser->errors;
    }
    void XmlScannerData::Add(const wchar_t* NamespaceURI, XmlLangDefinition *xldef){
        map<const wchar_t*, XmlLangDefinition*>::iterator it=XmlLangMap.find(NamespaceURI);
        if (it != XmlLangMap.end()){
            wchar_t fmt[200];
            coco_swprintf(fmt, 200, L"Namespace '%ls' declared twice.", NamespaceURI);
            /* unsupported option */
            errors->Exception(fmt);
        }
        XmlLangMap.insert(make_pair(NamespaceURI, xldef));
    }
    void XmlScannerData::OpenGen(const wchar_t *genName, bool backUp) { /* pdt */
        wchar_t *fn = coco_string_create_append(tab->outDir, genName); /* pdt */
        char *chFn = coco_string_create_char(fn);
        FILE* tmp;
        if (backUp && ((tmp = fopen(chFn, "r")) != NULL)) {
            fclose(tmp);
            wchar_t *oldName = coco_string_create_append(fn, L".old");
            char *chOldName = coco_string_create_char(oldName);
            remove(chOldName); rename(chFn, chOldName); // copy with overwrite
            coco_string_delete(chOldName);
            coco_string_delete(oldName);
        }
        if ((gen = fopen(chFn, "w")) == NULL) {
            errors->Exception(L"-- Cannot generate scanner file");
        }
        coco_string_delete(chFn);
        coco_string_delete(fn);
    }
    void XmlScannerData::CopyFramePart(const wchar_t * stop){
        wchar_t startCh = stop[0];
        int endOfStopString = coco_string_length(stop)-1;
        wchar_t ch = 0;

        fwscanf(fram, L"%lc", &ch); //fram.ReadByte();
        while (!feof(fram)) // ch != EOF
            if (ch == startCh) {
                int i = 0;
                do {
                    if (i == endOfStopString) return; // stop[0..i] found
                    fwscanf(fram, L"%lc", &ch); i++;
                } while (ch == stop[i]);
                // stop[0..i-1] found; continue with last read character
                wchar_t *subStop = coco_string_create(stop, 0, i);
                fwprintf(gen, L"%ls", subStop);
                coco_string_delete(subStop);
            } else {
                fwprintf(gen, L"%lc", ch);
                fwscanf(fram, L"%lc", &ch);
            }
        wprintf(L"CopyFramePart:[%ls]\n", stop);
        errors->Exception(L" -- incomplete or corrupt scanner frame file\n");
    }
    void XmlScannerData::WriteOptions(){
        fwprintf(gen, L"    public enum Options {");
        map<const wchar_t*, int>::iterator iter;
        for (iter=optionsMap.begin(); iter!=optionsMap.end(); iter++)
            fwprintf(gen, L"        %ls,", iter->first);
        fwprintf(gen, L"    };");
        fwprintf(gen, L"    public const int numOptions = %d;", optionsMap.size());
    }
    void XmlScannerData::WriteDeclarations(){
        Symbol *sym;
        fwprintf(gen, L"    static readonly int[] useKindVector = new int[] {");
        map<const wchar_t*, int>::iterator iter;
        for (iter=optionsMap.begin(); iter!=optionsMap.end(); ++iter){
            sym = tab->FindSym(iter->first);
            fwprintf(gen, L"        %d, // %ls", sym == NULL ? -1 : sym->n, iter->first);
        }
        fwprintf(gen, L"    };");
    }
    void XmlScannerData::WriteInitialization(){
        map<const wchar_t *, XmlLangDefinition*>::iterator iter;
        for (iter=XmlLangMap.begin(); iter!=XmlLangMap.end(); ++iter){
            fwprintf(gen, L"\tcurXLDef = new XmlLangDefinition();");
            iter->second->Write(gen);
            fwprintf(gen, L"\tXmlLangMap.Add(\"%ls\", curXLDef);", iter->first);
        }
    }
    int XmlScannerData::GenNamespaceOpen(const wchar_t *nsName) {
        if (nsName == NULL || coco_string_length(nsName) == 0) {
            return 0;
        }
        int len = coco_string_length(nsName);
        int startPos = 0, endPos;
        int nrOfNs = 0;
        do {
            endPos = coco_string_indexof(nsName + startPos, COCO_CPP_NAMESPACE_SEPARATOR);
            if (endPos == -1) { endPos = len; }
            wchar_t *curNs = coco_string_create(nsName, startPos, endPos - startPos);
            fwprintf(gen, L"namespace %ls {\n", curNs);
            coco_string_delete(curNs);
            startPos = endPos + 1;
            ++nrOfNs;
        } while (startPos < len);
        return nrOfNs;
    }
    void XmlScannerData::GenNamespaceClose(int nrOfNs) {
        for (int i = 0; i < nrOfNs; ++i) {
            fwprintf(gen, L"}; // namespace\n");
        }
    }
    void XmlScannerData::WriteXmlScanner(){
        wchar_t *fr = coco_string_create_append(tab->srcDir, L"XmlScanner.frame");
        char *chFr = coco_string_create_char(fr);

        FILE* tmp;
        if ((tmp = fopen(chFr, "r")) == NULL) {
            if (coco_string_length(tab->frameDir) != 0) {
                delete [] fr;
                fr = coco_string_create(tab->frameDir);
                coco_string_merge(fr, L"/");
                coco_string_merge(fr, L"XmlScanner.frame");
            }
            coco_string_delete(chFr);
            chFr = coco_string_create_char(fr);
            if ((tmp = fopen(chFr, "r")) == NULL) {
                errors->Exception(L"-- Cannot find XmlScanner.frame\n");
            } else {
                fclose(tmp);
            }
        } else {
            fclose(tmp);
        }
        if ((fram = fopen(chFr, "r")) == NULL) {
            errors->Exception(L"-- Cannot open XmlScanner.frame.\n");
        }
        coco_string_delete(chFr);
        coco_string_delete(fr);

        /* to generate header file */
        {
            OpenGen(L"XmlScanner.h", true);
            CopyFramePart(L"/*---- Namespace Begin ----*/");
            int nrOfNs = GenNamespaceOpen(tab->nsName);

            CopyFramePart(L"/*---- Options ----*/"); WriteOptions();

            CopyFramePart(L"/*---- Declarations ----*/"); WriteDeclarations();

            CopyFramePart(L"/*---- Namespace End ----*/");

            GenNamespaceClose(nrOfNs);

            CopyFramePart(L"/*---- Implementation ----*/");
            fclose(gen);
        }

        /* to generate source file */
        {
            OpenGen(L"XmlScanner.cpp", true);
            CopyFramePart(L"/*---- Begin ----*/");
            fwprintf(gen, L"// THIS FILE IS GENERATED BY CocoXml AUTOMATICALLY, DO NOT EDIT IT MANUALLY.");
            CopyFramePart(L"/*---- Namespace Begin ----*/");
            int nrOfNs = GenNamespaceOpen(tab->nsName);

            CopyFramePart(L"/*---- Initialization ----*/"); WriteInitialization();

            CopyFramePart(L"/*---- Namespace End ----*/");

            CopyFramePart(L"/*---- $$$ ----*/");

            GenNamespaceClose(nrOfNs);
            fclose(gen);
        }
    }
} //namespace
<commit_msg>add line character<commit_after>#include <stdlib.h>
#include <wchar.h>
#include "XSData.h"
#include "Tab.h"
#include "Parser.h"
#include "Scanner.h"

namespace CocoXml{
    struct options_s enum_options[]={
        {"UNKNOWN_NAMESPACE", UNKNOWN_NAMESPACE},
        {"END_UNKNOWN_NAMESPACE", END_UNKNOWN_NAMESPACE},
        {"UNKNOWN_TAG", UNKNOWN_TAG},
        {"END_UNKNOWN_TAG", END_UNKNOWN_TAG},
        {"UNKNOWN_ATTR_NAMESPACE", UNKNOWN_ATTR_NAMESPACE},
        {"UNKNOWN_ATTR", UNKNOWN_ATTR},
        {"UNKNOWN_PROCESSING_INSTRUCTION", UNKNOWN_PROCESSING_INSTRUCTION},
        // For the nodes in common status.
        {"TEXT", TEXT},
        {"CDATA", CDATA},
        {"COMMENT", COMMENT},
        {"WHITESPACE", WHITESPACE},
        // For the nodes in Unknown namespaces.
        {"UNS_TEXT", UNS_TEXT},
        {"UNS_CDATA", UNS_CDATA},
        {"UNS_COMMENT", UNS_COMMENT},
        {"UNS_WHITESPACE", UNS_WHITESPACE},
        // For the nodes in Unknown tags.
        {"UT_TEXT", UT_TEXT},
        {"UT_CDATA", UT_CDATA},
        {"UT_COMMENT", UT_COMMENT},
        {"UT_WHITESPACE", UT_WHITESPACE},
        {NULL, -1}
    };
    map<const wchar_t*, int, wcharCmp> optionsMap;

    XmlLangDefinition::XmlLangDefinition(Tab *tab, Errors *errors){
        int i;
        this->tab = tab;
        this->errors = errors;

        for (i=0; enum_options[i].value>=0; i++){
            optionsMap.insert(make_pair(coco_string_create(enum_options[i].name), enum_options[i].value));
        }
        useVector.resize(optionsMap.size());
    }

    void XmlLangDefinition::AddOption(const wchar_t* optname, int line){
        int optval; Symbol* sym;

        map<const wchar_t*, int>::iterator it = optionsMap.find(optname);
        if(it != optionsMap.end()){
            /* find it */
            optval = it->second;
        }else{
            wchar_t fmt[200];
            coco_swprintf(fmt, 200, L"Unsupported option '%ls' encountered. Line(%d)\n", optname, line);
            for(it=optionsMap.begin();it!=optionsMap.end();it++){
                wprintf(L"Options:%ls,%d\n", it->first, it->second);
            }
            /* unsupported option */
            errors->Exception(fmt);
            return;
        }
        useVector[optval] = true;

        sym = tab->FindSym(optname);
        if (sym == NULL) tab->NewSym(Node::t, optname, line);
    }
    int XmlLangDefinition::addUniqueToken(const wchar_t* tokenName, const wchar_t* typeName, int line){
        Symbol *sym = tab->FindSym(tokenName);
        if (sym != NULL){
            wprintf(L"typeName '%ls' declared twice\n", tokenName);
            errors->count ++;
        }
        sym = tab->NewSym(Node::t, tokenName, line);
        return sym->n;
    }
    void XmlLangDefinition::AddTag(const wchar_t* tagname, wchar_t* tokenname, int line){
        TagInfo *tinfo = new TagInfo();
        wchar_t tmp[200];

        tinfo->startToken = addUniqueToken(tokenname, L"Tag", line);

        coco_swprintf(tmp, 200, L"END_%ls", tokenname);
        tinfo->endToken = addUniqueToken(tmp, L"Tag", line);

        map<const wchar_t*, TagInfo*>::iterator it = Tags.find(tagname);
        if (it != Tags.end()){
            wprintf(L"Tag '%ls' declared twice\n", tagname);
            errors->count ++;
        }
        Tags.insert(make_pair(tagname, tinfo));
    }
    void XmlLangDefinition::AddAttr(const wchar_t* attrname, const wchar_t* tokenname, int line){
        int value = addUniqueToken(tokenname, L"Attribute", line);
        map<const wchar_t*, int>::iterator it = Attrs.find(attrname);
        if (it!=Attrs.end()){
            wprintf(L"Attribute '%ls' declared twice\n", attrname);
            errors->count ++;
        }
        Attrs.insert(make_pair(attrname, value));
    }
    void XmlLangDefinition::AddProcessingInstruction(const wchar_t* pinstruction, const wchar_t* tokenname, int line){
        int value = addUniqueToken(tokenname, L"Processing instruction", line);
        map<const wchar_t*, int>::iterator it = PInstructions.find(pinstruction);
        if (it!=PInstructions.end()){
            wprintf(L"Processing Instruction '%ls' declared twice\n", pinstruction);
            errors->count ++;
        }
        PInstructions.insert(make_pair(pinstruction, value));
    }
    void XmlLangDefinition::Write(FILE* gen){
        int option;
        for (option = 0; option < useVector.size(); ++option)
            if (useVector[option]){
                const char *optionname = enum_options[option].name;
                map<const wchar_t*, int>::iterator iter=optionsMap.find(coco_string_create(optionname));
                if(iter!=optionsMap.end()){
                    fwprintf(gen, L"    curXLDef.useVector[%d] = true; // %ls\n", option, iter->first);
                }else{
                    errors->Exception(L"Why come here? it MUST have some error in somwhere?\n");
                }
            }
        map<const wchar_t*, TagInfo*>::iterator iter;
        for (iter=Tags.begin(); iter!=Tags.end(); ++iter)
            fwprintf(gen, L"    curXLDef.AddTag(L%ls, %d, %d);\n",
                     iter->first, iter->second->startToken, iter->second->endToken);
        map<const wchar_t*, int>::iterator iter1;
        for (iter1=Attrs.begin(); iter1!=Attrs.end(); ++iter1)
            fwprintf(gen, L"    curXLDef.AddAttr(L%ls, %d);\n", iter1->first, iter1->second);
        map<const wchar_t*, int>::iterator iter2;
        for (iter2=PInstructions.begin(); iter2!=PInstructions.end(); ++iter2)
            fwprintf(gen, L"    curXLDef.AddProcessInstruction(L%ls, %d);\n", iter2->first, iter2->second);
    }
    XmlScannerData::XmlScannerData(Parser *parser){
        this->tab = parser->tab;
        this->errors = parser->errors;
    }
    void XmlScannerData::Add(const wchar_t* NamespaceURI, XmlLangDefinition *xldef){
        map<const wchar_t*, XmlLangDefinition*>::iterator it=XmlLangMap.find(NamespaceURI);
        if (it != XmlLangMap.end()){
            wchar_t fmt[200];
            coco_swprintf(fmt, 200, L"Namespace '%ls' declared twice.", NamespaceURI);
            /* unsupported option */
            errors->Exception(fmt);
        }
        XmlLangMap.insert(make_pair(NamespaceURI, xldef));
    }
    void XmlScannerData::OpenGen(const wchar_t *genName, bool backUp) { /* pdt */
        wchar_t *fn = coco_string_create_append(tab->outDir, genName); /* pdt */
        char *chFn = coco_string_create_char(fn);
        FILE* tmp;
        if (backUp && ((tmp = fopen(chFn, "r")) != NULL)) {
            fclose(tmp);
            wchar_t *oldName = coco_string_create_append(fn, L".old");
            char *chOldName = coco_string_create_char(oldName);
            remove(chOldName); rename(chFn, chOldName); // copy with overwrite
            coco_string_delete(chOldName);
            coco_string_delete(oldName);
        }
        if ((gen = fopen(chFn, "w")) == NULL) {
            errors->Exception(L"-- Cannot generate scanner file");
        }
        coco_string_delete(chFn);
        coco_string_delete(fn);
    }
    void XmlScannerData::CopyFramePart(const wchar_t * stop){
        wchar_t startCh = stop[0];
        int endOfStopString = coco_string_length(stop)-1;
        wchar_t ch = 0;

        fwscanf(fram, L"%lc", &ch); //fram.ReadByte();
        while (!feof(fram)) // ch != EOF
            if (ch == startCh) {
                int i = 0;
                do {
                    if (i == endOfStopString) return; // stop[0..i] found
                    fwscanf(fram, L"%lc", &ch); i++;
                } while (ch == stop[i]);
                // stop[0..i-1] found; continue with last read character
                wchar_t *subStop = coco_string_create(stop, 0, i);
                fwprintf(gen, L"%ls", subStop);
                coco_string_delete(subStop);
            } else {
                fwprintf(gen, L"%lc", ch);
                fwscanf(fram, L"%lc", &ch);
            }
        wprintf(L"CopyFramePart:[%ls]\n", stop);
        errors->Exception(L" -- incomplete or corrupt scanner frame file\n");
    }
    void XmlScannerData::WriteOptions(){
        fwprintf(gen, L"    enum Options {\n");
        map<const wchar_t*, int>::iterator iter;
        for (iter=optionsMap.begin(); iter!=optionsMap.end(); iter++)
            fwprintf(gen, L"        %ls,\n", iter->first);
        fwprintf(gen, L"    };\n");
        fwprintf(gen, L"    const int numOptions = %d;\n", optionsMap.size());
    }
    void XmlScannerData::WriteDeclarations(){
        Symbol *sym;
        fwprintf(gen, L"    int[] useKindVector = new int[] {\n");
        map<const wchar_t*, int>::iterator iter;
        for (iter=optionsMap.begin(); iter!=optionsMap.end(); ++iter){
            sym = tab->FindSym(iter->first);
            fwprintf(gen, L"        %d, // %ls\n", sym == NULL ? -1 : sym->n, iter->first);
        }
        fwprintf(gen, L"    };\n");
    }
    void XmlScannerData::WriteInitialization(){
        map<const wchar_t *, XmlLangDefinition*>::iterator iter;
        for (iter=XmlLangMap.begin(); iter!=XmlLangMap.end(); ++iter){
            fwprintf(gen, L"    curXLDef = new XmlLangDefinition();\n");
            iter->second->Write(gen);
            fwprintf(gen, L"    XmlLangMap.Add(\"%ls\", curXLDef);\n", iter->first);
        }
    }
    int XmlScannerData::GenNamespaceOpen(const wchar_t *nsName) {
        if (nsName == NULL || coco_string_length(nsName) == 0) {
            return 0;
        }
        int len = coco_string_length(nsName);
        int startPos = 0, endPos;
        int nrOfNs = 0;
        do {
            endPos = coco_string_indexof(nsName + startPos, COCO_CPP_NAMESPACE_SEPARATOR);
            if (endPos == -1) { endPos = len; }
            wchar_t *curNs = coco_string_create(nsName, startPos, endPos - startPos);
            fwprintf(gen, L"namespace %ls {\n", curNs);
            coco_string_delete(curNs);
            startPos = endPos + 1;
            ++nrOfNs;
        } while (startPos < len);
        return nrOfNs;
    }
    void XmlScannerData::GenNamespaceClose(int nrOfNs) {
        for (int i = 0; i < nrOfNs; ++i) {
            fwprintf(gen, L"}; // namespace\n");
        }
    }
    void XmlScannerData::WriteXmlScanner(){
        wchar_t *fr = coco_string_create_append(tab->srcDir, L"XmlScanner.frame");
        char *chFr = coco_string_create_char(fr);

        FILE* tmp;
        if ((tmp = fopen(chFr, "r")) == NULL) {
            if (coco_string_length(tab->frameDir) != 0) {
                delete [] fr;
                fr = coco_string_create(tab->frameDir);
                coco_string_merge(fr, L"/");
                coco_string_merge(fr, L"XmlScanner.frame");
            }
            coco_string_delete(chFr);
            chFr = coco_string_create_char(fr);
            if ((tmp = fopen(chFr, "r")) == NULL) {
                errors->Exception(L"-- Cannot find XmlScanner.frame\n");
            } else {
                fclose(tmp);
            }
        } else {
            fclose(tmp);
        }
        if ((fram = fopen(chFr, "r")) == NULL) {
            errors->Exception(L"-- Cannot open XmlScanner.frame.\n");
        }
        coco_string_delete(chFr);
        coco_string_delete(fr);

        /* to generate header file */
        {
            OpenGen(L"XmlScanner.h", true);
            CopyFramePart(L"/*---- Namespace Begin ----*/");
            int nrOfNs = GenNamespaceOpen(tab->nsName);

            CopyFramePart(L"/*---- Options ----*/"); WriteOptions();

            CopyFramePart(L"/*---- Declarations ----*/"); WriteDeclarations();

            CopyFramePart(L"/*---- Namespace End ----*/");

            GenNamespaceClose(nrOfNs);

            CopyFramePart(L"/*---- Implementation ----*/");
            fclose(gen);
        }

        /* to generate source file */
        {
            OpenGen(L"XmlScanner.cpp", true);
            CopyFramePart(L"/*---- Begin ----*/");
            fwprintf(gen, L"// THIS FILE IS GENERATED BY CocoXml AUTOMATICALLY, DO NOT EDIT IT MANUALLY.\n");
            CopyFramePart(L"/*---- Namespace Begin ----*/");
            int nrOfNs = GenNamespaceOpen(tab->nsName);

            CopyFramePart(L"/*---- Initialization ----*/"); WriteInitialization();

            CopyFramePart(L"/*---- Namespace End ----*/");

            CopyFramePart(L"/*---- $$$ ----*/");

            GenNamespaceClose(nrOfNs);
            fclose(gen);
        }
    }
} //namespace
<|endoftext|>