<commit_before>/*************************************************************************/
/*  register_types.cpp                                                   */
/*************************************************************************/
/*                       This file is part of:                           */
/*                           GODOT ENGINE                                */
/*                      https://godotengine.org                          */
/*************************************************************************/
/* Copyright (c) 2007-2017 Juan Linietsky, Ariel Manzur.                 */
/* Copyright (c) 2014-2017 Godot Engine contributors (cf. AUTHORS.md)    */
/*                                                                       */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the       */
/* "Software"), to deal in the Software without restriction, including   */
/* without limitation the rights to use, copy, modify, merge, publish,   */
/* distribute, sublicense, and/or sell copies of the Software, and to    */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions:                                             */
/*                                                                       */
/* The above copyright notice and this permission notice shall be        */
/* included in all copies or substantial portions of the Software.       */
/*                                                                       */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
/*************************************************************************/
#include "register_types.h"
#include "gdnative/gdnative.h"

#include "gdnative.h"

#include "io/resource_loader.h"
#include "io/resource_saver.h"

#include "arvr/register_types.h"
#include "nativescript/register_types.h"
#include "pluginscript/register_types.h"

#include "core/engine.h"
#include "core/os/os.h"
#include "core/project_settings.h"

#ifdef TOOLS_ENABLED
#include "editor/editor_node.h"
#include "gd_native_library_editor.h"
// Class used to discover singleton gdnative files

static void actual_discoverer_handler();

class GDNativeSingletonDiscover : public Object {
	// GDCLASS(GDNativeSingletonDiscover, Object)

	virtual String get_class() const {
		// okay, this is a really dirty hack.
		// We're overriding get_class so we can connect it to a signal
		// This works because get_class is a virtual method, so we don't
		// need to register a new class to ClassDB just for this one
		// little signal.

		actual_discoverer_handler();

		return "Object";
	}
};

static Set<String> get_gdnative_singletons(EditorFileSystemDirectory *p_dir) {

	Set<String> file_paths;

	// check children

	for (int i = 0; i < p_dir->get_file_count(); i++) {
		String file_name = p_dir->get_file(i);
		String file_type = p_dir->get_file_type(i);

		if (file_type != "GDNativeLibrary") {
			continue;
		}

		Ref<GDNativeLibrary> lib = ResourceLoader::load(p_dir->get_file_path(i));
		if (lib.is_valid() && lib->is_singleton()) {
			file_paths.insert(p_dir->get_file_path(i));
		}
	}

	// check subdirectories
	for (int i = 0; i < p_dir->get_subdir_count(); i++) {
		Set<String> paths = get_gdnative_singletons(p_dir->get_subdir(i));

		for (Set<String>::Element *E = paths.front(); E; E = E->next()) {
			file_paths.insert(E->get());
		}
	}

	return file_paths;
}

static void actual_discoverer_handler() {
	EditorFileSystemDirectory *dir = EditorFileSystem::get_singleton()->get_filesystem();

	Set<String> file_paths = get_gdnative_singletons(dir);

	Array files;
	files.resize(file_paths.size());
	int i = 0;
	for (Set<String>::Element *E = file_paths.front(); E; i++, E = E->next()) {
		files.set(i, E->get());
	}

	ProjectSettings::get_singleton()->set("gdnative/singletons", files);

	ProjectSettings::get_singleton()->save();
}

static GDNativeSingletonDiscover *discoverer = NULL;

class GDNativeExportPlugin : public EditorExportPlugin {

protected:
	virtual void _export_file(const String &p_path, const String &p_type, const Set<String> &p_features);
};

struct LibrarySymbol {
	char *name;
	bool is_required;
};

void GDNativeExportPlugin::_export_file(const String &p_path, const String &p_type, const Set<String> &p_features) {
	if (p_type != "GDNativeLibrary") {
		return;
	}

	Ref<GDNativeLibrary> lib = ResourceLoader::load(p_path);

	if (lib.is_null()) {
		return;
	}

	Ref<ConfigFile> config = lib->get_config_file();

	{

		List<String> entry_keys;
		config->get_section_keys("entry", &entry_keys);

		for (List<String>::Element *E = entry_keys.front(); E; E = E->next()) {
			String key = E->get();

			Vector<String> tags = key.split(".");

			bool skip = false;
			for (int i = 0; i < tags.size(); i++) {
				bool has_feature = p_features.has(tags[i]);

				if (!has_feature) {
					skip = true;
					break;
				}
			}

			if (skip) {
				continue;
			}

			String entry_lib_path = config->get_value("entry", key);
			add_shared_object(entry_lib_path, tags);
		}
	}

	{
		List<String> dependency_keys;
		config->get_section_keys("dependencies", &dependency_keys);

		for (List<String>::Element *E = dependency_keys.front(); E; E = E->next()) {
			String key = E->get();

			Vector<String> tags = key.split(".");

			bool skip = false;
			for (int i = 0; i < tags.size(); i++) {
				bool has_feature = p_features.has(tags[i]);

				if (!has_feature) {
					skip = true;
					break;
				}
			}

			if (skip) {
				continue;
			}

			Vector<String> dependency_paths = config->get_value("dependencies", key);
			for (int i = 0; i < dependency_paths.size(); i++) {
				add_shared_object(dependency_paths[i], tags);
			}
		}
	}

	if (p_features.has("iOS")) {
		// Register symbols in the "fake" dynamic lookup table, because dlsym does not work well on iOS.
		LibrarySymbol expected_symbols[] = {
			{ "gdnative_init", true },
			{ "gdnative_terminate", false },
			{ "nativescript_init", false },
			{ "nativescript_frame", false },
			{ "nativescript_thread_enter", false },
			{ "nativescript_thread_exit", false },
			{ "gdnative_singleton", false }
		};
		String declare_pattern = "extern \"C\" void $name(void)$weak;\n";
		String additional_code = "extern void register_dynamic_symbol(char *name, void *address);\n"
								 "extern void add_ios_init_callback(void (*cb)());\n";
		String linker_flags = "";
		for (int i = 0; i < sizeof(expected_symbols) / sizeof(expected_symbols[0]); ++i) {
			String full_name = lib->get_symbol_prefix() + expected_symbols[i].name;
			String code = declare_pattern.replace("$name", full_name);
			code = code.replace("$weak", expected_symbols[i].is_required ? "" : " __attribute__((weak))");
			additional_code += code;

			if (!expected_symbols[i].is_required) {
				if (linker_flags.length() > 0) {
					linker_flags += " ";
				}
				linker_flags += "-Wl,-U,_" + full_name;
			}
		}

		additional_code += String("void $prefixinit() {\n").replace("$prefix", lib->get_symbol_prefix());
		String register_pattern = "  if (&$name) register_dynamic_symbol((char *)\"$name\", (void *)$name);\n";
		for (int i = 0; i < sizeof(expected_symbols) / sizeof(expected_symbols[0]); ++i) {
			String full_name = lib->get_symbol_prefix() + expected_symbols[i].name;
			additional_code += register_pattern.replace("$name", full_name);
		}
		additional_code += "}\n";
		additional_code += String("struct $prefixstruct {$prefixstruct() {add_ios_init_callback($prefixinit);}};\n").replace("$prefix", lib->get_symbol_prefix());
		additional_code += String("$prefixstruct $prefixstruct_instance;\n").replace("$prefix", lib->get_symbol_prefix());

		add_ios_cpp_code(additional_code);
		add_ios_linker_flags(linker_flags);
	}
}

static void editor_init_callback() {

	GDNativeLibraryEditor *library_editor = memnew(GDNativeLibraryEditor);
	library_editor->set_name(TTR("GDNative"));
	ProjectSettingsEditor::get_singleton()->get_tabs()->add_child(library_editor);

	discoverer = memnew(GDNativeSingletonDiscover);
	EditorFileSystem::get_singleton()->connect("filesystem_changed", discoverer, "get_class");

	Ref<GDNativeExportPlugin> export_plugin;
	export_plugin.instance();

	EditorExport::get_singleton()->add_export_plugin(export_plugin);
}

#endif

static godot_variant cb_standard_varcall(void *p_procedure_handle, godot_array *p_args) {

	godot_gdnative_procedure_fn proc;
	proc = (godot_gdnative_procedure_fn)p_procedure_handle;

	return proc(p_args);
}

GDNativeCallRegistry *GDNativeCallRegistry::singleton;

Vector<Ref<GDNative> > singleton_gdnatives;

GDNativeLibraryResourceLoader *resource_loader_gdnlib = NULL;
GDNativeLibraryResourceSaver *resource_saver_gdnlib = NULL;

void register_gdnative_types() {

#ifdef TOOLS_ENABLED

	EditorNode::add_init_callback(editor_init_callback);
#endif

	ClassDB::register_class<GDNativeLibrary>();
	ClassDB::register_class<GDNative>();

	resource_loader_gdnlib = memnew(GDNativeLibraryResourceLoader);
	resource_saver_gdnlib = memnew(GDNativeLibraryResourceSaver);

	ResourceLoader::add_resource_format_loader(resource_loader_gdnlib);
	ResourceSaver::add_resource_format_saver(resource_saver_gdnlib);

	GDNativeCallRegistry::singleton = memnew(GDNativeCallRegistry);

	GDNativeCallRegistry::singleton->register_native_call_type("standard_varcall", cb_standard_varcall);

	register_arvr_types();
	register_nativescript_types();
	register_pluginscript_types();

	// run singletons

	Array singletons = Array();
	if (ProjectSettings::get_singleton()->has_setting("gdnative/singletons")) {
		singletons = ProjectSettings::get_singleton()->get("gdnative/singletons");
	}

	singleton_gdnatives.resize(singletons.size());

	for (int i = 0; i < singletons.size(); i++) {
		String path = singletons[i];

		Ref<GDNativeLibrary> lib = ResourceLoader::load(path);

		singleton_gdnatives[i].instance();
		singleton_gdnatives[i]->set_library(lib);

		if (!singleton_gdnatives[i]->initialize()) {
			// Can't initialize. Don't make a native_call then
			continue;
		}

		void *proc_ptr;
		Error err = singleton_gdnatives[i]->get_symbol(
				lib->get_symbol_prefix() + "gdnative_singleton",
				proc_ptr);

		if (err != OK) {
			ERR_PRINT((String("No godot_gdnative_singleton in \"" + singleton_gdnatives[i]->get_library()->get_current_library_path()) + "\" found").utf8().get_data());
		} else {
			((void (*)())proc_ptr)();
		}
	}
}

void unregister_gdnative_types() {

	for (int i = 0; i < singleton_gdnatives.size(); i++) {

		if (singleton_gdnatives[i].is_null()) {
			continue;
		}

		if (!singleton_gdnatives[i]->is_initialized()) {
			continue;
		}

		singleton_gdnatives[i]->terminate();
	}
	singleton_gdnatives.clear();

	unregister_pluginscript_types();
	unregister_nativescript_types();
	unregister_arvr_types();

	memdelete(GDNativeCallRegistry::singleton);

#ifdef TOOLS_ENABLED
	if (Engine::get_singleton()->is_editor_hint() && discoverer != NULL) {
		memdelete(discoverer);
	}
#endif

	memdelete(resource_loader_gdnlib);
	memdelete(resource_saver_gdnlib);

	// This is for printing out the sizes of the core types

	/*
	print_line(String("array:\t")     + itos(sizeof(Array)));
	print_line(String("basis:\t")     + itos(sizeof(Basis)));
	print_line(String("color:\t")     + itos(sizeof(Color)));
	print_line(String("dict:\t" )     + itos(sizeof(Dictionary)));
	print_line(String("node_path:\t") + itos(sizeof(NodePath)));
	print_line(String("plane:\t")     + itos(sizeof(Plane)));
	print_line(String("poolarray:\t") + itos(sizeof(PoolByteArray)));
	print_line(String("quat:\t")      + itos(sizeof(Quat)));
	print_line(String("rect2:\t")     + itos(sizeof(Rect2)));
	print_line(String("aabb:\t")     + itos(sizeof(AABB)));
	print_line(String("rid:\t")       + itos(sizeof(RID)));
	print_line(String("string:\t")    + itos(sizeof(String)));
	print_line(String("transform:\t") + itos(sizeof(Transform)));
	print_line(String("transfo2D:\t") + itos(sizeof(Transform2D)));
	print_line(String("variant:\t")   + itos(sizeof(Variant)));
	print_line(String("vector2:\t")   + itos(sizeof(Vector2)));
	print_line(String("vector3:\t")   + itos(sizeof(Vector3)));
	*/
}
<commit_msg>GDNative: Save singletons only if there's a change<commit_after>/*************************************************************************/
/*  register_types.cpp                                                   */
/*************************************************************************/
/*                       This file is part of:                           */
/*                           GODOT ENGINE                                */
/*                      https://godotengine.org                          */
/*************************************************************************/
/* Copyright (c) 2007-2017 Juan Linietsky, Ariel Manzur.                 */
/* Copyright (c) 2014-2017 Godot Engine contributors (cf. AUTHORS.md)    */
/*                                                                       */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the       */
/* "Software"), to deal in the Software without restriction, including   */
/* without limitation the rights to use, copy, modify, merge, publish,   */
/* distribute, sublicense, and/or sell copies of the Software, and to    */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions:                                             */
/*                                                                       */
/* The above copyright notice and this permission notice shall be        */
/* included in all copies or substantial portions of the Software.       */
/*                                                                       */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
/*************************************************************************/
#include "register_types.h"
#include "gdnative/gdnative.h"

#include "gdnative.h"

#include "io/resource_loader.h"
#include "io/resource_saver.h"

#include "arvr/register_types.h"
#include "nativescript/register_types.h"
#include "pluginscript/register_types.h"

#include "core/engine.h"
#include "core/os/os.h"
#include "core/project_settings.h"

#ifdef TOOLS_ENABLED
#include "editor/editor_node.h"
#include "gd_native_library_editor.h"
// Class used to discover singleton gdnative files

static void actual_discoverer_handler();

class GDNativeSingletonDiscover : public Object {
	// GDCLASS(GDNativeSingletonDiscover, Object)

	virtual String get_class() const {
		// okay, this is a really dirty hack.
		// We're overriding get_class so we can connect it to a signal
		// This works because get_class is a virtual method, so we don't
		// need to register a new class to ClassDB just for this one
		// little signal.

		actual_discoverer_handler();

		return "Object";
	}
};

static Set<String> get_gdnative_singletons(EditorFileSystemDirectory *p_dir) {

	Set<String> file_paths;

	// check children

	for (int i = 0; i < p_dir->get_file_count(); i++) {
		String file_name = p_dir->get_file(i);
		String file_type = p_dir->get_file_type(i);

		if (file_type != "GDNativeLibrary") {
			continue;
		}

		Ref<GDNativeLibrary> lib = ResourceLoader::load(p_dir->get_file_path(i));
		if (lib.is_valid() && lib->is_singleton()) {
			file_paths.insert(p_dir->get_file_path(i));
		}
	}

	// check subdirectories
	for (int i = 0; i < p_dir->get_subdir_count(); i++) {
		Set<String> paths = get_gdnative_singletons(p_dir->get_subdir(i));

		for (Set<String>::Element *E = paths.front(); E; E = E->next()) {
			file_paths.insert(E->get());
		}
	}

	return file_paths;
}

static void actual_discoverer_handler() {
	EditorFileSystemDirectory *dir = EditorFileSystem::get_singleton()->get_filesystem();

	Set<String> file_paths = get_gdnative_singletons(dir);

	bool changed = false;
	Array current_files = ProjectSettings::get_singleton()->get("gdnative/singletons");
	Array files;
	files.resize(file_paths.size());
	int i = 0;
	for (Set<String>::Element *E = file_paths.front(); E; i++, E = E->next()) {
		if (!current_files.has(E->get())) {
			changed = true;
		}
		files.set(i, E->get());
	}

	// Check for removed files
	if (!changed) {
		for (int i = 0; i < current_files.size(); i++) {
			if (!file_paths.has(current_files[i])) {
				changed = true;
				break;
			}
		}
	}

	if (changed) {

		ProjectSettings::get_singleton()->set("gdnative/singletons", files);

		ProjectSettings::get_singleton()->save();
	}
}

static GDNativeSingletonDiscover *discoverer = NULL;

class GDNativeExportPlugin : public EditorExportPlugin {

protected:
	virtual void _export_file(const String &p_path, const String &p_type, const Set<String> &p_features);
};

struct LibrarySymbol {
	char *name;
	bool is_required;
};

void GDNativeExportPlugin::_export_file(const String &p_path, const String &p_type, const Set<String> &p_features) {
	if (p_type != "GDNativeLibrary") {
		return;
	}

	Ref<GDNativeLibrary> lib = ResourceLoader::load(p_path);

	if (lib.is_null()) {
		return;
	}

	Ref<ConfigFile> config = lib->get_config_file();

	{

		List<String> entry_keys;
		config->get_section_keys("entry", &entry_keys);

		for (List<String>::Element *E = entry_keys.front(); E; E = E->next()) {
			String key = E->get();

			Vector<String> tags = key.split(".");

			bool skip = false;
			for (int i = 0; i < tags.size(); i++) {
				bool has_feature = p_features.has(tags[i]);

				if (!has_feature) {
					skip = true;
					break;
				}
			}

			if (skip) {
				continue;
			}

			String entry_lib_path = config->get_value("entry", key);
			add_shared_object(entry_lib_path, tags);
		}
	}

	{
		List<String> dependency_keys;
		config->get_section_keys("dependencies", &dependency_keys);

		for (List<String>::Element *E = dependency_keys.front(); E; E = E->next()) {
			String key = E->get();

			Vector<String> tags = key.split(".");

			bool skip = false;
			for (int i = 0; i < tags.size(); i++) {
				bool has_feature = p_features.has(tags[i]);

				if (!has_feature) {
					skip = true;
					break;
				}
			}

			if (skip) {
				continue;
			}

			Vector<String> dependency_paths = config->get_value("dependencies", key);
			for (int i = 0; i < dependency_paths.size(); i++) {
				add_shared_object(dependency_paths[i], tags);
			}
		}
	}

	if (p_features.has("iOS")) {
		// Register symbols in the "fake" dynamic lookup table, because dlsym does not work well on iOS.
		LibrarySymbol expected_symbols[] = {
			{ "gdnative_init", true },
			{ "gdnative_terminate", false },
			{ "nativescript_init", false },
			{ "nativescript_frame", false },
			{ "nativescript_thread_enter", false },
			{ "nativescript_thread_exit", false },
			{ "gdnative_singleton", false }
		};
		String declare_pattern = "extern \"C\" void $name(void)$weak;\n";
		String additional_code = "extern void register_dynamic_symbol(char *name, void *address);\n"
								 "extern void add_ios_init_callback(void (*cb)());\n";
		String linker_flags = "";
		for (int i = 0; i < sizeof(expected_symbols) / sizeof(expected_symbols[0]); ++i) {
			String full_name = lib->get_symbol_prefix() + expected_symbols[i].name;
			String code = declare_pattern.replace("$name", full_name);
			code = code.replace("$weak", expected_symbols[i].is_required ? "" : " __attribute__((weak))");
			additional_code += code;

			if (!expected_symbols[i].is_required) {
				if (linker_flags.length() > 0) {
					linker_flags += " ";
				}
				linker_flags += "-Wl,-U,_" + full_name;
			}
		}

		additional_code += String("void $prefixinit() {\n").replace("$prefix", lib->get_symbol_prefix());
		String register_pattern = "  if (&$name) register_dynamic_symbol((char *)\"$name\", (void *)$name);\n";
		for (int i = 0; i < sizeof(expected_symbols) / sizeof(expected_symbols[0]); ++i) {
			String full_name = lib->get_symbol_prefix() + expected_symbols[i].name;
			additional_code += register_pattern.replace("$name", full_name);
		}
		additional_code += "}\n";
		additional_code += String("struct $prefixstruct {$prefixstruct() {add_ios_init_callback($prefixinit);}};\n").replace("$prefix", lib->get_symbol_prefix());
		additional_code += String("$prefixstruct $prefixstruct_instance;\n").replace("$prefix", lib->get_symbol_prefix());

		add_ios_cpp_code(additional_code);
		add_ios_linker_flags(linker_flags);
	}
}

static void editor_init_callback() {

	GDNativeLibraryEditor *library_editor = memnew(GDNativeLibraryEditor);
	library_editor->set_name(TTR("GDNative"));
	ProjectSettingsEditor::get_singleton()->get_tabs()->add_child(library_editor);

	discoverer = memnew(GDNativeSingletonDiscover);
	EditorFileSystem::get_singleton()->connect("filesystem_changed", discoverer, "get_class");

	Ref<GDNativeExportPlugin> export_plugin;
	export_plugin.instance();

	EditorExport::get_singleton()->add_export_plugin(export_plugin);
}

#endif

static godot_variant cb_standard_varcall(void *p_procedure_handle, godot_array *p_args) {

	godot_gdnative_procedure_fn proc;
	proc = (godot_gdnative_procedure_fn)p_procedure_handle;

	return proc(p_args);
}

GDNativeCallRegistry *GDNativeCallRegistry::singleton;

Vector<Ref<GDNative> > singleton_gdnatives;

GDNativeLibraryResourceLoader *resource_loader_gdnlib = NULL;
GDNativeLibraryResourceSaver *resource_saver_gdnlib = NULL;

void register_gdnative_types() {

#ifdef TOOLS_ENABLED

	EditorNode::add_init_callback(editor_init_callback);
#endif

	ClassDB::register_class<GDNativeLibrary>();
	ClassDB::register_class<GDNative>();

	resource_loader_gdnlib = memnew(GDNativeLibraryResourceLoader);
	resource_saver_gdnlib = memnew(GDNativeLibraryResourceSaver);

	ResourceLoader::add_resource_format_loader(resource_loader_gdnlib);
	ResourceSaver::add_resource_format_saver(resource_saver_gdnlib);

	GDNativeCallRegistry::singleton = memnew(GDNativeCallRegistry);

	GDNativeCallRegistry::singleton->register_native_call_type("standard_varcall", cb_standard_varcall);

	register_arvr_types();
	register_nativescript_types();
	register_pluginscript_types();

	// run singletons

	Array singletons = Array();
	if (ProjectSettings::get_singleton()->has_setting("gdnative/singletons")) {
		singletons = ProjectSettings::get_singleton()->get("gdnative/singletons");
	}

	singleton_gdnatives.resize(singletons.size());

	for (int i = 0; i < singletons.size(); i++) {
		String path = singletons[i];

		Ref<GDNativeLibrary> lib = ResourceLoader::load(path);

		singleton_gdnatives[i].instance();
		singleton_gdnatives[i]->set_library(lib);

		if (!singleton_gdnatives[i]->initialize()) {
			// Can't initialize. Don't make a native_call then
			continue;
		}

		void *proc_ptr;
		Error err = singleton_gdnatives[i]->get_symbol(
				lib->get_symbol_prefix() + "gdnative_singleton",
				proc_ptr);

		if (err != OK) {
			ERR_PRINT((String("No godot_gdnative_singleton in \"" + singleton_gdnatives[i]->get_library()->get_current_library_path()) + "\" found").utf8().get_data());
		} else {
			((void (*)())proc_ptr)();
		}
	}
}

void unregister_gdnative_types() {

	for (int i = 0; i < singleton_gdnatives.size(); i++) {

		if (singleton_gdnatives[i].is_null()) {
			continue;
		}

		if (!singleton_gdnatives[i]->is_initialized()) {
			continue;
		}

		singleton_gdnatives[i]->terminate();
	}
	singleton_gdnatives.clear();

	unregister_pluginscript_types();
	unregister_nativescript_types();
	unregister_arvr_types();

	memdelete(GDNativeCallRegistry::singleton);

#ifdef TOOLS_ENABLED
	if (Engine::get_singleton()->is_editor_hint() && discoverer != NULL) {
		memdelete(discoverer);
	}
#endif

	memdelete(resource_loader_gdnlib);
	memdelete(resource_saver_gdnlib);

	// This is for printing out the sizes of the core types

	/*
	print_line(String("array:\t")     + itos(sizeof(Array)));
	print_line(String("basis:\t")     + itos(sizeof(Basis)));
	print_line(String("color:\t")     + itos(sizeof(Color)));
	print_line(String("dict:\t" )     + itos(sizeof(Dictionary)));
	print_line(String("node_path:\t") + itos(sizeof(NodePath)));
	print_line(String("plane:\t")     + itos(sizeof(Plane)));
	print_line(String("poolarray:\t") + itos(sizeof(PoolByteArray)));
	print_line(String("quat:\t")      + itos(sizeof(Quat)));
	print_line(String("rect2:\t")     + itos(sizeof(Rect2)));
	print_line(String("aabb:\t")     + itos(sizeof(AABB)));
	print_line(String("rid:\t")       + itos(sizeof(RID)));
	print_line(String("string:\t")    + itos(sizeof(String)));
	print_line(String("transform:\t") + itos(sizeof(Transform)));
	print_line(String("transfo2D:\t") + itos(sizeof(Transform2D)));
	print_line(String("variant:\t")   + itos(sizeof(Variant)));
	print_line(String("vector2:\t")   + itos(sizeof(Vector2)));
	print_line(String("vector3:\t")   + itos(sizeof(Vector3)));
	*/
}
<|endoftext|>


<commit_before>#include "partition/partitioner.hpp"
#include "partition/annotated_partition.hpp"
#include "partition/bisection_graph.hpp"
#include "partition/compressed_node_based_graph_reader.hpp"
#include "partition/edge_based_graph_reader.hpp"
#include "partition/node_based_graph_to_edge_based_graph_mapping_reader.hpp"
#include "partition/recursive_bisection.hpp"

#include "util/coordinate.hpp"
#include "util/geojson_debug_logger.hpp"
#include "util/geojson_debug_policies.hpp"
#include "util/integer_range.hpp"
#include "util/json_container.hpp"
#include "util/log.hpp"

#include <algorithm>
#include <iterator>
#include <vector>

#include <boost/assert.hpp>

#include "util/geojson_debug_logger.hpp"
#include "util/geojson_debug_policies.hpp"
#include "util/json_container.hpp"
#include "util/timing_util.hpp"

namespace osrm
{
namespace partition
{

void LogStatistics(const std::string &filename, std::vector<std::uint32_t> bisection_ids)
{
    auto compressed_node_based_graph = LoadCompressedNodeBasedGraph(filename);

    util::Log() << "Loaded compressed node based graph: "
                << compressed_node_based_graph.edges.size() << " edges, "
                << compressed_node_based_graph.coordinates.size() << " nodes";

    groupEdgesBySource(begin(compressed_node_based_graph.edges),
                       end(compressed_node_based_graph.edges));

    auto graph =
        makeBisectionGraph(compressed_node_based_graph.coordinates,
                           adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));

    TIMER_START(annotation);
    AnnotatedPartition partition(graph, bisection_ids);
    TIMER_STOP(annotation);
    std::cout << "Annotation took " << TIMER_SEC(annotation) << " seconds" << std::endl;
}

void LogGeojson(const std::string &filename, const std::vector<std::uint32_t> &bisection_ids)
{
    // reload graph, since we destroyed the old one
    auto compressed_node_based_graph = LoadCompressedNodeBasedGraph(filename);

    util::Log() << "Loaded compressed node based graph: "
                << compressed_node_based_graph.edges.size() << " edges, "
                << compressed_node_based_graph.coordinates.size() << " nodes";

    groupEdgesBySource(begin(compressed_node_based_graph.edges),
                       end(compressed_node_based_graph.edges));

    auto graph =
        makeBisectionGraph(compressed_node_based_graph.coordinates,
                           adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));

    const auto get_level = [](const std::uint32_t lhs, const std::uint32_t rhs) {
        auto xored = lhs ^ rhs;
        std::uint32_t level = log(xored) / log(2.0);
        return level;
    };

    std::vector<std::vector<util::Coordinate>> border_vertices(33);

    for (NodeID nid = 0; nid < graph.NumberOfNodes(); ++nid)
    {
        const auto source_id = bisection_ids[nid];
        for (const auto &edge : graph.Edges(nid))
        {
            const auto target_id = bisection_ids[edge.target];
            if (source_id != target_id)
            {
                auto level = get_level(source_id, target_id);
                border_vertices[level].push_back(graph.Node(nid).coordinate);
                border_vertices[level].push_back(graph.Node(edge.target).coordinate);
            }
        }
    }

    util::ScopedGeojsonLoggerGuard<util::CoordinateVectorToMultiPoint> guard(
        "border_vertices.geojson");
    std::size_t level = 0;
    for (auto &bv : border_vertices)
    {
        if (!bv.empty())
        {
            std::sort(bv.begin(), bv.end(), [](const auto lhs, const auto rhs) {
                return std::tie(lhs.lon, lhs.lat) < std::tie(rhs.lon, rhs.lat);
            });
            bv.erase(std::unique(bv.begin(), bv.end()), bv.end());

            util::json::Object jslevel;
            jslevel.values["level"] = util::json::Number(level++);
            guard.Write(bv, jslevel);
        }
    }
}

int Partitioner::Run(const PartitionConfig &config)
{
    auto compressed_node_based_graph =
        LoadCompressedNodeBasedGraph(config.compressed_node_based_graph_path.string());

    util::Log() << "Loaded compressed node based graph: "
                << compressed_node_based_graph.edges.size() << " edges, "
                << compressed_node_based_graph.coordinates.size() << " nodes";

    groupEdgesBySource(begin(compressed_node_based_graph.edges),
                       end(compressed_node_based_graph.edges));

    auto graph =
        makeBisectionGraph(compressed_node_based_graph.coordinates,
                           adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));

    util::Log() << " running partition: " << config.maximum_cell_size << " " << config.balance
                << " " << config.boundary_factor << " " << config.num_optimizing_cuts << " "
                << config.small_component_size
                << " # max_cell_size balance boundary cuts small_component_size";
    RecursiveBisection recursive_bisection(graph,
                                           config.maximum_cell_size,
                                           config.balance,
                                           config.boundary_factor,
                                           config.num_optimizing_cuts,
                                           config.small_component_size);

    LogStatistics(config.compressed_node_based_graph_path.string(),
                  recursive_bisection.BisectionIDs());

    // Up until now we worked on the compressed node based graph.
    // But what we actually need is a partition for the edge based graph to work on.
    // The following loads a mapping from node based graph to edge based graph.
    // Then loads the edge based graph tanslates the partition and modifies it.
    // For details see #3205

    auto mapping = LoadNodeBasedGraphToEdgeBasedGraphMapping(config.nbg_ebg_mapping_path.string());
    util::Log() << "Loaded node based graph to edge based graph mapping";

    auto edge_based_graph = LoadEdgeBasedGraph(config.edge_based_graph_path.string());
    util::Log() << "Loaded edge based graph for mapping partition ids: "
                << edge_based_graph->GetNumberOfEdges() << " edges, "
                << edge_based_graph->GetNumberOfNodes() << " nodes";

    // TODO: put translation into own function / file

    const auto &partition_ids = recursive_bisection.BisectionIDs();

    std::vector<NodeID> edge_based_border_nodes;

    // Extract edge based border nodes, based on node based partition and mapping.
    for (const auto node_id : util::irange(0u, edge_based_graph->GetNumberOfNodes()))
    {
        const auto node_based_nodes = mapping.Lookup(node_id);

        const auto u = node_based_nodes.u;
        const auto v = node_based_nodes.v;

        if (partition_ids[u] == partition_ids[v])
        {
            // Can use partition_ids[u/v] as partition for edge based graph `node_id`
        }
        else
        {
            // Border nodes u,v - need to be resolved.
            edge_based_border_nodes.push_back(node_id);
        }
    }

    util::Log() << "Fixing " << edge_based_border_nodes.size() << " edge based graph border nodes";

    std::vector<std::pair<NodeID, EdgeBasedGraphEdgeData>> incoming_edges;

    for (const auto border_node : edge_based_border_nodes)
    {
        for (const auto edge : edge_based_graph->GetAdjacentEdgeRange(border_node))
        {
            const auto &data = edge_based_graph->GetEdgeData(edge);

            if (data.backward)
            {
                incoming_edges.emplace_back(edge_based_graph->GetTarget(edge), data);
                edge_based_graph->DeleteEdge(border_node, edge);
            }
        }

        const auto artificial = edge_based_graph->InsertNode();

        EdgeBasedGraphEdgeData dummy{SPECIAL_EDGEID, /*is_boundary_arc=*/true, 0, 0, false, false};

        for (const auto edge : incoming_edges)
        {
            edge_based_graph->InsertEdge(edge.first, artificial, edge.second);
        }

        incoming_edges.clear();
    }

    return 0;
}

} // namespace partition
} // namespace osrm
<commit_msg>Sigh. Just randomly pick a side for border nodes for now..<commit_after>#include "partition/partitioner.hpp"
#include "partition/annotated_partition.hpp"
#include "partition/bisection_graph.hpp"
#include "partition/compressed_node_based_graph_reader.hpp"
#include "partition/edge_based_graph_reader.hpp"
#include "partition/node_based_graph_to_edge_based_graph_mapping_reader.hpp"
#include "partition/recursive_bisection.hpp"

#include "util/coordinate.hpp"
#include "util/geojson_debug_logger.hpp"
#include "util/geojson_debug_policies.hpp"
#include "util/integer_range.hpp"
#include "util/json_container.hpp"
#include "util/log.hpp"

#include <algorithm>
#include <iterator>
#include <vector>

#include <boost/assert.hpp>

#include "util/geojson_debug_logger.hpp"
#include "util/geojson_debug_policies.hpp"
#include "util/json_container.hpp"
#include "util/timing_util.hpp"

namespace osrm
{
namespace partition
{

void LogStatistics(const std::string &filename, std::vector<std::uint32_t> bisection_ids)
{
    auto compressed_node_based_graph = LoadCompressedNodeBasedGraph(filename);

    util::Log() << "Loaded compressed node based graph: "
                << compressed_node_based_graph.edges.size() << " edges, "
                << compressed_node_based_graph.coordinates.size() << " nodes";

    groupEdgesBySource(begin(compressed_node_based_graph.edges),
                       end(compressed_node_based_graph.edges));

    auto graph =
        makeBisectionGraph(compressed_node_based_graph.coordinates,
                           adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));

    TIMER_START(annotation);
    AnnotatedPartition partition(graph, bisection_ids);
    TIMER_STOP(annotation);
    std::cout << "Annotation took " << TIMER_SEC(annotation) << " seconds" << std::endl;
}

void LogGeojson(const std::string &filename, const std::vector<std::uint32_t> &bisection_ids)
{
    // reload graph, since we destroyed the old one
    auto compressed_node_based_graph = LoadCompressedNodeBasedGraph(filename);

    util::Log() << "Loaded compressed node based graph: "
                << compressed_node_based_graph.edges.size() << " edges, "
                << compressed_node_based_graph.coordinates.size() << " nodes";

    groupEdgesBySource(begin(compressed_node_based_graph.edges),
                       end(compressed_node_based_graph.edges));

    auto graph =
        makeBisectionGraph(compressed_node_based_graph.coordinates,
                           adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));

    const auto get_level = [](const std::uint32_t lhs, const std::uint32_t rhs) {
        auto xored = lhs ^ rhs;
        std::uint32_t level = log(xored) / log(2.0);
        return level;
    };

    std::vector<std::vector<util::Coordinate>> border_vertices(33);

    for (NodeID nid = 0; nid < graph.NumberOfNodes(); ++nid)
    {
        const auto source_id = bisection_ids[nid];
        for (const auto &edge : graph.Edges(nid))
        {
            const auto target_id = bisection_ids[edge.target];
            if (source_id != target_id)
            {
                auto level = get_level(source_id, target_id);
                border_vertices[level].push_back(graph.Node(nid).coordinate);
                border_vertices[level].push_back(graph.Node(edge.target).coordinate);
            }
        }
    }

    util::ScopedGeojsonLoggerGuard<util::CoordinateVectorToMultiPoint> guard(
        "border_vertices.geojson");
    std::size_t level = 0;
    for (auto &bv : border_vertices)
    {
        if (!bv.empty())
        {
            std::sort(bv.begin(), bv.end(), [](const auto lhs, const auto rhs) {
                return std::tie(lhs.lon, lhs.lat) < std::tie(rhs.lon, rhs.lat);
            });
            bv.erase(std::unique(bv.begin(), bv.end()), bv.end());

            util::json::Object jslevel;
            jslevel.values["level"] = util::json::Number(level++);
            guard.Write(bv, jslevel);
        }
    }
}

int Partitioner::Run(const PartitionConfig &config)
{
    auto compressed_node_based_graph =
        LoadCompressedNodeBasedGraph(config.compressed_node_based_graph_path.string());

    util::Log() << "Loaded compressed node based graph: "
                << compressed_node_based_graph.edges.size() << " edges, "
                << compressed_node_based_graph.coordinates.size() << " nodes";

    groupEdgesBySource(begin(compressed_node_based_graph.edges),
                       end(compressed_node_based_graph.edges));

    auto graph =
        makeBisectionGraph(compressed_node_based_graph.coordinates,
                           adaptToBisectionEdge(std::move(compressed_node_based_graph.edges)));

    util::Log() << " running partition: " << config.maximum_cell_size << " " << config.balance
                << " " << config.boundary_factor << " " << config.num_optimizing_cuts << " "
                << config.small_component_size
                << " # max_cell_size balance boundary cuts small_component_size";
    RecursiveBisection recursive_bisection(graph,
                                           config.maximum_cell_size,
                                           config.balance,
                                           config.boundary_factor,
                                           config.num_optimizing_cuts,
                                           config.small_component_size);

    LogStatistics(config.compressed_node_based_graph_path.string(),
                  recursive_bisection.BisectionIDs());

    // Up until now we worked on the compressed node based graph.
    // But what we actually need is a partition for the edge based graph to work on.
    // The following loads a mapping from node based graph to edge based graph.
    // Then loads the edge based graph tanslates the partition and modifies it.
    // For details see #3205

    auto mapping = LoadNodeBasedGraphToEdgeBasedGraphMapping(config.nbg_ebg_mapping_path.string());
    util::Log() << "Loaded node based graph to edge based graph mapping";

    auto edge_based_graph = LoadEdgeBasedGraph(config.edge_based_graph_path.string());
    util::Log() << "Loaded edge based graph for mapping partition ids: "
                << edge_based_graph->GetNumberOfEdges() << " edges, "
                << edge_based_graph->GetNumberOfNodes() << " nodes";

    // TODO: node based graph to edge based graph partition id mapping should be done split off.

    // Partition ids, keyed by node based graph nodes
    const auto &node_based_partition_ids = recursive_bisection.BisectionIDs();

    // Partition ids, keyed by edge based graph nodes
    std::vector<NodeID> edge_based_partition_ids(edge_based_graph->GetNumberOfNodes());

    // Extract edge based border nodes, based on node based partition and mapping.
    for (const auto node : util::irange(0u, edge_based_graph->GetNumberOfNodes()))
    {
        const auto node_based_nodes = mapping.Lookup(node);

        const auto u = node_based_nodes.u;
        const auto v = node_based_nodes.v;

        if (node_based_partition_ids[u] == node_based_partition_ids[v])
        {
            // Can use partition_ids[u/v] as partition for edge based graph `node_id`
            edge_based_partition_ids[node] = node_based_partition_ids[u];
        }
        else
        {
            // Border nodes u,v - need to be resolved.
            // FIXME: just pick one side for now. See #3205.
            edge_based_partition_ids[node] = node_based_partition_ids[u];
        }
    }

    return 0;
}

} // namespace partition
} // namespace osrm
<|endoftext|>


<commit_before>// This header intentionally has no include guards.

// Preparing ConfigIndex, etc for remapclasses_initialize_vector_loader.
// And, it's a valuable opportunity to traverse checkbox.xml. (it takes very long time!)
// We handle other data at this time.
//
// Targets:
//   - ConfigIndex (for <only>,<not> filter)
//   - KeyCode::VK_CONFIG_*
//   - essential_configurations
//   - preferences_node_tree
//
template <class preferences_node_tree_t>
class remapclasses_initialize_vector_prepare_loader final {
public:
  remapclasses_initialize_vector_prepare_loader(const xml_compiler& xml_compiler,
                                                symbol_map& symbol_map,
                                                std::vector<std::shared_ptr<essential_configuration>>& essential_configurations,
                                                preferences_node_tree_t* preferences_node_tree) : xml_compiler_(xml_compiler),
                                                                                symbol_map_(symbol_map),
                                                                                essential_configurations_(essential_configurations),
                                                                                preferences_node_tree_(preferences_node_tree),
                                                                                root_preferences_node_tree_(preferences_node_tree) {}

  void traverse(const extracted_ptree& pt) {
    for (const auto& it : pt) {
      // Hack for speed improvement.
      // We can stop traversing when we met <autogen>.
      if (it.get_tag_name() == "autogen") {
        continue;
      }

      // traverse
      {
        if (it.get_tag_name() != "item") {
          if (!it.children_empty()) {
            traverse(it.children_extracted_ptree());
          }
        } else {
          // preferences_node_tree
          assert(preferences_node_tree_);
          std::shared_ptr<preferences_node_tree_t> ptr(new preferences_node_tree_t(preferences_node_tree_->get_node()));
          assert(ptr);

          for (const auto& child : it.children_extracted_ptree()) {
            ptr->handle_item_child(child);

            if (child.get_tag_name() == "identifier") {
              auto raw_identifier = boost::trim_copy(child.get_data());
              if (xml_compiler_.valid_identifier_(raw_identifier, it.get_tag_name())) {
                auto identifier = raw_identifier;
                normalize_identifier_(identifier);

                // ----------------------------------------
                auto attr_essential = child.get_optional("<xmlattr>.essential");
                if (attr_essential) {
                  essential_configurations_.push_back(std::shared_ptr<essential_configuration>(new essential_configuration(ptr->get_node())));
                  // Do not treat essentials anymore.
                  continue;
                }

                // ----------------------------------------
                auto attr_vk_config = child.get_optional("<xmlattr>.vk_config");
                if (attr_vk_config) {
                  const char* names[] = {
                      "VK_CONFIG_TOGGLE_",
                      "VK_CONFIG_FORCE_ON_",
                      "VK_CONFIG_FORCE_OFF_",
                      "VK_CONFIG_SYNC_KEYDOWNUP_",
                  };
                  for (const auto& n : names) {
                    symbol_map_.add("KeyCode", std::string(n) + identifier);
                  }
                }

                // ----------------------------------------
                if (boost::starts_with(identifier, "notsave_")) {
                  identifiers_notsave_.push_back(identifier);
                } else {
                  identifiers_except_notsave_.push_back(identifier);
                }
              }
            }
          }

          auto saved_preferences_node_tree = preferences_node_tree_;
          preferences_node_tree_ = ptr.get();
          {
            if (!it.children_empty()) {
              traverse(it.children_extracted_ptree());
            }
          }
          preferences_node_tree_ = saved_preferences_node_tree;

          auto attr_hidden = it.get_optional("<xmlattr>.hidden");
          if (!attr_hidden) {
            preferences_node_tree_->push_back(ptr);
          }
        }
      }
    }
  }

  // call "fixup" at traversing each file.
  void fixup(void) {
    preferences_node_tree_ = root_preferences_node_tree_;
  }

  // call "cleanup" at finished traversing all files.
  void cleanup(void) {
    fixup();

    // "notsave" has higher priority.
    for (const auto& it : identifiers_notsave_) {
      symbol_map_.add("ConfigIndex", it);
    }
    identifiers_notsave_.clear();

    for (const auto& it : identifiers_except_notsave_) {
      symbol_map_.add("ConfigIndex", it);
    }
    identifiers_except_notsave_.clear();
  }

private:
  const xml_compiler& xml_compiler_;
  symbol_map& symbol_map_;
  std::vector<std::shared_ptr<essential_configuration>>& essential_configurations_;
  preferences_node_tree_t* preferences_node_tree_;
  preferences_node_tree_t* const root_preferences_node_tree_;

  std::vector<std::string> identifiers_notsave_;
  std::vector<std::string> identifiers_except_notsave_;
};
<commit_msg>add high_priority attribute<commit_after>// This header intentionally has no include guards.

// Preparing ConfigIndex, etc for remapclasses_initialize_vector_loader.
// And, it's a valuable opportunity to traverse checkbox.xml. (it takes very long time!)
// We handle other data at this time.
//
// Targets:
//   - ConfigIndex (for <only>,<not> filter)
//   - KeyCode::VK_CONFIG_*
//   - essential_configurations
//   - preferences_node_tree
//
template <class preferences_node_tree_t>
class remapclasses_initialize_vector_prepare_loader final {
public:
  remapclasses_initialize_vector_prepare_loader(const xml_compiler& xml_compiler,
                                                symbol_map& symbol_map,
                                                std::vector<std::shared_ptr<essential_configuration>>& essential_configurations,
                                                preferences_node_tree_t* preferences_node_tree) : xml_compiler_(xml_compiler),
                                                                                symbol_map_(symbol_map),
                                                                                essential_configurations_(essential_configurations),
                                                                                preferences_node_tree_(preferences_node_tree),
                                                                                root_preferences_node_tree_(preferences_node_tree) {}

  void traverse(const extracted_ptree& pt) {
    for (const auto& it : pt) {
      // Hack for speed improvement.
      // We can stop traversing when we met <autogen>.
      if (it.get_tag_name() == "autogen") {
        continue;
      }

      // traverse
      {
        if (it.get_tag_name() != "item") {
          if (!it.children_empty()) {
            traverse(it.children_extracted_ptree());
          }
        } else {
          // preferences_node_tree
          assert(preferences_node_tree_);
          std::shared_ptr<preferences_node_tree_t> ptr(new preferences_node_tree_t(preferences_node_tree_->get_node()));
          assert(ptr);

          for (const auto& child : it.children_extracted_ptree()) {
            ptr->handle_item_child(child);

            if (child.get_tag_name() == "identifier") {
              auto raw_identifier = boost::trim_copy(child.get_data());
              if (xml_compiler_.valid_identifier_(raw_identifier, it.get_tag_name())) {
                auto identifier = raw_identifier;
                normalize_identifier_(identifier);

                // ----------------------------------------
                auto attr_essential = child.get_optional("<xmlattr>.essential");
                if (attr_essential) {
                  essential_configurations_.push_back(std::shared_ptr<essential_configuration>(new essential_configuration(ptr->get_node())));
                  // Do not treat essentials anymore.
                  continue;
                }

                // ----------------------------------------
                auto attr_vk_config = child.get_optional("<xmlattr>.vk_config");
                if (attr_vk_config) {
                  const char* names[] = {
                      "VK_CONFIG_TOGGLE_",
                      "VK_CONFIG_FORCE_ON_",
                      "VK_CONFIG_FORCE_OFF_",
                      "VK_CONFIG_SYNC_KEYDOWNUP_",
                  };
                  for (const auto& n : names) {
                    symbol_map_.add("KeyCode", std::string(n) + identifier);
                  }
                }

                // ----------------------------------------
                auto attr_high_priority = child.get_optional("<xmlattr>.high_priority");
                if (boost::starts_with(identifier, "notsave_") ||
                    attr_high_priority) {
                  identifiers_high_priority_.push_back(identifier);
                } else {
                  identifiers_normal_priority_.push_back(identifier);
                }
              }
            }
          }

          auto saved_preferences_node_tree = preferences_node_tree_;
          preferences_node_tree_ = ptr.get();
          {
            if (!it.children_empty()) {
              traverse(it.children_extracted_ptree());
            }
          }
          preferences_node_tree_ = saved_preferences_node_tree;

          auto attr_hidden = it.get_optional("<xmlattr>.hidden");
          if (!attr_hidden) {
            preferences_node_tree_->push_back(ptr);
          }
        }
      }
    }
  }

  // call "fixup" at traversing each file.
  void fixup(void) {
    preferences_node_tree_ = root_preferences_node_tree_;
  }

  // call "cleanup" at finished traversing all files.
  void cleanup(void) {
    fixup();

    // "notsave" has higher priority.
    for (const auto& it : identifiers_high_priority_) {
      symbol_map_.add("ConfigIndex", it);
    }
    identifiers_high_priority_.clear();

    for (const auto& it : identifiers_normal_priority_) {
      symbol_map_.add("ConfigIndex", it);
    }
    identifiers_normal_priority_.clear();
  }

private:
  const xml_compiler& xml_compiler_;
  symbol_map& symbol_map_;
  std::vector<std::shared_ptr<essential_configuration>>& essential_configurations_;
  preferences_node_tree_t* preferences_node_tree_;
  preferences_node_tree_t* const root_preferences_node_tree_;

  std::vector<std::string> identifiers_high_priority_;
  std::vector<std::string> identifiers_normal_priority_;
};
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
#include <codecvt>
//#include <cstddef>
#include <locale>
#include <clocale>
#include <string>
#include <iomanip>
#include <ctime>

#include "cp-neural.h"

using std::wcout; using std::wstring;
using std::cerr; using std::vector;

class Text {
private:
    bool isinit=false;
    bool readDataFile(string inputfile) {
        std::wifstream txtfile(inputfile, std::ios::binary);
        if (!txtfile.is_open()) {
            return false;
        }
        txtfile.imbue(std::locale("en_US.UTF-8"));
        wstring ttext((std::istreambuf_iterator<wchar_t>(txtfile)),
                     std::istreambuf_iterator<wchar_t>());
        txtfile.close();
        text=ttext;
        filename=inputfile;
        return true;
    }
public:
    wstring text;
    string filename;
    std::map<wchar_t,int> freq;
    std::map<wchar_t,int> w2v;
    std::map<int,wchar_t> v2w;
    Text(string filename) {
        if (readDataFile(filename)) {
            for (auto wc : text) {
                freq[wc]++;
            }
            int it=0;
            for (auto wc : freq) {
                w2v[wc.first]=it;
                v2w[it]=wc.first;
                ++it;
            }
            isinit=true;
            cerr << "Freq:" << freq.size() << ", w2v:" << w2v.size() << ", v2w:" << v2w.size() << endl;
        }
    }
    ~Text() {

    }
    bool isInit() {
        return isinit;
    }
    int vocsize() {
        if (!isinit) return 0;
        return (int)v2w.size();
    }

    wstring sample(int len) {
        int p=std::rand()%(text.size()-len);
        return text.substr(p,len);
    }
};

void currentDateTime(wstring& timestr) {
    time_t     now = time(0);
    struct tm  tstruct;
    char       buf[80];
    tstruct = *localtime(&now);
    // Visit http://en.cppreference.com/w/cpp/chrono/c/strftime
    // for more information about date/time format
    strftime(buf, sizeof(buf), "%Y-%m-%d.%X", &tstruct);
    timestr=std::wstring_convert<std::codecvt_utf8<wchar_t>>().from_bytes(buf);
}

int main(int argc, char *argv[]) {
#ifndef __APPLE__
    std::setlocale(LC_ALL, "");
#else
    setlocale(LC_ALL, "");
    std::wcout.imbue(std::locale("en_US.UTF-8"));
#endif
    wcout << L"Rnn-Readär" << std::endl;

    bool allOk=true;
    if (argc!=2) {
        cerr << L"rnnreader <path-to-text-file>" << endl;
        exit(-1);
    }
    Text txt(argv[1]);
    if (!txt.isInit()) {
        cerr << "Cannot initialize Text object from inputfile: " << argv[1] << endl;
        exit(-1);
    }

    wcout << L"Text size: " << txt.text.size() << endl;

    wcout << L"Size of vocabulary:" << txt.freq.size() << std::endl;
/*    for (auto f : txt.freq) {
        int c=(int)f.first;
        wstring wc(1,f.first);
        wcout << wc << L"|" <<  wchar_t(f.first) << L"(0x" << std::hex << c << L")" ": " << std::dec <<  f.second << endl;
    }
*/
    Color::Modifier red(Color::FG_RED);
    Color::Modifier green(Color::FG_GREEN);
    Color::Modifier def(Color::FG_DEFAULT);

    int T=80;

    int N=(int)txt.text.size() / (T+1);
    cerr << N << " Max datasets" << endl;
    MatrixN Xr(N,T);
    MatrixN yr(N,T);

    wstring chunk,chunky;
    int n=0;
    for (int i=0; i<N; i++) {
        wstring smp = txt.sample(T+1);
        chunk=smp.substr(0,T);
        chunky=smp.substr(1,T);
        for (int t=0; t<T; t++) {
            Xr(i,t)=txt.w2v[chunk[t]];
            yr(i,t)=txt.w2v[chunky[t]];
        }
        ++n;
    }

    int maxN = 100000;
    if (n>maxN) n=maxN;
    int n1=n*0.9;
    int dn=(n-n1)/2;
    if (n1+2*dn > n) {
        cerr << "Math doesn't work out." << endl;
    }

    cerr << n1 << " datasets, " << dn << " test-sets, " << dn << " validation-sets" << endl;

/*    MatrixN X(n1,T);
    MatrixN y(n1,T);
    MatrixN Xv(dn,T);
    MatrixN yv(dn,T);
    MatrixN Xt(dn,T);
    MatrixN yt(dn,T);
*/
    MatrixN X=Xr.block(0,0,n1,T);
    MatrixN y=yr.block(0,0,n1,T);
    MatrixN Xv=Xr.block(n1,0,dn,T);
    MatrixN yv=yr.block(n1,0,dn,T);
    MatrixN Xt=Xr.block(n1+dn,0,dn,T);
    MatrixN yt=yr.block(n1+dn,0,dn,T);

    unsigned int ctm=(unsigned int)std::time(0);
    std::srand(ctm);

    cpInitCompute("Rnnreader");
    registerLayers();

    LayerBlock lb(R"({"name":"rnnreader","init":"orthonormal","initfactor":0.03})"_json);
    int VS=txt.vocsize();
    int H=512;

    int BS=128;
    float clip=5.0;

    //int D=64;
    // CpParams cp0;
    // cp0.setPar("inputShape",vector<int>{T});
    // cp0.setPar("V",VS);
    // cp0.setPar("D",D);
    // cp0.setPar("clip",clip);
    // cp0.setPar("init",(string)"orthonormal");
    // lb.addLayer("WordEmbedding","WE0",cp0,{"input"});

    string rnntype="LSTM"; // or "RNN"
    cerr << "RNN-type: " << rnntype << endl;

    json j0;
    string oName{"OH0"};
    j0["inputShape"]=vector<int>{T};
    j0["V"]=VS;
    lb.addLayer("OneHot",oName,j0,{"input"});

    string nName;
    json j1;
    j1["inputShape"]=vector<int>{VS,T};
    j1["N"]=BS;
    j1["H"]=H;
    j1["forgetgateinitones"]=true;
    j1["forgetbias"]=0.30;
    j1["clip"]=clip;
    int layer_depth1=4;
    j1["H"]=H;
    for (auto l=0; l<layer_depth1; l++) {
        if (l>0) j1["inputShape"]=vector<int>{H,T};
        nName="lstm"+std::to_string(l);
        lb.addLayer(rnntype,nName,j1,{oName});
        oName=nName;
    }

    json j10;
    j10["inputShape"]=vector<int>{H,T};
    j10["M"]=VS;
    lb.addLayer("TemporalAffine","af1",j10,{oName});

    json j11;
    j11["inputShape"]=vector<int>{VS,T};
    lb.addLayer("TemporalSoftmax","sm1",j11,{"af1"});

    if (!lb.checkTopology(true)) {
        allOk=false;
        cerr << red << "Topology-check for LayerBlock: ERROR." << def << endl;
    } else {
        cerr << green << "Topology-check for LayerBlock: ok." << def << endl;
    }

    //json jc;
    //lb.getLayerConfiguration(jc);
    //cerr << jc.dump(4) << endl;

    // preseverstates no longer necessary for training!
    json jo(R"({"verbose":true,"shuffle":false,"preservestates":false,"notests":false,"nofragmentbatches":true,"epsilon":1e-8})"_json);
    jo["lossfactor"]=1.0/(floatN)T;  // Allows to normalize the loss with T.
    jo["learning_rate"]=(floatN)2e-2; //2.2e-2);

    floatN dep=25.0;
    floatN sep=0.0;
    jo["epochs"]=(floatN)dep;
    jo["batch_size"]=BS;

    for (int i=0; i<10000; i++) {
        jo["startepoch"]=(floatN)sep;
        t_cppl states;
        t_cppl statesv;
        states["y"] = new MatrixN(y);
        statesv["y"] = new MatrixN(yv);
        lb.train(X, &states, Xv, &statesv, "Adam", jo);
        cppl_delete(&states);
        cppl_delete(&statesv);

        sep+=dep;

        int pos=rand() % 1000 + 5000;
        wstring instr=txt.text.substr(pos,T);

        MatrixN xg(1,T);
        for (int i=0; i<T; i++) {
            xg(0,i)=txt.w2v[instr[i]];
        }
        wstring sout{};
        Layer* plstm0=lb.layerMap["lstm0"];
        t_cppl statesg{};
        plstm0->genZeroStates(&statesg, 1);

        int g,t,v;
        for (g=0; g<500; g++) {
            t_cppl cache{};

            MatrixN probst=lb.forward(xg,&cache, &statesg);
            MatrixN probsd=MatrixN(T,VS);
            for (t=0; t<T; t++) {
                for (v=0; v<VS; v++) {
                    probsd(t,v)=probst(0,t*VS+v);
                }
            }
            int li=-1;
            for (t=0; t<T; t++) {
                vector<floatN> probs(VS);
                vector<floatN> index(VS);
                for (v=0; v<VS; v++) {
                    probs[v]=probsd(t,v);
                    index[v]=v;
                }
                li=(int)index[randomChoice(index, probs)];
            }
            cppl_delete(&cache);

            for (int t=0; t<T-1; t++) {
                xg(0,t)=xg(0,t+1);
            }
            xg(0,T-1)=li;
            sout += txt.v2w[li];
        }
        wcout << "output: " << sout << endl;
        wstring timestr;
        currentDateTime(timestr);
        std::wofstream fl("rnnreader.txt", std::ios_base::app);
        fl << "---- " << timestr << ", ep:" << sep << " ---" << endl;
        fl << sout << endl;
        fl.close();
        cppl_delete(&statesg);
    }
    cpExitCompute();
}
<commit_msg>smaller for M1<commit_after>#include <iostream>
#include <fstream>
#include <codecvt>
//#include <cstddef>
#include <locale>
#include <clocale>
#include <string>
#include <iomanip>
#include <ctime>

#include "cp-neural.h"

using std::wcout; using std::wstring;
using std::cerr; using std::vector;

class Text {
private:
    bool isinit=false;
    bool readDataFile(string inputfile) {
        std::wifstream txtfile(inputfile, std::ios::binary);
        if (!txtfile.is_open()) {
            return false;
        }
        txtfile.imbue(std::locale("en_US.UTF-8"));
        wstring ttext((std::istreambuf_iterator<wchar_t>(txtfile)),
                     std::istreambuf_iterator<wchar_t>());
        txtfile.close();
        text=ttext;
        filename=inputfile;
        return true;
    }
public:
    wstring text;
    string filename;
    std::map<wchar_t,int> freq;
    std::map<wchar_t,int> w2v;
    std::map<int,wchar_t> v2w;
    Text(string filename) {
        if (readDataFile(filename)) {
            for (auto wc : text) {
                freq[wc]++;
            }
            int it=0;
            for (auto wc : freq) {
                w2v[wc.first]=it;
                v2w[it]=wc.first;
                ++it;
            }
            isinit=true;
            cerr << "Freq:" << freq.size() << ", w2v:" << w2v.size() << ", v2w:" << v2w.size() << endl;
        }
    }
    ~Text() {

    }
    bool isInit() {
        return isinit;
    }
    int vocsize() {
        if (!isinit) return 0;
        return (int)v2w.size();
    }

    wstring sample(int len) {
        int p=std::rand()%(text.size()-len);
        return text.substr(p,len);
    }
};

void currentDateTime(wstring& timestr) {
    time_t     now = time(0);
    struct tm  tstruct;
    char       buf[80];
    tstruct = *localtime(&now);
    // Visit http://en.cppreference.com/w/cpp/chrono/c/strftime
    // for more information about date/time format
    strftime(buf, sizeof(buf), "%Y-%m-%d.%X", &tstruct);
    timestr=std::wstring_convert<std::codecvt_utf8<wchar_t>>().from_bytes(buf);
}

int main(int argc, char *argv[]) {
#ifndef __APPLE__
    std::setlocale(LC_ALL, "");
#else
    setlocale(LC_ALL, "");
    std::wcout.imbue(std::locale("en_US.UTF-8"));
#endif
    wcout << L"Rnn-Readär" << std::endl;

    bool allOk=true;
    if (argc!=2) {
        cerr << L"rnnreader <path-to-text-file>" << endl;
        exit(-1);
    }
    Text txt(argv[1]);
    if (!txt.isInit()) {
        cerr << "Cannot initialize Text object from inputfile: " << argv[1] << endl;
        exit(-1);
    }

    wcout << L"Text size: " << txt.text.size() << endl;

    wcout << L"Size of vocabulary:" << txt.freq.size() << std::endl;
/*    for (auto f : txt.freq) {
        int c=(int)f.first;
        wstring wc(1,f.first);
        wcout << wc << L"|" <<  wchar_t(f.first) << L"(0x" << std::hex << c << L")" ": " << std::dec <<  f.second << endl;
    }
*/
    Color::Modifier red(Color::FG_RED);
    Color::Modifier green(Color::FG_GREEN);
    Color::Modifier def(Color::FG_DEFAULT);

    int T=64;

    int N=(int)txt.text.size() / (T+1);
    cerr << N << " Max datasets" << endl;
    MatrixN Xr(N,T);
    MatrixN yr(N,T);

    wstring chunk,chunky;
    int n=0;
    for (int i=0; i<N; i++) {
        wstring smp = txt.sample(T+1);
        chunk=smp.substr(0,T);
        chunky=smp.substr(1,T);
        for (int t=0; t<T; t++) {
            Xr(i,t)=txt.w2v[chunk[t]];
            yr(i,t)=txt.w2v[chunky[t]];
        }
        ++n;
    }

    int maxN = 100000;
    if (n>maxN) n=maxN;
    int n1=n*0.9;
    int dn=(n-n1)/2;
    if (n1+2*dn > n) {
        cerr << "Math doesn't work out." << endl;
    }

    cerr << n1 << " datasets, " << dn << " test-sets, " << dn << " validation-sets" << endl;

/*    MatrixN X(n1,T);
    MatrixN y(n1,T);
    MatrixN Xv(dn,T);
    MatrixN yv(dn,T);
    MatrixN Xt(dn,T);
    MatrixN yt(dn,T);
*/
    MatrixN X=Xr.block(0,0,n1,T);
    MatrixN y=yr.block(0,0,n1,T);
    MatrixN Xv=Xr.block(n1,0,dn,T);
    MatrixN yv=yr.block(n1,0,dn,T);
    MatrixN Xt=Xr.block(n1+dn,0,dn,T);
    MatrixN yt=yr.block(n1+dn,0,dn,T);

    unsigned int ctm=(unsigned int)std::time(0);
    std::srand(ctm);

    cpInitCompute("Rnnreader");
    registerLayers();

    LayerBlock lb(R"({"name":"rnnreader","init":"orthonormal","initfactor":0.03})"_json);
    int VS=txt.vocsize();
    int H=384;

    int BS=128;
    float clip=5.0;

    //int D=64;
    // CpParams cp0;
    // cp0.setPar("inputShape",vector<int>{T});
    // cp0.setPar("V",VS);
    // cp0.setPar("D",D);
    // cp0.setPar("clip",clip);
    // cp0.setPar("init",(string)"orthonormal");
    // lb.addLayer("WordEmbedding","WE0",cp0,{"input"});

    string rnntype="LSTM"; // or "RNN"
    cerr << "RNN-type: " << rnntype << endl;

    json j0;
    string oName{"OH0"};
    j0["inputShape"]=vector<int>{T};
    j0["V"]=VS;
    lb.addLayer("OneHot",oName,j0,{"input"});

    string nName;
    json j1;
    j1["inputShape"]=vector<int>{VS,T};
    j1["N"]=BS;
    j1["H"]=H;
    j1["forgetgateinitones"]=true;
    j1["forgetbias"]=0.30;
    j1["clip"]=clip;
    int layer_depth1=4;
    j1["H"]=H;
    for (auto l=0; l<layer_depth1; l++) {
        if (l>0) j1["inputShape"]=vector<int>{H,T};
        nName="lstm"+std::to_string(l);
        lb.addLayer(rnntype,nName,j1,{oName});
        oName=nName;
    }

    json j10;
    j10["inputShape"]=vector<int>{H,T};
    j10["M"]=VS;
    lb.addLayer("TemporalAffine","af1",j10,{oName});

    json j11;
    j11["inputShape"]=vector<int>{VS,T};
    lb.addLayer("TemporalSoftmax","sm1",j11,{"af1"});

    if (!lb.checkTopology(true)) {
        allOk=false;
        cerr << red << "Topology-check for LayerBlock: ERROR." << def << endl;
    } else {
        cerr << green << "Topology-check for LayerBlock: ok." << def << endl;
    }

    //json jc;
    //lb.getLayerConfiguration(jc);
    //cerr << jc.dump(4) << endl;

    // preseverstates no longer necessary for training!
    json jo(R"({"verbose":true,"shuffle":false,"preservestates":false,"notests":false,"nofragmentbatches":true,"epsilon":1e-8})"_json);
    jo["lossfactor"]=1.0/(floatN)T;  // Allows to normalize the loss with T.
    jo["learning_rate"]=(floatN)2e-2; //2.2e-2);

    floatN dep=25.0;
    floatN sep=0.0;
    jo["epochs"]=(floatN)dep;
    jo["batch_size"]=BS;

    for (int i=0; i<10000; i++) {
        jo["startepoch"]=(floatN)sep;
        t_cppl states;
        t_cppl statesv;
        states["y"] = new MatrixN(y);
        statesv["y"] = new MatrixN(yv);
        lb.train(X, &states, Xv, &statesv, "Adam", jo);
        cppl_delete(&states);
        cppl_delete(&statesv);

        sep+=dep;

        int pos=rand() % 1000 + 5000;
        wstring instr=txt.text.substr(pos,T);

        MatrixN xg(1,T);
        for (int i=0; i<T; i++) {
            xg(0,i)=txt.w2v[instr[i]];
        }
        wstring sout{};
        Layer* plstm0=lb.layerMap["lstm0"];
        t_cppl statesg{};
        plstm0->genZeroStates(&statesg, 1);

        int g,t,v;
        for (g=0; g<500; g++) {
            t_cppl cache{};

            MatrixN probst=lb.forward(xg,&cache, &statesg);
            MatrixN probsd=MatrixN(T,VS);
            for (t=0; t<T; t++) {
                for (v=0; v<VS; v++) {
                    probsd(t,v)=probst(0,t*VS+v);
                }
            }
            int li=-1;
            for (t=0; t<T; t++) {
                vector<floatN> probs(VS);
                vector<floatN> index(VS);
                for (v=0; v<VS; v++) {
                    probs[v]=probsd(t,v);
                    index[v]=v;
                }
                li=(int)index[randomChoice(index, probs)];
            }
            cppl_delete(&cache);

            for (int t=0; t<T-1; t++) {
                xg(0,t)=xg(0,t+1);
            }
            xg(0,T-1)=li;
            sout += txt.v2w[li];
        }
        wcout << "output: " << sout << endl;
        wstring timestr;
        currentDateTime(timestr);
        std::wofstream fl("rnnreader.txt", std::ios_base::app);
        fl << "---- " << timestr << ", ep:" << sep << " ---" << endl;
        fl << sout << endl;
        fl.close();
        cppl_delete(&statesg);
    }
    cpExitCompute();
}
<|endoftext|>


<commit_before>/*****************************************************************************
 * Project: RooFit                                                           *
 * Package: RooFitCore                                                       *
 *    File: $Id: RooProdPdf.cc,v 1.32 2002/11/18 19:57:44 wverkerke Exp $
 * Authors:                                                                  *
 *   WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu       *
 *   DK, David Kirkby,    UC Irvine,         dkirkby@uci.edu                 *
 *                                                                           *
 * Copyright (c) 2000-2002, Regents of the University of California          *
 *                          and Stanford University. All rights reserved.    *
 *                                                                           *
 * Redistribution and use in source and binary forms,                        *
 * with or without modification, are permitted according to the terms        *
 * listed in LICENSE (http://roofit.sourceforge.net/license.txt)             *
 *****************************************************************************/

// -- CLASS DESCRIPTION [PDF] --
// RooProdPdf is an efficient implementation of a product of PDFs of the form
//
//  PDF_1 * PDF_2 * ... * PDF_N
//
// RooProdPdf relies on each component PDF to be normalized and will perform no
// explicit normalization itself. No PDF may share any dependents with any other PDF.
//
// If exactly one of the component PDFs supports extended likelihood fits, the
// product will also be usable in extended mode, returning the number of expected
// events from the extendable component PDF. The extendable component does not
// have to appear in any specific place in the list.
//
// To construct a product of PDFs that share dependents, and thus require explicit
// normalization of the product, use RooGenericPdf.

#include "TIterator.h"
#include "RooFitCore/RooProdPdf.hh"
#include "RooFitCore/RooRealProxy.hh"
#include "RooFitCore/RooProdGenContext.hh"

ClassImp(RooProdPdf)
;

RooProdPdf::RooProdPdf(const char *name, const char *title, Double_t cutOff) :
  RooAbsPdf(name,title),
  _pdfList("_pdfList","List of PDFs",this),
  _cutOff(cutOff),
  _codeReg(10),
  _extendedIndex(-1),
  _partListMgr(10)
{
  // Dummy constructor
}

RooProdPdf::RooProdPdf(const char *name, const char *title,
		       RooAbsPdf& pdf1, RooAbsPdf& pdf2, Double_t cutOff) :
  RooAbsPdf(name,title),
  _pdfList("_pdfList","List of PDFs",this),
  _pdfIter(_pdfList.createIterator()),
  _cutOff(cutOff),
  _codeReg(10),
  _extendedIndex(-1),
  _partListMgr(10)
{
  // Constructor with 2 PDFs (most frequent use case).
  //
  // The optional cutOff parameter can be used as a speed optimization if
  // one or more of the PDF have sizable regions with very small values,
  // which would pull the entire product of PDFs to zero in those regions.
  //
  // After each PDF multiplication, the running product is compared with
  // the cutOff parameter. If the running product is smaller than the
  // cutOff value, the product series is terminated and remaining PDFs
  // are not evaluated.
  //
  // There is no magic value of the cutOff, the user should experiment
  // to find the appropriate balance between speed and precision.
  // If a cutoff is specified, the PDFs most likely to be small should
  // be put first in the product. The default cutOff value is zero.

  _pdfList.add(pdf1) ;
  if (pdf1.canBeExtended()) {
    _extendedIndex = _pdfList.index(&pdf1) ;
  }

  _pdfList.add(pdf2) ;
  if (pdf2.canBeExtended()) {
    if (_extendedIndex>=0) {
      // Protect against multiple extended terms
      cout << "RooProdPdf::RooProdPdf(" << GetName()
	   << ") multiple components with extended terms detected,"
	   << " product will not be extendible." << endl ;
      _extendedIndex=-1 ;
    } else {
      _extendedIndex=_pdfList.index(&pdf2) ;
    }
  }
}

RooProdPdf::RooProdPdf(const char* name, const char* title, const RooArgList& pdfList, Double_t cutOff) :
  RooAbsPdf(name,title),
  _pdfList("_pdfList","List of PDFs",this),
  _pdfIter(_pdfList.createIterator()),
  _cutOff(cutOff),
  _codeReg(10),
  _extendedIndex(-1),
  _partListMgr(10)
{
  // Constructor from a list of PDFs
  //
  // The optional cutOff parameter can be used as a speed optimization if
  // one or more of the PDF have sizable regions with very small values,
  // which would pull the entire product of PDFs to zero in those regions.
  //
  // After each PDF multiplication, the running product is compared with
  // the cutOff parameter. If the running product is smaller than the
  // cutOff value, the product series is terminated and remaining PDFs
  // are not evaluated.
  //
  // There is no magic value of the cutOff, the user should experiment
  // to find the appropriate balance between speed and precision.
  // If a cutoff is specified, the PDFs most likely to be small should
  // be put first in the product. The default cutOff value is zero.

  TIterator* iter = pdfList.createIterator() ;
  RooAbsPdf* pdf ;
  Int_t numExtended(0) ;
  while(pdf=(RooAbsPdf*)iter->Next()) {
    if (!dynamic_cast<RooAbsPdf*>(pdf)) {
      cout << "RooProdPdf::RooProdPdf(" << GetName() << ") list arg "
	   << pdf->GetName() << " is not a PDF, ignored" << endl ;
      continue ;
    }
    _pdfList.add(*pdf) ;
    if (pdf->canBeExtended()) {
      _extendedIndex = _pdfList.index(pdf) ;
      numExtended++ ;
    }
  }

  // Protect against multiple extended terms
  if (numExtended>1) {
    cout << "RooProdPdf::RooProdPdf(" << GetName()
	 << ") WARNING: multiple components with extended terms detected,"
	 << " product will not be extendible." << endl ;
    _extendedIndex = -1 ;
  }

  delete iter ;
}

RooProdPdf::RooProdPdf(const RooProdPdf& other, const char* name) :
  RooAbsPdf(other,name),
  _pdfList("_pdfList",this,other._pdfList),
  _pdfIter(_pdfList.createIterator()),
  _cutOff(other._cutOff),
  _codeReg(other._codeReg),
  _extendedIndex(other._extendedIndex),
  _partListMgr(other._partListMgr)
{
  // Copy constructor
}

RooProdPdf::~RooProdPdf()
{
  // Destructor

  delete _pdfIter ;
}

Double_t RooProdPdf::evaluate() const
{
  // Calculate current unnormalized value of object

  const RooArgSet* nset = _pdfList.nset() ;
  Int_t code ;
  RooArgList* plist = getPartIntList(nset,0,code) ;

  return calculate(plist,nset) ;
}

Double_t RooProdPdf::calculate(const RooArgList* partIntList, const RooArgSet* normSet) const
{
  // Calculate running product of pdfs, skipping factorized components
  RooAbsReal* partInt ;
  Double_t value(1.0) ;
  Int_t n = partIntList->getSize() ;

  Int_t i ;
  for (i=0 ; i<n ; i++) {
    partInt = ((RooAbsReal*)partIntList->at(i)) ;
    Double_t piVal = partInt->getVal(normSet) ;
    value *= piVal ;
//     if (_verboseEval<0) {
//       cout << "RPP:calc(" << GetName() << "): value *= " << piVal << " (" << partInt->GetName() << ")" << endl ;
//     }
    if (value<_cutOff) {
      //cout << "RooProdPdf::calculate(" << GetName() << ") calculation cut off after " << partInt->GetName() << endl ;
      break ;
    }
  }

  return value ;
}

RooArgList* RooProdPdf::getPartIntList(const RooArgSet* nset, const RooArgSet* iset, Int_t& code) const
{
  // Check if this configuration was created before
  RooArgList* partIntList = _partListMgr.getNormList(this,nset,iset) ;
  if (partIntList) {
    code = _partListMgr.lastIndex() ;
    return partIntList ;
  }

  // Create the partial integral set for this request
  _pdfIter->Reset() ;
  partIntList = new RooArgList("partIntList") ;
  RooAbsPdf* pdf ;
  while(pdf=(RooAbsPdf*)_pdfIter->Next()) {

    // Check if all dependents of this PDF component appear in the normalization set
    Bool_t fact(kFALSE) ;
    RooArgSet *pdfDepList = pdf->getDependents(nset) ;
    if (pdfDepList->getSize()>0) {
      fact=kTRUE ;
      TIterator* depIter = pdfDepList->createIterator() ;
      RooAbsArg* dep ;
      while(dep=(RooAbsArg*)depIter->Next()) {
	if (!iset || !iset->find(dep->GetName())) {
	  fact=kFALSE ;
	}
      }
      delete depIter ;
    }
    // fact=true -> all integrated pdf dependents are in normalization set

    if (fact) {
      // This product term factorizes, no partial integral needs to be created
    } else if (nset && pdfDepList->getSize()==0) {
    } else {
      if (iset && iset->getSize()>0) {
	RooArgSet* iSet = pdf->getDependents(iset) ;
	RooAbsReal* partInt = pdf->createIntegral(*iSet,*pdfDepList) ;
	partInt->setOperMode(operMode()) ;
	partIntList->addOwned(*partInt) ;
	delete iSet ;
      } else {
	partIntList->add(*pdf) ;
      }
    }

    delete pdfDepList ;
  }

  // Store the partial integral list and return the assigned code ;
  code = _partListMgr.setNormList(this,nset,iset,partIntList) ;
//   cout << "RooProdPdf::getPIL(" << GetName() << ") creating new configuration with code " << code << endl
//        << "    nset = " ; if (nset) nset->Print("1") ; else cout << "<none>" << endl ;
//   cout << "    iset = " ; if (iset) iset->Print("1") ; else cout << "<none>" << endl ;
//   cout << "    Partial Integral List:" << endl ;
//   partIntList->Print("1") ;

  return partIntList ;
}

Int_t RooProdPdf::getAnalyticalIntegralWN(RooArgSet& allVars, RooArgSet& analVars, const RooArgSet* normSet) const
{
  // Determine which part (if any) of given integral can be performed analytically.
  // If any analytical integration is possible, return integration scenario code.
  //
  // RooProdPdf implements two strategies in implementing analytical integrals
  //
  // First, PDF components whose entire set of dependents are requested to be integrated
  // can be dropped from the product, as they will integrate out to 1 by construction
  //
  // Second, RooProdPdf queries each remaining component PDF for its analytical integration
  // capability of the requested set ('allVars'). It finds the largest common set of variables
  // that can be integrated by all remaining components. If such a set exists, it reconfirms that
  // each component is capable of analytically integrating the common set, and combines the components
  // individual integration codes into a single integration code valid for RooProdPdf.

  if (_forceNumInt) return 0 ;

  // Declare that we can analytically integrate all requested observables
  analVars.add(allVars) ;

  // Retrieve (or create) the required partial integral list
  Int_t code ;
  getPartIntList(normSet,&allVars,code) ;

  return code+1 ;
}

Double_t RooProdPdf::analyticalIntegralWN(Int_t code, const RooArgSet* normSet) const
{
  // Return analytical integral defined by given scenario code

  // No integration scenario
  if (code==0) {
    return getVal(normSet) ;
  }

  // Partial integration scenarios
  RooArgList* partIntList = _partListMgr.getNormListByIndex(code-1) ;
  return calculate(partIntList,normSet) ;
}

Bool_t RooProdPdf::checkDependents(const RooArgSet* nset) const
{
  // Check that none of the PDFs have overlapping dependents

  Bool_t ret(kFALSE) ;

  _pdfIter->Reset() ;
  RooAbsPdf* pdf, *pdf2 ;
  TIterator* iter2 = _pdfList.createIterator() ;
  while(pdf = (RooAbsPdf*)_pdfIter->Next()) {
    *iter2 = *_pdfIter ;
    while(pdf2 = (RooAbsPdf*)iter2->Next()) {
      if (pdf->dependentOverlaps(nset,*pdf2)) {
	cout << "RooProdPdf::checkDependents(" << GetName() << "): ERROR: PDFs " << pdf->GetName()
	     << " and " << pdf2->GetName() << " have one or more dependents in common" << endl ;
	ret = kTRUE ;
      }
    }
  }
  delete iter2 ;
  return ret ;
}

RooAbsPdf::ExtendMode RooProdPdf::extendMode() const
{
  return (_extendedIndex>=0) ? ((RooAbsPdf*)_pdfList.at(_extendedIndex))->extendMode() : CanNotBeExtended ;
}

Double_t RooProdPdf::expectedEvents() const
{
  assert(_extendedIndex>=0) ;
  return ((RooAbsPdf*)_pdfList.at(_extendedIndex))->expectedEvents() ;
}

RooAbsGenContext* RooProdPdf::genContext(const RooArgSet &vars, const RooDataSet *prototype, Bool_t verbose) const
{
  return new RooProdGenContext(*this,vars,prototype,verbose) ;
}

void RooProdPdf::operModeHook()
{
  Int_t i ;
  for (i=0 ; i<_partListMgr.cacheSize() ; i++) {
    RooArgList* plist = _partListMgr.getNormListByIndex(i) ;
   if (plist->isOwning()) {
     TIterator* iter = plist->createIterator() ;
     RooAbsArg* arg ;
     while(arg=(RooAbsArg*)iter->Next()) {
       arg->setOperMode(_operMode) ;
     }
     delete iter ;
   }
  }
  return ;
}

Bool_t RooProdPdf::redirectServersHook(const RooAbsCollection& newServerList, Bool_t mustReplaceAll, Bool_t nameChange, Bool_t isRecursive)
{
  Bool_t ret(kFALSE) ;

  Int_t i ;
  for (i=0 ; i<_partListMgr.cacheSize() ; i++) {
    RooArgList* plist = _partListMgr.getNormListByIndex(i) ;

    // Only redirect owned lists in recursive mode
    if (!isRecursive && !plist->isOwning()) continue ;

    TIterator* iter = plist->createIterator() ;
    RooAbsArg* arg ;
    while(arg=(RooAbsArg*)iter->Next()) {
      ret |= arg->recursiveRedirectServers(newServerList,mustReplaceAll,nameChange) ;
    }
    delete iter ;
  }
  return ret ;
}
<commit_msg><commit_after>/*****************************************************************************
 * Project: RooFit                                                           *
 * Package: RooFitCore                                                       *
 *    File: $Id: RooProdPdf.cc,v 1.33 2003/01/14 00:07:54 wverkerke Exp $
 * Authors:                                                                  *
 *   WV, Wouter Verkerke, UC Santa Barbara, verkerke@slac.stanford.edu       *
 *   DK, David Kirkby,    UC Irvine,         dkirkby@uci.edu                 *
 *                                                                           *
 * Copyright (c) 2000-2002, Regents of the University of California          *
 *                          and Stanford University. All rights reserved.    *
 *                                                                           *
 * Redistribution and use in source and binary forms,                        *
 * with or without modification, are permitted according to the terms        *
 * listed in LICENSE (http://roofit.sourceforge.net/license.txt)             *
 *****************************************************************************/

// -- CLASS DESCRIPTION [PDF] --
// RooProdPdf is an efficient implementation of a product of PDFs of the form
//
//  PDF_1 * PDF_2 * ... * PDF_N
//
// RooProdPdf relies on each component PDF to be normalized and will perform no
// explicit normalization itself. No PDF may share any dependents with any other PDF.
//
// If exactly one of the component PDFs supports extended likelihood fits, the
// product will also be usable in extended mode, returning the number of expected
// events from the extendable component PDF. The extendable component does not
// have to appear in any specific place in the list.
//
// To construct a product of PDFs that share dependents, and thus require explicit
// normalization of the product, use RooGenericPdf.

#include "TIterator.h"
#include "RooFitCore/RooProdPdf.hh"
#include "RooFitCore/RooRealProxy.hh"
#include "RooFitCore/RooProdGenContext.hh"

ClassImp(RooProdPdf)
;

RooProdPdf::RooProdPdf(const char *name, const char *title, Double_t cutOff) :
  RooAbsPdf(name,title),
  _pdfList("_pdfList","List of PDFs",this),
  _cutOff(cutOff),
  _codeReg(10),
  _extendedIndex(-1),
  _partListMgr(10)
{
  // Dummy constructor
}

RooProdPdf::RooProdPdf(const char *name, const char *title,
		       RooAbsPdf& pdf1, RooAbsPdf& pdf2, Double_t cutOff) :
  RooAbsPdf(name,title),
  _pdfList("_pdfList","List of PDFs",this),
  _pdfIter(_pdfList.createIterator()),
  _cutOff(cutOff),
  _codeReg(10),
  _extendedIndex(-1),
  _partListMgr(10)
{
  // Constructor with 2 PDFs (most frequent use case).
  //
  // The optional cutOff parameter can be used as a speed optimization if
  // one or more of the PDF have sizable regions with very small values,
  // which would pull the entire product of PDFs to zero in those regions.
  //
  // After each PDF multiplication, the running product is compared with
  // the cutOff parameter. If the running product is smaller than the
  // cutOff value, the product series is terminated and remaining PDFs
  // are not evaluated.
  //
  // There is no magic value of the cutOff, the user should experiment
  // to find the appropriate balance between speed and precision.
  // If a cutoff is specified, the PDFs most likely to be small should
  // be put first in the product. The default cutOff value is zero.

  _pdfList.add(pdf1) ;
  if (pdf1.canBeExtended()) {
    _extendedIndex = _pdfList.index(&pdf1) ;
  }

  _pdfList.add(pdf2) ;
  if (pdf2.canBeExtended()) {
    if (_extendedIndex>=0) {
      // Protect against multiple extended terms
      cout << "RooProdPdf::RooProdPdf(" << GetName()
	   << ") multiple components with extended terms detected,"
	   << " product will not be extendible." << endl ;
      _extendedIndex=-1 ;
    } else {
      _extendedIndex=_pdfList.index(&pdf2) ;
    }
  }
}

RooProdPdf::RooProdPdf(const char* name, const char* title, const RooArgList& pdfList, Double_t cutOff) :
  RooAbsPdf(name,title),
  _pdfList("_pdfList","List of PDFs",this),
  _pdfIter(_pdfList.createIterator()),
  _cutOff(cutOff),
  _codeReg(10),
  _extendedIndex(-1),
  _partListMgr(10)
{
  // Constructor from a list of PDFs
  //
  // The optional cutOff parameter can be used as a speed optimization if
  // one or more of the PDF have sizable regions with very small values,
  // which would pull the entire product of PDFs to zero in those regions.
  //
  // After each PDF multiplication, the running product is compared with
  // the cutOff parameter. If the running product is smaller than the
  // cutOff value, the product series is terminated and remaining PDFs
  // are not evaluated.
  //
  // There is no magic value of the cutOff, the user should experiment
  // to find the appropriate balance between speed and precision.
  // If a cutoff is specified, the PDFs most likely to be small should
  // be put first in the product. The default cutOff value is zero.

  TIterator* iter = pdfList.createIterator() ;
  RooAbsPdf* pdf ;
  Int_t numExtended(0) ;
  while(pdf=(RooAbsPdf*)iter->Next()) {
    if (!dynamic_cast<RooAbsPdf*>(pdf)) {
      cout << "RooProdPdf::RooProdPdf(" << GetName() << ") list arg "
	   << pdf->GetName() << " is not a PDF, ignored" << endl ;
      continue ;
    }
    _pdfList.add(*pdf) ;
    if (pdf->canBeExtended()) {
      _extendedIndex = _pdfList.index(pdf) ;
      numExtended++ ;
    }
  }

  // Protect against multiple extended terms
  if (numExtended>1) {
    cout << "RooProdPdf::RooProdPdf(" << GetName()
	 << ") WARNING: multiple components with extended terms detected,"
	 << " product will not be extendible." << endl ;
    _extendedIndex = -1 ;
  }

  delete iter ;
}

RooProdPdf::RooProdPdf(const RooProdPdf& other, const char* name) :
  RooAbsPdf(other,name),
  _pdfList("_pdfList",this,other._pdfList),
  _pdfIter(_pdfList.createIterator()),
  _cutOff(other._cutOff),
  _codeReg(other._codeReg),
  _extendedIndex(other._extendedIndex),
  _partListMgr(other._partListMgr)
{
  // Copy constructor
}

RooProdPdf::~RooProdPdf()
{
  // Destructor

  delete _pdfIter ;
}

Double_t RooProdPdf::evaluate() const
{
  // Calculate current unnormalized value of object

  const RooArgSet* nset = _pdfList.nset() ;
  Int_t code ;
  RooArgList* plist = getPartIntList(nset,0,code) ;

  return calculate(plist,nset) ;
}

Double_t RooProdPdf::calculate(const RooArgList* partIntList, const RooArgSet* normSet) const
{
  // Calculate running product of pdfs, skipping factorized components
  RooAbsReal* partInt ;
  Double_t value(1.0) ;
  Int_t n = partIntList->getSize() ;

  Int_t i ;
  for (i=0 ; i<n ; i++) {
    partInt = ((RooAbsReal*)partIntList->at(i)) ;
    Double_t piVal = partInt->getVal(normSet) ;
    value *= piVal ;
//     if (_verboseEval<0) {
//       cout << "RPP:calc(" << GetName() << "): value *= " << piVal << " (" << partInt->GetName() << ")" << endl ;
//     }
    if (value<_cutOff) {
      //cout << "RooProdPdf::calculate(" << GetName() << ") calculation cut off after " << partInt->GetName() << endl ;
      break ;
    }
  }

  return value ;
}

RooArgList* RooProdPdf::getPartIntList(const RooArgSet* nset, const RooArgSet* iset, Int_t& code) const
{
  // Check if this configuration was created before
  RooArgList* partIntList = _partListMgr.getNormList(this,nset,iset) ;
  if (partIntList) {
    code = _partListMgr.lastIndex() ;
    return partIntList ;
  }

  // Create the partial integral set for this request
  _pdfIter->Reset() ;
  partIntList = new RooArgList("partIntList") ;
  RooAbsPdf* pdf ;
  while(pdf=(RooAbsPdf*)_pdfIter->Next()) {

    // Check if all dependents of this PDF component appear in the normalization set
    Bool_t fact(kFALSE) ;
    RooArgSet *pdfDepList = pdf->getDependents(nset) ;
    if (pdfDepList->getSize()>0) {
      fact=kTRUE ;
      TIterator* depIter = pdfDepList->createIterator() ;
      RooAbsArg* dep ;
      while(dep=(RooAbsArg*)depIter->Next()) {
	if (!iset || !iset->find(dep->GetName())) {
	  fact=kFALSE ;
	}
      }
      delete depIter ;
    }
    // fact=true -> all integrated pdf dependents are in normalization set

    if (fact) {
      // This product term factorizes, no partial integral needs to be created
    } else if (nset && pdfDepList->getSize()==0) {
    } else {
      if (iset && iset->getSize()>0) {
	RooArgSet* iSet = pdf->getDependents(iset) ;
	RooAbsReal* partInt = pdf->createIntegral(*iSet,*pdfDepList) ;
	partInt->setOperMode(operMode()) ;
	partIntList->addOwned(*partInt) ;
	delete iSet ;
      } else {
	partIntList->add(*pdf) ;
      }
    }

    delete pdfDepList ;
  }

  // Store the partial integral list and return the assigned code ;
  code = _partListMgr.setNormList(this,nset,iset,partIntList) ;
//   cout << "RooProdPdf::getPIL(" << GetName() << ") creating new configuration with code " << code << endl
//        << "    nset = " ; if (nset) nset->Print("1") ; else cout << "<none>" << endl ;
//   cout << "    iset = " ; if (iset) iset->Print("1") ; else cout << "<none>" << endl ;
//   cout << "    Partial Integral List:" << endl ;
//   partIntList->Print("1") ;

  return partIntList ;
}

Int_t RooProdPdf::getAnalyticalIntegralWN(RooArgSet& allVars, RooArgSet& analVars, const RooArgSet* normSet) const
{
  // Determine which part (if any) of given integral can be performed analytically.
  // If any analytical integration is possible, return integration scenario code.
  //
  // RooProdPdf implements two strategies in implementing analytical integrals
  //
  // First, PDF components whose entire set of dependents are requested to be integrated
  // can be dropped from the product, as they will integrate out to 1 by construction
  //
  // Second, RooProdPdf queries each remaining component PDF for its analytical integration
  // capability of the requested set ('allVars'). It finds the largest common set of variables
  // that can be integrated by all remaining components. If such a set exists, it reconfirms that
  // each component is capable of analytically integrating the common set, and combines the components
  // individual integration codes into a single integration code valid for RooProdPdf.

  if (_forceNumInt) return 0 ;

  // Declare that we can analytically integrate all requested observables
  analVars.add(allVars) ;

  // Retrieve (or create) the required partial integral list
  Int_t code ;
  getPartIntList(normSet,&allVars,code) ;

  return code+1 ;
}

Double_t RooProdPdf::analyticalIntegralWN(Int_t code, const RooArgSet* normSet) const
{
  // Return analytical integral defined by given scenario code

  // No integration scenario
  if (code==0) {
    return getVal(normSet) ;
  }

  // Partial integration scenarios
  RooArgList* partIntList = _partListMgr.getNormListByIndex(code-1) ;
  return calculate(partIntList,normSet) ;
}

Bool_t RooProdPdf::checkDependents(const RooArgSet* nset) const
{
  // Check that none of the PDFs have overlapping dependents

  Bool_t ret(kFALSE) ;

  _pdfIter->Reset() ;
  RooAbsPdf* pdf, *pdf2 ;
  TIterator* iter2 = _pdfList.createIterator() ;
  while(pdf = (RooAbsPdf*)_pdfIter->Next()) {
    *iter2 = *_pdfIter ;
    while(pdf2 = (RooAbsPdf*)iter2->Next()) {
      if (pdf->dependentOverlaps(nset,*pdf2)) {
	cout << "RooProdPdf::checkDependents(" << GetName() << "): ERROR: PDFs " << pdf->GetName()
	     << " and " << pdf2->GetName() << " have one or more dependents in common" << endl ;
	ret = kTRUE ;
      }
    }
  }
  delete iter2 ;
  return ret ;
}

RooAbsPdf::ExtendMode RooProdPdf::extendMode() const
{
  return (_extendedIndex>=0) ? ((RooAbsPdf*)_pdfList.at(_extendedIndex))->extendMode() : CanNotBeExtended ;
}

Double_t RooProdPdf::expectedEvents() const
{
  assert(_extendedIndex>=0) ;
  return ((RooAbsPdf*)_pdfList.at(_extendedIndex))->expectedEvents() ;
}

RooAbsGenContext* RooProdPdf::genContext(const RooArgSet &vars, const RooDataSet *prototype, Bool_t verbose) const
{
  return new RooProdGenContext(*this,vars,prototype,verbose) ;
}

void RooProdPdf::operModeHook()
{
  Int_t i ;
  for (i=0 ; i<_partListMgr.cacheSize() ; i++) {
    RooArgList* plist = _partListMgr.getNormListByIndex(i) ;
   if (plist->isOwning()) {
     TIterator* iter = plist->createIterator() ;
     RooAbsArg* arg ;
     while(arg=(RooAbsArg*)iter->Next()) {
       arg->setOperMode(_operMode) ;
     }
     delete iter ;
   }
  }
  return ;
}

Bool_t RooProdPdf::redirectServersHook(const RooAbsCollection& newServerList, Bool_t mustReplaceAll, Bool_t nameChange, Bool_t isRecursive)
{
  Bool_t ret(kFALSE) ;

  Int_t i ;
  for (i=0 ; i<_partListMgr.cacheSize() ; i++) {
    RooArgList* plist = _partListMgr.getNormListByIndex(i) ;

    // Only redirect owned lists in recursive mode
    if (!isRecursive || !plist->isOwning()) continue ;

    TIterator* iter = plist->createIterator() ;
    RooAbsArg* arg ;
    while(arg=(RooAbsArg*)iter->Next()) {
      ret |= arg->recursiveRedirectServers(newServerList,mustReplaceAll,nameChange) ;
    }
    delete iter ;
  }
  return ret ;
}
<|endoftext|>