<commit_before>/* Ingen.LV2 - A thin wrapper which allows Ingen to run as an LV2 plugin.
 * Copyright 2008-2011 David Robillard <http://drobilla.net>
 *
 * This library is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * This library is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <stdlib.h>

#include <string>
#include <vector>

#include <glib.h>
#include <glibmm/convert.h>
#include <glibmm/miscutils.h>
#include <glibmm/timer.h>

#include "lv2/lv2plug.in/ns/lv2core/lv2.h"

#include "ingen/ServerInterface.hpp"
#include "ingen/serialisation/Parser.hpp"
#include "ingen/shared/Configuration.hpp"
#include "ingen/shared/World.hpp"
#include "ingen/shared/runtime_paths.hpp"
#include "raul/SharedPtr.hpp"
#include "raul/Thread.hpp"
#include "raul/log.hpp"

#include "AudioBuffer.hpp"
#include "Driver.hpp"
#include "Engine.hpp"
#include "PatchImpl.hpp"
#include "PostProcessor.hpp"
#include "ProcessContext.hpp"
#include "ServerInterfaceImpl.hpp"
#include "ThreadManager.hpp"

/** Record of a patch in this Ingen LV2 bundle */
struct LV2Patch {
	LV2Patch(const std::string& u, const std::string& f);

	const std::string uri;
	const std::string filename;
	LV2_Descriptor    descriptor;
};

class Lib {
public:
	Lib();
	~Lib();

	typedef std::vector< SharedPtr<const LV2Patch> > Patches;

	Patches                      patches;
	Ingen::Shared::Configuration conf;
	int                          argc;
	char**                       argv;
};

/** Library state (constructed/destructed on library load/unload) */
Lib lib;

namespace Ingen {
namespace Server {

class LV2Driver;

class LV2Port : public DriverPort
{
public:
	LV2Port(LV2Driver* driver, DuplexPort* patch_port)
		: DriverPort(patch_port)
		, _driver(driver)
		, _buffer(NULL)
	{}

	// TODO: LV2 dynamic ports
	void create() {}
	void destroy() {}
	void move(const Raul::Path& path) {}

	void pre_process(ProcessContext& context) {
		if (!is_input() || !_buffer)
			return;

		if (_patch_port->is_a(PortType::AUDIO)) {
			AudioBuffer* patch_buf = (AudioBuffer*)_patch_port->buffer(0).get();
			patch_buf->copy((Sample*)_buffer, 0, context.nframes() - 1);
		} else if (_patch_port->is_a(PortType::EVENTS)) {
			//Raul::warn << "TODO: LV2 event I/O" << std::endl;
		}
	}

	void post_process(ProcessContext& context) {
		if (is_input() || !_buffer)
			return;

		if (_patch_port->is_a(PortType::AUDIO)) {
			AudioBuffer* patch_buf = (AudioBuffer*)_patch_port->buffer(0).get();
			memcpy((Sample*)_buffer, patch_buf->data(), context.nframes() * sizeof(Sample));
		} else if (_patch_port->is_a(PortType::EVENTS)) {
			//Raul::warn << "TODO: LV2 event I/O" << std::endl;
		}
	}

	void* buffer() const        { return _buffer; }
	void  set_buffer(void* buf) { _buffer = buf; }

private:
	LV2Driver* _driver;
	void*      _buffer;
};

class LV2Driver : public Ingen::Server::Driver {
private:
	typedef std::vector<LV2Port*> Ports;

public:
	LV2Driver(Engine& engine, SampleCount buffer_size, SampleCount sample_rate)
		: _context(engine)
		, _root_patch(NULL)
		, _buffer_size(buffer_size)
		, _sample_rate(sample_rate)
		, _frame_time(0)
	{}

	void run(uint32_t nframes) {
		_context.locate(_frame_time, nframes, 0);

		for (Ports::iterator i = _ports.begin(); i != _ports.end(); ++i)
			(*i)->pre_process(_context);

		_context.engine().process_events(_context);

		if (_root_patch)
			_root_patch->process(_context);

		for (Ports::iterator i = _ports.begin(); i != _ports.end(); ++i)
			(*i)->post_process(_context);

		_frame_time += nframes;
	}

	virtual void       set_root_patch(PatchImpl* patch) { _root_patch = patch; }
	virtual PatchImpl* root_patch()                     { return _root_patch; }

	virtual void add_port(DriverPort* port) {
		// Note this doesn't have to be realtime safe since there's no dynamic LV2 ports
		ThreadManager::assert_thread(THREAD_PROCESS);
		assert(dynamic_cast<LV2Port*>(port));
		assert(port->patch_port()->index() == _ports.size());
		_ports.push_back((LV2Port*)port);
	}

	virtual Raul::Deletable* remove_port(const Raul::Path& path,
	                                     DriverPort** port=NULL) {
		// Note this doesn't have to be realtime safe since there's no dynamic LV2 ports
		ThreadManager::assert_thread(THREAD_PROCESS);

		for (Ports::iterator i = _ports.begin(); i != _ports.end(); ++i) {
			if ((*i)->patch_port()->path() == path) {
				_ports.erase(i);
				return NULL;
			}
		}

		Raul::warn << "Unable to find port " << path << std::endl;
		return NULL;
	}

	virtual DriverPort* create_port(DuplexPort* patch_port) {
		return new LV2Port(this, patch_port);
	}

	virtual DriverPort* driver_port(const Raul::Path& path) {
		ThreadManager::assert_thread(THREAD_PROCESS);

		for (Ports::iterator i = _ports.begin(); i != _ports.end(); ++i)
			if ((*i)->patch_port()->path() == path)
				return (*i);

		return NULL;
	}

	virtual SampleCount block_length() const { return _buffer_size; }
	virtual SampleCount sample_rate()  const { return _sample_rate; }
	virtual SampleCount frame_time()   const { return _frame_time;}

	virtual bool            is_realtime() const { return true; }
	virtual ProcessContext& context()           { return _context; }

	Ports& ports() { return _ports; }

private:
	ProcessContext _context;
	PatchImpl*     _root_patch;
	SampleCount    _buffer_size;
	SampleCount    _sample_rate;
	SampleCount    _frame_time;
	Ports          _ports;
};

} // namespace Server
} // namespace Ingen

extern "C" {

using namespace Ingen;
using namespace Ingen::Server;

/** LV2 plugin library entry point */
LV2_SYMBOL_EXPORT
const LV2_Descriptor*
lv2_descriptor(uint32_t index)
{
	return index < lib.patches.size() ? &lib.patches[index]->descriptor : NULL;
}

class MainThread : public Raul::Thread
{
public:
	MainThread(SharedPtr<Engine> engine) : _engine(engine) {}

private:
	virtual void _run() {
		while (_engine->main_iteration()) {
			Glib::usleep(125000);  // 1/8 second
		}
	}

	SharedPtr<Engine> _engine;
};

struct IngenPlugin {
	Ingen::Shared::World* world;
	MainThread*           main;
};

static LV2_Handle
ingen_instantiate(const LV2_Descriptor*    descriptor,
                  double                   rate,
                  const char*              bundle_path,
                  const LV2_Feature*const* features)
{
	Shared::set_bundle_path(bundle_path);

	const LV2Patch* patch = NULL;
	for (Lib::Patches::iterator i = lib.patches.begin(); i != lib.patches.end(); ++i) {
		if (&(*i)->descriptor == descriptor) {
			patch = (*i).get();
			break;
		}
	}

	if (!patch) {
		Raul::error << "Could not find patch " << descriptor->URI << std::endl;
		return NULL;
	}

	IngenPlugin* plugin = (IngenPlugin*)malloc(sizeof(IngenPlugin));
	plugin->world = new Ingen::Shared::World(&lib.conf, lib.argc, lib.argv);
	if (!plugin->world->load_module("serialisation")) {
		delete plugin->world;
		return NULL;
	}

	SharedPtr<Server::Engine> engine(new Server::Engine(plugin->world));
	plugin->world->set_local_engine(engine);
	plugin->main = new MainThread(engine);
	plugin->main->set_name("Main");

	SharedPtr<Server::ServerInterfaceImpl> interface(
		new Server::ServerInterfaceImpl(*engine.get()));

	plugin->world->set_engine(interface);
	engine->add_event_source(interface);

	Raul::Thread::get().set_context(Server::THREAD_PRE_PROCESS);
	Server::ThreadManager::single_threaded = true;

	// FIXME: fixed (or at least maximum) buffer size
	LV2Driver* driver = new LV2Driver(*engine.get(), rate, 4096);
	engine->set_driver(SharedPtr<Ingen::Server::Driver>(driver));

	engine->activate();
	Server::ThreadManager::single_threaded = true;

	ProcessContext context(*engine.get());
	context.locate(0, UINT_MAX, 0);

	engine->post_processor()->set_end_time(UINT_MAX);

	// TODO: Load only necessary plugins
	plugin->world->engine()->get("ingen:plugins");
	interface->process(*engine->post_processor(), context, false);
	engine->post_processor()->process();

	plugin->world->parser()->parse_file(plugin->world,
	                                    plugin->world->engine().get(),
	                                    patch->filename);

	while (!interface->empty()) {
		interface->process(*engine->post_processor(), context, false);
		engine->post_processor()->process();
	}

	engine->deactivate();

	plugin->world->load_module("osc_server");

	return (LV2_Handle)plugin;
}

static void
ingen_connect_port(LV2_Handle instance, uint32_t port, void* data)
{
	using namespace Ingen::Server;

	IngenPlugin*    me     = (IngenPlugin*)instance;
	Server::Engine* engine = (Server::Engine*)me->world->local_engine().get();
	LV2Driver*      driver = (LV2Driver*)engine->driver();
	if (port < driver->ports().size()) {
		driver->ports().at(port)->set_buffer(data);
		assert(driver->ports().at(port)->patch_port()->index() == port);
	} else {
		Raul::warn << "Connect to non-existent port " << port << std::endl;
	}
}

static void
ingen_activate(LV2_Handle instance)
{
	IngenPlugin* me = (IngenPlugin*)instance;
	me->world->local_engine()->activate();
	me->main->start();
}

static void
ingen_run(LV2_Handle instance, uint32_t sample_count)
{
	IngenPlugin*    me     = (IngenPlugin*)instance;
	Server::Engine* engine = (Server::Engine*)me->world->local_engine().get();
	// FIXME: don't do this every call
	Raul::Thread::get().set_context(Ingen::Server::THREAD_PROCESS);
	((LV2Driver*)engine->driver())->run(sample_count);
}

static void
ingen_deactivate(LV2_Handle instance)
{
	IngenPlugin* me = (IngenPlugin*)instance;
	me->world->local_engine()->deactivate();
}

static void
ingen_cleanup(LV2_Handle instance)
{
	IngenPlugin* me = (IngenPlugin*)instance;
	me->world->set_local_engine(SharedPtr<Ingen::Server::Engine>());
	me->world->set_engine(SharedPtr<Ingen::ServerInterface>());
	delete me->world;
	free(instance);
}

static const void*
ingen_extension_data(const char* uri)
{
	return NULL;
}

LV2Patch::LV2Patch(const std::string& u, const std::string& f)
	: uri(u)
	, filename(f)
{
	descriptor.URI            = uri.c_str();
	descriptor.instantiate    = ingen_instantiate;
	descriptor.connect_port   = ingen_connect_port;
	descriptor.activate       = ingen_activate;
	descriptor.run            = ingen_run;
	descriptor.deactivate     = ingen_deactivate;
	descriptor.cleanup        = ingen_cleanup;
	descriptor.extension_data = ingen_extension_data;
}

/** Library constructor (called on shared library load) */
Lib::Lib()
	: argc(0)
	, argv(NULL)
{
	if (!Glib::thread_supported())
		Glib::thread_init();

	using namespace Ingen;

	Ingen::Shared::set_bundle_path_from_code((void*)&lv2_descriptor);

	Ingen::Shared::World* world = new Ingen::Shared::World(&conf, argc, argv);
	if (!world->load_module("serialisation")) {
		delete world;
		return;
	}

	assert(world->parser());

	typedef Serialisation::Parser::PatchRecords Records;

	const std::string manifest_path = Shared::bundle_file_path("manifest.ttl");
	const SerdNode    base_node     = serd_node_from_string(
		SERD_URI, (const uint8_t*)manifest_path.c_str());

	SerdEnv* env = serd_env_new(&base_node);
	Records records(
		world->parser()->find_patches(world, env,
		                              Glib::filename_to_uri(manifest_path)));
	serd_env_free(env);

	for (Records::iterator i = records.begin(); i != records.end(); ++i) {
		patches.push_back(
			SharedPtr<const LV2Patch>(new LV2Patch(i->patch_uri.str(),
			                                       i->file_uri)));
	}

	delete world;
}

/** Library destructor (called on shared library unload) */
Lib::~Lib()
{
}

} // extern "C"
<commit_msg>Fix loading as an LV2 plugin.<commit_after>/* Ingen.LV2 - A thin wrapper which allows Ingen to run as an LV2 plugin.
 * Copyright 2008-2011 David Robillard <http://drobilla.net>
 *
 * This library is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * This library is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <stdlib.h>

#include <string>
#include <vector>

#include <glib.h>
#include <glibmm/convert.h>
#include <glibmm/miscutils.h>
#include <glibmm/timer.h>

#include "lv2/lv2plug.in/ns/lv2core/lv2.h"

#include "ingen/ServerInterface.hpp"
#include "ingen/serialisation/Parser.hpp"
#include "ingen/shared/Configuration.hpp"
#include "ingen/shared/World.hpp"
#include "ingen/shared/runtime_paths.hpp"
#include "raul/SharedPtr.hpp"
#include "raul/Thread.hpp"
#include "raul/log.hpp"

#include "AudioBuffer.hpp"
#include "Driver.hpp"
#include "Engine.hpp"
#include "PatchImpl.hpp"
#include "PostProcessor.hpp"
#include "ProcessContext.hpp"
#include "ServerInterfaceImpl.hpp"
#include "ThreadManager.hpp"

/** Record of a patch in this Ingen LV2 bundle */
struct LV2Patch {
	LV2Patch(const std::string& u, const std::string& f);

	const std::string uri;
	const std::string filename;
	LV2_Descriptor    descriptor;
};

class Lib {
public:
	Lib();
	~Lib();

	typedef std::vector< SharedPtr<const LV2Patch> > Patches;

	Patches                      patches;
	Ingen::Shared::Configuration conf;
	int                          argc;
	char**                       argv;
};

/** Library state (constructed/destructed on library load/unload) */
Lib lib;

namespace Ingen {
namespace Server {

class LV2Driver;

class LV2Port : public DriverPort
{
public:
	LV2Port(LV2Driver* driver, DuplexPort* patch_port)
		: DriverPort(patch_port)
		, _driver(driver)
		, _buffer(NULL)
	{}

	// TODO: LV2 dynamic ports
	void create() {}
	void destroy() {}
	void move(const Raul::Path& path) {}

	void pre_process(ProcessContext& context) {
		if (!is_input() || !_buffer)
			return;

		if (_patch_port->is_a(PortType::AUDIO)) {
			AudioBuffer* patch_buf = (AudioBuffer*)_patch_port->buffer(0).get();
			patch_buf->copy((Sample*)_buffer, 0, context.nframes() - 1);
		} else if (_patch_port->is_a(PortType::EVENTS)) {
			//Raul::warn << "TODO: LV2 event I/O" << std::endl;
		}
	}

	void post_process(ProcessContext& context) {
		if (is_input() || !_buffer)
			return;

		if (_patch_port->is_a(PortType::AUDIO)) {
			AudioBuffer* patch_buf = (AudioBuffer*)_patch_port->buffer(0).get();
			memcpy((Sample*)_buffer, patch_buf->data(), context.nframes() * sizeof(Sample));
		} else if (_patch_port->is_a(PortType::EVENTS)) {
			//Raul::warn << "TODO: LV2 event I/O" << std::endl;
		}
	}

	void* buffer() const        { return _buffer; }
	void  set_buffer(void* buf) { _buffer = buf; }

private:
	LV2Driver* _driver;
	void*      _buffer;
};

class LV2Driver : public Ingen::Server::Driver {
private:
	typedef std::vector<LV2Port*> Ports;

public:
	LV2Driver(Engine& engine, SampleCount buffer_size, SampleCount sample_rate)
		: _context(engine)
		, _root_patch(NULL)
		, _buffer_size(buffer_size)
		, _sample_rate(sample_rate)
		, _frame_time(0)
	{}

	void run(uint32_t nframes) {
		_context.locate(_frame_time, nframes, 0);

		for (Ports::iterator i = _ports.begin(); i != _ports.end(); ++i)
			(*i)->pre_process(_context);

		_context.engine().process_events(_context);

		if (_root_patch)
			_root_patch->process(_context);

		for (Ports::iterator i = _ports.begin(); i != _ports.end(); ++i)
			(*i)->post_process(_context);

		_frame_time += nframes;
	}

	virtual void       set_root_patch(PatchImpl* patch) { _root_patch = patch; }
	virtual PatchImpl* root_patch()                     { return _root_patch; }

	virtual void add_port(DriverPort* port) {
		// Note this doesn't have to be realtime safe since there's no dynamic LV2 ports
		ThreadManager::assert_thread(THREAD_PROCESS);
		assert(dynamic_cast<LV2Port*>(port));
		assert(port->patch_port()->index() == _ports.size());
		_ports.push_back((LV2Port*)port);
	}

	virtual Raul::Deletable* remove_port(const Raul::Path& path,
	                                     DriverPort** port=NULL) {
		// Note this doesn't have to be realtime safe since there's no dynamic LV2 ports
		ThreadManager::assert_thread(THREAD_PROCESS);

		for (Ports::iterator i = _ports.begin(); i != _ports.end(); ++i) {
			if ((*i)->patch_port()->path() == path) {
				_ports.erase(i);
				return NULL;
			}
		}

		Raul::warn << "Unable to find port " << path << std::endl;
		return NULL;
	}

	virtual DriverPort* create_port(DuplexPort* patch_port) {
		return new LV2Port(this, patch_port);
	}

	virtual DriverPort* driver_port(const Raul::Path& path) {
		ThreadManager::assert_thread(THREAD_PROCESS);

		for (Ports::iterator i = _ports.begin(); i != _ports.end(); ++i)
			if ((*i)->patch_port()->path() == path)
				return (*i);

		return NULL;
	}

	virtual SampleCount block_length() const { return _buffer_size; }
	virtual SampleCount sample_rate()  const { return _sample_rate; }
	virtual SampleCount frame_time()   const { return _frame_time;}

	virtual bool            is_realtime() const { return true; }
	virtual ProcessContext& context()           { return _context; }

	Ports& ports() { return _ports; }

private:
	ProcessContext _context;
	PatchImpl*     _root_patch;
	SampleCount    _buffer_size;
	SampleCount    _sample_rate;
	SampleCount    _frame_time;
	Ports          _ports;
};

} // namespace Server
} // namespace Ingen

extern "C" {

using namespace Ingen;
using namespace Ingen::Server;

/** LV2 plugin library entry point */
LV2_SYMBOL_EXPORT
const LV2_Descriptor*
lv2_descriptor(uint32_t index)
{
	return index < lib.patches.size() ? &lib.patches[index]->descriptor : NULL;
}

class MainThread : public Raul::Thread
{
public:
	MainThread(SharedPtr<Engine> engine) : _engine(engine) {}

private:
	virtual void _run() {
		while (_engine->main_iteration()) {
			Glib::usleep(125000);  // 1/8 second
		}
	}

	SharedPtr<Engine> _engine;
};

struct IngenPlugin {
	Ingen::Shared::World* world;
	MainThread*           main;
};

static LV2_Handle
ingen_instantiate(const LV2_Descriptor*    descriptor,
                  double                   rate,
                  const char*              bundle_path,
                  const LV2_Feature*const* features)
{
	Shared::set_bundle_path(bundle_path);

	const LV2Patch* patch = NULL;
	for (Lib::Patches::iterator i = lib.patches.begin(); i != lib.patches.end(); ++i) {
		if (&(*i)->descriptor == descriptor) {
			patch = (*i).get();
			break;
		}
	}

	if (!patch) {
		Raul::error << "Could not find patch " << descriptor->URI << std::endl;
		return NULL;
	}

	IngenPlugin* plugin = (IngenPlugin*)malloc(sizeof(IngenPlugin));
	plugin->world = new Ingen::Shared::World(&lib.conf, lib.argc, lib.argv);
	if (!plugin->world->load_module("serialisation")) {
		delete plugin->world;
		return NULL;
	}

	SharedPtr<Server::Engine> engine(new Server::Engine(plugin->world));
	plugin->world->set_local_engine(engine);
	plugin->main = new MainThread(engine);
	plugin->main->set_name("Main");

	SharedPtr<Server::ServerInterfaceImpl> interface(
		new Server::ServerInterfaceImpl(*engine.get()));

	plugin->world->set_engine(interface);
	engine->add_event_source(interface);

	Raul::Thread::get().set_context(Server::THREAD_PRE_PROCESS);
	Server::ThreadManager::single_threaded = true;

	// FIXME: fixed (or at least maximum) buffer size
	LV2Driver* driver = new LV2Driver(*engine.get(), rate, 4096);
	engine->set_driver(SharedPtr<Ingen::Server::Driver>(driver));

	engine->activate();
	Server::ThreadManager::single_threaded = true;

	ProcessContext context(*engine.get());
	context.locate(0, UINT_MAX, 0);

	engine->post_processor()->set_end_time(UINT_MAX);

	// TODO: Load only necessary plugins
	plugin->world->engine()->get("ingen:plugins");
	interface->process(*engine->post_processor(), context, false);
	engine->post_processor()->process();

	plugin->world->parser()->parse_file(plugin->world,
	                                    plugin->world->engine().get(),
	                                    patch->filename);

	while (!interface->empty()) {
		interface->process(*engine->post_processor(), context, false);
		engine->post_processor()->process();
	}

	engine->deactivate();

	plugin->world->load_module("osc_server");

	return (LV2_Handle)plugin;
}

static void
ingen_connect_port(LV2_Handle instance, uint32_t port, void* data)
{
	using namespace Ingen::Server;

	IngenPlugin*    me     = (IngenPlugin*)instance;
	Server::Engine* engine = (Server::Engine*)me->world->local_engine().get();
	LV2Driver*      driver = (LV2Driver*)engine->driver();
	if (port < driver->ports().size()) {
		driver->ports().at(port)->set_buffer(data);
		assert(driver->ports().at(port)->patch_port()->index() == port);
	} else {
		Raul::warn << "Connect to non-existent port " << port << std::endl;
	}
}

static void
ingen_activate(LV2_Handle instance)
{
	IngenPlugin* me = (IngenPlugin*)instance;
	me->world->local_engine()->activate();
	me->main->start();
}

static void
ingen_run(LV2_Handle instance, uint32_t sample_count)
{
	IngenPlugin*    me     = (IngenPlugin*)instance;
	Server::Engine* engine = (Server::Engine*)me->world->local_engine().get();
	// FIXME: don't do this every call
	Raul::Thread::get().set_context(Ingen::Server::THREAD_PROCESS);
	((LV2Driver*)engine->driver())->run(sample_count);
}

static void
ingen_deactivate(LV2_Handle instance)
{
	IngenPlugin* me = (IngenPlugin*)instance;
	me->world->local_engine()->deactivate();
}

static void
ingen_cleanup(LV2_Handle instance)
{
	IngenPlugin* me = (IngenPlugin*)instance;
	me->world->set_local_engine(SharedPtr<Ingen::Server::Engine>());
	me->world->set_engine(SharedPtr<Ingen::ServerInterface>());
	delete me->world;
	free(instance);
}

static const void*
ingen_extension_data(const char* uri)
{
	return NULL;
}

LV2Patch::LV2Patch(const std::string& u, const std::string& f)
	: uri(u)
	, filename(f)
{
	descriptor.URI            = uri.c_str();
	descriptor.instantiate    = ingen_instantiate;
	descriptor.connect_port   = ingen_connect_port;
	descriptor.activate       = ingen_activate;
	descriptor.run            = ingen_run;
	descriptor.deactivate     = ingen_deactivate;
	descriptor.cleanup        = ingen_cleanup;
	descriptor.extension_data = ingen_extension_data;
}

/** Library constructor (called on shared library load) */
Lib::Lib()
	: argc(0)
	, argv(NULL)
{
	if (!Glib::thread_supported())
		Glib::thread_init();

	using namespace Ingen;

	Ingen::Shared::set_bundle_path_from_code((void*)&lv2_descriptor);

	Ingen::Shared::World* world = new Ingen::Shared::World(&conf, argc, argv);
	if (!world->load_module("serialisation")) {
		delete world;
		return;
	}

	assert(world->parser());

	typedef Serialisation::Parser::PatchRecords Records;

	const std::string manifest_path = Shared::bundle_file_path("manifest.ttl");
	const std::string manifest_uri  = Glib::filename_to_uri(manifest_path);
	const SerdNode    base_node     = serd_node_from_string(
		SERD_URI, (const uint8_t*)manifest_uri.c_str());

	SerdEnv* env = serd_env_new(&base_node);
	Records records(
		world->parser()->find_patches(world, env, manifest_uri));
	serd_env_free(env);

	for (Records::iterator i = records.begin(); i != records.end(); ++i) {
		uint8_t* path = serd_file_uri_parse((const uint8_t*)i->file_uri.c_str(), NULL);
		if (path) {
			patches.push_back(
				SharedPtr<const LV2Patch>(
					new LV2Patch(i->patch_uri.str(), (const char*)path)));
		}
		free(path);
	}

	delete world;
}

/** Library destructor (called on shared library unload) */
Lib::~Lib()
{
}

} // extern "C"
<|endoftext|>


<commit_before>#ifndef UVPP_ZSOCK_HPP
#define UVPP_ZSOCK_HPP

#include "uvpp.hpp"

namespace uvpp
{
    class ZsockWatcher
    {
    public:
        ZsockWatcher(Loop& uvloop)
          : m_zsock(nullptr), m_events(0),
            m_prepare(uvloop), m_check(uvloop), m_idle(uvloop),
            m_poll(uvloop)
        {
            m_prepare.set_callback([this](){
                int revents = get_revents(m_zsock, m_events);
                if (revents) {
                    // idle ensures that libuv will not block
                    m_idle.start();
                }
            });

            m_check.set_callback([this](){
                m_idle.stop();
                int revents = get_revents(m_zsock, m_events);
                if (revents) {
                    m_callback();
                }
            });

            m_idle.set_callback([](){});
            m_poll.set_callback([](int status, int events){});
        }

        ZsockWatcher(Loop& uvloop, void *zsock)
          : ZsockWatcher(uvloop)
        {
            init(zsock);
        }

        void init(void *zsock)
        {
            assert(m_zsock==nullptr);

            m_zsock = zsock;

            uv_os_sock_t sockfd;
            size_t optlen = sizeof(sockfd);
            int rc = zmq_getsockopt(zsock, ZMQ_FD, &sockfd, &optlen);
            assert(rc==0);

            m_poll.init(sockfd);
        }

        void set_callback(BasicCallback cb)
        {
            m_callback = cb;
        }

        void start(int events = UV_READABLE)
        {
            assert(m_zsock != nullptr);

            m_events = events;
            m_prepare.start();
            m_check.start();
            m_poll.start(m_events ? UV_READABLE : 0);
        }

        void stop()
        {
            m_prepare.stop();
            m_check.stop();
            m_idle.stop();
            m_poll.stop();
        }

    private:
        int get_revents(void *zsock, int events)
        {
            int revents = 0;

            int zmq_events;
            size_t optlen = sizeof(zmq_events);
            int rc = zmq_getsockopt(zsock, ZMQ_EVENTS, &zmq_events, &optlen);

            if (rc==-1) {
                // on error, make callback get called
                return events;
            }

            if (zmq_events & ZMQ_POLLOUT)
                revents |= events & UV_WRITABLE;
            if (zmq_events & ZMQ_POLLIN)
                revents |= events & UV_READABLE;

            return revents;
        }

    private:
        BasicCallback m_callback;

        void *m_zsock;
        int m_events;
        Prepare m_prepare;
        Check m_check;
        Idle m_idle;
        Poll m_poll;
    };

    // alternate implementation that doesn't use prepare and check watchers.
    // after performing send on this zsock, you need to call Zsock::check()
    class Zsock
    {
    public:
        Zsock(Loop& uvloop)
          : m_zsock(nullptr), m_events(0),
            m_idle(uvloop), m_poll(uvloop)
        {
            m_idle.set_callback([this](){
                int revents = get_revents(m_zsock, m_events);
                if (revents) {
                    m_callback();
                } else {
                    // set level low
                    m_idle.stop();
                }
            });

            m_poll.set_callback([this](int status, int events){
                // set level high
                m_idle.start();
            });
        }

        Zsock(Loop& uvloop, void *zsock)
          : Zsock(uvloop)
        {
            init(zsock);
        }

        void check()
        {
            m_idle.start();
        }

        void init(void *zsock)
        {
            assert(m_zsock==nullptr);

            m_zsock = zsock;

            uv_os_sock_t sockfd;
            size_t optlen = sizeof(sockfd);
            int rc = zmq_getsockopt(zsock, ZMQ_FD, &sockfd, &optlen);
            assert(rc==0);

            m_poll.init(sockfd);
        }

        void set_callback(BasicCallback cb)
        {
            m_callback = cb;
        }

        void start(int events = UV_READABLE)
        {
            assert(m_zsock != nullptr);

            m_events = events;
            m_poll.start(m_events ? UV_READABLE : 0);
        }

        void stop()
        {
            m_idle.stop();
            m_poll.stop();
        }

    private:
        int get_revents(void *zsock, int events)
        {
            int revents = 0;

            int zmq_events;
            size_t optlen = sizeof(zmq_events);
            int rc = zmq_getsockopt(zsock, ZMQ_EVENTS, &zmq_events, &optlen);

            if (rc==-1) {
                // on error, make callback get called
                return events;
            }

            if (zmq_events & ZMQ_POLLOUT)
                revents |= events & UV_WRITABLE;
            if (zmq_events & ZMQ_POLLIN)
                revents |= events & UV_READABLE;

            return revents;
        }

    private:
        BasicCallback m_callback;

        void *m_zsock;
        int m_events;
        Idle m_idle;
        Poll m_poll;
    };
}

#endif
<commit_msg>call user callback in poll callback too<commit_after>#ifndef UVPP_ZSOCK_HPP
#define UVPP_ZSOCK_HPP

#include "uvpp.hpp"

namespace uvpp
{
    class ZsockWatcher
    {
    public:
        ZsockWatcher(Loop& uvloop)
          : m_zsock(nullptr), m_events(0),
            m_prepare(uvloop), m_check(uvloop), m_idle(uvloop),
            m_poll(uvloop)
        {
            m_prepare.set_callback([this](){
                int revents = get_revents(m_zsock, m_events);
                if (revents) {
                    // idle ensures that libuv will not block
                    m_idle.start();
                }
            });

            m_check.set_callback([this](){
                m_idle.stop();
                int revents = get_revents(m_zsock, m_events);
                if (revents) {
                    m_callback();
                }
            });

            m_idle.set_callback([](){});
            m_poll.set_callback([](int status, int events){});
        }

        ZsockWatcher(Loop& uvloop, void *zsock)
          : ZsockWatcher(uvloop)
        {
            init(zsock);
        }

        void init(void *zsock)
        {
            assert(m_zsock==nullptr);

            m_zsock = zsock;

            uv_os_sock_t sockfd;
            size_t optlen = sizeof(sockfd);
            int rc = zmq_getsockopt(zsock, ZMQ_FD, &sockfd, &optlen);
            assert(rc==0);

            m_poll.init(sockfd);
        }

        void set_callback(BasicCallback cb)
        {
            m_callback = cb;
        }

        void start(int events = UV_READABLE)
        {
            assert(m_zsock != nullptr);

            m_events = events;
            m_prepare.start();
            m_check.start();
            m_poll.start(m_events ? UV_READABLE : 0);
        }

        void stop()
        {
            m_prepare.stop();
            m_check.stop();
            m_idle.stop();
            m_poll.stop();
        }

    private:
        int get_revents(void *zsock, int events)
        {
            int revents = 0;

            int zmq_events;
            size_t optlen = sizeof(zmq_events);
            int rc = zmq_getsockopt(zsock, ZMQ_EVENTS, &zmq_events, &optlen);

            if (rc==-1) {
                // on error, make callback get called
                return events;
            }

            if (zmq_events & ZMQ_POLLOUT)
                revents |= events & UV_WRITABLE;
            if (zmq_events & ZMQ_POLLIN)
                revents |= events & UV_READABLE;

            return revents;
        }

    private:
        BasicCallback m_callback;

        void *m_zsock;
        int m_events;
        Prepare m_prepare;
        Check m_check;
        Idle m_idle;
        Poll m_poll;
    };

    // alternate implementation that doesn't use prepare and check watchers.
    // after performing send on this zsock, you need to call Zsock::check()
    class Zsock
    {
    public:
        Zsock(Loop& uvloop)
          : m_zsock(nullptr), m_events(0),
            m_idle(uvloop), m_poll(uvloop)
        {
            m_idle.set_callback([this](){
                int revents = get_revents(m_zsock, m_events);
                if (revents) {
                    m_callback();
                } else {
                    m_idle.stop();
                }
            });

            m_poll.set_callback([this](int status, int events){
                int revents = get_revents(m_zsock, m_events);
                if (revents) {
                    m_callback();
                    m_idle.start();
                }
            });
        }

        Zsock(Loop& uvloop, void *zsock)
          : Zsock(uvloop)
        {
            init(zsock);
        }

        void check()
        {
            m_idle.start();
        }

        void init(void *zsock)
        {
            assert(m_zsock==nullptr);

            m_zsock = zsock;

            uv_os_sock_t sockfd;
            size_t optlen = sizeof(sockfd);
            int rc = zmq_getsockopt(zsock, ZMQ_FD, &sockfd, &optlen);
            assert(rc==0);

            m_poll.init(sockfd);
        }

        void set_callback(BasicCallback cb)
        {
            m_callback = cb;
        }

        void start(int events = UV_READABLE)
        {
            assert(m_zsock != nullptr);

            m_events = events;
            m_poll.start(m_events ? UV_READABLE : 0);
        }

        void stop()
        {
            m_idle.stop();
            m_poll.stop();
        }

    private:
        int get_revents(void *zsock, int events)
        {
            int revents = 0;

            int zmq_events;
            size_t optlen = sizeof(zmq_events);
            int rc = zmq_getsockopt(zsock, ZMQ_EVENTS, &zmq_events, &optlen);

            if (rc==-1) {
                // on error, make callback get called
                return events;
            }

            if (zmq_events & ZMQ_POLLOUT)
                revents |= events & UV_WRITABLE;
            if (zmq_events & ZMQ_POLLIN)
                revents |= events & UV_READABLE;

            return revents;
        }

    private:
        BasicCallback m_callback;

        void *m_zsock;
        int m_events;
        Idle m_idle;
        Poll m_poll;
    };
}

#endif
<|endoftext|>


<commit_before>#include <sstream>

#include "ip_address.hpp"

#include <utki/config.hpp>
#include <utki/span.hpp>

#if M_OS == M_OS_LINUX || M_OS == M_OS_MACOSX
#	include <arpa/inet.h>
#elif M_OS == M_OS_WINDOWS
#	include <Ws2tcpip.h>
#else
#	error "Unknown OS"
#endif

using namespace setka;

namespace{
bool is_ip_v4_string(const char* str){
	for(const char* p = str; *p != 0; ++p){
		if(*p == '.'){
			return true;
		}
		if(*p == ':'){
			return false;
		}
	}
	return false;
}
}

ip_address::ip ip_address::ip::parse(const char* str){
	if(is_ip_v4_string(str)){
		return ip::parse_v4(str);
	}else{
		return ip::parse_v6(str);
	}
}

ip_address::ip ip_address::ip::parse_v4(const char* str){
	sockaddr_in a;

#if M_OS == M_OS_LINUX || M_OS == M_OS_MACOSX
	int res = inet_pton(
			AF_INET,
			str,
			&a.sin_addr
		);

	if(res != 1){
		throw std::runtime_error("bad IP address format");
	}

#elif M_OS == M_OS_WINDOWS
	INT len = sizeof(a);
	INT res = WSAStringToAddress(
			const_cast<char*>(str), // NOTE: when compiling in MS Visual Studio, set "Use multi-byte character set" in project properties to avoid usage of wchar_t
			AF_INET,
			NULL,
			reinterpret_cast<sockaddr*>(&a),
			&len
		);
	if(res != 0){
		throw std::runtime_error("bad IP address format");
	}
#else
#	error "Unknown OS"
#endif

	return ip_address::ip(ntohl(a.sin_addr.s_addr));
}

ip_address::ip ip_address::ip::parse_v6(const char* str){
#if M_OS == M_OS_WINDOWS
	sockaddr_in6 aa;
	in6_addr& a = aa.sin6_addr;
#else
	in6_addr a;
#endif

#if M_OS == M_OS_LINUX || M_OS == M_OS_MACOSX
	int res = inet_pton(
			AF_INET6,
			str,
			&a
		);

	if(res != 1){
		throw std::runtime_error("bad IP address format");
	}

#elif M_OS == M_OS_WINDOWS
	INT len = sizeof(aa);
	INT res = WSAStringToAddress(
			const_cast<char*>(str),
			AF_INET6,
			NULL,
			reinterpret_cast<sockaddr*>(&aa),
			&len
		);
	if(res != 0){
		TRACE(<< "ip_address::ip::ParseIPv6(): WSAStringToAddress() failed, error = " <<  WSAGetLastError()<< ". Maybe IPv6 protocol is not installed in the Windows system, WSAStringToAddress() only supports IPv6 if it is installed." << std::endl)
		throw std::runtime_error("invalid IPv6 address string");
	}
#else
#	error "Unknown OS"
#endif

#if M_OS == M_OS_MACOSX || M_OS == M_OS_WINDOWS || (M_OS == M_OS_LINUX && M_OS_NAME == M_OS_NAME_ANDROID)
	return ip_address::ip(
			a.s6_addr[0],
			a.s6_addr[1],
			a.s6_addr[2],
			a.s6_addr[3],
			a.s6_addr[4],
			a.s6_addr[5],
			a.s6_addr[6],
			a.s6_addr[7],
			a.s6_addr[8],
			a.s6_addr[9],
			a.s6_addr[10],
			a.s6_addr[11],
			a.s6_addr[12],
			a.s6_addr[13],
			a.s6_addr[14],
			a.s6_addr[15]
		);
#else
	return ip_address::ip(
			a.__in6_u.__u6_addr8[0],
			a.__in6_u.__u6_addr8[1],
			a.__in6_u.__u6_addr8[2],
			a.__in6_u.__u6_addr8[3],
			a.__in6_u.__u6_addr8[4],
			a.__in6_u.__u6_addr8[5],
			a.__in6_u.__u6_addr8[6],
			a.__in6_u.__u6_addr8[7],
			a.__in6_u.__u6_addr8[8],
			a.__in6_u.__u6_addr8[9],
			a.__in6_u.__u6_addr8[10],
			a.__in6_u.__u6_addr8[11],
			a.__in6_u.__u6_addr8[12],
			a.__in6_u.__u6_addr8[13],
			a.__in6_u.__u6_addr8[14],
			a.__in6_u.__u6_addr8[15]
		);
#endif
}

ip_address::ip_address(const char* host_str, uint16_t p) :
		host(ip_address::ip::parse(host_str)),
		port(p)
{}

ip_address::ip_address(const char* str){
	if(*str == 0){ // if zero length string
		throw std::runtime_error("bad IP address format");
	}

	if(*str == '['){ // IPv6 with port
		std::array<char, (4 * 6 + 6 + (3 * 4 + 3) + 1)> buf;

		++str;

		char* dst;
		for(dst = &*buf.begin(); *str != ']'; ++dst, ++str){
			if(*str == 0 || !utki::make_span(buf).overlaps(dst + 1)){
				throw std::runtime_error("bad IP address format");
			}

			*dst = *str;
		}

		ASSERT(utki::make_span(buf).overlaps(dst))
		*dst = 0; // null-terminate

		this->host = ip::parse_v6(&*buf.begin());

		++str; // move to port ':' separator
	}else{
		// IPv4 or IPv6 without port

		if(is_ip_v4_string(str)){
			std::array<char, (3 * 4 + 3 + 1)> buf;

			char* dst;
			for(dst = &*buf.begin(); *str != ':' && *str != 0; ++dst, ++str){
				if(!utki::make_span(buf).overlaps(dst + 1)){
					throw std::runtime_error("bad IP address format");
				}

				*dst = *str;
			}

			ASSERT(utki::make_span(buf).overlaps(dst))
			*dst = 0; // null-terminate

			this->host = ip::parse_v4(&*buf.begin());
		}else{
			// IPv6 without port
			this->host = ip::parse_v6(str);
			this->port = 0;
			return;
		}
	}

	// parse port

	if(*str != ':'){
		if(*str == 0){
			this->port = 0;
			return;
		}else{
	//		TRACE(<< "no colon, *ip = " << (*ip) << std::endl)
			throw std::runtime_error("bad IP address format");
		}
	}

	++str;

	// move to the end of port number, maximum 5 digits.
	for(unsigned i = 0; '0' <= *str && *str <= '9' && i < 5; ++i, ++str){
	}
	if('0' <= *str && *str <= '9'){//if still have one more digit
//		TRACE(<< "still have one more digit" << std::endl)
		throw std::runtime_error("bad IP address format");
	}

	--str;

	uint32_t port = 0;

	for(unsigned i = 0; *str != ':'; ++i, --str){
		uint32_t pow = 1;
		for(unsigned j = 0; j < i; ++j){
			pow *= 10;
		}

		ASSERT('0' <= *str && *str <= '9')

		port += (*str - '0') * pow;
	}

	if(port > 0xffff){
//		TRACE(<< "port number is bigger than 0xffff" << std::endl)
		throw std::runtime_error("bad IP address format");
	}

	this->port = uint16_t(port);
}

std::string ip_address::ip::to_string()const{
	std::stringstream ss;
	if(this->is_v4()){
		for(unsigned i = 4;;){
			--i;
			ss << (((this->get_v4()) >> (8 * i)) & 0xff);
			if(i == 0){
				break;
			}
			ss << '.';
		}
	}else{
		ss << std::hex;
		for(unsigned i = 8;;){
			--i;
			ss << ((this->quad[(i * 2) / 4] >> (16 * (i % 2))) & 0xffff);
			if(i == 0){
				break;
			}
			ss << ':';
		}
	}
	return ss.str();
}
<commit_msg>remove usage of deprecated function under Win32<commit_after>#include <sstream>

#include "ip_address.hpp"

#include <utki/config.hpp>
#include <utki/span.hpp>

#if M_OS == M_OS_LINUX || M_OS == M_OS_MACOSX
#	include <arpa/inet.h>
#elif M_OS == M_OS_WINDOWS
#	include <Ws2tcpip.h>
#else
#	error "Unknown OS"
#endif

using namespace setka;

namespace{
bool is_ip_v4_string(const char* str){
	for(const char* p = str; *p != 0; ++p){
		if(*p == '.'){
			return true;
		}
		if(*p == ':'){
			return false;
		}
	}
	return false;
}
}

ip_address::ip ip_address::ip::parse(const char* str){
	if(is_ip_v4_string(str)){
		return ip::parse_v4(str);
	}else{
		return ip::parse_v6(str);
	}
}

ip_address::ip ip_address::ip::parse_v4(const char* str){
	sockaddr_in a;

#if M_OS == M_OS_LINUX || M_OS == M_OS_MACOSX || M_OS == M_OS_WINDOWS
	int res = inet_pton(
			AF_INET,
			str,
			&a.sin_addr
		);

	if(res != 1){
		throw std::runtime_error("bad IP address format");
	}

#else
#	error "Unknown OS"
#endif

	return ip_address::ip(ntohl(a.sin_addr.s_addr));
}

ip_address::ip ip_address::ip::parse_v6(const char* str){
#if M_OS == M_OS_WINDOWS
	sockaddr_in6 aa;
	in6_addr& a = aa.sin6_addr;
#else
	in6_addr a;
#endif

#if M_OS == M_OS_LINUX || M_OS == M_OS_MACOSX || M_OS == M_OS_WINDOWS
	int res = inet_pton(
			AF_INET6,
			str,
			&a
		);

	if(res != 1){
		throw std::runtime_error("bad IP address format");
	}

#else
#	error "Unknown OS"
#endif

#if M_OS == M_OS_MACOSX || M_OS == M_OS_WINDOWS || (M_OS == M_OS_LINUX && M_OS_NAME == M_OS_NAME_ANDROID)
	return ip_address::ip(
			a.s6_addr[0],
			a.s6_addr[1],
			a.s6_addr[2],
			a.s6_addr[3],
			a.s6_addr[4],
			a.s6_addr[5],
			a.s6_addr[6],
			a.s6_addr[7],
			a.s6_addr[8],
			a.s6_addr[9],
			a.s6_addr[10],
			a.s6_addr[11],
			a.s6_addr[12],
			a.s6_addr[13],
			a.s6_addr[14],
			a.s6_addr[15]
		);
#else
	return ip_address::ip(
			a.__in6_u.__u6_addr8[0],
			a.__in6_u.__u6_addr8[1],
			a.__in6_u.__u6_addr8[2],
			a.__in6_u.__u6_addr8[3],
			a.__in6_u.__u6_addr8[4],
			a.__in6_u.__u6_addr8[5],
			a.__in6_u.__u6_addr8[6],
			a.__in6_u.__u6_addr8[7],
			a.__in6_u.__u6_addr8[8],
			a.__in6_u.__u6_addr8[9],
			a.__in6_u.__u6_addr8[10],
			a.__in6_u.__u6_addr8[11],
			a.__in6_u.__u6_addr8[12],
			a.__in6_u.__u6_addr8[13],
			a.__in6_u.__u6_addr8[14],
			a.__in6_u.__u6_addr8[15]
		);
#endif
}

ip_address::ip_address(const char* host_str, uint16_t p) :
		host(ip_address::ip::parse(host_str)),
		port(p)
{}

ip_address::ip_address(const char* str){
	if(*str == 0){ // if zero length string
		throw std::runtime_error("bad IP address format");
	}

	if(*str == '['){ // IPv6 with port
		std::array<char, (4 * 6 + 6 + (3 * 4 + 3) + 1)> buf;

		++str;

		char* dst;
		for(dst = &*buf.begin(); *str != ']'; ++dst, ++str){
			if(*str == 0 || !utki::make_span(buf).overlaps(dst + 1)){
				throw std::runtime_error("bad IP address format");
			}

			*dst = *str;
		}

		ASSERT(utki::make_span(buf).overlaps(dst))
		*dst = 0; // null-terminate

		this->host = ip::parse_v6(&*buf.begin());

		++str; // move to port ':' separator
	}else{
		// IPv4 or IPv6 without port

		if(is_ip_v4_string(str)){
			std::array<char, (3 * 4 + 3 + 1)> buf;

			char* dst;
			for(dst = &*buf.begin(); *str != ':' && *str != 0; ++dst, ++str){
				if(!utki::make_span(buf).overlaps(dst + 1)){
					throw std::runtime_error("bad IP address format");
				}

				*dst = *str;
			}

			ASSERT(utki::make_span(buf).overlaps(dst))
			*dst = 0; // null-terminate

			this->host = ip::parse_v4(&*buf.begin());
		}else{
			// IPv6 without port
			this->host = ip::parse_v6(str);
			this->port = 0;
			return;
		}
	}

	// parse port

	if(*str != ':'){
		if(*str == 0){
			this->port = 0;
			return;
		}else{
	//		TRACE(<< "no colon, *ip = " << (*ip) << std::endl)
			throw std::runtime_error("bad IP address format");
		}
	}

	++str;

	// move to the end of port number, maximum 5 digits.
	for(unsigned i = 0; '0' <= *str && *str <= '9' && i < 5; ++i, ++str){
	}
	if('0' <= *str && *str <= '9'){//if still have one more digit
//		TRACE(<< "still have one more digit" << std::endl)
		throw std::runtime_error("bad IP address format");
	}

	--str;

	uint32_t port = 0;

	for(unsigned i = 0; *str != ':'; ++i, --str){
		uint32_t pow = 1;
		for(unsigned j = 0; j < i; ++j){
			pow *= 10;
		}

		ASSERT('0' <= *str && *str <= '9')

		port += (*str - '0') * pow;
	}

	if(port > 0xffff){
//		TRACE(<< "port number is bigger than 0xffff" << std::endl)
		throw std::runtime_error("bad IP address format");
	}

	this->port = uint16_t(port);
}

std::string ip_address::ip::to_string()const{
	std::stringstream ss;
	if(this->is_v4()){
		for(unsigned i = 4;;){
			--i;
			ss << (((this->get_v4()) >> (8 * i)) & 0xff);
			if(i == 0){
				break;
			}
			ss << '.';
		}
	}else{
		ss << std::hex;
		for(unsigned i = 8;;){
			--i;
			ss << ((this->quad[(i * 2) / 4] >> (16 * (i % 2))) & 0xffff);
			if(i == 0){
				break;
			}
			ss << ':';
		}
	}
	return ss.str();
}
<|endoftext|>


<commit_before>/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2009 Soeren Sonnenburg
 * Copyright (C) 2009 Fraunhofer Institute FIRST and Max-Planck-Society
 */

#include <shogun/base/init.h>
#include <shogun/mathematics/Math.h>
#include <shogun/mathematics/Random.h>
#include <shogun/lib/common.h>
#include <shogun/lib/Map.h>
#include <shogun/base/Parallel.h>
#include <shogun/base/Version.h>

#ifdef TRACE_MEMORY_ALLOCS
shogun::CMap<void*, shogun::MemoryBlock>* sg_mallocs=NULL;
#endif

namespace shogun
{
	Parallel* sg_parallel=NULL;
	SGIO* sg_io=NULL;
	Version* sg_version=NULL;
	CMath* sg_math=NULL;
	CRandom* sg_rand=NULL;

	/// function called to print normal messages
	void (*sg_print_message)(FILE* target, const char* str) = NULL;

	/// function called to print warning messages
	void (*sg_print_warning)(FILE* target, const char* str) = NULL;

	/// function called to print error messages
	void (*sg_print_error)(FILE* target, const char* str) = NULL;

	/// function called to cancel things
	void (*sg_cancel_computations)(bool &delayed, bool &immediately)=NULL;

	void init_shogun(void (*print_message)(FILE* target, const char* str),
			void (*print_warning)(FILE* target, const char* str),
			void (*print_error)(FILE* target, const char* str),
			void (*cancel_computations)(bool &delayed, bool &immediately))
	{
		if (!sg_io)
			sg_io = new shogun::SGIO();
		if (!sg_parallel)
			sg_parallel=new shogun::Parallel();
		if (!sg_version)
			sg_version = new shogun::Version();
		if (!sg_math)
			sg_math = new shogun::CMath();
		if (!sg_rand)
			sg_rand = new shogun::CRandom();
#ifdef TRACE_MEMORY_ALLOCS
		if (!sg_mallocs)
			sg_mallocs = new shogun::CMap<void*, MemoryBlock>(631, 1024, false);

		SG_REF(sg_mallocs);
#endif
		SG_REF(sg_io);
		SG_REF(sg_parallel);
		SG_REF(sg_version);
		SG_REF(sg_math);
		SG_REF(sg_rand);

		sg_print_message=print_message;
		sg_print_warning=print_warning;
		sg_print_error=print_error;
		sg_cancel_computations=cancel_computations;
	}

	void sg_global_print_default(FILE* target, const char* str)
	{
		fprintf(target, "%s", str);
	}

	void init_shogun_with_defaults()
	{
		init_shogun(&sg_global_print_default, &sg_global_print_default,
				&sg_global_print_default);
	}

	void exit_shogun()
	{
#ifdef TRACE_MEMORY_ALLOCS
		list_memory_allocs();
		shogun::CMap<void*, shogun::MemoryBlock>* mallocs=sg_mallocs;
		sg_mallocs=NULL;
		SG_UNREF(mallocs);
#endif
		sg_print_message=NULL;
		sg_print_warning=NULL;
		sg_print_error=NULL;
		sg_cancel_computations=NULL;

		SG_UNREF(sg_rand);
		SG_UNREF(sg_math);
		SG_UNREF(sg_version);
		SG_UNREF(sg_parallel);
		SG_UNREF(sg_io);

	}

	void set_global_io(SGIO* io)
	{
		SG_REF(io);
		SG_UNREF(sg_io);
		sg_io=io;
	}

	SGIO* get_global_io()
	{
		SG_REF(sg_io);
		return sg_io;
	}

	void set_global_parallel(Parallel* parallel)
	{
		SG_REF(parallel);
		SG_UNREF(sg_parallel);
		sg_parallel=parallel;
	}

	Parallel* get_global_parallel()
	{
		SG_REF(sg_parallel);
		return sg_parallel;
	}

	void set_global_version(Version* version)
	{
		SG_REF(version);
		SG_UNREF(sg_version);
		sg_version=version;
	}

	Version* get_global_version()
	{
		SG_REF(sg_version);
		return sg_version;
	}

	void set_global_math(CMath* math)
	{
		SG_REF(math);
		SG_UNREF(sg_math);
		sg_math=math;
	}

	CMath* get_global_math()
	{
		SG_REF(sg_math);
		return sg_math;
	}

	void set_global_rand(CRandom* rand)
	{
		SG_REF(rand);
		SG_UNREF(sg_rand);
		sg_rand=rand;
	}

	CRandom* get_global_rand()
	{
		SG_REF(sg_rand);
		return sg_rand;
	}
}
<commit_msg>Issue #1955; change loglevel through environment variable<commit_after>/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2009 Soeren Sonnenburg
 * Copyright (C) 2009 Fraunhofer Institute FIRST and Max-Planck-Society
 */

#include <shogun/base/init.h>
#include <shogun/mathematics/Math.h>
#include <shogun/mathematics/Random.h>
#include <shogun/lib/common.h>
#include <shogun/lib/Map.h>
#include <shogun/base/Parallel.h>
#include <shogun/base/Version.h>
#include <stdlib.h>
#include <string.h>

#ifdef TRACE_MEMORY_ALLOCS
shogun::CMap<void*, shogun::MemoryBlock>* sg_mallocs=NULL;
#endif

namespace shogun
{
	Parallel* sg_parallel=NULL;
	SGIO* sg_io=NULL;
	Version* sg_version=NULL;
	CMath* sg_math=NULL;
	CRandom* sg_rand=NULL;

	/// function called to print normal messages
	void (*sg_print_message)(FILE* target, const char* str) = NULL;

	/// function called to print warning messages
	void (*sg_print_warning)(FILE* target, const char* str) = NULL;

	/// function called to print error messages
	void (*sg_print_error)(FILE* target, const char* str) = NULL;

	/// function called to cancel things
	void (*sg_cancel_computations)(bool &delayed, bool &immediately)=NULL;

	void init_shogun(void (*print_message)(FILE* target, const char* str),
			void (*print_warning)(FILE* target, const char* str),
			void (*print_error)(FILE* target, const char* str),
			void (*cancel_computations)(bool &delayed, bool &immediately))
	{
		char* value;

		if (!sg_io)
			sg_io = new shogun::SGIO();
		if (!sg_parallel)
			sg_parallel=new shogun::Parallel();
		if (!sg_version)
			sg_version = new shogun::Version();
		if (!sg_math)
			sg_math = new shogun::CMath();
		if (!sg_rand)
			sg_rand = new shogun::CRandom();
#ifdef TRACE_MEMORY_ALLOCS
		if (!sg_mallocs)
			sg_mallocs = new shogun::CMap<void*, MemoryBlock>(631, 1024, false);

		SG_REF(sg_mallocs);
#endif
		SG_REF(sg_io);
		SG_REF(sg_parallel);
		SG_REF(sg_version);
		SG_REF(sg_math);
		SG_REF(sg_rand);

		sg_print_message=print_message;
		sg_print_warning=print_warning;
		sg_print_error=print_error;
		sg_cancel_computations=cancel_computations;

		value = getenv("SHOGUN_LOG_LEVEL");
		if (value)
		{
			if(strncmp(value, "DEBUG", 5) == 0)
				sg_io->set_loglevel(MSG_DEBUG);
			else if(strncmp(value, "WARN", 4) == 0)
				sg_io->set_loglevel(MSG_WARN);
			else if(strncmp(value, "ERROR", 5) == 0)
				sg_io->set_loglevel(MSG_ERROR);
		}
	}

	void sg_global_print_default(FILE* target, const char* str)
	{
		fprintf(target, "%s", str);
	}

	void init_shogun_with_defaults()
	{
		init_shogun(&sg_global_print_default, &sg_global_print_default,
				&sg_global_print_default);
	}

	void exit_shogun()
	{
#ifdef TRACE_MEMORY_ALLOCS
		list_memory_allocs();
		shogun::CMap<void*, shogun::MemoryBlock>* mallocs=sg_mallocs;
		sg_mallocs=NULL;
		SG_UNREF(mallocs);
#endif
		sg_print_message=NULL;
		sg_print_warning=NULL;
		sg_print_error=NULL;
		sg_cancel_computations=NULL;

		SG_UNREF(sg_rand);
		SG_UNREF(sg_math);
		SG_UNREF(sg_version);
		SG_UNREF(sg_parallel);
		SG_UNREF(sg_io);

	}

	void set_global_io(SGIO* io)
	{
		SG_REF(io);
		SG_UNREF(sg_io);
		sg_io=io;
	}

	SGIO* get_global_io()
	{
		SG_REF(sg_io);
		return sg_io;
	}

	void set_global_parallel(Parallel* parallel)
	{
		SG_REF(parallel);
		SG_UNREF(sg_parallel);
		sg_parallel=parallel;
	}

	Parallel* get_global_parallel()
	{
		SG_REF(sg_parallel);
		return sg_parallel;
	}

	void set_global_version(Version* version)
	{
		SG_REF(version);
		SG_UNREF(sg_version);
		sg_version=version;
	}

	Version* get_global_version()
	{
		SG_REF(sg_version);
		return sg_version;
	}

	void set_global_math(CMath* math)
	{
		SG_REF(math);
		SG_UNREF(sg_math);
		sg_math=math;
	}

	CMath* get_global_math()
	{
		SG_REF(sg_math);
		return sg_math;
	}

	void set_global_rand(CRandom* rand)
	{
		SG_REF(rand);
		SG_UNREF(sg_rand);
		sg_rand=rand;
	}

	CRandom* get_global_rand()
	{
		SG_REF(sg_rand);
		return sg_rand;
	}
}
<|endoftext|>


<commit_before>#include <iostream>
#include <string>
#include <vector>

#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/videoio.hpp>

template<class T>
T base_name(T const & path, T const & delims = "/\\")
{
  return path.substr(path.find_last_of(delims) + 1);
}
template<class T>
T remove_extension(T const & filename)
{
  typename T::size_type const p(filename.find_last_of('.'));
  return p > 0 && p != T::npos ? filename.substr(0, p) : filename;
}

void help(const char* path) {

    std::string progname = remove_extension(base_name<std::string>(path));

    std::cout << std::endl
        << "Shows the view of the chosen camera" << std::endl
        << "usage: " << progname << " [options]" << std::endl
        << "usage: " << progname << " <camera-id> <width> <height> <fps>" << std::endl
        << "\toptions:" << std::endl
        << "\t -e: enumerates the cameras in the system" << std::endl;
}

struct camera_info {
    int32_t     id;
    cv::Size    frame_size;
    int32_t     fps;
};

std::vector<camera_info> enum_available_cameras() {
    std::vector<camera_info> cameras;

    int32_t device_counts = 0;
    while (true) {
        cv::VideoCapture camera;
        if (!camera.open(device_counts)) {
            break;
        }

        cameras.push_back({
            device_counts,
            cv::Size(
                (int32_t)camera.get(CV_CAP_PROP_FRAME_WIDTH),
                (int32_t)camera.get(CV_CAP_PROP_FRAME_HEIGHT)),
            (int32_t)camera.get(CV_CAP_PROP_FPS)
        });

        ++device_counts;
        camera.release();
    }

    return cameras;
}

int main(int32_t argc, char* argv[]) {

    try {
        std::string options =
            "{help h usage| |Program usage}"
            "{info i| |OpenCV build info}"
            "{enum e| |Enumerates available cameras|}"
            "{@camera|0|Camera to show|}"
            "{@width|1280|Desired frame width|}"
            "{@height|720|Desired frame height|}"
            "{@fps|30|Desired frame-rate|}";

        cv::CommandLineParser parser(argc, argv, options);
        parser.about("Shows the view of the chosen camera");

        if (parser.has("help")) {
            parser.printMessage();
            return EXIT_SUCCESS;
        }

        if (parser.has("info")) {
            std::cout << cv::getBuildInformation() << std::endl;
            return EXIT_SUCCESS;
        }

        if (!parser.check()) {
            parser.printErrors();
            return EXIT_FAILURE;
        }

        // Users wants to know the cameras availables
        if (parser.has("e")) {
            auto cameras = enum_available_cameras();
            return EXIT_SUCCESS;
        }

        // Create a camera info for the given command line's arguments
        camera_info ci = {
            parser.get<int32_t>("@camera"),
            cv::Size(
                parser.get<int32_t>("@width"),
                parser.get<int32_t>("@height")),
            parser.get<int32_t>("@fps")
        };

        cv::VideoCapture vc;
        if (!vc.open(ci.id)) {
            std::cerr << "Camera " << ci.id << " is not available!" << std::endl;
            help(argv[0]);
            return EXIT_FAILURE;
        }

        vc.set(CV_CAP_PROP_FRAME_WIDTH, (double)ci.frame_size.width);
        vc.set(CV_CAP_PROP_FRAME_HEIGHT, (double)ci.frame_size.height);
        vc.set(CV_CAP_PROP_FPS, (double)ci.fps);

        camera_info camera = {
            ci.id,
            cv::Size(
                (int32_t)vc.get(CV_CAP_PROP_FRAME_WIDTH),
                (int32_t)vc.get(CV_CAP_PROP_FRAME_HEIGHT)),
            (int32_t)vc.get(CV_CAP_PROP_FPS)
        };

        std::cout << std::endl
            << "Camera id: " << camera.id
            << " (" << camera.frame_size.width
            << "x" << camera.frame_size.height
            << "@" << camera.fps << ")"
            << std::endl;

        const std::string window_name("Camera Show");
        cv::startWindowThread();
        cv::namedWindow(window_name);

        cv::Mat frame;
        while (vc.grab() && vc.retrieve(frame)) {
            cv::imshow(window_name, frame);

            auto key = cv::waitKey((int32_t)(1000.0/camera.fps));
            if (key == 27) // ESCAPE
                break;

            printf("FPS: %d\r", camera.fps);
        }

    } catch (cv::Exception& cv_exc) {
        std::cerr << cv_exc.msg << std::endl;
        std::exit(EXIT_FAILURE);
    }

    return EXIT_SUCCESS;
}<commit_msg>Notify about available cameras<commit_after>#include <iostream>
#include <string>
#include <vector>

#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/videoio.hpp>

template<class T>
T base_name(T const & path, T const & delims = "/\\")
{
  return path.substr(path.find_last_of(delims) + 1);
}
template<class T>
T remove_extension(T const & filename)
{
  typename T::size_type const p(filename.find_last_of('.'));
  return p > 0 && p != T::npos ? filename.substr(0, p) : filename;
}

void help(const char* path) {

    std::string progname = remove_extension(base_name<std::string>(path));

    std::cout << std::endl
        << "Shows the view of the chosen camera" << std::endl
        << "usage: " << progname << " [options]" << std::endl
        << "usage: " << progname << " <camera-id> <width> <height> <fps>" << std::endl
        << "\toptions:" << std::endl
        << "\t -e: enumerates the cameras in the system" << std::endl;
}

struct camera_info {
    int32_t     id;
    cv::Size    frame_size;
    int32_t     fps;
};

std::vector<camera_info> enum_available_cameras() {
    std::vector<camera_info> cameras;

    int32_t device_counts = 0;
    while (true) {
        cv::VideoCapture camera;
        if (!camera.open(device_counts)) {
            break;
        }

        cameras.push_back({
            device_counts,
            cv::Size(
                (int32_t)camera.get(CV_CAP_PROP_FRAME_WIDTH),
                (int32_t)camera.get(CV_CAP_PROP_FRAME_HEIGHT)),
            (int32_t)camera.get(CV_CAP_PROP_FPS)
        });

        ++device_counts;
        camera.release();
    }

    return cameras;
}

void print_camera_info(const camera_info& camera) {
    std::cout << std::endl
        << "Camera id: " << camera.id
        << " (" << camera.frame_size.width
        << "x" << camera.frame_size.height
        << "@" << camera.fps << ")"
        << std::endl;
}

int main(int32_t argc, char* argv[]) {

    try {
        std::string options =
            "{help h usage| |Program usage}"
            "{info i| |OpenCV build info}"
            "{enum e| |Enumerates available cameras|}"
            "{@camera|0|Camera to show|}"
            "{@width|1280|Desired frame width|}"
            "{@height|720|Desired frame height|}"
            "{@fps|30|Desired frame-rate|}";

        cv::CommandLineParser parser(argc, argv, options);
        parser.about("Shows the view of the chosen camera");

        if (parser.has("help")) {
            parser.printMessage();
            return EXIT_SUCCESS;
        }

        if (parser.has("info")) {
            std::cout << cv::getBuildInformation() << std::endl;
            return EXIT_SUCCESS;
        }

        if (!parser.check()) {
            parser.printErrors();
            return EXIT_FAILURE;
        }

        // Users wants to know the cameras availables
        if (parser.has("e")) {
            auto cameras = enum_available_cameras();

            std::cout << "Available cameras: " << std::endl;
            for (auto camera : cameras) {
                print_camera_info(camera);
            }

            return EXIT_SUCCESS;
        }

        // Create a camera info for the given command line's arguments
        camera_info ci = {
            parser.get<int32_t>("@camera"),
            cv::Size(
                parser.get<int32_t>("@width"),
                parser.get<int32_t>("@height")),
            parser.get<int32_t>("@fps")
        };

        cv::VideoCapture vc;
        if (!vc.open(ci.id)) {
            std::cerr << "Camera " << ci.id << " is not available!" << std::endl;
            help(argv[0]);
            return EXIT_FAILURE;
        }

        vc.set(CV_CAP_PROP_FRAME_WIDTH, (double)ci.frame_size.width);
        vc.set(CV_CAP_PROP_FRAME_HEIGHT, (double)ci.frame_size.height);
        vc.set(CV_CAP_PROP_FPS, (double)ci.fps);

        camera_info camera = {
            ci.id,
            cv::Size(
                (int32_t)vc.get(CV_CAP_PROP_FRAME_WIDTH),
                (int32_t)vc.get(CV_CAP_PROP_FRAME_HEIGHT)),
            (int32_t)vc.get(CV_CAP_PROP_FPS)
        };

        // Print caps of current camera
        print_camera_info(camera);

        const std::string window_name("Camera Show");
        cv::startWindowThread();
        cv::namedWindow(window_name);

        cv::Mat frame;
        while (vc.grab() && vc.retrieve(frame)) {
            cv::imshow(window_name, frame);

            auto key = cv::waitKey((int32_t)(1000.0/camera.fps));
            if (key == 27) // ESCAPE
                break;

            printf("FPS: %d\r", camera.fps);
        }

    } catch (cv::Exception& cv_exc) {
        std::cerr << cv_exc.msg << std::endl;
        std::exit(EXIT_FAILURE);
    }

    return EXIT_SUCCESS;
}<|endoftext|>


<commit_before>#include <stdlib.h>
#include <stdio.h>
#include <string>
#include <unistd.h>
#include <algorithm>
#include <sstream>
#include <iostream>
#include <gflags/gflags.h>
#include <set>
#include <boost/algorithm/string.hpp>
#include "sort_file.h"
#include "logging.h"
#include "filesystem.h"
#include "common/tools_util.h"

DEFINE_int32(total, 0, "total numbers of map tasks");
DEFINE_int32(reduce_no, 0, "the reduce number of this reduce task");
DEFINE_string(work_dir, "/tmp", "the shuffle work dir");
DEFINE_int32(batch, 500, "merge how many maps output at the same time");
DEFINE_int32(attempt_id, 0, "the attempt_id of this reduce task");
DEFINE_string(dfs_host, "", "host name of dfs master");
DEFINE_string(dfs_port, "", "port of dfs master");
DEFINE_string(dfs_user, "", "user name of dfs master");
DEFINE_string(dfs_password, "", "password of dfs master");
DEFINE_string(format, "text", "output format: text/binary");

using baidu::common::Log;
using baidu::common::FATAL;
using baidu::common::INFO;
using baidu::common::WARNING;
using namespace baidu::shuttle;

std::set<std::string> g_merged;
int32_t g_file_no(0);
FileSystem* g_fs(NULL);

void FillParam(FileSystem::Param& param) {
    if (!FLAGS_dfs_user.empty()) {
        param["host"] = FLAGS_dfs_host;
        param["port"] = FLAGS_dfs_port;
        param["user"] = FLAGS_dfs_user;
        param["password"] = FLAGS_dfs_password;
    }
}

void CollectFilesToMerge(std::vector<std::string>* maps_to_merge) {
    assert(maps_to_merge);
    for (int i = 0; i < FLAGS_total; i++) {
        std::stringstream ss;
        ss << FLAGS_work_dir << "/map_" << i;
        std::string map_dir = ss.str();
        if (g_merged.find(map_dir) != g_merged.end()) {
            continue;
        }
        if (g_fs->Exist(map_dir)) {
            LOG(INFO, "maps_to_merge: %s", map_dir.c_str());
            maps_to_merge->push_back(map_dir);
        }
        if (maps_to_merge->size() >= (size_t)FLAGS_batch) {
            break;
        }
    }
}

void MergeMapOutput(const std::vector<std::string>& maps_to_merge) {
    std::vector<std::string> file_names;
    std::vector<std::string>::const_iterator it;
    std::vector<std::string>::iterator jt;
    std::vector<std::string> real_merged_maps;
    int map_ct = 0;
    for (it = maps_to_merge.begin(); it != maps_to_merge.end(); it++) {
        const std::string& map_dir = *it;
        std::vector<std::string> sort_files;
        if (g_fs->List(map_dir, &sort_files)) {
            for (jt = sort_files.begin(); jt != sort_files.end(); jt++) {
                const std::string& file_name = *jt;
                if (boost::ends_with(file_name, ".sort")) {
                    file_names.push_back(file_name);
                }
            }
        } else {
            LOG(FATAL, "fail to list %s", map_dir.c_str());
        }
        map_ct++;
        real_merged_maps.push_back(map_dir);
        if (map_ct >= FLAGS_batch){
            break;
        }
    }
    if (file_names.empty()) {
        LOG(WARNING, "not map output found");
        return;
    }
    MergeFileReader reader;
    FileSystem::Param param;
    FillParam(param);
    Status status = reader.Open(file_names, param, kHdfsFile);
    if (status != kOk) {
        LOG(FATAL, "fail to open: %s", reader.GetErrorFile().c_str());
    }
    char s_reduce_no[256];
    snprintf(s_reduce_no, sizeof(s_reduce_no), "%05d", FLAGS_reduce_no);
    std::string s_reduce_key(s_reduce_no);

    SortFileReader::Iterator* scan_it = reader.Scan(s_reduce_key,
                                                     s_reduce_key + "\xff");
    if (scan_it->Error() != kOk && scan_it->Error() != kNoMore) {
        LOG(FATAL, "fail to scan: %s", reader.GetErrorFile().c_str());
    }

    char output_file[4096];
    snprintf(output_file, sizeof(output_file), "%s/reduce_%d_%d/%d.sort",
             FLAGS_work_dir.c_str(), FLAGS_reduce_no, FLAGS_attempt_id,
             g_file_no++);
    SortFileWriter * writer = SortFileWriter::Create(kHdfsFile, &status);
    if (status != kOk) {
        LOG(FATAL, "fail to create writer");
    }
    FileSystem::Param param_write;
    FillParam(param_write);
    param_write["replica"] = "2";
    status = writer->Open(output_file, param_write);
    if (status != kOk) {
        LOG(FATAL, "fail to open %s for write", output_file);
    }
    while (!scan_it->Done()) {
        status = writer->Put(scan_it->Key(), scan_it->Value());
        if (status != kOk) {
            LOG(FATAL, "fail to put: %s", output_file);
        }
        scan_it->Next();
    }
    if (scan_it->Error() != kOk && scan_it->Error() != kNoMore) {
        LOG(FATAL, "fail to scan: %s", reader.GetErrorFile().c_str());
    }
    status = writer->Close();
    if (status != kOk) {
        LOG(FATAL, "fail to close writer: %s", output_file);
    }
    delete scan_it;
    status = reader.Close();
    if (status != kOk) {
        LOG(FATAL, "fail to close reader: %s", reader.GetErrorFile().c_str());
    }
    delete writer;
    for (it = real_merged_maps.begin(); it != real_merged_maps.end(); it++) {
        LOG(INFO, "g_merged insert: %s", it->c_str());
        g_merged.insert(*it);
    }
}

void MergeAndPrint() {
    char reduce_merge_dir[4096];
    snprintf(reduce_merge_dir, sizeof(reduce_merge_dir),
             "%s/reduce_%d_%d",
             FLAGS_work_dir.c_str(), FLAGS_reduce_no, FLAGS_attempt_id);
    std::vector<std::string> children;
    if (!g_fs->List(reduce_merge_dir, &children)) {
        LOG(FATAL, "fail to list: %s", reduce_merge_dir);
    }
    std::vector<std::string> file_names;
    std::vector<std::string>::iterator it;
    for (it = children.begin(); it != children.end(); it++) {
        const std::string& file_name = *it;
        if (boost::ends_with(file_name, ".sort")) {
            file_names.push_back(file_name);
        }
    }
    if (file_names.empty()) {
        LOG(WARNING, "no data for this reduce task");
        return;
    }
    MergeFileReader reader;
    FileSystem::Param param;
    FillParam(param);
    Status status = reader.Open(file_names, param, kHdfsFile);
    if (status != kOk) {
        LOG(FATAL, "fail to open: %s", reader.GetErrorFile().c_str());
    }
    SortFileReader::Iterator* scan_it = reader.Scan("", "");
    if (scan_it->Error() != kOk && scan_it->Error() != kNoMore) {
        LOG(FATAL, "fail to scan: %s", reader.GetErrorFile().c_str());
    }
    while (!scan_it->Done()) {
        if (FLAGS_format == "text") {
            std::cout << scan_it->Value() << std::endl;
        } else {
            std::cout << scan_it->Value();
        }
        scan_it->Next();
    }
    if (scan_it->Error() != kOk && scan_it->Error() != kNoMore) {
        LOG(FATAL, "fail to scan: %s", reader.GetErrorFile().c_str());
    }
    reader.Close();
}

int main(int argc, char* argv[]) {
    baidu::common::SetLogFile(GetLogName("./shuffle_tool.log").c_str());
    baidu::common::SetWarningFile(GetLogName("./shuffle_tool.log.wf").c_str());
    google::ParseCommandLineFlags(&argc, &argv, true);
    FileSystem::Param param;
    FillParam(param);
    g_fs = FileSystem::CreateInfHdfs(param);
    if (FLAGS_total == 0 ) {
        LOG(FATAL, "invalid map task total");
    }
    while ((int32_t)g_merged.size() < FLAGS_total) {
        std::vector<std::string> maps_to_merge;
        CollectFilesToMerge(&maps_to_merge);
        if (maps_to_merge.empty()) {
            LOG(INFO, "map output is empty, wait 10 seconds and try...");
            sleep(10);
            continue;
        }
        if (maps_to_merge.size() >= (size_t)FLAGS_batch ||
            maps_to_merge.size() + g_merged.size() >= (size_t)FLAGS_total) {
            LOG(INFO, "try merge %d maps", maps_to_merge.size());
            std::random_shuffle(maps_to_merge.begin(), maps_to_merge.end());
            MergeMapOutput(maps_to_merge);
        } else {
            LOG(INFO, "wait for enough map-output to merge, sleep 5 second");
            sleep(5);
        }
        LOG(INFO, "merge progress: < %d/%d > ", g_merged.size(), FLAGS_total);
    }
    LOG(INFO, "merge and print out");
    MergeAndPrint();
    return 0;
}
<commit_msg>add more log<commit_after>#include <stdlib.h>
#include <stdio.h>
#include <string>
#include <unistd.h>
#include <algorithm>
#include <sstream>
#include <iostream>
#include <gflags/gflags.h>
#include <set>
#include <boost/algorithm/string.hpp>
#include "sort_file.h"
#include "logging.h"
#include "filesystem.h"
#include "common/tools_util.h"

DEFINE_int32(total, 0, "total numbers of map tasks");
DEFINE_int32(reduce_no, 0, "the reduce number of this reduce task");
DEFINE_string(work_dir, "/tmp", "the shuffle work dir");
DEFINE_int32(batch, 500, "merge how many maps output at the same time");
DEFINE_int32(attempt_id, 0, "the attempt_id of this reduce task");
DEFINE_string(dfs_host, "", "host name of dfs master");
DEFINE_string(dfs_port, "", "port of dfs master");
DEFINE_string(dfs_user, "", "user name of dfs master");
DEFINE_string(dfs_password, "", "password of dfs master");
DEFINE_string(format, "text", "output format: text/binary");

using baidu::common::Log;
using baidu::common::FATAL;
using baidu::common::INFO;
using baidu::common::WARNING;
using namespace baidu::shuttle;

std::set<std::string> g_merged;
int32_t g_file_no(0);
FileSystem* g_fs(NULL);

void FillParam(FileSystem::Param& param) {
    if (!FLAGS_dfs_user.empty()) {
        param["host"] = FLAGS_dfs_host;
        param["port"] = FLAGS_dfs_port;
        param["user"] = FLAGS_dfs_user;
        param["password"] = FLAGS_dfs_password;
    }
}

void CollectFilesToMerge(std::vector<std::string>* maps_to_merge) {
    assert(maps_to_merge);
    for (int i = 0; i < FLAGS_total; i++) {
        std::stringstream ss;
        ss << FLAGS_work_dir << "/map_" << i;
        std::string map_dir = ss.str();
        if (g_merged.find(map_dir) != g_merged.end()) {
            continue;
        }
        if (g_fs->Exist(map_dir)) {
            LOG(INFO, "maps_to_merge: %s", map_dir.c_str());
            maps_to_merge->push_back(map_dir);
        }
        if (maps_to_merge->size() >= (size_t)FLAGS_batch) {
            break;
        }
    }
}

void MergeMapOutput(const std::vector<std::string>& maps_to_merge) {
    std::vector<std::string> file_names;
    std::vector<std::string>::const_iterator it;
    std::vector<std::string>::iterator jt;
    std::vector<std::string> real_merged_maps;
    int map_ct = 0;
    for (it = maps_to_merge.begin(); it != maps_to_merge.end(); it++) {
        const std::string& map_dir = *it;
        std::vector<std::string> sort_files;
        if (g_fs->List(map_dir, &sort_files)) {
            for (jt = sort_files.begin(); jt != sort_files.end(); jt++) {
                const std::string& file_name = *jt;
                if (boost::ends_with(file_name, ".sort")) {
                    file_names.push_back(file_name);
                }
            }
        } else {
            LOG(FATAL, "fail to list %s", map_dir.c_str());
        }
        map_ct++;
        real_merged_maps.push_back(map_dir);
        if (map_ct >= FLAGS_batch){
            break;
        }
    }
    if (file_names.empty()) {
        LOG(WARNING, "not map output found");
        return;
    }
    MergeFileReader reader;
    FileSystem::Param param;
    FillParam(param);
    Status status = reader.Open(file_names, param, kHdfsFile);
    if (status != kOk) {
        LOG(FATAL, "fail to open: %s", reader.GetErrorFile().c_str());
    }
    char s_reduce_no[256];
    snprintf(s_reduce_no, sizeof(s_reduce_no), "%05d", FLAGS_reduce_no);
    std::string s_reduce_key(s_reduce_no);

    SortFileReader::Iterator* scan_it = reader.Scan(s_reduce_key,
                                                     s_reduce_key + "\xff");
    if (scan_it->Error() != kOk && scan_it->Error() != kNoMore) {
        LOG(FATAL, "fail to scan: %s", reader.GetErrorFile().c_str());
    }

    char output_file[4096];
    snprintf(output_file, sizeof(output_file), "%s/reduce_%d_%d/%d.sort",
             FLAGS_work_dir.c_str(), FLAGS_reduce_no, FLAGS_attempt_id,
             g_file_no++);
    SortFileWriter * writer = SortFileWriter::Create(kHdfsFile, &status);
    if (status != kOk) {
        LOG(FATAL, "fail to create writer");
    }
    FileSystem::Param param_write;
    FillParam(param_write);
    param_write["replica"] = "2";
    status = writer->Open(output_file, param_write);
    if (status != kOk) {
        LOG(FATAL, "fail to open %s for write", output_file);
    }
    while (!scan_it->Done()) {
        status = writer->Put(scan_it->Key(), scan_it->Value());
        if (status != kOk) {
            LOG(FATAL, "fail to put: %s", output_file);
        }
        scan_it->Next();
    }
    if (scan_it->Error() != kOk && scan_it->Error() != kNoMore) {
        LOG(FATAL, "fail to scan: %s", reader.GetErrorFile().c_str());
    }
    status = writer->Close();
    if (status != kOk) {
        LOG(FATAL, "fail to close writer: %s", output_file);
    }
    delete scan_it;
    status = reader.Close();
    if (status != kOk) {
        LOG(FATAL, "fail to close reader: %s", reader.GetErrorFile().c_str());
    }
    delete writer;
    for (it = real_merged_maps.begin(); it != real_merged_maps.end(); it++) {
        LOG(INFO, "g_merged insert: %s", it->c_str());
        g_merged.insert(*it);
    }
}

void MergeAndPrint() {
    char reduce_merge_dir[4096];
    snprintf(reduce_merge_dir, sizeof(reduce_merge_dir),
             "%s/reduce_%d_%d",
             FLAGS_work_dir.c_str(), FLAGS_reduce_no, FLAGS_attempt_id);
    std::vector<std::string> children;
    if (!g_fs->List(reduce_merge_dir, &children)) {
        LOG(FATAL, "fail to list: %s", reduce_merge_dir);
    }
    std::vector<std::string> file_names;
    std::vector<std::string>::iterator it;
    for (it = children.begin(); it != children.end(); it++) {
        const std::string& file_name = *it;
        if (boost::ends_with(file_name, ".sort")) {
            file_names.push_back(file_name);
        }
    }
    if (file_names.empty()) {
        LOG(WARNING, "no data for this reduce task");
        return;
    }
    MergeFileReader reader;
    FileSystem::Param param;
    FillParam(param);
    Status status = reader.Open(file_names, param, kHdfsFile);
    if (status != kOk) {
        LOG(FATAL, "fail to open: %s", reader.GetErrorFile().c_str());
    }
    SortFileReader::Iterator* scan_it = reader.Scan("", "");
    if (scan_it->Error() != kOk && scan_it->Error() != kNoMore) {
        LOG(FATAL, "fail to scan: %s", reader.GetErrorFile().c_str());
    }
    while (!scan_it->Done()) {
        if (FLAGS_format == "text") {
            std::cout << scan_it->Value() << std::endl;
        } else {
            std::cout << scan_it->Value();
        }
        scan_it->Next();
    }
    if (scan_it->Error() != kOk && scan_it->Error() != kNoMore) {
        LOG(FATAL, "fail to scan: %s", reader.GetErrorFile().c_str());
    }
    reader.Close();
}

int main(int argc, char* argv[]) {
    baidu::common::SetLogFile(GetLogName("./shuffle_tool.log").c_str());
    baidu::common::SetWarningFile(GetLogName("./shuffle_tool.log.wf").c_str());
    google::ParseCommandLineFlags(&argc, &argv, true);
    FileSystem::Param param;
    FillParam(param);
    g_fs = FileSystem::CreateInfHdfs(param);
    if (FLAGS_total == 0 ) {
        LOG(FATAL, "invalid map task total");
    }
    while ((int32_t)g_merged.size() < FLAGS_total) {
        std::vector<std::string> maps_to_merge;
        CollectFilesToMerge(&maps_to_merge);
        if (maps_to_merge.empty()) {
            LOG(INFO, "map output is empty, wait 10 seconds and try...");
            sleep(10);
            continue;
        }
        if (maps_to_merge.size() >= (size_t)FLAGS_batch ||
            maps_to_merge.size() + g_merged.size() >= (size_t)FLAGS_total) {
            LOG(INFO, "try merge %d maps", maps_to_merge.size());
            std::random_shuffle(maps_to_merge.begin(), maps_to_merge.end());
            MergeMapOutput(maps_to_merge);
        } else {
            LOG(INFO, "merged: %d, wait-for merge: %d", g_merged.size(), maps_to_merge.size());
            LOG(INFO, "wait for enough map-output to merge, sleep 5 second");
            sleep(5);
        }
        LOG(INFO, "merge progress: < %d/%d > ", g_merged.size(), FLAGS_total);
    }
    LOG(INFO, "merge and print out");
    MergeAndPrint();
    return 0;
}
<|endoftext|>