<commit_before>/*
 * Asynchronous local file access.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "istream_file.hxx"
#include "istream_buffer.hxx"
#include "istream_oo.hxx"
#include "buffered_io.hxx"
#include "system/fd_util.h"
#include "gerrno.h"
#include "pool.hxx"
#include "fb_pool.hxx"
#include "SliceFifoBuffer.hxx"
#include "event/Event.hxx"
#include "event/Callback.hxx"

#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <limits.h>

/**
 * If EAGAIN occurs (on NFS), we try again after 100ms.  We can't
 * check EV_READ, because the kernel always indicates VFS files as
 * "readable without blocking".
 */
static const struct timeval file_retry_timeout = {
    .tv_sec = 0,
    .tv_usec = 100000,
};

struct FileIstream final : public Istream {
    int fd;

    FdType fd_type;

    /**
     * A timer to retry reading after EAGAIN.
     */
    Event event;

    off_t rest;
    SliceFifoBuffer buffer;
    const char *path;

    FileIstream(struct pool &p, int _fd, FdType _fd_type, off_t _length,
                const char *_path)
        :Istream(p),
         fd(_fd), fd_type(_fd_type), rest(_length),
         path(_path) {
        event.SetTimer(MakeSimpleEventCallback(FileIstream, EventCallback),
                       this);
    }

    static FileIstream &Cast2(struct istream &i) {
        return (FileIstream &)Istream::Cast(i);
    }

    void CloseHandle() {
        if (fd < 0)
            return;

        event.Delete();

        close(fd);
        fd = -1;

        buffer.FreeIfDefined(fb_pool_get());
    }

    void Abort(GError *error) {
        CloseHandle();
        DestroyError(error);
    }

    /**
     * @return the number of bytes still in the buffer
     */
    size_t SubmitBuffer() {
        return ConsumeFromBuffer(buffer);
    }

    void EofDetected() {
        assert(fd >= 0);

        CloseHandle();
        DestroyEof();
    }

    gcc_pure
    size_t GetMaxRead() const {
        if (rest != (off_t)-1 && rest < (off_t)INT_MAX)
            return (size_t)rest;
        else
            return INT_MAX;
    }

    void TryData();
    void TryDirect();

    void TryRead() {
        if (CheckDirect(fd_type))
            TryDirect();
        else
            TryData();
    }

    void EventCallback() {
        TryRead();
    }

    /* virtual methods from class Istream */

    off_t GetAvailable(bool partial) override;
    off_t Skip(gcc_unused off_t length) override;

    void Read() override {
        event.Delete();
        TryRead();
    }

    int AsFd() override;
    void Close() override {
        CloseHandle();
        Destroy();
    }
};

inline void
FileIstream::TryData()
{
    size_t buffer_rest = 0;

    if (buffer.IsNull()) {
        if (rest != 0)
            buffer.Allocate(fb_pool_get());
    } else {
        const size_t available = buffer.GetAvailable();
        if (available > 0) {
            buffer_rest = SubmitBuffer();
            if (buffer_rest == available)
                /* not a single byte was consumed: we may have been
                   closed, and we must bail out now */
                return;
        }
    }

    if (rest == 0) {
        if (buffer_rest == 0)
            EofDetected();
        return;
    }

    ssize_t nbytes = read_to_buffer(fd, buffer, GetMaxRead());
    if (nbytes == 0) {
        if (rest == (off_t)-1) {
            rest = 0;
            if (buffer_rest == 0)
                EofDetected();
        } else {
            GError *error =
                g_error_new(g_file_error_quark(), 0,
                            "premature end of file in '%s'", path);
            Abort(error);
        }
        return;
    } else if (nbytes == -1) {
        GError *error =
            g_error_new(errno_quark(), errno,
                        "failed to read from '%s': %s",
                        path, strerror(errno));
        Abort(error);
        return;
    } else if (nbytes > 0 && rest != (off_t)-1) {
        rest -= (off_t)nbytes;
        assert(rest >= 0);
    }

    assert(!buffer.IsEmpty());

    buffer_rest = SubmitBuffer();
    if (buffer_rest == 0 && rest == 0)
        EofDetected();
}

inline void
FileIstream::TryDirect()
{
    /* first consume the rest of the buffer */
    if (SubmitBuffer() > 0)
        return;

    if (rest == 0) {
        EofDetected();
        return;
    }

    ssize_t nbytes = InvokeDirect(fd_type, fd, GetMaxRead());
    if (nbytes == ISTREAM_RESULT_CLOSED)
        /* this stream was closed during the direct() callback */
        return;

    if (nbytes > 0 || nbytes == ISTREAM_RESULT_BLOCKING) {
        /* -2 means the callback wasn't able to consume any data right
           now */
        if (nbytes > 0 && rest != (off_t)-1) {
            rest -= (off_t)nbytes;
            assert(rest >= 0);
            if (rest == 0)
                EofDetected();
        }
    } else if (nbytes == ISTREAM_RESULT_EOF) {
        if (rest == (off_t)-1) {
            EofDetected();
        } else {
            GError *error =
                g_error_new(g_file_error_quark(), 0,
                            "premature end of file in '%s'", path);
            Abort(error);
        }
    } else if (errno == EAGAIN) {
        /* this should only happen for splice(SPLICE_F_NONBLOCK) from
           NFS files - unfortunately we cannot use EV_READ here, so we
           just install a timer which retries after 100ms */

        event.Add(file_retry_timeout);
    } else {
        /* XXX */
        GError *error =
            g_error_new(errno_quark(), errno,
                        "failed to read from '%s': %s",
                        path, strerror(errno));
        Abort(error);
    }
}

/*
 * istream implementation
 *
 */

off_t
FileIstream::GetAvailable(bool partial)
{
    off_t available;
    if (rest != (off_t)-1)
        available = rest;
    else if (!partial)
        return (off_t)-1;
    else
        available = 0;

    available += buffer.GetAvailable();
    return available;
}

off_t
FileIstream::Skip(off_t length)
{
    event.Delete();

    if (rest == (off_t)-1)
        return (off_t)-1;

    if (length == 0)
        return 0;

    /* clear the buffer; later we could optimize this function by
       flushing only the skipped number of bytes */
    buffer.Clear();

    if (length >= rest) {
        /* skip beyond EOF */

        length = rest;
        rest = 0;
    } else {
        /* seek the file descriptor */

        off_t ret = lseek(fd, length, SEEK_CUR);
        if (ret < 0)
            return -1;
        rest -= length;
    }

    return length;
}

int
FileIstream::AsFd()
{
    int result_fd = fd;

    event.Delete();
    Destroy();

    return result_fd;
}

/*
 * constructor and public methods
 *
 */

struct istream *
istream_file_fd_new(struct pool *pool, const char *path,
                    int fd, FdType fd_type, off_t length)
{
    assert(fd >= 0);
    assert(length >= -1);

    return NewIstream<FileIstream>(*pool, fd, fd_type, length, path);
}

struct istream *
istream_file_stat_new(struct pool *pool, const char *path, struct stat *st,
                      GError **error_r)
{
    assert(path != nullptr);
    assert(st != nullptr);

    int fd = open_cloexec(path, O_RDONLY|O_NOCTTY, 0);
    if (fd < 0) {
        set_error_errno(error_r);
        g_prefix_error(error_r, "Failed to open %s: ", path);
        return nullptr;
    }

    if (fstat(fd, st) < 0) {
        set_error_errno(error_r);
        g_prefix_error(error_r, "Failed to stat %s: ", path);
        close(fd);
        return nullptr;
    }

    FdType fd_type = FdType::FD_FILE;
    off_t size = st->st_size;

    if (S_ISCHR(st->st_mode)) {
        fd_type = FdType::FD_CHARDEV;
        size = -1;
    }

    return istream_file_fd_new(pool, path, fd, fd_type, size);
}

struct istream *
istream_file_new(struct pool *pool, const char *path, off_t length,
                 GError **error_r)
{
    assert(length >= -1);

    int fd = open_cloexec(path, O_RDONLY|O_NOCTTY, 0);
    if (fd < 0) {
        set_error_errno(error_r);
        g_prefix_error(error_r, "Failed to open %s: ", path);
        return nullptr;
    }

    return istream_file_fd_new(pool, path, fd, FdType::FD_FILE, length);
}

int
istream_file_fd(struct istream *istream)
{
    assert(istream != nullptr);

    auto &file = FileIstream::Cast2(*istream);
    assert(file.fd >= 0);
    return file.fd;
}

bool
istream_file_set_range(struct istream *istream, off_t start, off_t end)
{
    assert(istream != nullptr);
    assert(start >= 0);
    assert(end >= start);

    auto &file = FileIstream::Cast2(*istream);
    assert(file.fd >= 0);
    assert(file.rest >= 0);
    assert(file.buffer.IsNull());
    assert(end <= file.rest);

    if (start > 0 && lseek(file.fd, start, SEEK_CUR) < 0)
        return false;

    file.rest = end - start;
    return true;
}
<commit_msg>istream_file: fix Skip() implementation<commit_after>/*
 * Asynchronous local file access.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "istream_file.hxx"
#include "istream_buffer.hxx"
#include "istream_oo.hxx"
#include "buffered_io.hxx"
#include "system/fd_util.h"
#include "gerrno.h"
#include "pool.hxx"
#include "fb_pool.hxx"
#include "SliceFifoBuffer.hxx"
#include "event/Event.hxx"
#include "event/Callback.hxx"

#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <limits.h>

/**
 * If EAGAIN occurs (on NFS), we try again after 100ms.  We can't
 * check EV_READ, because the kernel always indicates VFS files as
 * "readable without blocking".
 */
static const struct timeval file_retry_timeout = {
    .tv_sec = 0,
    .tv_usec = 100000,
};

struct FileIstream final : public Istream {
    int fd;

    FdType fd_type;

    /**
     * A timer to retry reading after EAGAIN.
     */
    Event event;

    off_t rest;
    SliceFifoBuffer buffer;
    const char *path;

    FileIstream(struct pool &p, int _fd, FdType _fd_type, off_t _length,
                const char *_path)
        :Istream(p),
         fd(_fd), fd_type(_fd_type), rest(_length),
         path(_path) {
        event.SetTimer(MakeSimpleEventCallback(FileIstream, EventCallback),
                       this);
    }

    static FileIstream &Cast2(struct istream &i) {
        return (FileIstream &)Istream::Cast(i);
    }

    void CloseHandle() {
        if (fd < 0)
            return;

        event.Delete();

        close(fd);
        fd = -1;

        buffer.FreeIfDefined(fb_pool_get());
    }

    void Abort(GError *error) {
        CloseHandle();
        DestroyError(error);
    }

    /**
     * @return the number of bytes still in the buffer
     */
    size_t SubmitBuffer() {
        return ConsumeFromBuffer(buffer);
    }

    void EofDetected() {
        assert(fd >= 0);

        CloseHandle();
        DestroyEof();
    }

    gcc_pure
    size_t GetMaxRead() const {
        if (rest != (off_t)-1 && rest < (off_t)INT_MAX)
            return (size_t)rest;
        else
            return INT_MAX;
    }

    void TryData();
    void TryDirect();

    void TryRead() {
        if (CheckDirect(fd_type))
            TryDirect();
        else
            TryData();
    }

    void EventCallback() {
        TryRead();
    }

    /* virtual methods from class Istream */

    off_t GetAvailable(bool partial) override;
    off_t Skip(gcc_unused off_t length) override;

    void Read() override {
        event.Delete();
        TryRead();
    }

    int AsFd() override;
    void Close() override {
        CloseHandle();
        Destroy();
    }
};

inline void
FileIstream::TryData()
{
    size_t buffer_rest = 0;

    if (buffer.IsNull()) {
        if (rest != 0)
            buffer.Allocate(fb_pool_get());
    } else {
        const size_t available = buffer.GetAvailable();
        if (available > 0) {
            buffer_rest = SubmitBuffer();
            if (buffer_rest == available)
                /* not a single byte was consumed: we may have been
                   closed, and we must bail out now */
                return;
        }
    }

    if (rest == 0) {
        if (buffer_rest == 0)
            EofDetected();
        return;
    }

    ssize_t nbytes = read_to_buffer(fd, buffer, GetMaxRead());
    if (nbytes == 0) {
        if (rest == (off_t)-1) {
            rest = 0;
            if (buffer_rest == 0)
                EofDetected();
        } else {
            GError *error =
                g_error_new(g_file_error_quark(), 0,
                            "premature end of file in '%s'", path);
            Abort(error);
        }
        return;
    } else if (nbytes == -1) {
        GError *error =
            g_error_new(errno_quark(), errno,
                        "failed to read from '%s': %s",
                        path, strerror(errno));
        Abort(error);
        return;
    } else if (nbytes > 0 && rest != (off_t)-1) {
        rest -= (off_t)nbytes;
        assert(rest >= 0);
    }

    assert(!buffer.IsEmpty());

    buffer_rest = SubmitBuffer();
    if (buffer_rest == 0 && rest == 0)
        EofDetected();
}

inline void
FileIstream::TryDirect()
{
    /* first consume the rest of the buffer */
    if (SubmitBuffer() > 0)
        return;

    if (rest == 0) {
        EofDetected();
        return;
    }

    ssize_t nbytes = InvokeDirect(fd_type, fd, GetMaxRead());
    if (nbytes == ISTREAM_RESULT_CLOSED)
        /* this stream was closed during the direct() callback */
        return;

    if (nbytes > 0 || nbytes == ISTREAM_RESULT_BLOCKING) {
        /* -2 means the callback wasn't able to consume any data right
           now */
        if (nbytes > 0 && rest != (off_t)-1) {
            rest -= (off_t)nbytes;
            assert(rest >= 0);
            if (rest == 0)
                EofDetected();
        }
    } else if (nbytes == ISTREAM_RESULT_EOF) {
        if (rest == (off_t)-1) {
            EofDetected();
        } else {
            GError *error =
                g_error_new(g_file_error_quark(), 0,
                            "premature end of file in '%s'", path);
            Abort(error);
        }
    } else if (errno == EAGAIN) {
        /* this should only happen for splice(SPLICE_F_NONBLOCK) from
           NFS files - unfortunately we cannot use EV_READ here, so we
           just install a timer which retries after 100ms */

        event.Add(file_retry_timeout);
    } else {
        /* XXX */
        GError *error =
            g_error_new(errno_quark(), errno,
                        "failed to read from '%s': %s",
                        path, strerror(errno));
        Abort(error);
    }
}

/*
 * istream implementation
 *
 */

off_t
FileIstream::GetAvailable(bool partial)
{
    off_t available;
    if (rest != (off_t)-1)
        available = rest;
    else if (!partial)
        return (off_t)-1;
    else
        available = 0;

    available += buffer.GetAvailable();
    return available;
}

off_t
FileIstream::Skip(off_t length)
{
    event.Delete();

    if (rest == (off_t)-1)
        return (off_t)-1;

    if (length == 0)
        return 0;

    const size_t buffer_available = buffer.GetAvailable();
    if (length < off_t(buffer_available)) {
        buffer.Consume(length);
        return length;
    }

    length -= buffer_available;
    buffer.Clear();

    if (length >= rest) {
        /* skip beyond EOF */

        length = rest;
        rest = 0;
    } else {
        /* seek the file descriptor */

        off_t ret = lseek(fd, length, SEEK_CUR);
        if (ret < 0)
            return -1;
        rest -= length;
    }

    return buffer_available + length;
}

int
FileIstream::AsFd()
{
    int result_fd = fd;

    event.Delete();
    Destroy();

    return result_fd;
}

/*
 * constructor and public methods
 *
 */

struct istream *
istream_file_fd_new(struct pool *pool, const char *path,
                    int fd, FdType fd_type, off_t length)
{
    assert(fd >= 0);
    assert(length >= -1);

    return NewIstream<FileIstream>(*pool, fd, fd_type, length, path);
}

struct istream *
istream_file_stat_new(struct pool *pool, const char *path, struct stat *st,
                      GError **error_r)
{
    assert(path != nullptr);
    assert(st != nullptr);

    int fd = open_cloexec(path, O_RDONLY|O_NOCTTY, 0);
    if (fd < 0) {
        set_error_errno(error_r);
        g_prefix_error(error_r, "Failed to open %s: ", path);
        return nullptr;
    }

    if (fstat(fd, st) < 0) {
        set_error_errno(error_r);
        g_prefix_error(error_r, "Failed to stat %s: ", path);
        close(fd);
        return nullptr;
    }

    FdType fd_type = FdType::FD_FILE;
    off_t size = st->st_size;

    if (S_ISCHR(st->st_mode)) {
        fd_type = FdType::FD_CHARDEV;
        size = -1;
    }

    return istream_file_fd_new(pool, path, fd, fd_type, size);
}

struct istream *
istream_file_new(struct pool *pool, const char *path, off_t length,
                 GError **error_r)
{
    assert(length >= -1);

    int fd = open_cloexec(path, O_RDONLY|O_NOCTTY, 0);
    if (fd < 0) {
        set_error_errno(error_r);
        g_prefix_error(error_r, "Failed to open %s: ", path);
        return nullptr;
    }

    return istream_file_fd_new(pool, path, fd, FdType::FD_FILE, length);
}

int
istream_file_fd(struct istream *istream)
{
    assert(istream != nullptr);

    auto &file = FileIstream::Cast2(*istream);
    assert(file.fd >= 0);
    return file.fd;
}

bool
istream_file_set_range(struct istream *istream, off_t start, off_t end)
{
    assert(istream != nullptr);
    assert(start >= 0);
    assert(end >= start);

    auto &file = FileIstream::Cast2(*istream);
    assert(file.fd >= 0);
    assert(file.rest >= 0);
    assert(file.buffer.IsNull());
    assert(end <= file.rest);

    if (start > 0 && lseek(file.fd, start, SEEK_CUR) < 0)
        return false;

    file.rest = end - start;
    return true;
}
<|endoftext|>


<commit_before>/*

Copyright (c) 2006, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include "libtorrent/pch.hpp"
#include "libtorrent/socket.hpp"

#include <boost/bind.hpp>
#include <boost/mpl/max_element.hpp>
#include <boost/mpl/vector.hpp>
#include <boost/mpl/sizeof.hpp>
#include <boost/mpl/transform_view.hpp>
#include <boost/mpl/deref.hpp>
#include <boost/lexical_cast.hpp>

#include <libtorrent/io.hpp>
#include <libtorrent/invariant_check.hpp>
#include <libtorrent/kademlia/rpc_manager.hpp>
#include <libtorrent/kademlia/logging.hpp>
#include <libtorrent/kademlia/routing_table.hpp>
#include <libtorrent/kademlia/find_data.hpp>
#include <libtorrent/kademlia/closest_nodes.hpp>
#include <libtorrent/kademlia/refresh.hpp>
#include <libtorrent/kademlia/node.hpp>
#include <libtorrent/kademlia/observer.hpp>
#include <libtorrent/hasher.hpp>

#include <fstream>

using boost::shared_ptr;
using boost::bind;

namespace libtorrent { namespace dht
{

namespace io = libtorrent::detail;
namespace mpl = boost::mpl;

#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_DEFINE_LOG(rpc)
#endif

void intrusive_ptr_add_ref(observer const* o)
{
	assert(o->m_refs >= 0);
	assert(o != 0);
	++o->m_refs;
}

void intrusive_ptr_release(observer const* o)
{
	assert(o->m_refs > 0);
	assert(o != 0);
	if (--o->m_refs == 0)
	{
		boost::pool<>& p = o->pool_allocator;
		o->~observer();
		p.ordered_free(const_cast<observer*>(o));
	}
}

node_id generate_id();

typedef mpl::vector<
	closest_nodes_observer
	, find_data_observer
	, announce_observer
	, get_peers_observer
	, refresh_observer
	, ping_observer
	, null_observer
	> observer_types;

typedef mpl::max_element<
	mpl::transform_view<observer_types, mpl::sizeof_<mpl::_1> >
    >::type max_observer_type_iter;

rpc_manager::rpc_manager(fun const& f, node_id const& our_id
	, routing_table& table, send_fun const& sf)
	: m_pool_allocator(sizeof(mpl::deref<max_observer_type_iter::base>::type))
	, m_next_transaction_id(rand() % max_transactions)
	, m_oldest_transaction_id(m_next_transaction_id)
	, m_incoming(f)
	, m_send(sf)
	, m_our_id(our_id)
	, m_table(table)
	, m_timer(time_now())
	, m_random_number(generate_id())
	, m_destructing(false)
{
	std::srand(time(0));
}

rpc_manager::~rpc_manager()
{
	m_destructing = true;
#ifdef TORRENT_DHT_VERBOSE_LOGGING
	TORRENT_LOG(rpc) << "Destructing";
#endif
	std::for_each(m_aborted_transactions.begin(), m_aborted_transactions.end()
		, bind(&observer::abort, _1));

	for (transactions_t::iterator i = m_transactions.begin()
		, end(m_transactions.end()); i != end; ++i)
	{
		if (*i) (*i)->abort();
	}
}

#ifndef NDEBUG
void rpc_manager::check_invariant() const
{
	assert(m_oldest_transaction_id >= 0);
	assert(m_oldest_transaction_id < max_transactions);
	assert(m_next_transaction_id >= 0);
	assert(m_next_transaction_id < max_transactions);
	assert(!m_transactions[m_next_transaction_id]);

	for (int i = (m_next_transaction_id + 1) % max_transactions;
		i != m_oldest_transaction_id; i = (i + 1) % max_transactions)
	{
		assert(!m_transactions[i]);
	}
}
#endif

bool rpc_manager::incoming(msg const& m)
{
	INVARIANT_CHECK;

	if (m_destructing) return false;

	if (m.reply)
	{
		// if we don't have the transaction id in our
		// request list, ignore the packet

		if (m.transaction_id.size() < 2)
		{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
			TORRENT_LOG(rpc) << "Reply with invalid transaction id size: "
				<< m.transaction_id.size() << " from " << m.addr;
#endif
			msg reply;
			reply.reply = true;
			reply.message_id = messages::error;
			reply.error_code = 203; // Protocol error
			reply.error_msg = "reply with invalid transaction id, size "
				+ boost::lexical_cast<std::string>(m.transaction_id.size());
			reply.addr = m.addr;
			reply.transaction_id = "";
			m_send(reply);
			return false;
		}

		std::string::const_iterator i = m.transaction_id.begin();
		int tid = io::read_uint16(i);

		if (tid >= (int)m_transactions.size()
			|| tid < 0)
		{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
			TORRENT_LOG(rpc) << "Reply with invalid transaction id: "
				<< tid << " from " << m.addr;
#endif
			msg reply;
			reply.reply = true;
			reply.message_id = messages::error;
			reply.error_code = 203; // Protocol error
			reply.error_msg = "reply with invalid transaction id";
			reply.addr = m.addr;
			reply.transaction_id = "";
			m_send(reply);
			return false;
		}

		observer_ptr o = m_transactions[tid];

		if (!o)
		{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
			TORRENT_LOG(rpc) << "Reply with unknown transaction id: "
				<< tid << " from " << m.addr << " (possibly timed out)";
#endif
			return false;
		}

		if (m.addr != o->target_addr)
		{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
			TORRENT_LOG(rpc) << "Reply with incorrect address and valid transaction id: "
				<< tid << " from " << m.addr;
#endif
			return false;
		}

#ifdef TORRENT_DHT_VERBOSE_LOGGING
		std::ofstream reply_stats("libtorrent_logs/round_trip_ms.log", std::ios::app);
		reply_stats << m.addr << "\t" << total_milliseconds(time_now() - o->sent)
			<< std::endl;
#endif
		o->reply(m);
		m_transactions[tid] = 0;

		if (m.piggy_backed_ping)
		{
			// there is a ping request piggy
			// backed in this reply
			msg ph;
			ph.message_id = messages::ping;
			ph.transaction_id = m.ping_transaction_id;
			ph.addr = m.addr;
			ph.reply = true;

			reply(ph);
		}
		return m_table.node_seen(m.id, m.addr);
	}
	else
	{
		assert(m.message_id != messages::error);
		// this is an incoming request
		m_incoming(m);
	}
	return false;
}

time_duration rpc_manager::tick()
{
	INVARIANT_CHECK;

	const int timeout_ms = 10 * 1000;

	//	look for observers that has timed out

	if (m_next_transaction_id == m_oldest_transaction_id) return milliseconds(timeout_ms);

	std::vector<observer_ptr > timeouts;

	for (;m_next_transaction_id != m_oldest_transaction_id;
		m_oldest_transaction_id = (m_oldest_transaction_id + 1) % max_transactions)
	{
		assert(m_oldest_transaction_id >= 0);
		assert(m_oldest_transaction_id < max_transactions);

		observer_ptr o = m_transactions[m_oldest_transaction_id];
		if (!o) continue;

		time_duration diff = o->sent + milliseconds(timeout_ms) - time_now();
		if (diff > seconds(0))
		{
			if (diff < seconds(1)) return seconds(1);
			return diff;
		}

		try
		{
			m_transactions[m_oldest_transaction_id] = 0;
			timeouts.push_back(o);
		} catch (std::exception) {}
	}

	std::for_each(timeouts.begin(), timeouts.end(), bind(&observer::timeout, _1));
	timeouts.clear();

	// clear the aborted transactions, will likely
	// generate new requests. We need to swap, since the
	// destrutors may add more observers to the m_aborted_transactions
	std::vector<observer_ptr >().swap(m_aborted_transactions);
	return milliseconds(timeout_ms);
}

unsigned int rpc_manager::new_transaction_id(observer_ptr o)
{
	INVARIANT_CHECK;

	unsigned int tid = m_next_transaction_id;
	m_next_transaction_id = (m_next_transaction_id + 1) % max_transactions;
	if (m_transactions[m_next_transaction_id])
	{
		// moving the observer into the set of aborted transactions
		// it will prevent it from spawning new requests right now,
		// since that would break the invariant
		m_aborted_transactions.push_back(m_transactions[m_next_transaction_id]);
		m_transactions[m_next_transaction_id] = 0;
		assert(m_oldest_transaction_id == m_next_transaction_id);
	}
	assert(!m_transactions[tid]);
	m_transactions[tid] = o;
	if (m_oldest_transaction_id == m_next_transaction_id)
	{
		m_oldest_transaction_id = (m_oldest_transaction_id + 1) % max_transactions;
#ifdef TORRENT_DHT_VERBOSE_LOGGING
		TORRENT_LOG(rpc) << "WARNING: transaction limit reached! Too many concurrent"
			" messages! limit: " << (int)max_transactions;
#endif
		update_oldest_transaction_id();
	}

	return tid;
}

void rpc_manager::update_oldest_transaction_id()
{
	INVARIANT_CHECK;

	assert(m_oldest_transaction_id != m_next_transaction_id);
	while (!m_transactions[m_oldest_transaction_id])
	{
		m_oldest_transaction_id = (m_oldest_transaction_id + 1)
			% max_transactions;
		if (m_oldest_transaction_id == m_next_transaction_id)
			break;
	}
}

void rpc_manager::invoke(int message_id, udp::endpoint target_addr
	, observer_ptr o)
{
	INVARIANT_CHECK;

	if (m_destructing)
	{
		o->abort();
		return;
	}

	msg m;
	m.message_id = message_id;
	m.reply = false;
	m.id = m_our_id;
	m.addr = target_addr;
	assert(!m_transactions[m_next_transaction_id]);
#ifndef NDEBUG
	int potential_new_id = m_next_transaction_id;
#endif
	try
	{
		m.transaction_id.clear();
		std::back_insert_iterator<std::string> out(m.transaction_id);
		io::write_uint16(m_next_transaction_id, out);

		o->send(m);

		o->sent = time_now();
		o->target_addr = target_addr;

	#ifdef TORRENT_DHT_VERBOSE_LOGGING
		TORRENT_LOG(rpc) << "Invoking " << messages::ids[message_id]
			<< " -> " << target_addr;
	#endif
		m_send(m);
		new_transaction_id(o);
	}
	catch (std::exception& e)
	{
		// m_send may fail with "no route to host"
		assert(potential_new_id == m_next_transaction_id);
		o->abort();
	}
}

void rpc_manager::reply(msg& m)
{
	INVARIANT_CHECK;

	if (m_destructing) return;

	assert(m.reply);
	m.piggy_backed_ping = false;
	m.id = m_our_id;

	m_send(m);
}

void rpc_manager::reply_with_ping(msg& m)
{
	INVARIANT_CHECK;

	if (m_destructing) return;
	assert(m.reply);

	m.piggy_backed_ping = true;
	m.id = m_our_id;

	m.ping_transaction_id.clear();
	std::back_insert_iterator<std::string> out(m.ping_transaction_id);
	io::write_uint16(m_next_transaction_id, out);

	observer_ptr o(new (allocator().malloc()) null_observer(allocator()));
	assert(!m_transactions[m_next_transaction_id]);
	o->sent = time_now();
	o->target_addr = m.addr;

	m_send(m);
	new_transaction_id(o);
}

} } // namespace libtorrent::dht

<commit_msg>DHT optimization by using unordered free instead of ordered free on the pool<commit_after>/*

Copyright (c) 2006, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include "libtorrent/pch.hpp"
#include "libtorrent/socket.hpp"

#include <boost/bind.hpp>
#include <boost/mpl/max_element.hpp>
#include <boost/mpl/vector.hpp>
#include <boost/mpl/sizeof.hpp>
#include <boost/mpl/transform_view.hpp>
#include <boost/mpl/deref.hpp>
#include <boost/lexical_cast.hpp>

#include <libtorrent/io.hpp>
#include <libtorrent/invariant_check.hpp>
#include <libtorrent/kademlia/rpc_manager.hpp>
#include <libtorrent/kademlia/logging.hpp>
#include <libtorrent/kademlia/routing_table.hpp>
#include <libtorrent/kademlia/find_data.hpp>
#include <libtorrent/kademlia/closest_nodes.hpp>
#include <libtorrent/kademlia/refresh.hpp>
#include <libtorrent/kademlia/node.hpp>
#include <libtorrent/kademlia/observer.hpp>
#include <libtorrent/hasher.hpp>

#include <fstream>

using boost::shared_ptr;
using boost::bind;

namespace libtorrent { namespace dht
{

namespace io = libtorrent::detail;
namespace mpl = boost::mpl;

#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_DEFINE_LOG(rpc)
#endif

void intrusive_ptr_add_ref(observer const* o)
{
	assert(o->m_refs >= 0);
	assert(o != 0);
	++o->m_refs;
}

void intrusive_ptr_release(observer const* o)
{
	assert(o->m_refs > 0);
	assert(o != 0);
	if (--o->m_refs == 0)
	{
		boost::pool<>& p = o->pool_allocator;
		o->~observer();
		p.free(const_cast<observer*>(o));
	}
}

node_id generate_id();

typedef mpl::vector<
	closest_nodes_observer
	, find_data_observer
	, announce_observer
	, get_peers_observer
	, refresh_observer
	, ping_observer
	, null_observer
	> observer_types;

typedef mpl::max_element<
	mpl::transform_view<observer_types, mpl::sizeof_<mpl::_1> >
    >::type max_observer_type_iter;

rpc_manager::rpc_manager(fun const& f, node_id const& our_id
	, routing_table& table, send_fun const& sf)
	: m_pool_allocator(sizeof(mpl::deref<max_observer_type_iter::base>::type))
	, m_next_transaction_id(rand() % max_transactions)
	, m_oldest_transaction_id(m_next_transaction_id)
	, m_incoming(f)
	, m_send(sf)
	, m_our_id(our_id)
	, m_table(table)
	, m_timer(time_now())
	, m_random_number(generate_id())
	, m_destructing(false)
{
	std::srand(time(0));
}

rpc_manager::~rpc_manager()
{
	m_destructing = true;
#ifdef TORRENT_DHT_VERBOSE_LOGGING
	TORRENT_LOG(rpc) << "Destructing";
#endif
	std::for_each(m_aborted_transactions.begin(), m_aborted_transactions.end()
		, bind(&observer::abort, _1));

	for (transactions_t::iterator i = m_transactions.begin()
		, end(m_transactions.end()); i != end; ++i)
	{
		if (*i) (*i)->abort();
	}
}

#ifndef NDEBUG
void rpc_manager::check_invariant() const
{
	assert(m_oldest_transaction_id >= 0);
	assert(m_oldest_transaction_id < max_transactions);
	assert(m_next_transaction_id >= 0);
	assert(m_next_transaction_id < max_transactions);
	assert(!m_transactions[m_next_transaction_id]);

	for (int i = (m_next_transaction_id + 1) % max_transactions;
		i != m_oldest_transaction_id; i = (i + 1) % max_transactions)
	{
		assert(!m_transactions[i]);
	}
}
#endif

bool rpc_manager::incoming(msg const& m)
{
	INVARIANT_CHECK;

	if (m_destructing) return false;

	if (m.reply)
	{
		// if we don't have the transaction id in our
		// request list, ignore the packet

		if (m.transaction_id.size() < 2)
		{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
			TORRENT_LOG(rpc) << "Reply with invalid transaction id size: "
				<< m.transaction_id.size() << " from " << m.addr;
#endif
			msg reply;
			reply.reply = true;
			reply.message_id = messages::error;
			reply.error_code = 203; // Protocol error
			reply.error_msg = "reply with invalid transaction id, size "
				+ boost::lexical_cast<std::string>(m.transaction_id.size());
			reply.addr = m.addr;
			reply.transaction_id = "";
			m_send(reply);
			return false;
		}

		std::string::const_iterator i = m.transaction_id.begin();
		int tid = io::read_uint16(i);

		if (tid >= (int)m_transactions.size()
			|| tid < 0)
		{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
			TORRENT_LOG(rpc) << "Reply with invalid transaction id: "
				<< tid << " from " << m.addr;
#endif
			msg reply;
			reply.reply = true;
			reply.message_id = messages::error;
			reply.error_code = 203; // Protocol error
			reply.error_msg = "reply with invalid transaction id";
			reply.addr = m.addr;
			reply.transaction_id = "";
			m_send(reply);
			return false;
		}

		observer_ptr o = m_transactions[tid];

		if (!o)
		{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
			TORRENT_LOG(rpc) << "Reply with unknown transaction id: "
				<< tid << " from " << m.addr << " (possibly timed out)";
#endif
			return false;
		}

		if (m.addr != o->target_addr)
		{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
			TORRENT_LOG(rpc) << "Reply with incorrect address and valid transaction id: "
				<< tid << " from " << m.addr;
#endif
			return false;
		}

#ifdef TORRENT_DHT_VERBOSE_LOGGING
		std::ofstream reply_stats("libtorrent_logs/round_trip_ms.log", std::ios::app);
		reply_stats << m.addr << "\t" << total_milliseconds(time_now() - o->sent)
			<< std::endl;
#endif
		o->reply(m);
		m_transactions[tid] = 0;

		if (m.piggy_backed_ping)
		{
			// there is a ping request piggy
			// backed in this reply
			msg ph;
			ph.message_id = messages::ping;
			ph.transaction_id = m.ping_transaction_id;
			ph.addr = m.addr;
			ph.reply = true;

			reply(ph);
		}
		return m_table.node_seen(m.id, m.addr);
	}
	else
	{
		assert(m.message_id != messages::error);
		// this is an incoming request
		m_incoming(m);
	}
	return false;
}

time_duration rpc_manager::tick()
{
	INVARIANT_CHECK;

	const int timeout_ms = 10 * 1000;

	//	look for observers that has timed out

	if (m_next_transaction_id == m_oldest_transaction_id) return milliseconds(timeout_ms);

	std::vector<observer_ptr > timeouts;

	for (;m_next_transaction_id != m_oldest_transaction_id;
		m_oldest_transaction_id = (m_oldest_transaction_id + 1) % max_transactions)
	{
		assert(m_oldest_transaction_id >= 0);
		assert(m_oldest_transaction_id < max_transactions);

		observer_ptr o = m_transactions[m_oldest_transaction_id];
		if (!o) continue;

		time_duration diff = o->sent + milliseconds(timeout_ms) - time_now();
		if (diff > seconds(0))
		{
			if (diff < seconds(1)) return seconds(1);
			return diff;
		}

		try
		{
			m_transactions[m_oldest_transaction_id] = 0;
			timeouts.push_back(o);
		} catch (std::exception) {}
	}

	std::for_each(timeouts.begin(), timeouts.end(), bind(&observer::timeout, _1));
	timeouts.clear();

	// clear the aborted transactions, will likely
	// generate new requests. We need to swap, since the
	// destrutors may add more observers to the m_aborted_transactions
	std::vector<observer_ptr >().swap(m_aborted_transactions);
	return milliseconds(timeout_ms);
}

unsigned int rpc_manager::new_transaction_id(observer_ptr o)
{
	INVARIANT_CHECK;

	unsigned int tid = m_next_transaction_id;
	m_next_transaction_id = (m_next_transaction_id + 1) % max_transactions;
	if (m_transactions[m_next_transaction_id])
	{
		// moving the observer into the set of aborted transactions
		// it will prevent it from spawning new requests right now,
		// since that would break the invariant
		m_aborted_transactions.push_back(m_transactions[m_next_transaction_id]);
		m_transactions[m_next_transaction_id] = 0;
		assert(m_oldest_transaction_id == m_next_transaction_id);
	}
	assert(!m_transactions[tid]);
	m_transactions[tid] = o;
	if (m_oldest_transaction_id == m_next_transaction_id)
	{
		m_oldest_transaction_id = (m_oldest_transaction_id + 1) % max_transactions;
#ifdef TORRENT_DHT_VERBOSE_LOGGING
		TORRENT_LOG(rpc) << "WARNING: transaction limit reached! Too many concurrent"
			" messages! limit: " << (int)max_transactions;
#endif
		update_oldest_transaction_id();
	}

	return tid;
}

void rpc_manager::update_oldest_transaction_id()
{
	INVARIANT_CHECK;

	assert(m_oldest_transaction_id != m_next_transaction_id);
	while (!m_transactions[m_oldest_transaction_id])
	{
		m_oldest_transaction_id = (m_oldest_transaction_id + 1)
			% max_transactions;
		if (m_oldest_transaction_id == m_next_transaction_id)
			break;
	}
}

void rpc_manager::invoke(int message_id, udp::endpoint target_addr
	, observer_ptr o)
{
	INVARIANT_CHECK;

	if (m_destructing)
	{
		o->abort();
		return;
	}

	msg m;
	m.message_id = message_id;
	m.reply = false;
	m.id = m_our_id;
	m.addr = target_addr;
	assert(!m_transactions[m_next_transaction_id]);
#ifndef NDEBUG
	int potential_new_id = m_next_transaction_id;
#endif
	try
	{
		m.transaction_id.clear();
		std::back_insert_iterator<std::string> out(m.transaction_id);
		io::write_uint16(m_next_transaction_id, out);

		o->send(m);

		o->sent = time_now();
		o->target_addr = target_addr;

	#ifdef TORRENT_DHT_VERBOSE_LOGGING
		TORRENT_LOG(rpc) << "Invoking " << messages::ids[message_id]
			<< " -> " << target_addr;
	#endif
		m_send(m);
		new_transaction_id(o);
	}
	catch (std::exception& e)
	{
		// m_send may fail with "no route to host"
		assert(potential_new_id == m_next_transaction_id);
		o->abort();
	}
}

void rpc_manager::reply(msg& m)
{
	INVARIANT_CHECK;

	if (m_destructing) return;

	assert(m.reply);
	m.piggy_backed_ping = false;
	m.id = m_our_id;

	m_send(m);
}

void rpc_manager::reply_with_ping(msg& m)
{
	INVARIANT_CHECK;

	if (m_destructing) return;
	assert(m.reply);

	m.piggy_backed_ping = true;
	m.id = m_our_id;

	m.ping_transaction_id.clear();
	std::back_insert_iterator<std::string> out(m.ping_transaction_id);
	io::write_uint16(m_next_transaction_id, out);

	observer_ptr o(new (allocator().malloc()) null_observer(allocator()));
	assert(!m_transactions[m_next_transaction_id]);
	o->sent = time_now();
	o->target_addr = m.addr;

	m_send(m);
	new_transaction_id(o);
}

} } // namespace libtorrent::dht

<|endoftext|>


<commit_before>/******************************************************************************
** Copyright (c) 2014-2015, Intel Corporation                                **
** All rights reserved.                                                      **
**                                                                           **
** Redistribution and use in source and binary forms, with or without        **
** modification, are permitted provided that the following conditions        **
** are met:                                                                  **
** 1. Redistributions of source code must retain the above copyright         **
**    notice, this list of conditions and the following disclaimer.          **
** 2. Redistributions in binary form must reproduce the above copyright      **
**    notice, this list of conditions and the following disclaimer in the    **
**    documentation and/or other materials provided with the distribution.   **
** 3. Neither the name of the copyright holder nor the names of its          **
**    contributors may be used to endorse or promote products derived        **
**    from this software without specific prior written permission.          **
**                                                                           **
** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS       **
** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT         **
** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR     **
** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT      **
** HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,    **
** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED  **
** TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR    **
** PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF    **
** LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING      **
** NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS        **
** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.              **
******************************************************************************/
/* Hans Pabst (Intel Corp.)
******************************************************************************/
#if defined(LIBXSTREAM_EXPORTED) || defined(__LIBXSTREAM)
#include "libxstream_workqueue.hpp"
#include "libxstream_workitem.hpp"

#include <libxstream_begin.h>
#include <algorithm>
#include <cstdio>
#if defined(LIBXSTREAM_STDFEATURES)
# include <atomic>
#endif
#include <libxstream_end.h>

void libxstream_workqueue::entry_type::push(libxstream_workitem& workitem)
{
  delete m_dangling;
  const bool async = 0 == (LIBXSTREAM_CALL_WAIT & workitem.flags());
  const bool enone = 0 == (LIBXSTREAM_CALL_ERROR & workitem.flags());
  libxstream_workitem *const item = async ? workitem.clone() : &workitem;
  m_status = enone ? LIBXSTREAM_ERROR_NONE : LIBXSTREAM_ERROR_CONDITION;
  m_dangling = async ? item : 0;
  m_item = item;
}

int libxstream_workqueue::entry_type::wait() const
{
#if defined(LIBXSTREAM_SLEEP_CLIENT)
  size_t cycle = 0;
  while (0 != m_item) this_thread_wait(cycle);
#else
  while (0 != m_item) this_thread_yield();
#endif
  const int result = m_status;
  m_status = LIBXSTREAM_ERROR_NONE;
  LIBXSTREAM_ASSERT(LIBXSTREAM_ERROR_NONE == result);
  return result;
}

void libxstream_workqueue::entry_type::execute()
{
  (*m_item)(*this);
}

void libxstream_workqueue::entry_type::pop()
{
  m_item = 0;
  LIBXSTREAM_ASSERT(0 != m_queue);
  m_queue->pop();
}

libxstream_workqueue::libxstream_workqueue()
  : m_index(0)
#if defined(LIBXSTREAM_STDFEATURES)
  , m_size(new std::atomic<size_t>(0))
#else
  , m_size(new size_t(0))
#endif
{
  std::fill_n(m_buffer, LIBXSTREAM_MAX_QSIZE, entry_type(this, 0));
}

libxstream_workqueue::~libxstream_workqueue()
{
#if defined(LIBXSTREAM_DEBUG)
  size_t pending = 0;
  for (size_t i = 0; i < LIBXSTREAM_MAX_QSIZE; ++i) {
    pending += (0 == m_buffer[i].item()) ? 0 : 1;
    delete m_buffer[i].dangling();
  }
  if (0 < pending) {
    LIBXSTREAM_PRINT(1, "%lu work item%s pending!", static_cast<unsigned long>(pending), 1 < pending ? "s are" : " is");
  }
#endif
#if defined(LIBXSTREAM_STDFEATURES)
  delete static_cast<std::atomic<size_t>*>(m_size);
#else
  delete static_cast<size_t*>(m_size);
#endif
}

size_t libxstream_workqueue::size() const
{
  const size_t index = m_index;
#if defined(LIBXSTREAM_STDFEATURES)
  const size_t offset = *static_cast<const std::atomic<size_t>*>(m_size);
#else
  const size_t offset = *static_cast<const size_t*>(m_size);
#endif
  const entry_type& entry = m_buffer[LIBXSTREAM_MOD(index, LIBXSTREAM_MAX_QSIZE)];
  return 0 != entry.item() ? (offset - index) : (std::max<size_t>(offset - index, 1) - 1);
}

libxstream_workqueue::entry_type& libxstream_workqueue::allocate_entry_mt()
{
  entry_type* result = 0;
#if defined(LIBXSTREAM_STDFEATURES)
  result = m_buffer + LIBXSTREAM_MOD((*static_cast<std::atomic<size_t>*>(m_size))++, LIBXSTREAM_MAX_QSIZE);
#elif defined(_OPENMP)
  size_t size1 = 0;
  size_t& size = *static_cast<size_t*>(m_size);
# if (201107 <= _OPENMP)
#   pragma omp atomic capture
# else
#   pragma omp critical
# endif
  size1 = ++size;
  result = m_buffer + LIBXSTREAM_MOD(size1 - 1, LIBXSTREAM_MAX_QSIZE);
#else // generic
  size_t& size = *static_cast<size_t*>(m_size);
  libxstream_lock *const lock = libxstream_lock_get(this);
  libxstream_lock_acquire(lock);
  result = m_buffer + LIBXSTREAM_MOD(size++, LIBXSTREAM_MAX_QSIZE);
  libxstream_lock_release(lock);
#endif
  LIBXSTREAM_ASSERT(0 != result && result->queue() == this);

  if (0 != result->item()) {
    LIBXSTREAM_PRINT0(1, "queuing work is stalled!");
    do { // stall if capacity is exceeded
      this_thread_sleep();
    }
    while (0 != result->item());
  }

  return *result;
}

libxstream_workqueue::entry_type& libxstream_workqueue::allocate_entry()
{
  entry_type* result = 0;
#if defined(LIBXSTREAM_STDFEATURES)
  result = m_buffer + LIBXSTREAM_MOD(static_cast<std::atomic<size_t>*>(m_size)->fetch_add(1, std::memory_order_relaxed), LIBXSTREAM_MAX_QSIZE);
#else
  size_t& size = *static_cast<size_t*>(m_size);
  result = m_buffer + LIBXSTREAM_MOD(size++, LIBXSTREAM_MAX_QSIZE);
#endif
  LIBXSTREAM_ASSERT(0 != result && result->queue() == this);

  if (0 != result->item()) {
    LIBXSTREAM_PRINT0(1, "queuing work is stalled!");
    do { // stall if capacity is exceeded
      this_thread_sleep();
    }
    while (0 != result->item());
  }

  return *result;
}

#endif // defined(LIBXSTREAM_EXPORTED) || defined(__LIBXSTREAM)
<commit_msg>Introduced assertion.<commit_after>/******************************************************************************
** Copyright (c) 2014-2015, Intel Corporation                                **
** All rights reserved.                                                      **
**                                                                           **
** Redistribution and use in source and binary forms, with or without        **
** modification, are permitted provided that the following conditions        **
** are met:                                                                  **
** 1. Redistributions of source code must retain the above copyright         **
**    notice, this list of conditions and the following disclaimer.          **
** 2. Redistributions in binary form must reproduce the above copyright      **
**    notice, this list of conditions and the following disclaimer in the    **
**    documentation and/or other materials provided with the distribution.   **
** 3. Neither the name of the copyright holder nor the names of its          **
**    contributors may be used to endorse or promote products derived        **
**    from this software without specific prior written permission.          **
**                                                                           **
** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS       **
** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT         **
** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR     **
** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT      **
** HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,    **
** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED  **
** TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR    **
** PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF    **
** LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING      **
** NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS        **
** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.              **
******************************************************************************/
/* Hans Pabst (Intel Corp.)
******************************************************************************/
#if defined(LIBXSTREAM_EXPORTED) || defined(__LIBXSTREAM)
#include "libxstream_workqueue.hpp"
#include "libxstream_workitem.hpp"

#include <libxstream_begin.h>
#include <algorithm>
#include <cstdio>
#if defined(LIBXSTREAM_STDFEATURES)
# include <atomic>
#endif
#include <libxstream_end.h>

void libxstream_workqueue::entry_type::push(libxstream_workitem& workitem)
{
  delete m_dangling;
  const bool async = 0 == (LIBXSTREAM_CALL_WAIT & workitem.flags());
  const bool enone = 0 == (LIBXSTREAM_CALL_ERROR & workitem.flags());
  libxstream_workitem *const item = async ? workitem.clone() : &workitem;
  m_status = enone ? LIBXSTREAM_ERROR_NONE : LIBXSTREAM_ERROR_CONDITION;
  m_dangling = async ? item : 0;
  m_item = item;
}

int libxstream_workqueue::entry_type::wait() const
{
#if defined(LIBXSTREAM_SLEEP_CLIENT)
  size_t cycle = 0;
  while (0 != m_item) this_thread_wait(cycle);
#else
  while (0 != m_item) this_thread_yield();
#endif
  const int result = m_status;
  m_status = LIBXSTREAM_ERROR_NONE;
  LIBXSTREAM_ASSERT(LIBXSTREAM_ERROR_NONE == result);
  return result;
}

void libxstream_workqueue::entry_type::execute()
{
  LIBXSTREAM_ASSERT(0 != m_item && (m_item == m_dangling || 0 == m_dangling));
  (*m_item)(*this);
}

void libxstream_workqueue::entry_type::pop()
{
  m_item = 0;
  LIBXSTREAM_ASSERT(0 != m_queue);
  m_queue->pop();
}

libxstream_workqueue::libxstream_workqueue()
  : m_index(0)
#if defined(LIBXSTREAM_STDFEATURES)
  , m_size(new std::atomic<size_t>(0))
#else
  , m_size(new size_t(0))
#endif
{
  std::fill_n(m_buffer, LIBXSTREAM_MAX_QSIZE, entry_type(this, 0));
}

libxstream_workqueue::~libxstream_workqueue()
{
#if defined(LIBXSTREAM_DEBUG)
  size_t pending = 0;
  for (size_t i = 0; i < LIBXSTREAM_MAX_QSIZE; ++i) {
    pending += (0 == m_buffer[i].item()) ? 0 : 1;
    delete m_buffer[i].dangling();
  }
  if (0 < pending) {
    LIBXSTREAM_PRINT(1, "%lu work item%s pending!", static_cast<unsigned long>(pending), 1 < pending ? "s are" : " is");
  }
#endif
#if defined(LIBXSTREAM_STDFEATURES)
  delete static_cast<std::atomic<size_t>*>(m_size);
#else
  delete static_cast<size_t*>(m_size);
#endif
}

size_t libxstream_workqueue::size() const
{
  const size_t index = m_index;
#if defined(LIBXSTREAM_STDFEATURES)
  const size_t offset = *static_cast<const std::atomic<size_t>*>(m_size);
#else
  const size_t offset = *static_cast<const size_t*>(m_size);
#endif
  const entry_type& entry = m_buffer[LIBXSTREAM_MOD(index, LIBXSTREAM_MAX_QSIZE)];
  return 0 != entry.item() ? (offset - index) : (std::max<size_t>(offset - index, 1) - 1);
}

libxstream_workqueue::entry_type& libxstream_workqueue::allocate_entry_mt()
{
  entry_type* result = 0;
#if defined(LIBXSTREAM_STDFEATURES)
  result = m_buffer + LIBXSTREAM_MOD((*static_cast<std::atomic<size_t>*>(m_size))++, LIBXSTREAM_MAX_QSIZE);
#elif defined(_OPENMP)
  size_t size1 = 0;
  size_t& size = *static_cast<size_t*>(m_size);
# if (201107 <= _OPENMP)
#   pragma omp atomic capture
# else
#   pragma omp critical
# endif
  size1 = ++size;
  result = m_buffer + LIBXSTREAM_MOD(size1 - 1, LIBXSTREAM_MAX_QSIZE);
#else // generic
  size_t& size = *static_cast<size_t*>(m_size);
  libxstream_lock *const lock = libxstream_lock_get(this);
  libxstream_lock_acquire(lock);
  result = m_buffer + LIBXSTREAM_MOD(size++, LIBXSTREAM_MAX_QSIZE);
  libxstream_lock_release(lock);
#endif
  LIBXSTREAM_ASSERT(0 != result && result->queue() == this);

  if (0 != result->item()) {
    LIBXSTREAM_PRINT0(1, "queuing work is stalled!");
    do { // stall if capacity is exceeded
      this_thread_sleep();
    }
    while (0 != result->item());
  }

  return *result;
}

libxstream_workqueue::entry_type& libxstream_workqueue::allocate_entry()
{
  entry_type* result = 0;
#if defined(LIBXSTREAM_STDFEATURES)
  result = m_buffer + LIBXSTREAM_MOD(static_cast<std::atomic<size_t>*>(m_size)->fetch_add(1, std::memory_order_relaxed), LIBXSTREAM_MAX_QSIZE);
#else
  size_t& size = *static_cast<size_t*>(m_size);
  result = m_buffer + LIBXSTREAM_MOD(size++, LIBXSTREAM_MAX_QSIZE);
#endif
  LIBXSTREAM_ASSERT(0 != result && result->queue() == this);

  if (0 != result->item()) {
    LIBXSTREAM_PRINT0(1, "queuing work is stalled!");
    do { // stall if capacity is exceeded
      this_thread_sleep();
    }
    while (0 != result->item());
  }

  return *result;
}

#endif // defined(LIBXSTREAM_EXPORTED) || defined(__LIBXSTREAM)
<|endoftext|>


<commit_before>/****************************************************************************
**
** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
** All rights reserved.
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** This file is part of the Qt Mobility Components.
**
** $QT_BEGIN_LICENSE:LGPL$
** No Commercial Usage
** This file contains pre-release code and may not be distributed.
** You may use this file in accordance with the terms and conditions
** contained in the Technology Preview License Agreement accompanying
** this package.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Nokia gives you certain additional
** rights.  These rights are described in the Nokia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** If you have questions regarding the use of this file, please contact
** Nokia at qt-info@nokia.com.
**
**
**
**
**
**
**
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include "qlandmarkid.h"
#include "qlandmarkid_p.h"
#include <QString>

QTM_USE_NAMESPACE

/*!
    \class QLandmarkId
    \ingroup location

    \brief The QLandmarkId class provides a unique identifier for
    a category.

    It consists of a manager URI which identifies the manager which contains
    the category, and the id for that category.
*/

/*!

    Creates an invalid identifier, id() and managerUri() will return null strings
*/
QLandmarkId::QLandmarkId()
    :d(new QLandmarkIdPrivate)
{
}

/*!
    Constructs a copy of \a other.
*/
QLandmarkId::QLandmarkId(const QLandmarkId &other)
    :d(other.d)
{
}

/*!
    Destroys the identifier.
*/
QLandmarkId::~QLandmarkId()
{
}

/*!
    Returns true if the identifier is valid, meaning that both
    a manager URI and id have been set
*/
bool QLandmarkId::isValid() const
{
    return (!d->m_databaseName.isEmpty() && !d->m_id.isEmpty());
}

/*!
    Returns the id of the category identifier.
*/
QString QLandmarkId::id() const
{
    return d->m_id;
}

/*!
    Sets the \a id of the category identifier
*/
void QLandmarkId::setId(const QString &id)
{
    d->m_id = id;
}

/*!
    Returns the URI of the manager which this category identifer
    belongs to.
*/
QString QLandmarkId::managerUri() const
{
    return d->m_databaseName;
}

/*!
    Sets the manager \a uri which this category identifier
    belongs to.
*/
void QLandmarkId::setManagerUri(const QString &uri)
{
    d->m_databaseName = uri;
}

/*!
    Assigns \a other to this category identifier and returns a reference
    to this category identifier.
*/
QLandmarkId &QLandmarkId::operator=(const QLandmarkId &other)
{
    d = other.d;
    return *this;
}

/*!
    Returns true if this category identifer is equal to \a other, otherwise
    returns false.

    Two QLandmarkIds are considered equal if both have the same
    manager URI and id.
*/
bool QLandmarkId::operator==(const QLandmarkId &other) const
{
    return ((d->m_id == other.d->m_id)
            && (d->m_databaseName == other.d->m_databaseName));
}

/*!
    Returns true if this category identifier is not equal to \a other,
    otherwise returns false.

    If either the manager URIs or ids differ, then the
    QLandmarkIds are not considered equal.
*/
bool QLandmarkId::operator!=(const QLandmarkId &other) const
{
    return !this->operator ==(other);
}

<commit_msg>fix qlandmarkid docs.<commit_after>/****************************************************************************
**
** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
** All rights reserved.
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** This file is part of the Qt Mobility Components.
**
** $QT_BEGIN_LICENSE:LGPL$
** No Commercial Usage
** This file contains pre-release code and may not be distributed.
** You may use this file in accordance with the terms and conditions
** contained in the Technology Preview License Agreement accompanying
** this package.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Nokia gives you certain additional
** rights.  These rights are described in the Nokia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** If you have questions regarding the use of this file, please contact
** Nokia at qt-info@nokia.com.
**
**
**
**
**
**
**
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include "qlandmarkid.h"
#include "qlandmarkid_p.h"
#include <QString>

QTM_USE_NAMESPACE

/*!
    \class QLandmarkId
    \ingroup location

    \brief The QLandmarkId class provides a unique identifier for
    a landmark.

    It consists of a manager URI which identifies the manager which contains
    the landmark, and a manager specific id for that landmark.
*/

/*!

    Creates an invalid identifier.

     id() and managerUri() will return null strings
*/
QLandmarkId::QLandmarkId()
    :d(new QLandmarkIdPrivate)
{
}

/*!
    Constructs a copy of \a other.
*/
QLandmarkId::QLandmarkId(const QLandmarkId &other)
    :d(other.d)
{
}

/*!
    Destroys the identifier.
*/
QLandmarkId::~QLandmarkId()
{
}

/*!
    Returns true if the identifier is valid, meaning that both
    a manager URI and id have been set.
*/
bool QLandmarkId::isValid() const
{
    return (!d->m_databaseName.isEmpty() && !d->m_id.isEmpty());
}

/*!
    Returns the id of the landmark identifier.

    This id is specific to a particular landmark manager.
*/
QString QLandmarkId::id() const
{
    return d->m_id;
}

/*!
    Sets the \a id of the landmark identifier.
*/
void QLandmarkId::setId(const QString &id)
{
    d->m_id = id;
}

/*!
    Returns the URI of the manager which this landmark identifer
    belongs to.
*/
QString QLandmarkId::managerUri() const
{
    return d->m_databaseName;
}

/*!
    Sets the manager \a uri which this landmark identifier
    belongs to.
*/
void QLandmarkId::setManagerUri(const QString &uri)
{
    d->m_databaseName = uri;
}

/*!
    Assigns \a other to this landmark identifier and returns a reference
    to this landmark identifier.
*/
QLandmarkId &QLandmarkId::operator=(const QLandmarkId &other)
{
    d = other.d;
    return *this;
}

/*!
    Returns true if this landmark identifer is equal to \a other, otherwise
    returns false.

    Two QLandmarkIds are considered equal if both have the same
    manager URI and id.

    \sa operator!=()
*/
bool QLandmarkId::operator==(const QLandmarkId &other) const
{
    return ((d->m_id == other.d->m_id)
            && (d->m_databaseName == other.d->m_databaseName));
}

/*!
    Returns true if this landmark identifier is not equal to \a other,
    otherwise returns false.

    If either the manager URIs or ids differ, then the
    QLandmarkIds are not considered equal.

    \sa operator==()
*/
bool QLandmarkId::operator!=(const QLandmarkId &other) const
{
    return !this->operator ==(other);
}

<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "logjoin/LogJoinUpload.h"
#include "fnord-base/uri.h"
#include "fnord-base/util/binarymessagewriter.h"
#include "fnord-http/httprequest.h"

using namespace fnord;

namespace cm {

LogJoinUpload::LogJoinUpload(
    RefPtr<mdb::MDB> db,
    const String& feedserver_url,
    http::HTTPConnectionPool* http) :
    db_(db),
    feedserver_url_(feedserver_url),
    http_(http),
    batch_size_(kDefaultBatchSize) {}

void LogJoinUpload::upload() {
  while (scanQueue("__uploadq-sessions") > 0);
}

size_t LogJoinUpload::scanQueue(const String& queue_name) {
  auto txn = db_->startTransaction();
  auto cursor = txn->getCursor();

  Buffer key;
  Buffer value;
  bool eof = false;
  Vector<Buffer> batch;
  for (int i = 0; i < batch_size_ && !eof; ++i) {
    if (i == 0) {
      key.append(queue_name);
      eof = !cursor->getFirstOrGreater(&key, &value);
    } else {
      eof = !cursor->getNext(&key, &value);
    }

    if (!StringUtil::beginsWith(key.toString(), queue_name)) {
      eof = true;
    }

    if (!eof) {
      batch.emplace_back(value);
      txn->del(key);
    }
  }

  cursor->close();

  try {
    if (batch.size() > 0) {
      uploadBatch(queue_name, batch);
    }
  } catch (...) {
    txn->abort();
    throw;
  }

  txn->commit();

  return batch.size();
}

void LogJoinUpload::uploadBatch(
    const String& queue_name,
    const Vector<Buffer>& batch) {
  URI uri(feedserver_url_ + "/tsdb/insert_batch?stream=joined_sessions.dawanda");

  http::HTTPRequest req(http::HTTPMessage::M_POST, uri.pathAndQuery());
  req.addHeader("Host", uri.hostAndPort());
  req.addHeader("Content-Type", "application/fnord-msg");

  util::BinaryMessageWriter body;
  for (const auto& b : batch) {
    body.append(b.data(), b.size());
  }

  req.addBody(body.data(), body.size());

  auto res = http_->executeRequest(req);
  res.wait();

  const auto& r = res.get();
  if (r.statusCode() != 201) {
    RAISEF(kRuntimeError, "received non-201 response: $0", r.body().toString());
  }
}

} // namespace cm

<commit_msg>use mdb_cursor_del as mdb_del has some kind of bug<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "logjoin/LogJoinUpload.h"
#include "fnord-base/uri.h"
#include "fnord-base/util/binarymessagewriter.h"
#include "fnord-http/httprequest.h"

using namespace fnord;

namespace cm {

LogJoinUpload::LogJoinUpload(
    RefPtr<mdb::MDB> db,
    const String& feedserver_url,
    http::HTTPConnectionPool* http) :
    db_(db),
    feedserver_url_(feedserver_url),
    http_(http),
    batch_size_(kDefaultBatchSize) {}

void LogJoinUpload::upload() {
  while (scanQueue("__uploadq-sessions") > 0);
}

size_t LogJoinUpload::scanQueue(const String& queue_name) {
  auto txn = db_->startTransaction();
  auto cursor = txn->getCursor();

  Buffer key;
  Buffer value;
  Vector<Buffer> batch;
  for (int i = 0; i < batch_size_; ++i) {
    if (i == 0) {
      key.append(queue_name);

      if (!cursor->getFirstOrGreater(&key, &value)) {
        break;
      }
    } else {
      if (!cursor->getNext(&key, &value)) {
        break;
      }
    }

    if (!StringUtil::beginsWith(key.toString(), queue_name)) {
      break;
    }

    batch.emplace_back(value);
    cursor->del();
  }

  cursor->close();

  try {
    if (batch.size() > 0) {
      uploadBatch(queue_name, batch);
    }
  } catch (...) {
    txn->abort();
    throw;
  }

  txn->commit();

  return batch.size();
}

void LogJoinUpload::uploadBatch(
    const String& queue_name,
    const Vector<Buffer>& batch) {
  URI uri(feedserver_url_ + "/tsdb/insert_batch?stream=joined_sessions.dawanda");

  http::HTTPRequest req(http::HTTPMessage::M_POST, uri.pathAndQuery());
  req.addHeader("Host", uri.hostAndPort());
  req.addHeader("Content-Type", "application/fnord-msg");

  util::BinaryMessageWriter body;
  for (const auto& b : batch) {
    body.append(b.data(), b.size());
  }

  req.addBody(body.data(), body.size());

  auto res = http_->executeRequest(req);
  res.wait();

  const auto& r = res.get();
  if (r.statusCode() != 201) {
    RAISEF(kRuntimeError, "received non-201 response: $0", r.body().toString());
  }
}

} // namespace cm

<|endoftext|>


<commit_before>#include "lxc_isolation_module.hpp"

#include <algorithm>

#include "foreach.hpp"

using std::cerr;
using std::cout;
using std::endl;
using std::list;
using std::make_pair;
using std::max;
using std::ostringstream;
using std::pair;
using std::queue;
using std::string;
using std::vector;

using boost::lexical_cast;
using boost::unordered_map;
using boost::unordered_set;

using namespace nexus;
using namespace nexus::internal;
using namespace nexus::internal::slave;

LxcIsolationModule::LxcIsolationModule(Slave* slave)
{
  this->slave = slave;
  reaper = new Reaper(this);
  Process::spawn(reaper);
}

LxcIsolationModule::~LxcIsolationModule()
{
  // We want to wait until the reaper has completed because it
  // accesses 'this' in order to make callbacks ... deleting 'this'
  // could thus lead to a seg fault!
  Process::post(reaper->getPID(), SHUTDOWN_REAPER);
  Process::wait(reaper);
  delete reaper;
}

void LxcIsolationModule::frameworkAdded(Framework* framework)
{
  lxcExecutePid[framework->id] = -1;
  container[framework->id] = "";
  framework->executorStatus = "No executor running";
}

void LxcIsolationModule::frameworkRemoved(Framework *framework)
{
  lxcExecutePid.erase(framework->id);
  container.erase(framework->id);
}

void LxcIsolationModule::startExecutor(Framework *fw)
{
  LOG(INFO) << "Starting executor for framework " << fw->id << ": "
            << fw->executorInfo.uri;
  CHECK(lxcExecutePid[fw->id] == -1 && container[fw->id] == "");

  // Get location of Nexus install in order to find nexus-launcher.
  const char *nexusHome = getenv("NEXUS_HOME");
  if (!nexusHome)
    nexusHome = "..";
  string nexusLauncher = string(nexusHome) + "/src/nexus-launcher";

  // Create a name for the container
  ostringstream oss;
  oss << "nexus.slave-" << slave->id << ".framework-" << fw->id;
  string containerName = oss.str();

  container[fw->id] = containerName;
  fw->executorStatus = "Container: " + containerName;

  // Run lxc-execute nexus-launcher using a fork-exec (since lxc-execute
  // does not return until the container is finished). Note that lxc-execute
  // automatically creates the container and will delete it when finished.
  pid_t pid;
  if ((pid = fork()) == -1)
    PLOG(FATAL) << "Failed to fork to launch lxc-execute";

  if (pid) {
    // In parent process
    lxcExecutePid[fw->id] = pid;
    LOG(INFO) << "Started lxc-execute, pid = " << pid;
    int status;
  } else {
    // Set up environment variables.
    setenv("NEXUS_FRAMEWORK_ID", lexical_cast<string>(fw->id).c_str(), 1);
    setenv("NEXUS_EXECUTOR_URI", fw->executorInfo.uri.c_str(), 1);
    setenv("NEXUS_USER", fw->user.c_str(), 1);
    setenv("NEXUS_SLAVE_PID", lexical_cast<string>(slave->self()).c_str(), 1);
    setenv("NEXUS_REDIRECT_IO", slave->local ? "1" : "0", 1);
    setenv("NEXUS_WORK_DIRECTORY", slave->getWorkDirectory(fw->id).c_str(), 1);

    // Run lxc-execute.
    execlp("lxc-execute", "lxc-execute", "-n", containerName.c_str(),
           nexusLauncher.c_str(), (char *) NULL);
    // If we get here, the execl call failed.
    fatalerror("Could not exec lxc-execute");
    // TODO: Exit the slave if this happens
  }
}

void LxcIsolationModule::killExecutor(Framework* fw)
{
  if (container[fw->id] != "") {
    LOG(INFO) << "Stopping container " << container[fw->id];
    int ret = shell("lxc-stop -n %s", container[fw->id].c_str());
    if (ret != 0)
      LOG(ERROR) << "lxc-stop returned " << ret;
    container[fw->id] = "";
    fw->executorStatus = "No executor running";
  }
}

void LxcIsolationModule::resourcesChanged(Framework* fw)
{
  if (container[fw->id] != "") {
    // For now, just try setting the CPUs and mem right away.
    // A slightly smarter thing might be to only update them periodically.
    int ret;

    int32_t cpuShares = max(1024 * fw->resources.cpus, 10);
    LOG(INFO) << "Setting CPU shares for " << fw->id << " to " << cpuShares;
    ret = shell("lxc-cgroup -n %s cpu.shares %d",
                container[fw->id].c_str(), cpuShares);
    if (ret != 0)
      LOG(ERROR) << "lxc-cgroup returned " << ret;

    int64_t rssLimit = max(fw->resources.mem, 128 * Megabyte);
    LOG(INFO) << "Setting RSS limit for " << fw->id << " to " << rssLimit;
    ret = shell("lxc-cgroup -n %s memory.limit_in_bytes %lld",
                container[fw->id].c_str(), rssLimit);
    if (ret != 0)
      LOG(ERROR) << "lxc-cgroup returned " << ret;

    // TODO: Decreasing the RSS limit will fail if the current RSS is too
    // large and memory can't be swapped out. In that case, we should
    // either freeze the container before changing RSS, or just kill it.
  }
}

int LxcIsolationModule::shell(const char* fmt, ...)
{
  char *cmd;
  FILE *f;
  int ret;
  va_list args;
  va_start(args, fmt);
  if (vasprintf(&cmd, fmt, args) == -1)
    return -1;
  if ((f = popen(cmd, "w")) == NULL)
    return -1;
  ret = pclose(f);
  if (ret == -1)
    LOG(INFO) << "pclose error: " << strerror(errno);
  free(cmd);
  va_end(args);
  return ret;
}

LxcIsolationModule::Reaper::Reaper(LxcIsolationModule* m)
  : module(m)
{}

void LxcIsolationModule::Reaper::operator () ()
{
  pause(1); // HACK(matei)
  link(module->slave->getPID());
  while (true) {
    switch (receive(1)) {
    case PROCESS_TIMEOUT: {
      // Check whether any child process has exited
      pid_t pid;
      int status;
      if ((pid = waitpid((pid_t) -1, &status, WNOHANG)) > 0) {
	foreachpair (FrameworkID fid, pid_t& fwPid, module->lxcExecutePid) {
	  if (fwPid == pid) {
	    module->container[fid] = "";
	    module->lxcExecutePid[fid] = -1;
	    LOG(INFO) << "Telling slave of lost framework " << fid;
	    module->slave->executorExited(fid, status);
	    break;
	  }
	}
      }
    }
    case SHUTDOWN_REAPER:
    case PROCESS_EXIT:
      return;
    }
  }
}
<commit_msg>Revert non-working fix<commit_after>#include "lxc_isolation_module.hpp"

#include <algorithm>

#include "foreach.hpp"

using std::cerr;
using std::cout;
using std::endl;
using std::list;
using std::make_pair;
using std::max;
using std::ostringstream;
using std::pair;
using std::queue;
using std::string;
using std::vector;

using boost::lexical_cast;
using boost::unordered_map;
using boost::unordered_set;

using namespace nexus;
using namespace nexus::internal;
using namespace nexus::internal::slave;

LxcIsolationModule::LxcIsolationModule(Slave* slave)
{
  this->slave = slave;
  reaper = new Reaper(this);
  Process::spawn(reaper);
}

LxcIsolationModule::~LxcIsolationModule()
{
  // We want to wait until the reaper has completed because it
  // accesses 'this' in order to make callbacks ... deleting 'this'
  // could thus lead to a seg fault!
  Process::post(reaper->getPID(), SHUTDOWN_REAPER);
  Process::wait(reaper);
  delete reaper;
}

void LxcIsolationModule::frameworkAdded(Framework* framework)
{
  lxcExecutePid[framework->id] = -1;
  container[framework->id] = "";
  framework->executorStatus = "No executor running";
}

void LxcIsolationModule::frameworkRemoved(Framework *framework)
{
  lxcExecutePid.erase(framework->id);
  container.erase(framework->id);
}

void LxcIsolationModule::startExecutor(Framework *fw)
{
  LOG(INFO) << "Starting executor for framework " << fw->id << ": "
            << fw->executorInfo.uri;
  CHECK(lxcExecutePid[fw->id] == -1 && container[fw->id] == "");

  // Get location of Nexus install in order to find nexus-launcher.
  const char *nexusHome = getenv("NEXUS_HOME");
  if (!nexusHome)
    nexusHome = "..";
  string nexusLauncher = string(nexusHome) + "/src/nexus-launcher";

  // Create a name for the container
  ostringstream oss;
  oss << "nexus.slave-" << slave->id << ".framework-" << fw->id;
  string containerName = oss.str();

  container[fw->id] = containerName;
  fw->executorStatus = "Container: " + containerName;

  // Run lxc-execute nexus-launcher using a fork-exec (since lxc-execute
  // does not return until the container is finished). Note that lxc-execute
  // automatically creates the container and will delete it when finished.
  pid_t pid;
  if ((pid = fork()) == -1)
    PLOG(FATAL) << "Failed to fork to launch lxc-execute";

  if (pid) {
    // In parent process
    lxcExecutePid[fw->id] = pid;
    LOG(INFO) << "Started lxc-execute, pid = " << pid;
    int status;
  } else {
    // Set up environment variables.
    setenv("NEXUS_FRAMEWORK_ID", lexical_cast<string>(fw->id).c_str(), 1);
    setenv("NEXUS_EXECUTOR_URI", fw->executorInfo.uri.c_str(), 1);
    setenv("NEXUS_USER", fw->user.c_str(), 1);
    setenv("NEXUS_SLAVE_PID", lexical_cast<string>(slave->self()).c_str(), 1);
    setenv("NEXUS_REDIRECT_IO", slave->local ? "1" : "0", 1);
    setenv("NEXUS_WORK_DIRECTORY", slave->getWorkDirectory(fw->id).c_str(), 1);

    // Run lxc-execute.
    execlp("lxc-execute", "lxc-execute", "-n", containerName.c_str(),
           nexusLauncher.c_str(), (char *) NULL);
    // If we get here, the execl call failed.
    fatalerror("Could not exec lxc-execute");
    // TODO: Exit the slave if this happens
  }
}

void LxcIsolationModule::killExecutor(Framework* fw)
{
  if (container[fw->id] != "") {
    LOG(INFO) << "Stopping container " << container[fw->id];
    int ret = shell("lxc-stop -n %s", container[fw->id].c_str());
    if (ret != 0)
      LOG(ERROR) << "lxc-stop returned " << ret;
    container[fw->id] = "";
    fw->executorStatus = "No executor running";
  }
}

void LxcIsolationModule::resourcesChanged(Framework* fw)
{
  if (container[fw->id] != "") {
    // For now, just try setting the CPUs and mem right away.
    // A slightly smarter thing might be to only update them periodically.
    int ret;

    int32_t cpuShares = max(1024 * fw->resources.cpus, 10);
    LOG(INFO) << "Setting CPU shares for " << fw->id << " to " << cpuShares;
    ret = shell("lxc-cgroup -n %s cpu.shares %d",
                container[fw->id].c_str(), cpuShares);
    if (ret != 0)
      LOG(ERROR) << "lxc-cgroup returned " << ret;

    int64_t rssLimit = max(fw->resources.mem, 128 * Megabyte);
    LOG(INFO) << "Setting RSS limit for " << fw->id << " to " << rssLimit;
    ret = shell("lxc-cgroup -n %s memory.limit_in_bytes %lld",
                container[fw->id].c_str(), rssLimit);
    if (ret != 0)
      LOG(ERROR) << "lxc-cgroup returned " << ret;

    // TODO: Decreasing the RSS limit will fail if the current RSS is too
    // large and memory can't be swapped out. In that case, we should
    // either freeze the container before changing RSS, or just kill it.
  }
}

int LxcIsolationModule::shell(const char* fmt, ...)
{
  char *cmd;
  FILE *f;
  int ret;
  va_list args;
  va_start(args, fmt);
  if (vasprintf(&cmd, fmt, args) == -1)
    return -1;
  if ((f = popen(cmd, "w")) == NULL)
    return -1;
  ret = pclose(f);
  if (ret == -1)
    LOG(INFO) << "pclose error: " << strerror(errno);
  free(cmd);
  va_end(args);
  return ret;
}

LxcIsolationModule::Reaper::Reaper(LxcIsolationModule* m)
  : module(m)
{}

void LxcIsolationModule::Reaper::operator () ()
{
  link(module->slave->getPID());
  while (true) {
    switch (receive(1)) {
    case PROCESS_TIMEOUT: {
      // Check whether any child process has exited
      pid_t pid;
      int status;
      if ((pid = waitpid((pid_t) -1, &status, WNOHANG)) > 0) {
	foreachpair (FrameworkID fid, pid_t& fwPid, module->lxcExecutePid) {
	  if (fwPid == pid) {
	    module->container[fid] = "";
	    module->lxcExecutePid[fid] = -1;
	    LOG(INFO) << "Telling slave of lost framework " << fid;
	    module->slave->executorExited(fid, status);
	    break;
	  }
	}
      }
    }
    case SHUTDOWN_REAPER:
    case PROCESS_EXIT:
      return;
    }
  }
}
<|endoftext|>