<commit_before>/*******************************************************************************
 * c7a/api/reduce_node.hpp
 *
 * DIANode for a reduce operation. Performs the actual reduce operation
 *
 * Part of Project c7a.
 *
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef C7A_API_REDUCE_NODE_HEADER
#define C7A_API_REDUCE_NODE_HEADER

#include <c7a/api/dop_node.hpp>
#include <c7a/api/context.hpp>
#include <c7a/api/function_stack.hpp>
#include <c7a/common/logger.hpp>
#include <c7a/core/reduce_pre_table.hpp>
#include <c7a/core/reduce_post_table.hpp>

#include <unordered_map>
#include <functional>
#include <string>
#include <vector>

namespace c7a {
//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a Reduce operation. Reduce groups the elements in a
 * DIA by their key and reduces every key bucket to a single element each. The
 * ReduceNode stores the key_extractor and the reduce_function UDFs. The
 * chainable LOps ahead of the Reduce operation are stored in the Stack. The
 * ReduceNode has the type Output, which is the result type of the
 * reduce_function.
 *
 * \tparam Input Input type of the Reduce operation
 * \tparam Output Output type of the Reduce operation
 * \tparam Stack Function stack, which contains the chained lambdas between the last and this DIANode.
 * \tparam KeyExtractor Type of the key_extractor function.
 * \tparam ReduceFunction Type of the reduce_function
 */
template <typename Input, typename Output, typename Stack, typename KeyExtractor, typename ReduceFunction>
class ReduceNode : public DOpNode<Output>
{
    static const bool debug = true;

    typedef DOpNode<Output> Super;

    using reduce_arg_t = typename FunctionTraits<ReduceFunction>::template arg<0>;

    using Super::context_;
    using Super::data_id_;

public:
    /*!
     * Constructor for a ReduceNode. Sets the DataManager, parent, stack,
     * key_extractor and reduce_function.
     *
     * \param ctx Reference to Context, which holds references to data and network.
     * \param parent Parent DIANode.
     * \param stack Function chain with all lambdas between the parent and this node
     * \param key_extractor Key extractor function
     * \param reduce_function Reduce function
     */
    ReduceNode(Context& ctx,
               DIANode<Input>* parent,
               Stack& stack,
               KeyExtractor key_extractor,
               ReduceFunction reduce_function)
        : DOpNode<Output>(ctx, { parent }),
          local_stack_(stack),
          key_extractor_(key_extractor),
          reduce_function_(reduce_function),
          channel_id_(ctx.get_data_manager().AllocateNetworkChannel()),
          reduce_pre_table_(ctx.number_worker(), key_extractor, reduce_function_, ctx.get_data_manager().template GetNetworkEmitters<Output>(channel_id_))
    {
        // Hook PreOp
        auto pre_op_fn = [ = ](reduce_arg_t input) {
                             PreOp(input);
                         };
        auto lop_chain = local_stack_.push(pre_op_fn).emit();

        parent->RegisterChild(lop_chain);
    }

    //! Virtual destructor for a ReduceNode.
    virtual ~ReduceNode() { }

    /*!
     * Actually executes the reduce operation. Uses the member functions PreOp,
     * MainOp and PostOp.
     */
    void execute() override {
        //Flush hash table to send data before main op begins
        reduce_pre_table_.Flush();
        reduce_pre_table_.CloseEmitter();

        MainOp();
    }

    /*!
     * Produces a function stack, which only contains the PostOp function.
     * \return PostOp function stack
     */
    auto ProduceStack() {
        // Hook PostOp
        auto post_op_fn = [ = ](Output elem, std::function<void(Output)> emit_func) {
                              return PostOp(elem, emit_func);
                          };

        FunctionStack<> stack;
        return stack.push(post_op_fn);
    }

    /*!
     * Returns "[ReduceNode]" and its id as a string.
     * \return "[ReduceNode]"
     */
    std::string ToString() override {
        return "[ReduceNode] Id: " + std::to_string(data_id_);
    }

private:
    //! Local stack
    Stack local_stack_;
    //!Key extractor function
    KeyExtractor key_extractor_;
    //!Reduce function
    ReduceFunction reduce_function_;

    data::ChannelId channel_id_;

    core::ReducePreTable<KeyExtractor, ReduceFunction, data::BlockEmitter<Output> > reduce_pre_table_;

    //! Locally hash elements of the current DIA onto buckets and reduce each
    //! bucket to a single value, afterwards send data to another worker given
    //! by the shuffle algorithm.
    void PreOp(reduce_arg_t input) {
        reduce_pre_table_.Insert(input);
    }

    //!Recieve elements from other workers.
    auto MainOp() {
        std::function<void(Output)> print =
            [](Output elem) {
                LOG << elem.first << " " << elem.second;
            };

        using ReduceTable
                  = core::ReducePostTable<KeyExtractor,
                                          ReduceFunction,
                                          std::function<void(Output)> >;

        ReduceTable table(key_extractor_, reduce_function_, DIANode<Output>::callbacks());

        auto it = context_.get_data_manager().template GetRemoteBlocks<Output>(channel_id_);

        while (!it.IsClosed()) {
            it.WaitForMore();
            while (it.HasNext()) {
                table.Insert(it.Next());
            }
        }

        table.Flush();
    }

    //! Hash recieved elements onto buckets and reduce each bucket to a single value.
    void PostOp(Output input, std::function<void(Output)> emit_func) {
        emit_func(input);
    }
};

//! \}
} // namespace c7a

#endif // !C7A_API_REDUCE_NODE_HEADER

/******************************************************************************/
<commit_msg>add comment on WriteNode<commit_after>/*******************************************************************************
 * c7a/api/reduce_node.hpp
 *
 * DIANode for a reduce operation. Performs the actual reduce operation
 *
 * Part of Project c7a.
 *
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef C7A_API_REDUCE_NODE_HEADER
#define C7A_API_REDUCE_NODE_HEADER

#include <c7a/api/dop_node.hpp>
#include <c7a/api/context.hpp>
#include <c7a/api/function_stack.hpp>
#include <c7a/common/logger.hpp>
#include <c7a/core/reduce_pre_table.hpp>
#include <c7a/core/reduce_post_table.hpp>

#include <unordered_map>
#include <functional>
#include <string>
#include <vector>

namespace c7a {
//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a Reduce operation. Reduce groups the elements in a
 * DIA by their key and reduces every key bucket to a single element each. The
 * ReduceNode stores the key_extractor and the reduce_function UDFs. The
 * chainable LOps ahead of the Reduce operation are stored in the Stack. The
 * ReduceNode has the type Output, which is the result type of the
 * reduce_function.
 *
 * \tparam Input Input type of the Reduce operation
 * \tparam Output Output type of the Reduce operation
 * \tparam Stack Function stack, which contains the chained lambdas between the last and this DIANode.
 * \tparam KeyExtractor Type of the key_extractor function.
 * \tparam ReduceFunction Type of the reduce_function
 */
template <typename Input, typename Output, typename Stack, typename KeyExtractor, typename ReduceFunction>
class ReduceNode : public DOpNode<Output>
{
    static const bool debug = true;

    typedef DOpNode<Output> Super;

    using reduce_arg_t = typename FunctionTraits<ReduceFunction>::template arg<0>;

    using Super::context_;
    using Super::data_id_;

public:
    /*!
     * Constructor for a ReduceNode. Sets the DataManager, parent, stack,
     * key_extractor and reduce_function.
     *
     * \param ctx Reference to Context, which holds references to data and network.
     * \param parent Parent DIANode.
     * \param stack Function chain with all lambdas between the parent and this node
     * \param key_extractor Key extractor function
     * \param reduce_function Reduce function
     */
    ReduceNode(Context& ctx,
               DIANode<Input>* parent,
               Stack& stack,
               KeyExtractor key_extractor,
               ReduceFunction reduce_function)
        : DOpNode<Output>(ctx, { parent }),
          local_stack_(stack),
          key_extractor_(key_extractor),
          reduce_function_(reduce_function),
          channel_id_(ctx.get_data_manager().AllocateNetworkChannel()),
          reduce_pre_table_(ctx.number_worker(), key_extractor, reduce_function_, ctx.get_data_manager().template GetNetworkEmitters<Output>(channel_id_))
    {
        // Hook PreOp
        auto pre_op_fn = [ = ](reduce_arg_t input) {
                             PreOp(input);
                         };
        auto lop_chain = local_stack_.push(pre_op_fn).emit();

        parent->RegisterChild(lop_chain);
    }

    //! Virtual destructor for a ReduceNode.
    virtual ~ReduceNode() { }

    /*!
     * Actually executes the reduce operation. Uses the member functions PreOp,
     * MainOp and PostOp.
     */
    void execute() override {
        LOG << ToString() << " flushing pre tables";
        //Flush hash table to send data before main op begins
        reduce_pre_table_.Flush();
        reduce_pre_table_.CloseEmitter();

        LOG << ToString() << " running main op";
        MainOp();
    }

    /*!
     * Produces a function stack, which only contains the PostOp function.
     * \return PostOp function stack
     */
    auto ProduceStack() {
        // Hook PostOp
        auto post_op_fn = [ = ](Output elem, std::function<void(Output)> emit_func) {
                              return PostOp(elem, emit_func);
                          };

        FunctionStack<> stack;
        return stack.push(post_op_fn);
    }

    /*!
     * Returns "[ReduceNode]" and its id as a string.
     * \return "[ReduceNode]"
     */
    std::string ToString() override {
        return "[ReduceNode] Id: " + std::to_string(data_id_);
    }

private:
    //! Local stack
    Stack local_stack_;
    //!Key extractor function
    KeyExtractor key_extractor_;
    //!Reduce function
    ReduceFunction reduce_function_;

    data::ChannelId channel_id_;

    core::ReducePreTable<KeyExtractor, ReduceFunction, data::BlockEmitter<Output> > reduce_pre_table_;

    //! Locally hash elements of the current DIA onto buckets and reduce each
    //! bucket to a single value, afterwards send data to another worker given
    //! by the shuffle algorithm.
    void PreOp(reduce_arg_t input) {
        reduce_pre_table_.Insert(input);
    }

    //!Recieve elements from other workers.
    auto MainOp() {
        std::function<void(Output)> print =
            [](Output elem) {
                LOG << elem.first << " " << elem.second;
            };

        using ReduceTable
                  = core::ReducePostTable<KeyExtractor,
                                          ReduceFunction,
                                          std::function<void(Output)> >;

        ReduceTable table(key_extractor_, reduce_function_, DIANode<Output>::callbacks());

        auto it = context_.get_data_manager().template GetRemoteBlocks<Output>(channel_id_);

        while (!it.IsClosed()) {
            it.WaitForMore();
            while (it.HasNext()) {
                table.Insert(it.Next());
            }
        }

        table.Flush();
    }

    //! Hash recieved elements onto buckets and reduce each bucket to a single value.
    void PostOp(Output input, std::function<void(Output)> emit_func) {
        emit_func(input);
    }
};

//! \}
} // namespace c7a

#endif // !C7A_API_REDUCE_NODE_HEADER

/******************************************************************************/
<|endoftext|>


<commit_before>#include "master.hpp"

namespace factor
{

cell factor_vm::compute_entry_point_address(cell obj)
{
	switch(tagged<object>(obj).type())
	{
	case WORD_TYPE:
		return (cell)untag<word>(obj)->entry_point;
	case QUOTATION_TYPE:
		return (cell)untag<quotation>(obj)->entry_point;
	default:
		critical_error("Expected word or quotation",obj);
		return 0;
	}
}

cell factor_vm::compute_entry_point_pic_address(word *w, cell tagged_quot)
{
	if(!to_boolean(tagged_quot) || max_pic_size == 0)
		return (cell)w->entry_point;
	else
	{
		quotation *quot = untag<quotation>(tagged_quot);
		if(quot_compiled_p(quot))
			return (cell)quot->entry_point;
		else
			return (cell)w->entry_point;
	}
}

cell factor_vm::compute_entry_point_pic_address(cell w_)
{
	tagged<word> w(w_);
	return compute_entry_point_pic_address(w.untagged(),w->pic_def);
}

cell factor_vm::compute_entry_point_pic_tail_address(cell w_)
{
	tagged<word> w(w_);
	return compute_entry_point_pic_address(w.untagged(),w->pic_tail_def);
}

cell factor_vm::code_block_owner(code_block *compiled)
{
	tagged<object> owner(compiled->owner);

	/* Cold generic word call sites point to quotations that call the
	inline-cache-miss and inline-cache-miss-tail primitives. */
	if(owner.type_p(QUOTATION_TYPE))
	{
		tagged<quotation> quot(owner.as<quotation>());
		tagged<array> elements(quot->array);
#ifdef FACTOR_DEBUG
		assert(array_capacity(elements.untagged()) == 5);
		assert(array_nth(elements.untagged(),4) == special_objects[PIC_MISS_WORD]
			|| array_nth(elements.untagged(),4) == special_objects[PIC_MISS_TAIL_WORD]);
#endif
		tagged<wrapper> word_wrapper(array_nth(elements.untagged(),0));
		return word_wrapper->object;
	}
	else
		return compiled->owner;
}

struct update_word_references_relocation_visitor {
	factor_vm *parent;
	bool reset_inline_caches;

	update_word_references_relocation_visitor(
		factor_vm *parent_,
		bool reset_inline_caches_) :
		parent(parent_),
		reset_inline_caches(reset_inline_caches_) {}

	void operator()(instruction_operand op)
	{
		switch(op.rel_type())
		{
		case RT_ENTRY_POINT:
			{
				code_block *compiled = op.load_code_block();
				cell owner = compiled->owner;
				if(to_boolean(owner))
					op.store_value(parent->compute_entry_point_address(owner));
				break;
			}
		case RT_ENTRY_POINT_PIC:
			{
				code_block *compiled = op.load_code_block();
				if(reset_inline_caches || !compiled->pic_p())
				{
					cell owner = parent->code_block_owner(compiled);
					if(to_boolean(owner))
						op.store_value(parent->compute_entry_point_pic_address(owner));
				}
				break;
			}
		case RT_ENTRY_POINT_PIC_TAIL:
			{
				code_block *compiled = op.load_code_block();
				if(reset_inline_caches || !compiled->pic_p())
				{
					cell owner = parent->code_block_owner(compiled);
					if(to_boolean(owner))
						op.store_value(parent->compute_entry_point_pic_tail_address(owner));
				}
				break;
			}
		default:
			break;
		}
	}
};

/* Relocate new code blocks completely; updating references to literals,
dlsyms, and words. For all other words in the code heap, we only need
to update references to other words, without worrying about literals
or dlsyms. */
void factor_vm::update_word_references(code_block *compiled, bool reset_inline_caches)
{
	if(code->uninitialized_p(compiled))
		initialize_code_block(compiled);
	/* update_word_references() is always applied to every block in
	   the code heap. Since it resets all call sites to point to
	   their canonical entry point (cold entry point for non-tail calls,
	   standard entry point for tail calls), it means that no PICs
	   are referenced after this is done. So instead of polluting
	   the code heap with dead PICs that will be freed on the next
	   GC, we add them to the free list immediately. */
	else if(reset_inline_caches && compiled->pic_p())
		code->free(compiled);
	else
	{
		update_word_references_relocation_visitor visitor(this,reset_inline_caches);
		compiled->each_instruction_operand(visitor);
		compiled->flush_icache();
	}
}

/* References to undefined symbols are patched up to call this function on
image load */
void factor_vm::undefined_symbol()
{
	general_error(ERROR_UNDEFINED_SYMBOL,false_object,false_object);
}

void undefined_symbol()
{
	return current_vm()->undefined_symbol();
}

/* Look up an external library symbol referenced by a compiled code block */
cell factor_vm::compute_dlsym_address(array *literals, cell index)
{
	cell symbol = array_nth(literals,index);
	cell library = array_nth(literals,index + 1);

	dll *d = (to_boolean(library) ? untag<dll>(library) : NULL);

	if(d != NULL && !d->handle)
		return (cell)factor::undefined_symbol;

	switch(tagged<object>(symbol).type())
	{
	case BYTE_ARRAY_TYPE:
		{
			symbol_char *name = alien_offset(symbol);
			void *sym = ffi_dlsym(d,name);

			if(sym)
				return (cell)sym;
			else
				return (cell)factor::undefined_symbol;
		}
	case ARRAY_TYPE:
		{
			array *names = untag<array>(symbol);
			for(cell i = 0; i < array_capacity(names); i++)
			{
				symbol_char *name = alien_offset(array_nth(names,i));
				void *sym = ffi_dlsym(d,name);

				if(sym)
					return (cell)sym;
			}
			return (cell)factor::undefined_symbol;
		}
	default:
		critical_error("Bad symbol specifier",symbol);
		return (cell)factor::undefined_symbol;
	}
}

cell factor_vm::compute_vm_address(cell arg)
{
	return (cell)this + untag_fixnum(arg);
}

void factor_vm::store_external_address(instruction_operand op)
{
	code_block *compiled = op.parent_code_block();
	array *parameters = (to_boolean(compiled->parameters) ? untag<array>(compiled->parameters) : NULL);
	cell index = op.parameter_index();

	switch(op.rel_type())
	{
	case RT_DLSYM:
		op.store_value(compute_dlsym_address(parameters,index));
		break;
	case RT_THIS:
		op.store_value((cell)compiled->entry_point());
		break;
	case RT_MEGAMORPHIC_CACHE_HITS:
		op.store_value((cell)&dispatch_stats.megamorphic_cache_hits);
		break;
	case RT_VM:
		op.store_value(compute_vm_address(array_nth(parameters,index)));
		break;
	case RT_CARDS_OFFSET:
		op.store_value(cards_offset);
		break;
	case RT_DECKS_OFFSET:
		op.store_value(decks_offset);
		break;
#ifdef WINDOWS
	case RT_EXCEPTION_HANDLER:
		op.store_value(&factor::exception_handler);
		break;
#endif
	default:
		critical_error("Bad rel type",op.rel_type());
		break;
	}
}

cell factor_vm::compute_here_address(cell arg, cell offset, code_block *compiled)
{
	fixnum n = untag_fixnum(arg);
	if(n >= 0)
		return (cell)compiled->entry_point() + offset + n;
	else
		return (cell)compiled->entry_point() - n;
}

struct initial_code_block_visitor {
	factor_vm *parent;
	cell literals;
	cell literal_index;

	explicit initial_code_block_visitor(factor_vm *parent_, cell literals_)
		: parent(parent_), literals(literals_), literal_index(0) {}

	cell next_literal()
	{
		return array_nth(untag<array>(literals),literal_index++);
	}

	void operator()(instruction_operand op)
	{
		switch(op.rel_type())
		{
		case RT_LITERAL:
			op.store_value(next_literal());
			break;
		case RT_ENTRY_POINT:
			op.store_value(parent->compute_entry_point_address(next_literal()));
			break;
		case RT_ENTRY_POINT_PIC:
			op.store_value(parent->compute_entry_point_pic_address(next_literal()));
			break;
		case RT_ENTRY_POINT_PIC_TAIL:
			op.store_value(parent->compute_entry_point_pic_tail_address(next_literal()));
			break;
		case RT_HERE:
			op.store_value(parent->compute_here_address(next_literal(),op.rel_offset(),op.parent_code_block()));
			break;
		case RT_UNTAGGED:
			op.store_value(untag_fixnum(next_literal()));
			break;
		default:
			parent->store_external_address(op);
			break;
		}
	}
};

/* Perform all fixups on a code block */
void factor_vm::initialize_code_block(code_block *compiled, cell literals)
{
	initial_code_block_visitor visitor(this,literals);
	compiled->each_instruction_operand(visitor);
	compiled->flush_icache();

	/* next time we do a minor GC, we have to trace this code block, since
	the newly-installed instruction operands might point to literals in
	nursery or aging */
	code->write_barrier(compiled);
}

void factor_vm::initialize_code_block(code_block *compiled)
{
	std::map<code_block *,cell>::iterator iter = code->uninitialized_blocks.find(compiled);
	initialize_code_block(compiled,iter->second);
	code->uninitialized_blocks.erase(iter);
}

/* Fixup labels. This is done at compile time, not image load time */
void factor_vm::fixup_labels(array *labels, code_block *compiled)
{
	cell size = array_capacity(labels);

	for(cell i = 0; i < size; i += 3)
	{
		relocation_class rel_class = (relocation_class)untag_fixnum(array_nth(labels,i));
		cell offset = untag_fixnum(array_nth(labels,i + 1));
		cell target = untag_fixnum(array_nth(labels,i + 2));

		relocation_entry new_entry(RT_HERE,rel_class,offset);

		instruction_operand op(new_entry,compiled,0);
		op.store_value(target + (cell)compiled->entry_point());
	}
}

/* Might GC */
code_block *factor_vm::allot_code_block(cell size, code_block_type type)
{
	code_block *block = code->allocator->allot(size + sizeof(code_block));

	/* If allocation failed, do a full GC and compact the code heap.
	A full GC that occurs as a result of the data heap filling up does not
	trigger a compaction. This setup ensures that most GCs do not compact
	the code heap, but if the code fills up, it probably means it will be
	fragmented after GC anyway, so its best to compact. */
	if(block == NULL)
	{
		primitive_compact_gc();
		block = code->allocator->allot(size + sizeof(code_block));

		/* Insufficient room even after code GC, give up */
		if(block == NULL)
		{
			std::cout << "Code heap used: " << code->allocator->occupied_space() << "\n";
			std::cout << "Code heap free: " << code->allocator->free_space() << "\n";
			fatal_error("Out of memory in add-compiled-block",0);
		}
	}

	block->set_type(type);
	return block;
}

/* Might GC */
code_block *factor_vm::add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell parameters_, cell literals_)
{
	data_root<byte_array> code(code_,this);
	data_root<object> labels(labels_,this);
	data_root<object> owner(owner_,this);
	data_root<byte_array> relocation(relocation_,this);
	data_root<array> parameters(parameters_,this);
	data_root<array> literals(literals_,this);

	cell code_length = array_capacity(code.untagged());
	code_block *compiled = allot_code_block(code_length,type);

	compiled->owner = owner.value();

	/* slight space optimization */
	if(relocation.type() == BYTE_ARRAY_TYPE && array_capacity(relocation.untagged()) == 0)
		compiled->relocation = false_object;
	else
		compiled->relocation = relocation.value();

	if(parameters.type() == ARRAY_TYPE && array_capacity(parameters.untagged()) == 0)
		compiled->parameters = false_object;
	else
		compiled->parameters = parameters.value();

	/* code */
	memcpy(compiled + 1,code.untagged() + 1,code_length);

	/* fixup labels */
	if(to_boolean(labels.value()))
		fixup_labels(labels.as<array>().untagged(),compiled);

	/* Once we are ready, fill in literal and word references in this code
	block's instruction operands. In most cases this is done right after this
	method returns, except when compiling words with the non-optimizing
	compiler at the beginning of bootstrap */
	this->code->uninitialized_blocks.insert(std::make_pair(compiled,literals.value()));

	/* next time we do a minor GC, we have to trace this code block, since
	the fields of the code_block struct might point into nursery or aging */
	this->code->write_barrier(compiled);

	return compiled;
}

}
<commit_msg>vm: fix compile error<commit_after>#include "master.hpp"

namespace factor
{

cell factor_vm::compute_entry_point_address(cell obj)
{
	switch(tagged<object>(obj).type())
	{
	case WORD_TYPE:
		return (cell)untag<word>(obj)->entry_point;
	case QUOTATION_TYPE:
		return (cell)untag<quotation>(obj)->entry_point;
	default:
		critical_error("Expected word or quotation",obj);
		return 0;
	}
}

cell factor_vm::compute_entry_point_pic_address(word *w, cell tagged_quot)
{
	if(!to_boolean(tagged_quot) || max_pic_size == 0)
		return (cell)w->entry_point;
	else
	{
		quotation *quot = untag<quotation>(tagged_quot);
		if(quot_compiled_p(quot))
			return (cell)quot->entry_point;
		else
			return (cell)w->entry_point;
	}
}

cell factor_vm::compute_entry_point_pic_address(cell w_)
{
	tagged<word> w(w_);
	return compute_entry_point_pic_address(w.untagged(),w->pic_def);
}

cell factor_vm::compute_entry_point_pic_tail_address(cell w_)
{
	tagged<word> w(w_);
	return compute_entry_point_pic_address(w.untagged(),w->pic_tail_def);
}

cell factor_vm::code_block_owner(code_block *compiled)
{
	tagged<object> owner(compiled->owner);

	/* Cold generic word call sites point to quotations that call the
	inline-cache-miss and inline-cache-miss-tail primitives. */
	if(owner.type_p(QUOTATION_TYPE))
	{
		tagged<quotation> quot(owner.as<quotation>());
		tagged<array> elements(quot->array);
#ifdef FACTOR_DEBUG
		assert(array_capacity(elements.untagged()) == 5);
		assert(array_nth(elements.untagged(),4) == special_objects[PIC_MISS_WORD]
			|| array_nth(elements.untagged(),4) == special_objects[PIC_MISS_TAIL_WORD]);
#endif
		tagged<wrapper> word_wrapper(array_nth(elements.untagged(),0));
		return word_wrapper->object;
	}
	else
		return compiled->owner;
}

struct update_word_references_relocation_visitor {
	factor_vm *parent;
	bool reset_inline_caches;

	update_word_references_relocation_visitor(
		factor_vm *parent_,
		bool reset_inline_caches_) :
		parent(parent_),
		reset_inline_caches(reset_inline_caches_) {}

	void operator()(instruction_operand op)
	{
		switch(op.rel_type())
		{
		case RT_ENTRY_POINT:
			{
				code_block *compiled = op.load_code_block();
				cell owner = compiled->owner;
				if(to_boolean(owner))
					op.store_value(parent->compute_entry_point_address(owner));
				break;
			}
		case RT_ENTRY_POINT_PIC:
			{
				code_block *compiled = op.load_code_block();
				if(reset_inline_caches || !compiled->pic_p())
				{
					cell owner = parent->code_block_owner(compiled);
					if(to_boolean(owner))
						op.store_value(parent->compute_entry_point_pic_address(owner));
				}
				break;
			}
		case RT_ENTRY_POINT_PIC_TAIL:
			{
				code_block *compiled = op.load_code_block();
				if(reset_inline_caches || !compiled->pic_p())
				{
					cell owner = parent->code_block_owner(compiled);
					if(to_boolean(owner))
						op.store_value(parent->compute_entry_point_pic_tail_address(owner));
				}
				break;
			}
		default:
			break;
		}
	}
};

/* Relocate new code blocks completely; updating references to literals,
dlsyms, and words. For all other words in the code heap, we only need
to update references to other words, without worrying about literals
or dlsyms. */
void factor_vm::update_word_references(code_block *compiled, bool reset_inline_caches)
{
	if(code->uninitialized_p(compiled))
		initialize_code_block(compiled);
	/* update_word_references() is always applied to every block in
	   the code heap. Since it resets all call sites to point to
	   their canonical entry point (cold entry point for non-tail calls,
	   standard entry point for tail calls), it means that no PICs
	   are referenced after this is done. So instead of polluting
	   the code heap with dead PICs that will be freed on the next
	   GC, we add them to the free list immediately. */
	else if(reset_inline_caches && compiled->pic_p())
		code->free(compiled);
	else
	{
		update_word_references_relocation_visitor visitor(this,reset_inline_caches);
		compiled->each_instruction_operand(visitor);
		compiled->flush_icache();
	}
}

/* References to undefined symbols are patched up to call this function on
image load */
void factor_vm::undefined_symbol()
{
	general_error(ERROR_UNDEFINED_SYMBOL,false_object,false_object);
}

void undefined_symbol()
{
	return current_vm()->undefined_symbol();
}

/* Look up an external library symbol referenced by a compiled code block */
cell factor_vm::compute_dlsym_address(array *literals, cell index)
{
	cell symbol = array_nth(literals,index);
	cell library = array_nth(literals,index + 1);

	dll *d = (to_boolean(library) ? untag<dll>(library) : NULL);

	if(d != NULL && !d->handle)
		return (cell)factor::undefined_symbol;

	switch(tagged<object>(symbol).type())
	{
	case BYTE_ARRAY_TYPE:
		{
			symbol_char *name = alien_offset(symbol);
			void *sym = ffi_dlsym(d,name);

			if(sym)
				return (cell)sym;
			else
				return (cell)factor::undefined_symbol;
		}
	case ARRAY_TYPE:
		{
			array *names = untag<array>(symbol);
			for(cell i = 0; i < array_capacity(names); i++)
			{
				symbol_char *name = alien_offset(array_nth(names,i));
				void *sym = ffi_dlsym(d,name);

				if(sym)
					return (cell)sym;
			}
			return (cell)factor::undefined_symbol;
		}
	default:
		critical_error("Bad symbol specifier",symbol);
		return (cell)factor::undefined_symbol;
	}
}

cell factor_vm::compute_vm_address(cell arg)
{
	return (cell)this + untag_fixnum(arg);
}

void factor_vm::store_external_address(instruction_operand op)
{
	code_block *compiled = op.parent_code_block();
	array *parameters = (to_boolean(compiled->parameters) ? untag<array>(compiled->parameters) : NULL);
	cell index = op.parameter_index();

	switch(op.rel_type())
	{
	case RT_DLSYM:
		op.store_value(compute_dlsym_address(parameters,index));
		break;
	case RT_THIS:
		op.store_value((cell)compiled->entry_point());
		break;
	case RT_MEGAMORPHIC_CACHE_HITS:
		op.store_value((cell)&dispatch_stats.megamorphic_cache_hits);
		break;
	case RT_VM:
		op.store_value(compute_vm_address(array_nth(parameters,index)));
		break;
	case RT_CARDS_OFFSET:
		op.store_value(cards_offset);
		break;
	case RT_DECKS_OFFSET:
		op.store_value(decks_offset);
		break;
#ifdef WINDOWS
	case RT_EXCEPTION_HANDLER:
		op.store_value((cell)&factor::exception_handler);
		break;
#endif
	default:
		critical_error("Bad rel type",op.rel_type());
		break;
	}
}

cell factor_vm::compute_here_address(cell arg, cell offset, code_block *compiled)
{
	fixnum n = untag_fixnum(arg);
	if(n >= 0)
		return (cell)compiled->entry_point() + offset + n;
	else
		return (cell)compiled->entry_point() - n;
}

struct initial_code_block_visitor {
	factor_vm *parent;
	cell literals;
	cell literal_index;

	explicit initial_code_block_visitor(factor_vm *parent_, cell literals_)
		: parent(parent_), literals(literals_), literal_index(0) {}

	cell next_literal()
	{
		return array_nth(untag<array>(literals),literal_index++);
	}

	void operator()(instruction_operand op)
	{
		switch(op.rel_type())
		{
		case RT_LITERAL:
			op.store_value(next_literal());
			break;
		case RT_ENTRY_POINT:
			op.store_value(parent->compute_entry_point_address(next_literal()));
			break;
		case RT_ENTRY_POINT_PIC:
			op.store_value(parent->compute_entry_point_pic_address(next_literal()));
			break;
		case RT_ENTRY_POINT_PIC_TAIL:
			op.store_value(parent->compute_entry_point_pic_tail_address(next_literal()));
			break;
		case RT_HERE:
			op.store_value(parent->compute_here_address(next_literal(),op.rel_offset(),op.parent_code_block()));
			break;
		case RT_UNTAGGED:
			op.store_value(untag_fixnum(next_literal()));
			break;
		default:
			parent->store_external_address(op);
			break;
		}
	}
};

/* Perform all fixups on a code block */
void factor_vm::initialize_code_block(code_block *compiled, cell literals)
{
	initial_code_block_visitor visitor(this,literals);
	compiled->each_instruction_operand(visitor);
	compiled->flush_icache();

	/* next time we do a minor GC, we have to trace this code block, since
	the newly-installed instruction operands might point to literals in
	nursery or aging */
	code->write_barrier(compiled);
}

void factor_vm::initialize_code_block(code_block *compiled)
{
	std::map<code_block *,cell>::iterator iter = code->uninitialized_blocks.find(compiled);
	initialize_code_block(compiled,iter->second);
	code->uninitialized_blocks.erase(iter);
}

/* Fixup labels. This is done at compile time, not image load time */
void factor_vm::fixup_labels(array *labels, code_block *compiled)
{
	cell size = array_capacity(labels);

	for(cell i = 0; i < size; i += 3)
	{
		relocation_class rel_class = (relocation_class)untag_fixnum(array_nth(labels,i));
		cell offset = untag_fixnum(array_nth(labels,i + 1));
		cell target = untag_fixnum(array_nth(labels,i + 2));

		relocation_entry new_entry(RT_HERE,rel_class,offset);

		instruction_operand op(new_entry,compiled,0);
		op.store_value(target + (cell)compiled->entry_point());
	}
}

/* Might GC */
code_block *factor_vm::allot_code_block(cell size, code_block_type type)
{
	code_block *block = code->allocator->allot(size + sizeof(code_block));

	/* If allocation failed, do a full GC and compact the code heap.
	A full GC that occurs as a result of the data heap filling up does not
	trigger a compaction. This setup ensures that most GCs do not compact
	the code heap, but if the code fills up, it probably means it will be
	fragmented after GC anyway, so its best to compact. */
	if(block == NULL)
	{
		primitive_compact_gc();
		block = code->allocator->allot(size + sizeof(code_block));

		/* Insufficient room even after code GC, give up */
		if(block == NULL)
		{
			std::cout << "Code heap used: " << code->allocator->occupied_space() << "\n";
			std::cout << "Code heap free: " << code->allocator->free_space() << "\n";
			fatal_error("Out of memory in add-compiled-block",0);
		}
	}

	block->set_type(type);
	return block;
}

/* Might GC */
code_block *factor_vm::add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell parameters_, cell literals_)
{
	data_root<byte_array> code(code_,this);
	data_root<object> labels(labels_,this);
	data_root<object> owner(owner_,this);
	data_root<byte_array> relocation(relocation_,this);
	data_root<array> parameters(parameters_,this);
	data_root<array> literals(literals_,this);

	cell code_length = array_capacity(code.untagged());
	code_block *compiled = allot_code_block(code_length,type);

	compiled->owner = owner.value();

	/* slight space optimization */
	if(relocation.type() == BYTE_ARRAY_TYPE && array_capacity(relocation.untagged()) == 0)
		compiled->relocation = false_object;
	else
		compiled->relocation = relocation.value();

	if(parameters.type() == ARRAY_TYPE && array_capacity(parameters.untagged()) == 0)
		compiled->parameters = false_object;
	else
		compiled->parameters = parameters.value();

	/* code */
	memcpy(compiled + 1,code.untagged() + 1,code_length);

	/* fixup labels */
	if(to_boolean(labels.value()))
		fixup_labels(labels.as<array>().untagged(),compiled);

	/* Once we are ready, fill in literal and word references in this code
	block's instruction operands. In most cases this is done right after this
	method returns, except when compiling words with the non-optimizing
	compiler at the beginning of bootstrap */
	this->code->uninitialized_blocks.insert(std::make_pair(compiled,literals.value()));

	/* next time we do a minor GC, we have to trace this code block, since
	the fields of the code_block struct might point into nursery or aging */
	this->code->write_barrier(compiled);

	return compiled;
}

}
<|endoftext|>


<commit_before><commit_msg>non-functional update<commit_after><|endoftext|>


<commit_before>/* bzflag
 * Copyright (c) 1993 - 2005 Tim Riker
 *
 * This package is free software;  you can redistribute it and/or
 * modify it under the terms of the license found in the file
 * named LICENSE that should have accompanied this file.
 *
 * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

// Provide BZFS with a list server connection

/* class header */
#include "ListServerConnection.h"

/* system implementation headers */
#include <string.h>
#include <string>
#include <math.h>
#include <errno.h>

/* common implementation headers */
#include "bzfio.h"
#include "version.h"
#include "TextUtils.h"
#include "Protocol.h"
#include "GameKeeper.h"

/* local implementation headers */
#include "CmdLineOptions.h"

// FIXME remove externs!
extern Address serverAddress;
extern PingPacket getTeamCounts();
extern uint16_t curMaxPlayers;
extern int getTarget(const char *victimname);
extern void sendMessage(int playerIndex, PlayerId targetPlayer, const char *message);
extern void sendPlayerInfo(void);
extern void sendIPUpdate(int targetPlayer, int playerIndex);
extern CmdLineOptions *clOptions;

ListServerLink::ListServerLink(std::string listServerURL, std::string publicizedAddress, std::string publicizedTitle)
{
  // parse url
  std::string protocol, hostname, pathname;
  int port = 80;
  bool useDefault = false;

  // use default if it can't be parsed
  if (!BzfNetwork::parseURL(listServerURL, protocol, hostname, port, pathname))
    useDefault = true;

  // use default if wrong protocol or invalid port
  if ((protocol != "http") || (port < 1) || (port > 65535))
    useDefault = true;

  // use default if bad address
  Address address = Address::getHostAddress(hostname);
  if (address.isAny())
    useDefault = true;

  // parse default list server URL if we need to; assume default works
  if (useDefault) {
    BzfNetwork::parseURL(DefaultListServerURL, protocol, hostname, port, pathname);
    DEBUG1("Provided list server URL (%s) is invalid.  Using default of %s.\n", listServerURL.c_str(), DefaultListServerURL);
  }

  // add to list
  this->address	   = address;
  this->port	   = port;
  this->pathname   = pathname;
  this->hostname   = hostname;
  this->linkSocket = NotConnected;

  this->publicizeAddress     = publicizedAddress;
  this->publicizeDescription = publicizedTitle;
  this->publicizeServer	     = true;  //if this c'tor is called, it's safe to publicize

  // schedule initial ADD message
  queueMessage(ListServerLink::ADD);
}

ListServerLink::ListServerLink()
{
  // does not create a usable link, so checks should be placed
  // in  all public member functions to ensure that nothing tries
  // to happen if publicizeServer is false
  this->linkSocket = NotConnected;
  this->publicizeServer = false;
}

ListServerLink::~ListServerLink()
{
  // now tell the list server that we're going away.  this can
  // take some time but we don't want to wait too long.  we do
  // our own multiplexing loop and wait for a maximum of 3 seconds
  // total.

  // if we aren't supposed to be publicizing, skip the whole thing
  // and don't waste 3 seconds.
  if (!publicizeServer)
    return;

  queueMessage(ListServerLink::REMOVE);
  TimeKeeper start = TimeKeeper::getCurrent();
  do {
    // compute timeout
    float waitTime = 3.0f - (TimeKeeper::getCurrent() - start);
    if (waitTime <= 0.0f)
      break;
    if (!isConnected()) //queueMessage should have connected us
      break;
    // check for list server socket connection
    int fdMax = -1;
    fd_set write_set;
    fd_set read_set;
    FD_ZERO(&write_set);
    FD_ZERO(&read_set);
    if (phase == ListServerLink::CONNECTING)
      _FD_SET(linkSocket, &write_set);
    else
      _FD_SET(linkSocket, &read_set);
    fdMax = linkSocket;

    // wait for socket to connect or timeout
    struct timeval timeout;
    timeout.tv_sec = long(floorf(waitTime));
    timeout.tv_usec = long(1.0e+6f * (waitTime - floorf(waitTime)));
    int nfound = select(fdMax + 1, (fd_set*)&read_set, (fd_set*)&write_set,
			0, &timeout);
    if (nfound == 0)
      // Time has gone, close and go
      break;
    // check for connection to list server
    if (FD_ISSET(linkSocket, &write_set))
      sendQueuedMessages();
    else if (FD_ISSET(linkSocket, &read_set))
      read();
  } while (true);

  // stop list server communication
  closeLink();
}

void ListServerLink::closeLink()
{
  if (isConnected()) {
    close(linkSocket);
    DEBUG4("Closing List Server\n");
    linkSocket = NotConnected;
  }
}

void ListServerLink::read()
{
  if (isConnected()) {
    char buf[2048];
    int bytes = recv(linkSocket, buf, sizeof(buf)-1, 0);
    // TODO don't close unless we've got it all
    closeLink();
    buf[bytes]=0;
    char* base = buf;
    static char *tokGoodIdentifier = "TOKGOOD: ";
    static char *tokBadIdentifier = "TOKBAD: ";
    // walks entire reply including HTTP headers
    while (*base) {
      // find next newline
      char* scan = base;
      while (*scan && *scan != '\r' && *scan != '\n') scan++;
      // if no newline then no more complete replies
      if (*scan != '\r' && *scan != '\n') break;
      while (*scan && (*scan == '\r' || *scan == '\n')) *scan++ = '\0';
      DEBUG4("Got line: \"%s\"\n", base);
      // TODO don't do this if we don't want central logins
      if (strncmp(base, tokGoodIdentifier, strlen(tokGoodIdentifier)) == 0) {
	char *callsign, *group;
	callsign = base + strlen(tokGoodIdentifier);
	DEBUG3("Got: %s\n", base);
	group = callsign;
	while (*group && (*group != ':')) group++;
	while (*group && (*group == ':')) *group++ = 0;
	int playerIndex = getTarget(callsign);
	if (playerIndex < curMaxPlayers) {
	  GameKeeper::Player *playerData = GameKeeper::Player::getPlayerByIndex(playerIndex);
	  if (!playerData->accessInfo.isRegistered())
	    playerData->accessInfo.storeInfo(NULL);
	  playerData->accessInfo.setPermissionRights();
	  while (*group) {
	    char *nextgroup = group;
	    while (*nextgroup && (*nextgroup != ':')) nextgroup++;
	    while (*nextgroup && (*nextgroup == ':')) *nextgroup++ = 0;
	    playerData->accessInfo.addGroup(group);
	    //DEBUG3("Got: [%d] \"%s\" \"%s\"\n", playerIndex, callsign, group);
	    group = nextgroup;
	  }
	  sendMessage(ServerPlayer, playerIndex, "Global login approved!");
	  sendIPUpdate(playerIndex, -1);
	  sendPlayerInfo();
	  playerData->player.clearToken();
	}
      } else if (strncmp(base, tokBadIdentifier, strlen(tokBadIdentifier)) == 0) {
	char *callsign;
	callsign = base + strlen(tokBadIdentifier);
	// find end of callsign (string is "callsign=token")
	char *p = callsign;
	while (*p && (*p != '=')) p++;
	*p = 0;
	int playerIndex = getTarget(callsign);
	DEBUG3("Got: [%d] %s\n", playerIndex, base);
	if (playerIndex < curMaxPlayers) {
	  GameKeeper::Player *playerData = GameKeeper::Player::getPlayerByIndex(playerIndex);
	  sendMessage(ServerPlayer, playerIndex, "Global login rejected, bad token.");
	  playerData->player.clearToken();
	}
      }

      // next reply
      base = scan;
    }
    if (nextMessageType != ListServerLink::NONE) {
      // There was a pending request arrived after we write:
      // we should redo all the stuff
      openLink();
    }
  }
}

void ListServerLink::openLink()
{
  // start opening connection if not already doing so
  if (!isConnected()) {
    linkSocket = socket(AF_INET, SOCK_STREAM, 0);
    DEBUG4("Opening List Server\n");
    if (!isConnected()) {
      return;
    }

    // set to non-blocking for connect
    if (BzfNetwork::setNonBlocking(linkSocket) < 0) {
      closeLink();
      return;
    }

    // Make our connection come from our serverAddress in case we have
    // multiple/masked IPs so the list server can verify us.
    struct sockaddr_in addr;
    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_addr = serverAddress;

    // assign the address to the socket
    if (bind(linkSocket, (CNCTType*)&addr, sizeof(addr)) < 0) {
      closeLink();
      return;
    }

    // connect.  this should fail with EINPROGRESS but check for
    // success just in case.
    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port   = htons(port);
    addr.sin_addr   = address;
    if (connect(linkSocket, (CNCTType*)&addr, sizeof(addr)) < 0) {
#if defined(_WIN32)
#undef EINPROGRESS
#define EINPROGRESS EWOULDBLOCK
#endif
      if (getErrno() != EINPROGRESS) {
	nerror("connecting to list server");
	// try to lookup dns name again in case it moved
	this->address = Address::getHostAddress(this->hostname);
	closeLink();
      } else {
	phase = CONNECTING;
      }
    } else {
      // shouldn't arrive here. Just in case, clean
      DEBUG3("list server connect and close?\n");
      closeLink();
    }
  }
}

void ListServerLink::queueMessage(MessageType type)
{
  // ignore if the server is not public
  if (!publicizeServer) return;

  // Open network connection only if closed
  if (!isConnected()) openLink();

  // record next message to send.
  nextMessageType = type;
}

void ListServerLink::sendQueuedMessages()
{
  if (!isConnected())
    return;

  if (nextMessageType == ListServerLink::ADD) {
    DEBUG3("Queuing ADD message to list server\n");
    addMe(getTeamCounts(), publicizeAddress, TextUtils::url_encode(publicizeDescription));
    lastAddTime = TimeKeeper::getCurrent();
  } else if (nextMessageType == ListServerLink::REMOVE) {
    DEBUG3("Queuing REMOVE message to list server\n");
    removeMe(publicizeAddress);
  }
}

void ListServerLink::addMe(PingPacket pingInfo,
			   std::string publicizedAddress,
			   std::string publicizedTitle)
{
  std::string msg;

  // encode ping reply as ascii hex digits plus NULL
  char gameInfo[PingPacketHexPackedSize + 1];
  pingInfo.packHex(gameInfo);

  // TODO we probably should convert to a POST. List server now allows either
  // send ADD message (must send blank line)
  msg = TextUtils::format("GET %s?action=ADD&nameport=%s&version=%s&gameinfo=%s&build=%s",
    pathname.c_str(), publicizedAddress.c_str(),
    getServerVersion(), gameInfo,
    getAppVersion());
  msg += "&checktokens=";
  for (int i = 0; i < curMaxPlayers; i++) {
    GameKeeper::Player *playerData = GameKeeper::Player::getPlayerByIndex(i);
    if (!playerData)
      continue;
    std::string encodedCallsign = TextUtils::url_encode(playerData->player.getCallSign());
    if (strlen(playerData->player.getCallSign()) && strlen(playerData->player.getToken())) {
      msg += encodedCallsign;
      msg += "=";
      msg += playerData->player.getToken();
      msg += "%0D%0A";
    }
  }
  // *groups=GROUP0%0D%0AGROUP1%0D%0A
  msg += "&groups=";
  std::vector<std::string>::iterator itr = clOptions->remoteGroups.begin();
  for ( ; itr != clOptions->remoteGroups.end(); itr++) {
    if (*itr != "ADMIN") {
      msg += *itr;
      msg += "%0D%0A";
    }
  }

  msg += TextUtils::format("&title=%s HTTP/1.1\r\n"
      "User-Agent: bzfs %s\r\n"
      "Host: %s\r\n"
      "Cache-Control: no-cache\r\n"
      "Connection: close\r\n"
      "\r\n",
    publicizedTitle.c_str(),
    getAppVersion(),
    hostname.c_str());
  sendLSMessage(msg);
}

void ListServerLink::removeMe(std::string publicizedAddress)
{
  std::string msg;
  // send REMOVE (must send blank line)
  msg = TextUtils::format("GET %s?action=REMOVE&nameport=%s HTTP/1.1\r\n"
    "User-Agent: bzfs %s\r\n"
    "Host: %s\r\n"
    "Cache-Control: no-cache\r\n"
    "Connection: close\r\n"
    "\r\n",
    pathname.c_str(),
    publicizedAddress.c_str(),
    getAppVersion(),
    hostname.c_str());
  sendLSMessage(msg);
}

void ListServerLink::sendLSMessage(std::string message)
{
  const int bufsize = 4096;
  char msg[bufsize];
  strncpy(msg, message.c_str(), bufsize);
  msg[bufsize - 1] = 0;
  if (strlen(msg) > 0) {
    DEBUG3("%s\n", msg);
    if (send(linkSocket, msg, strlen(msg), 0) == -1) {
      perror("List server send failed");
      DEBUG3("Unable to send to the list server!\n");
      closeLink();
    } else {
      nextMessageType = ListServerLink::NONE;
      phase	   = ListServerLink::WRITTEN;
    }
  } else {
    closeLink();
  }
}

// Local Variables: ***
// mode:C++ ***
// tab-width: 8 ***
// c-basic-offset: 2 ***
// indent-tabs-mode: t ***
// End: ***
// ex: shiftwidth=2 tabstop=8
<commit_msg>bzfs asks list server to check whether token and client IP match<commit_after>/* bzflag
 * Copyright (c) 1993 - 2005 Tim Riker
 *
 * This package is free software;  you can redistribute it and/or
 * modify it under the terms of the license found in the file
 * named LICENSE that should have accompanied this file.
 *
 * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

// Provide BZFS with a list server connection

/* class header */
#include "ListServerConnection.h"

/* system implementation headers */
#include <string.h>
#include <string>
#include <math.h>
#include <errno.h>

/* common implementation headers */
#include "bzfio.h"
#include "version.h"
#include "TextUtils.h"
#include "Protocol.h"
#include "GameKeeper.h"

/* local implementation headers */
#include "CmdLineOptions.h"

// FIXME remove externs!
extern Address serverAddress;
extern PingPacket getTeamCounts();
extern uint16_t curMaxPlayers;
extern int getTarget(const char *victimname);
extern void sendMessage(int playerIndex, PlayerId targetPlayer, const char *message);
extern void sendPlayerInfo(void);
extern void sendIPUpdate(int targetPlayer, int playerIndex);
extern CmdLineOptions *clOptions;

ListServerLink::ListServerLink(std::string listServerURL, std::string publicizedAddress, std::string publicizedTitle)
{
  // parse url
  std::string protocol, hostname, pathname;
  int port = 80;
  bool useDefault = false;

  // use default if it can't be parsed
  if (!BzfNetwork::parseURL(listServerURL, protocol, hostname, port, pathname))
    useDefault = true;

  // use default if wrong protocol or invalid port
  if ((protocol != "http") || (port < 1) || (port > 65535))
    useDefault = true;

  // use default if bad address
  Address address = Address::getHostAddress(hostname);
  if (address.isAny())
    useDefault = true;

  // parse default list server URL if we need to; assume default works
  if (useDefault) {
    BzfNetwork::parseURL(DefaultListServerURL, protocol, hostname, port, pathname);
    DEBUG1("Provided list server URL (%s) is invalid.  Using default of %s.\n", listServerURL.c_str(), DefaultListServerURL);
  }

  // add to list
  this->address	   = address;
  this->port	   = port;
  this->pathname   = pathname;
  this->hostname   = hostname;
  this->linkSocket = NotConnected;

  this->publicizeAddress     = publicizedAddress;
  this->publicizeDescription = publicizedTitle;
  this->publicizeServer	     = true;  //if this c'tor is called, it's safe to publicize

  // schedule initial ADD message
  queueMessage(ListServerLink::ADD);
}

ListServerLink::ListServerLink()
{
  // does not create a usable link, so checks should be placed
  // in  all public member functions to ensure that nothing tries
  // to happen if publicizeServer is false
  this->linkSocket = NotConnected;
  this->publicizeServer = false;
}

ListServerLink::~ListServerLink()
{
  // now tell the list server that we're going away.  this can
  // take some time but we don't want to wait too long.  we do
  // our own multiplexing loop and wait for a maximum of 3 seconds
  // total.

  // if we aren't supposed to be publicizing, skip the whole thing
  // and don't waste 3 seconds.
  if (!publicizeServer)
    return;

  queueMessage(ListServerLink::REMOVE);
  TimeKeeper start = TimeKeeper::getCurrent();
  do {
    // compute timeout
    float waitTime = 3.0f - (TimeKeeper::getCurrent() - start);
    if (waitTime <= 0.0f)
      break;
    if (!isConnected()) //queueMessage should have connected us
      break;
    // check for list server socket connection
    int fdMax = -1;
    fd_set write_set;
    fd_set read_set;
    FD_ZERO(&write_set);
    FD_ZERO(&read_set);
    if (phase == ListServerLink::CONNECTING)
      _FD_SET(linkSocket, &write_set);
    else
      _FD_SET(linkSocket, &read_set);
    fdMax = linkSocket;

    // wait for socket to connect or timeout
    struct timeval timeout;
    timeout.tv_sec = long(floorf(waitTime));
    timeout.tv_usec = long(1.0e+6f * (waitTime - floorf(waitTime)));
    int nfound = select(fdMax + 1, (fd_set*)&read_set, (fd_set*)&write_set,
			0, &timeout);
    if (nfound == 0)
      // Time has gone, close and go
      break;
    // check for connection to list server
    if (FD_ISSET(linkSocket, &write_set))
      sendQueuedMessages();
    else if (FD_ISSET(linkSocket, &read_set))
      read();
  } while (true);

  // stop list server communication
  closeLink();
}

void ListServerLink::closeLink()
{
  if (isConnected()) {
    close(linkSocket);
    DEBUG4("Closing List Server\n");
    linkSocket = NotConnected;
  }
}

void ListServerLink::read()
{
  if (isConnected()) {
    char buf[2048];
    int bytes = recv(linkSocket, buf, sizeof(buf)-1, 0);
    // TODO don't close unless we've got it all
    closeLink();
    buf[bytes]=0;
    char* base = buf;
    static char *tokGoodIdentifier = "TOKGOOD: ";
    static char *tokBadIdentifier = "TOKBAD: ";
    // walks entire reply including HTTP headers
    while (*base) {
      // find next newline
      char* scan = base;
      while (*scan && *scan != '\r' && *scan != '\n') scan++;
      // if no newline then no more complete replies
      if (*scan != '\r' && *scan != '\n') break;
      while (*scan && (*scan == '\r' || *scan == '\n')) *scan++ = '\0';
      DEBUG4("Got line: \"%s\"\n", base);
      // TODO don't do this if we don't want central logins
      if (strncmp(base, tokGoodIdentifier, strlen(tokGoodIdentifier)) == 0) {
	char *callsign, *group;
	callsign = base + strlen(tokGoodIdentifier);
	DEBUG3("Got: %s\n", base);
	group = callsign;
	while (*group && (*group != ':')) group++;
	while (*group && (*group == ':')) *group++ = 0;
	int playerIndex = getTarget(callsign);
	if (playerIndex < curMaxPlayers) {
	  GameKeeper::Player *playerData = GameKeeper::Player::getPlayerByIndex(playerIndex);
	  if (!playerData->accessInfo.isRegistered())
	    playerData->accessInfo.storeInfo(NULL);
	  playerData->accessInfo.setPermissionRights();
	  while (*group) {
	    char *nextgroup = group;
	    while (*nextgroup && (*nextgroup != ':')) nextgroup++;
	    while (*nextgroup && (*nextgroup == ':')) *nextgroup++ = 0;
	    playerData->accessInfo.addGroup(group);
	    //DEBUG3("Got: [%d] \"%s\" \"%s\"\n", playerIndex, callsign, group);
	    group = nextgroup;
	  }
	  sendMessage(ServerPlayer, playerIndex, "Global login approved!");
	  sendIPUpdate(playerIndex, -1);
	  sendPlayerInfo();
	  playerData->player.clearToken();
	}
      } else if (strncmp(base, tokBadIdentifier, strlen(tokBadIdentifier)) == 0) {
	char *callsign;
	callsign = base + strlen(tokBadIdentifier);
	// find end of callsign (string is "callsign=token")
	char *p = callsign;
	while (*p && (*p != '=')) p++;
	*p = 0;
	int playerIndex = getTarget(callsign);
	DEBUG3("Got: [%d] %s\n", playerIndex, base);
	if (playerIndex < curMaxPlayers) {
	  GameKeeper::Player *playerData = GameKeeper::Player::getPlayerByIndex(playerIndex);
	  sendMessage(ServerPlayer, playerIndex, "Global login rejected, bad token.");
	  playerData->player.clearToken();
	}
      }

      // next reply
      base = scan;
    }
    if (nextMessageType != ListServerLink::NONE) {
      // There was a pending request arrived after we write:
      // we should redo all the stuff
      openLink();
    }
  }
}

void ListServerLink::openLink()
{
  // start opening connection if not already doing so
  if (!isConnected()) {
    linkSocket = socket(AF_INET, SOCK_STREAM, 0);
    DEBUG4("Opening List Server\n");
    if (!isConnected()) {
      return;
    }

    // set to non-blocking for connect
    if (BzfNetwork::setNonBlocking(linkSocket) < 0) {
      closeLink();
      return;
    }

    // Make our connection come from our serverAddress in case we have
    // multiple/masked IPs so the list server can verify us.
    struct sockaddr_in addr;
    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_addr = serverAddress;

    // assign the address to the socket
    if (bind(linkSocket, (CNCTType*)&addr, sizeof(addr)) < 0) {
      closeLink();
      return;
    }

    // connect.  this should fail with EINPROGRESS but check for
    // success just in case.
    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port   = htons(port);
    addr.sin_addr   = address;
    if (connect(linkSocket, (CNCTType*)&addr, sizeof(addr)) < 0) {
#if defined(_WIN32)
#undef EINPROGRESS
#define EINPROGRESS EWOULDBLOCK
#endif
      if (getErrno() != EINPROGRESS) {
	nerror("connecting to list server");
	// try to lookup dns name again in case it moved
	this->address = Address::getHostAddress(this->hostname);
	closeLink();
      } else {
	phase = CONNECTING;
      }
    } else {
      // shouldn't arrive here. Just in case, clean
      DEBUG3("list server connect and close?\n");
      closeLink();
    }
  }
}

void ListServerLink::queueMessage(MessageType type)
{
  // ignore if the server is not public
  if (!publicizeServer) return;

  // Open network connection only if closed
  if (!isConnected()) openLink();

  // record next message to send.
  nextMessageType = type;
}

void ListServerLink::sendQueuedMessages()
{
  if (!isConnected())
    return;

  if (nextMessageType == ListServerLink::ADD) {
    DEBUG3("Queuing ADD message to list server\n");
    addMe(getTeamCounts(), publicizeAddress, TextUtils::url_encode(publicizeDescription));
    lastAddTime = TimeKeeper::getCurrent();
  } else if (nextMessageType == ListServerLink::REMOVE) {
    DEBUG3("Queuing REMOVE message to list server\n");
    removeMe(publicizeAddress);
  }
}

void ListServerLink::addMe(PingPacket pingInfo,
			   std::string publicizedAddress,
			   std::string publicizedTitle)
{
  std::string msg;

  // encode ping reply as ascii hex digits plus NULL
  char gameInfo[PingPacketHexPackedSize + 1];
  pingInfo.packHex(gameInfo);

  // TODO we probably should convert to a POST. List server now allows either
  // send ADD message (must send blank line)
  msg = TextUtils::format("GET %s?action=ADD&nameport=%s&version=%s&gameinfo=%s&build=%s",
    pathname.c_str(), publicizedAddress.c_str(),
    getServerVersion(), gameInfo,
    getAppVersion());
  msg += "&checktokens=";
  // callsign1@ip1=token1%0D%0Acallsign2@ip2=token2%0D%0A
  for (int i = 0; i < curMaxPlayers; i++) {
    GameKeeper::Player *playerData = GameKeeper::Player::getPlayerByIndex(i);
    if (!playerData)
      continue;
    NetHandler *handler = playerData->netHandler;
    std::string encodedCallsign = TextUtils::url_encode(playerData->player.getCallSign());
    if (strlen(playerData->player.getCallSign()) && strlen(playerData->player.getToken())) {
      msg += encodedCallsign;
      msg += "@";
      msg += handler->getTargetIP();
      msg += "=";
      msg += playerData->player.getToken();
      msg += "%0D%0A";
    }
  }
  // *groups=GROUP0%0D%0AGROUP1%0D%0A
  msg += "&groups=";
  std::vector<std::string>::iterator itr = clOptions->remoteGroups.begin();
  for ( ; itr != clOptions->remoteGroups.end(); itr++) {
    if (*itr != "ADMIN") {
      msg += *itr;
      msg += "%0D%0A";
    }
  }

  msg += TextUtils::format("&title=%s HTTP/1.1\r\n"
      "User-Agent: bzfs %s\r\n"
      "Host: %s\r\n"
      "Cache-Control: no-cache\r\n"
      "Connection: close\r\n"
      "\r\n",
    publicizedTitle.c_str(),
    getAppVersion(),
    hostname.c_str());
  sendLSMessage(msg);
}

void ListServerLink::removeMe(std::string publicizedAddress)
{
  std::string msg;
  // send REMOVE (must send blank line)
  msg = TextUtils::format("GET %s?action=REMOVE&nameport=%s HTTP/1.1\r\n"
    "User-Agent: bzfs %s\r\n"
    "Host: %s\r\n"
    "Cache-Control: no-cache\r\n"
    "Connection: close\r\n"
    "\r\n",
    pathname.c_str(),
    publicizedAddress.c_str(),
    getAppVersion(),
    hostname.c_str());
  sendLSMessage(msg);
}

void ListServerLink::sendLSMessage(std::string message)
{
  const int bufsize = 4096;
  char msg[bufsize];
  strncpy(msg, message.c_str(), bufsize);
  msg[bufsize - 1] = 0;
  if (strlen(msg) > 0) {
    DEBUG3("%s\n", msg);
    if (send(linkSocket, msg, strlen(msg), 0) == -1) {
      perror("List server send failed");
      DEBUG3("Unable to send to the list server!\n");
      closeLink();
    } else {
      nextMessageType = ListServerLink::NONE;
      phase	   = ListServerLink::WRITTEN;
    }
  } else {
    closeLink();
  }
}

// Local Variables: ***
// mode:C++ ***
// tab-width: 8 ***
// c-basic-offset: 2 ***
// indent-tabs-mode: t ***
// End: ***
// ex: shiftwidth=2 tabstop=8
<|endoftext|>


<commit_before>/*
 * Funambol is a mobile platform developed by Funambol, Inc.
 * Copyright (C) 2003 - 2007 Funambol, Inc.
 *
 * This program is free software; you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License version 3 as published by
 * the Free Software Foundation with the addition of the following permission
 * added to Section 15 as permitted in Section 7(a): FOR ANY PART OF THE COVERED
 * WORK IN WHICH THE COPYRIGHT IS OWNED BY FUNAMBOL, FUNAMBOL DISCLAIMS THE
 * WARRANTY OF NON INFRINGEMENT  OF THIRD PARTY RIGHTS.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, see http://www.gnu.org/licenses or write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 * MA 02110-1301 USA.
 *
 * You can contact Funambol, Inc. headquarters at 643 Bair Island Road, Suite
 * 305, Redwood City, CA 94063, USA, or at email address info@funambol.com.
 *
 * The interactive user interfaces in modified source and object code versions
 * of this program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU Affero General Public License version 3.
 *
 * In accordance with Section 7(b) of the GNU Affero General Public License
 * version 3, these Appropriate Legal Notices must retain the display of the
 * "Powered by Funambol" logo. If the display of the logo is not reasonably
 * feasible for technical reasons, the Appropriate Legal Notices must display
 * the words "Powered by Funambol".
 */

#if 0
#include "base/EncodingUtils.h"
#endif

#include "base/util/StringBuffer.h"
#include "spdm/constants.h"
#include "push/CTPConfig.h"
#include "base/globalsdef.h"

USE_NAMESPACE

#define CTP_QUEUE_PUSH                  "queuePush"
#define CTP_RETRY                       "ctpRetry"
#define CTP_MAX_RETRY_TIMEOUT           "maxCtpRetry"
#define CTP_CMD_TIMEOUT                 "ctpCmdTimeout"
#define CTP_CONN_TIMEOUT                "ctpConnTimeout"
#define CTP_PORT                        "ctpPort"
#define CTP_READY                       "ctpReady"
#define PROPERTY_CTP_SERVER             "ctpServer"
#define PROPERTY_NOTIFY_TIMEOUT         "notifyTimeout"

CTPConfig::CTPConfig(const char* application_uri)
                    : DMTClientConfig(application_uri) {
}

CTPConfig::~CTPConfig() {}

char* CTPConfig::decodePassword(const char* password) {

    char* decripted = NULL;
#if 0
    // Encryption not supported yet!!! TODO FIXME
    if (password && strlen(password) > 0) {
         decripted = decryptData(password);
    }

    if (decripted == NULL || getLastErrorCode() == 801) {
        decripted = new char[1];
        decripted[0] = 0;
        setError(ERR_NONE, "");
    }
#endif
    return decripted;
}

StringBuffer CTPConfig::encodePassword(const char* password) {

    StringBuffer buffer("");
#if 0
    // Encryption not supported yet!!! TODO FIXME
    if (password && strlen(password) > 0) {
        char* encoded = encryptData(password);
        buffer = encoded;
    }
#endif
    return buffer;
}

void CTPConfig::readCTPConfig() {

    if (!open()) {
        LOG.error("Impossible read the ctp configuration. exit");
        return;
    }

    if (readAccessConfig(*syncMLNode)) {
        setUsername(accessConfig.getUsername());
        // urlTo is loaded from 'ctpServer' key
        // nonce is read calling 'readAccessConfig()'
    }

    bool passEncoded = false;
    // TODO we should stringbuffer instead of malloc/new strings
    char* passDecoded;
    if (passEncoded) {
#if 0
        // FIXME TODO
        passDecoded = decodePassword(accessConfig.getPassword());
        accessConfig.setPassword(passDecoded);
#endif
    } else {
        passDecoded = stringdup(accessConfig.getPassword());
        accessConfig.setPassword(passDecoded);
    }
    delete [] passDecoded;

    if (readDeviceConfig(*syncMLNode)) {
        setDeviceId(deviceConfig.getDevID());

    }

    // now read the single CTP properties
    ManagementNode* node;
    char nodeName[DIM_MANAGEMENT_PATH];
    nodeName[0] = 0;
    sprintf(nodeName, "%s%s", rootContext, CONTEXT_PUSH_CTP);

    node = dmt->readManagementNode(nodeName);
    if (node)
    {
        char* tmp;
        tmp = node->readPropertyValue(PROPERTY_PUSH_NOTIFICATION);
        if (tmp) {
            setPush(atoi(tmp));
        } else {
            setPush(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(PROPERTY_POLLING_NOTIFICATION);
        if (tmp) {
            setPolling(atoi(tmp));
        } else {
            setPolling(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_QUEUE_PUSH);
        if (tmp) {
            setQueuePush(atoi(tmp) == 0 ? false : true);
        } else {
            setQueuePush(false);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(PROPERTY_NOTIFY_TIMEOUT);
        if (tmp) {
            setNotifyTimeout(atoi(tmp));
        } else {
            setNotifyTimeout(180);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_RETRY);
        if (tmp) {
            setCtpRetry(atoi(tmp));
        } else {
            setCtpRetry(5);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_MAX_RETRY_TIMEOUT);
        if (tmp) {
            setMaxCtpRetry(atoi(tmp));
        } else {
            setMaxCtpRetry(900); // 15 min
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_CMD_TIMEOUT);
        if (tmp) {
            setCtpCmdTimeout(atoi(tmp));
        } else {
            setCtpCmdTimeout(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_CONN_TIMEOUT);
        if (tmp) {
            setCtpConnTimeout(atoi(tmp));
        } else {
            setCtpConnTimeout(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_PORT);
        if (tmp) {
            setCtpPort(atoi(tmp));
        } else {
            setCtpPort(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_READY);
        if (tmp) {
            setCtpReady(atoi(tmp));
        } else {
            setCtpReady(5);
        }
        delete [] tmp;

        // server url is loaded from regstry key 'ctpServer'
        tmp = node->readPropertyValue(PROPERTY_CTP_SERVER);
        if (tmp && strlen(tmp)>0) {
            // *** workaround for prefix: remove this when not necessary! ***
            StringBuffer url = checkPrefix(tmp);
            LOG.debug("urlTo = %s", url.c_str());
            setUrlTo(url);
        } else {
            // If 'ctpServer' reg value is empty, extract the url from 'syncUrl'
            setUrlTo(getHostName(accessConfig.getSyncURL()));
        }
        delete [] tmp;

        delete node;
        node = NULL;
    }

    close();
    return;

}

void CTPConfig::saveCTPConfig() {

    ManagementNode* node = NULL;
    if (!open()) {
        return;
    }

    bool passwordEncoded = false;

    StringBuffer buffer;
    if (passwordEncoded) {
        buffer = encodePassword(accessConfig.getPassword());
    } else {
        buffer = accessConfig.getPassword();
    }
    accessConfig.setPassword(buffer.c_str());
    // Save the nonce: save AuthConfig properties
    char nodeName[DIM_MANAGEMENT_PATH];
    sprintf(nodeName, "%s%s%s", APPLICATION_URI, CONTEXT_SPDS_SYNCML, CONTEXT_AUTH);
    node = dmt->readManagementNode(nodeName);
    if (node) {
        saveAuthConfig(*syncMLNode, *node);
        delete node;
    }

    char* passDecoded;
    if (passwordEncoded) {
#if 0
        // TODO FIXME
        passDecoded = decodePassword(accessConfig.getPassword());
        accessConfig.setPassword(passDecoded);
        delete [] passDecoded;
#endif
    } else {
        accessConfig.setPassword(accessConfig.getPassword());
    }
    close();
}

StringBuffer CTPConfig::getHostName(StringBuffer syncUrl) {

    size_t start, end;
    StringBuffer host;
    // Extract the hostName from syncUrl: "http://<hostName>:8080/funambol/ds"
    start = syncUrl.find("://", 0);
    if (start != StringBuffer::npos) {
        start += 3;
    } else {
        // try to extract hostname from <hostName>:8080/funambol/ds"
        start = 0;
    }

    end = syncUrl.find(":", start);       // stop if ":" or "/" found
    if (end == StringBuffer::npos) {
        end = syncUrl.find("/", start);
        if (end == StringBuffer::npos) {  // so the url is only <hostName>
            end = syncUrl.length();
        }
    }

    if (end > start) {
        host = syncUrl.substr(start, end-start);
    }

    return host;
}

int CTPConfig::getHostPort(StringBuffer syncUrl) {

    size_t start, endSlash, endColon, urlLength;
    StringBuffer hostPort = "";
    urlLength = syncUrl.length();
    int port = 0;
    // Extract the port from syncUrl: "http://<hostName>:8080/funambol/ds"
    start = syncUrl.find("://", 0);
    if (start != StringBuffer::npos) {
        start += 3;
    } else {
        // try to extract the port from <hostName>:8080/funambol/ds"
        start = 0;
    }

    endSlash = syncUrl.find("/", start);       // stop if ":" or "/" found
    endColon = syncUrl.find(":", start);       // stop if ":" or "/" found

    if (endSlash == StringBuffer::npos && endColon == StringBuffer::npos) {
        // there is no port
        // hostname
    } else if (endSlash != StringBuffer::npos && endColon == StringBuffer::npos) {
        // there is no port
        // hostname/funambol
    } else if (endSlash == StringBuffer::npos && endColon != StringBuffer::npos) {
        // there is port
        // hostname:8080
        hostPort = syncUrl.substr(endColon + 1, urlLength - start);
    } else {
        if (endSlash > endColon) {
            // there is port
            // hostname:8080/funambol
            hostPort = syncUrl.substr(endColon + 1, endSlash - endColon - 1);
        }
    }

    if (hostPort != "") {
        port = atoi(hostPort.c_str());
    }

    return port;
}

/// If passed 'url' contains "prefix:" the value is appended
/// next to the host name: <prefix><hostName> is returned
/// (hostName is retrieved from SyncUrl)
/// Otherwise the passed 'url' is returned as StringBuffer.
StringBuffer CTPConfig::checkPrefix(char* url) {

    StringBuffer ctpUrl = url;
    const StringBuffer prefix = "prefix:";

    size_t pos = ctpUrl.find(prefix, 0);
    if (pos == StringBuffer::npos) {
        // Not found: keep the url passed
        return ctpUrl;
    }

    // Go after the ":"
    pos += prefix.length();
    if (pos != StringBuffer::npos) {
        // Add the prefix
        ctpUrl = ctpUrl.substr(pos, ctpUrl.length());
        // Append the syncUrl
        ctpUrl += getHostName(accessConfig.getSyncURL());
    }

    return ctpUrl;
}

<commit_msg>fixed bug. don't use setVar(getVar()), the same pointer is deleted during the set operation<commit_after>/*
 * Funambol is a mobile platform developed by Funambol, Inc.
 * Copyright (C) 2003 - 2007 Funambol, Inc.
 *
 * This program is free software; you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License version 3 as published by
 * the Free Software Foundation with the addition of the following permission
 * added to Section 15 as permitted in Section 7(a): FOR ANY PART OF THE COVERED
 * WORK IN WHICH THE COPYRIGHT IS OWNED BY FUNAMBOL, FUNAMBOL DISCLAIMS THE
 * WARRANTY OF NON INFRINGEMENT  OF THIRD PARTY RIGHTS.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, see http://www.gnu.org/licenses or write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 * MA 02110-1301 USA.
 *
 * You can contact Funambol, Inc. headquarters at 643 Bair Island Road, Suite
 * 305, Redwood City, CA 94063, USA, or at email address info@funambol.com.
 *
 * The interactive user interfaces in modified source and object code versions
 * of this program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU Affero General Public License version 3.
 *
 * In accordance with Section 7(b) of the GNU Affero General Public License
 * version 3, these Appropriate Legal Notices must retain the display of the
 * "Powered by Funambol" logo. If the display of the logo is not reasonably
 * feasible for technical reasons, the Appropriate Legal Notices must display
 * the words "Powered by Funambol".
 */

#if 0
#include "base/EncodingUtils.h"
#endif

#include "base/util/StringBuffer.h"
#include "spdm/constants.h"
#include "push/CTPConfig.h"
#include "base/globalsdef.h"

USE_NAMESPACE

#define CTP_QUEUE_PUSH                  "queuePush"
#define CTP_RETRY                       "ctpRetry"
#define CTP_MAX_RETRY_TIMEOUT           "maxCtpRetry"
#define CTP_CMD_TIMEOUT                 "ctpCmdTimeout"
#define CTP_CONN_TIMEOUT                "ctpConnTimeout"
#define CTP_PORT                        "ctpPort"
#define CTP_READY                       "ctpReady"
#define PROPERTY_CTP_SERVER             "ctpServer"
#define PROPERTY_NOTIFY_TIMEOUT         "notifyTimeout"

CTPConfig::CTPConfig(const char* application_uri)
                    : DMTClientConfig(application_uri) {
}

CTPConfig::~CTPConfig() {}

char* CTPConfig::decodePassword(const char* password) {

    char* decripted = NULL;
#if 0
    // Encryption not supported yet!!! TODO FIXME
    if (password && strlen(password) > 0) {
         decripted = decryptData(password);
    }

    if (decripted == NULL || getLastErrorCode() == 801) {
        decripted = new char[1];
        decripted[0] = 0;
        setError(ERR_NONE, "");
    }
#endif
    return decripted;
}

StringBuffer CTPConfig::encodePassword(const char* password) {

    StringBuffer buffer("");
#if 0
    // Encryption not supported yet!!! TODO FIXME
    if (password && strlen(password) > 0) {
        char* encoded = encryptData(password);
        buffer = encoded;
    }
#endif
    return buffer;
}

void CTPConfig::readCTPConfig() {

    if (!open()) {
        LOG.error("Impossible read the ctp configuration. exit");
        return;
    }

    if (readAccessConfig(*syncMLNode)) {
        setUsername(accessConfig.getUsername());
        // urlTo is loaded from 'ctpServer' key
        // nonce is read calling 'readAccessConfig()'
    }

    bool passEncoded = false;
    // TODO we should stringbuffer instead of malloc/new strings
    char* passDecoded;
    if (passEncoded) {
#if 0
        // FIXME TODO
        passDecoded = decodePassword(accessConfig.getPassword());
        accessConfig.setPassword(passDecoded);
#endif
    } else {
        passDecoded = stringdup(accessConfig.getPassword());
        accessConfig.setPassword(passDecoded);
    }
    delete [] passDecoded;

    if (readDeviceConfig(*syncMLNode)) {
        setDeviceId(deviceConfig.getDevID());

    }

    // now read the single CTP properties
    ManagementNode* node;
    char nodeName[DIM_MANAGEMENT_PATH];
    nodeName[0] = 0;
    sprintf(nodeName, "%s%s", rootContext, CONTEXT_PUSH_CTP);

    node = dmt->readManagementNode(nodeName);
    if (node)
    {
        char* tmp;
        tmp = node->readPropertyValue(PROPERTY_PUSH_NOTIFICATION);
        if (tmp) {
            setPush(atoi(tmp));
        } else {
            setPush(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(PROPERTY_POLLING_NOTIFICATION);
        if (tmp) {
            setPolling(atoi(tmp));
        } else {
            setPolling(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_QUEUE_PUSH);
        if (tmp) {
            setQueuePush(atoi(tmp) == 0 ? false : true);
        } else {
            setQueuePush(false);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(PROPERTY_NOTIFY_TIMEOUT);
        if (tmp) {
            setNotifyTimeout(atoi(tmp));
        } else {
            setNotifyTimeout(180);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_RETRY);
        if (tmp) {
            setCtpRetry(atoi(tmp));
        } else {
            setCtpRetry(5);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_MAX_RETRY_TIMEOUT);
        if (tmp) {
            setMaxCtpRetry(atoi(tmp));
        } else {
            setMaxCtpRetry(900); // 15 min
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_CMD_TIMEOUT);
        if (tmp) {
            setCtpCmdTimeout(atoi(tmp));
        } else {
            setCtpCmdTimeout(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_CONN_TIMEOUT);
        if (tmp) {
            setCtpConnTimeout(atoi(tmp));
        } else {
            setCtpConnTimeout(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_PORT);
        if (tmp) {
            setCtpPort(atoi(tmp));
        } else {
            setCtpPort(0);
        }
        delete [] tmp;

        tmp = node->readPropertyValue(CTP_READY);
        if (tmp) {
            setCtpReady(atoi(tmp));
        } else {
            setCtpReady(5);
        }
        delete [] tmp;

        // server url is loaded from regstry key 'ctpServer'
        tmp = node->readPropertyValue(PROPERTY_CTP_SERVER);
        if (tmp && strlen(tmp)>0) {
            // *** workaround for prefix: remove this when not necessary! ***
            StringBuffer url = checkPrefix(tmp);
            LOG.debug("urlTo = %s", url.c_str());
            setUrlTo(url);
        } else {
            // If 'ctpServer' reg value is empty, extract the url from 'syncUrl'
            setUrlTo(getHostName(accessConfig.getSyncURL()));
        }
        delete [] tmp;

        delete node;
        node = NULL;
    }

    close();
    return;

}

void CTPConfig::saveCTPConfig() {

    ManagementNode* node = NULL;
    if (!open()) {
        return;
    }

    bool passwordEncoded = false;

    StringBuffer buffer;
    if (passwordEncoded) {
        buffer = encodePassword(accessConfig.getPassword());
    } else {
        buffer = accessConfig.getPassword();
    }
    accessConfig.setPassword(buffer.c_str());
    // Save the nonce: save AuthConfig properties
    char nodeName[DIM_MANAGEMENT_PATH];
    sprintf(nodeName, "%s%s%s", APPLICATION_URI, CONTEXT_SPDS_SYNCML, CONTEXT_AUTH);
    node = dmt->readManagementNode(nodeName);
    if (node) {
        saveAuthConfig(*syncMLNode, *node);
        delete node;
    }

    char* passDecoded;
    if (passwordEncoded) {
#if 0
        // TODO FIXME
        passDecoded = decodePassword(accessConfig.getPassword());
        accessConfig.setPassword(passDecoded);
        delete [] passDecoded;
#endif
    } else {
        // Nothing to do: the password is already in clear text.
    }
    close();
}

StringBuffer CTPConfig::getHostName(StringBuffer syncUrl) {

    size_t start, end;
    StringBuffer host;
    // Extract the hostName from syncUrl: "http://<hostName>:8080/funambol/ds"
    start = syncUrl.find("://", 0);
    if (start != StringBuffer::npos) {
        start += 3;
    } else {
        // try to extract hostname from <hostName>:8080/funambol/ds"
        start = 0;
    }

    end = syncUrl.find(":", start);       // stop if ":" or "/" found
    if (end == StringBuffer::npos) {
        end = syncUrl.find("/", start);
        if (end == StringBuffer::npos) {  // so the url is only <hostName>
            end = syncUrl.length();
        }
    }

    if (end > start) {
        host = syncUrl.substr(start, end-start);
    }

    return host;
}

int CTPConfig::getHostPort(StringBuffer syncUrl) {

    size_t start, endSlash, endColon, urlLength;
    StringBuffer hostPort = "";
    urlLength = syncUrl.length();
    int port = 0;
    // Extract the port from syncUrl: "http://<hostName>:8080/funambol/ds"
    start = syncUrl.find("://", 0);
    if (start != StringBuffer::npos) {
        start += 3;
    } else {
        // try to extract the port from <hostName>:8080/funambol/ds"
        start = 0;
    }

    endSlash = syncUrl.find("/", start);       // stop if ":" or "/" found
    endColon = syncUrl.find(":", start);       // stop if ":" or "/" found

    if (endSlash == StringBuffer::npos && endColon == StringBuffer::npos) {
        // there is no port
        // hostname
    } else if (endSlash != StringBuffer::npos && endColon == StringBuffer::npos) {
        // there is no port
        // hostname/funambol
    } else if (endSlash == StringBuffer::npos && endColon != StringBuffer::npos) {
        // there is port
        // hostname:8080
        hostPort = syncUrl.substr(endColon + 1, urlLength - start);
    } else {
        if (endSlash > endColon) {
            // there is port
            // hostname:8080/funambol
            hostPort = syncUrl.substr(endColon + 1, endSlash - endColon - 1);
        }
    }

    if (hostPort != "") {
        port = atoi(hostPort.c_str());
    }

    return port;
}

/// If passed 'url' contains "prefix:" the value is appended
/// next to the host name: <prefix><hostName> is returned
/// (hostName is retrieved from SyncUrl)
/// Otherwise the passed 'url' is returned as StringBuffer.
StringBuffer CTPConfig::checkPrefix(char* url) {

    StringBuffer ctpUrl = url;
    const StringBuffer prefix = "prefix:";

    size_t pos = ctpUrl.find(prefix, 0);
    if (pos == StringBuffer::npos) {
        // Not found: keep the url passed
        return ctpUrl;
    }

    // Go after the ":"
    pos += prefix.length();
    if (pos != StringBuffer::npos) {
        // Add the prefix
        ctpUrl = ctpUrl.substr(pos, ctpUrl.length());
        // Append the syncUrl
        ctpUrl += getHostName(accessConfig.getSyncURL());
    }

    return ctpUrl;
}

<|endoftext|>


<commit_before>#include <caffe2/core/init.h>
#include <caffe2/core/net.h>
#include <caffe2/utils/proto_utils.h>

#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>

#include <fstream>

CAFFE2_DEFINE_string(init_net, "res/squeezenet_init_net.pb",
                     "The given path to the init protobuffer.");
CAFFE2_DEFINE_string(predict_net, "res/squeezenet_predict_net.pb",
                     "The given path to the predict protobuffer.");
CAFFE2_DEFINE_string(file, "res/image_file.jpg", "The image file.");
CAFFE2_DEFINE_string(classes, "res/imagenet_classes.txt", "The classes file.");
CAFFE2_DEFINE_int(size, 227, "The image file.");

namespace caffe2 {

void run() {
  std::cout << std::endl;
  std::cout << "## Caffe2 Loading Pre-Trained Models Tutorial ##" << std::endl;
  std::cout << "https://caffe2.ai/docs/zoo.html" << std::endl;
  std::cout << "https://caffe2.ai/docs/tutorial-loading-pre-trained-models.html"
            << std::endl;
  std::cout << "https://caffe2.ai/docs/tutorial-image-pre-processing.html"
            << std::endl;
  std::cout << std::endl;

  if (!std::ifstream(FLAGS_init_net).good() ||
      !std::ifstream(FLAGS_predict_net).good()) {
    std::cerr << "error: Squeezenet model file missing: "
              << (std::ifstream(FLAGS_init_net).good() ? FLAGS_predict_net
                                                       : FLAGS_init_net)
              << std::endl;
    std::cerr << "Make sure to first run ./script/download_resource.sh"
              << std::endl;
    return;
  }

  if (!std::ifstream(FLAGS_file).good()) {
    std::cerr << "error: Image file missing: " << FLAGS_file << std::endl;
    return;
  }

  if (!std::ifstream(FLAGS_classes).good()) {
    std::cerr << "error: Classes file invalid: " << FLAGS_classes << std::endl;
    return;
  }

  std::cout << "init-net: " << FLAGS_init_net << std::endl;
  std::cout << "predict-net: " << FLAGS_predict_net << std::endl;
  std::cout << "file: " << FLAGS_file << std::endl;
  std::cout << "size: " << FLAGS_size << std::endl;

  std::cout << std::endl;

  // >>> img =
  // skimage.img_as_float(skimage.io.imread(IMAGE_LOCATION)).astype(np.float32)
  auto image = cv::imread(FLAGS_file);  // CV_8UC3
  std::cout << "image size: " << image.size() << std::endl;

  // scale image to fit
  cv::Size scale(std::max(FLAGS_size * image.cols / image.rows, FLAGS_size),
                 std::max(FLAGS_size, FLAGS_size * image.rows / image.cols));
  cv::resize(image, image, scale);
  std::cout << "scaled size: " << image.size() << std::endl;

  // crop image to fit
  cv::Rect crop((image.cols - FLAGS_size) / 2, (image.rows - FLAGS_size) / 2,
                FLAGS_size, FLAGS_size);
  image = image(crop);
  std::cout << "cropped size: " << image.size() << std::endl;

  // convert to float, normalize to mean 128
  image.convertTo(image, CV_32FC3, 1.0, -128);
  std::cout << "value range: ("
            << *std::min_element((float *)image.datastart,
                                 (float *)image.dataend)
            << ", "
            << *std::max_element((float *)image.datastart,
                                 (float *)image.dataend)
            << ")" << std::endl;

  // convert NHWC to NCHW
  vector<cv::Mat> channels(3);
  cv::split(image, channels);
  std::vector<float> data;
  for (auto &c : channels) {
    data.insert(data.end(), (float *)c.datastart, (float *)c.dataend);
  }
  std::vector<TIndex> dims({1, image.channels(), image.rows, image.cols});
  TensorCPU tensor(dims, data, NULL);

  // Load Squeezenet model
  NetDef init_net, predict_net;

  // >>> with open(path_to_INIT_NET) as f:
  CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &init_net));

  // >>> with open(path_to_PREDICT_NET) as f:
  CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predict_net));

  // >>> p = workspace.Predictor(init_net, predict_net)
  Workspace workspace("tmp");
  CAFFE_ENFORCE(workspace.RunNetOnce(init_net));
  auto input = workspace.CreateBlob("data")->GetMutable<TensorCPU>();
  input->ResizeLike(tensor);
  input->ShareData(tensor);
  CAFFE_ENFORCE(workspace.RunNetOnce(predict_net));

  // >>> results = p.run([img])
  auto output = workspace.GetBlob("softmaxout")->Get<TensorCPU>();

  // sort top results
  const auto &probs = output.data<float>();
  std::vector<std::pair<int, int>> pairs;
  for (auto i = 0; i < output.size(); i++) {
    if (probs[i] > 0.01) {
      pairs.push_back(std::make_pair(probs[i] * 100, i));
    }
  }

  std::sort(pairs.begin(), pairs.end());

  std::cout << std::endl;

  // read classes
  std::ifstream file(FLAGS_classes);
  std::string temp;
  std::vector<std::string> classes;
  while (std::getline(file, temp)) {
    classes.push_back(temp);
  }

  // show results
  std::cout << "output: " << std::endl;
  for (auto pair : pairs) {
    std::cout << "  " << pair.first << "% '" << classes[pair.second] << "' ("
              << pair.second << ")" << std::endl;
  }
}

}  // namespace caffe2

int main(int argc, char **argv) {
  caffe2::GlobalInit(&argc, &argv);
  caffe2::run();
  google::protobuf::ShutdownProtobufLibrary();
  return 0;
}
<commit_msg>fix pretrained on googlenet<commit_after>#include <caffe2/core/init.h>
#include <caffe2/core/net.h>
#include <caffe2/utils/proto_utils.h>

#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>

#include <fstream>

CAFFE2_DEFINE_string(init_net, "res/squeezenet_init_net.pb",
                     "The given path to the init protobuffer.");
CAFFE2_DEFINE_string(predict_net, "res/squeezenet_predict_net.pb",
                     "The given path to the predict protobuffer.");
CAFFE2_DEFINE_string(file, "res/image_file.jpg", "The image file.");
CAFFE2_DEFINE_string(classes, "res/imagenet_classes.txt", "The classes file.");
CAFFE2_DEFINE_int(size, 227, "The image file.");

namespace caffe2 {

void run() {
  std::cout << std::endl;
  std::cout << "## Caffe2 Loading Pre-Trained Models Tutorial ##" << std::endl;
  std::cout << "https://caffe2.ai/docs/zoo.html" << std::endl;
  std::cout << "https://caffe2.ai/docs/tutorial-loading-pre-trained-models.html"
            << std::endl;
  std::cout << "https://caffe2.ai/docs/tutorial-image-pre-processing.html"
            << std::endl;
  std::cout << std::endl;

  if (!std::ifstream(FLAGS_init_net).good() ||
      !std::ifstream(FLAGS_predict_net).good()) {
    std::cerr << "error: Squeezenet model file missing: "
              << (std::ifstream(FLAGS_init_net).good() ? FLAGS_predict_net
                                                       : FLAGS_init_net)
              << std::endl;
    std::cerr << "Make sure to first run ./script/download_resource.sh"
              << std::endl;
    return;
  }

  if (!std::ifstream(FLAGS_file).good()) {
    std::cerr << "error: Image file missing: " << FLAGS_file << std::endl;
    return;
  }

  if (!std::ifstream(FLAGS_classes).good()) {
    std::cerr << "error: Classes file invalid: " << FLAGS_classes << std::endl;
    return;
  }

  std::cout << "init-net: " << FLAGS_init_net << std::endl;
  std::cout << "predict-net: " << FLAGS_predict_net << std::endl;
  std::cout << "file: " << FLAGS_file << std::endl;
  std::cout << "size: " << FLAGS_size << std::endl;

  std::cout << std::endl;

  // >>> img =
  // skimage.img_as_float(skimage.io.imread(IMAGE_LOCATION)).astype(np.float32)
  auto image = cv::imread(FLAGS_file);  // CV_8UC3
  std::cout << "image size: " << image.size() << std::endl;

  // scale image to fit
  cv::Size scale(std::max(FLAGS_size * image.cols / image.rows, FLAGS_size),
                 std::max(FLAGS_size, FLAGS_size * image.rows / image.cols));
  cv::resize(image, image, scale);
  std::cout << "scaled size: " << image.size() << std::endl;

  // crop image to fit
  cv::Rect crop((image.cols - FLAGS_size) / 2, (image.rows - FLAGS_size) / 2,
                FLAGS_size, FLAGS_size);
  image = image(crop);
  std::cout << "cropped size: " << image.size() << std::endl;

  // convert to float, normalize to mean 128
  image.convertTo(image, CV_32FC3, 1.0, -128);
  std::cout << "value range: ("
            << *std::min_element((float *)image.datastart,
                                 (float *)image.dataend)
            << ", "
            << *std::max_element((float *)image.datastart,
                                 (float *)image.dataend)
            << ")" << std::endl;

  // convert NHWC to NCHW
  vector<cv::Mat> channels(3);
  cv::split(image, channels);
  std::vector<float> data;
  for (auto &c : channels) {
    data.insert(data.end(), (float *)c.datastart, (float *)c.dataend);
  }
  std::vector<TIndex> dims({1, image.channels(), image.rows, image.cols});
  TensorCPU tensor(dims, data, NULL);

  // Load Squeezenet model
  NetDef init_net, predict_net;

  // >>> with open(path_to_INIT_NET) as f:
  CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &init_net));

  // >>> with open(path_to_PREDICT_NET) as f:
  CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predict_net));

  // >>> p = workspace.Predictor(init_net, predict_net)
  Workspace workspace("tmp");
  CAFFE_ENFORCE(workspace.RunNetOnce(init_net));
  auto input = workspace.CreateBlob("data")->GetMutable<TensorCPU>();
  input->ResizeLike(tensor);
  input->ShareData(tensor);
  CAFFE_ENFORCE(workspace.RunNetOnce(predict_net));

  // >>> results = p.run([img])
  auto &output_name = predict_net.external_output(0);
  auto output = workspace.GetBlob(output_name)->Get<TensorCPU>();

  // sort top results
  const auto &probs = output.data<float>();
  std::vector<std::pair<int, int>> pairs;
  for (auto i = 0; i < output.size(); i++) {
    if (probs[i] > 0.01) {
      pairs.push_back(std::make_pair(probs[i] * 100, i));
    }
  }

  std::sort(pairs.begin(), pairs.end());

  std::cout << std::endl;

  // read classes
  std::ifstream file(FLAGS_classes);
  std::string temp;
  std::vector<std::string> classes;
  while (std::getline(file, temp)) {
    classes.push_back(temp);
  }

  // show results
  std::cout << "output: " << std::endl;
  for (auto pair : pairs) {
    std::cout << "  " << pair.first << "% '" << classes[pair.second] << "' ("
              << pair.second << ")" << std::endl;
  }
}

}  // namespace caffe2

int main(int argc, char **argv) {
  caffe2::GlobalInit(&argc, &argv);
  caffe2::run();
  google::protobuf::ShutdownProtobufLibrary();
  return 0;
}
<|endoftext|>