<commit_before>/***********************************************************\
 * structures that describe the collected enum/struct info *
 *     ___________ _____                                   *
 *    /  ___|  ___/ __  \                                  *
 *    \ `--.| |_  `' / /'                                  *
 *     `--. \  _|   / /                                    *
 *    /\__/ / |   ./ /___                                  *
 *    \____/\_|   \_____/                                  *
 *                                                         *
 *                                                         *
 *  Copyright (c) 2014 Florian Oetke                       *
 *                                                         *
 *  This file is part of SF2 and distributed under         *
 *  the MIT License. See LICENSE file for details.         *
\***********************************************************/

#pragma once

#include <string>
#include <map>
#include <unordered_map>
#include <array>

#include <iostream>

namespace sf2 {

	namespace details {
		constexpr auto calc_hash(const char* data, std::size_t len)noexcept -> std::size_t {
			std::size_t h = 0;
			for(std::size_t i=0; i<len; ++i)
				h = h*101 + data[i];

			return h;
		}
	}

	struct String_literal {
		const char* data;
		std::size_t len;
		std::size_t hash;

		template<std::size_t N>
		constexpr String_literal(const char (&data)[N]) : data(data), len(N-1), hash(details::calc_hash(data, len)) {

		}
		String_literal(const std::string& str) : data(str.data()), len(str.size()), hash(details::calc_hash(data,len)) {
		}

		bool operator==(const String_literal& rhs)const {
			if(len!=rhs.len) {
				return false;
			}
			if(hash!=rhs.hash)
				return false;

			for(std::size_t i=0; i<len; ++i) {
				if(data[i]!=rhs.data[i])
					return false;
			}

			return true;
		}
		bool operator==(const char* rhs)const {
			for(std::size_t i=0; i<len; ++i) {
				if(rhs[i]==0 || data[i]!=rhs[i])
					return false;
			}

			if(rhs[len]!=0)
				return false;

			return true;
		}
	};
}
namespace std {
	template<>
	struct hash<sf2::String_literal> {
		size_t operator()(const sf2::String_literal& s) const {
			return s.hash;
		}
	};
}

namespace sf2 {
	template<typename T>
	class Enum_info {
		public:
			using Value_type = std::pair<T, String_literal>;

		public:
			Enum_info(String_literal name, std::initializer_list<Value_type> v)
			    : _name(name) {
				for(auto& e : v) {
					_names.emplace(e.first, e.second);
					_values.emplace(e.second, e.first);
				}
			}

			auto name()const noexcept {return _name;}

			auto value_of(const String_literal& name)const noexcept -> T {
				auto i = _values.find(name);
				assert(i!=_values.end());
				return i->second;
			}
			auto value_of(const std::string& name)const noexcept -> T {
				auto i = _values.find(String_literal{name});
				assert(i!=_values.end());
				return i->second;
			}

			auto name_of (T value)const noexcept -> String_literal {
				auto i = _names.find(value);
				assert(i!=_names.end());
				return i->second;
			}

		private:
			struct Enum_hash {
				auto operator()(T v)const {
					return static_cast<std::size_t>(v);
				}
			};

			String_literal _name;
			std::unordered_map<T, String_literal, Enum_hash> _names;
			std::unordered_map<String_literal, T> _values;
	};

	template<typename ST, typename MT>
	using Member_ptr = MT ST::*;

	template<typename T, typename MemberT>
	using Member_data = std::tuple<MemberT T::*, String_literal>;

	template<typename T, typename... MemberT>
	class Struct_info {
		public:
			static constexpr std::size_t member_count = sizeof...(MemberT);

			constexpr Struct_info(String_literal name, Member_data<T,MemberT>... members) : _name(name), _member_names{{std::get<1>(members)...}}, _members(std::get<0>(members)...) {
			}

			constexpr auto name()const {return _name;}
			constexpr auto& members()const {return _member_names;}
			constexpr auto& member_ptrs()const {return _members;}
			constexpr auto size()const {return member_count;}

			template<std::size_t I = 0, typename FuncT>
			inline typename std::enable_if<I == member_count, void>::type
			for_each( FuncT)const {}

			template<std::size_t I = 0, typename FuncT>
			inline typename std::enable_if<I < member_count, void>::type
			for_each(FuncT f)const {
				f(_member_names[I], std::get<I>(_members));
				for_each<I + 1, FuncT>(f);
			}

		private:
			String_literal _name;
			std::array<String_literal, member_count> _member_names;
			std::tuple<Member_ptr<T,MemberT>...> _members;
	};

	template<typename T>
	auto get_enum_info() -> decltype(sf2_enum_info_factory((T*)nullptr)) {
		return sf2_enum_info_factory((T*)nullptr);
	}

	template<typename T>
	auto get_struct_info() -> decltype(sf2_struct_info_factory((T*)nullptr)) {
		return sf2_struct_info_factory((T*)nullptr);
	}

	template<class T>
	struct is_annotated_struct {
		private:
			typedef char one;
			typedef long two;

			template <typename C> static one test( std::remove_reference_t<decltype(sf2_struct_info_factory((C*)nullptr))>* ) ;
			template <typename C> static two test(...);

		public:
			enum { value = sizeof(test<T>(0)) == sizeof(char) };
	};

	template<class T>
	struct is_annotated_enum {
		private:
			typedef char one;
			typedef long two;

			template <typename C> static one test( std::remove_reference_t<decltype(sf2_enum_info_factory((C*)nullptr))>* ) ;
			template <typename C> static two test(...);

		public:
			enum { value = sizeof(test<T>(0)) == sizeof(char) };
	};

	template<class T>
	struct is_annotated {
		enum { value = is_annotated_struct<T>::value ||
		               is_annotated_enum<T>::value };
	};

}
<commit_msg>added missing include for cassert<commit_after>/***********************************************************\
 * structures that describe the collected enum/struct info *
 *     ___________ _____                                   *
 *    /  ___|  ___/ __  \                                  *
 *    \ `--.| |_  `' / /'                                  *
 *     `--. \  _|   / /                                    *
 *    /\__/ / |   ./ /___                                  *
 *    \____/\_|   \_____/                                  *
 *                                                         *
 *                                                         *
 *  Copyright (c) 2014 Florian Oetke                       *
 *                                                         *
 *  This file is part of SF2 and distributed under         *
 *  the MIT License. See LICENSE file for details.         *
\***********************************************************/

#pragma once

#include <string>
#include <map>
#include <unordered_map>
#include <array>
#include <cassert>

#include <iostream>

namespace sf2 {

	namespace details {
		constexpr auto calc_hash(const char* data, std::size_t len)noexcept -> std::size_t {
			std::size_t h = 0;
			for(std::size_t i=0; i<len; ++i)
				h = h*101 + data[i];

			return h;
		}
	}

	struct String_literal {
		const char* data;
		std::size_t len;
		std::size_t hash;

		template<std::size_t N>
		constexpr String_literal(const char (&data)[N]) : data(data), len(N-1), hash(details::calc_hash(data, len)) {

		}
		String_literal(const std::string& str) : data(str.data()), len(str.size()), hash(details::calc_hash(data,len)) {
		}

		bool operator==(const String_literal& rhs)const {
			if(len!=rhs.len) {
				return false;
			}
			if(hash!=rhs.hash)
				return false;

			for(std::size_t i=0; i<len; ++i) {
				if(data[i]!=rhs.data[i])
					return false;
			}

			return true;
		}
		bool operator==(const char* rhs)const {
			for(std::size_t i=0; i<len; ++i) {
				if(rhs[i]==0 || data[i]!=rhs[i])
					return false;
			}

			if(rhs[len]!=0)
				return false;

			return true;
		}
	};
}
namespace std {
	template<>
	struct hash<sf2::String_literal> {
		size_t operator()(const sf2::String_literal& s) const {
			return s.hash;
		}
	};
}

namespace sf2 {
	template<typename T>
	class Enum_info {
		public:
			using Value_type = std::pair<T, String_literal>;

		public:
			Enum_info(String_literal name, std::initializer_list<Value_type> v)
			    : _name(name) {
				for(auto& e : v) {
					_names.emplace(e.first, e.second);
					_values.emplace(e.second, e.first);
				}
			}

			auto name()const noexcept {return _name;}

			auto value_of(const String_literal& name)const noexcept -> T {
				auto i = _values.find(name);
				assert(i!=_values.end());
				return i->second;
			}
			auto value_of(const std::string& name)const noexcept -> T {
				auto i = _values.find(String_literal{name});
				assert(i!=_values.end());
				return i->second;
			}

			auto name_of (T value)const noexcept -> String_literal {
				auto i = _names.find(value);
				assert(i!=_names.end());
				return i->second;
			}

		private:
			struct Enum_hash {
				auto operator()(T v)const {
					return static_cast<std::size_t>(v);
				}
			};

			String_literal _name;
			std::unordered_map<T, String_literal, Enum_hash> _names;
			std::unordered_map<String_literal, T> _values;
	};

	template<typename ST, typename MT>
	using Member_ptr = MT ST::*;

	template<typename T, typename MemberT>
	using Member_data = std::tuple<MemberT T::*, String_literal>;

	template<typename T, typename... MemberT>
	class Struct_info {
		public:
			static constexpr std::size_t member_count = sizeof...(MemberT);

			constexpr Struct_info(String_literal name, Member_data<T,MemberT>... members) : _name(name), _member_names{{std::get<1>(members)...}}, _members(std::get<0>(members)...) {
			}

			constexpr auto name()const {return _name;}
			constexpr auto& members()const {return _member_names;}
			constexpr auto& member_ptrs()const {return _members;}
			constexpr auto size()const {return member_count;}

			template<std::size_t I = 0, typename FuncT>
			inline typename std::enable_if<I == member_count, void>::type
			for_each( FuncT)const {}

			template<std::size_t I = 0, typename FuncT>
			inline typename std::enable_if<I < member_count, void>::type
			for_each(FuncT f)const {
				f(_member_names[I], std::get<I>(_members));
				for_each<I + 1, FuncT>(f);
			}

		private:
			String_literal _name;
			std::array<String_literal, member_count> _member_names;
			std::tuple<Member_ptr<T,MemberT>...> _members;
	};

	template<typename T>
	auto get_enum_info() -> decltype(sf2_enum_info_factory((T*)nullptr)) {
		return sf2_enum_info_factory((T*)nullptr);
	}

	template<typename T>
	auto get_struct_info() -> decltype(sf2_struct_info_factory((T*)nullptr)) {
		return sf2_struct_info_factory((T*)nullptr);
	}

	template<class T>
	struct is_annotated_struct {
		private:
			typedef char one;
			typedef long two;

			template <typename C> static one test( std::remove_reference_t<decltype(sf2_struct_info_factory((C*)nullptr))>* ) ;
			template <typename C> static two test(...);

		public:
			enum { value = sizeof(test<T>(0)) == sizeof(char) };
	};

	template<class T>
	struct is_annotated_enum {
		private:
			typedef char one;
			typedef long two;

			template <typename C> static one test( std::remove_reference_t<decltype(sf2_enum_info_factory((C*)nullptr))>* ) ;
			template <typename C> static two test(...);

		public:
			enum { value = sizeof(test<T>(0)) == sizeof(char) };
	};

	template<class T>
	struct is_annotated {
		enum { value = is_annotated_struct<T>::value ||
		               is_annotated_enum<T>::value };
	};

}
<|endoftext|>


<commit_before>//----------------------------------------------------------------------------
/// \file   rate_throttler.hpp
/// \author Serge Aleynikov
//----------------------------------------------------------------------------
/// \brief Efficiently calculates the throttling rate over a number of seconds.
///
/// The algorithm implements a variation of token bucket algorithm that
/// doesn't require to add tokens to the bucket on a timer but rather it
/// maintains a circular buffer of tokens with resolution of 1/BucketsPerSec.
/// The basic_rate_throttler::add() function is used to adds items to a bucket
/// associated with the timestamp passed as the first argument to the
/// function.  The throttler::running_sum() returns the total number of
/// items over the given interval of seconds.
/// \note See also this article on a throttling algorithm
/// http://www.devquotes.com/2010/11/24/an-efficient-network-throttling-algorithm.
/// \note Another article on rate limiting
/// http://www.pennedobjects.com/2010/10/better-rate-limiting-with-dot-net.
//----------------------------------------------------------------------------
// Created: 2011-01-20
//----------------------------------------------------------------------------
/*
***** BEGIN LICENSE BLOCK *****

This file is part of the utxx open-source project.

Copyright (C) 2011 Serge Aleynikov <saleyn@gmail.com>

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

***** END LICENSE BLOCK *****
*/

#ifndef _UTXX_THROTTLER_HPP_
#define _UTXX_THROTTLER_HPP_

#include <utxx/error.hpp>
#include <utxx/meta.hpp>
#include <utxx/time_val.hpp>
#include <utxx/compiler_hints.hpp>
#include <time.h>

namespace utxx {

/// @brief Throttle given rate over a number of seconds.
/// Implementation uses time spacing reservation algorithm where each
/// allocation of samples reserves a fraction of space in the throttling
/// window. The reservation gets freed as the time goes by.  No more than
/// the "rate()" number of samples are allowed to fit in the "window_msec()"
/// window.
template <typename T = uint32_t>
class basic_time_spacing_throttle {
public:
    basic_time_spacing_throttle(uint32_t a_rate, uint32_t a_window_msec = 1000,
                                time_val a_now = now_utc())
        : m_rate        (a_rate)
        , m_window_ns   (long(a_window_msec) * 1000000)
        , m_step_ns     (m_window_ns / m_rate)
        , m_next_time   (a_now)
    {}

    /// Add \a a_samples to the throtlle's counter.
    /// @return number of samples that fit in the throttling window. 0 means
    /// that the throttler is fully congested, and more time needs to elapse
    /// before the throttles gets reset to accept more samples.
    T add(T a_samples  = 1, time_val a_now = now_utc()) {
        auto next_time = m_next_time;
        next_time.add_nsec(a_samples * m_step_ns);
        auto diff      = next_time.nanoseconds()
                       - (a_now.nanoseconds() + m_window_ns);
        if (diff < 0)  {
            // All samples fit the throttling threshold
            m_next_time = next_time;
            return a_samples;
        }

        // Figure out how many samples fit the throttling threshold
        auto n = std::max<T>(0, a_samples - (T(diff) / m_step_ns));
        m_next_time.add_nsec(n * m_step_ns);
        return n;
    }

    T        rate()        const { return m_rate;                }
    long     step_msec()   const { return m_step_ns   / 1000000; }
    long     step_usec()   const { return m_step_ns   / 1000;    }
    long     window_msec() const { return m_window_ns / 1000000; }
    long     window_usec() const { return m_window_ns;           }
    time_val next_time()   const { return m_next_time;           }

    /// Return the number of available samples given \a a_now current time.
    T        available(time_val a_now = now_utc()) const      {
        auto   diff = (a_now - m_next_time).nanoseconds();
        return diff >= 0 ? m_rate : T(m_window_ns + diff) / m_step_ns;
    }

private:
    T        m_rate;
    long     m_window_ns;
    long     m_step_ns;
    time_val m_next_time;
};

using time_spacing_throttle = basic_time_spacing_throttle<>;

/**
 * \brief Efficiently calculates the throttling rate over a number of seconds.
 * The algorithm implements a variation of token bucket algorithm that
 * doesn't require to add tokens to the bucket on a timer but rather it
 * maintains a cirtular buffer of tokens with resolution of 1/BucketsPerSec.
 * The basic_rate_throttler::add() function is used to adds items to a bucket
 * associated with the timestamp passed as the first argument to the
 * function.  The throttler::running_sum() returns the total number of
 * items over the given interval of seconds. The items automatically expire
 * when the time moves on the successive invocations of the add() method.
 * @tparam MaxSeconds defines the max number of seconds of data to hold
 *          in the circuling buffer.
 * @tparam BucketsPerSec defines the number of bucket slots per second.
 *          The larger the number the more accurate the running sum will be.
 */
template<size_t MaxSeconds = 16, size_t BucketsPerSec = 2>
class basic_rate_throttler {
public:
    static const size_t s_max_seconds       = upper_power<MaxSeconds,2>::value;
    static const size_t s_buckets_per_sec   = upper_power<BucketsPerSec,2>::value;
    static const size_t s_log_buckets_sec   = log<s_buckets_per_sec,2>::value;
    static const size_t s_bucket_count      = s_max_seconds * s_buckets_per_sec;
    static const size_t s_bucket_mask       = s_bucket_count - 1;

    explicit basic_rate_throttler(int a_interval = 1)
        : m_interval(-1)
    {
        init(a_interval);
    }

    basic_rate_throttler           (const basic_rate_throttler&) = default;
    basic_rate_throttler           (basic_rate_throttler&&)      = default;
    basic_rate_throttler& operator=(const basic_rate_throttler&) = default;

    /// Initialize the internal buffer setting the throttling interval
    /// measured in seconds.
    void init(int a_throttle_interval)
        throw(badarg_error)
    {
        if (a_throttle_interval == m_interval)
            return;
        m_interval = a_throttle_interval << s_log_buckets_sec;
        if (a_throttle_interval < 0 ||
            (size_t)a_throttle_interval > s_bucket_count / s_buckets_per_sec)
            throw badarg_error("Invalid throttle interval:", a_throttle_interval);
        reset();
    }

    /// Reset the inturnal circular buffer.
    void reset() {
        memset(m_buckets, 0, sizeof(m_buckets));
        m_last_time     = 0;
        m_sum           = 0;
    }

    /// Return running interval.
    long    interval()     const { return m_interval >> s_log_buckets_sec; }
    /// Return current running sum over the interval.
    long    running_sum()  const { return m_sum; }
    /// Return current running sum over the interval.
    double  running_avg()  const { return (double)m_sum / (m_interval >> s_log_buckets_sec); }

    /// Add \a a_count number of items to the bucket associated with \a a_time.
    /// @param a_time is monotonically increasing time value.
    /// @param a_count is the count to add to the bucket associated with \a a_time.
    /// @return current running sum.
    long   add(time_val a_time, int a_count = 1);

    /// Update current timestamp.
    long   refresh(time_val a_time) { return add(a_time, 0); }

    /// Dump the internal state to stream
    void   dump(std::ostream& out, time_val a_time);
private:
    size_t  m_buckets[s_bucket_count];
    time_t  m_last_time;
    long    m_sum;
    long    m_interval;

    // must be power of two
    BOOST_STATIC_ASSERT((s_bucket_count & (s_bucket_count - 1)) == 0);
};

//------------------------------------------------------------------------------
// Implementation
//------------------------------------------------------------------------------
template<size_t MaxSeconds, size_t BucketsPerSec>
long basic_rate_throttler<MaxSeconds, BucketsPerSec>::
add(time_val a_time, int a_count)
{
    time_t l_now = (time_t)(a_time.seconds() * s_buckets_per_sec);
    if (unlikely(!m_last_time))
       m_last_time = l_now;
    int  l_bucket    = l_now & s_bucket_mask;
    long l_time_diff = l_now - m_last_time;
    if (unlikely(l_now < m_last_time)) {
        // Clock was adjusted
        m_buckets[l_bucket] = a_count;
        m_sum = a_count;
    } else if (!l_time_diff) {
        m_sum += a_count;
        m_buckets[l_bucket] += a_count;
    } else if (l_time_diff >= m_interval) {
        int l_start = (l_now - m_interval + 1) & s_bucket_mask;
        int l_end   = l_now & s_bucket_mask;
        for (int i = l_start; i != l_end; i = (i+1) & s_bucket_mask)
            m_buckets[i] = 0;
        m_buckets[l_bucket] = a_count;
        m_sum = a_count;
    } else {
        // Calculate sum of buckets from
        int l_valid_buckets = m_interval - l_time_diff;
        int l_start, l_end;
        if (l_valid_buckets <= (m_interval>>1)) {
            l_start = (l_now - m_interval + 1) & s_bucket_mask;
            l_end   = (m_last_time+1) & s_bucket_mask;
            m_sum = a_count;
            #ifdef THROTTLE_DEBUG
            printf("Summing %d through %d\n", l_start, l_end);
            #endif
            for (int i = l_start; i != l_end; i = (i+1) & s_bucket_mask) {
                m_sum += m_buckets[i];
            }
            l_start = l_end;
            l_end   = l_now & s_bucket_mask;
        } else {
            l_start = (m_last_time - m_interval + 1) & s_bucket_mask;
            l_end   = (l_now - m_interval + 1) & s_bucket_mask;
            #ifdef THROTTLE_DEBUG
            printf("Subtracting/resetting %d through %d\n", l_start, l_end);
            #endif
            for (int i = l_start; i != l_end; i = (i+1) & s_bucket_mask) {
                m_sum -= m_buckets[i];
                // BOOST_ASSERT(m_sum >= 0);
                if (m_sum < 0) {
                    m_sum = 0;
                    std::cerr << "ERROR " << __FILE__ << ':' << __LINE__ << std::endl;
                    dump(std::cerr, a_time);
                }
                m_buckets[i] = 0;
            }
            l_start = (m_last_time + 1) & s_bucket_mask;
            l_end   = l_now & s_bucket_mask;
            m_sum += a_count;
        }
        #ifdef THROTTLE_DEBUG
        printf("Resetting %d through %d\n", l_start, l_end);
        #endif
        // Reset values in intermediate buckets since there was no activity there
        for (int i = l_start; i != l_end; i = (i+1) & s_bucket_mask)
            m_buckets[i] = 0;
        m_buckets[l_bucket] = a_count;
    }
    m_last_time = l_now;
    #ifdef THROTTLE_DEBUG
    dump(std::cout);
    #endif

    return m_sum;
}

template<size_t MaxSeconds, size_t BucketsPerSec>
void basic_rate_throttler<MaxSeconds, BucketsPerSec>::
dump(std::ostream& out, time_val a_time)
{
    time_t l_now    = (time_t)(a_time.seconds() * s_buckets_per_sec);
    size_t l_bucket = l_now & s_bucket_mask;

    char buf[256];
    std::stringstream s;
    sprintf(buf, "last_time=%ld, last_bucket=%3zu, sum=%ld (interval=%ld)\n",
            m_last_time, l_bucket, m_sum, m_interval);
    s << buf;
    int k = 0;
    size_t n = (l_bucket-m_interval) & s_bucket_mask;
    for (size_t j=0; j < s_bucket_count; j++) {
        k += snprintf(buf+k, sizeof(buf)-k, "%3zu%c", j, (j == l_bucket || j == n) ? '|' : ' ');
        k += snprintf(buf+k, sizeof(buf)-k, "%s", (j < s_bucket_count-1) ? "" : "\n");
    }
    s << buf;
    k = 0;
    for (size_t j=0; j < s_bucket_count; j++) {
        k += snprintf(buf+k, sizeof(buf)-k, "%3zu%c", m_buckets[j], (j == l_bucket || j == n) ? '|' : ' ');
        k += snprintf(buf+k, sizeof(buf)-k, "%s", (j < s_bucket_count-1) ? "" : "\n");
    }
    s << buf;
    out << s.str();
}

} // namespace utxx

#endif // _UTXX_THROTTLER_HPP_
<commit_msg>Update rate_throttler.hpp<commit_after>//----------------------------------------------------------------------------
/// \file   rate_throttler.hpp
/// \author Serge Aleynikov
//----------------------------------------------------------------------------
/// \brief Efficiently calculates the throttling rate over a number of seconds.
///
/// The algorithm implements a variation of token bucket algorithm that
/// doesn't require to add tokens to the bucket on a timer but rather it
/// maintains a circular buffer of tokens with resolution of 1/BucketsPerSec.
/// The basic_rate_throttler::add() function is used to adds items to a bucket
/// associated with the timestamp passed as the first argument to the
/// function.  The throttler::running_sum() returns the total number of
/// items over the given interval of seconds.
/// \note See also this article on a throttling algorithm
/// http://www.devquotes.com/2010/11/24/an-efficient-network-throttling-algorithm.
/// \note Another article on rate limiting
/// http://www.pennedobjects.com/2010/10/better-rate-limiting-with-dot-net.
//----------------------------------------------------------------------------
// Created: 2011-01-20
//----------------------------------------------------------------------------
/*
***** BEGIN LICENSE BLOCK *****

This file is part of the utxx open-source project.

Copyright (C) 2011 Serge Aleynikov <saleyn@gmail.com>

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

***** END LICENSE BLOCK *****
*/

#pragma once

#include <utxx/error.hpp>
#include <utxx/meta.hpp>
#include <utxx/time_val.hpp>
#include <utxx/compiler_hints.hpp>
#include <time.h>

namespace utxx {

/// @brief Throttle given rate over a number of seconds.
/// Implementation uses time spacing reservation algorithm where each
/// allocation of samples reserves a fraction of space in the throttling
/// window. The reservation gets freed as the time goes by.  No more than
/// the "rate()" number of samples are allowed to fit in the "window_msec()"
/// window.
template <typename T = uint32_t>
class basic_time_spacing_throttle {
public:
    basic_time_spacing_throttle(uint32_t a_rate, uint32_t a_window_msec = 1000,
                                time_val a_now = now_utc())
        : m_rate        (a_rate)
        , m_window_ns   (long(a_window_msec) * 1000000)
        , m_step_ns     (m_window_ns / m_rate)
        , m_next_time   (a_now)
    {}

    /// Add \a a_samples to the throtlle's counter.
    /// @return number of samples that fit in the throttling window. 0 means
    /// that the throttler is fully congested, and more time needs to elapse
    /// before the throttles gets reset to accept more samples.
    T add(T a_samples  = 1, time_val a_now = now_utc()) {
        auto next_time = m_next_time;
        next_time.add_nsec(a_samples * m_step_ns);
        auto diff      = next_time.nanoseconds()
                       - (a_now.nanoseconds() + m_window_ns);
        if (diff < 0)  {
            // All samples fit the throttling threshold
            m_next_time = next_time;
            return a_samples;
        }

        // Figure out how many samples fit the throttling threshold
        auto n = std::max<T>(0, a_samples - (T(diff) / m_step_ns));
        m_next_time.add_nsec(n * m_step_ns);
        return n;
    }

    T        rate()        const { return m_rate;                }
    long     step_msec()   const { return m_step_ns   / 1000000; }
    long     step_usec()   const { return m_step_ns   / 1000;    }
    long     window_msec() const { return m_window_ns / 1000000; }
    long     window_usec() const { return m_window_ns;           }
    time_val next_time()   const { return m_next_time;           }

    /// Return the number of available samples given \a a_now current time.
    T        available(time_val a_now = now_utc()) const      {
        auto   diff = (a_now - m_next_time).nanoseconds();
        return diff >= 0 ? m_rate : T(m_window_ns + diff) / m_step_ns;
    }

private:
    T        m_rate;
    long     m_window_ns;
    long     m_step_ns;
    time_val m_next_time;
};

using time_spacing_throttle = basic_time_spacing_throttle<>;

/**
 * \brief Efficiently calculates the throttling rate over a number of seconds.
 * The algorithm implements a variation of token bucket algorithm that
 * doesn't require to add tokens to the bucket on a timer but rather it
 * maintains a cirtular buffer of tokens with resolution of 1/BucketsPerSec.
 * The basic_rate_throttler::add() function is used to adds items to a bucket
 * associated with the timestamp passed as the first argument to the
 * function.  The throttler::running_sum() returns the total number of
 * items over the given interval of seconds. The items automatically expire
 * when the time moves on the successive invocations of the add() method.
 * @tparam MaxSeconds defines the max number of seconds of data to hold
 *          in the circuling buffer.
 * @tparam BucketsPerSec defines the number of bucket slots per second.
 *          The larger the number the more accurate the running sum will be.
 */
template<size_t MaxSeconds = 16, size_t BucketsPerSec = 2>
class basic_rate_throttler {
public:
    static const size_t s_max_seconds       = upper_power<MaxSeconds,2>::value;
    static const size_t s_buckets_per_sec   = upper_power<BucketsPerSec,2>::value;
    static const size_t s_log_buckets_sec   = log<s_buckets_per_sec,2>::value;
    static const size_t s_bucket_count      = s_max_seconds * s_buckets_per_sec;
    static const size_t s_bucket_mask       = s_bucket_count - 1;

    explicit basic_rate_throttler(int a_interval = 1)
        : m_interval(-1)
    {
        init(a_interval);
    }

    basic_rate_throttler           (const basic_rate_throttler&) = default;
    basic_rate_throttler           (basic_rate_throttler&&)      = default;
    basic_rate_throttler& operator=(const basic_rate_throttler&) = default;

    /// Initialize the internal buffer setting the throttling interval
    /// measured in seconds.
    void init(int a_throttle_interval)
        throw(badarg_error)
    {
        if (a_throttle_interval == m_interval)
            return;
        m_interval = a_throttle_interval << s_log_buckets_sec;
        if (a_throttle_interval < 0 ||
            (size_t)a_throttle_interval > s_bucket_count / s_buckets_per_sec)
            throw badarg_error("Invalid throttle interval:", a_throttle_interval);
        reset();
    }

    /// Reset the inturnal circular buffer.
    void reset() {
        memset(m_buckets, 0, sizeof(m_buckets));
        m_last_time     = 0;
        m_sum           = 0;
    }

    /// Return running interval.
    long    interval()     const { return m_interval >> s_log_buckets_sec; }
    /// Return current running sum over the interval.
    long    running_sum()  const { return m_sum; }
    /// Return current running sum over the interval.
    double  running_avg()  const { return (double)m_sum / (m_interval >> s_log_buckets_sec); }

    /// Add \a a_count number of items to the bucket associated with \a a_time.
    /// @param a_time is monotonically increasing time value.
    /// @param a_count is the count to add to the bucket associated with \a a_time.
    /// @return current running sum.
    long   add(time_val a_time, int a_count = 1);

    /// Update current timestamp.
    long   refresh(time_val a_time) { return add(a_time, 0); }

    /// Dump the internal state to stream
    void   dump(std::ostream& out, time_val a_time);
private:
    size_t  m_buckets[s_bucket_count];
    time_t  m_last_time;
    long    m_sum;
    long    m_interval;

    // must be power of two
    BOOST_STATIC_ASSERT((s_bucket_count & (s_bucket_count - 1)) == 0);
};

//------------------------------------------------------------------------------
// Implementation
//------------------------------------------------------------------------------
template<size_t MaxSeconds, size_t BucketsPerSec>
long basic_rate_throttler<MaxSeconds, BucketsPerSec>::
add(time_val a_time, int a_count)
{
    time_t l_now = (time_t)(a_time.seconds() * s_buckets_per_sec);
    if (unlikely(!m_last_time))
       m_last_time = l_now;
    int  l_bucket    = l_now & s_bucket_mask;
    long l_time_diff = l_now - m_last_time;
    if (unlikely(l_now < m_last_time)) {
        // Clock was adjusted
        m_buckets[l_bucket] = a_count;
        m_sum = a_count;
    } else if (!l_time_diff) {
        m_sum += a_count;
        m_buckets[l_bucket] += a_count;
    } else if (l_time_diff >= m_interval) {
        int l_start = (l_now - m_interval + 1) & s_bucket_mask;
        int l_end   = l_now & s_bucket_mask;
        for (int i = l_start; i != l_end; i = (i+1) & s_bucket_mask)
            m_buckets[i] = 0;
        m_buckets[l_bucket] = a_count;
        m_sum = a_count;
    } else {
        // Calculate sum of buckets from
        int l_valid_buckets = m_interval - l_time_diff;
        int l_start, l_end;
        if (l_valid_buckets <= (m_interval>>1)) {
            l_start = (l_now - m_interval + 1) & s_bucket_mask;
            l_end   = (m_last_time+1) & s_bucket_mask;
            m_sum = a_count;
            #ifdef THROTTLE_DEBUG
            printf("Summing %d through %d\n", l_start, l_end);
            #endif
            for (int i = l_start; i != l_end; i = (i+1) & s_bucket_mask) {
                m_sum += m_buckets[i];
            }
            l_start = l_end;
            l_end   = l_now & s_bucket_mask;
        } else {
            l_start = (m_last_time - m_interval + 1) & s_bucket_mask;
            l_end   = (l_now - m_interval + 1) & s_bucket_mask;
            #ifdef THROTTLE_DEBUG
            printf("Subtracting/resetting %d through %d\n", l_start, l_end);
            #endif
            for (int i = l_start; i != l_end; i = (i+1) & s_bucket_mask) {
                m_sum -= m_buckets[i];
                // BOOST_ASSERT(m_sum >= 0);
                if (m_sum < 0) {
                    m_sum = 0;
                    std::cerr << "ERROR " << __FILE__ << ':' << __LINE__ << std::endl;
                    dump(std::cerr, a_time);
                }
                m_buckets[i] = 0;
            }
            l_start = (m_last_time + 1) & s_bucket_mask;
            l_end   = l_now & s_bucket_mask;
            m_sum += a_count;
        }
        #ifdef THROTTLE_DEBUG
        printf("Resetting %d through %d\n", l_start, l_end);
        #endif
        // Reset values in intermediate buckets since there was no activity there
        for (int i = l_start; i != l_end; i = (i+1) & s_bucket_mask)
            m_buckets[i] = 0;
        m_buckets[l_bucket] = a_count;
    }
    m_last_time = l_now;
    #ifdef THROTTLE_DEBUG
    dump(std::cout);
    #endif

    return m_sum;
}

template<size_t MaxSeconds, size_t BucketsPerSec>
void basic_rate_throttler<MaxSeconds, BucketsPerSec>::
dump(std::ostream& out, time_val a_time)
{
    time_t l_now    = (time_t)(a_time.seconds() * s_buckets_per_sec);
    size_t l_bucket = l_now & s_bucket_mask;

    char buf[256];
    std::stringstream s;
    sprintf(buf, "last_time=%ld, last_bucket=%3zu, sum=%ld (interval=%ld)\n",
            m_last_time, l_bucket, m_sum, m_interval);
    s << buf;
    int k = 0;
    size_t n = (l_bucket-m_interval) & s_bucket_mask;
    for (size_t j=0; j < s_bucket_count; j++) {
        k += snprintf(buf+k, sizeof(buf)-k, "%3zu%c", j, (j == l_bucket || j == n) ? '|' : ' ');
        k += snprintf(buf+k, sizeof(buf)-k, "%s", (j < s_bucket_count-1) ? "" : "\n");
    }
    s << buf;
    k = 0;
    for (size_t j=0; j < s_bucket_count; j++) {
        k += snprintf(buf+k, sizeof(buf)-k, "%3zu%c", m_buckets[j], (j == l_bucket || j == n) ? '|' : ' ');
        k += snprintf(buf+k, sizeof(buf)-k, "%s", (j < s_bucket_count-1) ? "" : "\n");
    }
    s << buf;
    out << s.str();
}

} // namespace utxx
<|endoftext|>


<commit_before>/*************************************************************************
*    UrBackup - Client/Server backup system
*    Copyright (C) 2011  Martin Raiber
*
*    This program is free software: you can redistribute it and/or modify
*    it under the terms of the GNU General Public License as published by
*    the Free Software Foundation, either version 3 of the License, or
*    (at your option) any later version.
*
*    This program is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU General Public License for more details.
*
*    You should have received a copy of the GNU General Public License
*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
**************************************************************************/

#include "vld.h"

#include <map>
#include "WorkerThread.h"
#include "Client.h"
#include "Server.h"
#include "libfastcgi/fastcgi.hpp"
#include "SelectThread.h"
#include "stringtools.h"
#include "Interface/File.h"

//#define EXTENSIVE_DEBUGGING

extern std::deque<CClient*> client_queue;
extern IMutex* clients_mutex;
extern ICondition* clients_cond;

CWorkerThread::CWorkerThread(CSelectThread *pMaster)
{
	stop_mutex=Server->createMutex();
	stop_cond=Server->createCondition();
	Master=pMaster;
	keep_alive=true;
	run=true;
}

CWorkerThread::~CWorkerThread()
{
  shutdown();
  Server->destroy(stop_cond);
  Server->destroy(stop_mutex);
}

void CWorkerThread::shutdown(void)
{
    IScopedLock slock(stop_mutex);
    Server->Log("waiting for worker...");
    run=false;
    clients_cond->notify_all();
    stop_cond->wait(&slock);
    Server->Log("done.");
}

void CWorkerThread::operator()()
{
	while(run)
	{
		size_t nq=0;
		IScopedLock lock(clients_mutex);

		while(client_queue.size()==0 )
		{
			clients_cond->wait(&lock);
			if(!run)
			{
			  IScopedLock slock(stop_mutex);
			  stop_cond->notify_one();
			  return;
			}
		}

		{
			while( client_queue.size()>0 )
			{
				char buffer[WT_BUFFERSIZE];
				CClient *client=client_queue[0];
				client_queue.erase(client_queue.begin());
				SOCKET s=client->getSocket();

				lock.relock(NULL);

				_i32 rc=recv(s, buffer, WT_BUFFERSIZE, MSG_NOSIGNAL);

				if( rc<1 )
				{
					keep_alive=true;
					//Server->Log("Client disconnected", LL_INFO);
					Master->RemoveClient( client );
					lock.relock(clients_mutex);
				}
				else
				{
#ifdef EXTENSIVE_DEBUGGING
					std::string lbuf;
					for(_i32 i=0;i<rc;++i)
					{
						if( buffer[i]==0 )
							lbuf+='#';
						else
							lbuf+=buffer[i];
					}
					Server->Log("Incoming data: "+lbuf, LL_INFO);
#endif
					client->lock();
					try
					{
						client->getFCGIProtocolDriver()->process_input(buffer, rc);
					}catch(...)
					{
						client->unlock();
						Master->RemoveClient(client);
						lock.relock(clients_mutex);
						continue;
					}

					FCGIRequest* req=NULL;
					try
					{
						req=client->getFCGIProtocolDriver()->get_request();
					}catch(...)
					{
						client->unlock();
						Master->RemoveClient(client);
						lock.relock(clients_mutex);
						continue;
					}

					client->unlock();

					if( req!=NULL )
						client->addRequest(req);

					while( (req=client->getAndRemoveReadyRequest())!=NULL )
					{
						Server->addRequest();
						client->lock();
						ProcessRequest(client, req);
						client->unlock();
					}

					if( keep_alive==false )
					{
						keep_alive=true;
						Server->Log("Client disconnected", LL_INFO);
						Master->RemoveClient( client );
					}
					else
					{
						client->setProcessing(false);
						Master->WakeUp();
					}

					lock.relock(clients_mutex);
				}
			}
		}
	}
	IScopedLock slock(stop_mutex);
	stop_cond->notify_one();
}

void CWorkerThread::ProcessRequest(CClient *client, FCGIRequest *req)
{
	if( req->keep_connection )
	{
		keep_alive=true;
	}
	else
	{
		keep_alive=false;
	}

	if( req->role != FCGIRequest::RESPONDER )
	{
		Server->Log("Role ist not Responder", LL_ERROR);
		return;
	}

	str_map GET,POST;

	str_nmap::iterator iter=req->params.find("QUERY_STRING");
	if( iter!=req->params.end() )
	{
		for(size_t i=0,size=iter->second.size();i<size;++i)
		{
			if( iter->second[i]=='+' )
				iter->second[i]=' ';
		}
		ParseParamStr(iter->second, &GET );
		req->params.erase( iter );
	}

	std::string ct=req->params["CONTENT_TYPE"];
	std::string lct=ct;
	strlower(lct);
	bool postfile=false;
	POSTFILE_KEY pfkey;
	if(lct.find("multipart/form-data")==std::string::npos)
	{
		if( req->stdin_stream.size()>0 && req->stdin_stream.size()<1048576 )
		{
			for(size_t i=0,size=req->stdin_stream.size();i<size;++i)
			{
				if( req->stdin_stream[i]=='+' )
					req->stdin_stream[i]=' ';
			}
			ParseParamStr(req->stdin_stream, &POST );
		}
	}
	else
	{
		std::string boundary=getafter("boundary=",ct);
		pfkey=ParseMultipartData(req->stdin_stream, boundary);
	        req->params["POSTFILEKEY"]=nconvert(pfkey);
	        postfile=true;
	}

	str_map::iterator iter2=GET.find(L"a");

	if( iter2!=GET.end() )
	{
		int starttime=Server->getTimeMS();

		str_map::iterator iter3=GET.find(L"c");

		std::wstring context;
		if( iter3!=GET.end() )
			context=iter3->second;

		THREAD_ID tid=Server->Execute(iter2->second, context, GET, POST, req->params, req );

		if( tid==0 )
		{
			std::wstring error=L"Error: Unknown action ["+iter2->second+L"]";
			Server->Log(error, LL_WARNING);
			req->write("Content-type: text/html; charset=UTF-8\r\n\r\n"+wnarrow(error));
		}

		starttime=Server->getTimeMS()-starttime;
		//Server->Log("Execution Time: "+nconvert(starttime)+" ms - time="+nconvert(Server->getTimeMS() ), LL_INFO);
	}
	else
	{
		std::string error="Error: Parameter 'action' not given.";
		req->write("Content-type: text/html; charset=UTF-8\r\n\r\n"+error);
	}

	if(postfile)
	{
		Server->clearPostFiles(pfkey);
	}

	req->end_request(0, FCGIRequest::REQUEST_COMPLETE);
}

POSTFILE_KEY CWorkerThread::ParseMultipartData(const std::string &data, const std::string &boundary)
{
	std::string rboundary="--"+boundary;
	int state=0;
	std::string key;
	std::string value;
	std::string filename;
	std::string name;
	std::string filedata;
	std::string contenttype;
	size_t start;
	POSTFILE_KEY pfilekey=Server->getPostFileKey();
	for(size_t i=0;i<data.size();++i)
	{
	    switch(state)
	    {
		case 0:
		    if(next(data,i,rboundary))
		    {
				i+=rboundary.size()+1;
				state=2;
		    }
		    break;
		case 1:
		    if(data[i]=='\n' || data[i]=='\r' )
			{
				if(data[i]=='\n')
				{
					state=4;
					rboundary+="--";
					start=i+1;
				}
				else
					break;
			}
		    else
				state=1;
		case 2:
			if(data[i]!=':')
				key+=toupper(data[i]);
		    else
				state=3;
		    break;
		case 3:
		    if(data[i]!='\n' && data[i]!='\r' )
				value+=data[i];
		    else if(data[i]=='\n')
			{
				if(key=="CONTENT-DISPOSITION")
				{
					name=getbetween("name=\"","\"", value);
					filename=getbetween("filename=\"","\"", value);
				}
				else if(key=="CONTENT-TYPE")
				{
					contenttype=value;
				}
				value.clear();
				key.clear();
				state=1;
			}
		    break;
		case 4:
			if(next(data,i,rboundary)==true)
			{
            	IFile *memfile=Server->openMemoryFile();
				memfile->Write(data.substr(start,i-start-2) );
				memfile->Seek(0);
				Server->addPostFile(pfilekey, name, SPostfile(memfile, widen(filename), widen(contenttype)) );
				state=0;
				rboundary.erase(rboundary.size()-2,2);
				i+=rboundary.size()+2;
				state=0;
			}
	    }
	}
    return pfilekey;
}<commit_msg>Less info messages for fastcgi<commit_after>/*************************************************************************
*    UrBackup - Client/Server backup system
*    Copyright (C) 2011  Martin Raiber
*
*    This program is free software: you can redistribute it and/or modify
*    it under the terms of the GNU General Public License as published by
*    the Free Software Foundation, either version 3 of the License, or
*    (at your option) any later version.
*
*    This program is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU General Public License for more details.
*
*    You should have received a copy of the GNU General Public License
*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
**************************************************************************/

#include "vld.h"

#include <map>
#include "WorkerThread.h"
#include "Client.h"
#include "Server.h"
#include "libfastcgi/fastcgi.hpp"
#include "SelectThread.h"
#include "stringtools.h"
#include "Interface/File.h"

//#define EXTENSIVE_DEBUGGING

extern std::deque<CClient*> client_queue;
extern IMutex* clients_mutex;
extern ICondition* clients_cond;

CWorkerThread::CWorkerThread(CSelectThread *pMaster)
{
	stop_mutex=Server->createMutex();
	stop_cond=Server->createCondition();
	Master=pMaster;
	keep_alive=true;
	run=true;
}

CWorkerThread::~CWorkerThread()
{
  shutdown();
  Server->destroy(stop_cond);
  Server->destroy(stop_mutex);
}

void CWorkerThread::shutdown(void)
{
    IScopedLock slock(stop_mutex);
    Server->Log("waiting for worker...");
    run=false;
    clients_cond->notify_all();
    stop_cond->wait(&slock);
    Server->Log("done.");
}

void CWorkerThread::operator()()
{
	while(run)
	{
		size_t nq=0;
		IScopedLock lock(clients_mutex);

		while(client_queue.size()==0 )
		{
			clients_cond->wait(&lock);
			if(!run)
			{
			  IScopedLock slock(stop_mutex);
			  stop_cond->notify_one();
			  return;
			}
		}

		{
			while( client_queue.size()>0 )
			{
				char buffer[WT_BUFFERSIZE];
				CClient *client=client_queue[0];
				client_queue.erase(client_queue.begin());
				SOCKET s=client->getSocket();

				lock.relock(NULL);

				_i32 rc=recv(s, buffer, WT_BUFFERSIZE, MSG_NOSIGNAL);

				if( rc<1 )
				{
					keep_alive=true;
					//Server->Log("Client disconnected", LL_INFO);
					Master->RemoveClient( client );
					lock.relock(clients_mutex);
				}
				else
				{
#ifdef EXTENSIVE_DEBUGGING
					std::string lbuf;
					for(_i32 i=0;i<rc;++i)
					{
						if( buffer[i]==0 )
							lbuf+='#';
						else
							lbuf+=buffer[i];
					}
					Server->Log("Incoming data: "+lbuf, LL_INFO);
#endif
					client->lock();
					try
					{
						client->getFCGIProtocolDriver()->process_input(buffer, rc);
					}catch(...)
					{
						client->unlock();
						Master->RemoveClient(client);
						lock.relock(clients_mutex);
						continue;
					}

					FCGIRequest* req=NULL;
					try
					{
						req=client->getFCGIProtocolDriver()->get_request();
					}catch(...)
					{
						client->unlock();
						Master->RemoveClient(client);
						lock.relock(clients_mutex);
						continue;
					}

					client->unlock();

					if( req!=NULL )
						client->addRequest(req);

					while( (req=client->getAndRemoveReadyRequest())!=NULL )
					{
						Server->addRequest();
						client->lock();
						ProcessRequest(client, req);
						client->unlock();
					}

					if( keep_alive==false )
					{
						keep_alive=true;
						//Server->Log("Client disconnected", LL_INFO);
						Master->RemoveClient( client );
					}
					else
					{
						client->setProcessing(false);
						Master->WakeUp();
					}

					lock.relock(clients_mutex);
				}
			}
		}
	}
	IScopedLock slock(stop_mutex);
	stop_cond->notify_one();
}

void CWorkerThread::ProcessRequest(CClient *client, FCGIRequest *req)
{
	if( req->keep_connection )
	{
		keep_alive=true;
	}
	else
	{
		keep_alive=false;
	}

	if( req->role != FCGIRequest::RESPONDER )
	{
		Server->Log("Role ist not Responder", LL_ERROR);
		return;
	}

	str_map GET,POST;

	str_nmap::iterator iter=req->params.find("QUERY_STRING");
	if( iter!=req->params.end() )
	{
		for(size_t i=0,size=iter->second.size();i<size;++i)
		{
			if( iter->second[i]=='+' )
				iter->second[i]=' ';
		}
		ParseParamStr(iter->second, &GET );
		req->params.erase( iter );
	}

	std::string ct=req->params["CONTENT_TYPE"];
	std::string lct=ct;
	strlower(lct);
	bool postfile=false;
	POSTFILE_KEY pfkey;
	if(lct.find("multipart/form-data")==std::string::npos)
	{
		if( req->stdin_stream.size()>0 && req->stdin_stream.size()<1048576 )
		{
			for(size_t i=0,size=req->stdin_stream.size();i<size;++i)
			{
				if( req->stdin_stream[i]=='+' )
					req->stdin_stream[i]=' ';
			}
			ParseParamStr(req->stdin_stream, &POST );
		}
	}
	else
	{
		std::string boundary=getafter("boundary=",ct);
		pfkey=ParseMultipartData(req->stdin_stream, boundary);
	        req->params["POSTFILEKEY"]=nconvert(pfkey);
	        postfile=true;
	}

	str_map::iterator iter2=GET.find(L"a");

	if( iter2!=GET.end() )
	{
		int starttime=Server->getTimeMS();

		str_map::iterator iter3=GET.find(L"c");

		std::wstring context;
		if( iter3!=GET.end() )
			context=iter3->second;

		THREAD_ID tid=Server->Execute(iter2->second, context, GET, POST, req->params, req );

		if( tid==0 )
		{
			std::wstring error=L"Error: Unknown action ["+iter2->second+L"]";
			Server->Log(error, LL_WARNING);
			req->write("Content-type: text/html; charset=UTF-8\r\n\r\n"+wnarrow(error));
		}

		starttime=Server->getTimeMS()-starttime;
		//Server->Log("Execution Time: "+nconvert(starttime)+" ms - time="+nconvert(Server->getTimeMS() ), LL_INFO);
	}
	else
	{
		std::string error="Error: Parameter 'action' not given.";
		req->write("Content-type: text/html; charset=UTF-8\r\n\r\n"+error);
	}

	if(postfile)
	{
		Server->clearPostFiles(pfkey);
	}

	req->end_request(0, FCGIRequest::REQUEST_COMPLETE);
}

POSTFILE_KEY CWorkerThread::ParseMultipartData(const std::string &data, const std::string &boundary)
{
	std::string rboundary="--"+boundary;
	int state=0;
	std::string key;
	std::string value;
	std::string filename;
	std::string name;
	std::string filedata;
	std::string contenttype;
	size_t start;
	POSTFILE_KEY pfilekey=Server->getPostFileKey();
	for(size_t i=0;i<data.size();++i)
	{
	    switch(state)
	    {
		case 0:
		    if(next(data,i,rboundary))
		    {
				i+=rboundary.size()+1;
				state=2;
		    }
		    break;
		case 1:
		    if(data[i]=='\n' || data[i]=='\r' )
			{
				if(data[i]=='\n')
				{
					state=4;
					rboundary+="--";
					start=i+1;
				}
				else
					break;
			}
		    else
				state=1;
		case 2:
			if(data[i]!=':')
				key+=toupper(data[i]);
		    else
				state=3;
		    break;
		case 3:
		    if(data[i]!='\n' && data[i]!='\r' )
				value+=data[i];
		    else if(data[i]=='\n')
			{
				if(key=="CONTENT-DISPOSITION")
				{
					name=getbetween("name=\"","\"", value);
					filename=getbetween("filename=\"","\"", value);
				}
				else if(key=="CONTENT-TYPE")
				{
					contenttype=value;
				}
				value.clear();
				key.clear();
				state=1;
			}
		    break;
		case 4:
			if(next(data,i,rboundary)==true)
			{
            	IFile *memfile=Server->openMemoryFile();
				memfile->Write(data.substr(start,i-start-2) );
				memfile->Seek(0);
				Server->addPostFile(pfilekey, name, SPostfile(memfile, widen(filename), widen(contenttype)) );
				state=0;
				rboundary.erase(rboundary.size()-2,2);
				i+=rboundary.size()+2;
				state=0;
			}
	    }
	}
    return pfilekey;
}<|endoftext|>


<commit_before>#ifndef NAMED_TUPLES_TUPLE_HEADER
#define NAMED_TUPLES_TUPLE_HEADER

#include <type_traits>
#include <utility>
#include <stdexcept>
#include <tuple>
#include "type_traits.hpp"
#include "const_string.hpp"

namespace named_tuples {
using std::is_same;
using std::enable_if;

template <unsigned Id> struct hash;
template <unsigned long long Id, unsigned long long ... Ids> struct str8;
template <unsigned long long Id, unsigned long long ... Ids> struct str12;
template <typename Id, typename ValueType> struct attribute_holder;

template <typename Id> struct attribute_init_placeholder {
  template <typename ValueType> attribute_holder< Id, ValueType> inline operator=(ValueType const& value) const {
    return attribute_holder< Id, ValueType>(value);
  }
};

template <unsigned Id> struct attribute_init_int_placeholder {
  template <typename ValueType> attribute_holder< hash<Id>, ValueType> inline operator=(ValueType const& value) const {
    return attribute_holder< hash<Id>, ValueType>(value);
  }
};

template <typename Id, typename ValueType> struct attribute_holder {
  using id_type = Id;
  using value_type = ValueType;
  attribute_holder(ValueType const& value) : value_(value) {}
  attribute_holder(ValueType&& value) : value_(std::move(value)) {}
  ValueType value_;
};

template <typename ... Attributes> class named_tuple;

template <typename ... Ids, typename ... Types> class named_tuple<Types(Ids)...>
{
  using IdList = type_list<Ids ...>;
  using Tuple = std::tuple<Types ...>;
  static_assert(sizeof ... (Ids) == sizeof ... (Types), "Template failed to resolve : it must have exactly one Id for each Type.");
  static_assert(!has_duplicates<IdList>(), "A named tuple cannot have two parameters with the same identifier.");
  Tuple values_;

 public:
  using tuple_type = Tuple;

  static constexpr std::size_t size = sizeof ... (Types);

  // Test member existance at compile time

  // Type members
  template <typename Id>
  static inline constexpr auto has_member() ->
  typename enable_if<(contains<IdList,Id>()), bool>::type
  { return true; }

  template <typename Id>
  static inline constexpr auto has_member() ->
  typename enable_if<!(contains<IdList, Id>()), bool>::type
  { return false; }

  // Hash members
  template <unsigned Id>
  static inline constexpr auto has_member() ->
  typename enable_if<(contains<IdList, hash<Id>>()), bool>::type
  { return true; }

  template <unsigned Id>
  static inline constexpr auto has_member() ->
  typename enable_if<!(contains<IdList, hash<Id>>()), bool>::type
  { return false; }

  // Ctors
  named_tuple() {};
  named_tuple(attribute_holder<Ids,Types>&& ... args) : values_(std::make_tuple(std::move(args.value_) ...)) {};
  named_tuple(Types&& ... values) : values_(std::forward<Types>(values)...) {};
  named_tuple(tuple_type const& values) : values_(values) {};
  named_tuple(tuple_type && values) : values_(std::move(values)) {};
  named_tuple(named_tuple const& other) : values_(other) {};
  named_tuple(named_tuple && other) : values_(std::move(other)) {};

  named_tuple& operator=(tuple_type const& values) { values_ = values; return *this; }
  named_tuple& operator=(tuple_type&& values) { values_ = std::move(values); return *this; }

  // Conversion
  tuple_type const& as_tuple() const { return values_; }
  tuple_type& as_tuple() { return values_; }

  // Assignment
  operator tuple_type const& () const { return values_; }
  operator tuple_type& () { return values_; }

  // Access by name as a type
  template <typename Id>
  inline auto _() const ->
  typename enable_if<(contains<IdList,Id>()), decltype(std::get<index_of<IdList,Id>::value>(values_))>::type
  { return std::get<index_of<IdList,Id>::value>(values_); }

  template <typename Id>
  inline auto _() ->
  typename enable_if<(contains<IdList,Id>()), decltype(std::get<index_of<IdList,Id>::value>(values_))>::type
  { return std::get<index_of<IdList,Id>::value>(values_); }

  // Access by name as a integral (ex: hash)
  template <unsigned Id>
  inline auto _() const ->
  typename enable_if<(contains<IdList, hash<Id>>()), decltype(std::get<index_of<IdList,hash<Id>>::value>(values_))>::type
  { return std::get<index_of<IdList,hash<Id>>::value>(values_); }

  template <unsigned Id>
  inline auto _() ->
  typename enable_if<(contains<IdList, hash<Id>>()), decltype(std::get<index_of<IdList,hash<Id>>::value>(values_))>::type
  { return std::get<index_of<IdList,hash<Id>>::value>(values_); }

  // Access by index
  template <unsigned Index>
  inline auto get() const ->
  typename enable_if<(Index < size), decltype(std::get<Index>(values_))>::type
  { return std::get<Index>(values_); }

  template <unsigned Index>
  inline auto get() ->
  typename enable_if<(Index < size), decltype(std::get<Index>(values_))>::type
  { return std::get<Index>(values_); }
};

// Tuple cast forwards std::tuple and converts named_tuple
template <typename ... Types>
inline auto tuple_cast(std::tuple<Types...> const & tuple) ->
std::tuple<Types...> const
{ return tuple; }

template <typename ... Types>
inline auto tuple_cast(std::tuple<Types...>&& tuple) ->
std::tuple<Types...>&&
{ return std::move(tuple); }

template <typename ... Ids, typename ... Types>
inline auto tuple_cast(named_tuple<Types(Ids)...> const& tuple) ->
std::tuple<Types ...> const&
{ return tuple; }

template <typename ... Ids, typename ... Types>
inline auto tuple_cast(named_tuple<Types(Ids)...> && tuple) ->
std::tuple<Types ...>&&
{ return std::move(tuple); }

// Access by index
template <unsigned Index, typename ... Ids, typename ... Types>
inline auto get(named_tuple<Types(Ids)...> const& tuple)  ->
typename enable_if<(Index < named_tuple<Types(Ids)...>::size), decltype(std::get<Index>(tuple_cast(tuple)))>::type
{ return std::get<Index>(tuple_cast(tuple)); }

template <unsigned Index, typename ... Ids, typename ... Types>
inline auto get(named_tuple<Types(Ids)...>& tuple)  ->
typename enable_if<(Index < named_tuple<Types(Ids)...>::size), decltype(std::get<Index>(tuple_cast(tuple)))>::type
{ return std::get<Index>(tuple_cast(tuple)); }

// Make tuple
template <typename ... T> inline named_tuple<typename T::value_type(typename T::id_type) ...> make_named_tuple(T&& ... args) {
  return named_tuple<typename T::value_type(typename T::id_type) ... >(std::forward<T>(args)...);
}

// Helpers for make_tuples
namespace attribute_helper {

template <typename Id> inline attribute_init_placeholder<Id> _() { return attribute_init_placeholder<Id>(); }
template <unsigned Id> inline attribute_init_int_placeholder<Id> _() { return attribute_init_int_placeholder<Id>(); }

namespace hash {
unsigned constexpr generate_id(const char* c, size_t s) { return const_string(c, s); }
}

}  // namespace attribute_helper

}  // namespace name_tuple

#endif  // NAMED_TUPLES_TUPLE_HEADER
<commit_msg>preparatory work<commit_after>#ifndef NAMED_TUPLES_TUPLE_HEADER
#define NAMED_TUPLES_TUPLE_HEADER

#include <type_traits>
#include <utility>
#include <stdexcept>
#include <tuple>
#include "type_traits.hpp"
#include "const_string.hpp"

namespace named_tuples {
using std::is_same;
using std::enable_if;

template <unsigned Id> struct hash;
template <unsigned long long Id, unsigned long long ... Ids> struct str8;
template <unsigned long long Id, unsigned long long ... Ids> struct str12;
template <typename Id, typename ValueType> struct attribute_holder;

template <typename Id> struct attribute_init_placeholder {
  template <typename ValueType> attribute_holder< Id, ValueType> inline operator=(ValueType const& value) const {
    return attribute_holder< Id, ValueType>(value);
  }
};

template <unsigned Id> struct attribute_init_int_placeholder {
  template <typename ValueType> attribute_holder< hash<Id>, ValueType> inline operator=(ValueType const& value) const {
    return attribute_holder< hash<Id>, ValueType>(value);
  }
};

template <typename Id, typename ValueType> struct attribute_holder {
  using id_type = Id;
  using value_type = ValueType;
  attribute_holder(ValueType const& value) : value_(value) {}
  attribute_holder(ValueType&& value) : value_(std::move(value)) {}
  ValueType value_;
};

template <typename ... Attributes> class named_tuple;

template <typename ... Ids, typename ... Types> class named_tuple<Types(Ids)...>
{
  using IdList = type_list<Ids ...>;
  using Tuple = std::tuple<Types ...>;
  static_assert(sizeof ... (Ids) == sizeof ... (Types), "Template failed to resolve : it must have exactly one Id for each Type.");
  static_assert(!has_duplicates<IdList>(), "A named tuple cannot have two parameters with the same identifier.");
  Tuple values_;

 public:
  using tuple_type = Tuple;

  static constexpr std::size_t size = sizeof ... (Types);

  // Test member existance at compile time

  // Type members
  template <typename Id>
  static inline constexpr auto has_member() ->
  typename enable_if<(contains<IdList,Id>()), bool>::type
  { return true; }

  template <typename Id>
  static inline constexpr auto has_member() ->
  typename enable_if<!(contains<IdList, Id>()), bool>::type
  { return false; }

  // Hash members
  template <unsigned Id>
  static inline constexpr auto has_member() ->
  typename enable_if<(contains<IdList, hash<Id>>()), bool>::type
  { return true; }

  template <unsigned Id>
  static inline constexpr auto has_member() ->
  typename enable_if<!(contains<IdList, hash<Id>>()), bool>::type
  { return false; }

  // Ctors
  named_tuple() {};
  named_tuple(attribute_holder<Ids,Types>&& ... args) : values_(std::make_tuple(std::move(args.value_) ...)) {};
  named_tuple(Types&& ... values) : values_(std::forward<Types>(values)...) {};
  named_tuple(tuple_type const& values) : values_(values) {};
  named_tuple(tuple_type && values) : values_(std::move(values)) {};
  named_tuple(named_tuple const& other) : values_(other) {};
  named_tuple(named_tuple && other) : values_(std::move(other)) {};

  named_tuple& operator=(tuple_type const& values) { values_ = values; return *this; }
  named_tuple& operator=(tuple_type&& values) { values_ = std::move(values); return *this; }

  // Conversion
  tuple_type const& as_tuple() const { return values_; }
  tuple_type& as_tuple() { return values_; }

  // Assignment
  operator tuple_type const& () const { return values_; }
  operator tuple_type& () { return values_; }

  // Access by name as a type
  template <typename Id>
  inline auto _() const ->
  typename enable_if<(contains<IdList,Id>()), decltype(std::get<index_of<IdList,Id>::value>(values_))>::type
  { return std::get<index_of<IdList,Id>::value>(values_); }

  template <typename Id>
  inline auto _() ->
  typename enable_if<(contains<IdList,Id>()), decltype(std::get<index_of<IdList,Id>::value>(values_))>::type
  { return std::get<index_of<IdList,Id>::value>(values_); }

  // Access by name as a integral (ex: hash)
  template <unsigned Id>
  inline auto _() const ->
  typename enable_if<(contains<IdList, hash<Id>>()), decltype(std::get<index_of<IdList,hash<Id>>::value>(values_))>::type
  { return std::get<index_of<IdList,hash<Id>>::value>(values_); }

  template <unsigned Id>
  inline auto _() ->
  typename enable_if<(contains<IdList, hash<Id>>()), decltype(std::get<index_of<IdList,hash<Id>>::value>(values_))>::type
  { return std::get<index_of<IdList,hash<Id>>::value>(values_); }

  // Access by index
  template <unsigned Index>
  inline auto get() const ->
  typename enable_if<(Index < size), decltype(std::get<Index>(values_))>::type
  { return std::get<Index>(values_); }

  template <unsigned Index>
  inline auto get() ->
  typename enable_if<(Index < size), decltype(std::get<Index>(values_))>::type
  { return std::get<Index>(values_); }
};

// Tuple cast forwards std::tuple and converts named_tuple
template <typename ... Types>
inline auto tuple_cast(std::tuple<Types...> const & tuple) ->
std::tuple<Types...> const
{ return tuple; }

template <typename ... Types>
inline auto tuple_cast(std::tuple<Types...>&& tuple) ->
std::tuple<Types...>&&
{ return std::move(tuple); }

template <typename ... Ids, typename ... Types>
inline auto tuple_cast(named_tuple<Types(Ids)...> const& tuple) ->
std::tuple<Types ...> const&
{ return tuple; }

template <typename ... Ids, typename ... Types>
inline auto tuple_cast(named_tuple<Types(Ids)...> && tuple) ->
std::tuple<Types ...>&&
{ return std::move(tuple); }

template <typename ... TargetTypes, typename ... SourceTypes>
inline auto operator<< (named_tuple<TargetTypes ...>& target, named_tuple<SourceTypes ...> const& source) ->
named_tuple<TargetTypes ...>&
{
  return named_tuple<TargetTypes...>();
}

// Access by index
template <unsigned Index, typename ... Ids, typename ... Types>
inline auto get(named_tuple<Types(Ids)...> const& tuple)  ->
typename enable_if<(Index < named_tuple<Types(Ids)...>::size), decltype(std::get<Index>(tuple_cast(tuple)))>::type
{ return std::get<Index>(tuple_cast(tuple)); }

template <unsigned Index, typename ... Ids, typename ... Types>
inline auto get(named_tuple<Types(Ids)...>& tuple)  ->
typename enable_if<(Index < named_tuple<Types(Ids)...>::size), decltype(std::get<Index>(tuple_cast(tuple)))>::type
{ return std::get<Index>(tuple_cast(tuple)); }

// Make tuple
template <typename ... T> inline named_tuple<typename T::value_type(typename T::id_type) ...> make_named_tuple(T&& ... args) {
  return named_tuple<typename T::value_type(typename T::id_type) ... >(std::forward<T>(args)...);
}

// Helpers for make_tuples
namespace attribute_helper {

template <typename Id> inline attribute_init_placeholder<Id> _() { return attribute_init_placeholder<Id>(); }
template <unsigned Id> inline attribute_init_int_placeholder<Id> _() { return attribute_init_int_placeholder<Id>(); }

namespace hash {
unsigned constexpr generate_id(const char* c, size_t s) { return const_string(c, s); }
}

}  // namespace attribute_helper

}  // namespace name_tuple

#endif  // NAMED_TUPLES_TUPLE_HEADER
<|endoftext|>


<commit_before>/*
 * Copyright 2007-2021 CM4all GmbH
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#pragma once

#include "net/SocketAddress.hxx"
#include "util/StaticArray.hxx"
#include "util/ShallowCopy.hxx"
#include "StickyMode.hxx"

#include <stddef.h>
#include <assert.h>

class AllocatorPtr;
class AddressInfoList;

/**
 * Store a URI along with a list of socket addresses.
 */
struct AddressList {
	static constexpr size_t MAX_ADDRESSES = 16;

	StickyMode sticky_mode = StickyMode::NONE;

	using Array = StaticArray<SocketAddress, MAX_ADDRESSES>;
	using size_type = Array::size_type;
	using const_iterator = Array::const_iterator;
	using const_reference = Array::const_reference;

	Array addresses;

	AddressList() = default;

	constexpr AddressList(ShallowCopy, const AddressList &src) noexcept
		:sticky_mode(src.sticky_mode),
		 addresses(src.addresses)
	{
	}

	AddressList(ShallowCopy, const AddressInfoList &src) noexcept;

	AddressList(AllocatorPtr alloc, const AddressList &src) noexcept;

	void SetStickyMode(StickyMode _sticky_mode) noexcept {
		sticky_mode = _sticky_mode;
	}

	constexpr
	bool empty() const noexcept {
		return addresses.empty();
	}

	constexpr size_type size() const noexcept {
		return addresses.size();
	}

	/**
	 * Is there no more than one address?
	 */
	constexpr
	bool IsSingle() const noexcept {
		return addresses.size() == 1;
	}

	constexpr const_iterator begin() const noexcept {
		return addresses.begin();
	}

	constexpr const_iterator end() const noexcept {
		return addresses.end();
	}

	constexpr const_reference front() const noexcept {
		return addresses.front();
	}

	/**
	 * @return false if the list is full
	 */
	bool AddPointer(SocketAddress address) noexcept {
		return addresses.checked_append(address);
	}

	bool Add(AllocatorPtr alloc, SocketAddress address) noexcept;
	bool Add(AllocatorPtr alloc, const AddressInfoList &list) noexcept;

	const SocketAddress &operator[](unsigned n) const noexcept {
		assert(addresses[n].IsDefined());

		return addresses[n];
	}
};
<commit_msg>cluster/AddressList: include cleanup<commit_after>/*
 * Copyright 2007-2021 CM4all GmbH
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#pragma once

#include "StickyMode.hxx"
#include "net/SocketAddress.hxx"
#include "util/StaticArray.hxx"
#include "util/ShallowCopy.hxx"

class AllocatorPtr;
class AddressInfoList;

/**
 * Store a URI along with a list of socket addresses.
 */
struct AddressList {
	StickyMode sticky_mode = StickyMode::NONE;

	using Array = StaticArray<SocketAddress, 16>;
	using size_type = Array::size_type;
	using const_iterator = Array::const_iterator;
	using const_reference = Array::const_reference;

	Array addresses;

	AddressList() = default;

	constexpr AddressList(ShallowCopy, const AddressList &src) noexcept
		:sticky_mode(src.sticky_mode),
		 addresses(src.addresses)
	{
	}

	AddressList(ShallowCopy, const AddressInfoList &src) noexcept;

	AddressList(AllocatorPtr alloc, const AddressList &src) noexcept;

	void SetStickyMode(StickyMode _sticky_mode) noexcept {
		sticky_mode = _sticky_mode;
	}

	constexpr
	bool empty() const noexcept {
		return addresses.empty();
	}

	constexpr size_type size() const noexcept {
		return addresses.size();
	}

	/**
	 * Is there no more than one address?
	 */
	constexpr
	bool IsSingle() const noexcept {
		return addresses.size() == 1;
	}

	constexpr const_iterator begin() const noexcept {
		return addresses.begin();
	}

	constexpr const_iterator end() const noexcept {
		return addresses.end();
	}

	constexpr const_reference front() const noexcept {
		return addresses.front();
	}

	/**
	 * @return false if the list is full
	 */
	bool AddPointer(SocketAddress address) noexcept {
		return addresses.checked_append(address);
	}

	bool Add(AllocatorPtr alloc, SocketAddress address) noexcept;
	bool Add(AllocatorPtr alloc, const AddressInfoList &list) noexcept;

	const SocketAddress &operator[](unsigned n) const noexcept {
		assert(addresses[n].IsDefined());

		return addresses[n];
	}
};
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014 liblcf authors
 * This file is released under the MIT License
 * http://opensource.org/licenses/MIT
 */

#include "reader_options.h"
#ifdef LCF_SUPPORT_ICU
#  include "unicode/ucsdet.h"
#  include "unicode/ucnv.h"
#else
#  ifdef _WIN32
#    include <cstdio>
#    define WIN32_LEAN_AND_MEAN
#    ifndef NOMINMAX
#      define NOMINMAX
#    endif
#    include <windows.h>
#  endif
#endif
#ifndef _WIN32
#  include <locale>
#endif

#include <cstdlib>
#include <sstream>
#include <vector>

#include "data.h"
#include "inireader.h"
#include "ldb_reader.h"
#include "reader_util.h"

namespace ReaderUtil {
}

std::string ReaderUtil::CodepageToEncoding(int codepage) {
	if (codepage == 0)
		return std::string();
#ifndef LCF_SUPPORT_ICU
#  ifdef _WIN32
	if (codepage > 0) {
		// Looks like a valid codepage
		std::stringstream encoding;
		encoding << codepage;
		return std::string(encoding.str());
	}
#  endif
#endif
	if (codepage == 932) {
#ifdef LCF_SUPPORT_ICU
		return "cp943";
#else
		return "SHIFT_JIS";
#endif
	}
	if (codepage == 949) {
#ifdef LCF_SUPPORT_ICU
		return "cp949";
#else
		return "cp1361";
#endif
	}
	std::ostringstream out;
#ifdef LCF_SUPPORT_ICU
	out << "windows-" << codepage;
#else
	out << "CP" << codepage;
#endif
	// Check at first if the ini value is a codepage
	if (!out.str().empty()) {
		// Looks like a valid codepage
		return out.str();
	}

	return std::string();
}

std::string ReaderUtil::DetectEncoding(const std::string& database_file) {
#ifdef LCF_SUPPORT_ICU
	std::ostringstream text;
	std::string encoding;

	//Populate Data::terms or will empty by default even if load fails
	LDB_Reader::Load(database_file, "");

	text <<
	Data::terms.menu_save << " " <<
	Data::terms.menu_quit << " " <<
	Data::terms.new_game << " " <<
	Data::terms.load_game << " " <<
	Data::terms.exit_game << " " <<
	Data::terms.status << " " <<
	Data::terms.row << " " <<
	Data::terms.order << " " <<
	Data::terms.wait_on << " " <<
	Data::terms.wait_off << " " <<
	Data::terms.level << " " <<
	Data::terms.health_points << " " <<
	Data::terms.spirit_points << " " <<
	Data::terms.normal_status << " " <<
	Data::terms.exp_short << " " <<
	Data::terms.lvl_short << " " <<
	Data::terms.hp_short << " " <<
	Data::terms.sp_short << " " <<
	Data::terms.sp_cost << " " <<
	Data::terms.attack << " " <<
	Data::terms.defense << " " <<
	Data::terms.spirit << " " <<
	Data::terms.agility << " " <<
	Data::terms.weapon << " " <<
	Data::terms.shield << " " <<
	Data::terms.armor << " " <<
	Data::terms.helmet << " " <<
	Data::terms.accessory << " " <<
	Data::terms.save_game_message << " " <<
	Data::terms.load_game_message << " " <<
	Data::terms.file << " " <<
	Data::terms.exit_game_message << " " <<
	Data::terms.yes << " " <<
	Data::terms.no;

	// Checks if there are more than the above 33 spaces (no data)
	if (text.str().size() > 33)
	{
		UErrorCode status = U_ZERO_ERROR;
		UCharsetDetector* detector = ucsdet_open(&status);

		ucsdet_setText(detector, text.str().data(), text.str().length(), &status);
		const UCharsetMatch* match = ucsdet_detect(detector, &status);
		if (match != NULL)
		{
			encoding = ucsdet_getName(match, &status);
		}
		ucsdet_close(detector);

		// Fixes to ensure proper Windows encodings
		if (encoding == "Shift_JIS")
		{
			encoding = "cp943"; // Japanese
		}
		else if (encoding == "EUC-KR")
		{
			encoding = "cp949"; // Korean
		}
		else if (encoding == "ISO-8859-1")
		{
			encoding = "windows-1252"; // Occidental
		}
		else if (encoding == "ISO-8859-2")
		{
			encoding = "windows-1250"; // Central Europe
		}
		else if (encoding == "ISO-8859-5")
		{
			encoding = "windows-1251"; // Cyrillic
		}
		else if (encoding == "ISO-8859-6")
		{
			encoding = "windows-1256"; // Arabic
		}
		else if (encoding == "ISO-8859-7")
		{
			encoding = "windows-1253"; // Greek
		}
		else if (encoding == "ISO-8859-8")
		{
			encoding = "windows-1255"; // Hebrew
		}
	}
#endif

	return encoding;
}

std::string ReaderUtil::GetEncoding(const std::string& ini_file) {
	INIReader ini(ini_file);
	if (ini.ParseError() != -1) {
		std::string encoding = ini.Get("EasyRPG", "Encoding", std::string());
		if (!encoding.empty()) {
			return ReaderUtil::CodepageToEncoding(atoi(encoding.c_str()));
		}
	}
	return std::string();
}

std::string ReaderUtil::GetLocaleEncoding() {
#ifdef _WIN32
	// On Windows means current system locale.
	int codepage = 0;
#else
	int codepage = 1252;

	std::locale loc = std::locale("");
	// Gets the language and culture part only
	std::string loc_full = loc.name().substr(0, loc.name().find_first_of("@."));
	// Gets the language part only
	std::string loc_lang = loc.name().substr(0, loc.name().find_first_of("_"));

	if      (loc_lang == "th")    codepage = 874;
	else if (loc_lang == "ja")    codepage = 932;
	else if (loc_full == "zh_CN" ||
	         loc_full == "zh_SG") codepage = 936;
	else if (loc_lang == "ko")    codepage = 949;
	else if (loc_full == "zh_TW" ||
	         loc_full == "zh_HK") codepage = 950;
	else if (loc_lang == "cs" ||
	         loc_lang == "hu" ||
	         loc_lang == "pl" ||
	         loc_lang == "ro" ||
	         loc_lang == "hr" ||
	         loc_lang == "sk" ||
	         loc_lang == "sl")    codepage = 1250;
	else if (loc_lang == "ru")    codepage = 1251;
	else if (loc_lang == "ca" ||
	         loc_lang == "da" ||
	         loc_lang == "de" ||
	         loc_lang == "en" ||
	         loc_lang == "es" ||
	         loc_lang == "fi" ||
	         loc_lang == "fr" ||
	         loc_lang == "it" ||
	         loc_lang == "nl" ||
	         loc_lang == "nb" ||
	         loc_lang == "pt" ||
	         loc_lang == "sv" ||
	         loc_lang == "eu")    codepage = 1252;
	else if (loc_lang == "el")    codepage = 1253;
	else if (loc_lang == "tr")    codepage = 1254;
	else if (loc_lang == "he")    codepage = 1255;
	else if (loc_lang == "ar")    codepage = 1256;
	else if (loc_lang == "et" ||
	         loc_lang == "lt" ||
	         loc_lang == "lv")    codepage = 1257;
	else if (loc_lang == "vi")    codepage = 1258;
#endif
	return CodepageToEncoding(codepage);
}

std::string ReaderUtil::Recode(const std::string& str_to_encode, const std::string& source_encoding) {
#ifdef _WIN32
	return ReaderUtil::Recode(str_to_encode, source_encoding, "65001");
#else
	return ReaderUtil::Recode(str_to_encode, source_encoding, "UTF-8");
#endif
}

std::string ReaderUtil::Recode(const std::string& str_to_encode,
                               const std::string& src_enc,
                               const std::string& dst_enc) {
	std::string encoding_str = src_enc;

	if (src_enc.empty()) {
		return str_to_encode;
	}
	if (atoi(src_enc.c_str()) > 0) {
		encoding_str = ReaderUtil::CodepageToEncoding(atoi(src_enc.c_str()));
	}
#ifdef LCF_SUPPORT_ICU
	UErrorCode status = U_ZERO_ERROR;
	int size = str_to_encode.size() * 4;
	UChar* unicode_str = new UChar[size];
	UConverter *conv;
	int length;
	std::string result_str;

	conv = ucnv_open(encoding_str.c_str(), &status);
	length = ucnv_toUChars(conv, unicode_str, size, str_to_encode.c_str(), -1, &status);
	ucnv_close(conv);
	if (status != U_ZERO_ERROR) return std::string();

	char* result = new char[length * 4];

	conv = ucnv_open(dst_enc.c_str(), &status);
	ucnv_fromUChars(conv, result, length * 4, unicode_str, -1, &status);
	ucnv_close(conv);
	if (status != U_ZERO_ERROR) return std::string();

	result_str = result;

	delete[] unicode_str;
	delete[] result;

	return std::string(result_str);
#else
#  ifdef _WIN32
	size_t strsize = str_to_encode.size();

	wchar_t* widechar = new wchar_t[strsize * 5 + 1];

	// To UTF-16
	// Default codepage is 0, so we dont need a check here
	int res = MultiByteToWideChar(atoi(encoding_str.c_str()), 0, str_to_encode.c_str(), strsize, widechar, strsize * 5 + 1);
	if (res == 0) {
		// Invalid codepage
		delete [] widechar;
		return str_to_encode;
	}
	widechar[res] = '\0';

	// Back to UTF-8...
	char* utf8char = new char[strsize * 5 + 1];
	res = WideCharToMultiByte(atoi(dst_enc.c_str()), 0, widechar, res, utf8char, strsize * 5 + 1, NULL, NULL);
	utf8char[res] = '\0';

	// Result in str
	std::string str = std::string(utf8char, res);

	delete [] widechar;
	delete [] utf8char;

	return str;
#  else
	iconv_t cd = iconv_open(dst_enc.c_str(), encoding_str.c_str());
	if (cd == (iconv_t)-1)
		return str_to_encode;
	char *src = const_cast<char *>(str_to_encode.c_str());
	size_t src_left = str_to_encode.size();
	size_t dst_size = str_to_encode.size() * 5 + 10;
	char *dst = new char[dst_size];
	size_t dst_left = dst_size;
#    ifdef ICONV_CONST
	char ICONV_CONST *p = src;
#    else
	char *p = src;
#    endif
	char *q = dst;
	size_t status = iconv(cd, &p, &src_left, &q, &dst_left);
	iconv_close(cd);
	if (status == (size_t) -1 || src_left > 0) {
	delete[] dst;
		return std::string();
	}
	*q++ = '\0';
	std::string result(dst);
	delete[] dst;
	return result;
#  endif
#endif
}
<commit_msg>Fix build when ICU is not enabled<commit_after>/*
 * Copyright (c) 2014 liblcf authors
 * This file is released under the MIT License
 * http://opensource.org/licenses/MIT
 */

#include "reader_options.h"
#ifdef LCF_SUPPORT_ICU
#  include "unicode/ucsdet.h"
#  include "unicode/ucnv.h"
#else
#  ifdef _WIN32
#    include <cstdio>
#    define WIN32_LEAN_AND_MEAN
#    ifndef NOMINMAX
#      define NOMINMAX
#    endif
#    include <windows.h>
#  endif
#endif
#ifndef _WIN32
#  include <locale>
#endif

#include <cstdlib>
#include <sstream>
#include <vector>

#include "data.h"
#include "inireader.h"
#include "ldb_reader.h"
#include "reader_util.h"

namespace ReaderUtil {
}

std::string ReaderUtil::CodepageToEncoding(int codepage) {
	if (codepage == 0)
		return std::string();
#ifndef LCF_SUPPORT_ICU
#  ifdef _WIN32
	if (codepage > 0) {
		// Looks like a valid codepage
		std::stringstream encoding;
		encoding << codepage;
		return std::string(encoding.str());
	}
#  endif
#endif
	if (codepage == 932) {
#ifdef LCF_SUPPORT_ICU
		return "cp943";
#else
		return "SHIFT_JIS";
#endif
	}
	if (codepage == 949) {
#ifdef LCF_SUPPORT_ICU
		return "cp949";
#else
		return "cp1361";
#endif
	}
	std::ostringstream out;
#ifdef LCF_SUPPORT_ICU
	out << "windows-" << codepage;
#else
	out << "CP" << codepage;
#endif
	// Check at first if the ini value is a codepage
	if (!out.str().empty()) {
		// Looks like a valid codepage
		return out.str();
	}

	return std::string();
}

std::string ReaderUtil::DetectEncoding(const std::string& database_file) {
	std::string encoding;
#ifdef LCF_SUPPORT_ICU
	std::ostringstream text;

	//Populate Data::terms or will empty by default even if load fails
	LDB_Reader::Load(database_file, "");

	text <<
	Data::terms.menu_save << " " <<
	Data::terms.menu_quit << " " <<
	Data::terms.new_game << " " <<
	Data::terms.load_game << " " <<
	Data::terms.exit_game << " " <<
	Data::terms.status << " " <<
	Data::terms.row << " " <<
	Data::terms.order << " " <<
	Data::terms.wait_on << " " <<
	Data::terms.wait_off << " " <<
	Data::terms.level << " " <<
	Data::terms.health_points << " " <<
	Data::terms.spirit_points << " " <<
	Data::terms.normal_status << " " <<
	Data::terms.exp_short << " " <<
	Data::terms.lvl_short << " " <<
	Data::terms.hp_short << " " <<
	Data::terms.sp_short << " " <<
	Data::terms.sp_cost << " " <<
	Data::terms.attack << " " <<
	Data::terms.defense << " " <<
	Data::terms.spirit << " " <<
	Data::terms.agility << " " <<
	Data::terms.weapon << " " <<
	Data::terms.shield << " " <<
	Data::terms.armor << " " <<
	Data::terms.helmet << " " <<
	Data::terms.accessory << " " <<
	Data::terms.save_game_message << " " <<
	Data::terms.load_game_message << " " <<
	Data::terms.file << " " <<
	Data::terms.exit_game_message << " " <<
	Data::terms.yes << " " <<
	Data::terms.no;

	// Checks if there are more than the above 33 spaces (no data)
	if (text.str().size() > 33)
	{
		UErrorCode status = U_ZERO_ERROR;
		UCharsetDetector* detector = ucsdet_open(&status);

		ucsdet_setText(detector, text.str().data(), text.str().length(), &status);
		const UCharsetMatch* match = ucsdet_detect(detector, &status);
		if (match != NULL)
		{
			encoding = ucsdet_getName(match, &status);
		}
		ucsdet_close(detector);

		// Fixes to ensure proper Windows encodings
		if (encoding == "Shift_JIS")
		{
			encoding = "cp943"; // Japanese
		}
		else if (encoding == "EUC-KR")
		{
			encoding = "cp949"; // Korean
		}
		else if (encoding == "ISO-8859-1")
		{
			encoding = "windows-1252"; // Occidental
		}
		else if (encoding == "ISO-8859-2")
		{
			encoding = "windows-1250"; // Central Europe
		}
		else if (encoding == "ISO-8859-5")
		{
			encoding = "windows-1251"; // Cyrillic
		}
		else if (encoding == "ISO-8859-6")
		{
			encoding = "windows-1256"; // Arabic
		}
		else if (encoding == "ISO-8859-7")
		{
			encoding = "windows-1253"; // Greek
		}
		else if (encoding == "ISO-8859-8")
		{
			encoding = "windows-1255"; // Hebrew
		}
	}
#endif

	return encoding;
}

std::string ReaderUtil::GetEncoding(const std::string& ini_file) {
	INIReader ini(ini_file);
	if (ini.ParseError() != -1) {
		std::string encoding = ini.Get("EasyRPG", "Encoding", std::string());
		if (!encoding.empty()) {
			return ReaderUtil::CodepageToEncoding(atoi(encoding.c_str()));
		}
	}
	return std::string();
}

std::string ReaderUtil::GetLocaleEncoding() {
#ifdef _WIN32
	// On Windows means current system locale.
	int codepage = 0;
#else
	int codepage = 1252;

	std::locale loc = std::locale("");
	// Gets the language and culture part only
	std::string loc_full = loc.name().substr(0, loc.name().find_first_of("@."));
	// Gets the language part only
	std::string loc_lang = loc.name().substr(0, loc.name().find_first_of("_"));

	if      (loc_lang == "th")    codepage = 874;
	else if (loc_lang == "ja")    codepage = 932;
	else if (loc_full == "zh_CN" ||
	         loc_full == "zh_SG") codepage = 936;
	else if (loc_lang == "ko")    codepage = 949;
	else if (loc_full == "zh_TW" ||
	         loc_full == "zh_HK") codepage = 950;
	else if (loc_lang == "cs" ||
	         loc_lang == "hu" ||
	         loc_lang == "pl" ||
	         loc_lang == "ro" ||
	         loc_lang == "hr" ||
	         loc_lang == "sk" ||
	         loc_lang == "sl")    codepage = 1250;
	else if (loc_lang == "ru")    codepage = 1251;
	else if (loc_lang == "ca" ||
	         loc_lang == "da" ||
	         loc_lang == "de" ||
	         loc_lang == "en" ||
	         loc_lang == "es" ||
	         loc_lang == "fi" ||
	         loc_lang == "fr" ||
	         loc_lang == "it" ||
	         loc_lang == "nl" ||
	         loc_lang == "nb" ||
	         loc_lang == "pt" ||
	         loc_lang == "sv" ||
	         loc_lang == "eu")    codepage = 1252;
	else if (loc_lang == "el")    codepage = 1253;
	else if (loc_lang == "tr")    codepage = 1254;
	else if (loc_lang == "he")    codepage = 1255;
	else if (loc_lang == "ar")    codepage = 1256;
	else if (loc_lang == "et" ||
	         loc_lang == "lt" ||
	         loc_lang == "lv")    codepage = 1257;
	else if (loc_lang == "vi")    codepage = 1258;
#endif
	return CodepageToEncoding(codepage);
}

std::string ReaderUtil::Recode(const std::string& str_to_encode, const std::string& source_encoding) {
#ifdef _WIN32
	return ReaderUtil::Recode(str_to_encode, source_encoding, "65001");
#else
	return ReaderUtil::Recode(str_to_encode, source_encoding, "UTF-8");
#endif
}

std::string ReaderUtil::Recode(const std::string& str_to_encode,
                               const std::string& src_enc,
                               const std::string& dst_enc) {
	std::string encoding_str = src_enc;

	if (src_enc.empty()) {
		return str_to_encode;
	}
	if (atoi(src_enc.c_str()) > 0) {
		encoding_str = ReaderUtil::CodepageToEncoding(atoi(src_enc.c_str()));
	}
#ifdef LCF_SUPPORT_ICU
	UErrorCode status = U_ZERO_ERROR;
	int size = str_to_encode.size() * 4;
	UChar* unicode_str = new UChar[size];
	UConverter *conv;
	int length;
	std::string result_str;

	conv = ucnv_open(encoding_str.c_str(), &status);
	length = ucnv_toUChars(conv, unicode_str, size, str_to_encode.c_str(), -1, &status);
	ucnv_close(conv);
	if (status != U_ZERO_ERROR) return std::string();

	char* result = new char[length * 4];

	conv = ucnv_open(dst_enc.c_str(), &status);
	ucnv_fromUChars(conv, result, length * 4, unicode_str, -1, &status);
	ucnv_close(conv);
	if (status != U_ZERO_ERROR) return std::string();

	result_str = result;

	delete[] unicode_str;
	delete[] result;

	return std::string(result_str);
#else
#  ifdef _WIN32
	size_t strsize = str_to_encode.size();

	wchar_t* widechar = new wchar_t[strsize * 5 + 1];

	// To UTF-16
	// Default codepage is 0, so we dont need a check here
	int res = MultiByteToWideChar(atoi(encoding_str.c_str()), 0, str_to_encode.c_str(), strsize, widechar, strsize * 5 + 1);
	if (res == 0) {
		// Invalid codepage
		delete [] widechar;
		return str_to_encode;
	}
	widechar[res] = '\0';

	// Back to UTF-8...
	char* utf8char = new char[strsize * 5 + 1];
	res = WideCharToMultiByte(atoi(dst_enc.c_str()), 0, widechar, res, utf8char, strsize * 5 + 1, NULL, NULL);
	utf8char[res] = '\0';

	// Result in str
	std::string str = std::string(utf8char, res);

	delete [] widechar;
	delete [] utf8char;

	return str;
#  else
	iconv_t cd = iconv_open(dst_enc.c_str(), encoding_str.c_str());
	if (cd == (iconv_t)-1)
		return str_to_encode;
	char *src = const_cast<char *>(str_to_encode.c_str());
	size_t src_left = str_to_encode.size();
	size_t dst_size = str_to_encode.size() * 5 + 10;
	char *dst = new char[dst_size];
	size_t dst_left = dst_size;
#    ifdef ICONV_CONST
	char ICONV_CONST *p = src;
#    else
	char *p = src;
#    endif
	char *q = dst;
	size_t status = iconv(cd, &p, &src_left, &q, &dst_left);
	iconv_close(cd);
	if (status == (size_t) -1 || src_left > 0) {
	delete[] dst;
		return std::string();
	}
	*q++ = '\0';
	std::string result(dst);
	delete[] dst;
	return result;
#  endif
#endif
}
<|endoftext|>