<commit_before>#ifndef GHULBUS_LIBRARY_INCLUDE_GUARD_BASE_LOG_HANDLERS_HPP
#define GHULBUS_LIBRARY_INCLUDE_GUARD_BASE_LOG_HANDLERS_HPP

/** @file
*
* @brief Log Handlers.
* @author Andreas Weis (der_ghulbus@ghulbus-inc.de)
*/

#include <gbBase/config.hpp>
#include <gbBase/Log.hpp>

#include <condition_variable>
#include <deque>
#include <fstream>
#include <mutex>
#include <thread>
#include <tuple>

namespace GHULBUS_BASE_NAMESPACE
{
namespace Log
{
/** Handlers for use with Ghulbus::Log::setLogHandler().
 */
namespace Handlers
{

/** Simple, unsynchronized logging to `std::cout` and `std::cerr`.
 * Log messages of Ghulbus::LogLevel::Error or higher go to `std::cerr`, all others to `std::cout`.
 * Logging is not synchronized, so logging concurrently from multiple threads is not safe with this handler.
 * Use one of the \ref log_handler_adapters "thread-safe adapters" LogSynchronizeMutex or LogAsync if you need to
 * log from more than one thread.
 * @note Logging to the console is usually quite slow and can slow down an application significantly if many log
 *       messages are produces. Use the LogAsync adapter if performance is an issue.
 */
GHULBUS_BASE_API void logToCout(LogLevel log_level, std::stringstream&& log_stream);

#ifdef WIN32
/** Logs to an attached debugger via the OutputDebugString() Win32 API function.
*/
GHULBUS_BASE_API void logToWindowsDebugger(LogLevel log_level, std::stringstream&& log_stream);
#endif

/** Unsynchronized file logging.
 * All log messages will be appended to the given log file. %Log messages may be buffered in memory but will be
 * flushed upon destruction of the handler object.
 * Logging is not synchronized, so logging concurrently from multiple threads is not safe with this handler.
 * Use one of the \ref log_handler_adapters "thread-safe adapters" LogSynchronizeMutex or LogAsync if you need to
 * log from more than one thread.
 * @note This handler uses `std::fstream` for file I/O which might perform less than ideal on certain
 *       (*cough* Windows) implementations.
 *       Use the LogAsync adapter if performance is an issue.
 */
class LogToFile {
private:
    std::ofstream m_logFile;
public:
    /** Construct a logger for logging to a file.
     * @param[in] filename Path to the log file. This file will be opened in append mode.
     * @throw Exceptions::IOError If file could not be opened for writing.
     */
    GHULBUS_BASE_API LogToFile(char const* filename);

    /** Convert to a LogHandler function to pass to Ghulbus::Log::setLogHandler().
     * @attention Note that an object must not be destroyed while it is set as log handler.
     */
    GHULBUS_BASE_API operator LogHandler();
};

/** @defgroup log_handler_adapters Handler adapters.
 * A handler adapter is not a full handler by itself, but rather wraps around a downstream handler to add
 * additional functionality (like thread safety).
 * @{
 */
/** Simple synchronization.
 * The mutex adapter synchronizes concurrent access to the downstream handler via a mutex.
 * The resulting handler is thread safe.
 */
class LogSynchronizeMutex {
private:
    std::mutex m_mutex;
    LogHandler m_downstreamHandler;
public:
    /** Adapting Constructor.
     * @param[in] downstream_handler The log handler that is to be wrapped.
     *                               The downstream handler need not be thread safe.
     *                               The downstream handler must not be empty. If the downstream handler dies before
     *                               this adapter, the behavior is undefined.
     */
    GHULBUS_BASE_API LogSynchronizeMutex(LogHandler downstream_handler);

    /** Convert to a LogHandler function to pass to Ghulbus::Log::setLogHandler().
     * @attention Note that an object must not be destroyed while it is set as log handler.
     */
    GHULBUS_BASE_API operator LogHandler();
};

/** Asynchronous logging.
 * This adapter defers execution of the downstream handler in a separate thread. This allows the logging thread
 * to continue immediately after assembling the log message without having to wait for the log I/O to complete.
 * The impact on the runtime of the logging thread is far lower than with the other handlers. The downside is
 * that async logging requires spawning an additional I/O thread to execute the downstream handler.
 * Note that all access to the downstream handler is serialized through this I/O thread, so you can use a thread
 * unsafe handler as downstream handler.
 *
 * @note Messages will be queued in memory for processing. If an application produces log messages faster than the
 *       adapted handler can process them, this can in theory consume all available memory on the machine. Be careful
 *       not to overload the logging thread in this way.
 *
 * @note While the async handler works very well with small numbers of threads, it still requires concurrent access
 *       to the single queue of log messages. If you need highly concurrent logging, consider using an adapter that
 *       uses several thread-local queues instead.
 */
class LogAsync {
private:
    typedef std::tuple<LogLevel, std::stringstream> QueueElement;
private:
    std::mutex m_mutex;                     ///< mutex protexting access to the m_queue
    std::deque<QueueElement> m_queue;       ///< queue of log messages
    bool m_stopRequested;                   ///< flag indicating that the user called stop() to stop the I/O thread
    std::condition_variable m_condvar;      ///< signal that a message was pushed to the queue or stop was requested
    LogHandler m_downstreamHandler;
    std::thread m_ioThread;
public:
    /** @copydoc LogSynchronizeMutex::LogSynchronizeMutex(LogHandler)
     */
    GHULBUS_BASE_API LogAsync(LogHandler downstream_handler);

    /** Start the I/O thread.
     * Invoking the log handler obtained from this class will put the respective log message to an in-memory queue.
     * This function will spawn a thread that waits for messages to be put into that queue and forwards them to the
     * downstream handler. Note that while that thread is not running, messages will just keep piling up in memory,
     * so it is best to start the adapter *before* setting it as the active log handler.
     * Note that the object must not be destroyed while the I/O thread is running.
     * @see stop()
     * @pre The I/O thread is not already running.
     * @note This function is thread-safe.
     */
    GHULBUS_BASE_API void start();

    /** Stop the I/O thread.
     * Processes all outstanding log messages still in the queue and then joins the I/O thread.
     * %Log messages arriving after stop() has started executing will remain unprocessed. So you will probably want
     * to remove the adapter from active log handler before calling stop().
     * @see start()
     * @pre The I/O thread is currently running.
     * @note This function is thread-safe.
     */
    GHULBUS_BASE_API void stop();

    /** Convert to a LogHandler function to pass to Ghulbus::Log::setLogHandler().
     * @note Note that the handler needs to be \ref start() "started" for any log messages to be processed.
     * @attention Note that an object must not be destroyed while it is set as log handler.
     */
    GHULBUS_BASE_API operator LogHandler();
};

/** Forwards each log message to two downstream handlers.
 * Use this if you want to log to two different sinks, for instance to a log file and the console.
 * @note This handler will duplicate the log message before passing it on to the downstream handlers,
 *       which is potentially expensive. When combining the LogMultiSink with the LogAsync it is
 *       therefore desirable to have the LogMultiSink as the downstream and the LogAsync as the
 *       top-level handler.
 */
class LogMultiSink
{
private:
    LogHandler m_downstreamHandlers[2];
public:
    /** Adapting Constructor.
     * @param[in] first_downstream_handler The first log handler that is to be wrapped.
     * @param[in] first_downstream_handler The second log handler that is to be wrapped.
     */
    GHULBUS_BASE_API LogMultiSink(LogHandler first_downstream_handler,
                                  LogHandler second_downstream_handler);

    /** Convert to a LogHandler function to pass to Ghulbus::Log::setLogHandler().
     * @attention Note that an object must not be destroyed while it is set as log handler.
     */
    GHULBUS_BASE_API operator LogHandler();
};
/** @} */
}
}
}
#endif
<commit_msg>clarified documentation<commit_after>#ifndef GHULBUS_LIBRARY_INCLUDE_GUARD_BASE_LOG_HANDLERS_HPP
#define GHULBUS_LIBRARY_INCLUDE_GUARD_BASE_LOG_HANDLERS_HPP

/** @file
*
* @brief Log Handlers.
* @author Andreas Weis (der_ghulbus@ghulbus-inc.de)
*/

#include <gbBase/config.hpp>
#include <gbBase/Log.hpp>

#include <condition_variable>
#include <deque>
#include <fstream>
#include <mutex>
#include <thread>
#include <tuple>

namespace GHULBUS_BASE_NAMESPACE
{
namespace Log
{
/** Handlers for use with Ghulbus::Log::setLogHandler().
 */
namespace Handlers
{

/** Simple, unsynchronized logging to `std::cout` and `std::cerr`.
 * Log messages of Ghulbus::LogLevel::Error or higher go to `std::cerr`, all others to `std::cout`.
 * Logging is not synchronized, so logging concurrently from multiple threads is not safe with this handler.
 * Use one of the \ref log_handler_adapters "thread-safe adapters" LogSynchronizeMutex or LogAsync if you need to
 * log from more than one thread.
 * @note Logging to the console is usually quite slow and can slow down an application significantly if many log
 *       messages are produces. Use the LogAsync adapter if performance is an issue.
 */
GHULBUS_BASE_API void logToCout(LogLevel log_level, std::stringstream&& log_stream);

#ifdef WIN32
/** Logs to an attached debugger via the OutputDebugString() Win32 API function.
 */
GHULBUS_BASE_API void logToWindowsDebugger(LogLevel log_level, std::stringstream&& log_stream);
#endif

/** Unsynchronized file logging.
 * All log messages will be appended to the given log file. %Log messages may be buffered in memory but will be
 * flushed upon destruction of the handler object.
 * Logging is not synchronized, so logging concurrently from multiple threads is not safe with this handler.
 * Use one of the \ref log_handler_adapters "thread-safe adapters" LogSynchronizeMutex or LogAsync if you need to
 * log from more than one thread.
 * @note This handler uses `std::fstream` for file I/O which might perform less than ideal on certain
 *       (*cough* Windows) implementations.
 *       Use the LogAsync adapter if performance is an issue.
 */
class LogToFile {
private:
    std::ofstream m_logFile;
public:
    /** Construct a logger for logging to a file.
     * @param[in] filename Path to the log file. This file will be opened in append mode.
     * @throw Exceptions::IOError If file could not be opened for writing.
     */
    GHULBUS_BASE_API LogToFile(char const* filename);

    /** Convert to a LogHandler function to pass to Ghulbus::Log::setLogHandler().
     * @attention Note that an object must not be destroyed while it is set as log handler.
     */
    GHULBUS_BASE_API operator LogHandler();
};

/** @defgroup log_handler_adapters Handler adapters.
 * A handler adapter is not a full handler by itself, but rather wraps around a downstream handler to add
 * additional functionality (like thread safety).
 * @{
 */
/** Simple synchronization.
 * The mutex adapter synchronizes concurrent access to the downstream handler via a mutex.
 * The resulting handler is thread safe.
 */
class LogSynchronizeMutex {
private:
    std::mutex m_mutex;
    LogHandler m_downstreamHandler;
public:
    /** Adapting Constructor.
     * @param[in] downstream_handler The log handler that is to be wrapped.
     *                               The downstream handler need not be thread safe.
     *                               The downstream handler must not be empty. If the downstream handler dies before
     *                               this adapter, the behavior is undefined.
     */
    GHULBUS_BASE_API LogSynchronizeMutex(LogHandler downstream_handler);

    /** Convert to a LogHandler function to pass to Ghulbus::Log::setLogHandler().
     * @attention Note that an object must not be destroyed while it is set as log handler.
     */
    GHULBUS_BASE_API operator LogHandler();
};

/** Asynchronous logging.
 * This adapter defers execution of the downstream handler in a separate thread. This allows the logging thread
 * to continue immediately after assembling the log message without having to wait for the log I/O to complete.
 * The impact on the runtime of the logging thread is far lower than with the other handlers. The downside is
 * that async logging requires spawning an additional I/O thread to execute the downstream handler.
 * Note that all access to the downstream handler is serialized through this I/O thread, so you can use a thread
 * unsafe handler as downstream handler.
 *
 * @note Messages will be queued in memory for processing. If an application produces log messages faster than the
 *       adapted handler can process them, this can in theory consume all available memory on the machine. Be careful
 *       not to overload the logging thread in this way.
 *
 * @note While the async handler works very well with small numbers of threads, it still requires concurrent access
 *       to the single queue of log messages. If you need highly concurrent logging, consider using an adapter that
 *       uses several thread-local queues instead.
 */
class LogAsync {
private:
    typedef std::tuple<LogLevel, std::stringstream> QueueElement;
private:
    std::mutex m_mutex;                     ///< mutex protexting access to the m_queue
    std::deque<QueueElement> m_queue;       ///< queue of log messages
    bool m_stopRequested;                   ///< flag indicating that the user called stop() to stop the I/O thread
    std::condition_variable m_condvar;      ///< signal that a message was pushed to the queue or stop was requested
    LogHandler m_downstreamHandler;
    std::thread m_ioThread;
public:
    /** @copydoc LogSynchronizeMutex::LogSynchronizeMutex(LogHandler)
     */
    GHULBUS_BASE_API LogAsync(LogHandler downstream_handler);

    /** Start the I/O thread.
     * Invoking the log handler obtained from this class will put the respective log message to an in-memory queue.
     * This function will spawn a thread that waits for messages to be put into that queue and forwards them to the
     * downstream handler. Note that while that thread is not running, messages will just keep piling up in memory,
     * so it is best to start the adapter *before* setting it as the active log handler.
     * Note that the object must not be destroyed while the I/O thread is running.
     * @see stop()
     * @pre The I/O thread is not already running.
     * @note This function is thread-safe.
     */
    GHULBUS_BASE_API void start();

    /** Stop the I/O thread.
     * Processes all outstanding log messages still in the queue and then joins the I/O thread.
     * %Log messages arriving after stop() has started executing will remain unprocessed. So you will probably want
     * to remove the adapter from active log handler before calling stop().
     * @see start()
     * @pre The I/O thread is currently running.
     * @note This function is thread-safe.
     */
    GHULBUS_BASE_API void stop();

    /** Convert to a LogHandler function to pass to Ghulbus::Log::setLogHandler().
     * @note Note that the handler needs to be \ref start() "started" for any log messages to be processed.
     * @attention Note that an object must not be destroyed while it is set as log handler.
     */
    GHULBUS_BASE_API operator LogHandler();
};

/** Forwards each log message to two downstream handlers.
 * Use this if you want to log to two different sinks, for instance to a log file and the console.
 * @note This handler will duplicate the log message before passing it on to the downstream handlers,
 *       which is potentially expensive. When combining the LogMultiSink with the LogAsync it is
 *       therefore desirable to have the LogMultiSink as the downstream and the LogAsync as the
 *       top-level handler.
 */
class LogMultiSink
{
private:
    LogHandler m_downstreamHandlers[2];
public:
    /** Adapting Constructor.
     * Neither of the downstream handlers shall be empty.
     * @param[in] first_downstream_handler The first log handler that is to be wrapped.
     * @param[in] second_downstream_handler The second log handler that is to be wrapped.
     */
    GHULBUS_BASE_API LogMultiSink(LogHandler first_downstream_handler,
                                  LogHandler second_downstream_handler);

    /** Convert to a LogHandler function to pass to Ghulbus::Log::setLogHandler().
     * @attention Note that an object must not be destroyed while it is set as log handler.
     */
    GHULBUS_BASE_API operator LogHandler();
};
/** @} */
}
}
}
#endif
<|endoftext|>


<commit_before>//
// Copyright (c) 2014 CNRS
// Authors: Florent Lamiraux
//
// This file is part of hpp-core
// hpp-core is free software: you can redistribute it
// and/or modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation, either version
// 3 of the License, or (at your option) any later version.
//
// hpp-core is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Lesser Public License for more details.  You should have
// received a copy of the GNU Lesser General Public License along with
// hpp-core  If not, see
// <http://www.gnu.org/licenses/>.

#ifndef HPP_CORE_LOCKED_DOF_HH
# define HPP_CORE_LOCKED_DOF_HH

# include <roboptim/core/differentiable-function.hh>
# include <hpp/core/constraint-set.hh>

namespace hpp {
  namespace core {
    /// Constraint that locks a degree of freedom to a constant value.
    class HPP_CORE_DLLAPI LockedDof : public Constraint
    {
    public:
      /// Return shared pointer to new object
      static LockedDofPtr_t create (const std::string& name, size_type index,
				    value_type value)
      {
	LockedDof* ptr = new LockedDof (name, index, value);
	LockedDofPtr_t shPtr (ptr);
	ptr->init (shPtr);
	return shPtr;
      }
      /// Get index of the locked degree of freedom in the configuration vector.
      std::size_t index () const
      {
	return index_;
      }
      /// Get the value of the configuration locked component.
      value_type value () const
      {
	return value_;
      }
    protected:
      LockedDof (const std::string& name, size_type index, value_type value) :
	Constraint (name), index_ (index), value_ (value)
      {
      }
      void init (const LockedDofPtr_t& self)
      {
	weak_ = self;
      }
      bool impl_compute (Configuration_t& configuration)
      {
	configuration [index_] = value_;
	return true;
      }
    private:
      virtual std::ostream& print (std::ostream& os) const
      {
	os << "Locked degree of freedom " << name () << ": index = "
	   << index_ << ", value = " << value_ << std::endl;
	return os;
      }
      virtual void addToConstraintSet (const ConstraintSetPtr_t& constraintSet)
      {
	constraintSet->addLockedDof (weak_.lock ());
	constraintSet->hasLockedDofs_ = true;
	Constraint::addToConstraintSet (constraintSet);
      }

      std::size_t index_;
      value_type value_;
      /// Weak pointer to itself
      LockedDofWkPtr weak_;
    }; // class LockedDof
  } // namespace core
} // namespace hpp
#endif // HPP_CORE_LOCKED_DOF_HH
<commit_msg>Give access to reference of the value of a locked dof.<commit_after>//
// Copyright (c) 2014 CNRS
// Authors: Florent Lamiraux
//
// This file is part of hpp-core
// hpp-core is free software: you can redistribute it
// and/or modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation, either version
// 3 of the License, or (at your option) any later version.
//
// hpp-core is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Lesser Public License for more details.  You should have
// received a copy of the GNU Lesser General Public License along with
// hpp-core  If not, see
// <http://www.gnu.org/licenses/>.

#ifndef HPP_CORE_LOCKED_DOF_HH
# define HPP_CORE_LOCKED_DOF_HH

# include <roboptim/core/differentiable-function.hh>
# include <hpp/core/constraint-set.hh>

namespace hpp {
  namespace core {
    /// Constraint that locks a degree of freedom to a constant value.
    class HPP_CORE_DLLAPI LockedDof : public Constraint
    {
    public:
      /// Return shared pointer to new object
      static LockedDofPtr_t create (const std::string& name, size_type index,
				    value_type value)
      {
	LockedDof* ptr = new LockedDof (name, index, value);
	LockedDofPtr_t shPtr (ptr);
	ptr->init (shPtr);
	return shPtr;
      }
      /// Get index of the locked degree of freedom in the configuration vector.
      std::size_t index () const
      {
	return index_;
      }
      /// Get the value of the configuration locked component.
      const value_type& value () const
      {
	return value_;
      }
      /// Get the value of the configuration locked component.
      value_type& value ()
      {
	return value_;
      }
    protected:
      LockedDof (const std::string& name, size_type index, value_type value) :
	Constraint (name), index_ (index), value_ (value)
      {
      }
      void init (const LockedDofPtr_t& self)
      {
	weak_ = self;
      }
      bool impl_compute (Configuration_t& configuration)
      {
	configuration [index_] = value_;
	return true;
      }
    private:
      virtual std::ostream& print (std::ostream& os) const
      {
	os << "Locked degree of freedom " << name () << ": index = "
	   << index_ << ", value = " << value_ << std::endl;
	return os;
      }
      virtual void addToConstraintSet (const ConstraintSetPtr_t& constraintSet)
      {
	constraintSet->addLockedDof (weak_.lock ());
	constraintSet->hasLockedDofs_ = true;
	Constraint::addToConstraintSet (constraintSet);
      }

      std::size_t index_;
      value_type value_;
      /// Weak pointer to itself
      LockedDofWkPtr weak_;
    }; // class LockedDof
  } // namespace core
} // namespace hpp
#endif // HPP_CORE_LOCKED_DOF_HH
<|endoftext|>


<commit_before>/*******************************************************************************
 * include/huffman/huff_merge.hpp
 *
 * Copyright (C) 2017 Marvin Löbel <loebel.marvin@gmail.com>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#pragma once

#include <cassert>
#include <climits>
#include <omp.h>

#include "huffman/huff_bit_vectors.hpp"
#include "util/common.hpp"
#include "util/macros.hpp"
#include "construction/merge.hpp"

template<typename ContextType, typename Rho>
inline auto huff_merge_bit_vectors(std::vector<uint64_t> const& level_sizes,
                                   uint64_t const shards,
                                   const std::vector<ContextType>& src_ctxs,
                                   const Rho& rho)
{
  assert(shards == src_ctxs.size());
  assert(shards > 1);

  uint64_t const levels = level_sizes.size();

  // Allocate data structures centrally

  struct MergeLevelCtx {
    std::vector<uint64_t> read_offsets;
    uint64_t initial_read_offset;
    uint64_t first_read_block;
    uint64_t write_end_offset;
  };

  struct MergeCtx {
    std::vector<MergeLevelCtx> levels;
  };

  auto ctxs = std::vector<MergeCtx> { shards,
    {
      std::vector<MergeLevelCtx> { levels,
        {
          std::vector<uint64_t>(shards),
          0,
          0,
          0,
        }
      },
    }
  };

  /*
    Visualization of ctxs for levels = 2 and shards = 2:
    ┌──────────────┬───────────────────────┐
    │ctxs: MergeCtx│ x shards              │
    │ ┌────────────┴────────┬──────────────┤
    │ │levels: MergeLevelCtx│ x levels     │
    │ │ ┌───────────────────┴──────────────┤
    │ │ │write_end_offset: uint64_t        │
    │ │ ├──────────────────────────────────┤
    │ │ │initial_read_offset: uint64_t     │
    │ │ ├──────────────────────────────────┤
    │ │ │first_read_block: uint64_t        │
    │ │ ├──────────────────────┬───────────┤
    │ │ │read_offsets: uint64_t│ x shards  │
    │ │ │ ┌────────────────────┴───────────┤
    │ │ │ │                                │
    │ │ │ ╞════════════════════════════════╡
    │ │ │ │                                │
    │ │ ╞═╧════════════════════════════════╡
    │ │ │initial_read_offset: uint64_t     │
    │ │ ├──────────────────────────────────┤
    │ │ │first_read_block: uint64_t        │
    │ │ ├──────────────────────┬───────────┤
    │ │ │read_offsets: uint64_t│ x shards  │
    │ │ │ ┌────────────────────┴───────────┤
    │ │ │ │                                │
    │ │ │ ╞════════════════════════════════╡
    │ │ │ │                                │
    │ ╞═╧═╧════════════════════════════════╡
    │ │levels: MergeLevelCtx│ x levels     │
    │ │ ┌───────────────────┴──────────────┤
    │ │ │write_end_offset: uint64_t        │
    │ │ ├──────────────────────────────────┤
    │ │ │initial_read_offset: uint64_t     │
    │ │ ├──────────────────────────────────┤
    │ │ │first_read_block: uint64_t        │
    │ │ ├──────────────────────┬───────────┤
    │ │ │read_offsets: uint64_t│ x shards  │
    │ │ │ ┌────────────────────┴───────────┤
    │ │ │ │                                │
    │ │ │ ╞════════════════════════════════╡
    │ │ │ │                                │
    │ │ ╞═╧════════════════════════════════╡
    │ │ │initial_read_offset: uint64_t     │
    │ │ ├──────────────────────────────────┤
    │ │ │first_read_block: uint64_t        │
    │ │ ├──────────────────────┬───────────┤
    │ │ │read_offsets: uint64_t│ x shards  │
    │ │ │ ┌────────────────────┴───────────┤
    │ │ │ │                                │
    │ │ │ ╞════════════════════════════════╡
    │ │ │ │                                │
    └─┴─┴─┴────────────────────────────────┘
  */

  // Calculate end bit offset per merge shard (thread).
  // It is an multiple of 64, to ensure no interference
  // in writing bits to the left or right of it in parallel
  for(size_t level = 0; level < levels; level++) {
    const size_t size = level_sizes[level];
    for (size_t shard = 1; shard < shards; shard++) {
      const uint64_t offset = (shard * (word_size(size) / shards)) +
        std::min<uint64_t>(shard, word_size(size) % shards);

      ctxs[shard - 1].levels[level].write_end_offset
        = std::min<uint64_t>(offset * 64ull, size);
    }
    ctxs[shards - 1].levels[level].write_end_offset
      = std::min<uint64_t>(word_size(size) * 64ull, size);
  }

  for(size_t level = 0; level < levels; level++) {
    // number of tree nodes on the current level
    const size_t blocks = 1ull << level;

    size_t write_offset = 0; // bit offset in destination bv
    size_t merge_shard = 0; // index of merge thread

    // iterate over all blocks of all shards on the current level
    //
    // this gradually assigns the bits of the
    // blocks to each of the aviable merge shards of the current level:
    //
    // |   read_shard  |   read_shard  |
    // +---------------+---------------+
    // |     block     |     block     |
    // | block | block | block | block | <- current level
    // +----------+----------+---------+
    // | m. shard | m. shard |m. shard |
    //
    for(size_t i = 0; i < blocks * shards; i++) {
      // std::cout << "[offsets]   i: " << i << "\n";

      // returns merge level context of merge_shard
      auto lctx = [&level, &ctxs](auto merge_shard) -> MergeLevelCtx& {
          return ctxs[merge_shard].levels[level];
      };

      // returns merge level context of merge_shard+1
      auto nxt_lctx = [&level, &ctxs](auto merge_shard) -> MergeLevelCtx& {
          return ctxs[merge_shard + 1].levels[level];
      };

      // which block (node on current level of tree)
      const auto block = i / shards;

      // which shard to read from
      const auto read_shard = i % shards;

      // map logical block index to
      // actual block index (rho depends on WT vs WM)
      const auto permuted_block = rho(level, block);

      // block size == number of entries in the block on this level
      auto block_size = src_ctxs[read_shard].hist(level, permuted_block);

      // advance global write offset by the number of bits assigned for
      // this block
      write_offset += block_size;

      // advance local read offset of next merge shard
      // for the curent read shard
      //
      // this way, all merge shard start reading
      // at different offsets from the read shards
      nxt_lctx(merge_shard).read_offsets[read_shard] += block_size;

      // If we passed the current right border, split up the block
      if (write_offset > lctx(merge_shard).write_end_offset) {
        // Take back the last step
        write_offset -= block_size;
        nxt_lctx(merge_shard).read_offsets[read_shard] -= block_size;

        uint64_t initial_read_offset = 0;
        do {
          // Split up the block like this:
          //     [  left_block_size  |    right_block_size    ]
          //     ^                   ^                        ^
          // (write_offset)          |                        |
          //           (ctxs[merge_shard].write_end_offset)   |
          //                              (write_offset + block_size)
          //
          // this loop iterates multiple times, to ensure right_block_size
          // did not also overlap the next write_end_offset

          auto const left_block_size
            = lctx(merge_shard).write_end_offset - write_offset;

          // advance global and local read offsets to end exactly at
          // write_end_offset
          write_offset += left_block_size;
          nxt_lctx(merge_shard).read_offsets[read_shard] += left_block_size;

          initial_read_offset += left_block_size;

          // from which offset in which block to start reading
          nxt_lctx(merge_shard).initial_read_offset = initial_read_offset;
          nxt_lctx(merge_shard).first_read_block = i;

          // if there is a merge_shard to the right
          // of the next merge shard, intialize
          // its local read offsets with those of the next shard
          if (merge_shard + 2 < shards) {
            for(size_t s = 0; s < shards; s++) {
              nxt_lctx(merge_shard + 1).read_offsets[s]
                = nxt_lctx(merge_shard).read_offsets[s];
            }
          }

          merge_shard++;

          // Once we have calculated the offsets for all merge threads,
          // break out of the whole nested loop
          if (merge_shard + 1 == shards) {
            goto triple_loop_exit;
          }

          // Iterate on remaining block, because one block might
          // span multiple threads
          block_size -= left_block_size;
        } while ((write_offset + block_size) > lctx(merge_shard).write_end_offset);

        // Process remainder of block
        write_offset += block_size;
        nxt_lctx(merge_shard).read_offsets[read_shard] += block_size;

        assert(write_offset <= lctx(merge_shard).write_end_offset);
      }
    }
    triple_loop_exit:; // we are done
  }

  // TODO: remove redundant argument
  auto r =  huff_bit_vectors(levels, level_sizes);
  auto& _bv = r;

  // TODO: Bugs

  //#pragma omp parallel
  for(size_t merge_shard = 0; merge_shard < shards; merge_shard++)
  {
    auto& ctx = ctxs[merge_shard];

    for (size_t level = 0; level < levels; level++) {
      const auto target_right = std::min(
        ctx.levels[level].write_end_offset,
        level_sizes[level]);
      const auto target_left = std::min(
        (merge_shard > 0 ? ctxs[merge_shard - 1].levels[level].write_end_offset : 0),
        target_right);

      // std::cout << "\n";
      // std::cout << "[merge] target_{left,right}: " << target_left;
      // std::cout << ", " << target_right << "\n";

      auto i = ctx.levels[level].first_read_block;
      uint64_t write_offset = target_left;

      auto copy_next_block = [&](size_t const initial_read_offset) {
        // std::cout << "\n";
        // std::cout << "[merge] i: " << i << "\n";
        // std::cout << "[merge] write_offset: " << write_offset << "\n";
        // std::cout << "[merge] initial_read_offset: " << initial_read_offset << "\n";

        const auto block = i / shards;
        const auto read_shard = i % shards;
        i++;

        // std::cout << "[merge] block: " << block << "\n";
        // std::cout << "[merge] read_shard: " << read_shard << "\n";

        const auto& local_bv = src_ctxs[read_shard].bv()[level];
        const auto& h = src_ctxs[read_shard];
        auto const permuted_block = rho(level, block);
        uint64_t block_size = h.hist(level, permuted_block) - initial_read_offset;
        uint64_t distance_to_end = target_right - write_offset;

        // std::cout << "[merge] block_size: " << block_size << "\n";
        // std::cout << "[merge] distance_to_end: " << distance_to_end << "\n";

        uint64_t copy_size;
        if (PWM_LIKELY(block_size <= distance_to_end)) {
            copy_size = block_size;
        } else {
            copy_size = distance_to_end;
        }

        auto& local_cursor = ctx.levels[level].read_offsets[read_shard];

        // std::cout << "[merge] copy_bits("
        //     << "write_offset=" << write_offset << ", "
        //     << "local_cursor=" << local_cursor << ", "
        //     << "copy_size=" << copy_size << ")"
        //     << "\n";
        copy_bits<uint64_t>(
          _bv[level].data(),
          local_bv.data(),
          write_offset,
          local_cursor,
          copy_size
        );
      };

      if (write_offset < target_right) {
        // the first block might start somewhere in the middle
        copy_next_block(ctx.levels[level].initial_read_offset);
      }
      while (write_offset < target_right) {
        // other blocks start at 0
        copy_next_block(0);
      }
      assert(write_offset == target_right);
    }
  }

  return r;
}

/******************************************************************************/
<commit_msg>FIx parallelism in huff_merge<commit_after>/*******************************************************************************
 * include/huffman/huff_merge.hpp
 *
 * Copyright (C) 2017 Marvin Löbel <loebel.marvin@gmail.com>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#pragma once

#include <cassert>
#include <climits>
#include <omp.h>

#include "huffman/huff_bit_vectors.hpp"
#include "util/common.hpp"
#include "util/macros.hpp"
#include "construction/merge.hpp"

template<typename ContextType, typename Rho>
inline auto huff_merge_bit_vectors(std::vector<uint64_t> const& level_sizes,
                                   uint64_t const shards,
                                   const std::vector<ContextType>& src_ctxs,
                                   const Rho& rho)
{
  assert(shards == src_ctxs.size());
  assert(shards > 1);

  uint64_t const levels = level_sizes.size();

  // Allocate data structures centrally

  struct MergeLevelCtx {
    std::vector<uint64_t> read_offsets;
    uint64_t initial_read_offset;
    uint64_t first_read_block;
    uint64_t write_end_offset;
  };

  struct MergeCtx {
    std::vector<MergeLevelCtx> levels;
  };

  auto ctxs = std::vector<MergeCtx> { shards,
    {
      std::vector<MergeLevelCtx> { levels,
        {
          std::vector<uint64_t>(shards),
          0,
          0,
          0,
        }
      },
    }
  };

  /*
    Visualization of ctxs for levels = 2 and shards = 2:
    ┌──────────────┬───────────────────────┐
    │ctxs: MergeCtx│ x shards              │
    │ ┌────────────┴────────┬──────────────┤
    │ │levels: MergeLevelCtx│ x levels     │
    │ │ ┌───────────────────┴──────────────┤
    │ │ │write_end_offset: uint64_t        │
    │ │ ├──────────────────────────────────┤
    │ │ │initial_read_offset: uint64_t     │
    │ │ ├──────────────────────────────────┤
    │ │ │first_read_block: uint64_t        │
    │ │ ├──────────────────────┬───────────┤
    │ │ │read_offsets: uint64_t│ x shards  │
    │ │ │ ┌────────────────────┴───────────┤
    │ │ │ │                                │
    │ │ │ ╞════════════════════════════════╡
    │ │ │ │                                │
    │ │ ╞═╧════════════════════════════════╡
    │ │ │initial_read_offset: uint64_t     │
    │ │ ├──────────────────────────────────┤
    │ │ │first_read_block: uint64_t        │
    │ │ ├──────────────────────┬───────────┤
    │ │ │read_offsets: uint64_t│ x shards  │
    │ │ │ ┌────────────────────┴───────────┤
    │ │ │ │                                │
    │ │ │ ╞════════════════════════════════╡
    │ │ │ │                                │
    │ ╞═╧═╧════════════════════════════════╡
    │ │levels: MergeLevelCtx│ x levels     │
    │ │ ┌───────────────────┴──────────────┤
    │ │ │write_end_offset: uint64_t        │
    │ │ ├──────────────────────────────────┤
    │ │ │initial_read_offset: uint64_t     │
    │ │ ├──────────────────────────────────┤
    │ │ │first_read_block: uint64_t        │
    │ │ ├──────────────────────┬───────────┤
    │ │ │read_offsets: uint64_t│ x shards  │
    │ │ │ ┌────────────────────┴───────────┤
    │ │ │ │                                │
    │ │ │ ╞════════════════════════════════╡
    │ │ │ │                                │
    │ │ ╞═╧════════════════════════════════╡
    │ │ │initial_read_offset: uint64_t     │
    │ │ ├──────────────────────────────────┤
    │ │ │first_read_block: uint64_t        │
    │ │ ├──────────────────────┬───────────┤
    │ │ │read_offsets: uint64_t│ x shards  │
    │ │ │ ┌────────────────────┴───────────┤
    │ │ │ │                                │
    │ │ │ ╞════════════════════════════════╡
    │ │ │ │                                │
    └─┴─┴─┴────────────────────────────────┘
  */

  // Calculate end bit offset per merge shard (thread).
  // It is an multiple of 64, to ensure no interference
  // in writing bits to the left or right of it in parallel
  for(size_t level = 0; level < levels; level++) {
    const size_t size = level_sizes[level];
    for (size_t shard = 1; shard < shards; shard++) {
      const uint64_t offset = (shard * (word_size(size) / shards)) +
        std::min<uint64_t>(shard, word_size(size) % shards);

      ctxs[shard - 1].levels[level].write_end_offset
        = std::min<uint64_t>(offset * 64ull, size);
    }
    ctxs[shards - 1].levels[level].write_end_offset
      = std::min<uint64_t>(word_size(size) * 64ull, size);
  }

  for(size_t level = 0; level < levels; level++) {
    // number of tree nodes on the current level
    const size_t blocks = 1ull << level;

    size_t write_offset = 0; // bit offset in destination bv
    size_t merge_shard = 0; // index of merge thread

    // iterate over all blocks of all shards on the current level
    //
    // this gradually assigns the bits of the
    // blocks to each of the aviable merge shards of the current level:
    //
    // |   read_shard  |   read_shard  |
    // +---------------+---------------+
    // |     block     |     block     |
    // | block | block | block | block | <- current level
    // +----------+----------+---------+
    // | m. shard | m. shard |m. shard |
    //
    for(size_t i = 0; i < blocks * shards; i++) {
      // std::cout << "[offsets]   i: " << i << "\n";

      // returns merge level context of merge_shard
      auto lctx = [&level, &ctxs](auto merge_shard) -> MergeLevelCtx& {
          return ctxs[merge_shard].levels[level];
      };

      // returns merge level context of merge_shard+1
      auto nxt_lctx = [&level, &ctxs](auto merge_shard) -> MergeLevelCtx& {
          return ctxs[merge_shard + 1].levels[level];
      };

      // which block (node on current level of tree)
      const auto block = i / shards;

      // which shard to read from
      const auto read_shard = i % shards;

      // map logical block index to
      // actual block index (rho depends on WT vs WM)
      const auto permuted_block = rho(level, block);

      // block size == number of entries in the block on this level
      auto block_size = src_ctxs[read_shard].hist(level, permuted_block);

      // advance global write offset by the number of bits assigned for
      // this block
      write_offset += block_size;

      // advance local read offset of next merge shard
      // for the curent read shard
      //
      // this way, all merge shard start reading
      // at different offsets from the read shards
      nxt_lctx(merge_shard).read_offsets[read_shard] += block_size;

      // If we passed the current right border, split up the block
      if (write_offset > lctx(merge_shard).write_end_offset) {
        // Take back the last step
        write_offset -= block_size;
        nxt_lctx(merge_shard).read_offsets[read_shard] -= block_size;

        uint64_t initial_read_offset = 0;
        do {
          // Split up the block like this:
          //     [  left_block_size  |    right_block_size    ]
          //     ^                   ^                        ^
          // (write_offset)          |                        |
          //           (ctxs[merge_shard].write_end_offset)   |
          //                              (write_offset + block_size)
          //
          // this loop iterates multiple times, to ensure right_block_size
          // did not also overlap the next write_end_offset

          auto const left_block_size
            = lctx(merge_shard).write_end_offset - write_offset;

          // advance global and local read offsets to end exactly at
          // write_end_offset
          write_offset += left_block_size;
          nxt_lctx(merge_shard).read_offsets[read_shard] += left_block_size;

          initial_read_offset += left_block_size;

          // from which offset in which block to start reading
          nxt_lctx(merge_shard).initial_read_offset = initial_read_offset;
          nxt_lctx(merge_shard).first_read_block = i;

          // if there is a merge_shard to the right
          // of the next merge shard, intialize
          // its local read offsets with those of the next shard
          if (merge_shard + 2 < shards) {
            for(size_t s = 0; s < shards; s++) {
              nxt_lctx(merge_shard + 1).read_offsets[s]
                = nxt_lctx(merge_shard).read_offsets[s];
            }
          }

          merge_shard++;

          // Once we have calculated the offsets for all merge threads,
          // break out of the whole nested loop
          if (merge_shard + 1 == shards) {
            goto triple_loop_exit;
          }

          // Iterate on remaining block, because one block might
          // span multiple threads
          block_size -= left_block_size;
        } while ((write_offset + block_size) > lctx(merge_shard).write_end_offset);

        // Process remainder of block
        write_offset += block_size;
        nxt_lctx(merge_shard).read_offsets[read_shard] += block_size;

        assert(write_offset <= lctx(merge_shard).write_end_offset);
      }
    }
    triple_loop_exit:; // we are done
  }

  // TODO: remove redundant argument
  auto r =  huff_bit_vectors(levels, level_sizes);
  auto& _bv = r;

  #pragma omp parallel for
  for(size_t merge_shard = 0; merge_shard < shards; merge_shard++) {
    for (size_t level = 0; level < levels; level++) {
      auto& lctx = ctxs[merge_shard].levels[level];

      const auto target_right = std::min(
        lctx.write_end_offset,
        level_sizes[level]);
      const auto target_left = std::min(
        (merge_shard > 0
          ? ctxs[merge_shard - 1].levels[level].write_end_offset
          : 0),
        target_right);

      uint64_t i = lctx.first_read_block;
      uint64_t write_offset = target_left;

      auto copy_next_block = [&](size_t const initial_read_offset) {
        const auto block = i / shards;
        const auto read_shard = i % shards;
        i++;

        const auto& local_bv = src_ctxs[read_shard].bv()[level];
        const auto& h = src_ctxs[read_shard];
        auto const permuted_block = rho(level, block);
        uint64_t block_size = h.hist(level, permuted_block) - initial_read_offset;
        uint64_t distance_to_end = target_right - write_offset;

        uint64_t copy_size;
        if (PWM_LIKELY(block_size <= distance_to_end)) {
            copy_size = block_size;
        } else {
            copy_size = distance_to_end;
        }

        auto& local_cursor = lctx.read_offsets[read_shard];

        copy_bits<uint64_t>(
          _bv[level].data(),
          local_bv.data(),
          write_offset,
          local_cursor,
          copy_size
        );
      };

      if (write_offset < target_right) {
        // the first block might start somewhere in the middle
        copy_next_block(lctx.initial_read_offset);
      }
      while (write_offset < target_right) {
        // other blocks start at 0
        copy_next_block(0);
      }
      assert(write_offset == target_right);
    }
  }

  return r;
}

/******************************************************************************/
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#ifndef MNIST_READER_HPP
#define MNIST_READER_HPP

#include <fstream>
#include <iostream>
#include <vector>
#include <cstdint>
#include <memory>

namespace mnist {

template<template<typename...> class  Container = std::vector, template<typename...> class  Sub = std::vector, typename Pixel = uint8_t, typename Label = uint8_t>
struct MNIST_dataset {
    Container<Sub<Pixel>> training_images;
    Container<Sub<Pixel>> test_images;
    Container<Label> training_labels;
    Container<Label> test_labels;

    void resize_training(std::size_t new_size){
        if(training_images.size() > new_size){
            training_images.resize(new_size);
            training_labels.resize(new_size);
        }
    }

    void resize_test(std::size_t new_size){
        if(test_images.size() > new_size){
            test_images.resize(new_size);
            test_labels.resize(new_size);
        }
    }
};

inline uint32_t read_header(const std::unique_ptr<char[]>& buffer, size_t position){
    auto header = reinterpret_cast<uint32_t*>(buffer.get());

    auto value = *(header + position);
    return (value << 24) | ((value << 8) & 0x00FF0000) | ((value >> 8) & 0X0000FF00) | (value >> 24);
}

template<template<typename...> class Container = std::vector, template<typename...> class Sub = std::vector, typename Pixel = uint8_t>
Container<Sub<Pixel>> read_mnist_image_file(const std::string& path, std::size_t limit = 0){
    std::ifstream file;
    file.open(path, std::ios::in | std::ios::binary | std::ios::ate);

    if(!file){
        std::cout << "Error opening file" << std::endl;
    } else {
        auto size = file.tellg();
        std::unique_ptr<char[]> buffer(new char[size]);

        //Read the entire file at once
        file.seekg(0, std::ios::beg);
        file.read(buffer.get(), size);
        file.close();

        auto magic = read_header(buffer, 0);

        if(magic != 0x803){
            std::cout << "Invalid magic number, probably not a MNIST file" << std::endl;
        } else {
            auto count = read_header(buffer, 1);
            auto rows = read_header(buffer, 2);
            auto columns = read_header(buffer, 3);

            if(size < count * rows * columns + 16){
                std::cout << "The file is not large enough to hold all the data, probably corrupted" << std::endl;
            } else {
                //Skip the header
                //Cast to unsigned char is necessary cause signedness of char is
                //platform-specific
                auto image_buffer = reinterpret_cast<unsigned char*>(buffer.get() + 16);

                if(limit > 0 && count > limit){
                    count = limit;
                }

                Container<Sub<Pixel>> images;
                images.reserve(count);

                for(size_t i = 0; i < count; ++i){
                    images.emplace_back(rows * columns);

                    for(size_t j = 0; j < rows * columns; ++j){
                        auto pixel = *image_buffer++;
                        images[i][j] = static_cast<Pixel>(pixel);
                    }
                }

                return images;
            }
        }
    }

    return {};
}

template<template<typename...> class  Container = std::vector, typename Label = uint8_t>
Container<Label> read_mnist_label_file(const std::string& path, std::size_t limit = 0){
    std::ifstream file;
    file.open(path, std::ios::in | std::ios::binary | std::ios::ate);

    if(!file){
        std::cout << "Error opening file" << std::endl;
    } else {
        auto size = file.tellg();
        std::unique_ptr<char[]> buffer(new char[size]);

        //Read the entire file at once
        file.seekg(0, std::ios::beg);
        file.read(buffer.get(), size);
        file.close();

        auto magic = read_header(buffer, 0);

        if(magic != 0x801){
            std::cout << "Invalid magic number, probably not a MNIST file" << std::endl;
        } else {
            auto count = read_header(buffer, 1);

            if(size < count + 8){
                std::cout << "The file is not large enough to hold all the data, probably corrupted" << std::endl;
            } else {
                //Skip the header
                //Cast to unsigned char is necessary cause signedness of char is
                //platform-specific
                auto label_buffer = reinterpret_cast<unsigned char*>(buffer.get() + 8);

                if(limit > 0 && count > limit){
                    count = limit;
                }

                Container<Label> labels(count);

                for(size_t i = 0; i < count; ++i){
                    auto label = *label_buffer++;
                    labels[i] = static_cast<Label>(label);
                }

                return labels;
            }
        }
    }

    return {};
}

template<template<typename...> class Container = std::vector, template<typename...> class Sub = std::vector, typename Pixel = uint8_t>
Container<Sub<Pixel>> read_training_images(std::size_t limit = 0){
    return read_mnist_image_file<Container,Sub,Pixel>("mnist/train-images-idx3-ubyte", limit);
}

template<template<typename...> class Container = std::vector, template<typename...> class Sub = std::vector, typename Pixel = uint8_t>
Container<Sub<Pixel>> read_test_images(std::size_t limit = 0){
    return read_mnist_image_file<Container,Sub,Pixel>("mnist/t10k-images-idx3-ubyte", limit);
}

template<template<typename...> class Container = std::vector, typename Label = uint8_t>
Container<Label> read_training_labels(std::size_t limit = 0){
    return read_mnist_label_file<Container, Label>("mnist/train-labels-idx1-ubyte", limit);
}

template<template<typename...> class Container = std::vector, typename Label = uint8_t>
Container<Label> read_test_labels(std::size_t limit = 0){
    return read_mnist_label_file<Container, Label>("mnist/t10k-labels-idx1-ubyte", limit);
}

template<template<typename...> class Container = std::vector, template<typename...> class Sub = std::vector, typename Pixel = uint8_t, typename Label = uint8_t>
MNIST_dataset<Container, Sub, Pixel, Label> read_dataset(std::size_t training_limit = 0, std::size_t test_limit = 0){
    MNIST_dataset<Container, Sub, Pixel, Label> dataset;

    dataset.training_images = read_training_images<Container, Sub, Pixel>(training_limit);
    dataset.training_labels = read_training_labels<Container, Label>(training_limit);

    dataset.test_images = read_test_images<Container, Sub, Pixel>(test_limit);
    dataset.test_labels = read_test_labels<Container, Label>(test_limit);

    return dataset;
}

} //end of namespace mnist

#endif
<commit_msg>Make the types more flexible<commit_after>//=======================================================================
// Copyright (c) 2014 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#ifndef MNIST_READER_HPP
#define MNIST_READER_HPP

#include <fstream>
#include <iostream>
#include <vector>
#include <cstdint>
#include <memory>

namespace mnist {

template<template<typename...> class Container, typename Image, typename Label>
struct MNIST_dataset {
    Container<Image> training_images;
    Container<Image> test_images;
    Container<Label> training_labels;
    Container<Label> test_labels;

    void resize_training(std::size_t new_size){
        if(training_images.size() > new_size){
            training_images.resize(new_size);
            training_labels.resize(new_size);
        }
    }

    void resize_test(std::size_t new_size){
        if(test_images.size() > new_size){
            test_images.resize(new_size);
            test_labels.resize(new_size);
        }
    }
};

inline uint32_t read_header(const std::unique_ptr<char[]>& buffer, size_t position){
    auto header = reinterpret_cast<uint32_t*>(buffer.get());

    auto value = *(header + position);
    return (value << 24) | ((value << 8) & 0x00FF0000) | ((value >> 8) & 0X0000FF00) | (value >> 24);
}

template<template<typename...> class Container = std::vector, typename Image>
Container<Image> read_mnist_image_file(const std::string& path, std::size_t limit = 0){
    std::ifstream file;
    file.open(path, std::ios::in | std::ios::binary | std::ios::ate);

    if(!file){
        std::cout << "Error opening file" << std::endl;
    } else {
        auto size = file.tellg();
        std::unique_ptr<char[]> buffer(new char[size]);

        //Read the entire file at once
        file.seekg(0, std::ios::beg);
        file.read(buffer.get(), size);
        file.close();

        auto magic = read_header(buffer, 0);

        if(magic != 0x803){
            std::cout << "Invalid magic number, probably not a MNIST file" << std::endl;
        } else {
            auto count = read_header(buffer, 1);
            auto rows = read_header(buffer, 2);
            auto columns = read_header(buffer, 3);

            if(size < count * rows * columns + 16){
                std::cout << "The file is not large enough to hold all the data, probably corrupted" << std::endl;
            } else {
                //Skip the header
                //Cast to unsigned char is necessary cause signedness of char is
                //platform-specific
                auto image_buffer = reinterpret_cast<unsigned char*>(buffer.get() + 16);

                if(limit > 0 && count > limit){
                    count = limit;
                }

                Container<Image> images;
                images.reserve(count);

                for(size_t i = 0; i < count; ++i){
                    images.emplace_back(rows * columns);

                    for(size_t j = 0; j < rows * columns; ++j){
                        auto pixel = *image_buffer++;
                        images[i][j] = static_cast<typename Image::value_type>(pixel);
                    }
                }

                return std::move(images);
            }
        }
    }

    return {};
}

template<template<typename...> class  Container = std::vector, typename Label = uint8_t>
Container<Label> read_mnist_label_file(const std::string& path, std::size_t limit = 0){
    std::ifstream file;
    file.open(path, std::ios::in | std::ios::binary | std::ios::ate);

    if(!file){
        std::cout << "Error opening file" << std::endl;
    } else {
        auto size = file.tellg();
        std::unique_ptr<char[]> buffer(new char[size]);

        //Read the entire file at once
        file.seekg(0, std::ios::beg);
        file.read(buffer.get(), size);
        file.close();

        auto magic = read_header(buffer, 0);

        if(magic != 0x801){
            std::cout << "Invalid magic number, probably not a MNIST file" << std::endl;
        } else {
            auto count = read_header(buffer, 1);

            if(size < count + 8){
                std::cout << "The file is not large enough to hold all the data, probably corrupted" << std::endl;
            } else {
                //Skip the header
                //Cast to unsigned char is necessary cause signedness of char is
                //platform-specific
                auto label_buffer = reinterpret_cast<unsigned char*>(buffer.get() + 8);

                if(limit > 0 && count > limit){
                    count = limit;
                }

                Container<Label> labels(count);

                for(size_t i = 0; i < count; ++i){
                    auto label = *label_buffer++;
                    labels[i] = static_cast<Label>(label);
                }

                return std::move(labels);
            }
        }
    }

    return {};
}

template<template<typename...> class Container = std::vector, typename Image>
Container<Image> read_training_images(std::size_t limit = 0){
    return read_mnist_image_file<Container, Image>("mnist/train-images-idx3-ubyte", limit);
}

template<template<typename...> class Container = std::vector, typename Image>
Container<Image> read_test_images(std::size_t limit = 0){
    return read_mnist_image_file<Container, Image>("mnist/t10k-images-idx3-ubyte", limit);
}

template<template<typename...> class Container = std::vector, typename Label = uint8_t>
Container<Label> read_training_labels(std::size_t limit = 0){
    return read_mnist_label_file<Container, Label>("mnist/train-labels-idx1-ubyte", limit);
}

template<template<typename...> class Container = std::vector, typename Label = uint8_t>
Container<Label> read_test_labels(std::size_t limit = 0){
    return read_mnist_label_file<Container, Label>("mnist/t10k-labels-idx1-ubyte", limit);
}

template<template<typename...> class Container, typename Image, typename Label = uint8_t>
MNIST_dataset<Container, Image, Label> read_dataset_direct(std::size_t training_limit = 0, std::size_t test_limit = 0){
    MNIST_dataset<Container, Image, Label> dataset;

    dataset.training_images = read_training_images<Container, Image>(training_limit);
    dataset.training_labels = read_training_labels<Container, Label>(training_limit);

    dataset.test_images = read_test_images<Container, Image>(test_limit);
    dataset.test_labels = read_test_labels<Container, Label>(test_limit);

    return dataset;
}

template<template<typename...> class Container = std::vector, template<typename...> class Sub = std::vector, typename Pixel = uint8_t, typename Label = uint8_t>
MNIST_dataset<Container, Sub<Pixel>, Label> read_dataset(std::size_t training_limit = 0, std::size_t test_limit = 0){
    return read_dataset_direct<Container, Sub<Pixel>>(training_limit, test_limit);
}

} //end of namespace mnist

#endif
<|endoftext|>


<commit_before>#pragma once

#include <utility>
#include <stdexcept>
#include <vector>

#include <array_view.hpp>
#include "reflection_info.hpp"
#include "api_types.hpp"
#include "info_iterators.hpp"

namespace shadow
{

namespace reflection_initialization_detail
{
// used to handle cases where reflection_manager is initialized with nullptrs
// (ie empty) instead of arrays
template <class T>
struct array_selector
{
    static helene::array_view<std::remove_pointer_t<std::remove_const_t<T>>>
        initialize(T)
    {
        return helene::array_view<
            std::remove_pointer_t<std::remove_const_t<T>>>();
    }
};

template <class I, std::size_t N>
struct array_selector<I[N]>
{
    static helene::array_view<I> initialize(I (&arr)[N])
    {
        return helene::make_array_view(arr);
    }
};
} // namespace reflection_initialization_detail

// point of interaction with the reflection system
class reflection_manager
{
public:
    typedef info_iterator_<const type_info, type_tag> const_type_iterator;
    typedef indexed_info_iterator_<const type_info, type_tag>
        const_indexed_type_iterator;
    typedef info_iterator_<const constructor_info, constructor_tag>
        const_constructor_iterator;

    typedef info_iterator_<const conversion_info, conversion_tag>
        const_conversion_iterator;

    typedef info_iterator_<const free_function_info, free_function_tag>
        const_free_function_iterator;

public:
    template <class TypeInfoArray,
              class ConstructorInfoArray,
              class ConversionInfoArray,
              class FreeFunctionArray,
              class MemberFunctionArray,
              class MemberVariableArray,
              class StringSerializationArray>
    reflection_manager(TypeInfoArray& ti_arr,
                       ConstructorInfoArray& ci_arr,
                       ConversionInfoArray& cv_arr,
                       FreeFunctionArray& ff_arr,
                       MemberFunctionArray& mf_arr,
                       MemberVariableArray& mv_arr,
                       StringSerializationArray& ss_arr);

    // queries on types
    std::pair<const_type_iterator, const_type_iterator> types() const;

    std::string type_name(const type_tag& tag) const;

    std::size_t type_size(const type_tag& tag) const;

    // returns range of all constructors available
    std::pair<const_constructor_iterator, const_constructor_iterator>
    constructors() const;

    // returns the type that the given constructor belongs to
    type_tag constructor_type(const constructor_tag& tag) const;

    // returns range of the types of the parameters of the given constructor
    std::pair<const_indexed_type_iterator, const_indexed_type_iterator>
    constructor_parameter_types(const constructor_tag& tag) const;

    template <class Iterator>
    object construct_object(const constructor_tag& tag,
                            Iterator first,
                            Iterator last) const;

    // type conversion functions
    std::pair<const_conversion_iterator, const_conversion_iterator>
    conversions() const;

    std::pair<type_tag, type_tag>
    conversion_types(const conversion_tag& tag) const;

    object convert(const conversion_tag& tag, const object& val) const;

    // returns range of tags to all free functions available
    std::pair<const_free_function_iterator, const_free_function_iterator>
    free_functions() const;

    // returns name of function associated with the free_function_tag
    std::string free_function_name(const free_function_tag& tag) const;

    // returns return type of function associated with the free_function_tag
    type_tag free_function_return_type(const free_function_tag& tag) const;

    // return range of type tags of parameters of free function associated with
    // the tag
    std::pair<const_indexed_type_iterator, const_indexed_type_iterator>
    free_function_parameter_types(const free_function_tag& tag) const;

    template <class Iterator>
    object call_free_function(const free_function_tag& tag,
                              Iterator first,
                              Iterator last) const;

public:
    // unchecked operations
    template <class T>
    T& get(object& obj) const;

    template <class T>
    const T& get(const object& obj) const;

private:
    template <class Iterator, class InfoType>
    bool
    check_arguments(Iterator first, Iterator last, const InfoType& info) const;

    bool compare_type(const type_tag& tag, std::size_t index) const;

    template <class Iterator, class OutputIterator, class InfoType>
    void construct_argument_array(Iterator first,
                                  Iterator last,
                                  OutputIterator out,
                                  const InfoType& info) const;

private:
    // array_views of reflection information generated at compile time
    helene::array_view<const type_info> type_info_view_;
    helene::array_view<const constructor_info> constructor_info_view_;
    helene::array_view<const conversion_info> conversion_info_view_;
    helene::array_view<const free_function_info> free_function_info_view_;
    helene::array_view<const member_function_info> member_function_info_view_;
    helene::array_view<const member_variable_info> member_variable_info_view_;
    helene::array_view<const string_serialization_info>
        string_serialization_info_view_;
};
} // namespace shadow

namespace shadow
{
template <class TypeInfoArray,
          class ConstructorInfoArray,
          class ConversionInfoArray,
          class FreeFunctionArray,
          class MemberFunctionArray,
          class MemberVariableArray,
          class StringSerializationArray>
inline reflection_manager::reflection_manager(TypeInfoArray& ti_arr,
                                              ConstructorInfoArray& ci_arr,
                                              ConversionInfoArray& cv_arr,
                                              FreeFunctionArray& ff_arr,
                                              MemberFunctionArray& mf_arr,
                                              MemberVariableArray& mv_arr,
                                              StringSerializationArray& ss_arr)
    : type_info_view_(reflection_initialization_detail::array_selector<
                      TypeInfoArray>::initialize(ti_arr)),
      constructor_info_view_(reflection_initialization_detail::array_selector<
                             ConstructorInfoArray>::initialize(ci_arr)),
      conversion_info_view_(reflection_initialization_detail::array_selector<
                            ConversionInfoArray>::initialize(cv_arr)),
      free_function_info_view_(reflection_initialization_detail::array_selector<
                               FreeFunctionArray>::initialize(ff_arr)),
      member_function_info_view_(
          reflection_initialization_detail::array_selector<
              MemberFunctionArray>::initialize(mf_arr)),
      member_variable_info_view_(
          reflection_initialization_detail::array_selector<
              MemberVariableArray>::initialize(mv_arr)),
      string_serialization_info_view_(
          reflection_initialization_detail::array_selector<
              StringSerializationArray>::initialize(ss_arr))
{
}

inline std::pair<typename reflection_manager::const_type_iterator,
                 typename reflection_manager::const_type_iterator>
reflection_manager::types() const
{
    return std::make_pair(const_type_iterator(type_info_view_.cbegin()),
                          const_type_iterator(type_info_view_.cend()));
}

inline std::string
reflection_manager::type_name(const type_tag& tag) const
{
    return tag.name();
}

inline std::size_t
reflection_manager::type_size(const type_tag& tag) const
{
    return tag.size();
}

inline std::pair<typename reflection_manager::const_constructor_iterator,
                 typename reflection_manager::const_constructor_iterator>
reflection_manager::constructors() const
{
    return std::make_pair(
        const_constructor_iterator(constructor_info_view_.cbegin()),
        const_constructor_iterator(constructor_info_view_.cend()));
}

inline type_tag
reflection_manager::constructor_type(const constructor_tag& tag) const
{
    return type_tag(type_info_view_[tag.info_ptr_->type_index]);
}

inline std::pair<reflection_manager::const_indexed_type_iterator,
                 reflection_manager::const_indexed_type_iterator>
reflection_manager::constructor_parameter_types(
    const constructor_tag& tag) const
{
    auto index_buffer = tag.info_ptr_->parameter_type_indices;
    auto data_buffer = type_info_view_.data();

    return std::make_pair(
        const_indexed_type_iterator(0, index_buffer, data_buffer),
        const_indexed_type_iterator(
            tag.info_ptr_->num_parameters, index_buffer, data_buffer));
}

inline bool
reflection_manager::compare_type(const type_tag& tag, std::size_t index) const
{
    type_tag tag_from_index(type_info_view_[index]);

    return tag == tag_from_index;
}

template <class Iterator, class InfoType>
inline bool
reflection_manager::check_arguments(Iterator first,
                                    Iterator last,
                                    const InfoType& info) const
{
    if(info.num_parameters != std::distance(first, last))
    {
        return false;
    }

    for(auto index_ptr = info.parameter_type_indices; first != last;
        ++index_ptr, ++first)
    {
        if(compare_type(first->type(), *index_ptr) == false)
        {
            return false;
        }
    }

    return true;
}

template <class Iterator, class OutputIterator, class InfoType>
inline void
reflection_manager::construct_argument_array(Iterator first,
                                             Iterator last,
                                             OutputIterator out,
                                             const InfoType& info) const
{
    for(auto pointer_flag_ptr = info.parameter_pointer_flags; first != last;
        ++pointer_flag_ptr, ++first, ++out)
    {
        if(*pointer_flag_ptr)
        {
            // construct any with pointer to value
            auto address_bind_point = first->type_info_->address_of_bind_point;

            out = address_bind_point(first->value_);
        }
        else
        {
            out = first->value_;
        }
    }
}

template <class Iterator>
inline object
reflection_manager::construct_object(const constructor_tag& tag,
                                     Iterator first,
                                     Iterator last) const
{
    if(check_arguments(first, last, *tag.info_ptr_) == false)
    {
        throw std::runtime_error("wrong argument types");
    }

    std::vector<any> arg_vec;
    arg_vec.reserve(std::distance(first, last));

    construct_argument_array(
        first, last, std::back_inserter(arg_vec), *tag.info_ptr_);

    auto return_value = tag.info_ptr_->bind_point(arg_vec.data());

    return object(
        return_value, type_info_view_.data() + tag.info_ptr_->type_index, this);
}

inline std::pair<reflection_manager::const_conversion_iterator,
                 reflection_manager::const_conversion_iterator>
reflection_manager::conversions() const
{
    return std::make_pair(
        const_conversion_iterator(conversion_info_view_.cbegin()),
        const_conversion_iterator(conversion_info_view_.cend()));
}

inline std::pair<type_tag, type_tag>
reflection_manager::conversion_types(const conversion_tag& tag) const
{
    return std::make_pair(
        type_tag(type_info_view_[tag.info_ptr_->from_type_index]),
        type_tag(type_info_view_[tag.info_ptr_->to_type_index]));
}

inline object
reflection_manager::convert(const conversion_tag& tag, const object& val) const
{
    if(val.type() != type_tag(type_info_view_[tag.info_ptr_->from_type_index]))
    {
        throw std::runtime_error(
            "type of object doesn't match conversion binding");
    }

    return object(tag.info_ptr_->bind_point(val.value_),
                  type_info_view_.data() + tag.info_ptr_->to_type_index,
                  this);
}
template <class T>
inline T&
reflection_manager::get(object& obj) const
{
    return obj.value_.get<T>();
}

template <class T>
inline const T&
reflection_manager::get(const object& obj) const
{
    return obj.value_.get<T>();
}

inline std::pair<reflection_manager::const_free_function_iterator,
                 reflection_manager::const_free_function_iterator>
reflection_manager::free_functions() const
{
    return std::make_pair(
        const_free_function_iterator(free_function_info_view_.cbegin()),
        const_free_function_iterator(free_function_info_view_.cend()));
}

inline std::string
reflection_manager::free_function_name(const free_function_tag& tag) const
{
    return std::string(tag.info_ptr_->name);
}

inline type_tag
reflection_manager::free_function_return_type(
    const free_function_tag& tag) const
{
    return type_tag(type_info_view_[tag.info_ptr_->return_type_index]);
}

inline std::pair<reflection_manager::const_indexed_type_iterator,
                 reflection_manager::const_indexed_type_iterator>
reflection_manager::free_function_parameter_types(
    const free_function_tag& tag) const
{
    return std::make_pair(
        const_indexed_type_iterator(
            0, tag.info_ptr_->parameter_type_indices, type_info_view_.data()),
        const_indexed_type_iterator(tag.info_ptr_->num_parameters,
                                    tag.info_ptr_->parameter_type_indices,
                                    type_info_view_.data()));
}

template <class Iterator>
inline object
reflection_manager::call_free_function(const free_function_tag& tag,
                                       Iterator first,
                                       Iterator last) const
{
    if(!check_arguments(first, last, *tag.info_ptr_))
    {
        throw std::runtime_error(
            "attempting to call free function with arguments of wrong type");
    }

    std::vector<shadow::any> args;
    args.reserve(std::distance(first, last));

    construct_argument_array(
        first, last, std::back_inserter(args), *tag.info_ptr_);

    auto return_value = tag.info_ptr_->bind_point(args.data());

    return object(return_value,
                  type_info_view_.data() + tag.info_ptr_->return_type_index,
                  this);
}
} // namespace shadow
<commit_msg>Fix passing parameters out in case of out parameters. 	modified:   include/reflection_manager.hpp<commit_after>#pragma once

#include <utility>
#include <stdexcept>
#include <vector>

#include <array_view.hpp>
#include "reflection_info.hpp"
#include "api_types.hpp"
#include "info_iterators.hpp"

namespace shadow
{

namespace reflection_initialization_detail
{
// used to handle cases where reflection_manager is initialized with nullptrs
// (ie empty) instead of arrays
template <class T>
struct array_selector
{
    static helene::array_view<std::remove_pointer_t<std::remove_const_t<T>>>
        initialize(T)
    {
        return helene::array_view<
            std::remove_pointer_t<std::remove_const_t<T>>>();
    }
};

template <class I, std::size_t N>
struct array_selector<I[N]>
{
    static helene::array_view<I> initialize(I (&arr)[N])
    {
        return helene::make_array_view(arr);
    }
};
} // namespace reflection_initialization_detail

// point of interaction with the reflection system
class reflection_manager
{
public:
    typedef info_iterator_<const type_info, type_tag> const_type_iterator;
    typedef indexed_info_iterator_<const type_info, type_tag>
        const_indexed_type_iterator;
    typedef info_iterator_<const constructor_info, constructor_tag>
        const_constructor_iterator;

    typedef info_iterator_<const conversion_info, conversion_tag>
        const_conversion_iterator;

    typedef info_iterator_<const free_function_info, free_function_tag>
        const_free_function_iterator;

public:
    template <class TypeInfoArray,
              class ConstructorInfoArray,
              class ConversionInfoArray,
              class FreeFunctionArray,
              class MemberFunctionArray,
              class MemberVariableArray,
              class StringSerializationArray>
    reflection_manager(TypeInfoArray& ti_arr,
                       ConstructorInfoArray& ci_arr,
                       ConversionInfoArray& cv_arr,
                       FreeFunctionArray& ff_arr,
                       MemberFunctionArray& mf_arr,
                       MemberVariableArray& mv_arr,
                       StringSerializationArray& ss_arr);

    // queries on types
    std::pair<const_type_iterator, const_type_iterator> types() const;

    std::string type_name(const type_tag& tag) const;

    std::size_t type_size(const type_tag& tag) const;

    // returns range of all constructors available
    std::pair<const_constructor_iterator, const_constructor_iterator>
    constructors() const;

    // returns the type that the given constructor belongs to
    type_tag constructor_type(const constructor_tag& tag) const;

    // returns range of the types of the parameters of the given constructor
    std::pair<const_indexed_type_iterator, const_indexed_type_iterator>
    constructor_parameter_types(const constructor_tag& tag) const;

    template <class Iterator>
    object construct_object(const constructor_tag& tag,
                            Iterator first,
                            Iterator last) const;

    // type conversion functions
    std::pair<const_conversion_iterator, const_conversion_iterator>
    conversions() const;

    std::pair<type_tag, type_tag>
    conversion_types(const conversion_tag& tag) const;

    object convert(const conversion_tag& tag, const object& val) const;

    // returns range of tags to all free functions available
    std::pair<const_free_function_iterator, const_free_function_iterator>
    free_functions() const;

    // returns name of function associated with the free_function_tag
    std::string free_function_name(const free_function_tag& tag) const;

    // returns return type of function associated with the free_function_tag
    type_tag free_function_return_type(const free_function_tag& tag) const;

    // return range of type tags of parameters of free function associated with
    // the tag
    std::pair<const_indexed_type_iterator, const_indexed_type_iterator>
    free_function_parameter_types(const free_function_tag& tag) const;

    template <class Iterator>
    object call_free_function(const free_function_tag& tag,
                              Iterator first,
                              Iterator last) const;

public:
    // unchecked operations
    template <class T>
    T& get(object& obj) const;

    template <class T>
    const T& get(const object& obj) const;

private:
    template <class Iterator, class InfoType>
    bool
    check_arguments(Iterator first, Iterator last, const InfoType& info) const;

    bool compare_type(const type_tag& tag, std::size_t index) const;

    template <class Iterator, class OutputIterator, class InfoType>
    void construct_argument_array(Iterator first,
                                  Iterator last,
                                  OutputIterator out,
                                  const InfoType& info) const;

    template <class Iterator, class OutputIterator, class InfoType>
    void pass_parameters_out(Iterator first,
                             Iterator last,
                             OutputIterator out,
                             const InfoType& info) const;

private:
    // array_views of reflection information generated at compile time
    helene::array_view<const type_info> type_info_view_;
    helene::array_view<const constructor_info> constructor_info_view_;
    helene::array_view<const conversion_info> conversion_info_view_;
    helene::array_view<const free_function_info> free_function_info_view_;
    helene::array_view<const member_function_info> member_function_info_view_;
    helene::array_view<const member_variable_info> member_variable_info_view_;
    helene::array_view<const string_serialization_info>
        string_serialization_info_view_;
};
} // namespace shadow

namespace shadow
{
template <class TypeInfoArray,
          class ConstructorInfoArray,
          class ConversionInfoArray,
          class FreeFunctionArray,
          class MemberFunctionArray,
          class MemberVariableArray,
          class StringSerializationArray>
inline reflection_manager::reflection_manager(TypeInfoArray& ti_arr,
                                              ConstructorInfoArray& ci_arr,
                                              ConversionInfoArray& cv_arr,
                                              FreeFunctionArray& ff_arr,
                                              MemberFunctionArray& mf_arr,
                                              MemberVariableArray& mv_arr,
                                              StringSerializationArray& ss_arr)
    : type_info_view_(reflection_initialization_detail::array_selector<
                      TypeInfoArray>::initialize(ti_arr)),
      constructor_info_view_(reflection_initialization_detail::array_selector<
                             ConstructorInfoArray>::initialize(ci_arr)),
      conversion_info_view_(reflection_initialization_detail::array_selector<
                            ConversionInfoArray>::initialize(cv_arr)),
      free_function_info_view_(reflection_initialization_detail::array_selector<
                               FreeFunctionArray>::initialize(ff_arr)),
      member_function_info_view_(
          reflection_initialization_detail::array_selector<
              MemberFunctionArray>::initialize(mf_arr)),
      member_variable_info_view_(
          reflection_initialization_detail::array_selector<
              MemberVariableArray>::initialize(mv_arr)),
      string_serialization_info_view_(
          reflection_initialization_detail::array_selector<
              StringSerializationArray>::initialize(ss_arr))
{
}

inline std::pair<typename reflection_manager::const_type_iterator,
                 typename reflection_manager::const_type_iterator>
reflection_manager::types() const
{
    return std::make_pair(const_type_iterator(type_info_view_.cbegin()),
                          const_type_iterator(type_info_view_.cend()));
}

inline std::string
reflection_manager::type_name(const type_tag& tag) const
{
    return tag.name();
}

inline std::size_t
reflection_manager::type_size(const type_tag& tag) const
{
    return tag.size();
}

inline std::pair<typename reflection_manager::const_constructor_iterator,
                 typename reflection_manager::const_constructor_iterator>
reflection_manager::constructors() const
{
    return std::make_pair(
        const_constructor_iterator(constructor_info_view_.cbegin()),
        const_constructor_iterator(constructor_info_view_.cend()));
}

inline type_tag
reflection_manager::constructor_type(const constructor_tag& tag) const
{
    return type_tag(type_info_view_[tag.info_ptr_->type_index]);
}

inline std::pair<reflection_manager::const_indexed_type_iterator,
                 reflection_manager::const_indexed_type_iterator>
reflection_manager::constructor_parameter_types(
    const constructor_tag& tag) const
{
    auto index_buffer = tag.info_ptr_->parameter_type_indices;
    auto data_buffer = type_info_view_.data();

    return std::make_pair(
        const_indexed_type_iterator(0, index_buffer, data_buffer),
        const_indexed_type_iterator(
            tag.info_ptr_->num_parameters, index_buffer, data_buffer));
}

inline bool
reflection_manager::compare_type(const type_tag& tag, std::size_t index) const
{
    type_tag tag_from_index(type_info_view_[index]);

    return tag == tag_from_index;
}

template <class Iterator, class InfoType>
inline bool
reflection_manager::check_arguments(Iterator first,
                                    Iterator last,
                                    const InfoType& info) const
{
    if(info.num_parameters != std::distance(first, last))
    {
        return false;
    }

    for(auto index_ptr = info.parameter_type_indices; first != last;
        ++index_ptr, ++first)
    {
        if(compare_type(first->type(), *index_ptr) == false)
        {
            return false;
        }
    }

    return true;
}

template <class Iterator, class OutputIterator, class InfoType>
inline void
reflection_manager::construct_argument_array(Iterator first,
                                             Iterator last,
                                             OutputIterator out,
                                             const InfoType& info) const
{
    for(auto pointer_flag_ptr = info.parameter_pointer_flags; first != last;
        ++pointer_flag_ptr, ++first, ++out)
    {
        if(*pointer_flag_ptr)
        {
            // construct any with pointer to value
            auto address_bind_point = first->type_info_->address_of_bind_point;

            out = address_bind_point(first->value_);
        }
        else
        {
            out = first->value_;
        }
    }
}

template <class Iterator, class OutputIterator, class InfoType>
inline void
reflection_manager::pass_parameters_out(Iterator first,
                                        Iterator last,
                                        OutputIterator out,
                                        const InfoType& info) const
{
    for(auto ptr_flags = info.parameter_pointer_flags; first != last;
        ++first, ++out, ++ptr_flags)
    {
        if(*ptr_flags)
        {
            auto dereference_binding = out->type_info_->dereference_bind_point;
            out->value_ = dereference_binding(*first);
        }
        else
        {
            out->value_ = *first;
        }
    }
}

template <class Iterator>
inline object
reflection_manager::construct_object(const constructor_tag& tag,
                                     Iterator first,
                                     Iterator last) const
{
    if(check_arguments(first, last, *tag.info_ptr_) == false)
    {
        throw std::runtime_error("wrong argument types");
    }

    std::vector<any> arg_vec;
    arg_vec.reserve(std::distance(first, last));

    construct_argument_array(
        first, last, std::back_inserter(arg_vec), *tag.info_ptr_);

    auto return_value = tag.info_ptr_->bind_point(arg_vec.data());

    return object(
        return_value, type_info_view_.data() + tag.info_ptr_->type_index, this);
}

inline std::pair<reflection_manager::const_conversion_iterator,
                 reflection_manager::const_conversion_iterator>
reflection_manager::conversions() const
{
    return std::make_pair(
        const_conversion_iterator(conversion_info_view_.cbegin()),
        const_conversion_iterator(conversion_info_view_.cend()));
}

inline std::pair<type_tag, type_tag>
reflection_manager::conversion_types(const conversion_tag& tag) const
{
    return std::make_pair(
        type_tag(type_info_view_[tag.info_ptr_->from_type_index]),
        type_tag(type_info_view_[tag.info_ptr_->to_type_index]));
}

inline object
reflection_manager::convert(const conversion_tag& tag, const object& val) const
{
    if(val.type() != type_tag(type_info_view_[tag.info_ptr_->from_type_index]))
    {
        throw std::runtime_error(
            "type of object doesn't match conversion binding");
    }

    return object(tag.info_ptr_->bind_point(val.value_),
                  type_info_view_.data() + tag.info_ptr_->to_type_index,
                  this);
}
template <class T>
inline T&
reflection_manager::get(object& obj) const
{
    return obj.value_.get<T>();
}

template <class T>
inline const T&
reflection_manager::get(const object& obj) const
{
    return obj.value_.get<T>();
}

inline std::pair<reflection_manager::const_free_function_iterator,
                 reflection_manager::const_free_function_iterator>
reflection_manager::free_functions() const
{
    return std::make_pair(
        const_free_function_iterator(free_function_info_view_.cbegin()),
        const_free_function_iterator(free_function_info_view_.cend()));
}

inline std::string
reflection_manager::free_function_name(const free_function_tag& tag) const
{
    return std::string(tag.info_ptr_->name);
}

inline type_tag
reflection_manager::free_function_return_type(
    const free_function_tag& tag) const
{
    return type_tag(type_info_view_[tag.info_ptr_->return_type_index]);
}

inline std::pair<reflection_manager::const_indexed_type_iterator,
                 reflection_manager::const_indexed_type_iterator>
reflection_manager::free_function_parameter_types(
    const free_function_tag& tag) const
{
    return std::make_pair(
        const_indexed_type_iterator(
            0, tag.info_ptr_->parameter_type_indices, type_info_view_.data()),
        const_indexed_type_iterator(tag.info_ptr_->num_parameters,
                                    tag.info_ptr_->parameter_type_indices,
                                    type_info_view_.data()));
}

template <class Iterator>
inline object
reflection_manager::call_free_function(const free_function_tag& tag,
                                       Iterator first,
                                       Iterator last) const
{
    if(!check_arguments(first, last, *tag.info_ptr_))
    {
        throw std::runtime_error(
            "attempting to call free function with arguments of wrong type");
    }

    std::vector<shadow::any> args;
    args.reserve(std::distance(first, last));

    construct_argument_array(
        first, last, std::back_inserter(args), *tag.info_ptr_);

    auto return_value = tag.info_ptr_->bind_point(args.data());

    pass_parameters_out(args.begin(), args.end(), first, *tag.info_ptr_);

    return object(return_value,
                  type_info_view_.data() + tag.info_ptr_->return_type_index,
                  this);
}
} // namespace shadow
<|endoftext|>


<commit_before>/*
 * type.hpp
 *
 *  Copyright (c) 2015 Masatoshi Hanai
 *
 *  This software is released under MIT License.
 *  See LICENSE.
 *
 */
#ifndef SCALESIM_UTIL_TYPE_HPP_
#define SCALESIM_UTIL_TYPE_HPP_

#include <vector>
#include <boost/unordered_set.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/unordered_map.hpp>

template<class App> using event = typename App::Event;
template<class App> using state = typename App::State;

template<class App> using ev_ptr = boost::shared_ptr<const event<App> >;
template<class App> using st_ptr = boost::shared_ptr<const state<App> >;

template<class App> using ev_vec = std::vector<ev_ptr<App> >;
template<class App> using st_vec = std::vector<st_ptr<App> >;

using parti_ptr = boost::shared_ptr<std::vector<long> >;
using parti_indx = boost::unordered_multimap<long, long>;
using parti_indx_ptr = boost::shared_ptr<parti_indx>;

#endif /* SCALESIM_UTIL_TYPE_HPP_ */
<commit_msg>Add comment<commit_after>/*
 * type.hpp
 *
 *  Copyright (c) 2015 Masatoshi Hanai
 *
 *  This software is released under MIT License.
 *  See LICENSE.
 *
 */
#ifndef SCALESIM_UTIL_TYPE_HPP_
#define SCALESIM_UTIL_TYPE_HPP_

#include <vector>
#include <boost/unordered_set.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/unordered_map.hpp>

template<class App> using event = typename App::Event;
template<class App> using state = typename App::State;

template<class App> using ev_ptr = boost::shared_ptr<const event<App> >;
template<class App> using st_ptr = boost::shared_ptr<const state<App> >;

template<class App> using ev_vec = std::vector<ev_ptr<App> >;
template<class App> using st_vec = std::vector<st_ptr<App> >;

/* partition is a table from id to partition number */
using parti_ptr = boost::shared_ptr<std::vector<long> >;
/* partition index is a table from partition to involving ids */
using parti_indx = boost::unordered_multimap<long, long>;
using parti_indx_ptr = boost::shared_ptr<parti_indx>;

#endif /* SCALESIM_UTIL_TYPE_HPP_ */
<|endoftext|>