<commit_before>// Copyright (c) 2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/test_file_util.h"

#include <windows.h>

#include <vector>

#include "base/file_util.h"
#include "base/scoped_handle.h"

namespace file_util {

// We could use GetSystemInfo to get the page size, but this serves
// our purpose fine since 4K is the page size on x86 as well as x64.
static const ptrdiff_t kPageSize = 4096;

bool EvictFileFromSystemCache(const wchar_t* file) {
  // Request exclusive access to the file and overwrite it with no buffering.
  ScopedHandle file_handle(
      CreateFile(file, GENERIC_READ | GENERIC_WRITE, 0, NULL,
                 OPEN_EXISTING, FILE_FLAG_NO_BUFFERING, NULL));
  if (!file_handle)
    return false;

  // Get some attributes to restore later.
  BY_HANDLE_FILE_INFORMATION bhi = {0};
  CHECK(::GetFileInformationByHandle(file_handle, &bhi));

  // Execute in chunks. It could be optimized. We want to do few of these since
  // these operations will be slow without the cache.

  // Non-buffered reads and writes need to be sector aligned and since sector
  // sizes typically range from 512-4096 bytes, we just use the page size.
  // The buffer size is twice the size of a page (minus one) since we need to
  // get an aligned pointer into the buffer that we can use.
  char buffer[2 * kPageSize - 1];
  // Get an aligned pointer into buffer.
  char* read_write = reinterpret_cast<char*>(
      reinterpret_cast<ptrdiff_t>(buffer + kPageSize - 1) & ~(kPageSize - 1));
  DCHECK((reinterpret_cast<int>(read_write) % kPageSize) == 0);

  // If the file size isn't a multiple of kPageSize, we'll need special
  // processing.
  bool file_is_page_aligned = true;
  int total_bytes = 0;
  DWORD bytes_read, bytes_written;
  for (;;) {
    bytes_read = 0;
    ReadFile(file_handle, read_write, kPageSize, &bytes_read, NULL);
    if (bytes_read == 0)
      break;

    if (bytes_read < kPageSize) {
      // Zero out the remaining part of the buffer.
      // WriteFile will fail if we provide a buffer size that isn't a
      // sector multiple, so we'll have to write the entire buffer with
      // padded zeros and then use SetEndOfFile to truncate the file.
      ZeroMemory(read_write + bytes_read, kPageSize - bytes_read);
      file_is_page_aligned = false;
    }

    // Move back to the position we just read from.
    // Note that SetFilePointer will also fail if total_bytes isn't sector
    // aligned, but that shouldn't happen here.
    DCHECK((total_bytes % kPageSize) == 0);
    SetFilePointer(file_handle, total_bytes, NULL, FILE_BEGIN);
    if (!WriteFile(file_handle, read_write, kPageSize, &bytes_written, NULL) ||
        bytes_written != kPageSize) {
      DCHECK(false);
      return false;
    }

    total_bytes += bytes_read;

    // If this is false, then we just processed the last portion of the file.
    if (!file_is_page_aligned)
      break;
  }

  if (!file_is_page_aligned) {
    // The size of the file isn't a multiple of the page size, so we'll have
    // to open the file again, this time without the FILE_FLAG_NO_BUFFERING
    // flag and use SetEndOfFile to mark EOF.
    file_handle.Set(CreateFile(file, GENERIC_WRITE, 0, NULL, OPEN_EXISTING,
                               0, NULL));
    CHECK(SetFilePointer(file_handle, total_bytes, NULL, FILE_BEGIN) !=
          INVALID_SET_FILE_POINTER);
    CHECK(::SetEndOfFile(file_handle));
  }

  // Restore the file attributes.
  CHECK(::SetFileTime(file_handle, &bhi.ftCreationTime, &bhi.ftLastAccessTime,
                      &bhi.ftLastWriteTime));

  return true;
}

// Like CopyFileNoCache but recursively copies all files and subdirectories
// in the given input directory to the output directory.
bool CopyRecursiveDirNoCache(const std::wstring& source_dir,
                             const std::wstring& dest_dir) {
  // Try to create the directory if it doesn't already exist.
  if (!CreateDirectory(dest_dir)) {
    if (GetLastError() != ERROR_ALREADY_EXISTS)
      return false;
  }

  std::vector<std::wstring> files_copied;

  std::wstring src(source_dir);
  file_util::AppendToPath(&src, L"*");

  WIN32_FIND_DATA fd;
  HANDLE fh = FindFirstFile(src.c_str(), &fd);
  if (fh == INVALID_HANDLE_VALUE)
    return false;

  do {
    std::wstring cur_file(fd.cFileName);
    if (cur_file == L"." || cur_file == L"..")
      continue;  // Skip these special entries.

    std::wstring cur_source_path(source_dir);
    file_util::AppendToPath(&cur_source_path, cur_file);

    std::wstring cur_dest_path(dest_dir);
    file_util::AppendToPath(&cur_dest_path, cur_file);

    if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
      // Recursively copy a subdirectory. We stripped "." and ".." already.
      if (!CopyRecursiveDirNoCache(cur_source_path, cur_dest_path)) {
        FindClose(fh);
        return false;
      }
    } else {
      // Copy the file.
      if (!::CopyFile(cur_source_path.c_str(), cur_dest_path.c_str(), false)) {
        FindClose(fh);
        return false;
      }

      // We don't check for errors from this function, often, we are copying
      // files that are in the repository, and they will have read-only set.
      // This will prevent us from evicting from the cache, but these don't
      // matter anyway.
      EvictFileFromSystemCache(cur_dest_path.c_str());
    }
  } while (FindNextFile(fh, &fd));

  FindClose(fh);
  return true;
}

}  // namespace file_util

<commit_msg>Fix CloseHandle blunder.  I was thinking that the handle would be closed by Set, but of course I need to close the handle before the CreateFile call... doh!<commit_after>// Copyright (c) 2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/test_file_util.h"

#include <windows.h>

#include <vector>

#include "base/file_util.h"
#include "base/scoped_handle.h"

namespace file_util {

// We could use GetSystemInfo to get the page size, but this serves
// our purpose fine since 4K is the page size on x86 as well as x64.
static const ptrdiff_t kPageSize = 4096;

bool EvictFileFromSystemCache(const wchar_t* file) {
  // Request exclusive access to the file and overwrite it with no buffering.
  ScopedHandle file_handle(
      CreateFile(file, GENERIC_READ | GENERIC_WRITE, 0, NULL,
                 OPEN_EXISTING, FILE_FLAG_NO_BUFFERING, NULL));
  if (!file_handle)
    return false;

  // Get some attributes to restore later.
  BY_HANDLE_FILE_INFORMATION bhi = {0};
  CHECK(::GetFileInformationByHandle(file_handle, &bhi));

  // Execute in chunks. It could be optimized. We want to do few of these since
  // these operations will be slow without the cache.

  // Non-buffered reads and writes need to be sector aligned and since sector
  // sizes typically range from 512-4096 bytes, we just use the page size.
  // The buffer size is twice the size of a page (minus one) since we need to
  // get an aligned pointer into the buffer that we can use.
  char buffer[2 * kPageSize - 1];
  // Get an aligned pointer into buffer.
  char* read_write = reinterpret_cast<char*>(
      reinterpret_cast<ptrdiff_t>(buffer + kPageSize - 1) & ~(kPageSize - 1));
  DCHECK((reinterpret_cast<int>(read_write) % kPageSize) == 0);

  // If the file size isn't a multiple of kPageSize, we'll need special
  // processing.
  bool file_is_page_aligned = true;
  int total_bytes = 0;
  DWORD bytes_read, bytes_written;
  for (;;) {
    bytes_read = 0;
    ReadFile(file_handle, read_write, kPageSize, &bytes_read, NULL);
    if (bytes_read == 0)
      break;

    if (bytes_read < kPageSize) {
      // Zero out the remaining part of the buffer.
      // WriteFile will fail if we provide a buffer size that isn't a
      // sector multiple, so we'll have to write the entire buffer with
      // padded zeros and then use SetEndOfFile to truncate the file.
      ZeroMemory(read_write + bytes_read, kPageSize - bytes_read);
      file_is_page_aligned = false;
    }

    // Move back to the position we just read from.
    // Note that SetFilePointer will also fail if total_bytes isn't sector
    // aligned, but that shouldn't happen here.
    DCHECK((total_bytes % kPageSize) == 0);
    SetFilePointer(file_handle, total_bytes, NULL, FILE_BEGIN);
    if (!WriteFile(file_handle, read_write, kPageSize, &bytes_written, NULL) ||
        bytes_written != kPageSize) {
      DCHECK(false);
      return false;
    }

    total_bytes += bytes_read;

    // If this is false, then we just processed the last portion of the file.
    if (!file_is_page_aligned)
      break;
  }

  if (!file_is_page_aligned) {
    // The size of the file isn't a multiple of the page size, so we'll have
    // to open the file again, this time without the FILE_FLAG_NO_BUFFERING
    // flag and use SetEndOfFile to mark EOF.
    file_handle.Set(NULL);
    file_handle.Set(CreateFile(file, GENERIC_WRITE, 0, NULL, OPEN_EXISTING,
                               0, NULL));
    CHECK(SetFilePointer(file_handle, total_bytes, NULL, FILE_BEGIN) !=
          INVALID_SET_FILE_POINTER);
    CHECK(::SetEndOfFile(file_handle));
  }

  // Restore the file attributes.
  CHECK(::SetFileTime(file_handle, &bhi.ftCreationTime, &bhi.ftLastAccessTime,
                      &bhi.ftLastWriteTime));

  return true;
}

// Like CopyFileNoCache but recursively copies all files and subdirectories
// in the given input directory to the output directory.
bool CopyRecursiveDirNoCache(const std::wstring& source_dir,
                             const std::wstring& dest_dir) {
  // Try to create the directory if it doesn't already exist.
  if (!CreateDirectory(dest_dir)) {
    if (GetLastError() != ERROR_ALREADY_EXISTS)
      return false;
  }

  std::vector<std::wstring> files_copied;

  std::wstring src(source_dir);
  file_util::AppendToPath(&src, L"*");

  WIN32_FIND_DATA fd;
  HANDLE fh = FindFirstFile(src.c_str(), &fd);
  if (fh == INVALID_HANDLE_VALUE)
    return false;

  do {
    std::wstring cur_file(fd.cFileName);
    if (cur_file == L"." || cur_file == L"..")
      continue;  // Skip these special entries.

    std::wstring cur_source_path(source_dir);
    file_util::AppendToPath(&cur_source_path, cur_file);

    std::wstring cur_dest_path(dest_dir);
    file_util::AppendToPath(&cur_dest_path, cur_file);

    if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
      // Recursively copy a subdirectory. We stripped "." and ".." already.
      if (!CopyRecursiveDirNoCache(cur_source_path, cur_dest_path)) {
        FindClose(fh);
        return false;
      }
    } else {
      // Copy the file.
      if (!::CopyFile(cur_source_path.c_str(), cur_dest_path.c_str(), false)) {
        FindClose(fh);
        return false;
      }

      // We don't check for errors from this function, often, we are copying
      // files that are in the repository, and they will have read-only set.
      // This will prevent us from evicting from the cache, but these don't
      // matter anyway.
      EvictFileFromSystemCache(cur_dest_path.c_str());
    }
  } while (FindNextFile(fh, &fd));

  FindClose(fh);
  return true;
}

}  // namespace file_util

<|endoftext|>


<commit_before>#ifndef _TOOLBOX_CPP_FORMAT_HPP_
#define _TOOLBOX_CPP_FORMAT_HPP_

#include <toolbox/container/stack.hpp>

namespace toolbox
{

namespace format {
namespace detail {
struct flags {
    bool hex = 0;
};
} // namespace detail
template<typename Output, typename T>
void format_argument(Output &out, detail::flags flags, T number)
{
    // TODO: may be removed when we fix stack size
    static_assert(sizeof(number) <= 4, "does not support integer types > 4 bytes");

    if (flags.hex == false) {
        // TODO: allocate stack based on max dec representation
        // of type T, for now we assume 32 bit integer which
        // may have up to 10 digits in dec representation
        toolbox::stack<char, 10> stack;

        while (number != 0) {
            char c = number % 10;
            stack.emplace_back('0' + c);
            number = number / 10;
        }

        while (stack.size()) {
            out.put(stack.back());
            stack.pop_back();
        }
    } else if (flags.hex == true) {
        static char lt[] = {
            '0', '1', '2', '3', '4',
            '5', '6', '7', '8', '9',
            'A', 'B', 'C', 'D', 'E',
            'F'
        };

        constexpr auto hexSize = sizeof(T) * 2;
        for (int i = 0; i < hexSize; i ++) {
            out.put(
                lt[(number >> (hexSize-1-i)*4) & 0xf]
            );
        }
    }
}

template<typename Output>
void print(Output &out, const char *fmt)
{
    while (*fmt) {
        out.put(*fmt);
        fmt ++;
    }
}

template<typename Output, typename Next, typename... Args>
void print(Output &out, const char *fmt, Next next, Args... args)
{
    bool format = false;
    detail::flags flags;

    while (*fmt) {
        if (format) {
            if (*fmt == 'x')
                flags.hex = true;

            if (*fmt == '}') {
                format_argument(out, flags, next);
                print(out, fmt + 1, args...);
                return;
            }
        } else {
            if (*fmt != '{') {
                out.put(*fmt);
            } else {
                format = true;
                flags = detail::flags{};
            }
        }

        fmt ++;
    }
}

} // namespace format
} // namespace toolbox

#endif // _TOOLBOX_CPP_FORMAT_HPP_<commit_msg>format: separate fmt string parsing from actual print function<commit_after>#ifndef _TOOLBOX_CPP_FORMAT_HPP_
#define _TOOLBOX_CPP_FORMAT_HPP_

#include <toolbox/container/stack.hpp>
#include <toolbox/span.hpp>

namespace toolbox
{

namespace format {
namespace detail {
struct flags {
    bool hex = 0;
};
} // namespace detail
template<typename Output, typename T>
void format(Output &out, detail::flags flags, T number)
{
    // TODO: may be removed when we fix stack size
    static_assert(sizeof(number) <= 4, "does not support integer types > 4 bytes");

    if (flags.hex == false) {
        // TODO: allocate stack based on max dec representation
        // of type T, for now we assume 32 bit integer which
        // may have up to 10 digits in dec representation
        toolbox::stack<char, 10> stack;

        while (number != 0) {
            char c = number % 10;
            stack.emplace_back('0' + c);
            number = number / 10;
        }

        while (stack.size()) {
            out.put(stack.back());
            stack.pop_back();
        }
    } else if (flags.hex == true) {
        static char lt[] = {
            '0', '1', '2', '3', '4',
            '5', '6', '7', '8', '9',
            'A', 'B', 'C', 'D', 'E',
            'F'
        };

        constexpr auto hexSize = sizeof(T) * 2;
        for (int i = 0; i < hexSize; i ++) {
            out.put(
                lt[(number >> (hexSize-1-i)*4) & 0xf]
            );
        }
    }
}

template<typename Output>
void print(Output &out, const char *fmt)
{
    while (*fmt) {
        out.put(*fmt);
        fmt ++;
    }
}

template<typename Output, typename Next>
void format_argument(Output &out, span<const char> fmt, const Next &next)
{
    detail::flags flags;

    for (auto c : fmt)
    {
        if (c == 'x') {
            flags.hex = true;
        }
    }

    format(out, flags, next);
}

template<typename Output, typename Next, typename... Args>
void print(Output &out, const char *fmt, Next next, Args... args)
{
    bool format = false;
    const char *fmtBegin = nullptr;
    detail::flags flags;

    while (*fmt) {
        if (format) {
            if (*fmt == '}') {
                format_argument(out, span<const char>{fmtBegin, fmt}, next);
                print(out, fmt + 1, args...);
                return;
            }
        } else {
            if (*fmt != '{') {
                out.put(*fmt);
            } else {
                format = true;
                fmtBegin = fmt + 1;
            }
        }

        fmt ++;
    }
}

} // namespace format
} // namespace toolbox

#endif // _TOOLBOX_CPP_FORMAT_HPP_
<|endoftext|>


<commit_before>#pragma once

#include <vector>

#include <memory>

#include <valijson/constraints/constraint.hpp>
#include <valijson/internal/optional.hpp>
#include <valijson/exceptions.hpp>

namespace valijson {

/**
 * Represents a sub-schema within a JSON Schema
 *
 * While all JSON Schemas have at least one sub-schema, the root, some will
 * have additional sub-schemas that are defined as part of constraints that are
 * included in the schema. For example, a 'oneOf' constraint maintains a set of
 * references to one or more nested sub-schemas. As per the definition of a
 * oneOf constraint, a document is valid within that constraint if it validates
 * against one of the nested sub-schemas.
 */
class Subschema
{
public:

    /// Typedef for custom new-/malloc-like function
    typedef void * (*CustomAlloc)(size_t size);

    /// Typedef for custom free-like function
    typedef void (*CustomFree)(void *);

    /// Typedef the Constraint class into the local namespace for convenience
    typedef constraints::Constraint Constraint;

    /// Typedef for a function that can be applied to each of the Constraint
    /// instances owned by a Schema.
    typedef std::function<bool (const Constraint &)> ApplyFunction;

    // Disable copy construction
    Subschema(const Subschema &) = delete;

    // Disable copy assignment
    Subschema & operator=(const Subschema &) = delete;

    /**
     * @brief  Construct a new Subschema object
     */
    Subschema()
      : m_allocFn(::operator new)
      , m_freeFn(::operator delete)
      , m_alwaysInvalid(false) { }

    /**
     * @brief  Construct a new Subschema using custom memory management
     *         functions
     *
     * @param  allocFn  malloc- or new-like function to allocate memory
     *                  within Schema, such as for Subschema instances
     * @param  freeFn   free-like function to free memory allocated with
     *                  the `customAlloc` function
     */
    Subschema(CustomAlloc allocFn, CustomFree freeFn)
      : m_allocFn(allocFn)
      , m_freeFn(freeFn)
      , m_alwaysInvalid(false)
    {
        // explicitly initialise optionals. See: https://github.com/tristanpenman/valijson/issues/124
        m_description = opt::nullopt;
        m_id = opt::nullopt;
        m_title = opt::nullopt;
    }

    /**
     * @brief  Clean up and free all memory managed by the Subschema
     */
    virtual ~Subschema()
    {
#if VALIJSON_USE_EXCEPTIONS
        try {
#endif
            for (auto constConstraint : m_constraints) {
                auto *constraint = const_cast<Constraint *>(constConstraint);
                constraint->~Constraint();
                m_freeFn(constraint);
            }
            m_constraints.clear();
#if VALIJSON_USE_EXCEPTIONS
        } catch (const std::exception &e) {
            fprintf(stderr, "Caught an exception in Subschema destructor: %s",
                    e.what());
        }
#endif
    }

    /**
     * @brief  Add a constraint to this sub-schema
     *
     * The constraint will be copied before being added to the list of
     * constraints for this Subschema. Note that constraints will be copied
     * only as deep as references to other Subschemas - e.g. copies of
     * constraints that refer to sub-schemas, will continue to refer to the
     * same Subschema instances.
     *
     * @param  constraint  Reference to the constraint to copy
     */
    void addConstraint(const Constraint &constraint)
    {
        Constraint *newConstraint = constraint.clone(m_allocFn, m_freeFn);
#if VALIJSON_USE_EXCEPTIONS
        try {
#endif
            m_constraints.push_back(newConstraint);
#if VALIJSON_USE_EXCEPTIONS
        } catch (...) {
            newConstraint->~Constraint();
            m_freeFn(newConstraint);
            throw;
        }
#endif
    }

    /**
     * @brief  Invoke a function on each child Constraint
     *
     * This function will apply the callback function to each constraint in
     * the Subschema, even if one of the invokations returns \c false. However,
     * if one or more invokations of the callback function return \c false,
     * this function will also return \c false.
     *
     * @returns  \c true if all invokations of the callback function are
     *           successful, \c false otherwise
     */
    bool apply(ApplyFunction &applyFunction) const
    {
        bool allTrue = true;
        for (const Constraint *constraint : m_constraints) {
            allTrue = allTrue && applyFunction(*constraint);
        }

        return allTrue;
    }

    /**
     * @brief  Invoke a function on each child Constraint
     *
     * This is a stricter version of the apply() function that will return
     * immediately if any of the invokations of the callback function return
     * \c false.
     *
     * @returns  \c true if all invokations of the callback function are
     *           successful, \c false otherwise
     */
    bool applyStrict(ApplyFunction &applyFunction) const
    {
        for (const Constraint *constraint : m_constraints) {
            if (!applyFunction(*constraint)) {
                return false;
            }
        }

        return true;
    }

    bool getAlwaysInvalid() const
    {
        return m_alwaysInvalid;
    }

    /**
     * @brief  Get the description associated with this sub-schema
     *
     * @throws  std::runtime_error if a description has not been set
     *
     * @returns  string containing sub-schema description
     */
    std::string getDescription() const
    {
        if (m_description) {
            return *m_description;
        }

        throwRuntimeError("Schema does not have a description");
    }

    /**
     * @brief  Get the ID associated with this sub-schema
     *
     * @throws  std::runtime_error if an ID has not been set
     *
     * @returns  string containing sub-schema ID
     */
    std::string getId() const
    {
        if (m_id) {
            return *m_id;
        }

        throwRuntimeError("Schema does not have an ID");
    }

    /**
     * @brief  Get the title associated with this sub-schema
     *
     * @throws  std::runtime_error if a title has not been set
     *
     * @returns  string containing sub-schema title
     */
    std::string getTitle() const
    {
        if (m_title) {
            return *m_title;
        }

        throwRuntimeError("Schema does not have a title");
    }

    /**
     * @brief  Check whether this sub-schema has a description
     *
     * @return boolean value
     */
    bool hasDescription() const
    {
        return static_cast<bool>(m_description);
    }

    /**
     * @brief  Check whether this sub-schema has an ID
     *
     * @return  boolean value
     */
    bool hasId() const
    {
        return static_cast<bool>(m_id);
    }

    /**
     * @brief  Check whether this sub-schema has a title
     *
     * @return  boolean value
     */
    bool hasTitle() const
    {
        return static_cast<bool>(m_title);
    }

    void setAlwaysInvalid(bool value)
    {
        m_alwaysInvalid = value;
    }

    /**
     * @brief  Set the description for this sub-schema
     *
     * The description will not be used for validation, but may be used as part
     * of the user interface for interacting with schemas and sub-schemas. As
     * an example, it may be used as part of the validation error descriptions
     * that are produced by the Validator and ValidationVisitor classes.
     *
     * @param  description  new description
     */
    void setDescription(const std::string &description)
    {
        m_description = description;
    }

    void setId(const std::string &id)
    {
        m_id = id;
    }

    /**
     * @brief  Set the title for this sub-schema
     *
     * The title will not be used for validation, but may be used as part
     * of the user interface for interacting with schemas and sub-schema. As an
     * example, it may be used as part of the validation error descriptions
     * that are produced by the Validator and ValidationVisitor classes.
     *
     * @param  title  new title
     */
    void setTitle(const std::string &title)
    {
        m_title = title;
    }

protected:

    CustomAlloc m_allocFn;

    CustomFree m_freeFn;

private:

    bool m_alwaysInvalid;

    /// List of pointers to constraints that apply to this schema.
    std::vector<const Constraint *> m_constraints;

    /// Schema description (optional)
    opt::optional<std::string> m_description;

    /// Id to apply when resolving the schema URI
    opt::optional<std::string> m_id;

    /// Title string associated with the schema (optional)
    opt::optional<std::string> m_title;
};

} // namespace valijson
<commit_msg>Always apply callback function when validating schema<commit_after>#pragma once

#include <vector>

#include <memory>

#include <valijson/constraints/constraint.hpp>
#include <valijson/internal/optional.hpp>
#include <valijson/exceptions.hpp>

namespace valijson {

/**
 * Represents a sub-schema within a JSON Schema
 *
 * While all JSON Schemas have at least one sub-schema, the root, some will
 * have additional sub-schemas that are defined as part of constraints that are
 * included in the schema. For example, a 'oneOf' constraint maintains a set of
 * references to one or more nested sub-schemas. As per the definition of a
 * oneOf constraint, a document is valid within that constraint if it validates
 * against one of the nested sub-schemas.
 */
class Subschema
{
public:

    /// Typedef for custom new-/malloc-like function
    typedef void * (*CustomAlloc)(size_t size);

    /// Typedef for custom free-like function
    typedef void (*CustomFree)(void *);

    /// Typedef the Constraint class into the local namespace for convenience
    typedef constraints::Constraint Constraint;

    /// Typedef for a function that can be applied to each of the Constraint
    /// instances owned by a Schema.
    typedef std::function<bool (const Constraint &)> ApplyFunction;

    // Disable copy construction
    Subschema(const Subschema &) = delete;

    // Disable copy assignment
    Subschema & operator=(const Subschema &) = delete;

    /**
     * @brief  Construct a new Subschema object
     */
    Subschema()
      : m_allocFn(::operator new)
      , m_freeFn(::operator delete)
      , m_alwaysInvalid(false) { }

    /**
     * @brief  Construct a new Subschema using custom memory management
     *         functions
     *
     * @param  allocFn  malloc- or new-like function to allocate memory
     *                  within Schema, such as for Subschema instances
     * @param  freeFn   free-like function to free memory allocated with
     *                  the `customAlloc` function
     */
    Subschema(CustomAlloc allocFn, CustomFree freeFn)
      : m_allocFn(allocFn)
      , m_freeFn(freeFn)
      , m_alwaysInvalid(false)
    {
        // explicitly initialise optionals. See: https://github.com/tristanpenman/valijson/issues/124
        m_description = opt::nullopt;
        m_id = opt::nullopt;
        m_title = opt::nullopt;
    }

    /**
     * @brief  Clean up and free all memory managed by the Subschema
     */
    virtual ~Subschema()
    {
#if VALIJSON_USE_EXCEPTIONS
        try {
#endif
            for (auto constConstraint : m_constraints) {
                auto *constraint = const_cast<Constraint *>(constConstraint);
                constraint->~Constraint();
                m_freeFn(constraint);
            }
            m_constraints.clear();
#if VALIJSON_USE_EXCEPTIONS
        } catch (const std::exception &e) {
            fprintf(stderr, "Caught an exception in Subschema destructor: %s",
                    e.what());
        }
#endif
    }

    /**
     * @brief  Add a constraint to this sub-schema
     *
     * The constraint will be copied before being added to the list of
     * constraints for this Subschema. Note that constraints will be copied
     * only as deep as references to other Subschemas - e.g. copies of
     * constraints that refer to sub-schemas, will continue to refer to the
     * same Subschema instances.
     *
     * @param  constraint  Reference to the constraint to copy
     */
    void addConstraint(const Constraint &constraint)
    {
        Constraint *newConstraint = constraint.clone(m_allocFn, m_freeFn);
#if VALIJSON_USE_EXCEPTIONS
        try {
#endif
            m_constraints.push_back(newConstraint);
#if VALIJSON_USE_EXCEPTIONS
        } catch (...) {
            newConstraint->~Constraint();
            m_freeFn(newConstraint);
            throw;
        }
#endif
    }

    /**
     * @brief  Invoke a function on each child Constraint
     *
     * This function will apply the callback function to each constraint in
     * the Subschema, even if one of the invocations returns \c false. However,
     * if one or more invocations of the callback function return \c false,
     * this function will also return \c false.
     *
     * @returns  \c true if all invocations of the callback function are
     *           successful, \c false otherwise
     */
    bool apply(ApplyFunction &applyFunction) const
    {
        bool allTrue = true;
        for (const Constraint *constraint : m_constraints) {
            allTrue = applyFunction(*constraint) && allTrue;
        }

        return allTrue;
    }

    /**
     * @brief  Invoke a function on each child Constraint
     *
     * This is a stricter version of the apply() function that will return
     * immediately if any of the invocations of the callback function return
     * \c false.
     *
     * @returns  \c true if all invocations of the callback function are
     *           successful, \c false otherwise
     */
    bool applyStrict(ApplyFunction &applyFunction) const
    {
        for (const Constraint *constraint : m_constraints) {
            if (!applyFunction(*constraint)) {
                return false;
            }
        }

        return true;
    }

    bool getAlwaysInvalid() const
    {
        return m_alwaysInvalid;
    }

    /**
     * @brief  Get the description associated with this sub-schema
     *
     * @throws  std::runtime_error if a description has not been set
     *
     * @returns  string containing sub-schema description
     */
    std::string getDescription() const
    {
        if (m_description) {
            return *m_description;
        }

        throwRuntimeError("Schema does not have a description");
    }

    /**
     * @brief  Get the ID associated with this sub-schema
     *
     * @throws  std::runtime_error if an ID has not been set
     *
     * @returns  string containing sub-schema ID
     */
    std::string getId() const
    {
        if (m_id) {
            return *m_id;
        }

        throwRuntimeError("Schema does not have an ID");
    }

    /**
     * @brief  Get the title associated with this sub-schema
     *
     * @throws  std::runtime_error if a title has not been set
     *
     * @returns  string containing sub-schema title
     */
    std::string getTitle() const
    {
        if (m_title) {
            return *m_title;
        }

        throwRuntimeError("Schema does not have a title");
    }

    /**
     * @brief  Check whether this sub-schema has a description
     *
     * @return boolean value
     */
    bool hasDescription() const
    {
        return static_cast<bool>(m_description);
    }

    /**
     * @brief  Check whether this sub-schema has an ID
     *
     * @return  boolean value
     */
    bool hasId() const
    {
        return static_cast<bool>(m_id);
    }

    /**
     * @brief  Check whether this sub-schema has a title
     *
     * @return  boolean value
     */
    bool hasTitle() const
    {
        return static_cast<bool>(m_title);
    }

    void setAlwaysInvalid(bool value)
    {
        m_alwaysInvalid = value;
    }

    /**
     * @brief  Set the description for this sub-schema
     *
     * The description will not be used for validation, but may be used as part
     * of the user interface for interacting with schemas and sub-schemas. As
     * an example, it may be used as part of the validation error descriptions
     * that are produced by the Validator and ValidationVisitor classes.
     *
     * @param  description  new description
     */
    void setDescription(const std::string &description)
    {
        m_description = description;
    }

    void setId(const std::string &id)
    {
        m_id = id;
    }

    /**
     * @brief  Set the title for this sub-schema
     *
     * The title will not be used for validation, but may be used as part
     * of the user interface for interacting with schemas and sub-schema. As an
     * example, it may be used as part of the validation error descriptions
     * that are produced by the Validator and ValidationVisitor classes.
     *
     * @param  title  new title
     */
    void setTitle(const std::string &title)
    {
        m_title = title;
    }

protected:

    CustomAlloc m_allocFn;

    CustomFree m_freeFn;

private:

    bool m_alwaysInvalid;

    /// List of pointers to constraints that apply to this schema.
    std::vector<const Constraint *> m_constraints;

    /// Schema description (optional)
    opt::optional<std::string> m_description;

    /// Id to apply when resolving the schema URI
    opt::optional<std::string> m_id;

    /// Title string associated with the schema (optional)
    opt::optional<std::string> m_title;
};

} // namespace valijson
<|endoftext|>


<commit_before>#ifndef TREEHASH_UTIL
#define TREEHASH_UTIL

#include <cstdint>
#include "immintrin.h"

typedef uint64_t ui128[2];

// Multiply two unsigned 64-bit ints, producing a 128-bit int but
// returning only the high-order bits.
static inline uint64_t mulHi(uint64_t x, uint64_t y) {
  uint64_t lo, hi;
  __asm__("mulq %3" : "=a,a"(lo), "=d,d"(hi) : "%0,0"(x), "r,m"(y));
  return hi;
}
// Universal hashing of x and y
static inline uint64_t deltaDietz(const ui128 h, const uint64_t x,
                                   const uint64_t y) {
  return x + y * h[1] + mulHi(y, h[0]);
}

static inline uint64_t bigendian(const ui128 h, const uint64_t x,
                                 const uint64_t y) {
  const uint64_t hlo = h[0] | 0x1;
  return h[1] * x + hlo * y + mulHi(hlo, x);
}

// Universal hashing 512 bits down to 256 bits using 128 bits of
// random data, with collision probability 2^-64.
//
// r128x2 is 128 bits of random data, duplicated.
//
// The method we use is one that relies on:
//
// 1. CLNH
//
// 2. The fact that if H is a almost delta-universal family, we can
// get an almost universal family by using one value from H and then
// performing an addition in the codomain. This has been noted in the
// Badger paper, as well as by Woelfel in "A construction method for
// optimally universal hash families and its consequences for the
// existence of RBIBDs.".
static inline __m256i clUniv512(const __m256i r128x2, const __m256i d0, const __m256i d1) {
  __m256i result = _mm256_xor_si256(r128x2, d0);

  // I think cast gets the lower bits of an __m256i
  __m128i x = _mm256_castsi256_si128(result);
  __m128i y = _mm256_extracti128_si256(result, 1);

  x = _mm_clmulepi64_si128(x, x, 1);
  y = _mm_clmulepi64_si128(y, y, 1);

  // I believe cast puts into the lower order bits of an __m256i
  result = _mm256_castsi128_si256(x);
  result = _mm256_inserti128_si256(result, y, 1);

  return _mm256_xor_si256(result, d1);
}

// In the interest of reducing code duplication, it is useful to
// parameterize treehashes by the primitive they use. By "primitive",
// I mean the building block that hashes R^2 down to R for some
// universe R. or instance, deltaDietz hashes uint64_t^2 down to
// uint64_t.
//
// We'll need a few ingredients:
struct MultiplyShift {
  // First, we need to know what R is in the notation above. This is
  // called an Atom, because from the point of view of the
  // parameterized hash algorithm, it cannot be broken up into
  // component parts.
  typedef uint64_t Atom;

 private:
  // Second, we need to know the type of the random data needed to
  // hash Atom^2 down to Atom.
  typedef ui128 Rand;

 public:
  // Third, we need to know the size of Atom in bytes. The sizeof
  // keyword won't always be enough is clients of this class, since
  // sometimes Atom is an array and "decays" into pointers.
  const static size_t ATOM_SIZE = sizeof(uint64_t);
  // Fourth, we need a way to copy Atoms.
  inline static void AtomCopy(Atom *x, const Atom &y) { *x = y; }
  // Fifth, we need to expose the hash family itself.
  inline void Hash(Atom *out, int i, const Atom &in0, const Atom &in1) const {
    *out = deltaDietz(r[i], in0, in1);
  }

  // Sixth, we need to store a particular array of random values. We
  // have to store these because some primitives need to preprocess
  // the random bits into an array of Rands. There is an example
  // below, but an easy one to think of is hashing modulo a prime. For
  // those hash families, we get data in, say, 8-bit chunks, but we
  // need random values modulo p, so we have to pre-process.
  //
  // We will need one Rand for each level of the tree hash. When we
  // pre-process, these values may require extra storage space. Since
  // that depends on the primitive, we keep the data in this struct
  // itself.
 private:
  const Rand *r;

 public:
  // In a primitive that needs preprocessing, depth indicates how many
  // random items to take - it is the depth of the treehash. We take
  // as an argument a const void ** that we update to point to random
  // data that this part of the tree hash will not need, so that it is
  // fresh when it is needed at the end of the tree hash.
  explicit MultiplyShift(const void **rvoid, const size_t depth)
      : r(reinterpret_cast<const Rand *>(*rvoid)) {
    *rvoid = reinterpret_cast<const void *>(r + depth);
  }
};

struct NH {
  typedef ui128 Atom;

 private:
  typedef ui128 Rand;

 public:
  const static size_t ATOM_SIZE = sizeof(ui128);
  inline static void AtomCopy(Atom *x, const Atom &y) {
    (*x)[0] = y[0];
    (*x)[1] = y[1];
  }
  inline void Hash(Atom *out, const int i, const Atom &in0,
                   const Atom &in1) const {
    // Use a temporary Atom in case out == &in0 or out == &in1
    const Atom tmp = {in0[0] + r[i][0], in0[1] + r[i][1]};
    const unsigned __int128 tmp2 =
        *reinterpret_cast<const unsigned __int128 *>(in1) +
        (((unsigned __int128)(tmp[0])) * ((unsigned __int128)(tmp[1])));
    memcpy(out, &tmp2, sizeof(tmp2));
  }

  explicit NH(const void **rvoid, const size_t depth)
      : r(reinterpret_cast<const Rand *>(rvoid)) {
    *rvoid = reinterpret_cast<const void *>(r + depth);
  }

 private:
  const Rand *r;
};

struct CLNH {
  typedef __m128i Atom;

 private:
  typedef __m128i Rand;

 public:
  const static size_t ATOM_SIZE = sizeof(__m128i);
  inline static void AtomCopy(Atom *x, const Atom &y) { *x = y; }
  inline void Hash(Atom *out, const int i, const Atom &in0,
                   const Atom &in1) const {
    Atom tmp = _mm_xor_si128(in0, r[i]);
    tmp = _mm_clmulepi64_si128(tmp, tmp, 1);
    tmp = _mm_xor_si128(tmp, in1);
    *out = tmp;
  }

  explicit CLNH(const void **rvoid, const size_t depth)
      : r(reinterpret_cast<const Rand *>(rvoid)) {
    *rvoid = reinterpret_cast<const void *>(r + depth);
  }

 private:
  const Rand *r;
};

struct CLNHx2 {
  typedef __m256i Atom;

  const static size_t ATOM_SIZE = sizeof(__m256i);

  inline static void AtomCopy(Atom *x, const Atom &y) { *x = y; }

  explicit CLNHx2(const void **rvoid, const size_t depth) {
    const __m128i *const r128 = reinterpret_cast<const __m128i *>(*rvoid);
    for (size_t i = 0; i < depth; ++i) {
      r[i] = _mm256_broadcastsi128_si256(r128[i]);
    }
    *rvoid = reinterpret_cast<const void *>(r128 + depth);
  }

  inline void Hash(Atom *out, const int i, const Atom &in0,
                   const Atom &in1) const {
    *out = clUniv512(r[i], in0, in1);
  }

 private:
  typedef __m256i Rand;
  Rand r[62];
};

// This primitive just wraps another primitive, but operates on arrays
// of a given static size.
template <typename T, size_t n>
struct Wide {
  typedef typename T::Atom Atom[n];

  const static size_t ATOM_SIZE = T::ATOM_SIZE * n;

  inline static void AtomCopy(Atom *x, const Atom &y) {
    for (size_t i = 0; i < n; ++i) {
      T::AtomCopy(&(*x)[i], y[i]);
    }
  }

  explicit Wide(const void **rvoid, const size_t depth) : t(rvoid, depth) {}

  inline void Hash(Atom *out, const int j, const Atom &in0,
                   const Atom &in1) const {
    for (size_t i = 0; i < n; ++i) {
      t.Hash(&(*out)[i], j, in0[i], in1[i]);
    }
  }

 private:
  T t;
};

#endif
<commit_msg>BUG: was casting from void** to Rand*, needed to be void* to Rand*<commit_after>#ifndef TREEHASH_UTIL
#define TREEHASH_UTIL

#include <cstdint>
#include "immintrin.h"

typedef uint64_t ui128[2];

// Multiply two unsigned 64-bit ints, producing a 128-bit int but
// returning only the high-order bits.
static inline uint64_t mulHi(uint64_t x, uint64_t y) {
  uint64_t lo, hi;
  __asm__("mulq %3" : "=a,a"(lo), "=d,d"(hi) : "%0,0"(x), "r,m"(y));
  return hi;
}
// Universal hashing of x and y
static inline uint64_t deltaDietz(const ui128 h, const uint64_t x,
                                   const uint64_t y) {
  return x + y * h[1] + mulHi(y, h[0]);
}

static inline uint64_t bigendian(const ui128 h, const uint64_t x,
                                 const uint64_t y) {
  const uint64_t hlo = h[0] | 0x1;
  return h[1] * x + hlo * y + mulHi(hlo, x);
}

// Universal hashing 512 bits down to 256 bits using 128 bits of
// random data, with collision probability 2^-64.
//
// r128x2 is 128 bits of random data, duplicated.
//
// The method we use is one that relies on:
//
// 1. CLNH
//
// 2. The fact that if H is a almost delta-universal family, we can
// get an almost universal family by using one value from H and then
// performing an addition in the codomain. This has been noted in the
// Badger paper, as well as by Woelfel in "A construction method for
// optimally universal hash families and its consequences for the
// existence of RBIBDs.".
static inline __m256i clUniv512(const __m256i r128x2, const __m256i d0, const __m256i d1) {
  __m256i result = _mm256_xor_si256(r128x2, d0);

  // I think cast gets the lower bits of an __m256i
  __m128i x = _mm256_castsi256_si128(result);
  __m128i y = _mm256_extracti128_si256(result, 1);

  x = _mm_clmulepi64_si128(x, x, 1);
  y = _mm_clmulepi64_si128(y, y, 1);

  // I believe cast puts into the lower order bits of an __m256i
  result = _mm256_castsi128_si256(x);
  result = _mm256_inserti128_si256(result, y, 1);

  return _mm256_xor_si256(result, d1);
}

// In the interest of reducing code duplication, it is useful to
// parameterize treehashes by the primitive they use. By "primitive",
// I mean the building block that hashes R^2 down to R for some
// universe R. or instance, deltaDietz hashes uint64_t^2 down to
// uint64_t.
//
// We'll need a few ingredients:
struct MultiplyShift {
  // First, we need to know what R is in the notation above. This is
  // called an Atom, because from the point of view of the
  // parameterized hash algorithm, it cannot be broken up into
  // component parts.
  typedef uint64_t Atom;

 private:
  // Second, we need to know the type of the random data needed to
  // hash Atom^2 down to Atom.
  typedef ui128 Rand;

 public:
  // Third, we need to know the size of Atom in bytes. The sizeof
  // keyword won't always be enough is clients of this class, since
  // sometimes Atom is an array and "decays" into pointers.
  const static size_t ATOM_SIZE = sizeof(uint64_t);
  // Fourth, we need a way to copy Atoms.
  inline static void AtomCopy(Atom *x, const Atom &y) { *x = y; }
  // Fifth, we need to expose the hash family itself.
  inline void Hash(Atom *out, int i, const Atom &in0, const Atom &in1) const {
    *out = deltaDietz(r[i], in0, in1);
  }

  // Sixth, we need to store a particular array of random values. We
  // have to store these because some primitives need to preprocess
  // the random bits into an array of Rands. There is an example
  // below, but an easy one to think of is hashing modulo a prime. For
  // those hash families, we get data in, say, 8-bit chunks, but we
  // need random values modulo p, so we have to pre-process.
  //
  // We will need one Rand for each level of the tree hash. When we
  // pre-process, these values may require extra storage space. Since
  // that depends on the primitive, we keep the data in this struct
  // itself.
 private:
  const Rand *r;

 public:
  // In a primitive that needs preprocessing, depth indicates how many
  // random items to take - it is the depth of the treehash. We take
  // as an argument a const void ** that we update to point to random
  // data that this part of the tree hash will not need, so that it is
  // fresh when it is needed at the end of the tree hash.
  explicit MultiplyShift(const void **rvoid, const size_t depth)
      : r(reinterpret_cast<const Rand *>(*rvoid)) {
    *rvoid = reinterpret_cast<const void *>(r + depth);
  }
};

struct NH {
  typedef ui128 Atom;

 private:
  typedef ui128 Rand;

 public:
  const static size_t ATOM_SIZE = sizeof(ui128);
  inline static void AtomCopy(Atom *x, const Atom &y) {
    (*x)[0] = y[0];
    (*x)[1] = y[1];
  }
  inline void Hash(Atom *out, const int i, const Atom &in0,
                   const Atom &in1) const {
    // Use a temporary Atom in case out == &in0 or out == &in1
    const Atom tmp = {in0[0] + r[i][0], in0[1] + r[i][1]};
    const unsigned __int128 tmp2 =
        *reinterpret_cast<const unsigned __int128 *>(in1) +
        (((unsigned __int128)(tmp[0])) * ((unsigned __int128)(tmp[1])));
    memcpy(out, &tmp2, sizeof(tmp2));
  }

  explicit NH(const void **rvoid, const size_t depth)
      : r(reinterpret_cast<const Rand *>(*rvoid)) {
    *rvoid = reinterpret_cast<const void *>(r + depth);
  }

 private:
  const Rand *r;
};

struct CLNH {
  typedef __m128i Atom;

 private:
  typedef __m128i Rand;

 public:
  const static size_t ATOM_SIZE = sizeof(__m128i);
  inline static void AtomCopy(Atom *x, const Atom &y) { *x = y; }
  inline void Hash(Atom *out, const int i, const Atom &in0,
                   const Atom &in1) const {
    Atom tmp = _mm_xor_si128(in0, r[i]);
    tmp = _mm_clmulepi64_si128(tmp, tmp, 1);
    tmp = _mm_xor_si128(tmp, in1);
    *out = tmp;
  }

  explicit CLNH(const void **rvoid, const size_t depth)
      : r(reinterpret_cast<const Rand *>(*rvoid)) {
    *rvoid = reinterpret_cast<const void *>(r + depth);
  }

 private:
  const Rand *r;
};

struct CLNHx2 {
  typedef __m256i Atom;

  const static size_t ATOM_SIZE = sizeof(__m256i);

  inline static void AtomCopy(Atom *x, const Atom &y) { *x = y; }

  explicit CLNHx2(const void **rvoid, const size_t depth) {
    const __m128i *const r128 = reinterpret_cast<const __m128i *>(*rvoid);
    for (size_t i = 0; i < depth; ++i) {
      r[i] = _mm256_broadcastsi128_si256(r128[i]);
    }
    *rvoid = reinterpret_cast<const void *>(r128 + depth);
  }

  inline void Hash(Atom *out, const int i, const Atom &in0,
                   const Atom &in1) const {
    *out = clUniv512(r[i], in0, in1);
  }

 private:
  typedef __m256i Rand;
  Rand r[62];
};

// This primitive just wraps another primitive, but operates on arrays
// of a given static size.
template <typename T, size_t n>
struct Wide {
  typedef typename T::Atom Atom[n];

  const static size_t ATOM_SIZE = T::ATOM_SIZE * n;

  inline static void AtomCopy(Atom *x, const Atom &y) {
    for (size_t i = 0; i < n; ++i) {
      T::AtomCopy(&(*x)[i], y[i]);
    }
  }

  explicit Wide(const void **rvoid, const size_t depth) : t(rvoid, depth) {}

  inline void Hash(Atom *out, const int j, const Atom &in0,
                   const Atom &in1) const {
    for (size_t i = 0; i < n; ++i) {
      t.Hash(&(*out)[i], j, in0[i], in1[i]);
    }
  }

 private:
  T t;
};

#endif
<|endoftext|>


<commit_before>#include <gtest/gtest.h>
#include <teraranger_hub/tr_hub_parser.h>

class HubParserTest : public ::testing::Test{
protected:
  uint8_t input_buffer[BUFFER_SIZE] = {0x54,0x48,0x08,0xa7,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x0b};//expected result is one 2.22 meter measurement
};

TEST_F(HubParserTest, crc8Test){
  int16_t crc = tr_hub_parser::crc8(input_buffer, 18);
  EXPECT_EQ(crc, input_buffer[18]);
}

TEST_F(HubParserTest, parsingTest){
  float result = tr_hub_parser::two_chars_to_float(input_buffer[2],input_buffer[3]);
  ASSERT_FLOAT_EQ(result, 2215);
}

int main(int argc, char **argv) {
  testing::InitGoogleTest(&argc, argv);

  return RUN_ALL_TESTS();
}
<commit_msg>Fix test_tr_hub_parser<commit_after>#include <gtest/gtest.h>
#include <teraranger_hub/teraranger_one.h>

class HubParserTest : public ::testing::Test{
protected:
  uint8_t input_buffer[BUFFER_SIZE] = {0x54,0x48,0x08,0xa7,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x0b};//expected result is one 2.22 meter measurement
};

TEST_F(HubParserTest, crc8Test){
  int16_t crc = tr_hub_parser::crc8(input_buffer, 18);
  EXPECT_EQ(crc, input_buffer[18]);
}

TEST_F(HubParserTest, parsingTest){
  float result = tr_hub_parser::two_chars_to_float(input_buffer[2],input_buffer[3]);
  ASSERT_FLOAT_EQ(result, 2215);
}

int main(int argc, char **argv) {
  testing::InitGoogleTest(&argc, argv);

  return RUN_ALL_TESTS();
}
<|endoftext|>


<commit_before>#ifndef VSMC_CORE_PARTICLE_HPP
#define VSMC_CORE_PARTICLE_HPP

#include <vsmc/internal/common.hpp>
#include <vsmc/core/single_particle.hpp>
#include <vsmc/core/weight.hpp>
#include <vsmc/resample/basic.hpp>
#include <vsmc/utility/rng_set.hpp>
#include <vsmc/utility/seed.hpp>
#include <vsmc/utility/stop_watch.hpp>

namespace vsmc {

/// \brief Particle class representing the whole particle set
/// \ingroup Core
template <typename T>
class Particle
{
    public :

    typedef typename traits::SizeTypeTrait<T>::type size_type;
    typedef T value_type;
    typedef typename traits::WeightSetTypeTrait<T>::type weight_set_type;
    typedef typename traits::RngSetTypeTrait<T>::type rng_set_type;
    typedef typename rng_set_type::rng_type rng_type;
    typedef cxx11::function<
        void (std::size_t, typename traits::ResampleRngTypeTrait<T>::type &,
                double *, size_type *)>
        resample_type;
    typedef ParticleIterator<T, SingleParticle> iterator;
    typedef ParticleIterator<T, ConstSingleParticle> const_iterator;
    typedef std::reverse_iterator<iterator> reverse_iterator;
    typedef std::reverse_iterator<const_iterator> const_reverse_iterator;

    explicit Particle (size_type N) :
        size_(N), value_(N),
        weight_set_(static_cast<typename
                traits::SizeTypeTrait<weight_set_type>::type>(N)),
        rng_set_(static_cast<typename
                traits::SizeTypeTrait<rng_set_type>::type>(N)),
        resample_replication_(N), resample_copy_from_(N), resample_weight_(N),
        resample_rng_(Seed::instance().get()),
        sp_(N + 2, SingleParticle<T>(0, VSMC_NULLPTR)),
        csp_(N + 2, ConstSingleParticle<T>(0, VSMC_NULLPTR)),
        start_watch_(false)
    {
        weight_set_.set_equal_weight();
        sp_[0] = SingleParticle<T>(std::numeric_limits<size_type>::max
                VSMC_MACRO_NO_EXPANSION (), this);
        csp_[0] = ConstSingleParticle<T>(std::numeric_limits<size_type>::max
                VSMC_MACRO_NO_EXPANSION (), this);
        for (size_type i = 1; i != size_ + 2; ++i) {
            sp_[i] = SingleParticle<T>(i - 1, this);
            csp_[i] = ConstSingleParticle<T>(i - 1, this);
        }
    }

    /// \brief Number of particles
    size_type size () const {return size_;}

    const SingleParticle<T> &sp (size_type i) {return sp_[i + 1];}

    const ConstSingleParticle<T> &sp (size_type i) const {return csp_[i + 1];}

    iterator begin () {return iterator(&sp(0));}

    iterator end () {return iterator(&sp(size_));}

    const_iterator begin () const {return cosnt_iterator(&sp(0));}

    const_iterator end () const {return const_iterator(&sp(size_));}

    const_iterator cbegin () const {return const_iterator(&sp(0));}

    const_iterator cend () const {return const_iterator(&sp(size_));}

    reverse_iterator rbegin () {return reverse_iterator(end());}

    reverse_iterator rend () {return reverse_iterator(begin());}

    const_reverse_iterator rbegin () const
    {return const_reverse_iterator(end());}

    const_reverse_iterator rend () const
    {return const_reverse_iterator(begin());}

    const_reverse_iterator crbegin () const
    {return const_reverse_iterator(cend());}

    const_reverse_iterator crend () const
    {return const_reverse_iterator(cbegin());}

    /// \brief Read and write access to the value collection object
    value_type &value () {return value_;}

    /// \brief Read only access to the value collection object
    const value_type &value () const {return value_;}

    /// \brief Read and write access to the weight collection object
    weight_set_type &weight_set () {return weight_set_;}

    /// \brief Read only access to the weight collection object
    const weight_set_type &weight_set () const {return weight_set_;}

    /// \brief Read and write access to the RNG collection object
    rng_set_type &rng_set () {return rng_set_;}

    /// \brief Read only access to the RNG collection object
    const rng_set_type &rng_set () const {return rng_set_;}

    /// \brief Set normalized weight, unnormalized logarithm weight and ESS
    /// such that each particle has a equal weight
    void set_equal_weight () {weight_set_.set_equal_weight();}

    /// \brief Read normalized weights through an output iterator
    template <typename OutputIter>
    OutputIter read_weight (OutputIter first) const
    {return weight_set_.read_weight(first);}

    /// \brief Get the ESS of the particle collection based on the current
    /// weights
    double ess () const {return weight_set_.ess();}

    /// \brief Get an RNG stream for a given particle
    rng_type &rng (size_type id) {return rng_set_.rng(id);}

    /// \brief Performing resampling if ESS/N < threshold
    ///
    /// \param op The resampling operation funcitor
    /// \param threshold The threshold of ESS/N below which resampling will be
    /// performed
    ///
    /// \return true if resampling was performed
    ///
    /// \details
    /// The out of box behavior are suitable for most applications. However,
    /// the user can also gain full control over the internal of this
    /// function by changing the `weight_set_type` and other type traits for
    /// specific value collection type. The sequence of operations of this
    /// function is listed as in the following pseudo-code
    /// 1. Determine if resampling is required
    ///     * Set `N = weight_set.resample_size()`
    ///     * Set `resampled = weight_set.ess() < threshold * N`
    ///     * If `!resampled`, GO TO Step 8. Otherwise, GO TO Step 2
    /// 2. Determine if resampling algorithms shall be performed. This is only
    /// useful for MPI implementations. the boolean value `resampled` ensures
    /// that collective actions like `copy` will be called by all nodes.
    /// However, the resampling replication numbers can be computed by one
    /// node instead of by all. Therefore, the program can actually collect and
    /// read the resampling weights on one node, and do nothing on other nodes.
    /// This difference shall be reflected by the returning iterator of
    /// `read_resample_weight`
    ///     * (Allocate `double *weight` for size `N`)
    ///     * Set `double *end = weight_set.read_resample_weight(weight)`
    ///     * If `end != weight + N`, GO TO step 5. Otherwise, GO TO Step 3
    /// 3. Performing resampling, produce replication number of each particle
    ///     * (Allocate `size_type *replication` for size `N`)
    ///     * (Create RNG engine `resample_rng` according to
    ///     traits::ResampleRngTypeTrait)
    ///     * Call `op(N, resample_rng, weight, replication)`
    /// 4. Transform replication numbers into parent particle indices
    ///     * (Allocat `size_type *copy_from` for size `N`)
    ///     * (Create functor `r2c` according to
    ///     traits::ResampleCopyFromReplicationTypeTrait)
    ///     * Call `r2c(N, replication, copy_from)`
    /// 5. Set `const size_type *cptr` according to results of Step 2.
    ///     * If Step 3 and 4 are skipped according to Step 2, then set
    ///     `cptr = 0`. Otherwise,
    ///     * Set `cptr = copy_from`
    /// 6. Performing copying of particles
    ///     * Call `value.copy(N, cptr)`
    /// 7. Performing post resampling weight manipulation
    ///     * (Create functor `post` according to
    ///     traits::ResamplePostCopyTypeTrait)
    ///     * `post(weight_set)`
    /// 8. `return resampled`
    bool resample (const resample_type &op, double threshold)
    {
        size_type N = static_cast<size_type>(weight_set_.resample_size());
        bool resampled = weight_set_.ess() < threshold * N;
        if (resampled) {
            resample_copy_from_.resize(N);
            resample_replication_.resize(N);
            resample_weight_.resize(N);
            const size_type *cptr = VSMC_NULLPTR;
            double *end = VSMC_NULLPTR;
            {
                ScopedStopWatch<StopWatch> start(
                        resample_read_weight_watch_, start_watch_);
                end = weight_set_.read_resample_weight(&resample_weight_[0]);
            }
            if (end == &resample_weight_[0] + N) {
                {
                    ScopedStopWatch<StopWatch> start(
                            resample_get_replication_watch_, start_watch_);
                    op(static_cast<std::size_t>(N), resample_rng_,
                            &resample_weight_[0], &resample_replication_[0]);
                }
                {
                    ScopedStopWatch<StopWatch> start(
                            resample_get_copy_from_watch_, start_watch_);
                    resample_copy_from_replication_(N,
                            &resample_replication_[0],
                            &resample_copy_from_[0]);
                }
                cptr = &resample_copy_from_[0];
            }
            {
                ScopedStopWatch<StopWatch> start(
                        resample_copy_watch_, start_watch_);
                value_.copy(N, cptr);
            }
            {
                ScopedStopWatch<StopWatch> start(
                        resample_post_copy_watch_, start_watch_);
                resample_post_copy_(weight_set_);
            }
        }

        return resampled;
    }

    void start_watch () {start_watch_ = true;}

    void stop_watch () {start_watch_ = false;}

    void reset_watch ()
    {
        resample_read_weight_watch_.reset();
        resample_get_replication_watch_.reset();
        resample_get_copy_from_watch_.reset();
        resample_copy_watch_.reset();
        resample_post_copy_watch_.reset();
    }

    StopWatch &resample_read_weight_watch ()
    {return resample_read_weight_watch_;}

    const StopWatch &resample_read_weight_watch () const
    {return resample_read_weight_watch_;}

    StopWatch &resample_get_replication_watch ()
    {return resample_get_replication_watch_;}

    const StopWatch &resample_get_replication_watch () const
    {return resample_get_replication_watch_;}

    StopWatch &resample_get_copy_from_watch ()
    {return resample_get_copy_from_watch_;}

    const StopWatch &resample_get_copy_from_watch () const
    {return resample_get_copy_from_watch_;}

    StopWatch &resample_copy_watch ()
    {return resample_copy_watch_;}

    const StopWatch &resample_copy_watch () const
    {return resample_copy_watch_;}

    StopWatch &resample_post_copy_watch ()
    {return resample_post_copy_watch_;}

    const StopWatch &resample_post_copy_watch () const
    {return resample_post_copy_watch_;}

    private :

    size_type size_;
    value_type value_;
    weight_set_type weight_set_;
    rng_set_type rng_set_;

    std::vector<size_type> resample_replication_;
    std::vector<size_type> resample_copy_from_;
    std::vector<double> resample_weight_;
    typename traits::ResampleCopyFromReplicationTypeTrait<T>::type
        resample_copy_from_replication_;
    typename traits::ResamplePostCopyTypeTrait<T>::type resample_post_copy_;
    typename traits::ResampleRngTypeTrait<T>::type resample_rng_;
    std::vector<SingleParticle<T> > sp_;
    std::vector<ConstSingleParticle<T> > csp_;

    bool start_watch_;
    StopWatch resample_read_weight_watch_;
    StopWatch resample_get_replication_watch_;
    StopWatch resample_get_copy_from_watch_;
    StopWatch resample_copy_watch_;
    StopWatch resample_post_copy_watch_;
}; // class Particle

} // namespace vsmc

#endif // VSMC_CORE_PARTICLE_HPP
<commit_msg>match Particle::resample_type decl with impl<commit_after>#ifndef VSMC_CORE_PARTICLE_HPP
#define VSMC_CORE_PARTICLE_HPP

#include <vsmc/internal/common.hpp>
#include <vsmc/core/single_particle.hpp>
#include <vsmc/core/weight.hpp>
#include <vsmc/resample/basic.hpp>
#include <vsmc/utility/rng_set.hpp>
#include <vsmc/utility/seed.hpp>
#include <vsmc/utility/stop_watch.hpp>

namespace vsmc {

/// \brief Particle class representing the whole particle set
/// \ingroup Core
template <typename T>
class Particle
{
    public :

    typedef typename traits::SizeTypeTrait<T>::type size_type;
    typedef T value_type;
    typedef typename traits::WeightSetTypeTrait<T>::type weight_set_type;
    typedef typename traits::RngSetTypeTrait<T>::type rng_set_type;
    typedef typename rng_set_type::rng_type rng_type;
    typedef cxx11::function<
        void (std::size_t, typename traits::ResampleRngTypeTrait<T>::type &,
                const double *, size_type *)>
        resample_type;
    typedef ParticleIterator<T, SingleParticle> iterator;
    typedef ParticleIterator<T, ConstSingleParticle> const_iterator;
    typedef std::reverse_iterator<iterator> reverse_iterator;
    typedef std::reverse_iterator<const_iterator> const_reverse_iterator;

    explicit Particle (size_type N) :
        size_(N), value_(N),
        weight_set_(static_cast<typename
                traits::SizeTypeTrait<weight_set_type>::type>(N)),
        rng_set_(static_cast<typename
                traits::SizeTypeTrait<rng_set_type>::type>(N)),
        resample_replication_(N), resample_copy_from_(N), resample_weight_(N),
        resample_rng_(Seed::instance().get()),
        sp_(N + 2, SingleParticle<T>(0, VSMC_NULLPTR)),
        csp_(N + 2, ConstSingleParticle<T>(0, VSMC_NULLPTR)),
        start_watch_(false)
    {
        weight_set_.set_equal_weight();
        sp_[0] = SingleParticle<T>(std::numeric_limits<size_type>::max
                VSMC_MACRO_NO_EXPANSION (), this);
        csp_[0] = ConstSingleParticle<T>(std::numeric_limits<size_type>::max
                VSMC_MACRO_NO_EXPANSION (), this);
        for (size_type i = 1; i != size_ + 2; ++i) {
            sp_[i] = SingleParticle<T>(i - 1, this);
            csp_[i] = ConstSingleParticle<T>(i - 1, this);
        }
    }

    /// \brief Number of particles
    size_type size () const {return size_;}

    const SingleParticle<T> &sp (size_type i) {return sp_[i + 1];}

    const ConstSingleParticle<T> &sp (size_type i) const {return csp_[i + 1];}

    iterator begin () {return iterator(&sp(0));}

    iterator end () {return iterator(&sp(size_));}

    const_iterator begin () const {return cosnt_iterator(&sp(0));}

    const_iterator end () const {return const_iterator(&sp(size_));}

    const_iterator cbegin () const {return const_iterator(&sp(0));}

    const_iterator cend () const {return const_iterator(&sp(size_));}

    reverse_iterator rbegin () {return reverse_iterator(end());}

    reverse_iterator rend () {return reverse_iterator(begin());}

    const_reverse_iterator rbegin () const
    {return const_reverse_iterator(end());}

    const_reverse_iterator rend () const
    {return const_reverse_iterator(begin());}

    const_reverse_iterator crbegin () const
    {return const_reverse_iterator(cend());}

    const_reverse_iterator crend () const
    {return const_reverse_iterator(cbegin());}

    /// \brief Read and write access to the value collection object
    value_type &value () {return value_;}

    /// \brief Read only access to the value collection object
    const value_type &value () const {return value_;}

    /// \brief Read and write access to the weight collection object
    weight_set_type &weight_set () {return weight_set_;}

    /// \brief Read only access to the weight collection object
    const weight_set_type &weight_set () const {return weight_set_;}

    /// \brief Read and write access to the RNG collection object
    rng_set_type &rng_set () {return rng_set_;}

    /// \brief Read only access to the RNG collection object
    const rng_set_type &rng_set () const {return rng_set_;}

    /// \brief Set normalized weight, unnormalized logarithm weight and ESS
    /// such that each particle has a equal weight
    void set_equal_weight () {weight_set_.set_equal_weight();}

    /// \brief Read normalized weights through an output iterator
    template <typename OutputIter>
    OutputIter read_weight (OutputIter first) const
    {return weight_set_.read_weight(first);}

    /// \brief Get the ESS of the particle collection based on the current
    /// weights
    double ess () const {return weight_set_.ess();}

    /// \brief Get an RNG stream for a given particle
    rng_type &rng (size_type id) {return rng_set_.rng(id);}

    /// \brief Performing resampling if ESS/N < threshold
    ///
    /// \param op The resampling operation funcitor
    /// \param threshold The threshold of ESS/N below which resampling will be
    /// performed
    ///
    /// \return true if resampling was performed
    ///
    /// \details
    /// The out of box behavior are suitable for most applications. However,
    /// the user can also gain full control over the internal of this
    /// function by changing the `weight_set_type` and other type traits for
    /// specific value collection type. The sequence of operations of this
    /// function is listed as in the following pseudo-code
    /// 1. Determine if resampling is required
    ///     * Set `N = weight_set.resample_size()`
    ///     * Set `resampled = weight_set.ess() < threshold * N`
    ///     * If `!resampled`, GO TO Step 8. Otherwise, GO TO Step 2
    /// 2. Determine if resampling algorithms shall be performed. This is only
    /// useful for MPI implementations. the boolean value `resampled` ensures
    /// that collective actions like `copy` will be called by all nodes.
    /// However, the resampling replication numbers can be computed by one
    /// node instead of by all. Therefore, the program can actually collect and
    /// read the resampling weights on one node, and do nothing on other nodes.
    /// This difference shall be reflected by the returning iterator of
    /// `read_resample_weight`
    ///     * (Allocate `double *weight` for size `N`)
    ///     * Set `double *end = weight_set.read_resample_weight(weight)`
    ///     * If `end != weight + N`, GO TO step 5. Otherwise, GO TO Step 3
    /// 3. Performing resampling, produce replication number of each particle
    ///     * (Allocate `size_type *replication` for size `N`)
    ///     * (Create RNG engine `resample_rng` according to
    ///     traits::ResampleRngTypeTrait)
    ///     * Call `op(N, resample_rng, weight, replication)`
    /// 4. Transform replication numbers into parent particle indices
    ///     * (Allocat `size_type *copy_from` for size `N`)
    ///     * (Create functor `r2c` according to
    ///     traits::ResampleCopyFromReplicationTypeTrait)
    ///     * Call `r2c(N, replication, copy_from)`
    /// 5. Set `const size_type *cptr` according to results of Step 2.
    ///     * If Step 3 and 4 are skipped according to Step 2, then set
    ///     `cptr = 0`. Otherwise,
    ///     * Set `cptr = copy_from`
    /// 6. Performing copying of particles
    ///     * Call `value.copy(N, cptr)`
    /// 7. Performing post resampling weight manipulation
    ///     * (Create functor `post` according to
    ///     traits::ResamplePostCopyTypeTrait)
    ///     * `post(weight_set)`
    /// 8. `return resampled`
    bool resample (const resample_type &op, double threshold)
    {
        size_type N = static_cast<size_type>(weight_set_.resample_size());
        bool resampled = weight_set_.ess() < threshold * N;
        if (resampled) {
            resample_copy_from_.resize(N);
            resample_replication_.resize(N);
            resample_weight_.resize(N);
            const size_type *cptr = VSMC_NULLPTR;
            double *end = VSMC_NULLPTR;
            {
                ScopedStopWatch<StopWatch> start(
                        resample_read_weight_watch_, start_watch_);
                end = weight_set_.read_resample_weight(&resample_weight_[0]);
            }
            if (end == &resample_weight_[0] + N) {
                {
                    ScopedStopWatch<StopWatch> start(
                            resample_get_replication_watch_, start_watch_);
                    op(static_cast<std::size_t>(N), resample_rng_,
                            &resample_weight_[0], &resample_replication_[0]);
                }
                {
                    ScopedStopWatch<StopWatch> start(
                            resample_get_copy_from_watch_, start_watch_);
                    resample_copy_from_replication_(N,
                            &resample_replication_[0],
                            &resample_copy_from_[0]);
                }
                cptr = &resample_copy_from_[0];
            }
            {
                ScopedStopWatch<StopWatch> start(
                        resample_copy_watch_, start_watch_);
                value_.copy(N, cptr);
            }
            {
                ScopedStopWatch<StopWatch> start(
                        resample_post_copy_watch_, start_watch_);
                resample_post_copy_(weight_set_);
            }
        }

        return resampled;
    }

    void start_watch () {start_watch_ = true;}

    void stop_watch () {start_watch_ = false;}

    void reset_watch ()
    {
        resample_read_weight_watch_.reset();
        resample_get_replication_watch_.reset();
        resample_get_copy_from_watch_.reset();
        resample_copy_watch_.reset();
        resample_post_copy_watch_.reset();
    }

    StopWatch &resample_read_weight_watch ()
    {return resample_read_weight_watch_;}

    const StopWatch &resample_read_weight_watch () const
    {return resample_read_weight_watch_;}

    StopWatch &resample_get_replication_watch ()
    {return resample_get_replication_watch_;}

    const StopWatch &resample_get_replication_watch () const
    {return resample_get_replication_watch_;}

    StopWatch &resample_get_copy_from_watch ()
    {return resample_get_copy_from_watch_;}

    const StopWatch &resample_get_copy_from_watch () const
    {return resample_get_copy_from_watch_;}

    StopWatch &resample_copy_watch ()
    {return resample_copy_watch_;}

    const StopWatch &resample_copy_watch () const
    {return resample_copy_watch_;}

    StopWatch &resample_post_copy_watch ()
    {return resample_post_copy_watch_;}

    const StopWatch &resample_post_copy_watch () const
    {return resample_post_copy_watch_;}

    private :

    size_type size_;
    value_type value_;
    weight_set_type weight_set_;
    rng_set_type rng_set_;

    std::vector<size_type> resample_replication_;
    std::vector<size_type> resample_copy_from_;
    std::vector<double> resample_weight_;
    typename traits::ResampleCopyFromReplicationTypeTrait<T>::type
        resample_copy_from_replication_;
    typename traits::ResamplePostCopyTypeTrait<T>::type resample_post_copy_;
    typename traits::ResampleRngTypeTrait<T>::type resample_rng_;
    std::vector<SingleParticle<T> > sp_;
    std::vector<ConstSingleParticle<T> > csp_;

    bool start_watch_;
    StopWatch resample_read_weight_watch_;
    StopWatch resample_get_replication_watch_;
    StopWatch resample_get_copy_from_watch_;
    StopWatch resample_copy_watch_;
    StopWatch resample_post_copy_watch_;
}; // class Particle

} // namespace vsmc

#endif // VSMC_CORE_PARTICLE_HPP
<|endoftext|>


<commit_before>#include "page.h"
#include "bloom_filter.h"
#include <fstream>
#include <cassert>
#include <algorithm>
#include <cstring>

using namespace dariadb::storage;

class PageCursor : public dariadb::storage::Cursor {
public:
	PageCursor(Page*page, const dariadb::IdArray&ids, dariadb::Time from, dariadb::Time to, dariadb::Flag flag) :
		link(page),
		_ids(ids),
		_from(from),
		_to(to),
		_flag(flag)
	{
		reset_pos();
	}

	~PageCursor() {
		if (link != nullptr) {
			link->dec_reader();
			link = nullptr;
		}
	}

	bool is_end()const override {
		return _is_end;
	}

	void readNext(Cursor::Callback*cbk)  override {
		std::lock_guard<std::mutex> lg(_locker);
		for (; !_is_end; _index_it++) {
			if (_index_it == _index_end) {
				Chunk_Ptr empty;
				cbk->call(empty);
				_is_end = true;
				break;
			}
			if (!_index_it->is_init) {
				Chunk_Ptr empty;
				cbk->call(empty);
				_is_end = true;
				continue;
			}

			if ((_ids.size() != 0) && (std::find(_ids.begin(), _ids.end(), _index_it->info.first.id) == _ids.end())) {
				continue;
			}

			if (!dariadb::storage::bloom_check(_index_it->info.flag_bloom, _flag)) {
				continue;
			}

			if (check_index_rec(_index_it)) {
				auto ptr = new Chunk(_index_it->info, link->chunks + _index_it->offset, link->header->chunk_size);
				Chunk_Ptr c{ ptr };
				assert(c->last.time != 0);
				cbk->call(c);
				_index_it++;
				break;
			}
			else {//end of data;
				_is_end = true;
				Chunk_Ptr empty;
				cbk->call(empty);
				break;
			}
		}
	}

	bool check_index_rec(Page_ChunkIndex*it) const{
		return ((dariadb::utils::inInterval(_from, _to, it->info.minTime)) || (dariadb::utils::inInterval(_from, _to, it->info.maxTime)));
	}

	void reset_pos() override { //start read from begining;
		_is_end = false;

		uint32_t upPos = link->header->chunk_per_storage;
		//TODO lock this;
		auto sz = this->link->_itree.size();
		if (sz != 0) {
			auto it = this->link->_itree.upper_bound(this->_to);

			if (it != this->link->_itree.end()) {
				upPos = (*it).second;
			}
		}

		_index_end = link->index + upPos;
		_index_it = link->index;

		//move to data begining
		for (; !_is_end; _index_it++) {
			if (_index_it == _index_end) {
				_is_end = true;
				break;
			}
			if (check_index_rec(_index_it)) {
				break;
			}
		}
	}
protected:
	Page* link;
	bool _is_end;
	Page_ChunkIndex *_index_it, *_index_end;
	dariadb::IdArray _ids;
	dariadb::Time _from, _to;
	dariadb::Flag _flag;
	std::mutex _locker;
};

Page::~Page() {
	region=nullptr;
	header=nullptr;
	index=nullptr;
	chunks=nullptr;
	mmap->close();
}

Page* Page::create(std::string file_name, uint64_t sz, uint32_t chunk_per_storage, uint32_t chunk_size) {
	auto res = new Page;
	auto mmap = utils::fs::MappedFile::touch(file_name, sz);
	auto region = mmap->data();
	std::fill(region, region + sz, 0);

	res->mmap = mmap;
	res->region = region;
	res->header = reinterpret_cast<PageHeader*>(region);
	res->index = reinterpret_cast<Page_ChunkIndex*>(region + sizeof(PageHeader));
	res->chunks = reinterpret_cast<uint8_t*>(region + sizeof(PageHeader) + sizeof(Page_ChunkIndex)*chunk_per_storage);

	res->header->chunk_per_storage = chunk_per_storage;
	res->header->chunk_size = chunk_size;
	res->header->maxTime = dariadb::Time(0);
	res->header->minTime = std::numeric_limits<dariadb::Time>::max();
    res->header->is_overwrite=false;
	return res;
}

Page* Page::open(std::string file_name) {
	auto res = new Page;
	auto mmap = utils::fs::MappedFile::open(file_name);

	auto region = mmap->data();

	res->mmap = mmap;
	res->region = region;
	res->header = reinterpret_cast<PageHeader*>(region);
	res->index = reinterpret_cast<Page_ChunkIndex*>(region + sizeof(PageHeader));
	res->chunks = reinterpret_cast<uint8_t*>(region + sizeof(PageHeader) + sizeof(Page_ChunkIndex)*res->header->chunk_per_storage);
	if (res->header->chunk_size == 0) {
		throw MAKE_EXCEPTION("(res->header->chunk_size == 0)");
	}

	for (size_t i = 0; i < res->header->chunk_per_storage; ++i) {
		auto irec = &res->index[i];
		if (!irec->is_init) {
			break;
		}
		res->_itree.insert2(irec->info.maxTime, i);
	}
	return res;
}

PageHeader Page::readHeader(std::string file_name) {
	std::ifstream istream;
	istream.open(file_name, std::fstream::in | std::fstream::binary);
	if (!istream.is_open()) {
		std::stringstream ss;
		ss << "can't open file. filename=" << file_name;
		throw MAKE_EXCEPTION(ss.str());
	}
	PageHeader result;
	memset(&result, 0, sizeof(PageHeader));
	istream.read((char *)&result, sizeof(PageHeader));
	istream.close();
	return result;

}

uint32_t Page::get_oldes_index() {
    auto min_time = std::numeric_limits<dariadb::Time>::max();
    uint32_t pos = 0;
    auto index_end = index + header->chunk_per_storage;
    uint32_t i = 0;
    for (auto index_it = index; index_it != index_end; ++index_it, ++i) {
        if (index_it->info.minTime<min_time) {
            pos = i;
            min_time = index_it->info.minTime;
        }
    }
    return pos;
}

bool Page::append(const Chunk_Ptr&ch, MODE mode) {
    std::lock_guard<std::mutex> lg(_locker);
	auto index_rec = (ChunkIndexInfo*)ch.get();
	auto buffer = ch->_buffer_t.data();

	assert(ch->last.time != 0);
	assert(header->chunk_size == ch->_buffer_t.size());
	if (is_full()) {
        if (mode == MODE::SINGLE) {
            header->is_overwrite=true;
            header->pos_index=0;
        }
		else {
			return false;
		}
	}
	index[header->pos_index].info = *index_rec;

    index[header->pos_index].is_init = true;

    if(!header->is_overwrite){
        index[header->pos_index].offset = header->pos_chunks;
        header->pos_chunks += header->chunk_size;
        header->addeded_chunks++;
		_itree.insert2(index_rec->maxTime, header->pos_index);
	}
	else {
		for (auto it = _itree.begin(); it != _itree.end();++it) {
			if (it->second == header->pos_index) {
				_itree.erase(it);
			}
		}
		_itree.insert2(index_rec->maxTime, header->pos_index);
	}
	memcpy(this->chunks + index[header->pos_index].offset, buffer, sizeof(uint8_t)*header->chunk_size);

	header->pos_index++;
	header->minTime = std::min(header->minTime,ch->minTime);
	header->maxTime = std::max(header->maxTime, ch->maxTime);
	return true;
}

bool Page::is_full()const {
	return !(header->pos_index < header->chunk_per_storage);
}

Cursor_ptr Page::get_chunks(const dariadb::IdArray&ids, dariadb::Time from, dariadb::Time to, dariadb::Flag flag) {
    std::lock_guard<std::mutex> lg(_locker);

	auto raw_ptr = new PageCursor(this,ids,from,to,flag );
	Cursor_ptr result{ raw_ptr };

	header->count_readers++;

	return result;
}

ChuncksList Page::get_open_chunks() {
    std::lock_guard<std::mutex> lg(_locker);
	auto index_end = this->index + this->header->pos_index;
	auto index_it = this->index;
	ChuncksList result;
	for (; index_it != index_end; index_it++) {
		if (!index_it->is_init) {
			continue;
		}
		if (!index_it->info.is_readonly) {
			index_it->is_init = false;
            auto ptr=new Chunk(index_it->info, this->chunks + index_it->offset, this->header->chunk_size);
            Chunk_Ptr c = Chunk_Ptr(ptr);
			result.push_back(c);
		}
	}
	return result;
}

void Page::dec_reader(){
    std::lock_guard<std::mutex> lg(_locker);
	header->count_readers--;
}

Cursor_ptr dariadb::storage::Page::chunksByIterval(const IdArray & ids, Flag flag, Time from, Time to)
{
	return get_chunks(ids, from, to, flag);
}

IdToChunkMap dariadb::storage::Page::chunksBeforeTimePoint(const IdArray & ids, Flag flag, Time timePoint)
{
	IdToChunkMap result;

	ChuncksList ch_list;
	auto cursor = this->get_chunks(ids, header->minTime, timePoint, flag);
	if (cursor == nullptr) {
		return result;
	}
	cursor->readAll(&ch_list);

	for (auto&v : ch_list) {
		auto find_res = result.find(v->first.id);
		if (find_res == result.end()) {
			result.insert(std::make_pair(v->first.id, v));
		}
		else {
			if (find_res->second->maxTime < v->maxTime) {
				result[v->first.id] = v;
			}
		}
	}
	return result;
}
class CountOfIdCallback :public Cursor::Callback {
public:
	dariadb::IdSet ids;
	CountOfIdCallback() {
	}
	~CountOfIdCallback() {
	}

	virtual void call(Chunk_Ptr & ptr) override {
		if (ptr != nullptr) {
			ids.insert(ptr->first.id);
		}
	}
};

dariadb::IdArray dariadb::storage::Page::getIds()
{
	auto cursor = get_chunks(dariadb::IdArray{}, header->minTime, header->maxTime, 0);

	CountOfIdCallback*clbk_raw = new CountOfIdCallback;
	std::unique_ptr<Cursor::Callback> clbk{ clbk_raw };

	cursor->readAll(clbk.get());

	return dariadb::IdArray{ clbk_raw->ids.begin(),clbk_raw->ids.end() };
}<commit_msg>update todo.<commit_after>#include "page.h"
#include "bloom_filter.h"
#include <fstream>
#include <cassert>
#include <algorithm>
#include <cstring>

using namespace dariadb::storage;

class PageCursor : public dariadb::storage::Cursor {
public:
	PageCursor(Page*page, const dariadb::IdArray&ids, dariadb::Time from, dariadb::Time to, dariadb::Flag flag) :
		link(page),
		_ids(ids),
		_from(from),
		_to(to),
		_flag(flag)
	{
		reset_pos();
	}

	~PageCursor() {
		if (link != nullptr) {
			link->dec_reader();
			link = nullptr;
		}
	}

	bool is_end()const override {
		return _is_end;
	}

	void readNext(Cursor::Callback*cbk)  override {
		std::lock_guard<std::mutex> lg(_locker);
		for (; !_is_end; _index_it++) {
			if (_index_it == _index_end) {
				Chunk_Ptr empty;
				cbk->call(empty);
				_is_end = true;
				break;
			}
			if (!_index_it->is_init) {
				Chunk_Ptr empty;
				cbk->call(empty);
				_is_end = true;
				continue;
			}

			if ((_ids.size() != 0) && (std::find(_ids.begin(), _ids.end(), _index_it->info.first.id) == _ids.end())) {
				continue;
			}

			if (!dariadb::storage::bloom_check(_index_it->info.flag_bloom, _flag)) {
				continue;
			}

			if (check_index_rec(_index_it)) {
				auto ptr = new Chunk(_index_it->info, link->chunks + _index_it->offset, link->header->chunk_size);
				Chunk_Ptr c{ ptr };
				assert(c->last.time != 0);
				cbk->call(c);
				_index_it++;
				break;
			}
			else {//end of data;
				_is_end = true;
				Chunk_Ptr empty;
				cbk->call(empty);
				break;
			}
		}
	}

	bool check_index_rec(Page_ChunkIndex*it) const{
		return ((dariadb::utils::inInterval(_from, _to, it->info.minTime)) || (dariadb::utils::inInterval(_from, _to, it->info.maxTime)));
	}

	void reset_pos() override { //start read from begining;
		_is_end = false;

		uint32_t upPos = link->header->chunk_per_storage;

		auto sz = this->link->_itree.size();
		if (sz != 0) {
			auto it = this->link->_itree.upper_bound(this->_to);

			if (it != this->link->_itree.end()) {
				upPos = (*it).second;
			}
		}

		_index_end = link->index + upPos;
		_index_it = link->index;

		//move to data begining
		for (; !_is_end; _index_it++) {
			if (_index_it == _index_end) {
				_is_end = true;
				break;
			}
			if (check_index_rec(_index_it)) {
				break;
			}
		}
	}
protected:
	Page* link;
	bool _is_end;
	Page_ChunkIndex *_index_it, *_index_end;
	dariadb::IdArray _ids;
	dariadb::Time _from, _to;
	dariadb::Flag _flag;
	std::mutex _locker;
};

Page::~Page() {
	region=nullptr;
	header=nullptr;
	index=nullptr;
	chunks=nullptr;
	mmap->close();
}

Page* Page::create(std::string file_name, uint64_t sz, uint32_t chunk_per_storage, uint32_t chunk_size) {
	auto res = new Page;
	auto mmap = utils::fs::MappedFile::touch(file_name, sz);
	auto region = mmap->data();
	std::fill(region, region + sz, 0);

	res->mmap = mmap;
	res->region = region;
	res->header = reinterpret_cast<PageHeader*>(region);
	res->index = reinterpret_cast<Page_ChunkIndex*>(region + sizeof(PageHeader));
	res->chunks = reinterpret_cast<uint8_t*>(region + sizeof(PageHeader) + sizeof(Page_ChunkIndex)*chunk_per_storage);

	res->header->chunk_per_storage = chunk_per_storage;
	res->header->chunk_size = chunk_size;
	res->header->maxTime = dariadb::Time(0);
	res->header->minTime = std::numeric_limits<dariadb::Time>::max();
    res->header->is_overwrite=false;
	return res;
}

Page* Page::open(std::string file_name) {
	auto res = new Page;
	auto mmap = utils::fs::MappedFile::open(file_name);

	auto region = mmap->data();

	res->mmap = mmap;
	res->region = region;
	res->header = reinterpret_cast<PageHeader*>(region);
	res->index = reinterpret_cast<Page_ChunkIndex*>(region + sizeof(PageHeader));
	res->chunks = reinterpret_cast<uint8_t*>(region + sizeof(PageHeader) + sizeof(Page_ChunkIndex)*res->header->chunk_per_storage);
	if (res->header->chunk_size == 0) {
		throw MAKE_EXCEPTION("(res->header->chunk_size == 0)");
	}

	for (size_t i = 0; i < res->header->chunk_per_storage; ++i) {
		auto irec = &res->index[i];
		if (!irec->is_init) {
			break;
		}
		res->_itree.insert2(irec->info.maxTime, i);
	}
	return res;
}

PageHeader Page::readHeader(std::string file_name) {
	std::ifstream istream;
	istream.open(file_name, std::fstream::in | std::fstream::binary);
	if (!istream.is_open()) {
		std::stringstream ss;
		ss << "can't open file. filename=" << file_name;
		throw MAKE_EXCEPTION(ss.str());
	}
	PageHeader result;
	memset(&result, 0, sizeof(PageHeader));
	istream.read((char *)&result, sizeof(PageHeader));
	istream.close();
	return result;

}

uint32_t Page::get_oldes_index() {
    auto min_time = std::numeric_limits<dariadb::Time>::max();
    uint32_t pos = 0;
    auto index_end = index + header->chunk_per_storage;
    uint32_t i = 0;
    for (auto index_it = index; index_it != index_end; ++index_it, ++i) {
        if (index_it->info.minTime<min_time) {
            pos = i;
            min_time = index_it->info.minTime;
        }
    }
    return pos;
}

bool Page::append(const Chunk_Ptr&ch, MODE mode) {
    std::lock_guard<std::mutex> lg(_locker);
	auto index_rec = (ChunkIndexInfo*)ch.get();
	auto buffer = ch->_buffer_t.data();

	assert(ch->last.time != 0);
	assert(header->chunk_size == ch->_buffer_t.size());
	if (is_full()) {
        if (mode == MODE::SINGLE) {
            header->is_overwrite=true;
            header->pos_index=0;
        }
		else {
			return false;
		}
	}
	index[header->pos_index].info = *index_rec;

    index[header->pos_index].is_init = true;

    if(!header->is_overwrite){
        index[header->pos_index].offset = header->pos_chunks;
        header->pos_chunks += header->chunk_size;
        header->addeded_chunks++;
		_itree.insert2(index_rec->maxTime, header->pos_index);
	}
	else {
		for (auto it = _itree.begin(); it != _itree.end();++it) {
			if (it->second == header->pos_index) {
				_itree.erase(it);
			}
		}
		_itree.insert2(index_rec->maxTime, header->pos_index);
	}
	memcpy(this->chunks + index[header->pos_index].offset, buffer, sizeof(uint8_t)*header->chunk_size);

	header->pos_index++;
	header->minTime = std::min(header->minTime,ch->minTime);
	header->maxTime = std::max(header->maxTime, ch->maxTime);
	return true;
}

bool Page::is_full()const {
	return !(header->pos_index < header->chunk_per_storage);
}

Cursor_ptr Page::get_chunks(const dariadb::IdArray&ids, dariadb::Time from, dariadb::Time to, dariadb::Flag flag) {
    std::lock_guard<std::mutex> lg(_locker);

	auto raw_ptr = new PageCursor(this,ids,from,to,flag );
	Cursor_ptr result{ raw_ptr };

	header->count_readers++;

	return result;
}

ChuncksList Page::get_open_chunks() {
    std::lock_guard<std::mutex> lg(_locker);
	auto index_end = this->index + this->header->pos_index;
	auto index_it = this->index;
	ChuncksList result;
	for (; index_it != index_end; index_it++) {
		if (!index_it->is_init) {
			continue;
		}
		if (!index_it->info.is_readonly) {
			index_it->is_init = false;
            auto ptr=new Chunk(index_it->info, this->chunks + index_it->offset, this->header->chunk_size);
            Chunk_Ptr c = Chunk_Ptr(ptr);
			result.push_back(c);
		}
	}
	return result;
}

void Page::dec_reader(){
    std::lock_guard<std::mutex> lg(_locker);
	header->count_readers--;
}

Cursor_ptr dariadb::storage::Page::chunksByIterval(const IdArray & ids, Flag flag, Time from, Time to)
{
	return get_chunks(ids, from, to, flag);
}

IdToChunkMap dariadb::storage::Page::chunksBeforeTimePoint(const IdArray & ids, Flag flag, Time timePoint)
{
	IdToChunkMap result;

	ChuncksList ch_list;
	auto cursor = this->get_chunks(ids, header->minTime, timePoint, flag);
	if (cursor == nullptr) {
		return result;
	}
	cursor->readAll(&ch_list);

	for (auto&v : ch_list) {
		auto find_res = result.find(v->first.id);
		if (find_res == result.end()) {
			result.insert(std::make_pair(v->first.id, v));
		}
		else {
			if (find_res->second->maxTime < v->maxTime) {
				result[v->first.id] = v;
			}
		}
	}
	return result;
}
class CountOfIdCallback :public Cursor::Callback {
public:
	dariadb::IdSet ids;
	CountOfIdCallback() {
	}
	~CountOfIdCallback() {
	}

	virtual void call(Chunk_Ptr & ptr) override {
		if (ptr != nullptr) {
			ids.insert(ptr->first.id);
		}
	}
};

dariadb::IdArray dariadb::storage::Page::getIds()
{
	auto cursor = get_chunks(dariadb::IdArray{}, header->minTime, header->maxTime, 0);

	CountOfIdCallback*clbk_raw = new CountOfIdCallback;
	std::unique_ptr<Cursor::Callback> clbk{ clbk_raw };

	cursor->readAll(clbk.get());

	return dariadb::IdArray{ clbk_raw->ids.begin(),clbk_raw->ids.end() };
}<|endoftext|>


<commit_before>#ifndef V_SMC_HISTORY_HPP
#define V_SMC_HISTORY_HPP

#include <vector>
#include <cstddef>
#include <vSMC/particle.hpp>

namespace vSMC {

enum HistoryMode {HISTORY_RAM, HISTORY_FILE, HISTORY_NONE};

template <class T>
class HistoryElement
{
    public :

    HistoryElement (const Particle<T> &particle,
            bool resample, double ess, std::size_t accept) :
        e_particle(particle), e_resample(resample),
        e_ess(ess), e_accept(accept) {}

    const Particle<T> &particle () const
    {
        return e_particle;
    }

    double ESS () const
    {
        return e_ess;
    }

    bool resample () const
    {
        return e_resample;
    }

    std::size_t accept () const
    {
        return e_accept;
    }

    private :

    Particle<T> e_particle;
    bool e_resample;
    double e_ess;
    std::size_t e_accept;
}; // class HistoryElement

template <class T>
class History
{
    public :

    explicit History (HistoryMode history_mode) : mode(history_mode) {};

    inline void push_back (const HistoryElement<T> &element)
    {
        history.push_back(element);
    }

    inline void pop_back ()
    {
        history.pop_back();
    }

    inline void pop_back (HistoryElement<T> &element)
    {
        element = history.back();
        history.pop_back();
    }

    private :

    HistoryMode mode;
    std::vector<HistoryElement<T> > history;
}; // class History

} // namespace vSMC

#endif // V_SMC_HISTORY_HPP
<commit_msg>explicit set HISTORY_NONE mode to 0<commit_after>#ifndef V_SMC_HISTORY_HPP
#define V_SMC_HISTORY_HPP

#include <vector>
#include <cstddef>
#include <vSMC/particle.hpp>

namespace vSMC {

enum HistoryMode {HISTORY_NONE = 0, HISTORY_RAM, HISTORY_FILE};

template <class T>
class HistoryElement
{
    public :

    HistoryElement (const Particle<T> &particle,
            bool resample, double ess, std::size_t accept) :
        e_particle(particle), e_resample(resample),
        e_ess(ess), e_accept(accept) {}

    const Particle<T> &particle () const
    {
        return e_particle;
    }

    double ESS () const
    {
        return e_ess;
    }

    bool resample () const
    {
        return e_resample;
    }

    std::size_t accept () const
    {
        return e_accept;
    }

    private :

    Particle<T> e_particle;
    bool e_resample;
    double e_ess;
    std::size_t e_accept;
}; // class HistoryElement

template <class T>
class History
{
    public :

    explicit History (HistoryMode history_mode) : mode(history_mode) {};

    inline void push_back (const HistoryElement<T> &element)
    {
        history.push_back(element);
    }

    inline void pop_back ()
    {
        history.pop_back();
    }

    inline void pop_back (HistoryElement<T> &element)
    {
        element = history.back();
        history.pop_back();
    }

    private :

    HistoryMode mode;
    std::vector<HistoryElement<T> > history;
}; // class History

} // namespace vSMC

#endif // V_SMC_HISTORY_HPP
<|endoftext|>