<commit_before>#ifndef STAN_MATH_REV_MAT_FUN_GP_EXP_QUAD_COV_HPP
#define STAN_MATH_REV_MAT_FUN_GP_EXP_QUAD_COV_HPP

#include <boost/math/tools/promotion.hpp>
#include <boost/type_traits.hpp>
#include <boost/utility/enable_if.hpp>
#include <stan/math/prim/mat/fun/Eigen.hpp>
#include <stan/math/prim/scal/err/check_not_nan.hpp>
#include <stan/math/prim/scal/err/check_positive.hpp>
#include <stan/math/prim/scal/fun/exp.hpp>
#include <stan/math/prim/scal/fun/square.hpp>
#include <stan/math/prim/scal/fun/squared_distance.hpp>
#include <stan/math/prim/scal/meta/scalar_type.hpp>
#include <stan/math/rev/core.hpp>
#include <stan/math/rev/scal/fun/value_of.hpp>
#include <vector>
#include <cmath>

namespace stan {
namespace math {

/**
 * This is a subclass of the vari class for precomputed
 * gradients of gp_exp_quad_cov.
 *
 * The class stores the double values for the distance
 * matrix, pointers to the varis for the covariance
 * matrix, along with a pointer to the vari for sigma,
 * and the vari for length_scale.
 *
 * @tparam T_x type of std::vector of elements
 * @tparam T_sigma type of sigma
 * @tparam T_l type of length scale
 */
template <typename T_x, typename T_sigma, typename T_l>
class gp_exp_quad_cov_vari : public vari {
 public:
  const size_t size_;
  const size_t size_ltri_;
  const double l_d_;
  const double sigma_d_;
  const double sigma_sq_d_;
  double *dist_;
  vari *l_vari_;
  vari *sigma_vari_;
  vari **cov_lower_;
  vari **cov_diag_;

  /**
   * Constructor for gp_exp_quad_cov.
   *
   * All memory allocated in
   * ChainableStack's stack_alloc arena.
   *
   * It is critical for the efficiency of this object
   * that the constructor create new varis that aren't
   * popped onto the var_stack_, but rather are
   * popped onto the var_nochain_stack_. This is
   * controlled to the second argument to
   * vari's constructor.
   *
   * @param x std::vector input that can be used in square distance
   *    Assumes each element of x is the same size
   * @param sigma standard deviation
   * @param length_scale length scale
   */
  gp_exp_quad_cov_vari(const std::vector<T_x> &x, const T_sigma &sigma,
                       const T_l &length_scale)
      : vari(0.0),
        size_(x.size()),
        size_ltri_(size_ * (size_ - 1) / 2),
        l_d_(value_of(length_scale)),
        sigma_d_(value_of(sigma)),
        sigma_sq_d_(sigma_d_ * sigma_d_),
        dist_(ChainableStack::memalloc_.alloc_array<double>(size_ltri_)),
        l_vari_(length_scale.vi_),
        sigma_vari_(sigma.vi_),
        cov_lower_(ChainableStack::memalloc_.alloc_array<vari *>(size_ltri_)),
        cov_diag_(ChainableStack::memalloc_.alloc_array<vari *>(size_)) {
    double inv_half_sq_l_d = 0.5 / (l_d_ * l_d_);
    size_t pos = 0;
    for (size_t j = 0; j < size_ - 1; ++j) {
      for (size_t i = j + 1; i < size_; ++i) {
        double dist_sq = squared_distance(x[i], x[j]);
        dist_[pos] = dist_sq;
        cov_lower_[pos] = new vari(
            sigma_sq_d_ * std::exp(-dist_sq * inv_half_sq_l_d), false);
        ++pos;
      }
    }
    for (size_t i = 0; i < size_; ++i)
      cov_diag_[i] = new vari(sigma_sq_d_, false);
  }

  virtual void chain() {
    double adjl = 0;
    double adjsigma = 0;

    for (size_t i = 0; i < size_ltri_; ++i) {
      vari *el_low = cov_lower_[i];
      double prod_add = el_low->adj_ * el_low->val_;
      adjl += prod_add * dist_[i];
      adjsigma += prod_add;
    }
    for (size_t i = 0; i < size_; ++i) {
      vari *el = cov_diag_[i];
      adjsigma += el->adj_ * el->val_;
    }
    l_vari_->adj_ += adjl / (l_d_ * l_d_ * l_d_);
    sigma_vari_->adj_ += adjsigma * 2 / sigma_d_;
  }
};

/**
 * This is a subclass of the vari class for precomputed
 * gradients of gp_exp_quad_cov.
 *
 * The class stores the double values for the distance
 * matrix, pointers to the varis for the covariance
 * matrix, along with a pointer to the vari for sigma,
 * and the vari for length_scale.
 *
 * @tparam T_x type of std::vector of elements
 * @tparam T_l type of length scale
 */
template <typename T_x, typename T_l>
class gp_exp_quad_cov_vari<T_x, double, T_l> : public vari {
 public:
  const size_t size_;
  const size_t size_ltri_;
  const double l_d_;
  const double sigma_d_;
  const double sigma_sq_d_;
  double *dist_;
  vari *l_vari_;
  vari **cov_lower_;
  vari **cov_diag_;

  /**
   * Constructor for gp_exp_quad_cov.
   *
   * All memory allocated in
   * ChainableStack's stack_alloc arena.
   *
   * It is critical for the efficiency of this object
   * that the constructor create new varis that aren't
   * popped onto the var_stack_, but rather are
   * popped onto the var_nochain_stack_. This is
   * controlled to the second argument to
   * vari's constructor.
   *
   * @param x std::vector input that can be used in square distance
   *    Assumes each element of x is the same size
   * @param sigma standard deviation
   * @param length_scale length scale
   */
  gp_exp_quad_cov_vari(const std::vector<T_x> &x, double sigma,
                       const T_l &length_scale)
      : vari(0.0),
        size_(x.size()),
        size_ltri_(size_ * (size_ - 1) / 2),
        l_d_(value_of(length_scale)),
        sigma_d_(value_of(sigma)),
        sigma_sq_d_(sigma_d_ * sigma_d_),
        dist_(ChainableStack::memalloc_.alloc_array<double>(size_ltri_)),
        l_vari_(length_scale.vi_),
        cov_lower_(ChainableStack::memalloc_.alloc_array<vari *>(size_ltri_)),
        cov_diag_(ChainableStack::memalloc_.alloc_array<vari *>(size_)) {
    double inv_half_sq_l_d = 0.5 / (l_d_ * l_d_);
    size_t pos = 0;
    for (size_t j = 0; j < size_ - 1; ++j) {
      for (size_t i = j + 1; i < size_; ++i) {
        double dist_sq = squared_distance(x[i], x[j]);
        dist_[pos] = dist_sq;
        cov_lower_[pos] = new vari(
            sigma_sq_d_ * std::exp(-dist_sq * inv_half_sq_l_d), false);
        ++pos;
      }
    }
    for (size_t i = 0; i < size_; ++i)
      cov_diag_[i] = new vari(sigma_sq_d_, false);
  }

  virtual void chain() {
    double adjl = 0;

    for (size_t i = 0; i < size_ltri_; ++i) {
      vari *el_low = cov_lower_[i];
      adjl += el_low->adj_ * el_low->val_ * dist_[i];
    }
    l_vari_->adj_ += adjl / (l_d_ * l_d_ * l_d_);
  }
};

/**
 * Returns a squared exponential kernel.
 *
 * @param x std::vector input that can be used in square distance
 *    Assumes each element of x is the same size
 * @param sigma standard deviation
 * @param length_scale length scale
 * @return squared distance
 * @throw std::domain_error if sigma <= 0, l <= 0, or
 *   x is nan or infinite
 */
template <typename T_x>
inline typename boost::enable_if_c<
    boost::is_same<typename scalar_type<T_x>::type, double>::value,
    Eigen::Matrix<var, -1, -1>>::type
gp_exp_quad_cov(const std::vector<T_x> &x, const var &sigma,
                const var &length_scale) {
  check_positive("gp_exp_quad_cov", "sigma", sigma);
  check_positive("gp_exp_quad_cov", "length_scale", length_scale);
  size_t x_size = x.size();
  for (size_t i = 0; i < x_size; ++i)
    check_not_nan("gp_exp_quad_cov", "x", x[i]);

  Eigen::Matrix<var, -1, -1> cov(x_size, x_size);
  if (x_size == 0)
    return cov;

  gp_exp_quad_cov_vari<T_x, var, var> *baseVari
      = new gp_exp_quad_cov_vari<T_x, var, var>(x, sigma, length_scale);

  size_t pos = 0;
  for (size_t j = 0; j < x_size - 1; ++j) {
    for (size_t i = (j + 1); i < x_size; ++i) {
      cov.coeffRef(i, j).vi_ = baseVari->cov_lower_[pos];
      cov.coeffRef(j, i).vi_ = cov.coeffRef(i, j).vi_;
      ++pos;
    }
    cov.coeffRef(j, j).vi_ = baseVari->cov_diag_[j];
  }
  cov.coeffRef(x_size - 1, x_size - 1).vi_ = baseVari->cov_diag_[x_size - 1];
  return cov;
}

/**
 * Returns a squared exponential kernel.
 *
 * @param x std::vector input that can be used in square distance
 *    Assumes each element of x is the same size
 * @param sigma standard deviation
 * @param length_scale length scale
 * @return squared distance
 * @throw std::domain_error if sigma <= 0, l <= 0, or
 *   x is nan or infinite
 */
template <typename T_x>
inline typename boost::enable_if_c<
    boost::is_same<typename scalar_type<T_x>::type, double>::value,
    Eigen::Matrix<var, -1, -1>>::type
gp_exp_quad_cov(const std::vector<T_x> &x, double sigma,
                const var &length_scale) {
  check_positive("gp_exp_quad_cov", "marginal variance", sigma);
  check_positive("gp_exp_quad_cov", "length-scale", length_scale);
  size_t x_size = x.size();
  for (size_t i = 0; i < x_size; ++i)
    check_not_nan("gp_exp_quad_cov", "x", x[i]);

  Eigen::Matrix<var, -1, -1> cov(x_size, x_size);
  if (x_size == 0)
    return cov;

  gp_exp_quad_cov_vari<T_x, double, var> *baseVari
      = new gp_exp_quad_cov_vari<T_x, double, var>(x, sigma, length_scale);

  size_t pos = 0;
  for (size_t j = 0; j < x_size - 1; ++j) {
    for (size_t i = (j + 1); i < x_size; ++i) {
      cov.coeffRef(i, j).vi_ = baseVari->cov_lower_[pos];
      cov.coeffRef(j, i).vi_ = cov.coeffRef(i, j).vi_;
      ++pos;
    }
    cov.coeffRef(j, j).vi_ = baseVari->cov_diag_[j];
  }
  cov.coeffRef(x_size - 1, x_size - 1).vi_ = baseVari->cov_diag_[x_size - 1];
  return cov;
}

}  // namespace math
}  // namespace stan
#endif
<commit_msg>agree w jenkins<commit_after>#ifndef STAN_MATH_REV_MAT_FUN_GP_EXP_QUAD_COV_HPP
#define STAN_MATH_REV_MAT_FUN_GP_EXP_QUAD_COV_HPP

#include <boost/math/tools/promotion.hpp>
#include <boost/type_traits.hpp>
#include <boost/utility/enable_if.hpp>
#include <stan/math/prim/mat/fun/Eigen.hpp>
#include <stan/math/prim/scal/err/check_not_nan.hpp>
#include <stan/math/prim/scal/err/check_positive.hpp>
#include <stan/math/prim/scal/fun/exp.hpp>
#include <stan/math/prim/scal/fun/square.hpp>
#include <stan/math/prim/scal/fun/squared_distance.hpp>
#include <stan/math/prim/scal/meta/scalar_type.hpp>
#include <stan/math/rev/core.hpp>
#include <stan/math/rev/scal/fun/value_of.hpp>
#include <vector>
#include <cmath>

namespace stan {
namespace math {

/**
 * This is a subclass of the vari class for precomputed
 * gradients of gp_exp_quad_cov.
 *
 * The class stores the double values for the distance
 * matrix, pointers to the varis for the covariance
 * matrix, along with a pointer to the vari for sigma,
 * and the vari for length_scale.
 *
 * @tparam T_x type of std::vector of elements
 * @tparam T_sigma type of sigma
 * @tparam T_l type of length scale
 */
template <typename T_x, typename T_sigma, typename T_l>
class gp_exp_quad_cov_vari : public vari {
 public:
  const size_t size_;
  const size_t size_ltri_;
  const double l_d_;
  const double sigma_d_;
  const double sigma_sq_d_;
  double *dist_;
  vari *l_vari_;
  vari *sigma_vari_;
  vari **cov_lower_;
  vari **cov_diag_;

  /**
   * Constructor for gp_exp_quad_cov.
   *
   * All memory allocated in
   * ChainableStack's stack_alloc arena.
   *
   * It is critical for the efficiency of this object
   * that the constructor create new varis that aren't
   * popped onto the var_stack_, but rather are
   * popped onto the var_nochain_stack_. This is
   * controlled to the second argument to
   * vari's constructor.
   *
   * @param x std::vector input that can be used in square distance
   *    Assumes each element of x is the same size
   * @param sigma standard deviation
   * @param length_scale length scale
   */
  gp_exp_quad_cov_vari(const std::vector<T_x> &x, const T_sigma &sigma,
                       const T_l &length_scale)
      : vari(0.0),
        size_(x.size()),
        size_ltri_(size_ * (size_ - 1) / 2),
        l_d_(value_of(length_scale)),
        sigma_d_(value_of(sigma)),
        sigma_sq_d_(sigma_d_ * sigma_d_),
        dist_(ChainableStack::instance().memalloc_.alloc_array<double>(size_ltri_)),
        l_vari_(length_scale.vi_),
        sigma_vari_(sigma.vi_),
        cov_lower_(ChainableStack::instance().memalloc_.alloc_array<vari *>(size_ltri_)),
        cov_diag_(ChainableStack::instance().memalloc_.alloc_array<vari *>(size_)) {
    double inv_half_sq_l_d = 0.5 / (l_d_ * l_d_);
    size_t pos = 0;
    for (size_t j = 0; j < size_ - 1; ++j) {
      for (size_t i = j + 1; i < size_; ++i) {
        double dist_sq = squared_distance(x[i], x[j]);
        dist_[pos] = dist_sq;
        cov_lower_[pos] = new vari(
            sigma_sq_d_ * std::exp(-dist_sq * inv_half_sq_l_d), false);
        ++pos;
      }
    }
    for (size_t i = 0; i < size_; ++i)
      cov_diag_[i] = new vari(sigma_sq_d_, false);
  }

  virtual void chain() {
    double adjl = 0;
    double adjsigma = 0;

    for (size_t i = 0; i < size_ltri_; ++i) {
      vari *el_low = cov_lower_[i];
      double prod_add = el_low->adj_ * el_low->val_;
      adjl += prod_add * dist_[i];
      adjsigma += prod_add;
    }
    for (size_t i = 0; i < size_; ++i) {
      vari *el = cov_diag_[i];
      adjsigma += el->adj_ * el->val_;
    }
    l_vari_->adj_ += adjl / (l_d_ * l_d_ * l_d_);
    sigma_vari_->adj_ += adjsigma * 2 / sigma_d_;
  }
};

/**
 * This is a subclass of the vari class for precomputed
 * gradients of gp_exp_quad_cov.
 *
 * The class stores the double values for the distance
 * matrix, pointers to the varis for the covariance
 * matrix, along with a pointer to the vari for sigma,
 * and the vari for length_scale.
 *
 * @tparam T_x type of std::vector of elements
 * @tparam T_l type of length scale
 */
template <typename T_x, typename T_l>
class gp_exp_quad_cov_vari<T_x, double, T_l> : public vari {
 public:
  const size_t size_;
  const size_t size_ltri_;
  const double l_d_;
  const double sigma_d_;
  const double sigma_sq_d_;
  double *dist_;
  vari *l_vari_;
  vari **cov_lower_;
  vari **cov_diag_;

  /**
   * Constructor for gp_exp_quad_cov.
   *
   * All memory allocated in
   * ChainableStack's stack_alloc arena.
   *
   * It is critical for the efficiency of this object
   * that the constructor create new varis that aren't
   * popped onto the var_stack_, but rather are
   * popped onto the var_nochain_stack_. This is
   * controlled to the second argument to
   * vari's constructor.
   *
   * @param x std::vector input that can be used in square distance
   *    Assumes each element of x is the same size
   * @param sigma standard deviation
   * @param length_scale length scale
   */
  gp_exp_quad_cov_vari(const std::vector<T_x> &x, double sigma,
                       const T_l &length_scale)
      : vari(0.0),
        size_(x.size()),
        size_ltri_(size_ * (size_ - 1) / 2),
        l_d_(value_of(length_scale)),
        sigma_d_(value_of(sigma)),
        sigma_sq_d_(sigma_d_ * sigma_d_),
        dist_(ChainableStack::instance().memalloc_.alloc_array<double>(size_ltri_)),
        l_vari_(length_scale.vi_),
        cov_lower_(ChainableStack::instance().memalloc_.alloc_array<vari *>(size_ltri_)),
        cov_diag_(ChainableStack::instance().memalloc_.alloc_array<vari *>(size_)) {
    double inv_half_sq_l_d = 0.5 / (l_d_ * l_d_);
    size_t pos = 0;
    for (size_t j = 0; j < size_ - 1; ++j) {
      for (size_t i = j + 1; i < size_; ++i) {
        double dist_sq = squared_distance(x[i], x[j]);
        dist_[pos] = dist_sq;
        cov_lower_[pos] = new vari(
            sigma_sq_d_ * std::exp(-dist_sq * inv_half_sq_l_d), false);
        ++pos;
      }
    }
    for (size_t i = 0; i < size_; ++i)
      cov_diag_[i] = new vari(sigma_sq_d_, false);
  }

  virtual void chain() {
    double adjl = 0;

    for (size_t i = 0; i < size_ltri_; ++i) {
      vari *el_low = cov_lower_[i];
      adjl += el_low->adj_ * el_low->val_ * dist_[i];
    }
    l_vari_->adj_ += adjl / (l_d_ * l_d_ * l_d_);
  }
};

/**
 * Returns a squared exponential kernel.
 *
 * @param x std::vector input that can be used in square distance
 *    Assumes each element of x is the same size
 * @param sigma standard deviation
 * @param length_scale length scale
 * @return squared distance
 * @throw std::domain_error if sigma <= 0, l <= 0, or
 *   x is nan or infinite
 */
template <typename T_x>
inline typename boost::enable_if_c<
    boost::is_same<typename scalar_type<T_x>::type, double>::value,
    Eigen::Matrix<var, -1, -1>>::type
gp_exp_quad_cov(const std::vector<T_x> &x, const var &sigma,
                const var &length_scale) {
  check_positive("gp_exp_quad_cov", "sigma", sigma);
  check_positive("gp_exp_quad_cov", "length_scale", length_scale);
  size_t x_size = x.size();
  for (size_t i = 0; i < x_size; ++i)
    check_not_nan("gp_exp_quad_cov", "x", x[i]);

  Eigen::Matrix<var, -1, -1> cov(x_size, x_size);
  if (x_size == 0)
    return cov;

  gp_exp_quad_cov_vari<T_x, var, var> *baseVari
      = new gp_exp_quad_cov_vari<T_x, var, var>(x, sigma, length_scale);

  size_t pos = 0;
  for (size_t j = 0; j < x_size - 1; ++j) {
    for (size_t i = (j + 1); i < x_size; ++i) {
      cov.coeffRef(i, j).vi_ = baseVari->cov_lower_[pos];
      cov.coeffRef(j, i).vi_ = cov.coeffRef(i, j).vi_;
      ++pos;
    }
    cov.coeffRef(j, j).vi_ = baseVari->cov_diag_[j];
  }
  cov.coeffRef(x_size - 1, x_size - 1).vi_ = baseVari->cov_diag_[x_size - 1];
  return cov;
}

/**
 * Returns a squared exponential kernel.
 *
 * @param x std::vector input that can be used in square distance
 *    Assumes each element of x is the same size
 * @param sigma standard deviation
 * @param length_scale length scale
 * @return squared distance
 * @throw std::domain_error if sigma <= 0, l <= 0, or
 *   x is nan or infinite
 */
template <typename T_x>
inline typename boost::enable_if_c<
    boost::is_same<typename scalar_type<T_x>::type, double>::value,
    Eigen::Matrix<var, -1, -1>>::type
gp_exp_quad_cov(const std::vector<T_x> &x, double sigma,
                const var &length_scale) {
  check_positive("gp_exp_quad_cov", "marginal variance", sigma);
  check_positive("gp_exp_quad_cov", "length-scale", length_scale);
  size_t x_size = x.size();
  for (size_t i = 0; i < x_size; ++i)
    check_not_nan("gp_exp_quad_cov", "x", x[i]);

  Eigen::Matrix<var, -1, -1> cov(x_size, x_size);
  if (x_size == 0)
    return cov;

  gp_exp_quad_cov_vari<T_x, double, var> *baseVari
      = new gp_exp_quad_cov_vari<T_x, double, var>(x, sigma, length_scale);

  size_t pos = 0;
  for (size_t j = 0; j < x_size - 1; ++j) {
    for (size_t i = (j + 1); i < x_size; ++i) {
      cov.coeffRef(i, j).vi_ = baseVari->cov_lower_[pos];
      cov.coeffRef(j, i).vi_ = cov.coeffRef(i, j).vi_;
      ++pos;
    }
    cov.coeffRef(j, j).vi_ = baseVari->cov_diag_[j];
  }
  cov.coeffRef(x_size - 1, x_size - 1).vi_ = baseVari->cov_diag_[x_size - 1];
  return cov;
}

}  // namespace math
}  // namespace stan
#endif
<|endoftext|>


<commit_before>/*******************************************************************************
 * thrill/api/sort.hpp
 *
 * Part of Project Thrill - http://project-thrill.org
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 * Copyright (C) 2015 Michael Axtmann <michael.axtmann@kit.edu>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_SORT_HEADER
#define THRILL_API_SORT_HEADER

#include <thrill/api/context.hpp>
#include <thrill/api/dia.hpp>
#include <thrill/api/dop_node.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/common/stat_logger.hpp>
#include <thrill/net/flow_control_channel.hpp>
#include <thrill/net/flow_control_manager.hpp>
#include <thrill/net/group.hpp>

#include <algorithm>
#include <functional>
#include <string>
#include <vector>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a Sort operation. Sort sorts a DIA according to a given
 * compare function
 *
 * \tparam ValueType Type of DIA elements
 * \tparam Stack Function stack, which contains the chained lambdas between the last and this DIANode.
 * \tparam CompareFunction Type of the compare function
 */
template <typename ValueType, typename ParentDIA, typename CompareFunction>
class SortNode final : public DOpNode<ValueType>
{
    static const bool debug = false;

    using Super = DOpNode<ValueType>;
    using Super::context_;

public:
    /*!
     * Constructor for a sort node.
     *
     * \param parent DIA.
     * \param parent_stack Stack of lambda functions between parent and this node
     * \param compare_function Function comparing two elements.
     */
    SortNode(const ParentDIA& parent,
             CompareFunction compare_function,
             StatsNode* stats_node)
        : DOpNode<ValueType>(parent.ctx(), { parent.node() }, stats_node),
          compare_function_(compare_function),
          stream_id_samples_(parent.ctx().GetNewCatStream()),
          emitters_samples_(stream_id_samples_->OpenWriters()),
          stream_id_data_(parent.ctx().GetNewCatStream()),
          emitters_data_(stream_id_data_->OpenWriters())
    {
        // Hook PreOp(s)
        auto pre_op_fn = [=](const ValueType& input) {
                             PreOp(input);
                         };

        auto lop_chain = parent.stack().push(pre_op_fn).emit();
        parent.node()->RegisterChild(lop_chain, this->type());
    }

    //! Executes the sum operation.
    void Execute() final {
        MainOp();
    }

    void PushData(bool consume) final {

        for (size_t i = 0; i < data_.size(); i++) {
            this->PushItem(data_[i]);
        }

        if (consume) {
            std::vector<ValueType>().swap(data_);
        }
    }

    void Dispose() final {
        std::vector<ValueType>().swap(data_);
    }

private:
    //! The sum function which is applied to two elements.
    CompareFunction compare_function_;
    //! Local data
    std::vector<ValueType> data_;

    //! Emitter to send samples to process 0
    data::CatStreamPtr stream_id_samples_;
    std::vector<data::CatStream::Writer> emitters_samples_;

    //! Emitters to send data to other workers specified by splitters.
    data::CatStreamPtr stream_id_data_;
    std::vector<data::CatStream::Writer> emitters_data_;

    // epsilon
    static constexpr double desired_imbalance_ = 0.25;

    void PreOp(ValueType input) {
        data_.push_back(input);
    }

    void FindAndSendSplitters(
        std::vector<ValueType>& splitters, size_t sample_size) {
        // Get samples from other workers
        size_t num_total_workers = context_.num_workers();

        std::vector<ValueType> samples;
        samples.reserve(sample_size * num_total_workers);
        // TODO(tb): OpenConsumeReader
        auto reader = stream_id_samples_->OpenCatReader(true);

        while (reader.HasNext()) {
            samples.push_back(reader.template Next<ValueType>());
        }

        // Find splitters
        std::sort(samples.begin(), samples.end(), compare_function_);

        size_t splitting_size = samples.size() / num_total_workers;

        // Send splitters to other workers
        for (size_t i = 1; i < num_total_workers; i++) {
            splitters.push_back(samples[i * splitting_size]);
            for (size_t j = 1; j < num_total_workers; j++) {
                emitters_samples_[j].Put(samples[i * splitting_size]);
            }
        }

        for (size_t j = 1; j < num_total_workers; j++) {
            emitters_samples_[j].Close();
        }
    }

    class TreeBuilder
    {
    public:
        ValueType* tree_;
        ValueType* samples_;
        size_t index_ = 0;
        size_t ssplitter_;

        /*!
         * Target: tree. Size of 'number of splitter'
         * Source: sorted splitters. Size of 'number of splitter'
         * Number of splitter
         */
        TreeBuilder(ValueType* splitter_tree,
                    ValueType* samples,
                    size_t ssplitter)
            : tree_(splitter_tree),
              samples_(samples),
              ssplitter_(ssplitter) {
            if (ssplitter != 0)
                recurse(samples, samples + ssplitter, 1);
        }

        void recurse(ValueType* lo, ValueType* hi, unsigned int treeidx) {

            // pick middle element as splitter
            ValueType* mid = lo + (ssize_t)(hi - lo) / 2;
            assert(mid < samples_ + ssplitter_);
            tree_[treeidx] = *mid;

            ValueType* midlo = mid, * midhi = mid + 1;

            if (2 * treeidx < ssplitter_)
            {
                recurse(lo, midlo, 2 * treeidx + 0);
                recurse(midhi, hi, 2 * treeidx + 1);
            }
        }
    };

    bool Equal(const ValueType& ele1, const ValueType& ele2) {
        return !(compare_function_(ele1, ele2) || compare_function_(ele2, ele1));
    }

    //! round n down by k where k is a power of two.
    template <typename Integral>
    static inline size_t RoundDown(Integral n, Integral k) {
        return (n & ~(k - 1));
    }

    void EmitToBuckets(
        // Tree of splitters, sizeof |splitter|
        const ValueType* const tree,
        // Number of buckets: k = 2^{log_k}
        size_t k,
        size_t log_k,
        // Number of actual workers to send to
        size_t actual_k,
        const ValueType* const sorted_splitters,
        size_t prefix_elem,
        size_t total_elem) {

        // enlarge emitters array to next power of two to have direct access,
        // because we fill the splitter set up with sentinels == last splitter,
        // hence all items land in the last bucket.
        assert(emitters_data_.size() == actual_k);
        assert(actual_k <= k);

        while (emitters_data_.size() < k)
            emitters_data_.emplace_back(nullptr);

        std::swap(emitters_data_[actual_k - 1], emitters_data_[k - 1]);

        // classify all items (take two at once) and immediately transmit them.

        const size_t stepsize = 2;

        size_t i = 0;
        for ( ; i < RoundDown(data_.size(), stepsize); i += stepsize)
        {
            // take two items
            size_t j0 = 1;
            const ValueType& el0 = data_[i];

            size_t j1 = 1;
            const ValueType& el1 = data_[i + 1];

            // run items down the tree
            for (size_t l = 0; l < log_k; l++)
            {
                j0 = j0 * 2 + !(compare_function_(el0, tree[j0]));
                j1 = j1 * 2 + !(compare_function_(el1, tree[j1]));
            }

            size_t b0 = j0 - k;
            size_t b1 = j1 - k;

            if (b0 && Equal(el0, sorted_splitters[b0 - 1])) {
                while (b0 && Equal(el0, sorted_splitters[b0 - 1])
                       && (prefix_elem + i) * actual_k < b0 * total_elem) {
                    b0--;
                }

                if (b0 + 1 >= actual_k) {
                    b0 = k - 1;
                }
            }

            assert(emitters_data_[b0].IsValid());
            emitters_data_[b0].Put(el0);

            if (b1 && Equal(el1, sorted_splitters[b1 - 1])) {
                while (b1 && Equal(el1, sorted_splitters[b1 - 1])
                       && (prefix_elem + i + 1) * actual_k < b1 * total_elem) {
                    b1--;
                }

                if (b1 + 1 >= actual_k) {
                    b1 = k - 1;
                }
            }

            assert(emitters_data_[b1].IsValid());
            emitters_data_[b1].Put(el1);
        }

        // last iteration of loop if we have an odd number of items.
        for ( ; i < data_.size(); i++)
        {
            size_t j = 1;
            for (size_t l = 0; l < log_k; l++)
            {
                j = j * 2 + !(compare_function_(data_[i], tree[j]));
            }

            size_t b = j - k;

            while (b && Equal(data_[i], sorted_splitters[b - 1])
                   && (prefix_elem + i) * actual_k < b * total_elem) {
                b--;
            }

            if (b + 1 >= actual_k) {
                b = k - 1;
            }

            assert(emitters_data_[b].IsValid());
            emitters_data_[b].Put(data_[i]);
        }
    }

    void MainOp() {
        net::FlowControlChannel& channel = context_.flow_control_channel();

        size_t prefix_elem = channel.PrefixSum(data_.size(), (size_t)0, std::plus<size_t>(), false);
        size_t total_elem = channel.AllReduce(data_.size());

        size_t num_total_workers = context_.num_workers();
        size_t sample_size =
            std::max(common::IntegerLog2Ceil(total_elem) *
                     static_cast<size_t>(1 / (desired_imbalance_ * desired_imbalance_)),
                     (size_t)1);

        sample_size = std::min(data_.size(), sample_size);

        sample_size = std::min(data_.size(), sample_size);

        LOG << prefix_elem << " elements, out of " << total_elem;

        std::default_random_engine generator(std::random_device { } ());
        std::uniform_int_distribution<size_t> distribution(0, data_.size() - 1);

        // Send samples to worker 0
        for (size_t i = 0; i < sample_size; i++) {
            size_t sample = distribution(generator);
            emitters_samples_[0].Put(data_[sample]);
        }
        emitters_samples_[0].Close();

        // Get the ceiling of log(num_total_workers), as SSSS needs 2^n buckets.
        size_t ceil_log = common::IntegerLog2Ceil(num_total_workers);
        size_t workers_algo = 1 << ceil_log;
        size_t splitter_count_algo = workers_algo - 1;

        std::vector<ValueType> splitters;
        splitters.reserve(workers_algo);

        if (context_.my_rank() == 0) {
            FindAndSendSplitters(splitters, sample_size);
        }
        else {
            // Close unused emitters
            for (size_t j = 1; j < num_total_workers; j++) {
                emitters_samples_[j].Close();
            }
            bool consume = false;
            auto reader = stream_id_samples_->OpenCatReader(consume);
            while (reader.HasNext()) {
                splitters.push_back(reader.template Next<ValueType>());
            }
        }
        stream_id_samples_->Close();

        // code from SS2NPartition, slightly altered

        ValueType* splitter_tree = new ValueType[workers_algo + 1];

        // add sentinel splitters if fewer nodes than splitters.
        for (size_t i = num_total_workers; i < workers_algo; i++) {
            splitters.push_back(splitters.back());
        }

        TreeBuilder(splitter_tree,
                    splitters.data(),
                    splitter_count_algo);

        EmitToBuckets(
            splitter_tree, // Tree. sizeof |splitter|
            workers_algo,  // Number of buckets
            ceil_log,
            num_total_workers,
            splitters.data(),
            prefix_elem,
            total_elem);

        delete[] splitter_tree;

        // end of SS2N

        for (size_t i = 0; i < emitters_data_.size(); i++)
            emitters_data_[i].Close();

        data_.clear();

        bool consume = false;
        auto reader = stream_id_data_->OpenCatReader(consume);

        while (reader.HasNext()) {
            data_.push_back(reader.template Next<ValueType>());
        }
        stream_id_data_->Close();

        double balance = 0;
        if (data_.size() > 0) {
            balance = static_cast<double>(data_.size())
                      * static_cast<double>(num_total_workers)
                      / static_cast<double>(total_elem);
        }

        if (balance > 1) {
            balance = 1 / balance;
        }

        std::sort(data_.begin(), data_.end(), compare_function_);

        STATC << "NodeType" << "Sort"
              << "Workers" << num_total_workers
              << "LocalSize" << data_.size()
              << "Balance Factor" << balance
              << "Sample Size" << sample_size;

        this->WriteStreamStats(stream_id_data_);
        this->WriteStreamStats(stream_id_samples_);
    }

    void PostOp() { }
};

template <typename ValueType, typename Stack>
template <typename CompareFunction>
auto DIA<ValueType, Stack>::Sort(const CompareFunction &compare_function) const {
    assert(IsValid());

    using SortNode = api::SortNode<ValueType, DIA, CompareFunction>;

    static_assert(
        std::is_convertible<
            ValueType,
            typename FunctionTraits<CompareFunction>::template arg<0>
            >::value ||
        std::is_same<CompareFunction, std::less<ValueType> >::value,
        "CompareFunction has the wrong input type");

    static_assert(
        std::is_convertible<
            ValueType,
            typename FunctionTraits<CompareFunction>::template arg<1> >::value ||
        std::is_same<CompareFunction, std::less<ValueType> >::value,
        "CompareFunction has the wrong input type");

    static_assert(
        std::is_convertible<
            typename FunctionTraits<CompareFunction>::result_type,
            bool>::value,
        "CompareFunction has the wrong output type (should be bool)");

    StatsNode* stats_node = AddChildStatsNode("Sort", DIANodeType::DOP);
    auto shared_node
        = std::make_shared<SortNode>(*this, compare_function, stats_node);

    return DIA<ValueType>(shared_node, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_SORT_HEADER

/******************************************************************************/
<commit_msg>Sort() now uses files for the unsorted data<commit_after>/*******************************************************************************
 * thrill/api/sort.hpp
 *
 * Part of Project Thrill - http://project-thrill.org
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 * Copyright (C) 2015 Michael Axtmann <michael.axtmann@kit.edu>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_SORT_HEADER
#define THRILL_API_SORT_HEADER

#include <thrill/api/context.hpp>
#include <thrill/api/dia.hpp>
#include <thrill/api/dop_node.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/common/stat_logger.hpp>
#include <thrill/data/file.hpp>
#include <thrill/net/flow_control_channel.hpp>
#include <thrill/net/flow_control_manager.hpp>
#include <thrill/net/group.hpp>

#include <algorithm>
#include <functional>
#include <string>
#include <vector>
#include <cstdlib>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a Sort operation. Sort sorts a DIA according to a given
 * compare function
 *
 * \tparam ValueType Type of DIA elements
 * \tparam Stack Function stack, which contains the chained lambdas between the last and this DIANode.
 * \tparam CompareFunction Type of the compare function
 */
template <typename ValueType, typename ParentDIA, typename CompareFunction>
class SortNode final : public DOpNode<ValueType>
{
    static const bool debug = true;

    using Super = DOpNode<ValueType>;
    using Super::context_;

public:
    /*!
     * Constructor for a sort node.
     *
     * \param parent DIA.
     * \param parent_stack Stack of lambda functions between parent and this node
     * \param compare_function Function comparing two elements.
     */
    SortNode(const ParentDIA& parent,
             CompareFunction compare_function,
             StatsNode* stats_node)
        : DOpNode<ValueType>(parent.ctx(), { parent.node() }, stats_node),
          compare_function_(compare_function),
          stream_id_samples_(parent.ctx().GetNewCatStream()),
          emitters_samples_(stream_id_samples_->OpenWriters()),
          stream_id_data_(parent.ctx().GetNewCatStream()),
          emitters_data_(stream_id_data_->OpenWriters())
    {
        // Hook PreOp(s)
        auto pre_op_fn = [=](const ValueType& input) {
                             PreOp(input);
                         };

        unsorted_data_ = context_.GetFilePtr();
        unsorted_writer_ = unsorted_data_->GetWriter();

        auto lop_chain = parent.stack().push(pre_op_fn).emit();
        parent.node()->RegisterChild(lop_chain, this->type());
    }

    //! Executes the sum operation.
    void Execute() final {
        MainOp();
    }

    void PushData(bool consume) final {

        for (size_t i = 0; i < data_.size(); i++) {
            this->PushItem(data_[i]);
        }

        if (consume) {
            std::vector<ValueType>().swap(data_);
        }
    }

    void Dispose() final {
        std::vector<ValueType>().swap(data_);
    }

private:
    //! The sum function which is applied to two elements.
    CompareFunction compare_function_;
    //! Local data
    std::vector<ValueType> data_;
    std::vector<ValueType> samples_;

    data::FilePtr unsorted_data_;
    data::File::Writer unsorted_writer_;
    size_t total_items_ = 0;
    size_t after_sample_ = 0;
    size_t next_sample_ = 0;

    //! Emitter to send samples to process 0
    data::CatStreamPtr stream_id_samples_;
    std::vector<data::CatStream::Writer> emitters_samples_;

    //! Emitters to send data to other workers specified by splitters.
    data::CatStreamPtr stream_id_data_;
    std::vector<data::CatStream::Writer> emitters_data_;

    // epsilon
    static constexpr double desired_imbalance_ = 0.25;

    void PreOp(ValueType input) {
        unsorted_writer_.Put(input);
        total_items_++;
        after_sample_++;
        //In this stage we do not know how many elements are there in total.
        //Therefore we draw samples based on current number of elements and
        //randomly replace older samples when we have too many.
        if (after_sample_ > next_sample_) {
            if (samples_.size() >
                std::log2(total_items_ * context_.num_workers())
                * (1 / (desired_imbalance_ * desired_imbalance_))) {
                samples_[std::rand() % samples_.size()] = input;
            } else {
                samples_.push_back(input);
            }
            after_sample_ = 0;
            double sample = (double)total_items_ /
                (std::log2(total_items_ * context_.num_workers())
                 * (1 / (desired_imbalance_ * desired_imbalance_)));
            next_sample_ = std::floor(sample);
        }
    }

    void FindAndSendSplitters(
        std::vector<ValueType>& splitters, size_t sample_size) {
        // Get samples from other workers
        size_t num_total_workers = context_.num_workers();

        std::vector<ValueType> samples;
        samples.reserve(sample_size * num_total_workers);
        // TODO(tb): OpenConsumeReader
        auto reader = stream_id_samples_->OpenCatReader(true);

        while (reader.HasNext()) {
            samples.push_back(reader.template Next<ValueType>());
        }

        // Find splitters
        std::sort(samples.begin(), samples.end(), compare_function_);

        size_t splitting_size = samples.size() / num_total_workers;

        // Send splitters to other workers
        for (size_t i = 1; i < num_total_workers; i++) {
            splitters.push_back(samples[i * splitting_size]);
            for (size_t j = 1; j < num_total_workers; j++) {
                emitters_samples_[j].Put(samples[i * splitting_size]);
            }
        }

        for (size_t j = 1; j < num_total_workers; j++) {
            emitters_samples_[j].Close();
        }
    }

    class TreeBuilder
    {
    public:
        ValueType* tree_;
        ValueType* samples_;
        size_t index_ = 0;
        size_t ssplitter_;

        /*!
         * Target: tree. Size of 'number of splitter'
         * Source: sorted splitters. Size of 'number of splitter'
         * Number of splitter
         */
        TreeBuilder(ValueType* splitter_tree,
                    ValueType* samples,
                    size_t ssplitter)
            : tree_(splitter_tree),
              samples_(samples),
              ssplitter_(ssplitter) {
            if (ssplitter != 0)
                recurse(samples, samples + ssplitter, 1);
        }

        void recurse(ValueType* lo, ValueType* hi, unsigned int treeidx) {

            // pick middle element as splitter
            ValueType* mid = lo + (ssize_t)(hi - lo) / 2;
            assert(mid < samples_ + ssplitter_);
            tree_[treeidx] = *mid;

            ValueType* midlo = mid, * midhi = mid + 1;

            if (2 * treeidx < ssplitter_)
            {
                recurse(lo, midlo, 2 * treeidx + 0);
                recurse(midhi, hi, 2 * treeidx + 1);
            }
        }
    };

    bool Equal(const ValueType& ele1, const ValueType& ele2) {
        return !(compare_function_(ele1, ele2) || compare_function_(ele2, ele1));
    }

    //! round n down by k where k is a power of two.
    template <typename Integral>
    static inline size_t RoundDown(Integral n, Integral k) {
        return (n & ~(k - 1));
    }

    void EmitToBuckets(
        // Tree of splitters, sizeof |splitter|
        const ValueType* const tree,
        // Number of buckets: k = 2^{log_k}
        size_t k,
        size_t log_k,
        // Number of actual workers to send to
        size_t actual_k,
        const ValueType* const sorted_splitters,
        size_t prefix_elem,
        size_t total_elem) {

        auto unsorted_reader_ = unsorted_data_->GetReader(/* consume */ true);

        // enlarge emitters array to next power of two to have direct access,
        // because we fill the splitter set up with sentinels == last splitter,
        // hence all items land in the last bucket.
        assert(emitters_data_.size() == actual_k);
        assert(actual_k <= k);

        while (emitters_data_.size() < k)
            emitters_data_.emplace_back(nullptr);

        std::swap(emitters_data_[actual_k - 1], emitters_data_[k - 1]);

        // classify all items (take two at once) and immediately transmit them.

        const size_t stepsize = 2;

        size_t i = 0;
        for ( ; i < RoundDown(total_items_, stepsize); i += stepsize)
        {
            // take two items
            size_t j0 = 1;
            const ValueType& el0 = unsorted_reader_.Next<ValueType>();

            size_t j1 = 1;
            const ValueType& el1 = unsorted_reader_.Next<ValueType>();

            // run items down the tree
            for (size_t l = 0; l < log_k; l++)
            {
                j0 = j0 * 2 + !(compare_function_(el0, tree[j0]));
                j1 = j1 * 2 + !(compare_function_(el1, tree[j1]));
            }

            size_t b0 = j0 - k;
            size_t b1 = j1 - k;

            if (b0 && Equal(el0, sorted_splitters[b0 - 1])) {
                while (b0 && Equal(el0, sorted_splitters[b0 - 1])
                       && (prefix_elem + i) * actual_k < b0 * total_elem) {
                    b0--;
                }

                if (b0 + 1 >= actual_k) {
                    b0 = k - 1;
                }
            }

            assert(emitters_data_[b0].IsValid());
            emitters_data_[b0].Put(el0);

            if (b1 && Equal(el1, sorted_splitters[b1 - 1])) {
                while (b1 && Equal(el1, sorted_splitters[b1 - 1])
                       && (prefix_elem + i + 1) * actual_k < b1 * total_elem) {
                    b1--;
                }

                if (b1 + 1 >= actual_k) {
                    b1 = k - 1;
                }
            }

            assert(emitters_data_[b1].IsValid());
            emitters_data_[b1].Put(el1);
        }

        // last iteration of loop if we have an odd number of items.
        for ( ; i < total_items_; i++)
        {

            const ValueType& last_item = unsorted_reader_.Next<ValueType>();
            size_t j = 1;
            for (size_t l = 0; l < log_k; l++)
            {
                j = j * 2 + !(compare_function_(last_item, tree[j]));
            }

            size_t b = j - k;

            while (b && Equal(last_item, sorted_splitters[b - 1])
                   && (prefix_elem + i) * actual_k < b * total_elem) {
                b--;
            }

            if (b + 1 >= actual_k) {
                b = k - 1;
            }

            assert(emitters_data_[b].IsValid());
            emitters_data_[b].Put(last_item);
        }
    }

    void MainOp() {
        unsorted_writer_.Close();
        net::FlowControlChannel& channel = context_.flow_control_channel();

        size_t prefix_elem = channel.PrefixSum(total_items_, (size_t)0, std::plus<size_t>(), false);
        size_t total_elem = channel.AllReduce(total_items_);

        size_t num_total_workers = context_.num_workers();

        LOG << "Local sample size on worker " << context_.my_rank() <<
            ": " << samples_.size();
        LOG << "Number of elements: " << total_items_;

        //Send all samples to worker 0.
        for (auto sample : samples_) {
            emitters_samples_[0].Put(sample);
        }
        emitters_samples_[0].Close();
        std::vector<ValueType>().swap(samples_);

        // Get the ceiling of log(num_total_workers), as SSSS needs 2^n buckets.
        size_t ceil_log = common::IntegerLog2Ceil(num_total_workers);
        size_t workers_algo = 1 << ceil_log;
        size_t splitter_count_algo = workers_algo - 1;

        std::vector<ValueType> splitters;
        splitters.reserve(workers_algo);

        if (context_.my_rank() == 0) {
            FindAndSendSplitters(splitters, samples_.size());
        }
        else {
            // Close unused emitters
            for (size_t j = 1; j < num_total_workers; j++) {
                emitters_samples_[j].Close();
            }
            bool consume = false;
            auto reader = stream_id_samples_->OpenCatReader(consume);
            while (reader.HasNext()) {
                splitters.push_back(reader.template Next<ValueType>());
            }
        }
        stream_id_samples_->Close();

        // code from SS2NPartition, slightly altered

        ValueType* splitter_tree = new ValueType[workers_algo + 1];

        // add sentinel splitters if fewer nodes than splitters.
        for (size_t i = num_total_workers; i < workers_algo; i++) {
            splitters.push_back(splitters.back());
        }

        TreeBuilder(splitter_tree,
                    splitters.data(),
                    splitter_count_algo);

        EmitToBuckets(
            splitter_tree, // Tree. sizeof |splitter|
            workers_algo,  // Number of buckets
            ceil_log,
            num_total_workers,
            splitters.data(),
            prefix_elem,
            total_elem);

        delete[] splitter_tree;

        // end of SS2N

        for (size_t i = 0; i < emitters_data_.size(); i++)
            emitters_data_[i].Close();

        data_.clear();

        bool consume = false;
        auto reader = stream_id_data_->OpenCatReader(consume);

        while (reader.HasNext()) {
            data_.push_back(reader.template Next<ValueType>());
        }
        stream_id_data_->Close();

        double balance = 0;
        if (data_.size() > 0) {
            balance = static_cast<double>(data_.size())
                      * static_cast<double>(num_total_workers)
                      / static_cast<double>(total_elem);
        }

        if (balance > 1) {
            balance = 1 / balance;
        }

        std::sort(data_.begin(), data_.end(), compare_function_);

        STATC << "NodeType" << "Sort"
              << "Workers" << num_total_workers
              << "LocalSize" << data_.size()
              << "Balance Factor" << balance
              << "Sample Size" << samples_.size();

        this->WriteStreamStats(stream_id_data_);
        this->WriteStreamStats(stream_id_samples_);
    }

    void PostOp() { }
};

template <typename ValueType, typename Stack>
template <typename CompareFunction>
auto DIA<ValueType, Stack>::Sort(const CompareFunction &compare_function) const {
    assert(IsValid());

    using SortNode = api::SortNode<ValueType, DIA, CompareFunction>;

    static_assert(
        std::is_convertible<
            ValueType,
            typename FunctionTraits<CompareFunction>::template arg<0>
            >::value ||
        std::is_same<CompareFunction, std::less<ValueType> >::value,
        "CompareFunction has the wrong input type");

    static_assert(
        std::is_convertible<
            ValueType,
            typename FunctionTraits<CompareFunction>::template arg<1> >::value ||
        std::is_same<CompareFunction, std::less<ValueType> >::value,
        "CompareFunction has the wrong input type");

    static_assert(
        std::is_convertible<
            typename FunctionTraits<CompareFunction>::result_type,
            bool>::value,
        "CompareFunction has the wrong output type (should be bool)");

    StatsNode* stats_node = AddChildStatsNode("Sort", DIANodeType::DOP);
    auto shared_node
        = std::make_shared<SortNode>(*this, compare_function, stats_node);

    return DIA<ValueType>(shared_node, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_SORT_HEADER

/******************************************************************************/
<|endoftext|>


<commit_before>/*
 * (C) 2004 British Broadcasting Corporation and Kamaelia Contributors(1)
 *     All Rights Reserved.
 *
 * You may only modify and redistribute this under the terms of any of the
 * following licenses(2): Mozilla Public License, V1.1, GNU General
 * Public License, V2.0, GNU Lesser General Public License, V2.1
 *
 * (1) Kamaelia Contributors are listed in the AUTHORS file and at
 *     http://kamaelia.sourceforge.net/AUTHORS - please extend this file,
 *     not this notice.
 * (2) Reproduced in the COPYING file, and at:
 *     http://kamaelia.sourceforge.net/COPYING
 * Under section 3.5 of the MPL, we are using this text since we deem the MPL
 * notice inappropriate for this file. As per MPL/GPL/LGPL removal of this
 * notice is prohibited.
 *
 * Please contact us via: kamaelia-list-owner@lists.sourceforge.net
 * to discuss alternative licensing.
 *
 * -------------------------------------------------------------------------
 *
 * This file contains a microcosm proof of concept implementation of a
 * subset of Axon & a simple producer consumer system using this.
 *
 * Compile:
 *  > g++ -o miniaxon miniaxon.cpp
 *
 * Run:
 *  > ./miniaxon
 *
 *
 * This is a rather icky implementation of Axon now, which has many of the
 * normal features of axon - the ability to link arbitrary outboxes to
 * arbitrary inboxes, multiple named inboxes/outboxes, arbitrary message
 * passing. (Subclass IPC_S, and place what you want in a .payload
 * attribute. (doesn't have to have a .payload though).
 *
 */

#include "generators.hpp"
#include <iostream>
#include <list>
#include <map>

using namespace std;

/* Base class for IPC objects */
struct IPC_S {
   virtual string __str__() { }; // Force polymorphic
   int payload;
};

struct Microprocess : public Generator {
    Microprocess()  {     };
    ~Microprocess() {     };
    virtual int next() {
    GENERATOR_CODE_START
        cout << "X Hello!" << endl;
        YIELD(-1);
    GENERATOR_CODE_END
    };
    virtual void run() {
    // This would not normally be run on a microprocess, but may be useful
    // for some circumstances, such as testing.
       int r;
       while(r!= -1) {
          r = next();
       }
    }
};

struct Scheduler : public Microprocess {
    list<Microprocess*> active;
    list<Microprocess*> newqueue;
    list<Microprocess*>::iterator current;

    int i;
    virtual int next() {
    GENERATOR_CODE_START
        i = 0;
        YIELD(1);
        for(i=0; i< 100; i++) {
           newqueue.resize(0);
           for(current=active.begin(); current != active.end(); current++) {
               int result;
               try {
                  result = (*current)->next();
                  if (result != -1) {
                     newqueue.push_back(*current);
                  };
               } catch (StopIteration null){
                  int k;
                  k =1;
               };
           }
           active.resize(0);  // Empty the active queue (unnecessary?)
           active = newqueue; // _Copies_ vector newqueue to active
           YIELD(1);
        };
        YIELD(-1);
    GENERATOR_CODE_END
    };
    void activateMicroprocess(Microprocess* m) {
        active.push_back(m);
    };
};

/* Simplificaton of Component, need to check what it's now missing
 */
struct SimpleComponent : public Microprocess {
    map<string,list<IPC_S*> > outboxes;
    map <string, list<IPC_S*> > inboxes;

    SimpleComponent()  {
       inboxes["inbox"];
       outboxes["outbox"];
    };
    ~SimpleComponent() {     };

    virtual int next() {
    GENERATOR_CODE_START
        cout << "SimpleComponent" << endl;
        YIELD(-1);
    GENERATOR_CODE_END
    };

    void send(IPC_S *value) {   send(value,"outbox"); }
    void send(IPC_S *value, string somebox) {
       outboxes[somebox].push_back(value);
    }

    IPC_S *recv() {   return recv("inbox"); }
    IPC_S *recv(string somebox) {
        IPC_S *result;
        result = *(inboxes[somebox].begin());
        inboxes[somebox].erase(inboxes[somebox].begin());
        return result;
    }

    void deliver(IPC_S *value, string somebox) {
       inboxes[somebox].push_back(value);
    }
    IPC_S *collect(string somebox) {
        IPC_S *result;
        result = *(outboxes[somebox].begin());
        outboxes[somebox].erase(outboxes[somebox].begin());
        return result;
    }
    bool dataReady() { return dataReady("inbox");    }
    bool dataReady(string somebox) {
       return ! inboxes[somebox].empty();
    }
    bool dataOutReady(string somebox) {
       return ! outboxes[somebox].empty();
    }
};

struct Postman : public SimpleComponent {
    SimpleComponent* source;
    string sourcebox;
    SimpleComponent* destination;
    string destbox;
    IPC_S * temp;
    Postman(SimpleComponent* s, SimpleComponent* d)  {
       source = s;
       destination = d;
       sourcebox = "outbox";
       destbox = "inbox";
    };
    Postman(SimpleComponent* s, string sbox, SimpleComponent* d, string dbox)  {
       source = s;
       destination = d;
       sourcebox = sbox;
       destbox = dbox;
    };
    ~Postman() {     };

    virtual int next() {
    GENERATOR_CODE_START
        while (1) {
           if (source->dataOutReady(sourcebox)) {
              temp = source->collect(sourcebox);
              destination->deliver(temp,destbox);
           };
           YIELD(1);
        };
    GENERATOR_CODE_END
    };
};

/**********************************************************************
 *
 * Sample Producer/Consumer system, passing IPC messages with a string as
 * payload.
 *
 */

struct mystring : public IPC_S{
   string payload;
};

struct Producer : public SimpleComponent {

    mystring msg;
    Producer()  {     };
    ~Producer() {     };

    virtual int next() {
    GENERATOR_CODE_START
        while(1) {
           msg.payload = "hello world!";
           send(&msg);
           YIELD(1);
        };
    GENERATOR_CODE_END
    };
};

struct Consumer : public SimpleComponent {
    mystring *result;

    Consumer()  {     };
    ~Consumer() {     };

    virtual int next() {
    GENERATOR_CODE_START
        while (1) {
            if (dataReady()) {
               result = dynamic_cast<mystring *>(recv());
               cout << "! " << result->payload << endl;
            };
            YIELD(1);
        };
    GENERATOR_CODE_END
    };
};

/*
 * End of sample system.
 *
 **********************************************************************/

int main(int, char **) {
   Scheduler scheduler;

   Producer P;
   Consumer C;
   Postman postie(&P,"outbox", &C, "inbox"); // Same as postie(&P,&C)
   scheduler.activateMicroprocess(&P);
   scheduler.activateMicroprocess(&C);
   scheduler.activateMicroprocess(&postie);
   scheduler.run();

   return 0;
}
<commit_msg>License change to Apache 2 license<commit_after>/*
 * Copyright 2010 British Broadcasting Corporation and Kamaelia Contributors(1)
 *
 * (1) Kamaelia Contributors are listed in the AUTHORS file and at
 *     http://www.kamaelia.org/AUTHORS - please extend this file,
 *     not this notice.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * -------------------------------------------------------------------------
 *
 * This file contains a microcosm proof of concept implementation of a
 * subset of Axon & a simple producer consumer system using this.
 *
 * Compile:
 *  > g++ -o miniaxon miniaxon.cpp
 *
 * Run:
 *  > ./miniaxon
 *
 *
 * This is a rather icky implementation of Axon now, which has many of the
 * normal features of axon - the ability to link arbitrary outboxes to
 * arbitrary inboxes, multiple named inboxes/outboxes, arbitrary message
 * passing. (Subclass IPC_S, and place what you want in a .payload
 * attribute. (doesn't have to have a .payload though).
 *
 */

#include "generators.hpp"
#include <iostream>
#include <list>
#include <map>

using namespace std;

/* Base class for IPC objects */
struct IPC_S {
   virtual string __str__() { }; // Force polymorphic
   int payload;
};

struct Microprocess : public Generator {
    Microprocess()  {     };
    ~Microprocess() {     };
    virtual int next() {
    GENERATOR_CODE_START
        cout << "X Hello!" << endl;
        YIELD(-1);
    GENERATOR_CODE_END
    };
    virtual void run() {
    // This would not normally be run on a microprocess, but may be useful
    // for some circumstances, such as testing.
       int r;
       while(r!= -1) {
          r = next();
       }
    }
};

struct Scheduler : public Microprocess {
    list<Microprocess*> active;
    list<Microprocess*> newqueue;
    list<Microprocess*>::iterator current;

    int i;
    virtual int next() {
    GENERATOR_CODE_START
        i = 0;
        YIELD(1);
        for(i=0; i< 100; i++) {
           newqueue.resize(0);
           for(current=active.begin(); current != active.end(); current++) {
               int result;
               try {
                  result = (*current)->next();
                  if (result != -1) {
                     newqueue.push_back(*current);
                  };
               } catch (StopIteration null){
                  int k;
                  k =1;
               };
           }
           active.resize(0);  // Empty the active queue (unnecessary?)
           active = newqueue; // _Copies_ vector newqueue to active
           YIELD(1);
        };
        YIELD(-1);
    GENERATOR_CODE_END
    };
    void activateMicroprocess(Microprocess* m) {
        active.push_back(m);
    };
};

/* Simplificaton of Component, need to check what it's now missing
 */
struct SimpleComponent : public Microprocess {
    map<string,list<IPC_S*> > outboxes;
    map <string, list<IPC_S*> > inboxes;

    SimpleComponent()  {
       inboxes["inbox"];
       outboxes["outbox"];
    };
    ~SimpleComponent() {     };

    virtual int next() {
    GENERATOR_CODE_START
        cout << "SimpleComponent" << endl;
        YIELD(-1);
    GENERATOR_CODE_END
    };

    void send(IPC_S *value) {   send(value,"outbox"); }
    void send(IPC_S *value, string somebox) {
       outboxes[somebox].push_back(value);
    }

    IPC_S *recv() {   return recv("inbox"); }
    IPC_S *recv(string somebox) {
        IPC_S *result;
        result = *(inboxes[somebox].begin());
        inboxes[somebox].erase(inboxes[somebox].begin());
        return result;
    }

    void deliver(IPC_S *value, string somebox) {
       inboxes[somebox].push_back(value);
    }
    IPC_S *collect(string somebox) {
        IPC_S *result;
        result = *(outboxes[somebox].begin());
        outboxes[somebox].erase(outboxes[somebox].begin());
        return result;
    }
    bool dataReady() { return dataReady("inbox");    }
    bool dataReady(string somebox) {
       return ! inboxes[somebox].empty();
    }
    bool dataOutReady(string somebox) {
       return ! outboxes[somebox].empty();
    }
};

struct Postman : public SimpleComponent {
    SimpleComponent* source;
    string sourcebox;
    SimpleComponent* destination;
    string destbox;
    IPC_S * temp;
    Postman(SimpleComponent* s, SimpleComponent* d)  {
       source = s;
       destination = d;
       sourcebox = "outbox";
       destbox = "inbox";
    };
    Postman(SimpleComponent* s, string sbox, SimpleComponent* d, string dbox)  {
       source = s;
       destination = d;
       sourcebox = sbox;
       destbox = dbox;
    };
    ~Postman() {     };

    virtual int next() {
    GENERATOR_CODE_START
        while (1) {
           if (source->dataOutReady(sourcebox)) {
              temp = source->collect(sourcebox);
              destination->deliver(temp,destbox);
           };
           YIELD(1);
        };
    GENERATOR_CODE_END
    };
};

/**********************************************************************
 *
 * Sample Producer/Consumer system, passing IPC messages with a string as
 * payload.
 *
 */

struct mystring : public IPC_S{
   string payload;
};

struct Producer : public SimpleComponent {

    mystring msg;
    Producer()  {     };
    ~Producer() {     };

    virtual int next() {
    GENERATOR_CODE_START
        while(1) {
           msg.payload = "hello world!";
           send(&msg);
           YIELD(1);
        };
    GENERATOR_CODE_END
    };
};

struct Consumer : public SimpleComponent {
    mystring *result;

    Consumer()  {     };
    ~Consumer() {     };

    virtual int next() {
    GENERATOR_CODE_START
        while (1) {
            if (dataReady()) {
               result = dynamic_cast<mystring *>(recv());
               cout << "! " << result->payload << endl;
            };
            YIELD(1);
        };
    GENERATOR_CODE_END
    };
};

/*
 * End of sample system.
 *
 **********************************************************************/

int main(int, char **) {
   Scheduler scheduler;

   Producer P;
   Consumer C;
   Postman postie(&P,"outbox", &C, "inbox"); // Same as postie(&P,&C)
   scheduler.activateMicroprocess(&P);
   scheduler.activateMicroprocess(&C);
   scheduler.activateMicroprocess(&postie);
   scheduler.run();

   return 0;
}
<|endoftext|>


<commit_before>/*=========================================================================
*
*  Copyright Insight Software Consortium
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
*         http://www.apache.org/licenses/LICENSE-2.0.txt
*
*  Unless required by applicable law or agreed to in writing, software
*  distributed under the License is distributed on an "AS IS" BASIS,
*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*  See the License for the specific language governing permissions and
*  limitations under the License.
*
*=========================================================================*/
#ifndef sitkImage_hxx
#define sitkImage_hxx

#include "sitkImage.h"

#include "itkImage.h"
#include "itkVectorImage.h"
#include "itkLabelMap.h"
#include "itkLabelObject.h"

#include "sitkExceptionObject.h"
#include "sitkPimpleImageBase.hxx"
#include "sitkPixelIDTypeLists.h"

namespace itk
{
  namespace simple
  {

  // this is a little specialization just to get the
  // InternalInitialization method's PixelIDTpImageType lookup to get
  // a valid void type, so it'll dispatch to the a specialized
  // method. All this is just to instantiate something that will never
  // be actually used.
  template <unsigned int VImageDimension>
  struct PixelIDToImageType< typelist::NullType , VImageDimension >
  {
    typedef void ImageType;
  };

  // This method is explicitly instantiated, and in-turn implicitly
  // instantates the PipleImage for all used image types. This method
  // just dispatces to nother method, to aid in instantiating only the
  // images requested.
  template <int VPixelIDValue, unsigned int VImageDimension>
  void Image::InternalInitialization( typename PixelIDToImageType<typename typelist::TypeAt<InstantiatedPixelIDTypeList,
                                                                                VPixelIDValue>::Result,
                                                                  VImageDimension>::ImageType *i )
  {
    this->ConditionalInternalInitialization<VPixelIDValue>( i );
  }

  template<int VPixelIDValue, typename TImageType>
  typename DisableIf<nsstd::is_same<TImageType, void>::value>::Type
  Image::ConditionalInternalInitialization( TImageType *image )
  {
    // no need to check if null
    delete this->m_PimpleImage;
    this->m_PimpleImage = SITK_NULLPTR;

    this->m_PimpleImage = new PimpleImage<TImageType>( image );
  }

  template<class TImageType>
  typename EnableIf<IsBasic<TImageType>::Value>::Type
  Image::AllocateInternal ( unsigned int Width, unsigned int Height, unsigned int Depth, unsigned int dim4, unsigned int numberOfComponents )
  {
    if ( numberOfComponents != 1  && numberOfComponents != 0 )
      {
      sitkExceptionMacro( "Specified number of components as " << numberOfComponents
                          << " but did not specify pixelID as a vector type!" );
      }

    typename TImageType::IndexType  index;
    typename TImageType::SizeType   size;
    typename TImageType::RegionType region;

    index.Fill ( 0 );
    size.Fill(1);
    size[0] = Width;
    size[1] = Height;

    if ( TImageType::ImageDimension > 2 )
      {
      assert( Depth != 0 );
      size[2] = Depth;
      }

    if ( TImageType::ImageDimension > 3 )
      {
      assert(  dim4 != 0 );
      size[3] =  dim4;
      }

    region.SetSize ( size );
    region.SetIndex ( index );

    typename TImageType::Pointer image = TImageType::New();
    image->SetRegions ( region );
    image->Allocate();
    image->FillBuffer ( itk::NumericTraits<typename TImageType::PixelType>::Zero );

    delete this->m_PimpleImage;
    this->m_PimpleImage = SITK_NULLPTR;
    m_PimpleImage =  new PimpleImage<TImageType>( image );
  }

  template<class TImageType>
  typename EnableIf<IsVector<TImageType>::Value>::Type
  Image::AllocateInternal ( unsigned int Width, unsigned int Height, unsigned int Depth, unsigned int dim4, unsigned int numberOfComponents )
  {
    if ( numberOfComponents == 0 )
      {
      numberOfComponents = TImageType::ImageDimension;
      }

    typename TImageType::IndexType  index;
    typename TImageType::SizeType   size;
    typename TImageType::RegionType region;
    typename TImageType::PixelType  zero;

    index.Fill ( 0 );
    size.Fill(1);
    size[0] = Width;
    size[1] = Height;

    if ( TImageType::ImageDimension > 2 )
      {
      assert( Depth != 0 );
      size[2] = Depth;
      }

    if ( TImageType::ImageDimension > 3 )
      {
      assert(  dim4 != 0 );
      size[3] =  dim4;
      }

    region.SetSize ( size );
    region.SetIndex ( index );

    zero.SetSize( numberOfComponents );
    zero.Fill ( itk::NumericTraits<typename TImageType::PixelType::ValueType>::Zero );

    typename TImageType::Pointer image = TImageType::New();
    image->SetRegions ( region );
    image->SetVectorLength( numberOfComponents );
    image->Allocate();
    image->FillBuffer ( zero );

    delete this->m_PimpleImage;
    this->m_PimpleImage = SITK_NULLPTR;

    m_PimpleImage = new PimpleImage<TImageType>( image );
  }

  template<class TImageType>
  typename EnableIf<IsLabel<TImageType>::Value>::Type
  Image::AllocateInternal ( unsigned int Width, unsigned int Height, unsigned int Depth, unsigned int dim4, unsigned int numberOfComponents )
  {
    if ( numberOfComponents != 1 && numberOfComponents != 0 )
      {
      sitkExceptionMacro( "Specified number of components as " << numberOfComponents
                          << " but did not specify pixelID as a vector type!" );
      }

    typename TImageType::IndexType  index;
    typename TImageType::SizeType   size;
    typename TImageType::RegionType region;

    index.Fill ( 0 );
    size.Fill(1);
    size[0] = Width;
    size[1] = Height;

    if ( TImageType::ImageDimension > 2 )
      {
      assert( Depth != 0 );
      size[2] = Depth;
      }

    if ( TImageType::ImageDimension > 3 )
      {
      assert(  dim4 != 0 );
      size[3] =  dim4;
      }

    region.SetSize ( size );
    region.SetIndex ( index );

    typename TImageType::Pointer image = TImageType::New();
    image->SetRegions ( region );
    image->Allocate();
    image->SetBackgroundValue( 0 );

    delete this->m_PimpleImage;
    this->m_PimpleImage = SITK_NULLPTR;

    m_PimpleImage = new PimpleImage<TImageType>( image );
  }

  }
}

#endif // sitkImage_h
<commit_msg>Prefer using ZeroValue() function of Zero variable.<commit_after>/*=========================================================================
*
*  Copyright Insight Software Consortium
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
*         http://www.apache.org/licenses/LICENSE-2.0.txt
*
*  Unless required by applicable law or agreed to in writing, software
*  distributed under the License is distributed on an "AS IS" BASIS,
*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*  See the License for the specific language governing permissions and
*  limitations under the License.
*
*=========================================================================*/
#ifndef sitkImage_hxx
#define sitkImage_hxx

#include "sitkImage.h"

#include "itkImage.h"
#include "itkVectorImage.h"
#include "itkLabelMap.h"
#include "itkLabelObject.h"

#include "sitkExceptionObject.h"
#include "sitkPimpleImageBase.hxx"
#include "sitkPixelIDTypeLists.h"

namespace itk
{
  namespace simple
  {

  // this is a little specialization just to get the
  // InternalInitialization method's PixelIDTpImageType lookup to get
  // a valid void type, so it'll dispatch to the a specialized
  // method. All this is just to instantiate something that will never
  // be actually used.
  template <unsigned int VImageDimension>
  struct PixelIDToImageType< typelist::NullType , VImageDimension >
  {
    typedef void ImageType;
  };

  // This method is explicitly instantiated, and in-turn implicitly
  // instantates the PipleImage for all used image types. This method
  // just dispatces to nother method, to aid in instantiating only the
  // images requested.
  template <int VPixelIDValue, unsigned int VImageDimension>
  void Image::InternalInitialization( typename PixelIDToImageType<typename typelist::TypeAt<InstantiatedPixelIDTypeList,
                                                                                VPixelIDValue>::Result,
                                                                  VImageDimension>::ImageType *i )
  {
    this->ConditionalInternalInitialization<VPixelIDValue>( i );
  }

  template<int VPixelIDValue, typename TImageType>
  typename DisableIf<nsstd::is_same<TImageType, void>::value>::Type
  Image::ConditionalInternalInitialization( TImageType *image )
  {
    // no need to check if null
    delete this->m_PimpleImage;
    this->m_PimpleImage = SITK_NULLPTR;

    this->m_PimpleImage = new PimpleImage<TImageType>( image );
  }

  template<class TImageType>
  typename EnableIf<IsBasic<TImageType>::Value>::Type
  Image::AllocateInternal ( unsigned int Width, unsigned int Height, unsigned int Depth, unsigned int dim4, unsigned int numberOfComponents )
  {
    if ( numberOfComponents != 1  && numberOfComponents != 0 )
      {
      sitkExceptionMacro( "Specified number of components as " << numberOfComponents
                          << " but did not specify pixelID as a vector type!" );
      }

    typename TImageType::IndexType  index;
    typename TImageType::SizeType   size;
    typename TImageType::RegionType region;

    index.Fill ( 0 );
    size.Fill(1);
    size[0] = Width;
    size[1] = Height;

    if ( TImageType::ImageDimension > 2 )
      {
      assert( Depth != 0 );
      size[2] = Depth;
      }

    if ( TImageType::ImageDimension > 3 )
      {
      assert(  dim4 != 0 );
      size[3] =  dim4;
      }

    region.SetSize ( size );
    region.SetIndex ( index );

    typename TImageType::Pointer image = TImageType::New();
    image->SetRegions ( region );
    image->Allocate();
    image->FillBuffer ( itk::NumericTraits<typename TImageType::PixelType>::ZeroValue() );

    delete this->m_PimpleImage;
    this->m_PimpleImage = SITK_NULLPTR;
    m_PimpleImage =  new PimpleImage<TImageType>( image );
  }

  template<class TImageType>
  typename EnableIf<IsVector<TImageType>::Value>::Type
  Image::AllocateInternal ( unsigned int Width, unsigned int Height, unsigned int Depth, unsigned int dim4, unsigned int numberOfComponents )
  {
    if ( numberOfComponents == 0 )
      {
      numberOfComponents = TImageType::ImageDimension;
      }

    typename TImageType::IndexType  index;
    typename TImageType::SizeType   size;
    typename TImageType::RegionType region;
    typename TImageType::PixelType  zero;

    index.Fill ( 0 );
    size.Fill(1);
    size[0] = Width;
    size[1] = Height;

    if ( TImageType::ImageDimension > 2 )
      {
      assert( Depth != 0 );
      size[2] = Depth;
      }

    if ( TImageType::ImageDimension > 3 )
      {
      assert(  dim4 != 0 );
      size[3] =  dim4;
      }

    region.SetSize ( size );
    region.SetIndex ( index );

    zero.SetSize( numberOfComponents );
    zero.Fill ( itk::NumericTraits<typename TImageType::PixelType::ValueType>::Zero );

    typename TImageType::Pointer image = TImageType::New();
    image->SetRegions ( region );
    image->SetVectorLength( numberOfComponents );
    image->Allocate();
    image->FillBuffer ( zero );

    delete this->m_PimpleImage;
    this->m_PimpleImage = SITK_NULLPTR;

    m_PimpleImage = new PimpleImage<TImageType>( image );
  }

  template<class TImageType>
  typename EnableIf<IsLabel<TImageType>::Value>::Type
  Image::AllocateInternal ( unsigned int Width, unsigned int Height, unsigned int Depth, unsigned int dim4, unsigned int numberOfComponents )
  {
    if ( numberOfComponents != 1 && numberOfComponents != 0 )
      {
      sitkExceptionMacro( "Specified number of components as " << numberOfComponents
                          << " but did not specify pixelID as a vector type!" );
      }

    typename TImageType::IndexType  index;
    typename TImageType::SizeType   size;
    typename TImageType::RegionType region;

    index.Fill ( 0 );
    size.Fill(1);
    size[0] = Width;
    size[1] = Height;

    if ( TImageType::ImageDimension > 2 )
      {
      assert( Depth != 0 );
      size[2] = Depth;
      }

    if ( TImageType::ImageDimension > 3 )
      {
      assert(  dim4 != 0 );
      size[3] =  dim4;
      }

    region.SetSize ( size );
    region.SetIndex ( index );

    typename TImageType::Pointer image = TImageType::New();
    image->SetRegions ( region );
    image->Allocate();
    image->SetBackgroundValue( 0 );

    delete this->m_PimpleImage;
    this->m_PimpleImage = SITK_NULLPTR;

    m_PimpleImage = new PimpleImage<TImageType>( image );
  }

  }
}

#endif // sitkImage_h
<|endoftext|>


<commit_before>// Copyright 2016 The University of North Carolina at Chapel Hill
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Please send all BUG REPORTS to <pavel@cs.unc.edu>.
// <http://gamma.cs.unc.edu/FasTC/>

#include "FasTC/DXTCompressor.h"

#include <cassert>
#include <cstdlib>
#include <cstring>

#include "FasTC/Pixel.h"

namespace DXTC
{
  void DecompressDXT1Block(const uint8 *block, uint32 *outBuf) {
    // When we call FasTC::Pixel::FromBits, we expect the bits
    // to be read out of memory in LSB (byte) order first. Hence,
    // we can't read the blocks directly as uint16 values out of
    // the DXT buffer and we have to swap the bytes before hand.
    uint16 colorA = block[0];
    colorA <<= 8;
    colorA |= block[1];

    uint16 colorB = block[2];
    colorB <<= 8;
    colorB |= block[3];

    uint32 mod = reinterpret_cast<const uint32 *>(block + 4)[0];

    uint8 kFiveSixFive[4] = { 0, 5, 6, 5 };
    FasTC::Pixel a, b, c, d;
    a.FromBits(reinterpret_cast<const uint8 *>(&colorA), kFiveSixFive);
    b.FromBits(reinterpret_cast<const uint8 *>(&colorB), kFiveSixFive);

    uint8 kFullDepth[4] = {8, 8, 8, 8};
    a.ChangeBitDepth(kFullDepth);
    b.ChangeBitDepth(kFullDepth);

    d = (a + b*2) / 3;
    c = (a*2 + b) / 3;

    FasTC::Pixel *colors[4] = { &a, &b, &c, &d };

    uint32 *outPixels = reinterpret_cast<uint32 *>(outBuf);
    for(uint32 i = 0; i < 16; i++) {
      outPixels[i] = colors[(mod >> (i*2)) & 3]->Pack();
    }
  }

  void DecompressDXT1(const FasTC::DecompressionJob &dcj)
  {
    assert(!(dcj.Height() & 3));
    assert(!(dcj.Width() & 3));

    uint32 blockW = dcj.Width() >> 2;
    uint32 blockH = dcj.Height() >> 2;

    const uint32 blockSz = GetBlockSize(FasTC::eCompressionFormat_DXT1);

    uint32 *outPixels = reinterpret_cast<uint32 *>(dcj.OutBuf());

    uint32 outBlock[16];
    for(uint32 j = 0; j < blockH; j++) {
      for(uint32 i = 0; i < blockW; i++) {

        uint32 offset = (j * blockW + i) * blockSz;
        DecompressDXT1Block(dcj.InBuf() + offset, outBlock);

        for(uint32 y = 0; y < 4; y++)
        for(uint32 x = 0; x < 4; x++) {
          offset = (j*4 + y)*dcj.Width() + ((i*4)+x);
          outPixels[offset] = outBlock[y*4 + x];
        }
      }
    }
  }
}
<commit_msg>Fix bugs where we wouldn't properly decode punchthrough alpha in DXT<commit_after>// Copyright 2016 The University of North Carolina at Chapel Hill
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Please send all BUG REPORTS to <pavel@cs.unc.edu>.
// <http://gamma.cs.unc.edu/FasTC/>

#include "FasTC/DXTCompressor.h"

#include <cassert>
#include <cstdlib>
#include <cstring>

#include "FasTC/Pixel.h"

namespace {
  void DecompressDXT1Block(const uint8 *block, uint32 *outBuf, bool check_order) {
    // When we call FasTC::Pixel::FromBits, we expect the bits
    // to be read out of memory in LSB (byte) order first. Hence,
    // we can't read the blocks directly as uint16 values out of
    // the DXT buffer and we have to swap the bytes before hand.
    uint16 colorA = block[1] | block[0] << 8;
    uint16 colorB = block[3] | block[2] << 8;

    uint32 mod = reinterpret_cast<const uint32 *>(block + 4)[0];

    uint8 kFiveSixFive[4] = { 0, 5, 6, 5 };
    FasTC::Pixel a, b, c, d;
    a.FromBits(reinterpret_cast<const uint8 *>(&colorA), kFiveSixFive);
    b.FromBits(reinterpret_cast<const uint8 *>(&colorB), kFiveSixFive);

    uint8 kFullDepth[4] = { 8, 8, 8, 8 };
    a.ChangeBitDepth(kFullDepth);
    b.ChangeBitDepth(kFullDepth);

    // However, for the purposes of properly decoding DXT, we can read them as ints...
    const uint16 *block_ptr = reinterpret_cast<const uint16 *>(block);
    if (!check_order || block_ptr[0] > block_ptr[1]) {
      c = (a * 2 + b) / 3;
      d = (a + b * 2) / 3;
    }
    else {
      c = (a + b) / 2;
      // d already initialized to zero...
    }

    FasTC::Pixel *colors[4] = { &a, &b, &c, &d };

    uint32 *outPixels = reinterpret_cast<uint32 *>(outBuf);
    for(uint32 i = 0; i < 16; i++) {
      outPixels[i] = colors[(mod >> (i*2)) & 3]->Pack();
    }
  }

}  // namespace

namespace DXTC
{
  void DecompressDXT1(const FasTC::DecompressionJob &dcj)
  {
    assert(!(dcj.Height() & 3));
    assert(!(dcj.Width() & 3));

    uint32 blockW = dcj.Width() >> 2;
    uint32 blockH = dcj.Height() >> 2;

    const uint32 blockSz = GetBlockSize(FasTC::eCompressionFormat_DXT1);

    uint32 *outPixels = reinterpret_cast<uint32 *>(dcj.OutBuf());

    uint32 outBlock[16];
    memset(outBlock, 0xFF, sizeof(outBlock));

    for(uint32 j = 0; j < blockH; j++) {
      for(uint32 i = 0; i < blockW; i++) {

        uint32 offset = (j * blockW + i) * blockSz;
        DecompressDXT1Block(dcj.InBuf() + offset, outBlock, true);

        for(uint32 y = 0; y < 4; y++)
        for(uint32 x = 0; x < 4; x++) {
          offset = (j*4 + y)*dcj.Width() + ((i*4)+x);
          outPixels[offset] = outBlock[y*4 + x];
        }
      }
    }
  }
}
<|endoftext|>


<commit_before>/*++

Module Name:

    driver.c

Abstract:

    This file contains the driver entry points and callbacks.

Environment:

    Kernel-mode Driver Framework

--*/

#define INITGUID

#include "Precomp.h"

#include "driver.h"

#include <ktl\scope.h>

#include "Public.h"

// {27d1e96e-78c4-41d7-aba4-9f9ca3379a15}
TRACELOGGING_DEFINE_PROVIDER(
    OSRDriverTraceProvider,
    "OSRTraceProvider",
    (0x27d1e96e, 0x78c4, 0x41d7, 0xab, 0xa4, 0x9f, 0x9c, 0xa3, 0x37, 0x9a, 0x15));

PASSIVE INIT_CODE NTSTATUS DriverEntry(
    _In_ PDRIVER_OBJECT  DriverObject,
    _In_ PUNICODE_STRING RegistryPath)
/*++

Routine Description:
    DriverEntry initializes the driver and is the first routine called by the
    system after the driver is loaded. DriverEntry specifies the other entry
    points in the function driver, such as EvtDevice and DriverUnload.

Parameters Description:

    DriverObject - represents the instance of the function driver that is loaded
    into memory. DriverEntry must initialize members of DriverObject before it
    returns to the caller. DriverObject is allocated by the system before the
    driver is loaded, and it is released by the system after the system unloads
    the function driver from memory.

    RegistryPath - represents the driver specific path in the Registry.
    The function driver can use the path to store driver related data between
    reboots. The path does not store hardware instance specific data.

Return Value:

    STATUS_SUCCESS if successful,
    STATUS_UNSUCCESSFUL otherwise.

--*/
{
    // Initialize TraceLogging
    TraceLoggingRegister(OSRDriverTraceProvider);

    auto unregisterLoggingOnFailure = ktl::make_scope_exit_paged([]() PAGED {
        TraceLoggingUnregister(OSRDriverTraceProvider);
    });

    OSRLogEntry();

    //
    // Register a cleanup callback so that we can call WPP_CLEANUP when
    // the framework driver object is deleted during driver unload.
    //

    WDF_OBJECT_ATTRIBUTES attributes;
    WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
    attributes.EvtCleanupCallback = DriverEvtDriverContextCleanup;

    WDF_DRIVER_CONFIG config;
    WDF_DRIVER_CONFIG_INIT(&config, DriverEvtDeviceAdd);

    RETURN_IF_NT_FAILED(
        WdfDriverCreate(DriverObject, RegistryPath, &attributes, &config, WDF_NO_HANDLE));

    unregisterLoggingOnFailure.Dismiss();

    OSRLogExit();

    return STATUS_SUCCESS;
}

PASSIVE PAGED NTSTATUS DriverEvtDeviceAdd(
    _In_    WDFDRIVER       Driver,
    _Inout_ PWDFDEVICE_INIT DeviceInit)
/*++
Routine Description:

    EvtDeviceAdd is called by the framework in response to AddDevice
    call from the PnP manager. We create and initialize a device object to
    represent a new instance of the device.

Arguments:

    Driver - Handle to a framework driver object created in DriverEntry

    DeviceInit - Pointer to a framework-allocated WDFDEVICE_INIT structure.

Return Value:

    NTSTATUS

--*/
{
    UNREFERENCED_PARAMETER(Driver);

    PAGED_CODE();

    OSRLogEntry();

    RETURN_IF_NT_FAILED(DriverCreateDevice(DeviceInit));

    OSRLogExit();

    return STATUS_SUCCESS;
}

PASSIVE PAGED VOID DriverEvtDriverContextCleanup(
    _In_ WDFOBJECT DriverObject)
/*++
Routine Description:

    Free all the resources allocated in DriverEntry.

Arguments:

    DriverObject - handle to a WDF Driver object.

Return Value:

    VOID.

--*/
{
    UNREFERENCED_PARAMETER(DriverObject);

    PAGED_CODE ();

    OSRLogEntry();

    TraceLoggingUnregister(OSRDriverTraceProvider);
}
<commit_msg>TraceLoggingUnregister on DriverEntry failure: PAGED -> INIT_CODE<commit_after>/*++

Module Name:

    driver.c

Abstract:

    This file contains the driver entry points and callbacks.

Environment:

    Kernel-mode Driver Framework

--*/

#define INITGUID

#include "Precomp.h"

#include "driver.h"

#include <ktl\scope.h>

#include "Public.h"

// {27d1e96e-78c4-41d7-aba4-9f9ca3379a15}
TRACELOGGING_DEFINE_PROVIDER(
    OSRDriverTraceProvider,
    "OSRTraceProvider",
    (0x27d1e96e, 0x78c4, 0x41d7, 0xab, 0xa4, 0x9f, 0x9c, 0xa3, 0x37, 0x9a, 0x15));

PASSIVE INIT_CODE NTSTATUS DriverEntry(
    _In_ PDRIVER_OBJECT  DriverObject,
    _In_ PUNICODE_STRING RegistryPath)
/*++

Routine Description:
    DriverEntry initializes the driver and is the first routine called by the
    system after the driver is loaded. DriverEntry specifies the other entry
    points in the function driver, such as EvtDevice and DriverUnload.

Parameters Description:

    DriverObject - represents the instance of the function driver that is loaded
    into memory. DriverEntry must initialize members of DriverObject before it
    returns to the caller. DriverObject is allocated by the system before the
    driver is loaded, and it is released by the system after the system unloads
    the function driver from memory.

    RegistryPath - represents the driver specific path in the Registry.
    The function driver can use the path to store driver related data between
    reboots. The path does not store hardware instance specific data.

Return Value:

    STATUS_SUCCESS if successful,
    STATUS_UNSUCCESSFUL otherwise.

--*/
{
    // Initialize TraceLogging
    TraceLoggingRegister(OSRDriverTraceProvider);

    auto unregisterLoggingOnFailure = ktl::make_scope_exit_paged([]() INIT_CODE {
        TraceLoggingUnregister(OSRDriverTraceProvider);
    });

    OSRLogEntry();

    //
    // Register a cleanup callback so that we can call WPP_CLEANUP when
    // the framework driver object is deleted during driver unload.
    //

    WDF_OBJECT_ATTRIBUTES attributes;
    WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
    attributes.EvtCleanupCallback = DriverEvtDriverContextCleanup;

    WDF_DRIVER_CONFIG config;
    WDF_DRIVER_CONFIG_INIT(&config, DriverEvtDeviceAdd);

    RETURN_IF_NT_FAILED(
        WdfDriverCreate(DriverObject, RegistryPath, &attributes, &config, WDF_NO_HANDLE));

    unregisterLoggingOnFailure.Dismiss();

    OSRLogExit();

    return STATUS_SUCCESS;
}

PASSIVE PAGED NTSTATUS DriverEvtDeviceAdd(
    _In_    WDFDRIVER       Driver,
    _Inout_ PWDFDEVICE_INIT DeviceInit)
/*++
Routine Description:

    EvtDeviceAdd is called by the framework in response to AddDevice
    call from the PnP manager. We create and initialize a device object to
    represent a new instance of the device.

Arguments:

    Driver - Handle to a framework driver object created in DriverEntry

    DeviceInit - Pointer to a framework-allocated WDFDEVICE_INIT structure.

Return Value:

    NTSTATUS

--*/
{
    UNREFERENCED_PARAMETER(Driver);

    PAGED_CODE();

    OSRLogEntry();

    RETURN_IF_NT_FAILED(DriverCreateDevice(DeviceInit));

    OSRLogExit();

    return STATUS_SUCCESS;
}

PASSIVE PAGED VOID DriverEvtDriverContextCleanup(
    _In_ WDFOBJECT DriverObject)
/*++
Routine Description:

    Free all the resources allocated in DriverEntry.

Arguments:

    DriverObject - handle to a WDF Driver object.

Return Value:

    VOID.

--*/
{
    UNREFERENCED_PARAMETER(DriverObject);

    PAGED_CODE ();

    OSRLogEntry();

    TraceLoggingUnregister(OSRDriverTraceProvider);
}
<|endoftext|>