<commit_before>#include "system/env.h"
#include "system/van.h"
#include "base/common.h"
#include "base/local_machine.h"
#include "base/file.h"
// #include "data/common.h"
#include "proto/node.pb.h"
namespace ps {

DEFINE_int32(num_servers, 0, "number of servers");
DEFINE_int32(num_workers, 0, "number of clients");
DEFINE_int32(num_threads, 2, "number of computational threads");
DEFINE_int32(num_replicas, 0, "number of replicas per server node");

DEFINE_string(my_node, "role:SCHEDULER,hostname:'127.0.0.1',port:8000,id:'H'", "my node");
DEFINE_string(scheduler, "role:SCHEDULER,hostname:'127.0.0.1',port:8000,id:'H'", "the scheduler node");
DEFINE_int32(my_rank, -1, "my rank among MPI peers");
DEFINE_string(interface, "", "network interface");

void Env::Init(char* argv0) {
  if (getenv("DMLC_PS_ROOT_URI")) {
    InitDMLC();
  }

  InitGlog(argv0);
}

void Env::InitGlog(char* argv0) {
  if (FLAGS_log_dir.empty()) FLAGS_log_dir = "/tmp";
  if (!dirExists(FLAGS_log_dir)) { createDir(FLAGS_log_dir); }
  // change the hostname in default log filename to node id
  string logfile = FLAGS_log_dir + "/" + string(basename(argv0)) + "."
                   + Van::ParseNode(FLAGS_my_node).id() + ".log.";
  google::SetLogDestination(google::INFO, (logfile+"INFO.").c_str());
  google::SetLogDestination(google::WARNING, (logfile+"WARNING.").c_str());
  google::SetLogDestination(google::ERROR, (logfile+"ERROR.").c_str());
  google::SetLogDestination(google::FATAL, (logfile+"FATAL.").c_str());
  google::SetLogSymlink(google::INFO, "");
  google::SetLogSymlink(google::WARNING, "");
  google::SetLogSymlink(google::ERROR, "");
  google::SetLogSymlink(google::FATAL, "");
  FLAGS_logbuflevel = -1;
}

void Env::InitDMLC() {
  Node scheduler;
  scheduler.set_hostname(std::string(CHECK_NOTNULL(getenv("DMLC_PS_ROOT_URI"))));
  scheduler.set_port(atoi(CHECK_NOTNULL(getenv("DMLC_PS_ROOT_PORT"))));
  scheduler.set_role(Node::SCHEDULER);
  scheduler.set_id("H");
  CHECK(google::protobuf::TextFormat::PrintToString(scheduler, &FLAGS_scheduler));

  FLAGS_num_workers = atoi(CHECK_NOTNULL(getenv("DMLC_NUM_WORKER")));
  FLAGS_num_servers = atoi(CHECK_NOTNULL(getenv("DMLC_NUM_SERVER")));

  AssembleMyNode();
}

void Env::AssembleMyNode() {
  // get my role
  char* role_str = getenv("DMLC_ROLE");
  Node::Role role;
  string id;
  if (role_str != NULL) {
    if (!strcmp(role_str, "scheduler")) {
      FLAGS_my_node = FLAGS_scheduler;
      return;
    } else if (!strcmp(role_str, "worker")) {
      role = Node::WORKER;
    } else if (!strcmp(role_str, "server")) {
      role = Node::SERVER;
    } else {
      LOG(FATAL) << "unknown role: " << *role_str;
    }
  } else {
    int my_rank;
    char* rank_str = getenv("PMI_RANK");
    if (!rank_str) {
      rank_str = getenv("OMPI_COMM_WORLD_RANK");
    }
    CHECK(rank_str != NULL) << "fail to get rank size";
    my_rank = atoi(rank_str);

    // role and id
    CHECK_LT(my_rank, FLAGS_num_workers + FLAGS_num_servers);
    if (my_rank < FLAGS_num_workers) {
      role = Node::WORKER;
      id = "W" + std::to_string(my_rank);
    } else { // if (my_rank < FLAGS_num_workers + FLAGS_num_servers) {
      role = Node::SERVER;
      id = "S" + std::to_string(FLAGS_my_rank - FLAGS_num_workers);
    }
  }

  Node node; node.set_role(role);
  if (!id.empty()) node.set_id(id);

  // IP, port and interface
  string ip;
  string interface = FLAGS_interface;
  unsigned short port;

  if (interface.empty()) {
    LocalMachine::pickupAvailableInterfaceAndIP(interface, ip);
  } else {
    ip = LocalMachine::IP(interface);
  }
  CHECK(!ip.empty()) << "failed to got ip";
  CHECK(!interface.empty()) << "failed to got the interface";
  port = LocalMachine::pickupAvailablePort();
  CHECK_NE(port, 0) << "failed to get port";
  node.set_hostname(ip);
  node.set_port(static_cast<int32>(port));

  CHECK(google::protobuf::TextFormat::PrintToString(node, &FLAGS_my_node));
}

}  // namespace ps
<commit_msg>fix bug in env when launching using dmlc-*.py with multiple servers<commit_after>#include "system/env.h"
#include "system/van.h"
#include "base/common.h"
#include "base/local_machine.h"
#include "base/file.h"
// #include "data/common.h"
#include "proto/node.pb.h"
namespace ps {

DEFINE_int32(num_servers, 0, "number of servers");
DEFINE_int32(num_workers, 0, "number of clients");
DEFINE_int32(num_threads, 2, "number of computational threads");
DEFINE_int32(num_replicas, 0, "number of replicas per server node");

DEFINE_string(my_node, "role:SCHEDULER,hostname:'127.0.0.1',port:8000,id:'H'", "my node");
DEFINE_string(scheduler, "role:SCHEDULER,hostname:'127.0.0.1',port:8000,id:'H'", "the scheduler node");
DEFINE_int32(my_rank, -1, "my rank among MPI peers");
DEFINE_string(interface, "", "network interface");

void Env::Init(char* argv0) {
  if (getenv("DMLC_PS_ROOT_URI")) {
    InitDMLC();
  }

  InitGlog(argv0);
}

void Env::InitGlog(char* argv0) {
  if (FLAGS_log_dir.empty()) FLAGS_log_dir = "/tmp";
  if (!dirExists(FLAGS_log_dir)) { createDir(FLAGS_log_dir); }
  // change the hostname in default log filename to node id
  string logfile = FLAGS_log_dir + "/" + string(basename(argv0)) + "."
                   + Van::ParseNode(FLAGS_my_node).id() + ".log.";
  google::SetLogDestination(google::INFO, (logfile+"INFO.").c_str());
  google::SetLogDestination(google::WARNING, (logfile+"WARNING.").c_str());
  google::SetLogDestination(google::ERROR, (logfile+"ERROR.").c_str());
  google::SetLogDestination(google::FATAL, (logfile+"FATAL.").c_str());
  google::SetLogSymlink(google::INFO, "");
  google::SetLogSymlink(google::WARNING, "");
  google::SetLogSymlink(google::ERROR, "");
  google::SetLogSymlink(google::FATAL, "");
  FLAGS_logbuflevel = -1;
}

void Env::InitDMLC() {
  Node scheduler;
  scheduler.set_hostname(std::string(CHECK_NOTNULL(getenv("DMLC_PS_ROOT_URI"))));
  scheduler.set_port(atoi(CHECK_NOTNULL(getenv("DMLC_PS_ROOT_PORT"))));
  scheduler.set_role(Node::SCHEDULER);
  scheduler.set_id("H");
  CHECK(google::protobuf::TextFormat::PrintToString(scheduler, &FLAGS_scheduler));

  FLAGS_num_workers = atoi(CHECK_NOTNULL(getenv("DMLC_NUM_WORKER")));
  FLAGS_num_servers = atoi(CHECK_NOTNULL(getenv("DMLC_NUM_SERVER")));

  AssembleMyNode();
}

void Env::AssembleMyNode() {
  // get my role
  char* role_str = getenv("DMLC_ROLE");
  Node::Role role;
  string id;
  if (role_str != NULL) {
    if (!strcmp(role_str, "scheduler")) {
      FLAGS_my_node = FLAGS_scheduler;
      return;
    } else if (!strcmp(role_str, "worker")) {
      role = Node::WORKER;
    } else if (!strcmp(role_str, "server")) {
      role = Node::SERVER;
    } else {
      LOG(FATAL) << "unknown role: " << *role_str;
    }
  } else {
    int my_rank;
    char* rank_str = getenv("PMI_RANK");
    if (!rank_str) {
      rank_str = getenv("OMPI_COMM_WORLD_RANK");
    }
    CHECK(rank_str != NULL) << "fail to get rank size";
    my_rank = atoi(rank_str);

    // role and id
    CHECK_LT(my_rank, FLAGS_num_workers + FLAGS_num_servers);
    if (my_rank < FLAGS_num_workers) {
      role = Node::WORKER;
      id = "W" + std::to_string(my_rank);
    } else { // if (my_rank < FLAGS_num_workers + FLAGS_num_servers) {
      role = Node::SERVER;
      id = "S" + std::to_string(my_rank - FLAGS_num_workers);
    }
  }

  Node node; node.set_role(role);
  if (!id.empty()) node.set_id(id);

  // IP, port and interface
  string ip;
  string interface = FLAGS_interface;
  unsigned short port;

  if (interface.empty()) {
    LocalMachine::pickupAvailableInterfaceAndIP(interface, ip);
  } else {
    ip = LocalMachine::IP(interface);
  }
  CHECK(!ip.empty()) << "failed to got ip";
  CHECK(!interface.empty()) << "failed to got the interface";
  port = LocalMachine::pickupAvailablePort();
  CHECK_NE(port, 0) << "failed to get port";
  node.set_hostname(ip);
  node.set_port(static_cast<int32>(port));

  CHECK(google::protobuf::TextFormat::PrintToString(node, &FLAGS_my_node));
}

}  // namespace ps
<|endoftext|>


<commit_before>/*
 * TCP client connection pooling.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#ifndef BENG_PROXY_TCP_STOCK_HXX
#define BENG_PROXY_TCP_STOCK_HXX

struct pool;
struct balancer;
struct StockMap;
struct StockItem;
struct StockGetHandler;
struct async_operation_ref;
class SocketAddress;

/**
 * Creates a new TCP connection stock.
 *
 * @param pool the memory pool
 * @param limit the maximum number of connections per host
 * @return the new TCP connections stock (this function cannot fail)
 */
StockMap *
tcp_stock_new(struct pool *pool, unsigned limit);

/**
 * @param name the hstock name; it is auto-generated from the
 * #address_list if NULL is passed here
 * @param timeout the connect timeout in seconds
 */
void
tcp_stock_get(StockMap *tcp_stock, struct pool *pool, const char *name,
              bool ip_transparent,
              SocketAddress bind_address,
              SocketAddress address,
              unsigned timeout,
              const StockGetHandler *handler, void *handler_ctx,
              struct async_operation_ref *async_ref);

void
tcp_stock_put(StockMap *tcp_stock, StockItem &item, bool destroy);

int
tcp_stock_item_get(const StockItem &item);

int
tcp_stock_item_get_domain(const StockItem &item);

#endif
<commit_msg>tcp_stock: add "pure" attributes<commit_after>/*
 * TCP client connection pooling.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#ifndef BENG_PROXY_TCP_STOCK_HXX
#define BENG_PROXY_TCP_STOCK_HXX

#include <inline/compiler.h>

struct pool;
struct balancer;
struct StockMap;
struct StockItem;
struct StockGetHandler;
struct async_operation_ref;
class SocketAddress;

/**
 * Creates a new TCP connection stock.
 *
 * @param pool the memory pool
 * @param limit the maximum number of connections per host
 * @return the new TCP connections stock (this function cannot fail)
 */
StockMap *
tcp_stock_new(struct pool *pool, unsigned limit);

/**
 * @param name the hstock name; it is auto-generated from the
 * #address_list if NULL is passed here
 * @param timeout the connect timeout in seconds
 */
void
tcp_stock_get(StockMap *tcp_stock, struct pool *pool, const char *name,
              bool ip_transparent,
              SocketAddress bind_address,
              SocketAddress address,
              unsigned timeout,
              const StockGetHandler *handler, void *handler_ctx,
              struct async_operation_ref *async_ref);

void
tcp_stock_put(StockMap *tcp_stock, StockItem &item, bool destroy);

gcc_pure
int
tcp_stock_item_get(const StockItem &item);

gcc_pure
int
tcp_stock_item_get_domain(const StockItem &item);

#endif
<|endoftext|>


<commit_before>// (C) 2016 Marvin Sielenkemper

#include "db.hpp"

#include <mariadb/mysql.h>
#include <iostream>

struct Db::Impl : boost::noncopyable {
  MYSQL mysql;

  Impl() : mysql() {
    mysql_init(&mysql);
  }
};

Db::Db(char* passwd) : m_implPtr(new Impl()) {
  auto& mysql(m_implPtr->mysql);
  auto const p(mysql_real_connect(&mysql, "192.168.2.51", "furnace",
                                  passwd, "furnace", 3306, nullptr,
                                  CLIENT_COMPRESS));

  std::cout << p << " " << &mysql << std::endl << mysql_errno(&mysql)
            << std::endl << mysql_error(&mysql) << std::endl;
}

Db::~Db() {
}
<commit_msg>moved the inline constants into named constants, updated the error handling<commit_after>// (C) 2016 Marvin Sielenkemper

#include "db.hpp"

#include <mariadb/mysql.h>
#include <sstream>

namespace
{
  const char* const db_host("192.168.2.51");
  const char* const db_user("furnace");
  const char* const db_name("furnace");
  unsigned int db_port(3306);
}

struct Db::Impl : boost::noncopyable {
  MYSQL mysql;

  Impl() : mysql() {
    mysql_init(&mysql);
  }
};

Db::Db(char* passwd) : m_implPtr(new Impl()) {
  auto& mysql(m_implPtr->mysql);

  if (!mysql_real_connect(&mysql, db_host, db_user, passwd, db_name,
                                  db_port, nullptr, CLIENT_COMPRESS)) {
    std::ostringstream buffer;

    buffer << "failed to connect to database: '"
       << mysql_error(&mysql) << '\'';

    throw std::runtime_error(buffer.str());
  }
}

Db::~Db() {
}
<|endoftext|>


<commit_before>#include "chainerx/python/chainer_interop.h"

#include <algorithm>
#include <cstdint>
#include <iterator>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include <gsl/gsl>
#include <nonstd/optional.hpp>

#include "chainerx/array.h"
#include "chainerx/array_body.h"
#include "chainerx/array_fwd.h"
#include "chainerx/backward.h"
#include "chainerx/backward_builder.h"
#include "chainerx/backward_context.h"

#include "chainerx/python/common.h"

namespace chainerx {
namespace python {
namespace python_internal {

namespace py = pybind11;

using ArrayBodyPtr = std::shared_ptr<internal::ArrayBody>;

namespace {

inline bool IsUniqueAndIncreasingIndexes(const std::vector<size_t>& vec, size_t upper) {
    for (size_t i = 0; i < vec.size(); ++i) {
        if (i > 0 && vec[i] <= vec[i - 1]) {
            return false;
        }
        if (upper <= vec[i]) {
            return false;
        }
    }
    return true;
}

}  // namespace

void InitChainerxChainerInterop(pybind11::module& m) {
    m.def("_function_node_forward",
          [](py::object function_node,
             const std::vector<ArrayBodyPtr>& inputs,
             const std::vector<ArrayBodyPtr>& outputs,
             const std::vector<size_t>& input_indexes_to_retain,
             const std::vector<size_t>& output_indexes_to_retain) {
              // Implementation note:
              // There are two kinds of indices:
              // o: Original indices. This is the indices used in Python world. It includes None arrays.
              // r: Reduced indices. This is the indices that ChainerX C++ impl handles. None arrays are omitted.
              // o and r are used as abbreviations of these.
              // i and j are used as variables of respective kinds of indices.
              CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(input_indexes_to_retain, inputs.size()));
              CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(output_indexes_to_retain, outputs.size()));
              CHAINERX_ASSERT(std::all_of(outputs.begin(), outputs.end(), [](const ArrayBodyPtr& array_body) {
                  return array_body == nullptr || array_body->nodes().empty();
              }));
              const size_t chainer_output_count = outputs.size();

              // Prepare input/output arrays for BackwardBuilder
              // {in,out}put_index_map maps original input/output indices to reduced indices where None outputs are omitted

              auto get_reduced_arrays = [](const std::vector<ArrayBodyPtr>& array_bodies) {
                  // Given the input/output array bodies, construct an index mapping between original <o> indices and
                  // reduced <r> indices where None arrays are omitted.
                  std::vector<Array> reduced_arrays;
                  std::vector<size_t> index_r2o_map;
                  std::vector<nonstd::optional<size_t>> index_o2r_map(array_bodies.size());
                  reduced_arrays.reserve(array_bodies.size());
                  index_r2o_map.reserve(array_bodies.size());
                  for (size_t i = 0; i < array_bodies.size(); ++i) {
                      const ArrayBodyPtr& array_body = array_bodies[i];
                      if (array_body != nullptr) {
                          gsl::at(index_o2r_map, i) = index_r2o_map.size();
                          index_r2o_map.emplace_back(i);
                          reduced_arrays.emplace_back(array_body);
                      }
                  }
                  return std::make_tuple(std::move(reduced_arrays), std::move(index_r2o_map), std::move(index_o2r_map));
              };

              // Inputs
              std::vector<Array> reduced_input_arrays;
              std::vector<size_t> input_index_r2o_map;
              std::vector<nonstd::optional<size_t>> input_index_o2r_map;
              std::vector<ConstArrayRef> reduced_input_array_refs;
              std::tie(reduced_input_arrays, input_index_r2o_map, input_index_o2r_map) = get_reduced_arrays(inputs);
              CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(input_index_r2o_map, inputs.size()));
              CHAINERX_ASSERT(!reduced_input_arrays.empty());
              CHAINERX_ASSERT(std::all_of(reduced_input_arrays.begin(), reduced_input_arrays.end(), [](const Array& arr) {
                  return internal::GetArrayBody(arr) != nullptr;
              }));
              reduced_input_array_refs.insert(reduced_input_array_refs.begin(), reduced_input_arrays.begin(), reduced_input_arrays.end());

              // Outputs
              std::vector<Array> reduced_output_arrays;
              std::vector<size_t> output_index_r2o_map;
              std::vector<nonstd::optional<size_t>> output_index_o2r_map;
              std::vector<ConstArrayRef> reduced_output_array_refs;
              std::tie(reduced_output_arrays, output_index_r2o_map, output_index_o2r_map) = get_reduced_arrays(outputs);
              CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(output_index_r2o_map, outputs.size()));
              CHAINERX_ASSERT(!reduced_output_arrays.empty());
              CHAINERX_ASSERT(std::all_of(reduced_output_arrays.begin(), reduced_output_arrays.end(), [](const Array& arr) {
                  return internal::GetArrayBody(arr) != nullptr;
              }));
              reduced_output_array_refs.insert(
                      reduced_output_array_refs.begin(), reduced_output_arrays.begin(), reduced_output_arrays.end());

              // Insert backward function
              BackwardBuilder bb{"chainer_function", std::move(reduced_input_array_refs), std::move(reduced_output_array_refs)};
              if (BackwardBuilder::Target bt = bb.CreateTarget()) {
                  auto function_node_ptr = std::make_shared<py::object>(std::move(function_node), [](gsl::owner<py::object*> ptr) {
                      py::gil_scoped_acquire acquire;
                      delete ptr;
                  });

                  // Retain inputs/outputs
                  auto retain_arrays = [](auto retain,
                                          const std::vector<nonstd::optional<size_t>>& index_o2r_map,
                                          const std::vector<size_t>& indexes_to_retain) {
                      // Given the original indices to retain, retain the corresponding arrays and return the retain tokens. If the
                      // corresponding array was None, the token is nullopt.
                      using RetainedToken = decltype(retain(size_t{}));
                      std::vector<nonstd::optional<RetainedToken>> retained_tokens;
                      retained_tokens.reserve(indexes_to_retain.size());
                      for (size_t i : indexes_to_retain) {
                          if (auto j = index_o2r_map[i]) {
                              retained_tokens.emplace_back(retain(*j));
                          } else {
                              // Array to retain was None
                              retained_tokens.emplace_back(nonstd::nullopt);
                          }
                      }
                      return retained_tokens;
                  };

                  std::vector<nonstd::optional<RetainedInputToken>> retained_input_tokens =
                          retain_arrays([&bb](size_t j) { return bb.RetainInput(j); }, input_index_o2r_map, input_indexes_to_retain);
                  std::vector<nonstd::optional<RetainedOutputToken>> retained_output_tokens =
                          retain_arrays([&bb](size_t j) { return bb.RetainOutput(j); }, output_index_o2r_map, output_indexes_to_retain);

                  // Define backward function
                  bt.Define([chainer_output_count,
                             function_node_ptr = std::move(function_node_ptr),
                             input_index_r2o_map = std::move(input_index_r2o_map),
                             output_index_r2o_map = std::move(output_index_r2o_map),
                             in_toks = std::move(retained_input_tokens),
                             out_toks = std::move(retained_output_tokens)](BackwardContext& bctx) {
                      // Target input indices
                      // This is reduced <r> indices of grad-required inputs.
                      std::vector<size_t> target_input_indexes;
                      target_input_indexes.reserve(bctx.input_count());

                      for (size_t j = 0; j < bctx.input_count(); ++j) {
                          if (bctx.is_input_grad_required(j)) {
                              target_input_indexes.emplace_back(j);
                          }
                      }

                      CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(target_input_indexes, bctx.input_count()));

                      // Collect incoming output gradients
                      std::vector<ArrayBodyPtr> chainer_grad_outputs{chainer_output_count};
                      for (size_t j = 0; j < bctx.output_count(); ++j) {
                          if (auto& gy = bctx.output_grad(j)) {
                              size_t chainer_output_index = output_index_r2o_map[j];
                              chainer_grad_outputs[chainer_output_index] = internal::GetArrayBody(*gy);
                          }
                      }

                      // Get retained inputs and outputs
                      auto get_retained_arrays = [](auto get_retained, const auto& tokens) {
                          std::vector<ArrayBodyPtr> retained_arrays;
                          retained_arrays.reserve(tokens.size());
                          for (const auto& tok : tokens) {
                              if (tok.has_value()) {
                                  // Retrieve the retained array.
                                  retained_arrays.emplace_back(internal::GetArrayBody(get_retained(*tok)));
                              } else {
                                  // Array to retain was None.
                                  retained_arrays.emplace_back(nullptr);
                              }
                          }
                          return retained_arrays;
                      };

                      std::vector<ArrayBodyPtr> retained_inputs =
                              get_retained_arrays([&bctx](const RetainedInputToken& tok) { return bctx.GetRetainedInput(tok); }, in_toks);
                      std::vector<ArrayBodyPtr> retained_outputs = get_retained_arrays(
                              [&bctx](const RetainedOutputToken& tok) { return bctx.GetRetainedOutput(tok); }, out_toks);

                      // Call FunctionNode._backward_chainerx()
                      std::vector<ArrayBodyPtr> chainer_grad_inputs;

                      {
                          // Target input indices that are passed to backward() of Chainer.
                          // This is indices of <grad-required> inputs among <all> of the inputs.
                          std::vector<size_t> chainer_target_input_indexes{};
                          chainer_target_input_indexes.reserve(target_input_indexes.size());
                          for (size_t j : target_input_indexes) {
                              chainer_target_input_indexes.emplace_back(input_index_r2o_map[j]);
                          }

                          py::gil_scoped_acquire acquire;
                          py::object func_backward = function_node_ptr->attr("_backward_chainerx");
                          py::object py_grad_inputs =
                                  func_backward(chainer_target_input_indexes, chainer_grad_outputs, retained_inputs, retained_outputs);
                          chainer_grad_inputs = py::cast<std::vector<ArrayBodyPtr>>(py_grad_inputs);
                      }
                      CHAINERX_ASSERT(chainer_grad_inputs.size() == target_input_indexes.size());

                      // Store computed input gradients
                      for (size_t k = 0; k < target_input_indexes.size(); ++k) {
                          size_t j = gsl::at(target_input_indexes, k);
                          ArrayBodyPtr& gx = gsl::at(chainer_grad_inputs, k);
                          CHAINERX_ASSERT(gx != nullptr);

                          bctx.input_grad(j) = Array{gx};
                      }
                  });
              }
              bb.Finalize();
          },
          py::arg("function_node"),
          py::arg("inputs"),
          py::arg("outputs"),
          py::arg("input_indexes_to_retain"),
          py::arg("output_indexes_to_retain"));
}

}  // namespace python_internal
}  // namespace python
}  // namespace chainerx
<commit_msg>Fix construction of std::shared_ptr with custom deleter<commit_after>#include "chainerx/python/chainer_interop.h"

#include <algorithm>
#include <cstdint>
#include <iterator>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include <gsl/gsl>
#include <nonstd/optional.hpp>

#include "chainerx/array.h"
#include "chainerx/array_body.h"
#include "chainerx/array_fwd.h"
#include "chainerx/backward.h"
#include "chainerx/backward_builder.h"
#include "chainerx/backward_context.h"

#include "chainerx/python/common.h"

namespace chainerx {
namespace python {
namespace python_internal {

namespace py = pybind11;

using ArrayBodyPtr = std::shared_ptr<internal::ArrayBody>;

namespace {

inline bool IsUniqueAndIncreasingIndexes(const std::vector<size_t>& vec, size_t upper) {
    for (size_t i = 0; i < vec.size(); ++i) {
        if (i > 0 && vec[i] <= vec[i - 1]) {
            return false;
        }
        if (upper <= vec[i]) {
            return false;
        }
    }
    return true;
}

}  // namespace

void InitChainerxChainerInterop(pybind11::module& m) {
    m.def("_function_node_forward",
          [](py::object function_node,
             const std::vector<ArrayBodyPtr>& inputs,
             const std::vector<ArrayBodyPtr>& outputs,
             const std::vector<size_t>& input_indexes_to_retain,
             const std::vector<size_t>& output_indexes_to_retain) {
              // Implementation note:
              // There are two kinds of indices:
              // o: Original indices. This is the indices used in Python world. It includes None arrays.
              // r: Reduced indices. This is the indices that ChainerX C++ impl handles. None arrays are omitted.
              // o and r are used as abbreviations of these.
              // i and j are used as variables of respective kinds of indices.
              CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(input_indexes_to_retain, inputs.size()));
              CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(output_indexes_to_retain, outputs.size()));
              CHAINERX_ASSERT(std::all_of(outputs.begin(), outputs.end(), [](const ArrayBodyPtr& array_body) {
                  return array_body == nullptr || array_body->nodes().empty();
              }));
              const size_t chainer_output_count = outputs.size();

              // Prepare input/output arrays for BackwardBuilder
              // {in,out}put_index_map maps original input/output indices to reduced indices where None outputs are omitted

              auto get_reduced_arrays = [](const std::vector<ArrayBodyPtr>& array_bodies) {
                  // Given the input/output array bodies, construct an index mapping between original <o> indices and
                  // reduced <r> indices where None arrays are omitted.
                  std::vector<Array> reduced_arrays;
                  std::vector<size_t> index_r2o_map;
                  std::vector<nonstd::optional<size_t>> index_o2r_map(array_bodies.size());
                  reduced_arrays.reserve(array_bodies.size());
                  index_r2o_map.reserve(array_bodies.size());
                  for (size_t i = 0; i < array_bodies.size(); ++i) {
                      const ArrayBodyPtr& array_body = array_bodies[i];
                      if (array_body != nullptr) {
                          gsl::at(index_o2r_map, i) = index_r2o_map.size();
                          index_r2o_map.emplace_back(i);
                          reduced_arrays.emplace_back(array_body);
                      }
                  }
                  return std::make_tuple(std::move(reduced_arrays), std::move(index_r2o_map), std::move(index_o2r_map));
              };

              // Inputs
              std::vector<Array> reduced_input_arrays;
              std::vector<size_t> input_index_r2o_map;
              std::vector<nonstd::optional<size_t>> input_index_o2r_map;
              std::vector<ConstArrayRef> reduced_input_array_refs;
              std::tie(reduced_input_arrays, input_index_r2o_map, input_index_o2r_map) = get_reduced_arrays(inputs);
              CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(input_index_r2o_map, inputs.size()));
              CHAINERX_ASSERT(!reduced_input_arrays.empty());
              CHAINERX_ASSERT(std::all_of(reduced_input_arrays.begin(), reduced_input_arrays.end(), [](const Array& arr) {
                  return internal::GetArrayBody(arr) != nullptr;
              }));
              reduced_input_array_refs.insert(reduced_input_array_refs.begin(), reduced_input_arrays.begin(), reduced_input_arrays.end());

              // Outputs
              std::vector<Array> reduced_output_arrays;
              std::vector<size_t> output_index_r2o_map;
              std::vector<nonstd::optional<size_t>> output_index_o2r_map;
              std::vector<ConstArrayRef> reduced_output_array_refs;
              std::tie(reduced_output_arrays, output_index_r2o_map, output_index_o2r_map) = get_reduced_arrays(outputs);
              CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(output_index_r2o_map, outputs.size()));
              CHAINERX_ASSERT(!reduced_output_arrays.empty());
              CHAINERX_ASSERT(std::all_of(reduced_output_arrays.begin(), reduced_output_arrays.end(), [](const Array& arr) {
                  return internal::GetArrayBody(arr) != nullptr;
              }));
              reduced_output_array_refs.insert(
                      reduced_output_array_refs.begin(), reduced_output_arrays.begin(), reduced_output_arrays.end());

              // Insert backward function
              BackwardBuilder bb{"chainer_function", std::move(reduced_input_array_refs), std::move(reduced_output_array_refs)};
              if (BackwardBuilder::Target bt = bb.CreateTarget()) {
                  // Need to reallocate the function node in order to specify a custom deleter (that acquires the GIL before deletion).
                  auto function_node_ptr =
                          std::shared_ptr<py::object>{new py::object{std::move(function_node)}, [](gsl::owner<py::object*> ptr) {
                                                          py::gil_scoped_acquire acquire;
                                                          delete ptr;
                                                      }};

                  // Retain inputs/outputs
                  auto retain_arrays = [](auto retain,
                                          const std::vector<nonstd::optional<size_t>>& index_o2r_map,
                                          const std::vector<size_t>& indexes_to_retain) {
                      // Given the original indices to retain, retain the corresponding arrays and return the retain tokens. If the
                      // corresponding array was None, the token is nullopt.
                      using RetainedToken = decltype(retain(size_t{}));
                      std::vector<nonstd::optional<RetainedToken>> retained_tokens;
                      retained_tokens.reserve(indexes_to_retain.size());
                      for (size_t i : indexes_to_retain) {
                          if (auto j = index_o2r_map[i]) {
                              retained_tokens.emplace_back(retain(*j));
                          } else {
                              // Array to retain was None
                              retained_tokens.emplace_back(nonstd::nullopt);
                          }
                      }
                      return retained_tokens;
                  };

                  std::vector<nonstd::optional<RetainedInputToken>> retained_input_tokens =
                          retain_arrays([&bb](size_t j) { return bb.RetainInput(j); }, input_index_o2r_map, input_indexes_to_retain);
                  std::vector<nonstd::optional<RetainedOutputToken>> retained_output_tokens =
                          retain_arrays([&bb](size_t j) { return bb.RetainOutput(j); }, output_index_o2r_map, output_indexes_to_retain);

                  // Define backward function
                  bt.Define([chainer_output_count,
                             function_node_ptr = std::move(function_node_ptr),
                             input_index_r2o_map = std::move(input_index_r2o_map),
                             output_index_r2o_map = std::move(output_index_r2o_map),
                             in_toks = std::move(retained_input_tokens),
                             out_toks = std::move(retained_output_tokens)](BackwardContext& bctx) {
                      // Target input indices
                      // This is reduced <r> indices of grad-required inputs.
                      std::vector<size_t> target_input_indexes;
                      target_input_indexes.reserve(bctx.input_count());

                      for (size_t j = 0; j < bctx.input_count(); ++j) {
                          if (bctx.is_input_grad_required(j)) {
                              target_input_indexes.emplace_back(j);
                          }
                      }

                      CHAINERX_ASSERT(IsUniqueAndIncreasingIndexes(target_input_indexes, bctx.input_count()));

                      // Collect incoming output gradients
                      std::vector<ArrayBodyPtr> chainer_grad_outputs{chainer_output_count};
                      for (size_t j = 0; j < bctx.output_count(); ++j) {
                          if (auto& gy = bctx.output_grad(j)) {
                              size_t chainer_output_index = output_index_r2o_map[j];
                              chainer_grad_outputs[chainer_output_index] = internal::GetArrayBody(*gy);
                          }
                      }

                      // Get retained inputs and outputs
                      auto get_retained_arrays = [](auto get_retained, const auto& tokens) {
                          std::vector<ArrayBodyPtr> retained_arrays;
                          retained_arrays.reserve(tokens.size());
                          for (const auto& tok : tokens) {
                              if (tok.has_value()) {
                                  // Retrieve the retained array.
                                  retained_arrays.emplace_back(internal::GetArrayBody(get_retained(*tok)));
                              } else {
                                  // Array to retain was None.
                                  retained_arrays.emplace_back(nullptr);
                              }
                          }
                          return retained_arrays;
                      };

                      std::vector<ArrayBodyPtr> retained_inputs =
                              get_retained_arrays([&bctx](const RetainedInputToken& tok) { return bctx.GetRetainedInput(tok); }, in_toks);
                      std::vector<ArrayBodyPtr> retained_outputs = get_retained_arrays(
                              [&bctx](const RetainedOutputToken& tok) { return bctx.GetRetainedOutput(tok); }, out_toks);

                      // Call FunctionNode._backward_chainerx()
                      std::vector<ArrayBodyPtr> chainer_grad_inputs;

                      {
                          // Target input indices that are passed to backward() of Chainer.
                          // This is indices of <grad-required> inputs among <all> of the inputs.
                          std::vector<size_t> chainer_target_input_indexes{};
                          chainer_target_input_indexes.reserve(target_input_indexes.size());
                          for (size_t j : target_input_indexes) {
                              chainer_target_input_indexes.emplace_back(input_index_r2o_map[j]);
                          }

                          py::gil_scoped_acquire acquire;
                          py::object func_backward = function_node_ptr->attr("_backward_chainerx");
                          py::object py_grad_inputs =
                                  func_backward(chainer_target_input_indexes, chainer_grad_outputs, retained_inputs, retained_outputs);
                          chainer_grad_inputs = py::cast<std::vector<ArrayBodyPtr>>(py_grad_inputs);
                      }
                      CHAINERX_ASSERT(chainer_grad_inputs.size() == target_input_indexes.size());

                      // Store computed input gradients
                      for (size_t k = 0; k < target_input_indexes.size(); ++k) {
                          size_t j = gsl::at(target_input_indexes, k);
                          ArrayBodyPtr& gx = gsl::at(chainer_grad_inputs, k);
                          CHAINERX_ASSERT(gx != nullptr);

                          bctx.input_grad(j) = Array{gx};
                      }
                  });
              }
              bb.Finalize();
          },
          py::arg("function_node"),
          py::arg("inputs"),
          py::arg("outputs"),
          py::arg("input_indexes_to_retain"),
          py::arg("output_indexes_to_retain"));
}

}  // namespace python_internal
}  // namespace python
}  // namespace chainerx
<|endoftext|>


<commit_before>#include "chainerx/python/chainer_interop.h"

#include <algorithm>
#include <cstdint>
#include <iterator>
#include <memory>
#include <string>
#include <vector>

#include "chainerx/array.h"
#include "chainerx/array_body.h"
#include "chainerx/array_fwd.h"
#include "chainerx/backward.h"
#include "chainerx/backward_builder.h"
#include "chainerx/backward_context.h"

#include "chainerx/python/common.h"

namespace chainerx {
namespace python {
namespace python_internal {

namespace py = pybind11;

using ArrayBodyPtr = std::shared_ptr<internal::ArrayBody>;

void InitChainerxChainerInterop(pybind11::module& m) {
    m.def("_function_node_forward",
          [](py::object function_node,
             const std::vector<ArrayBodyPtr>& inputs,
             const std::vector<ArrayBodyPtr>& outputs,
             const std::vector<size_t>& input_indexes_to_retain,
             const std::vector<size_t>& output_indexes_to_retain) {
              CHAINERX_ASSERT(std::all_of(outputs.begin(), outputs.end(), [](const ArrayBodyPtr& array_body) {
                  return array_body == nullptr || array_body->nodes().empty();
              }));

              // Prepare input arrays for BackwardBuilder
              std::vector<Array> input_arrays;
              std::vector<ConstArrayRef> input_array_refs;
              input_arrays.reserve(inputs.size());
              input_array_refs.reserve(inputs.size());
              for (const ArrayBodyPtr& array_body : inputs) {
                  input_arrays.emplace_back(array_body);
                  input_array_refs.emplace_back(input_arrays.back());
              }

              // Prepare output arrays for BackwardBuilder
              std::vector<Array> output_arrays;
              std::vector<ConstArrayRef> output_array_refs;
              std::vector<nonstd::optional<size_t>> output_index_map;  // maps original output index to reduced index
              output_arrays.reserve(outputs.size());
              output_array_refs.reserve(outputs.size());
              output_index_map.reserve(outputs.size());
              for (const ArrayBodyPtr& array_body : outputs) {
                  if (array_body != nullptr) {
                      output_index_map.emplace_back(output_array_refs.size());
                      output_arrays.emplace_back(array_body);
                      output_array_refs.emplace_back(output_arrays.back());
                  } else {
                      output_index_map.emplace_back(nonstd::nullopt);
                  }
              }

              // Insert backward function
              BackwardBuilder bb{"chainer_function", std::move(input_array_refs), std::move(output_array_refs)};
              if (BackwardBuilder::Target bt = bb.CreateTarget()) {
                  auto function_node_ptr = std::make_shared<py::object>(std::move(function_node), [](py::object* ptr) {
                      py::gil_scoped_acquire acquire;
                      delete ptr;
                  });

                  std::vector<RetainedInputToken> retained_input_tokens;
                  std::transform(
                          input_indexes_to_retain.begin(),
                          input_indexes_to_retain.end(),
                          std::back_inserter(retained_input_tokens),
                          [&bb](size_t i) { return bb.RetainInput(i); });
                  std::vector<nonstd::optional<RetainedOutputToken>> retained_output_tokens;
                  retained_output_tokens.reserve(output_indexes_to_retain.size());
                  for (size_t i : output_indexes_to_retain) {
                      if (nonstd::optional<size_t> i_out_reduced = gsl::at(output_index_map, i)) {
                          retained_output_tokens.emplace_back(bb.RetainOutput(*i_out_reduced));
                      } else {
                          retained_output_tokens.emplace_back(nonstd::nullopt);
                      }
                  }

                  bt.Define([function_node_ptr = std::move(function_node_ptr),
                             output_index_map = std::move(output_index_map),
                             in_toks = std::move(retained_input_tokens),
                             out_toks = std::move(retained_output_tokens)](BackwardContext& bctx) {
                      // Target input indexes
                      std::vector<size_t> target_input_indexes;
                      target_input_indexes.reserve(bctx.input_count());

                      for (size_t i_in = 0; i_in < bctx.input_count(); ++i_in) {
                          if (bctx.is_input_grad_required(i_in)) {
                              target_input_indexes.emplace_back(i_in);
                          }
                      }

                      // Collect incoming output gradients
                      std::vector<ArrayBodyPtr> grad_outputs;
                      grad_outputs.reserve(output_index_map.size());
                      for (const nonstd::optional<size_t>& i_out : output_index_map) {
                          if (i_out.has_value()) {
                              CHAINERX_ASSERT(*i_out < bctx.output_count());
                              if (auto gy = bctx.output_grad(*i_out)) {
                                  grad_outputs.emplace_back(internal::GetArrayBody(*gy));
                              } else {
                                  grad_outputs.emplace_back(nullptr);
                              }
                          } else {
                              grad_outputs.emplace_back(nullptr);
                          }
                      }

                      // Get retained inputs and outputs
                      std::vector<ArrayBodyPtr> retained_inputs;
                      std::transform(
                              in_toks.begin(), in_toks.end(), std::back_inserter(retained_inputs), [&bctx](const RetainedInputToken& tok) {
                                  return internal::GetArrayBody(bctx.GetRetainedInput(tok));
                              });
                      std::vector<ArrayBodyPtr> retained_outputs;
                      retained_outputs.reserve(out_toks.size());
                      for (const nonstd::optional<RetainedOutputToken>& tok : out_toks) {
                          if (tok.has_value()) {
                              retained_outputs.emplace_back(internal::GetArrayBody(bctx.GetRetainedOutput(*tok)));
                          } else {
                              retained_outputs.emplace_back(nullptr);
                          }
                      }

                      // Call FunctionNode._backward_chainerx()
                      std::vector<ArrayBodyPtr> grad_inputs;
                      {
                          py::gil_scoped_acquire acquire;
                          py::object func_backward = function_node_ptr->attr("_backward_chainerx");
                          py::object py_grad_inputs = func_backward(target_input_indexes, grad_outputs, retained_inputs, retained_outputs);
                          grad_inputs = py::cast<std::vector<ArrayBodyPtr>>(py_grad_inputs);
                      }
                      CHAINERX_ASSERT(grad_inputs.size() == target_input_indexes.size());

                      // Store computed input gradients
                      for (size_t i = 0; i < target_input_indexes.size(); ++i) {
                          size_t i_in = gsl::at(target_input_indexes, i);
                          ArrayBodyPtr& gx = gsl::at(grad_inputs, i);

                          bctx.input_grad(i_in) = Array{gx};
                      }
                  });
              }
              bb.Finalize();
          },
          py::arg("function_node"),
          py::arg("inputs"),
          py::arg("outputs"),
          py::arg("input_indexes_to_retain"),
          py::arg("output_indexes_to_retain"));
}

}  // namespace python_internal
}  // namespace python
}  // namespace chainerx
<commit_msg>Add missing include<commit_after>#include "chainerx/python/chainer_interop.h"

#include <algorithm>
#include <cstdint>
#include <iterator>
#include <memory>
#include <string>
#include <vector>

#include <nonstd/optional.hpp>

#include "chainerx/array.h"
#include "chainerx/array_body.h"
#include "chainerx/array_fwd.h"
#include "chainerx/backward.h"
#include "chainerx/backward_builder.h"
#include "chainerx/backward_context.h"

#include "chainerx/python/common.h"

namespace chainerx {
namespace python {
namespace python_internal {

namespace py = pybind11;

using ArrayBodyPtr = std::shared_ptr<internal::ArrayBody>;

void InitChainerxChainerInterop(pybind11::module& m) {
    m.def("_function_node_forward",
          [](py::object function_node,
             const std::vector<ArrayBodyPtr>& inputs,
             const std::vector<ArrayBodyPtr>& outputs,
             const std::vector<size_t>& input_indexes_to_retain,
             const std::vector<size_t>& output_indexes_to_retain) {
              CHAINERX_ASSERT(std::all_of(outputs.begin(), outputs.end(), [](const ArrayBodyPtr& array_body) {
                  return array_body == nullptr || array_body->nodes().empty();
              }));

              // Prepare input arrays for BackwardBuilder
              std::vector<Array> input_arrays;
              std::vector<ConstArrayRef> input_array_refs;
              input_arrays.reserve(inputs.size());
              input_array_refs.reserve(inputs.size());
              for (const ArrayBodyPtr& array_body : inputs) {
                  input_arrays.emplace_back(array_body);
                  input_array_refs.emplace_back(input_arrays.back());
              }

              // Prepare output arrays for BackwardBuilder
              std::vector<Array> output_arrays;
              std::vector<ConstArrayRef> output_array_refs;
              std::vector<nonstd::optional<size_t>> output_index_map;  // maps original output index to reduced index
              output_arrays.reserve(outputs.size());
              output_array_refs.reserve(outputs.size());
              output_index_map.reserve(outputs.size());
              for (const ArrayBodyPtr& array_body : outputs) {
                  if (array_body != nullptr) {
                      output_index_map.emplace_back(output_array_refs.size());
                      output_arrays.emplace_back(array_body);
                      output_array_refs.emplace_back(output_arrays.back());
                  } else {
                      output_index_map.emplace_back(nonstd::nullopt);
                  }
              }

              // Insert backward function
              BackwardBuilder bb{"chainer_function", std::move(input_array_refs), std::move(output_array_refs)};
              if (BackwardBuilder::Target bt = bb.CreateTarget()) {
                  auto function_node_ptr = std::make_shared<py::object>(std::move(function_node), [](py::object* ptr) {
                      py::gil_scoped_acquire acquire;
                      delete ptr;
                  });

                  std::vector<RetainedInputToken> retained_input_tokens;
                  std::transform(
                          input_indexes_to_retain.begin(),
                          input_indexes_to_retain.end(),
                          std::back_inserter(retained_input_tokens),
                          [&bb](size_t i) { return bb.RetainInput(i); });
                  std::vector<nonstd::optional<RetainedOutputToken>> retained_output_tokens;
                  retained_output_tokens.reserve(output_indexes_to_retain.size());
                  for (size_t i : output_indexes_to_retain) {
                      if (nonstd::optional<size_t> i_out_reduced = gsl::at(output_index_map, i)) {
                          retained_output_tokens.emplace_back(bb.RetainOutput(*i_out_reduced));
                      } else {
                          retained_output_tokens.emplace_back(nonstd::nullopt);
                      }
                  }

                  bt.Define([function_node_ptr = std::move(function_node_ptr),
                             output_index_map = std::move(output_index_map),
                             in_toks = std::move(retained_input_tokens),
                             out_toks = std::move(retained_output_tokens)](BackwardContext& bctx) {
                      // Target input indexes
                      std::vector<size_t> target_input_indexes;
                      target_input_indexes.reserve(bctx.input_count());

                      for (size_t i_in = 0; i_in < bctx.input_count(); ++i_in) {
                          if (bctx.is_input_grad_required(i_in)) {
                              target_input_indexes.emplace_back(i_in);
                          }
                      }

                      // Collect incoming output gradients
                      std::vector<ArrayBodyPtr> grad_outputs;
                      grad_outputs.reserve(output_index_map.size());
                      for (const nonstd::optional<size_t>& i_out : output_index_map) {
                          if (i_out.has_value()) {
                              CHAINERX_ASSERT(*i_out < bctx.output_count());
                              if (auto gy = bctx.output_grad(*i_out)) {
                                  grad_outputs.emplace_back(internal::GetArrayBody(*gy));
                              } else {
                                  grad_outputs.emplace_back(nullptr);
                              }
                          } else {
                              grad_outputs.emplace_back(nullptr);
                          }
                      }

                      // Get retained inputs and outputs
                      std::vector<ArrayBodyPtr> retained_inputs;
                      std::transform(
                              in_toks.begin(), in_toks.end(), std::back_inserter(retained_inputs), [&bctx](const RetainedInputToken& tok) {
                                  return internal::GetArrayBody(bctx.GetRetainedInput(tok));
                              });
                      std::vector<ArrayBodyPtr> retained_outputs;
                      retained_outputs.reserve(out_toks.size());
                      for (const nonstd::optional<RetainedOutputToken>& tok : out_toks) {
                          if (tok.has_value()) {
                              retained_outputs.emplace_back(internal::GetArrayBody(bctx.GetRetainedOutput(*tok)));
                          } else {
                              retained_outputs.emplace_back(nullptr);
                          }
                      }

                      // Call FunctionNode._backward_chainerx()
                      std::vector<ArrayBodyPtr> grad_inputs;
                      {
                          py::gil_scoped_acquire acquire;
                          py::object func_backward = function_node_ptr->attr("_backward_chainerx");
                          py::object py_grad_inputs = func_backward(target_input_indexes, grad_outputs, retained_inputs, retained_outputs);
                          grad_inputs = py::cast<std::vector<ArrayBodyPtr>>(py_grad_inputs);
                      }
                      CHAINERX_ASSERT(grad_inputs.size() == target_input_indexes.size());

                      // Store computed input gradients
                      for (size_t i = 0; i < target_input_indexes.size(); ++i) {
                          size_t i_in = gsl::at(target_input_indexes, i);
                          ArrayBodyPtr& gx = gsl::at(grad_inputs, i);

                          bctx.input_grad(i_in) = Array{gx};
                      }
                  });
              }
              bb.Finalize();
          },
          py::arg("function_node"),
          py::arg("inputs"),
          py::arg("outputs"),
          py::arg("input_indexes_to_retain"),
          py::arg("output_indexes_to_retain"));
}

}  // namespace python_internal
}  // namespace python
}  // namespace chainerx
<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include <GL3DBarChart.hxx>

#include <GL/glew.h>

#include <glm/glm.hpp>
#include <glm/gtx/transform.hpp>

#include "3DChartObjects.hxx"
#include "GL3DRenderer.hxx"
#include <ExplicitCategoriesProvider.hxx>
#include <DataSeriesHelper.hxx>

using namespace com::sun::star;

namespace chart {

GL3DBarChart::GL3DBarChart(
    const css::uno::Reference<css::chart2::XChartType>& xChartType,
    OpenGLWindow& rWindow) :
    mxChartType(xChartType),
    mpRenderer(new opengl3D::OpenGL3DRenderer()),
    mrWindow(rWindow),
    mpCamera(NULL)
{
    mrWindow.setRenderer(this);
    mpRenderer->init();
}

GL3DBarChart::~GL3DBarChart()
{
    mrWindow.setRenderer(NULL);
}

void GL3DBarChart::create3DShapes(const boost::ptr_vector<VDataSeries>& rDataSeriesContainer,
        ExplicitCategoriesProvider& rCatProvider)
{
    // Each series of data flows from left to right, and multiple series are
    // stacked vertically along y axis.

    // NOTE: These objects are created and positioned in a totally blind
    // fashion since we don't even have a way to see them on screen.  So, no
    // guarantee they are positioned correctly.  In fact, they are guaranteed
    // to be positioned incorrectly.

    const float nBarSizeX = 10;
    const float nBarSizeY = 10;
    const float nBarDistanceX = nBarSizeX / 2;
    const float nBarDistanceY = nBarSizeY / 2;

    sal_uInt32 nId = 1;
    float nXEnd = 0.0;
    float nYPos = 0.0;

    const Color aSeriesColor[] = {
        COL_RED, COL_GREEN, COL_YELLOW, COL_BROWN, COL_GRAY
    };

    maShapes.clear();
    maShapes.push_back(new opengl3D::Camera(mpRenderer.get()));
    mpCamera = static_cast<opengl3D::Camera*>(&maShapes.back());

    sal_Int32 nSeriesIndex = 0;
    for (boost::ptr_vector<VDataSeries>::const_iterator itr = rDataSeriesContainer.begin(),
            itrEnd = rDataSeriesContainer.end(); itr != itrEnd; ++itr)
    {
        nYPos = nSeriesIndex * (nBarSizeY + nBarDistanceY);

        const VDataSeries& rDataSeries = *itr;
        sal_Int32 nPointCount = rDataSeries.getTotalPointCount();

        bool bMappedFillProperty = rDataSeries.hasPropertyMapping("FillColor");

        // Create series name text object.
        OUString aSeriesName =
            DataSeriesHelper::getDataSeriesLabel(
                rDataSeries.getModel(), mxChartType->getRoleOfSequenceForSeriesLabel());

        maShapes.push_back(new opengl3D::Text(mpRenderer.get(), aSeriesName, nId++));
        opengl3D::Text* p = static_cast<opengl3D::Text*>(&maShapes.back());
        Size aTextSize = p->getSize();
        glm::vec3 aTopLeft, aTopRight, aBottomRight;
        aTopLeft.x = aTextSize.getWidth() * -1.0;
        aTopLeft.y = nYPos;
        aTopRight.y = nYPos;
        aBottomRight = aTopRight;
        aBottomRight.y += aTextSize.getHeight();
        p->setPosition(aTopLeft, aTopRight, aBottomRight);

        sal_Int32 nColor = aSeriesColor[nSeriesIndex % SAL_N_ELEMENTS(aSeriesColor)].GetColor();
        for(sal_Int32 nIndex = 0; nIndex < nPointCount; ++nIndex)
        {
            if(bMappedFillProperty)
            {
                nColor = static_cast<sal_uInt32>(rDataSeries.getValueByProperty(nIndex, "FillColor"));
            }

            float nVal = rDataSeries.getYValue(nIndex);
            float nXPos = nIndex * (nBarSizeX + nBarDistanceX);

            glm::mat4 aScaleMatrix = glm::scale(nBarSizeX, nBarSizeY, nVal);
            glm::mat4 aTranslationMatrix = glm::translate(nXPos, nYPos, 0.0f);
            glm::mat4 aBarPosition = aTranslationMatrix * aScaleMatrix;

            maShapes.push_back(new opengl3D::Bar(mpRenderer.get(), aBarPosition, nColor, nId++));
        }

        float nThisXEnd = nPointCount * (nBarSizeX + nBarDistanceX);
        if (nXEnd < nThisXEnd)
            nXEnd = nThisXEnd;

        ++nSeriesIndex;
    }

    nYPos += nBarSizeY + nBarDistanceY;

    // X axis
    maShapes.push_back(new opengl3D::Line(mpRenderer.get(), nId++));
    opengl3D::Line* pAxis = static_cast<opengl3D::Line*>(&maShapes.back());
    glm::vec3 aBegin;
    aBegin.y = nYPos;
    glm::vec3 aEnd = aBegin;
    aEnd.x = nXEnd;
    pAxis->setPosition(aBegin, aEnd);
    pAxis->setLineColor(COL_BLUE);

    // Y axis
    maShapes.push_back(new opengl3D::Line(mpRenderer.get(), nId++));
    pAxis = static_cast<opengl3D::Line*>(&maShapes.back());
    aBegin.x = aBegin.y = 0;
    aEnd = aBegin;
    aEnd.y = nYPos;
    pAxis->setPosition(aBegin, aEnd);
    pAxis->setLineColor(COL_BLUE);

    // Chart background.
    maShapes.push_back(new opengl3D::Rectangle(mpRenderer.get(), nId++));
    opengl3D::Rectangle* pRect = static_cast<opengl3D::Rectangle*>(&maShapes.back());
    glm::vec3 aTopLeft;
    glm::vec3 aTopRight = aTopLeft;
    aTopRight.x = nXEnd;
    glm::vec3 aBottomRight = aTopRight;
    aBottomRight.y = nYPos;
    pRect->setPosition(aTopLeft, aTopRight, aBottomRight);
    pRect->setFillColor(COL_BLACK);
    pRect->setLineColor(COL_BLUE);

    // Create category texts along X-axis at the bottom.
    uno::Sequence<OUString> aCats = rCatProvider.getSimpleCategories();
    for (sal_Int32 i = 0; i < aCats.getLength(); ++i)
    {
        float nXPos = i * (nBarSizeX + nBarDistanceX);

        maShapes.push_back(new opengl3D::Text(mpRenderer.get(), aCats[i], nId++));
        opengl3D::Text* p = static_cast<opengl3D::Text*>(&maShapes.back());
        Size aTextSize = p->getSize();
        aTopLeft.x = nXPos;
        aTopLeft.y = nYPos;
        aTopRight = aTopLeft;
        aTopRight.x += aTextSize.getWidth();
        aBottomRight = aTopRight;
        aBottomRight.y += aTextSize.getHeight();
        p->setPosition(aTopLeft, aTopRight, aBottomRight);
    }
}

void GL3DBarChart::render()
{
    mrWindow.getContext()->makeCurrent();
    Size aSize = mrWindow.GetSizePixel();
    mpRenderer->SetSize(aSize);
    mrWindow.getContext()->setWinSize(aSize);
    for(boost::ptr_vector<opengl3D::Renderable3DObject>::iterator itr = maShapes.begin(),
            itrEnd = maShapes.end(); itr != itrEnd; ++itr)
    {
        itr->render();
    }
    mpRenderer->ProcessUnrenderedShape();
    mrWindow.getContext()->swapBuffers();
}

void GL3DBarChart::update()
{
    render();
}

namespace {

class PickingModeSetter
{
private:
    opengl3D::OpenGL3DRenderer* mpRenderer;

public:
    PickingModeSetter(opengl3D::OpenGL3DRenderer* pRenderer):
        mpRenderer(pRenderer)
    {
        mpRenderer->SetPickingMode(true);
    }

    ~PickingModeSetter()
    {
        mpRenderer->SetPickingMode(false);
    }
};

}

void GL3DBarChart::clickedAt(const Point& )
{
    {
        PickingModeSetter(mpRenderer.get());
        render();
    }
    if (mpCamera)
        mpCamera->zoom(1);
}

}

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<commit_msg>prevent access to uninitialized variables<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include <GL3DBarChart.hxx>

#include <GL/glew.h>

#include <glm/glm.hpp>
#include <glm/gtx/transform.hpp>

#include "3DChartObjects.hxx"
#include "GL3DRenderer.hxx"
#include <ExplicitCategoriesProvider.hxx>
#include <DataSeriesHelper.hxx>

using namespace com::sun::star;

namespace chart {

GL3DBarChart::GL3DBarChart(
    const css::uno::Reference<css::chart2::XChartType>& xChartType,
    OpenGLWindow& rWindow) :
    mxChartType(xChartType),
    mpRenderer(new opengl3D::OpenGL3DRenderer()),
    mrWindow(rWindow),
    mpCamera(NULL)
{
    Size aSize = mrWindow.GetSizePixel();
    mpRenderer->SetSize(aSize);
    mrWindow.setRenderer(this);
    mpRenderer->init();
}

GL3DBarChart::~GL3DBarChart()
{
    mrWindow.setRenderer(NULL);
}

void GL3DBarChart::create3DShapes(const boost::ptr_vector<VDataSeries>& rDataSeriesContainer,
        ExplicitCategoriesProvider& rCatProvider)
{
    // Each series of data flows from left to right, and multiple series are
    // stacked vertically along y axis.

    // NOTE: These objects are created and positioned in a totally blind
    // fashion since we don't even have a way to see them on screen.  So, no
    // guarantee they are positioned correctly.  In fact, they are guaranteed
    // to be positioned incorrectly.

    const float nBarSizeX = 10;
    const float nBarSizeY = 10;
    const float nBarDistanceX = nBarSizeX / 2;
    const float nBarDistanceY = nBarSizeY / 2;

    sal_uInt32 nId = 1;
    float nXEnd = 0.0;
    float nYPos = 0.0;

    const Color aSeriesColor[] = {
        COL_RED, COL_GREEN, COL_YELLOW, COL_BROWN, COL_GRAY
    };

    maShapes.clear();
    maShapes.push_back(new opengl3D::Camera(mpRenderer.get()));
    mpCamera = static_cast<opengl3D::Camera*>(&maShapes.back());

    sal_Int32 nSeriesIndex = 0;
    for (boost::ptr_vector<VDataSeries>::const_iterator itr = rDataSeriesContainer.begin(),
            itrEnd = rDataSeriesContainer.end(); itr != itrEnd; ++itr)
    {
        nYPos = nSeriesIndex * (nBarSizeY + nBarDistanceY);

        const VDataSeries& rDataSeries = *itr;
        sal_Int32 nPointCount = rDataSeries.getTotalPointCount();

        bool bMappedFillProperty = rDataSeries.hasPropertyMapping("FillColor");

        // Create series name text object.
        OUString aSeriesName =
            DataSeriesHelper::getDataSeriesLabel(
                rDataSeries.getModel(), mxChartType->getRoleOfSequenceForSeriesLabel());

        maShapes.push_back(new opengl3D::Text(mpRenderer.get(), aSeriesName, nId++));
        opengl3D::Text* p = static_cast<opengl3D::Text*>(&maShapes.back());
        Size aTextSize = p->getSize();
        glm::vec3 aTopLeft, aTopRight, aBottomRight;
        aTopLeft.x = aTextSize.getWidth() * -1.0;
        aTopLeft.y = nYPos;
        aTopRight.y = nYPos;
        aBottomRight = aTopRight;
        aBottomRight.y += aTextSize.getHeight();
        p->setPosition(aTopLeft, aTopRight, aBottomRight);

        sal_Int32 nColor = aSeriesColor[nSeriesIndex % SAL_N_ELEMENTS(aSeriesColor)].GetColor();
        for(sal_Int32 nIndex = 0; nIndex < nPointCount; ++nIndex)
        {
            if(bMappedFillProperty)
            {
                nColor = static_cast<sal_uInt32>(rDataSeries.getValueByProperty(nIndex, "FillColor"));
            }

            float nVal = rDataSeries.getYValue(nIndex);
            float nXPos = nIndex * (nBarSizeX + nBarDistanceX);

            glm::mat4 aScaleMatrix = glm::scale(nBarSizeX, nBarSizeY, nVal);
            glm::mat4 aTranslationMatrix = glm::translate(nXPos, nYPos, 0.0f);
            glm::mat4 aBarPosition = aTranslationMatrix * aScaleMatrix;

            maShapes.push_back(new opengl3D::Bar(mpRenderer.get(), aBarPosition, nColor, nId++));
        }

        float nThisXEnd = nPointCount * (nBarSizeX + nBarDistanceX);
        if (nXEnd < nThisXEnd)
            nXEnd = nThisXEnd;

        ++nSeriesIndex;
    }

    nYPos += nBarSizeY + nBarDistanceY;

    // X axis
    maShapes.push_back(new opengl3D::Line(mpRenderer.get(), nId++));
    opengl3D::Line* pAxis = static_cast<opengl3D::Line*>(&maShapes.back());
    glm::vec3 aBegin;
    aBegin.y = nYPos;
    glm::vec3 aEnd = aBegin;
    aEnd.x = nXEnd;
    pAxis->setPosition(aBegin, aEnd);
    pAxis->setLineColor(COL_BLUE);

    // Y axis
    maShapes.push_back(new opengl3D::Line(mpRenderer.get(), nId++));
    pAxis = static_cast<opengl3D::Line*>(&maShapes.back());
    aBegin.x = aBegin.y = 0;
    aEnd = aBegin;
    aEnd.y = nYPos;
    pAxis->setPosition(aBegin, aEnd);
    pAxis->setLineColor(COL_BLUE);

    // Chart background.
    maShapes.push_back(new opengl3D::Rectangle(mpRenderer.get(), nId++));
    opengl3D::Rectangle* pRect = static_cast<opengl3D::Rectangle*>(&maShapes.back());
    glm::vec3 aTopLeft;
    glm::vec3 aTopRight = aTopLeft;
    aTopRight.x = nXEnd;
    glm::vec3 aBottomRight = aTopRight;
    aBottomRight.y = nYPos;
    pRect->setPosition(aTopLeft, aTopRight, aBottomRight);
    pRect->setFillColor(COL_BLACK);
    pRect->setLineColor(COL_BLUE);

    // Create category texts along X-axis at the bottom.
    uno::Sequence<OUString> aCats = rCatProvider.getSimpleCategories();
    for (sal_Int32 i = 0; i < aCats.getLength(); ++i)
    {
        float nXPos = i * (nBarSizeX + nBarDistanceX);

        maShapes.push_back(new opengl3D::Text(mpRenderer.get(), aCats[i], nId++));
        opengl3D::Text* p = static_cast<opengl3D::Text*>(&maShapes.back());
        Size aTextSize = p->getSize();
        aTopLeft.x = nXPos;
        aTopLeft.y = nYPos;
        aTopRight = aTopLeft;
        aTopRight.x += aTextSize.getWidth();
        aBottomRight = aTopRight;
        aBottomRight.y += aTextSize.getHeight();
        p->setPosition(aTopLeft, aTopRight, aBottomRight);
    }
}

void GL3DBarChart::render()
{
    mrWindow.getContext()->makeCurrent();
    Size aSize = mrWindow.GetSizePixel();
    mpRenderer->SetSize(aSize);
    mrWindow.getContext()->setWinSize(aSize);
    for(boost::ptr_vector<opengl3D::Renderable3DObject>::iterator itr = maShapes.begin(),
            itrEnd = maShapes.end(); itr != itrEnd; ++itr)
    {
        itr->render();
    }
    mpRenderer->ProcessUnrenderedShape();
    mrWindow.getContext()->swapBuffers();
}

void GL3DBarChart::update()
{
    render();
}

namespace {

class PickingModeSetter
{
private:
    opengl3D::OpenGL3DRenderer* mpRenderer;

public:
    PickingModeSetter(opengl3D::OpenGL3DRenderer* pRenderer):
        mpRenderer(pRenderer)
    {
        mpRenderer->SetPickingMode(true);
    }

    ~PickingModeSetter()
    {
        mpRenderer->SetPickingMode(false);
    }
};

}

void GL3DBarChart::clickedAt(const Point& )
{
    {
        PickingModeSetter(mpRenderer.get());
        render();
    }
    if (mpCamera)
        mpCamera->zoom(1);
}

}

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<|endoftext|>


<commit_before>#include "Thread.hpp"

#include <Windows.h>

namespace Util
{
    namespace Thread
    {
        uintmax_t GetCurrentThreadId()
        {
#ifdef _WIN64
            /*
            mov         rax,qword ptr gs:[30h]
            mov         eax,dword ptr [rax+48h]
            */
            return Pointer(__readgsqword(0x30))(0x48).Read<uint64_t>();
#else
            /*
            mov         eax,dword ptr fs:[18h]
            mov         eax,dword ptr [eax+24h]
            */
            return Pointer(__readfsdword(0x18))(0x24).Read<uint32_t>();
#endif
        }

        Pointer GetThreadLocalStorage(size_t Index)
        {
            //#ifdef _WIN64
            //            return Pointer(
            //                __readgsqword(0x1480 + static_cast<uint32_t>(Index) * 8)
            //            );
            //#else
            //            return Pointer(
            //                __readfsdword(0x18)
            //            )(0xE10 + static_cast<uint32_t>(Index) * 4).Read<uintptr_t>();
            //#endif
            return TlsGetValue(Index);
        }
    }
}<commit_msg>TLS slot downcasted to dword<commit_after>#include "Thread.hpp"

#include <Windows.h>

namespace Util
{
    namespace Thread
    {
        uintmax_t GetCurrentThreadId()
        {
#ifdef _WIN64
            /*
            mov         rax,qword ptr gs:[30h]
            mov         eax,dword ptr [rax+48h]
            */
            return Pointer(__readgsqword(0x30))(0x48).Read<uint64_t>();
#else
            /*
            mov         eax,dword ptr fs:[18h]
            mov         eax,dword ptr [eax+24h]
            */
            return Pointer(__readfsdword(0x18))(0x24).Read<uint32_t>();
#endif
        }

        Pointer GetThreadLocalStorage(size_t Index)
        {
            //#ifdef _WIN64
            //            return Pointer(
            //                __readgsqword(0x1480 + static_cast<uint32_t>(Index) * 8)
            //            );
            //#else
            //            return Pointer(
            //                __readfsdword(0x18)
            //            )(0xE10 + static_cast<uint32_t>(Index) * 4).Read<uintptr_t>();
            //#endif
            return TlsGetValue(static_cast<uint32_t>(Index));
        }
    }
}<|endoftext|>


<commit_before>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/extensions/extension_creator.h"

#include <vector>
#include <string>

#include "base/crypto/rsa_private_key.h"
#include "base/crypto/signature_creator.h"
#include "base/file_util.h"
#include "base/scoped_handle.h"
#include "base/string_util.h"
#include "chrome/browser/extensions/sandboxed_extension_unpacker.h"
#include "chrome/common/extensions/extension.h"
#include "chrome/common/zip.h"
#include "net/base/base64.h"

namespace {
  const int kRSAKeySize = 1024;
};

bool ExtensionCreator::InitializeInput(
    const FilePath& extension_dir,
    const FilePath& private_key_path,
    const FilePath& private_key_output_path) {
  // Validate input |extension_dir|.
  if (extension_dir.value().empty() ||
      !file_util::DirectoryExists(extension_dir)) {
    error_message_ = "Input directory must exist.";
    return false;
  }

  // Validate input |private_key| (if provided).
  if (!private_key_path.value().empty() &&
      !file_util::PathExists(private_key_path)) {
    error_message_ = "Input value for private key must be a valid path.";
    return false;
  }

  // If an |output_private_key| path is given, make sure it doesn't over-write
  // an existing private key.
  if (private_key_path.value().empty() &&
      !private_key_output_path.value().empty() &&
      file_util::PathExists(private_key_output_path)) {
      error_message_ = "A private key for specified extension already exists. "
                       "Reuse that key or delete it first.";
      return false;
  }

  return true;
}

base::RSAPrivateKey* ExtensionCreator::ReadInputKey(const FilePath&
    private_key_path) {
  if (!file_util::PathExists(private_key_path)) {
    error_message_ = "Input value for private key must exist.";
    return false;
  }

  std::string private_key_contents;
  if (!file_util::ReadFileToString(private_key_path,
      &private_key_contents)) {
    error_message_ = "Failed to read private key.";
    return false;
  }

  std::string private_key_bytes;
  if (!Extension::ParsePEMKeyBytes(private_key_contents,
       &private_key_bytes)) {
    error_message_ = "Invalid private key.";
    return false;
  }

  return base::RSAPrivateKey::CreateFromPrivateKeyInfo(
      std::vector<uint8>(private_key_bytes.begin(), private_key_bytes.end()));
}

base::RSAPrivateKey* ExtensionCreator::GenerateKey(const FilePath&
    output_private_key_path) {
  scoped_ptr<base::RSAPrivateKey> key_pair(
      base::RSAPrivateKey::Create(kRSAKeySize));
  if (!key_pair.get()) {
    error_message_ = "Yikes! Failed to generate random RSA private key.";
    return NULL;
  }

  std::vector<uint8> private_key_vector;
  if (!key_pair->ExportPrivateKey(&private_key_vector)) {
    error_message_ = "Failed to export private key.";
    return NULL;
  }
  std::string private_key_bytes(
      reinterpret_cast<char*>(&private_key_vector.front()),
      private_key_vector.size());

  std::string private_key;
  if (!Extension::ProducePEM(private_key_bytes, &private_key)) {
    error_message_ = "Failed to output private key.";
    return NULL;
  }
  std::string pem_output;
  if (!Extension::FormatPEMForFileOutput(private_key, &pem_output,
       false)) {
    error_message_ = "Failed to output private key.";
    return NULL;
  }

  if (!output_private_key_path.empty()) {
    if (-1 == file_util::WriteFile(output_private_key_path,
        pem_output.c_str(), pem_output.size())) {
      error_message_ = "Failed to write private key.";
      return NULL;
    }
  }

  return key_pair.release();
}

bool ExtensionCreator::CreateZip(const FilePath& extension_dir,
                                 FilePath* zip_path) {
  file_util::CreateNewTempDirectory(FILE_PATH_LITERAL("chrome_"), zip_path);
  *zip_path = zip_path->Append(FILE_PATH_LITERAL("extension.zip"));

  if (!Zip(extension_dir, *zip_path)) {
    error_message_ = "Failed to create temporary zip file during packaging.";
    return false;
  }

  return true;
}

bool ExtensionCreator::SignZip(const FilePath& zip_path,
                               base::RSAPrivateKey* private_key,
                               std::vector<uint8>* signature) {
  scoped_ptr<base::SignatureCreator> signature_creator(
      base::SignatureCreator::Create(private_key));
  ScopedStdioHandle zip_handle(file_util::OpenFile(zip_path, "rb"));
  uint8 buffer[1 << 16];
  int bytes_read = -1;
  while ((bytes_read = fread(buffer, 1, sizeof(buffer),
       zip_handle.get())) > 0) {
    if (!signature_creator->Update(buffer, bytes_read)) {
      error_message_ = "Error while signing extension.";
      return false;
    }
  }
  zip_handle.Close();

  signature_creator->Final(signature);
  return true;
}

bool ExtensionCreator::WriteCRX(const FilePath& zip_path,
                                base::RSAPrivateKey* private_key,
                                const std::vector<uint8>& signature,
                                const FilePath& crx_path) {
  if (file_util::PathExists(crx_path))
    file_util::Delete(crx_path, false);
  ScopedStdioHandle crx_handle(file_util::OpenFile(crx_path, "wb"));

  std::vector<uint8> public_key;
  if (!private_key->ExportPublicKey(&public_key)) {
    error_message_ = "Failed to export public key.";
    return false;
  }

  SandboxedExtensionUnpacker::ExtensionHeader header;
  memcpy(&header.magic, SandboxedExtensionUnpacker::kExtensionHeaderMagic,
         SandboxedExtensionUnpacker::kExtensionHeaderMagicSize);
  header.version = SandboxedExtensionUnpacker::kCurrentVersion;
  header.key_size = public_key.size();
  header.signature_size = signature.size();

  fwrite(&header, sizeof(SandboxedExtensionUnpacker::ExtensionHeader), 1,
      crx_handle.get());
  fwrite(&public_key.front(), sizeof(uint8), public_key.size(),
      crx_handle.get());
  fwrite(&signature.front(), sizeof(uint8), signature.size(),
      crx_handle.get());

  uint8 buffer[1 << 16];
  int bytes_read = -1;
  ScopedStdioHandle zip_handle(file_util::OpenFile(zip_path, "rb"));
  while ((bytes_read = fread(buffer, 1, sizeof(buffer),
      zip_handle.get())) > 0) {
    fwrite(buffer, sizeof(char), bytes_read, crx_handle.get());
  }

  return true;
}

bool ExtensionCreator::Run(const FilePath& extension_dir,
                           const FilePath& crx_path,
                           const FilePath& private_key_path,
                           const FilePath& output_private_key_path) {
  // Check input diretory and read manifest.
  if (!InitializeInput(extension_dir, private_key_path,
                       output_private_key_path)) {
    return false;
  }

  // Initialize Key Pair
  scoped_ptr<base::RSAPrivateKey> key_pair;
  if (!private_key_path.value().empty())
    key_pair.reset(ReadInputKey(private_key_path));
  else
    key_pair.reset(GenerateKey(output_private_key_path));
  if (!key_pair.get())
    return false;

  // Zip up the extension.
  FilePath zip_path;
  std::vector<uint8> signature;
  bool result = false;
  if (CreateZip(extension_dir, &zip_path) &&
      SignZip(zip_path, key_pair.get(), &signature) &&
      WriteCRX(zip_path, key_pair.get(), signature, crx_path)) {
    result = true;
  }

  file_util::Delete(zip_path, false);
  return result;
}
<commit_msg>Use heap memory intead of stack memory to avoid having to grow the stack.<commit_after>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/extensions/extension_creator.h"

#include <vector>
#include <string>

#include "base/crypto/rsa_private_key.h"
#include "base/crypto/signature_creator.h"
#include "base/file_util.h"
#include "base/scoped_handle.h"
#include "base/string_util.h"
#include "chrome/browser/extensions/sandboxed_extension_unpacker.h"
#include "chrome/common/extensions/extension.h"
#include "chrome/common/zip.h"
#include "net/base/base64.h"

namespace {
  const int kRSAKeySize = 1024;
};

bool ExtensionCreator::InitializeInput(
    const FilePath& extension_dir,
    const FilePath& private_key_path,
    const FilePath& private_key_output_path) {
  // Validate input |extension_dir|.
  if (extension_dir.value().empty() ||
      !file_util::DirectoryExists(extension_dir)) {
    error_message_ = "Input directory must exist.";
    return false;
  }

  // Validate input |private_key| (if provided).
  if (!private_key_path.value().empty() &&
      !file_util::PathExists(private_key_path)) {
    error_message_ = "Input value for private key must be a valid path.";
    return false;
  }

  // If an |output_private_key| path is given, make sure it doesn't over-write
  // an existing private key.
  if (private_key_path.value().empty() &&
      !private_key_output_path.value().empty() &&
      file_util::PathExists(private_key_output_path)) {
      error_message_ = "A private key for specified extension already exists. "
                       "Reuse that key or delete it first.";
      return false;
  }

  return true;
}

base::RSAPrivateKey* ExtensionCreator::ReadInputKey(const FilePath&
    private_key_path) {
  if (!file_util::PathExists(private_key_path)) {
    error_message_ = "Input value for private key must exist.";
    return false;
  }

  std::string private_key_contents;
  if (!file_util::ReadFileToString(private_key_path,
      &private_key_contents)) {
    error_message_ = "Failed to read private key.";
    return false;
  }

  std::string private_key_bytes;
  if (!Extension::ParsePEMKeyBytes(private_key_contents,
       &private_key_bytes)) {
    error_message_ = "Invalid private key.";
    return false;
  }

  return base::RSAPrivateKey::CreateFromPrivateKeyInfo(
      std::vector<uint8>(private_key_bytes.begin(), private_key_bytes.end()));
}

base::RSAPrivateKey* ExtensionCreator::GenerateKey(const FilePath&
    output_private_key_path) {
  scoped_ptr<base::RSAPrivateKey> key_pair(
      base::RSAPrivateKey::Create(kRSAKeySize));
  if (!key_pair.get()) {
    error_message_ = "Yikes! Failed to generate random RSA private key.";
    return NULL;
  }

  std::vector<uint8> private_key_vector;
  if (!key_pair->ExportPrivateKey(&private_key_vector)) {
    error_message_ = "Failed to export private key.";
    return NULL;
  }
  std::string private_key_bytes(
      reinterpret_cast<char*>(&private_key_vector.front()),
      private_key_vector.size());

  std::string private_key;
  if (!Extension::ProducePEM(private_key_bytes, &private_key)) {
    error_message_ = "Failed to output private key.";
    return NULL;
  }
  std::string pem_output;
  if (!Extension::FormatPEMForFileOutput(private_key, &pem_output,
       false)) {
    error_message_ = "Failed to output private key.";
    return NULL;
  }

  if (!output_private_key_path.empty()) {
    if (-1 == file_util::WriteFile(output_private_key_path,
        pem_output.c_str(), pem_output.size())) {
      error_message_ = "Failed to write private key.";
      return NULL;
    }
  }

  return key_pair.release();
}

bool ExtensionCreator::CreateZip(const FilePath& extension_dir,
                                 FilePath* zip_path) {
  file_util::CreateNewTempDirectory(FILE_PATH_LITERAL("chrome_"), zip_path);
  *zip_path = zip_path->Append(FILE_PATH_LITERAL("extension.zip"));

  if (!Zip(extension_dir, *zip_path)) {
    error_message_ = "Failed to create temporary zip file during packaging.";
    return false;
  }

  return true;
}

bool ExtensionCreator::SignZip(const FilePath& zip_path,
                               base::RSAPrivateKey* private_key,
                               std::vector<uint8>* signature) {
  scoped_ptr<base::SignatureCreator> signature_creator(
      base::SignatureCreator::Create(private_key));
  ScopedStdioHandle zip_handle(file_util::OpenFile(zip_path, "rb"));
  size_t buffer_size = 1 << 16;
  scoped_array<uint8> buffer(new uint8[buffer_size]);
  int bytes_read = -1;
  while ((bytes_read = fread(buffer.get(), 1, buffer_size,
       zip_handle.get())) > 0) {
    if (!signature_creator->Update(buffer.get(), bytes_read)) {
      error_message_ = "Error while signing extension.";
      return false;
    }
  }
  zip_handle.Close();

  signature_creator->Final(signature);
  return true;
}

bool ExtensionCreator::WriteCRX(const FilePath& zip_path,
                                base::RSAPrivateKey* private_key,
                                const std::vector<uint8>& signature,
                                const FilePath& crx_path) {
  if (file_util::PathExists(crx_path))
    file_util::Delete(crx_path, false);
  ScopedStdioHandle crx_handle(file_util::OpenFile(crx_path, "wb"));

  std::vector<uint8> public_key;
  if (!private_key->ExportPublicKey(&public_key)) {
    error_message_ = "Failed to export public key.";
    return false;
  }

  SandboxedExtensionUnpacker::ExtensionHeader header;
  memcpy(&header.magic, SandboxedExtensionUnpacker::kExtensionHeaderMagic,
         SandboxedExtensionUnpacker::kExtensionHeaderMagicSize);
  header.version = SandboxedExtensionUnpacker::kCurrentVersion;
  header.key_size = public_key.size();
  header.signature_size = signature.size();

  fwrite(&header, sizeof(SandboxedExtensionUnpacker::ExtensionHeader), 1,
      crx_handle.get());
  fwrite(&public_key.front(), sizeof(uint8), public_key.size(),
      crx_handle.get());
  fwrite(&signature.front(), sizeof(uint8), signature.size(),
      crx_handle.get());

  size_t buffer_size = 1 << 16;
  scoped_array<uint8> buffer(new uint8[buffer_size]);
  int bytes_read = -1;
  ScopedStdioHandle zip_handle(file_util::OpenFile(zip_path, "rb"));
  while ((bytes_read = fread(buffer.get(), 1, buffer_size,
      zip_handle.get())) > 0) {
    fwrite(buffer.get(), sizeof(char), bytes_read, crx_handle.get());
  }

  return true;
}

bool ExtensionCreator::Run(const FilePath& extension_dir,
                           const FilePath& crx_path,
                           const FilePath& private_key_path,
                           const FilePath& output_private_key_path) {
  // Check input diretory and read manifest.
  if (!InitializeInput(extension_dir, private_key_path,
                       output_private_key_path)) {
    return false;
  }

  // Initialize Key Pair
  scoped_ptr<base::RSAPrivateKey> key_pair;
  if (!private_key_path.value().empty())
    key_pair.reset(ReadInputKey(private_key_path));
  else
    key_pair.reset(GenerateKey(output_private_key_path));
  if (!key_pair.get())
    return false;

  // Zip up the extension.
  FilePath zip_path;
  std::vector<uint8> signature;
  bool result = false;
  if (CreateZip(extension_dir, &zip_path) &&
      SignZip(zip_path, key_pair.get(), &signature) &&
      WriteCRX(zip_path, key_pair.get(), signature, crx_path)) {
    result = true;
  }

  file_util::Delete(zip_path, false);
  return result;
}
<|endoftext|>


<commit_before>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/browser.h"
#include "chrome/browser/browser_window.h"
#include "chrome/browser/gtk/view_id_util.h"
#include "chrome/common/url_constants.h"
#include "chrome/test/in_process_browser_test.h"

class ViewIDTest : public InProcessBrowserTest {
 public:
  ViewIDTest() : root_window_(NULL) {}

  void CheckViewID(ViewID id, bool should_have) {
    if (!root_window_)
      root_window_ = GTK_WIDGET(browser()->window()->GetNativeHandle());

    ASSERT_TRUE(root_window_);
    EXPECT_EQ(should_have, !!ViewIDUtil::GetWidget(root_window_, id));
  }

 private:
  GtkWidget* root_window_;
};

IN_PROC_BROWSER_TEST_F(ViewIDTest, Basic) {
  for (int i = VIEW_ID_TOOLBAR; i < VIEW_ID_PREDEFINED_COUNT; ++i) {
    CheckViewID(static_cast<ViewID>(i), true);
  }

  CheckViewID(VIEW_ID_PREDEFINED_COUNT, false);
}

IN_PROC_BROWSER_TEST_F(ViewIDTest, Delegate) {
  CheckViewID(VIEW_ID_TAB_0, true);
  CheckViewID(VIEW_ID_TAB_1, false);

  browser()->OpenURL(GURL(chrome::kAboutBlankURL), GURL(),
                     NEW_BACKGROUND_TAB, PageTransition::TYPED);

  CheckViewID(VIEW_ID_TAB_0, true);
  CheckViewID(VIEW_ID_TAB_1, true);
}
<commit_msg>Fix my view id util browsertest breakage.<commit_after>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/browser.h"
#include "chrome/browser/browser_window.h"
#include "chrome/browser/gtk/view_id_util.h"
#include "chrome/common/url_constants.h"
#include "chrome/test/in_process_browser_test.h"

class ViewIDTest : public InProcessBrowserTest {
 public:
  ViewIDTest() : root_window_(NULL) {}

  void CheckViewID(ViewID id, bool should_have) {
    if (!root_window_)
      root_window_ = GTK_WIDGET(browser()->window()->GetNativeHandle());

    ASSERT_TRUE(root_window_);
    EXPECT_EQ(should_have, !!ViewIDUtil::GetWidget(root_window_, id));
  }

 private:
  GtkWidget* root_window_;
};

IN_PROC_BROWSER_TEST_F(ViewIDTest, Basic) {
  for (int i = VIEW_ID_TOOLBAR; i < VIEW_ID_PREDEFINED_COUNT; ++i) {
    // http://crbug.com/21152
    if (i == VIEW_ID_BOOKMARK_MENU)
      continue;

    CheckViewID(static_cast<ViewID>(i), true);
  }

  CheckViewID(VIEW_ID_PREDEFINED_COUNT, false);
}

IN_PROC_BROWSER_TEST_F(ViewIDTest, Delegate) {
  CheckViewID(VIEW_ID_TAB_0, true);
  CheckViewID(VIEW_ID_TAB_1, false);

  browser()->OpenURL(GURL(chrome::kAboutBlankURL), GURL(),
                     NEW_BACKGROUND_TAB, PageTransition::TYPED);

  CheckViewID(VIEW_ID_TAB_0, true);
  CheckViewID(VIEW_ID_TAB_1, true);
}
<|endoftext|>


<commit_before>AliAnalysisTask *AddTaskHFECal()
{
  AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
  if (!mgr) {
    Error("AddTaskHFEECal", "No analysis manager found.");
    return NULL;
  }

  if (!mgr->GetInputEventHandler()) {
    ::Error("AddTaskHFECal", "This task requires an input event handler");
    return NULL;
  }
  TString type = mgr->GetInputEventHandler()->GetDataType(); // can be "ESD" or "AOD"
  if (type=="AOD"){
    ::Error("AddTaskHFECal", "The tasks exits because AODs are in input");
    return NULL;
  }
  Bool_t MCthere=kFALSE;
  AliMCEventHandler *mcH = dynamic_cast<AliMCEventHandler*>(mgr->GetMCtruthEventHandler());
  if(!mcH){
    MCthere=kFALSE;
  }else{
    MCthere=kTRUE;
  }

  //analysis task
  gROOT->LoadMacro("$ALICE_ROOT/PWGHF/hfe/macros/configs/PbPb/ConfigHFECal.C");

  AliAnalysisTaskHFECal *hfetaskCent = ConfigHFECal(MCthere);
  AliAnalysisTaskHFECal *hfetaskTrig= ConfigHFECal(MCthere);

  mgr->AddTask(hfetaskCent);
  mgr->AddTask(hfetaskTrig);

  // Semi-central trigger
  hfetaskCent->SelectCollisionCandidates(AliVEvent::kCentral);

  TString containerName = mgr->GetCommonFileName();
  containerName += ":PWGHF_hfeCalCentral";

  AliAnalysisDataContainer *cinput = mgr->GetCommonInputContainer();
  AliAnalysisDataContainer *coutput1 = mgr->CreateContainer("HFE_Results_EMCalCentral", TList::Class(),AliAnalysisManager::kOutputContainer, containerName.Data());
  mgr->ConnectInput(hfetaskCent, 0, cinput);
  mgr->ConnectOutput(hfetaskCent, 1, coutput1);

  //L1 gamma trigger
  hfetaskTrig->SelectCollisionCandidates(AliVEvent::kEMCEGA);

  TString containerName2 = mgr->GetCommonFileName();
  containerName2 += ":PWGHF_hfeCalTrigEGA";

  AliAnalysisDataContainer *cinput = mgr->GetCommonInputContainer();
  AliAnalysisDataContainer *coutput1 = mgr->CreateContainer("HFE_Results_EMCalTrigEGA", TList::Class(),AliAnalysisManager::kOutputContainer, containerName2.Data());
  mgr->ConnectInput(hfetaskTrig, 0, cinput);
  mgr->ConnectOutput(hfetaskTrig, 1, coutput1);

  return NULL;
}
<commit_msg>upadted for MC<commit_after>AliAnalysisTask *AddTaskHFECal()
{
  AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
  if (!mgr) {
    Error("AddTaskHFEECal", "No analysis manager found.");
    return NULL;
  }

  if (!mgr->GetInputEventHandler()) {
    ::Error("AddTaskHFECal", "This task requires an input event handler");
    return NULL;
  }
  TString type = mgr->GetInputEventHandler()->GetDataType(); // can be "ESD" or "AOD"
  if (type=="AOD"){
    ::Error("AddTaskHFECal", "The tasks exits because AODs are in input");
    return NULL;
  }
  Bool_t MCthere=kFALSE;
  AliMCEventHandler *mcH = dynamic_cast<AliMCEventHandler*>(mgr->GetMCtruthEventHandler());
  if(!mcH){
    MCthere=kFALSE;
  }else{
    MCthere=kTRUE;
  }

  //analysis task
  gROOT->LoadMacro("$ALICE_ROOT/PWGHF/hfe/macros/configs/PbPb/ConfigHFECal.C");

  AliAnalysisTaskHFECal *hfetaskCent = ConfigHFECal(MCthere);
  AliAnalysisTaskHFECal *hfetaskTrig= ConfigHFECal(MCthere);

  mgr->AddTask(hfetaskCent);
  mgr->AddTask(hfetaskTrig);

  // Semi-central trigger
  hfetaskCent->SelectCollisionCandidates(AliVEvent::kCentral);

  TString containerName = mgr->GetCommonFileName();
  containerName += ":PWGHF_hfeCalCentral";

  AliAnalysisDataContainer *cinput = mgr->GetCommonInputContainer();
  AliAnalysisDataContainer *coutput1 = mgr->CreateContainer("HFE_Results_EMCalCentral", TList::Class(),AliAnalysisManager::kOutputContainer, containerName.Data());
  mgr->ConnectInput(hfetaskCent, 0, cinput);
  mgr->ConnectOutput(hfetaskCent, 1, coutput1);

  //L1 gamma trigger
  hfetaskTrig->SelectCollisionCandidates(AliVEvent::kEMCEGA);

  TString containerName2 = mgr->GetCommonFileName();
  containerName2 += ":PWGHF_hfeCalTrigEGA";

  AliAnalysisDataContainer *cinput = mgr->GetCommonInputContainer();
  AliAnalysisDataContainer *coutput1 = mgr->CreateContainer("HFE_Results_EMCalTrigEGA", TList::Class(),AliAnalysisManager::kOutputContainer, containerName2.Data());
  mgr->ConnectInput(hfetaskTrig, 0, cinput);
  mgr->ConnectOutput(hfetaskTrig, 1, coutput1);

  if(MCthere)
    {
    //MB trigger
    AliAnalysisTaskHFECal *hfetaskMB = ConfigHFECal(MCthere);
    mgr->AddTask(hfetaskMB);
    hfetaskMB->SelectCollisionCandidates(AliVEvent::kMB);

    TString containerName3 = mgr->GetCommonFileName();
    containerName3 += ":PWGHF_hfeCalkMB";

     AliAnalysisDataContainer *cinput = mgr->GetCommonInputContainer();
     AliAnalysisDataContainer *coutput1 = mgr->CreateContainer("HFE_Results_EMCalMB", TList::Class(),AliAnalysisManager::kOutputContainer, containerName3.Data());
     mgr->ConnectInput(hfetaskMB, 0, cinput);
     mgr->ConnectOutput(hfetaskMB, 1, coutput1);
     }

  return NULL;
}
<|endoftext|>