<commit_before>#include "get_new_points.hxx"

#include "eval_weighted.hxx"
#include "poles_prefactor.hxx"
#include "power_prefactor.hxx"

#include "../sdp_solve.hxx"
#include "../ostream_set.hxx"

std::vector<El::BigFloat>
compute_optimal(const std::vector<Positive_Matrix_With_Prefactor> &matrices,
                const std::vector<El::BigFloat> &objectives,
                const std::vector<El::BigFloat> &normalization,
                const SDP_Solver_Parameters &parameters)
{
  size_t num_weights(normalization.size());

  const size_t num_blocks(matrices.size());
  std::vector<El::BigFloat> weights(num_weights, 0);
  std::vector<std::set<El::BigFloat>> points(num_blocks);
  std::vector<std::vector<El::BigFloat>> new_points(num_blocks);

  // Need to have a point at zero and infinity
  // GMP does not have a special infinity value, so we use max double.
  const El::BigFloat infinity(std::numeric_limits<double>::max());
  const El::BigFloat min_x(0), max_x(infinity);
  for(size_t block(0); block < num_blocks; ++block)
    {
      points.at(block).emplace(min_x);

      // points.at(block).emplace(0.1);
      // points.at(block).emplace(1);

      // const int64_t num_points(64);
      // const double dx(1.0/num_points);
      // for(double x(dx); x < 1; x +=dx)
      //   points.at(block).emplace(-log(1-x));

      // const int64_t num_points(64);
      // const double dx(64.0/num_points);
      // for(double x(dx); x <= 64; x +=dx)
      //   points.at(block).emplace(x);

      for(double x(1 / 64.0); x < 64; x *= 1.13878863476 * 1.13878863476)
        points.at(block).emplace(x);

      new_points.at(block).emplace_back(max_x);
    }

  bool has_new_points(true);

  bool is_first_iteration(true);
  while(has_new_points)
    {
      has_new_points = false;
      size_t num_constraints(0);
      std::vector<size_t> matrix_dimensions;
      for(size_t block(0); block != num_blocks; ++block)
        {
          for(auto &point : new_points.at(block))
            {
              points.at(block).emplace(point);
            }
          num_constraints += points.at(block).size();
          matrix_dimensions.insert(matrix_dimensions.end(),
                                   points.at(block).size(),
                                   matrices[block].polynomials.size());
          if(El::mpi::Rank() == 0)
            {
              std::cout << "points: " << block << " " << points.at(block)
                        << "\n";
            }
        }

      // if(El::mpi::Rank() == 0)
      //   {
      //     std::cout << "num_constraints: " << num_constraints << "\n";
      //   }

      std::vector<std::vector<El::BigFloat>> primal_objective_c;
      primal_objective_c.reserve(num_constraints);
      std::vector<El::Matrix<El::BigFloat>> free_var_matrix;
      free_var_matrix.reserve(num_constraints);

      // TODO: This is duplicated from sdp2input/write_output/write_output.cxx
      auto max_normalization(normalization.begin());
      for(auto n(normalization.begin()); n != normalization.end(); ++n)
        {
          if(Abs(*n) > Abs(*max_normalization))
            {
              max_normalization = n;
            }
        }
      const size_t max_index(
        std::distance(normalization.begin(), max_normalization));

      for(size_t block(0); block != num_blocks; ++block)
        {
          const int64_t max_degree([&]() {
            int64_t result(0);
            for(auto &row : matrices[block].polynomials)
              for(auto &column : row)
                for(auto &poly : column)
                  {
                    result = std::max(result, poly.degree());
                  }
            return result;
          }());

          for(auto &x : points.at(block))
            {
              El::BigFloat prefactor([&]() {
                if(x == infinity)
                  {
                    return El::BigFloat(1);
                  }
                return power_prefactor(matrices[block].damped_rational.base, x)
                       * poles_prefactor(matrices[block].damped_rational.poles,
                                         x);
              }());
              const size_t dim(matrices[block].polynomials.size());
              free_var_matrix.emplace_back(
                dim * (dim + 1) / 2,
                matrices[block].polynomials.at(0).at(0).size() - 1);
              auto &free_var(free_var_matrix.back());

              primal_objective_c.emplace_back();
              auto &primal(primal_objective_c.back());

              size_t flattened_matrix_row(0);
              for(size_t matrix_row(0); matrix_row != dim; ++matrix_row)
                for(size_t matrix_column(0); matrix_column <= matrix_row;
                    ++matrix_column)
                  {
                    if(x == infinity)
                      {
                        if(matrices[block]
                             .polynomials.at(matrix_row)
                             .at(matrix_column)
                             .at(max_index)
                             .degree()
                           < max_degree)
                          {
                            primal.push_back(0);
                          }
                        else
                          {
                            primal.push_back(matrices[block]
                                               .polynomials.at(matrix_row)
                                               .at(matrix_column)
                                               .at(max_index)
                                               .coefficients.at(max_degree)
                                             / normalization.at(max_index));
                          }
                        auto &primal_constant(primal.back());
                        for(size_t column(0);
                            column != size_t(free_var.Width()); ++column)
                          {
                            const size_t index(column
                                               + (column < max_index ? 0 : 1));
                            if(matrices[block]
                                 .polynomials.at(matrix_row)
                                 .at(matrix_column)
                                 .at(index)
                                 .degree()
                               < max_degree)
                              {
                                free_var(flattened_matrix_row, column)
                                  = primal_constant * normalization.at(index);
                              }
                            else
                              {
                                free_var(flattened_matrix_row, column)
                                  = primal_constant * normalization.at(index)
                                    - matrices[block]
                                        .polynomials.at(matrix_row)
                                        .at(matrix_column)
                                        .at(index)
                                        .coefficients.at(max_degree);
                              }
                          }
                      }
                    else
                      {
                        primal.push_back(prefactor
                                         * matrices[block]
                                             .polynomials.at(matrix_row)
                                             .at(matrix_column)
                                             .at(max_index)(x)
                                         / normalization.at(max_index));

                        auto &primal_constant(primal.back());
                        for(size_t column(0);
                            column != size_t(free_var.Width()); ++column)
                          {
                            const size_t index(column
                                               + (column < max_index ? 0 : 1));
                            free_var(flattened_matrix_row, column)
                              = primal_constant * normalization.at(index)
                                - prefactor
                                    * matrices[block]
                                        .polynomials.at(matrix_row)
                                        .at(matrix_column)
                                        .at(index)(x);
                          }
                      }
                    ++flattened_matrix_row;
                  }
              // Rescale rows
              const El::BigFloat scaling([&]() {
                El::BigFloat max_value(0);
                size_t flattened_matrix_row(0);
                for(size_t matrix_row(0); matrix_row != dim; ++matrix_row)
                  for(size_t matrix_column(0); matrix_column <= matrix_row;
                      ++matrix_column)
                    {
                      max_value = std::max(
                        max_value, El::Abs(primal.at(flattened_matrix_row)));
                      for(size_t column(0); column != size_t(free_var.Width());
                          ++column)
                        {
                          max_value = std::max(
                            max_value,
                            El::Abs(free_var(flattened_matrix_row, column)));
                        }
                      ++flattened_matrix_row;
                    }
                return 1 / max_value;
              }());
              {
                size_t flattened_matrix_row(0);
                for(size_t matrix_row(0); matrix_row != dim; ++matrix_row)
                  for(size_t matrix_column(0); matrix_column <= matrix_row;
                      ++matrix_column)
                    {
                      primal.at(flattened_matrix_row) *= scaling;
                      for(size_t column(0); column != size_t(free_var.Width());
                          ++column)
                        {
                          free_var(flattened_matrix_row, column) *= scaling;
                        }
                      ++flattened_matrix_row;
                    }
              }
            }
        }

      El::BigFloat objective_const(objectives.at(max_index)
                                   / normalization.at(max_index));
      std::vector<El::BigFloat> dual_objective_b;
      dual_objective_b.reserve(normalization.size() - 1);
      for(size_t index = 0; index < normalization.size(); ++index)
        {
          if(index != max_index)
            {
              dual_objective_b.push_back(objectives.at(index)
                                         - normalization.at(index)
                                             * objective_const);
            }
        }

      // Rescale columns
      std::vector<El::BigFloat> rescaling(num_weights - 1, 0);
      for(auto &matrix : free_var_matrix)
        {
          for(size_t column(0); column < size_t(matrix.Width()); ++column)
            {
              for(size_t row(0); row < size_t(matrix.Height()); ++row)
                {
                  rescaling.at(column) = std::max(
                    rescaling.at(column), El::Abs(matrix(row, column)));
                }
            }
        }

      for(size_t index(0); index != rescaling.size(); ++index)
        {
          rescaling[index] = 1 / rescaling[index];
          dual_objective_b[index] *= rescaling[index];
        }

      for(auto &matrix : free_var_matrix)
        {
          for(size_t row(0); row != size_t(matrix.Height()); ++row)
            {
              for(size_t column(0); column != size_t(matrix.Width()); ++column)
                {
                  matrix(row, column) *= rescaling[column];
                }
            }
        }

      if(is_first_iteration)
        {
          std::vector<size_t> elements_to_keep(1, 0);
          const El::BigFloat tolerance(0.1);
          for(size_t matrix_index(0);
              matrix_index + 1 < free_var_matrix.size();)
            {
              auto &matrix(free_var_matrix[matrix_index]);
              auto &c(primal_objective_c[matrix_index]);

              size_t offset_index(matrix_index + 1);
              for(; offset_index != free_var_matrix.size(); ++offset_index)
                {
                  if(free_var_matrix[offset_index].Height() != matrix.Height())
                    {
                      elements_to_keep.push_back(offset_index);
                      break;
                    }
                  bool should_erase(true);
                  for(size_t row(0);
                      should_erase && row != size_t(matrix.Height()); ++row)
                    {
                      should_erase
                        = should_erase
                          && (El::Abs(c[row]
                                      - primal_objective_c[offset_index][row])
                              < tolerance);
                      for(size_t column(0);
                          should_erase && column != size_t(matrix.Width());
                          ++column)
                        {
                          should_erase
                            = should_erase
                              && (El::Abs(matrix(row, column)
                                          - free_var_matrix[offset_index](
                                              row, column))
                                  < tolerance);
                        }
                    }
                  if(!should_erase)
                    {
                      elements_to_keep.push_back(offset_index);
                      break;
                    }
                }
              matrix_index = offset_index;
            }

          std::vector<std::vector<El::BigFloat>> temp_c;
          temp_c.reserve(elements_to_keep.size());
          std::vector<El::Matrix<El::BigFloat>> temp_matrix;
          temp_matrix.reserve(elements_to_keep.size());
          std::vector<size_t> temp_dimensions;
          temp_dimensions.reserve(elements_to_keep.size());

          for(auto element : elements_to_keep)
            {
              temp_c.push_back(primal_objective_c[element]);
              temp_matrix.push_back(free_var_matrix[element]);
              temp_dimensions.push_back(matrix_dimensions[element]);
            }

          std::vector<std::set<El::BigFloat>> temp_points(num_blocks);
          size_t index(0), temp_num_constraints(0);
          for(size_t block(0); block != num_blocks; ++block)
            {
              for(auto &point : points.at(block))
                {
                  if(std::find(elements_to_keep.begin(),
                               elements_to_keep.end(), index)
                     != elements_to_keep.end())
                    temp_points.at(block).emplace(point);
                  ++index;
                }
              temp_num_constraints += temp_points.at(block).size();
              if(El::mpi::Rank() == 0)
                {
                  std::cout << "points: " << block << " "
                            << temp_points.at(block) << "\n";
                }
            }

          std::swap(temp_c, primal_objective_c);
          std::swap(temp_matrix, free_var_matrix);
          std::swap(temp_dimensions, matrix_dimensions);
          std::swap(temp_points, points);

          is_first_iteration = false;
          // std::cout << "dimensions: " << temp_dimensions.size() << " "
          //           << matrix_dimensions.size() << " "
          //           << temp_num_constraints << "\n";
          // std::cout << matrix_dimensions << "\n";
          // std::cout << points << "\n";
        }
      std::cout << "num_constraints: " << free_var_matrix.size() << "\n";

      Block_Info block_info(matrix_dimensions, parameters.procs_per_node,
                            parameters.proc_granularity, parameters.verbosity);
      El::Grid grid(block_info.mpi_comm.value);

      SDP sdp(objective_const, dual_objective_b, primal_objective_c,
              free_var_matrix, block_info, grid);

      SDP_Solver solver(parameters, block_info, grid,
                        sdp.dual_objective_b.Height());

      for(auto &block : solver.y.blocks)
        {
          if(block.GlobalCol(0) == 0)
            {
              for(size_t row(0); row != size_t(block.LocalHeight()); ++row)
                {
                  size_t global_row(block.GlobalRow(row));
                  const size_t index(global_row
                                     + (global_row < max_index ? 0 : 1));
                  block.SetLocal(row, 0, weights.at(index));
                }
            }
        }

      Timers timers(parameters.verbosity >= Verbosity::debug);
      SDP_Solver_Terminate_Reason reason
        = solver.run(parameters, block_info, sdp, grid, timers);

      if(reason != SDP_Solver_Terminate_Reason::PrimalDualOptimal)
        {
          std::stringstream ss;
          ss << "Can not find solution: " << reason;
          throw std::runtime_error(ss.str());
        }
      // y is duplicated among cores, so only need to print out copy on
      // the root node.
      // THe weight at max_index is determined by the normalization condition
      // dot(norm,weights)=1
      weights.at(max_index) = 1;
      for(size_t block_row(0);
          block_row != size_t(solver.y.blocks.at(0).Height()); ++block_row)
        {
          const size_t index(block_row + (block_row < max_index ? 0 : 1));
          weights.at(index)
            = solver.y.blocks.at(0).Get(block_row, 0) * rescaling[block_row];
          weights.at(max_index) -= weights.at(index) * normalization.at(index);
        }
      weights.at(max_index) /= normalization.at(max_index);

      if(El::mpi::Rank() == 0)
        {
          std::cout.precision(20);
          std::cout << "weight: " << weights << "\n";

          El::BigFloat optimal(0);
          for(size_t index(0); index < objectives.size(); ++index)
            {
              optimal += objectives[index] * weights[index];
            }
          std::cout << "optimal: " << optimal << "\n";
        }
      for(size_t block(0); block != num_blocks; ++block)
        {
          // 0.01 should be a small enough relative error so that we are
          // in the regime of convergence.  Then the error estimates will
          // work
          // Mesh mesh(*(points.at(block).begin()), *(points.at(block).rbegin()),
          Mesh mesh(*(points.at(block).begin()), El::BigFloat(100),
                    [&](const El::BigFloat &x) {
                      return eval_weighted(matrices[block], x, weights);
                    },
                    0.01);
          new_points.at(block) = get_new_points(mesh);
          for(auto &point : new_points.at(block))
            {
              has_new_points
                = has_new_points || (points.at(block).count(point) == 0);
            }
        }
    }
  // if(El::mpi::Rank() == 0)
  //   {
  //     std::cout << "weights: " << weights << "\n";
  //   }
  return weights;
}
<commit_msg>Remove column scaling.  Add dynamics duality gap.<commit_after>#include "get_new_points.hxx"

#include "eval_weighted.hxx"
#include "poles_prefactor.hxx"
#include "power_prefactor.hxx"

#include "../sdp_solve.hxx"
#include "../ostream_set.hxx"

std::vector<El::BigFloat>
compute_optimal(const std::vector<Positive_Matrix_With_Prefactor> &matrices,
                const std::vector<El::BigFloat> &objectives,
                const std::vector<El::BigFloat> &normalization,
                const SDP_Solver_Parameters &parameters_in)
{
  SDP_Solver_Parameters parameters(parameters_in);

  size_t num_weights(normalization.size());

  const size_t num_blocks(matrices.size());
  std::vector<El::BigFloat> weights(num_weights, 0);
  std::vector<std::set<El::BigFloat>> points(num_blocks);
  std::vector<std::vector<El::BigFloat>> new_points(num_blocks);

  // Need to have a point at zero and infinity
  // GMP does not have a special infinity value, so we use max double.
  const El::BigFloat infinity(std::numeric_limits<double>::max());
  const El::BigFloat min_x(0), max_x(infinity);
  for(size_t block(0); block < num_blocks; ++block)
    {
      points.at(block).emplace(min_x);

      // points.at(block).emplace(0.1);
      // points.at(block).emplace(1);

      // const int64_t num_points(64);
      // const double dx(1.0/num_points);
      // for(double x(dx); x < 1; x +=dx)
      //   points.at(block).emplace(-log(1-x));

      // const int64_t num_points(64);
      // const double dx(64.0/num_points);
      // for(double x(dx); x <= 64; x +=dx)
      //   points.at(block).emplace(x);

      for(double x(1 / 64.0); x < 64; x *= 1.13878863476 * 1.13878863476)
        points.at(block).emplace(x);

      new_points.at(block).emplace_back(max_x);
    }

  parameters.duality_gap_threshold = 1.1;
  while(parameters.duality_gap_threshold > parameters_in.duality_gap_threshold)
    {
      size_t num_constraints(0);
      std::vector<size_t> matrix_dimensions;
      for(size_t block(0); block != num_blocks; ++block)
        {
          for(auto &point : new_points.at(block))
            {
              points.at(block).emplace(point);
            }
          num_constraints += points.at(block).size();
          matrix_dimensions.insert(matrix_dimensions.end(),
                                   points.at(block).size(),
                                   matrices[block].polynomials.size());
          if(El::mpi::Rank() == 0)
            {
              std::cout << "points: " << block << " " << points.at(block)
                        << "\n";
            }
        }

      if(El::mpi::Rank() == 0)
        {
          std::cout << "num_constraints: " << num_constraints << "\n";
        }

      std::vector<std::vector<El::BigFloat>> primal_objective_c;
      primal_objective_c.reserve(num_constraints);
      std::vector<El::Matrix<El::BigFloat>> free_var_matrix;
      free_var_matrix.reserve(num_constraints);

      // TODO: This is duplicated from sdp2input/write_output/write_output.cxx
      auto max_normalization(normalization.begin());
      for(auto n(normalization.begin()); n != normalization.end(); ++n)
        {
          if(Abs(*n) > Abs(*max_normalization))
            {
              max_normalization = n;
            }
        }
      const size_t max_index(
        std::distance(normalization.begin(), max_normalization));

      for(size_t block(0); block != num_blocks; ++block)
        {
          const int64_t max_degree([&]() {
            int64_t result(0);
            for(auto &row : matrices[block].polynomials)
              for(auto &column : row)
                for(auto &poly : column)
                  {
                    result = std::max(result, poly.degree());
                  }
            return result;
          }());

          for(auto &x : points.at(block))
            {
              El::BigFloat prefactor([&]() {
                if(x == infinity)
                  {
                    return El::BigFloat(1);
                  }
                return power_prefactor(matrices[block].damped_rational.base, x)
                       * poles_prefactor(matrices[block].damped_rational.poles,
                                         x);
              }());
              const size_t dim(matrices[block].polynomials.size());
              free_var_matrix.emplace_back(
                dim * (dim + 1) / 2,
                matrices[block].polynomials.at(0).at(0).size() - 1);
              auto &free_var(free_var_matrix.back());

              primal_objective_c.emplace_back();
              auto &primal(primal_objective_c.back());

              size_t flattened_matrix_row(0);
              for(size_t matrix_row(0); matrix_row != dim; ++matrix_row)
                for(size_t matrix_column(0); matrix_column <= matrix_row;
                    ++matrix_column)
                  {
                    if(x == infinity)
                      {
                        if(matrices[block]
                             .polynomials.at(matrix_row)
                             .at(matrix_column)
                             .at(max_index)
                             .degree()
                           < max_degree)
                          {
                            primal.push_back(0);
                          }
                        else
                          {
                            primal.push_back(matrices[block]
                                               .polynomials.at(matrix_row)
                                               .at(matrix_column)
                                               .at(max_index)
                                               .coefficients.at(max_degree)
                                             / normalization.at(max_index));
                          }
                        auto &primal_constant(primal.back());
                        for(size_t column(0);
                            column != size_t(free_var.Width()); ++column)
                          {
                            const size_t index(column
                                               + (column < max_index ? 0 : 1));
                            if(matrices[block]
                                 .polynomials.at(matrix_row)
                                 .at(matrix_column)
                                 .at(index)
                                 .degree()
                               < max_degree)
                              {
                                free_var(flattened_matrix_row, column)
                                  = primal_constant * normalization.at(index);
                              }
                            else
                              {
                                free_var(flattened_matrix_row, column)
                                  = primal_constant * normalization.at(index)
                                    - matrices[block]
                                        .polynomials.at(matrix_row)
                                        .at(matrix_column)
                                        .at(index)
                                        .coefficients.at(max_degree);
                              }
                          }
                      }
                    else
                      {
                        primal.push_back(prefactor
                                         * matrices[block]
                                             .polynomials.at(matrix_row)
                                             .at(matrix_column)
                                             .at(max_index)(x)
                                         / normalization.at(max_index));

                        auto &primal_constant(primal.back());
                        for(size_t column(0);
                            column != size_t(free_var.Width()); ++column)
                          {
                            const size_t index(column
                                               + (column < max_index ? 0 : 1));
                            free_var(flattened_matrix_row, column)
                              = primal_constant * normalization.at(index)
                                - prefactor
                                    * matrices[block]
                                        .polynomials.at(matrix_row)
                                        .at(matrix_column)
                                        .at(index)(x);
                          }
                      }
                    ++flattened_matrix_row;
                  }
              // Rescale rows
              const El::BigFloat scaling([&]() {
                El::BigFloat max_value(0);
                size_t flattened_matrix_row(0);
                for(size_t matrix_row(0); matrix_row != dim; ++matrix_row)
                  for(size_t matrix_column(0); matrix_column <= matrix_row;
                      ++matrix_column)
                    {
                      max_value = std::max(
                        max_value, El::Abs(primal.at(flattened_matrix_row)));
                      for(size_t column(0); column != size_t(free_var.Width());
                          ++column)
                        {
                          max_value = std::max(
                            max_value,
                            El::Abs(free_var(flattened_matrix_row, column)));
                        }
                      ++flattened_matrix_row;
                    }
                return 1 / max_value;
              }());
              {
                size_t flattened_matrix_row(0);
                for(size_t matrix_row(0); matrix_row != dim; ++matrix_row)
                  for(size_t matrix_column(0); matrix_column <= matrix_row;
                      ++matrix_column)
                    {
                      primal.at(flattened_matrix_row) *= scaling;
                      for(size_t column(0); column != size_t(free_var.Width());
                          ++column)
                        {
                          free_var(flattened_matrix_row, column) *= scaling;
                        }
                      ++flattened_matrix_row;
                    }
              }
            }
        }

      El::BigFloat objective_const(objectives.at(max_index)
                                   / normalization.at(max_index));
      std::vector<El::BigFloat> dual_objective_b;
      dual_objective_b.reserve(normalization.size() - 1);
      for(size_t index = 0; index < normalization.size(); ++index)
        {
          if(index != max_index)
            {
              dual_objective_b.push_back(objectives.at(index)
                                         - normalization.at(index)
                                             * objective_const);
            }
        }

      Block_Info block_info(matrix_dimensions, parameters.procs_per_node,
                            parameters.proc_granularity, parameters.verbosity);
      El::Grid grid(block_info.mpi_comm.value);

      SDP sdp(objective_const, dual_objective_b, primal_objective_c,
              free_var_matrix, block_info, grid);

      SDP_Solver solver(parameters, block_info, grid,
                        sdp.dual_objective_b.Height());

      for(auto &block : solver.y.blocks)
        {
          if(block.GlobalCol(0) == 0)
            {
              for(size_t row(0); row != size_t(block.LocalHeight()); ++row)
                {
                  size_t global_row(block.GlobalRow(row));
                  const size_t index(global_row
                                     + (global_row < max_index ? 0 : 1));
                  block.SetLocal(row, 0, weights.at(index));
                }
            }
        }

      Timers timers(parameters.verbosity >= Verbosity::debug);
      SDP_Solver_Terminate_Reason reason
        = solver.run(parameters, block_info, sdp, grid, timers);

      if(El::mpi::Rank() == 0)
        {
          set_stream_precision(std::cout);
          std::cout << "-----" << reason << "-----\n"
                    << '\n'
                    << "primalObjective = " << solver.primal_objective << '\n'
                    << "dualObjective   = " << solver.dual_objective << '\n'
                    << "dualityGap      = " << solver.duality_gap << '\n'
                    << "primalError     = " << solver.primal_error() << '\n'
                    << "dualError       = " << solver.dual_error << '\n'
                    << '\n';
        }

      // if(reason != SDP_Solver_Terminate_Reason::PrimalDualOptimal)
      //   {
      //     std::stringstream ss;
      //     ss << "Can not find solution: " << reason;
      //     throw std::runtime_error(ss.str());
      //   }

      // y is duplicated among cores, so only need to print out copy on
      // the root node.
      // THe weight at max_index is determined by the normalization condition
      // dot(norm,weights)=1
      weights.at(max_index) = 1;
      for(size_t block_row(0);
          block_row != size_t(solver.y.blocks.at(0).Height()); ++block_row)
        {
          const size_t index(block_row + (block_row < max_index ? 0 : 1));
          weights.at(index) = solver.y.blocks.at(0).Get(block_row, 0);
          // weights.at(index)
          //   = solver.y.blocks.at(0).Get(block_row, 0) * rescaling[block_row];
          weights.at(max_index) -= weights.at(index) * normalization.at(index);
        }
      weights.at(max_index) /= normalization.at(max_index);

      if(El::mpi::Rank() == 0)
        {
          std::cout.precision(20);
          std::cout << "weight: " << weights << "\n";

          El::BigFloat optimal(0);
          for(size_t index(0); index < objectives.size(); ++index)
            {
              optimal += objectives[index] * weights[index];
            }
          std::cout << "optimal: " << optimal << "\n";
        }
      bool has_new_points(false);
      for(size_t block(0); block != num_blocks; ++block)
        {
          // 0.01 should be a small enough relative error so that we are
          // in the regime of convergence.  Then the error estimates will
          // work
          // Mesh mesh(*(points.at(block).begin()), *(points.at(block).rbegin()),
          Mesh mesh(*(points.at(block).begin()), El::BigFloat(100),
                    [&](const El::BigFloat &x) {
                      return eval_weighted(matrices[block], x, weights);
                    },
                    (1.0 / 128));
          new_points.at(block) = get_new_points(mesh);
          for(auto &point : new_points.at(block))
            {
              has_new_points
                = has_new_points || (points.at(block).count(point) == 0);
            }
        }
      if(!has_new_points)
        {
          parameters.duality_gap_threshold *= (1.0 / 8);
        }
    }
  return weights;
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/logging.h"
#include "chrome/browser/extensions/extension_apitest.h"
#include "chrome/browser/ui/browser.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/test/base/ui_test_utils.h"
#include "content/public/browser/web_contents.h"
#include "googleurl/src/gurl.h"
#include "net/base/mock_host_resolver.h"

class ExtensionResourceRequestPolicyTest : public ExtensionApiTest {
 protected:
  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
    ExtensionApiTest::SetUpCommandLine(command_line);
    command_line->AppendSwitch(switches::kAllowLegacyExtensionManifests);
  }
};

// Note, this mostly tests the logic of chrome/renderer/extensions/
// extension_resource_request_policy.*, but we have it as a browser test so that
// can make sure it works end-to-end.
IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest, OriginPrivileges) {
  host_resolver()->AddRule("*", "127.0.0.1");
  ASSERT_TRUE(test_server()->Start());
  ASSERT_TRUE(LoadExtension(test_data_dir_
      .AppendASCII("extension_resource_request_policy")
      .AppendASCII("extension")));

  GURL web_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "index.html"));

  std::string host_a("a.com");
  GURL::Replacements make_host_a_com;
  make_host_a_com.SetHostStr(host_a);

  std::string host_b("b.com");
  GURL::Replacements make_host_b_com;
  make_host_b_com.SetHostStr(host_b);

  // A web host that has permission.
  ui_test_utils::NavigateToURL(
      browser(), web_resource.ReplaceComponents(make_host_a_com));
  std::string result;
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ(result, "Loaded");

  // A web host that loads a non-existent extension.
  GURL non_existent_extension(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "non_existent_extension.html"));
  ui_test_utils::NavigateToURL(browser(), non_existent_extension);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ(result, "Image failed to load");

  // A data URL. Data URLs should always be able to load chrome-extension://
  // resources.
  std::string file_source;
  ASSERT_TRUE(file_util::ReadFileToString(
      test_data_dir_.AppendASCII("extension_resource_request_policy")
                    .AppendASCII("index.html"), &file_source));
  ui_test_utils::NavigateToURL(browser(),
      GURL(std::string("data:text/html;charset=utf-8,") + file_source));
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
      browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
      &result));
  EXPECT_EQ(result, "Loaded");

  // A different extension. Extensions should always be able to load each
  // other's resources.
  ASSERT_TRUE(LoadExtension(test_data_dir_
      .AppendASCII("extension_resource_request_policy")
      .AppendASCII("extension2")));
  ui_test_utils::NavigateToURL(
      browser(),
      GURL("chrome-extension://pbkkcbgdkliohhfaeefcijaghglkahja/index.html"));
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
      browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
      &result));
  EXPECT_EQ(result, "Loaded");
}

IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest,
                       ExtensionCanLoadHostedAppIcons) {
  ASSERT_TRUE(LoadExtension(test_data_dir_
      .AppendASCII("extension_resource_request_policy")
      .AppendASCII("extension")));

  ASSERT_TRUE(RunExtensionSubtest(
      "extension_resource_request_policy/extension2/",
      "can_load_icons_from_hosted_apps.html"));
}

IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest, Audio) {
  EXPECT_TRUE(RunExtensionSubtest(
      "extension_resource_request_policy/extension2",
      "audio.html"));
}

#if defined(OS_MACOSX)
// http://crbug.com/95274 - Video is flaky on Mac.
#define MAYBE_Video DISABLED_Video
#else
#define MAYBE_Video Video
#endif

IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest, MAYBE_Video) {
  EXPECT_TRUE(RunExtensionSubtest(
      "extension_resource_request_policy/extension2",
      "video.html"));
}

IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest,
                       WebAccessibleResources) {
  std::string result;
  ASSERT_TRUE(test_server()->Start());
  ASSERT_TRUE(LoadExtension(test_data_dir_
      .AppendASCII("extension_resource_request_policy")
      .AppendASCII("web_accessible")));

  GURL accessible_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/accessible_resource.html"));
  ui_test_utils::NavigateToURL(browser(), accessible_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("Loaded", result);

  GURL xhr_accessible_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/xhr_accessible_resource.html"));
  ui_test_utils::NavigateToURL(
      browser(), xhr_accessible_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("XHR completed with status: 200", result);

  GURL xhr_inaccessible_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/xhr_inaccessible_resource.html"));
  ui_test_utils::NavigateToURL(
      browser(), xhr_inaccessible_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("XHR failed to load resource", result);

  GURL nonaccessible_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/nonaccessible_resource.html"));
  ui_test_utils::NavigateToURL(browser(), nonaccessible_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("Image failed to load", result);

  GURL nonexistent_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/nonexistent_resource.html"));
  ui_test_utils::NavigateToURL(browser(), nonexistent_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("Image failed to load", result);
}

<commit_msg>Disable ExtensionResourceRequestPolicyTest.WebAccessibleResources on Windows. It times out regularly on the win_rel trybots.<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/logging.h"
#include "chrome/browser/extensions/extension_apitest.h"
#include "chrome/browser/ui/browser.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/test/base/ui_test_utils.h"
#include "content/public/browser/web_contents.h"
#include "googleurl/src/gurl.h"
#include "net/base/mock_host_resolver.h"

class ExtensionResourceRequestPolicyTest : public ExtensionApiTest {
 protected:
  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
    ExtensionApiTest::SetUpCommandLine(command_line);
    command_line->AppendSwitch(switches::kAllowLegacyExtensionManifests);
  }
};

// Note, this mostly tests the logic of chrome/renderer/extensions/
// extension_resource_request_policy.*, but we have it as a browser test so that
// can make sure it works end-to-end.
IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest, OriginPrivileges) {
  host_resolver()->AddRule("*", "127.0.0.1");
  ASSERT_TRUE(test_server()->Start());
  ASSERT_TRUE(LoadExtension(test_data_dir_
      .AppendASCII("extension_resource_request_policy")
      .AppendASCII("extension")));

  GURL web_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "index.html"));

  std::string host_a("a.com");
  GURL::Replacements make_host_a_com;
  make_host_a_com.SetHostStr(host_a);

  std::string host_b("b.com");
  GURL::Replacements make_host_b_com;
  make_host_b_com.SetHostStr(host_b);

  // A web host that has permission.
  ui_test_utils::NavigateToURL(
      browser(), web_resource.ReplaceComponents(make_host_a_com));
  std::string result;
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ(result, "Loaded");

  // A web host that loads a non-existent extension.
  GURL non_existent_extension(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "non_existent_extension.html"));
  ui_test_utils::NavigateToURL(browser(), non_existent_extension);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ(result, "Image failed to load");

  // A data URL. Data URLs should always be able to load chrome-extension://
  // resources.
  std::string file_source;
  ASSERT_TRUE(file_util::ReadFileToString(
      test_data_dir_.AppendASCII("extension_resource_request_policy")
                    .AppendASCII("index.html"), &file_source));
  ui_test_utils::NavigateToURL(browser(),
      GURL(std::string("data:text/html;charset=utf-8,") + file_source));
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
      browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
      &result));
  EXPECT_EQ(result, "Loaded");

  // A different extension. Extensions should always be able to load each
  // other's resources.
  ASSERT_TRUE(LoadExtension(test_data_dir_
      .AppendASCII("extension_resource_request_policy")
      .AppendASCII("extension2")));
  ui_test_utils::NavigateToURL(
      browser(),
      GURL("chrome-extension://pbkkcbgdkliohhfaeefcijaghglkahja/index.html"));
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
      browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
      &result));
  EXPECT_EQ(result, "Loaded");
}

IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest,
                       ExtensionCanLoadHostedAppIcons) {
  ASSERT_TRUE(LoadExtension(test_data_dir_
      .AppendASCII("extension_resource_request_policy")
      .AppendASCII("extension")));

  ASSERT_TRUE(RunExtensionSubtest(
      "extension_resource_request_policy/extension2/",
      "can_load_icons_from_hosted_apps.html"));
}

IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest, Audio) {
  EXPECT_TRUE(RunExtensionSubtest(
      "extension_resource_request_policy/extension2",
      "audio.html"));
}

#if defined(OS_MACOSX)
// http://crbug.com/95274 - Video is flaky on Mac.
#define MAYBE_Video DISABLED_Video
#else
#define MAYBE_Video Video
#endif

IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest, MAYBE_Video) {
  EXPECT_TRUE(RunExtensionSubtest(
      "extension_resource_request_policy/extension2",
      "video.html"));
}

// This test times out regularly on win_rel trybots. See http://crbug.com/122154
#if defined(OS_WIN)
#define MAYBE_WebAccessibleResources DISABLED_WebAccessibleResources
#else
#define MAYBE_WebAccessibleResources WebAccessibleResources
#endif
IN_PROC_BROWSER_TEST_F(ExtensionResourceRequestPolicyTest,
                       MAYBE_WebAccessibleResources) {
  std::string result;
  ASSERT_TRUE(test_server()->Start());
  ASSERT_TRUE(LoadExtension(test_data_dir_
      .AppendASCII("extension_resource_request_policy")
      .AppendASCII("web_accessible")));

  GURL accessible_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/accessible_resource.html"));
  ui_test_utils::NavigateToURL(browser(), accessible_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("Loaded", result);

  GURL xhr_accessible_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/xhr_accessible_resource.html"));
  ui_test_utils::NavigateToURL(
      browser(), xhr_accessible_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("XHR completed with status: 200", result);

  GURL xhr_inaccessible_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/xhr_inaccessible_resource.html"));
  ui_test_utils::NavigateToURL(
      browser(), xhr_inaccessible_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("XHR failed to load resource", result);

  GURL nonaccessible_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/nonaccessible_resource.html"));
  ui_test_utils::NavigateToURL(browser(), nonaccessible_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("Image failed to load", result);

  GURL nonexistent_resource(
      test_server()->GetURL(
          "files/extensions/api_test/extension_resource_request_policy/"
          "web_accessible/nonexistent_resource.html"));
  ui_test_utils::NavigateToURL(browser(), nonexistent_resource);
  ASSERT_TRUE(ui_test_utils::ExecuteJavaScriptAndExtractString(
    browser()->GetSelectedWebContents()->GetRenderViewHost(), L"",
      L"window.domAutomationController.send(document.title)",
    &result));
  EXPECT_EQ("Image failed to load", result);
}

<|endoftext|>


<commit_before>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/net/websocket_experiment/websocket_experiment_runner.h"

#include "base/compiler_specific.h"
#include "base/histogram.h"
#include "base/message_loop.h"
#include "base/task.h"
#include "chrome/browser/chrome_thread.h"
#include "net/base/net_errors.h"

namespace chrome_browser_net_websocket_experiment {

static const char *kExperimentHost = "websocket-experiment.chromium.org";
static const int kAlternativePort = 61985;

static const int kUrlFetchDeadlineSec = 10;
static const int kWebSocketConnectDeadlineSec = 10;
static const int kWebSocketEchoDeadlineSec = 5;
static const int kWebSocketIdleSec = 1;
static const int kWebSocketPushDeadlineSec = 1;
static const int kWebSocketByeDeadlineSec = 10;
static const int kWebSocketCloseDeadlineSec = 5;
static const int kWebSocketTimeSec = 10;
static const int kTimeBucketCount = 50;

// TODO(ukai): Use new thread-safe-reference-counted Histograms.
#define UPDATE_HISTOGRAM(name, sample, min, max, bucket_count) do { \
    switch (task_state_) {                                          \
      case STATE_RUN_WS:                                            \
        {                                                           \
          static LinearHistogram counter(                               \
              "WebSocketExperiment.Basic." name, min, max, bucket_count); \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.Add(sample);                                          \
        }                                                               \
        break;                                                          \
      case STATE_RUN_WSS:                                               \
        {                                                               \
          static LinearHistogram counter(                               \
              "WebSocketExperiment.Secure." name, min, max, bucket_count); \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.Add(sample);                                          \
        }                                                               \
        break;                                                          \
      case STATE_RUN_WS_NODEFAULT_PORT:                                 \
        {                                                               \
          static LinearHistogram counter(                               \
              "WebSocketExperiment.NoDefaultPort." name,                \
              min, max, bucket_count);                                  \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.Add(sample);                                          \
        }                                                               \
        break;                                                          \
      default:                                                          \
        NOTREACHED();                                                   \
        break;                                                          \
    }                                                                   \
  } while (0)

#define UPDATE_HISTOGRAM_TIMES(name, sample, min, max, bucket_count) do {    \
    switch (task_state_) {                                              \
      case STATE_RUN_WS:                                                \
        {                                                               \
          static Histogram counter(                                     \
              "WebSocketExperiment.Basic." name, min, max, bucket_count); \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.AddTime(sample);                                      \
        }                                                               \
        break;                                                          \
      case STATE_RUN_WSS:                                               \
        {                                                               \
          static Histogram counter(                                     \
              "WebSocketExperiment.Secure." name, min, max, bucket_count); \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.AddTime(sample);                                      \
        }                                                               \
        break;                                                          \
      case STATE_RUN_WS_NODEFAULT_PORT:                                 \
        {                                                               \
          static Histogram counter(                                     \
              "WebSocketExperiment.NoDefaultPort." name,                \
              min, max, bucket_count);                                  \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.AddTime(sample);                                      \
        }                                                               \
        break;                                                          \
      default:                                                          \
        NOTREACHED();                                                   \
        break;                                                          \
    }                                                                   \
  } while (0);

// Hold reference while experiment is running.
static scoped_refptr<WebSocketExperimentRunner> runner;

/* static */
void WebSocketExperimentRunner::Start() {
  DCHECK(!runner.get());
  runner = new WebSocketExperimentRunner;
  runner->Run();
}

/* static */
void WebSocketExperimentRunner::Stop() {
  if (runner.get())
    runner->Cancel();
  runner = NULL;
}

WebSocketExperimentRunner::WebSocketExperimentRunner()
    : next_state_(STATE_NONE),
      task_state_(STATE_NONE),
      ALLOW_THIS_IN_INITIALIZER_LIST(
          task_callback_(this, &WebSocketExperimentRunner::OnTaskCompleted)) {
  InitConfig();
}

WebSocketExperimentRunner::~WebSocketExperimentRunner() {
  DCHECK(!task_.get());
}

void WebSocketExperimentRunner::Run() {
  DCHECK_EQ(next_state_, STATE_NONE);
  next_state_ = STATE_RUN_WS;
  ChromeThread::PostDelayedTask(
      ChromeThread::IO,
      FROM_HERE,
      NewRunnableMethod(this, &WebSocketExperimentRunner::DoLoop),
      config_.initial_delay_ms);
}

void WebSocketExperimentRunner::Cancel() {
  next_state_ = STATE_NONE;
  ChromeThread::PostTask(
      ChromeThread::IO,
      FROM_HERE,
      NewRunnableMethod(this, &WebSocketExperimentRunner::DoLoop));
}

void WebSocketExperimentRunner::InitConfig() {
  config_.initial_delay_ms = 5 * 60 * 1000;  // 5 mins
  config_.next_delay_ms = 12 * 60 * 60 * 1000;  // 12 hours

  WebSocketExperimentTask::Config task_config;
  task_config.ws_protocol = "google-websocket-liveexperiment";
  task_config.ws_origin = "http://dev.chromium.org/";
  task_config.url_fetch_deadline_ms = kUrlFetchDeadlineSec * 1000;
  task_config.websocket_onopen_deadline_ms =
      kWebSocketConnectDeadlineSec * 1000;
  task_config.websocket_hello_message = "Hello";
  task_config.websocket_hello_echoback_deadline_ms =
      kWebSocketEchoDeadlineSec * 1000;
  // Note: wait 1.5 sec in websocket_experiment_def.txt
  task_config.websocket_idle_ms = kWebSocketIdleSec * 1000;
  task_config.websocket_receive_push_message_deadline_ms =
      kWebSocketPushDeadlineSec * 1000;
  task_config.websocket_bye_message = "Bye";
  task_config.websocket_bye_deadline_ms =
      kWebSocketByeDeadlineSec * 1000;
  task_config.websocket_close_deadline_ms =
      kWebSocketCloseDeadlineSec * 1000;

  config_.ws_config = task_config;
  config_.ws_config.url =
      GURL(StringPrintf("ws://%s/live_exp", kExperimentHost));
  config_.ws_config.ws_location =
      StringPrintf("ws://%s/live_exp", kExperimentHost);
  config_.ws_config.http_url =
      GURL(StringPrintf("http://%s/", kExperimentHost));

  config_.wss_config = task_config;
  config_.wss_config.url =
      GURL(StringPrintf("wss://%s/live_exp", kExperimentHost));
  config_.wss_config.ws_location =
      StringPrintf("wss://%s/live_exp", kExperimentHost);
  config_.wss_config.http_url =
      GURL(StringPrintf("https://%s/", kExperimentHost));

  config_.ws_nondefault_config = task_config;
  config_.ws_nondefault_config.url =
      GURL(StringPrintf("ws://%s:%d/live_exp",
                        kExperimentHost, kAlternativePort));
  config_.ws_nondefault_config.ws_location =
      StringPrintf("ws://%s:%d/live_exp",
                   kExperimentHost, kAlternativePort);
  config_.ws_nondefault_config.http_url =
      GURL(StringPrintf("http://%s:%d/",
                        kExperimentHost, kAlternativePort));
}

void WebSocketExperimentRunner::DoLoop() {
  if (next_state_ == STATE_NONE) {
    if (task_.get()) {
      AddRef();  // Release in OnTaskCompleted.
      task_->Cancel();
    }
    return;
  }

  State state = next_state_;
  task_state_ = STATE_NONE;
  next_state_ = STATE_NONE;

  switch (state) {
    case STATE_IDLE:
      task_.reset();
      next_state_ = STATE_RUN_WS;
      ChromeThread::PostDelayedTask(
          ChromeThread::IO,
          FROM_HERE,
          NewRunnableMethod(this, &WebSocketExperimentRunner::DoLoop),
          config_.next_delay_ms);
      break;
    case STATE_RUN_WS:
      task_.reset(new WebSocketExperimentTask(config_.ws_config,
                                              &task_callback_));
      task_state_ = STATE_RUN_WS;
      next_state_ = STATE_RUN_WSS;
      break;
    case STATE_RUN_WSS:
      task_.reset(new WebSocketExperimentTask(config_.wss_config,
                                              &task_callback_));
      task_state_ = STATE_RUN_WSS;
      next_state_ = STATE_RUN_WS_NODEFAULT_PORT;
      break;
    case STATE_RUN_WS_NODEFAULT_PORT:
      task_.reset(new WebSocketExperimentTask(config_.ws_nondefault_config,
                                              &task_callback_));
      task_state_ = STATE_RUN_WS_NODEFAULT_PORT;
      next_state_ = STATE_IDLE;
      break;
    default:
      NOTREACHED();
      break;
  }
  if (task_.get())
    task_->Run();
}

void WebSocketExperimentRunner::OnTaskCompleted(int result) {
  if (result == net::ERR_ABORTED) {
    task_.reset();
    // Task is Canceled.
    DLOG(INFO) << "WebSocketExperiment Task is canceled.";
    Release();
    return;
  }
  UpdateTaskResultHistogram(task_.get());
  task_.reset();

  DoLoop();
}

void WebSocketExperimentRunner::UpdateTaskResultHistogram(
    const WebSocketExperimentTask* task) {
  DCHECK(task);
  const WebSocketExperimentTask::Result& task_result = task->result();

  UPDATE_HISTOGRAM("LastState", task_result.last_state,
                   1, WebSocketExperimentTask::NUM_STATES,
                   WebSocketExperimentTask::NUM_STATES + 1);

  UPDATE_HISTOGRAM_TIMES("UrlFetch", task_result.url_fetch,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(kUrlFetchDeadlineSec),
                         kTimeBucketCount);

  if (task_result.last_state <
      WebSocketExperimentTask::STATE_WEBSOCKET_CONNECT_COMPLETE)
    return;

  UPDATE_HISTOGRAM_TIMES("WebSocketConnect", task_result.websocket_connect,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(
                             kWebSocketConnectDeadlineSec),
                         kTimeBucketCount);

  if (task_result.last_state <
      WebSocketExperimentTask::STATE_WEBSOCKET_RECV_HELLO)
    return;

  UPDATE_HISTOGRAM_TIMES("WebSocketEcho", task_result.websocket_echo,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(
                             kWebSocketEchoDeadlineSec),
                         kTimeBucketCount);

  if (task_result.last_state <
      WebSocketExperimentTask::STATE_WEBSOCKET_KEEP_IDLE)
    return;

  UPDATE_HISTOGRAM_TIMES("WebSocketIdle", task_result.websocket_idle,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(
                             kWebSocketIdleSec + kWebSocketPushDeadlineSec),
                         kTimeBucketCount);

  if (task_result.last_state <
      WebSocketExperimentTask::STATE_WEBSOCKET_CLOSE_COMPLETE)
    return;

  UPDATE_HISTOGRAM_TIMES("WebSocketTotal", task_result.websocket_total,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(kWebSocketTimeSec),
                         kTimeBucketCount);
}

}  // namespace chrome_browser_net_websocket_experiment
<commit_msg>Throttle websocket live experiment<commit_after>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/net/websocket_experiment/websocket_experiment_runner.h"

#include "base/compiler_specific.h"
#include "base/field_trial.h"
#include "base/histogram.h"
#include "base/message_loop.h"
#include "base/task.h"
#include "chrome/browser/chrome_thread.h"
#include "net/base/net_errors.h"

namespace chrome_browser_net_websocket_experiment {

static const char *kExperimentHost = "websocket-experiment.chromium.org";
static const int kAlternativePort = 61985;

static const int kUrlFetchDeadlineSec = 10;
static const int kWebSocketConnectDeadlineSec = 10;
static const int kWebSocketEchoDeadlineSec = 5;
static const int kWebSocketIdleSec = 1;
static const int kWebSocketPushDeadlineSec = 1;
static const int kWebSocketByeDeadlineSec = 10;
static const int kWebSocketCloseDeadlineSec = 5;
static const int kWebSocketTimeSec = 10;
static const int kTimeBucketCount = 50;

// TODO(ukai): Use new thread-safe-reference-counted Histograms.
#define UPDATE_HISTOGRAM(name, sample, min, max, bucket_count) do { \
    switch (task_state_) {                                          \
      case STATE_RUN_WS:                                            \
        {                                                           \
          static LinearHistogram counter(                               \
              "WebSocketExperiment.Basic." name, min, max, bucket_count); \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.Add(sample);                                          \
        }                                                               \
        break;                                                          \
      case STATE_RUN_WSS:                                               \
        {                                                               \
          static LinearHistogram counter(                               \
              "WebSocketExperiment.Secure." name, min, max, bucket_count); \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.Add(sample);                                          \
        }                                                               \
        break;                                                          \
      case STATE_RUN_WS_NODEFAULT_PORT:                                 \
        {                                                               \
          static LinearHistogram counter(                               \
              "WebSocketExperiment.NoDefaultPort." name,                \
              min, max, bucket_count);                                  \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.Add(sample);                                          \
        }                                                               \
        break;                                                          \
      default:                                                          \
        NOTREACHED();                                                   \
        break;                                                          \
    }                                                                   \
  } while (0)

#define UPDATE_HISTOGRAM_TIMES(name, sample, min, max, bucket_count) do {    \
    switch (task_state_) {                                              \
      case STATE_RUN_WS:                                                \
        {                                                               \
          static Histogram counter(                                     \
              "WebSocketExperiment.Basic." name, min, max, bucket_count); \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.AddTime(sample);                                      \
        }                                                               \
        break;                                                          \
      case STATE_RUN_WSS:                                               \
        {                                                               \
          static Histogram counter(                                     \
              "WebSocketExperiment.Secure." name, min, max, bucket_count); \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.AddTime(sample);                                      \
        }                                                               \
        break;                                                          \
      case STATE_RUN_WS_NODEFAULT_PORT:                                 \
        {                                                               \
          static Histogram counter(                                     \
              "WebSocketExperiment.NoDefaultPort." name,                \
              min, max, bucket_count);                                  \
          counter.SetFlags(kUmaTargetedHistogramFlag);                  \
          counter.AddTime(sample);                                      \
        }                                                               \
        break;                                                          \
      default:                                                          \
        NOTREACHED();                                                   \
        break;                                                          \
    }                                                                   \
  } while (0);

// Hold reference while experiment is running.
static scoped_refptr<WebSocketExperimentRunner> runner;

/* static */
void WebSocketExperimentRunner::Start() {
  DCHECK(!runner.get());

  scoped_refptr<FieldTrial> trial = new FieldTrial("WebSocketExperiment", 1000);
  trial->AppendGroup("_active", 5);  // 0.5% in _active group.

  if (trial->group() == FieldTrial::kNotParticipating)
    return;

  runner = new WebSocketExperimentRunner;
  runner->Run();
}

/* static */
void WebSocketExperimentRunner::Stop() {
  if (runner.get())
    runner->Cancel();
  runner = NULL;
}

WebSocketExperimentRunner::WebSocketExperimentRunner()
    : next_state_(STATE_NONE),
      task_state_(STATE_NONE),
      ALLOW_THIS_IN_INITIALIZER_LIST(
          task_callback_(this, &WebSocketExperimentRunner::OnTaskCompleted)) {
  InitConfig();
}

WebSocketExperimentRunner::~WebSocketExperimentRunner() {
  DCHECK(!task_.get());
}

void WebSocketExperimentRunner::Run() {
  DCHECK_EQ(next_state_, STATE_NONE);
  next_state_ = STATE_RUN_WS;
  ChromeThread::PostDelayedTask(
      ChromeThread::IO,
      FROM_HERE,
      NewRunnableMethod(this, &WebSocketExperimentRunner::DoLoop),
      config_.initial_delay_ms);
}

void WebSocketExperimentRunner::Cancel() {
  next_state_ = STATE_NONE;
  ChromeThread::PostTask(
      ChromeThread::IO,
      FROM_HERE,
      NewRunnableMethod(this, &WebSocketExperimentRunner::DoLoop));
}

void WebSocketExperimentRunner::InitConfig() {
  config_.initial_delay_ms = 5 * 60 * 1000;  // 5 mins
  config_.next_delay_ms = 12 * 60 * 60 * 1000;  // 12 hours

  WebSocketExperimentTask::Config task_config;
  task_config.ws_protocol = "google-websocket-liveexperiment";
  task_config.ws_origin = "http://dev.chromium.org/";
  task_config.url_fetch_deadline_ms = kUrlFetchDeadlineSec * 1000;
  task_config.websocket_onopen_deadline_ms =
      kWebSocketConnectDeadlineSec * 1000;
  task_config.websocket_hello_message = "Hello";
  task_config.websocket_hello_echoback_deadline_ms =
      kWebSocketEchoDeadlineSec * 1000;
  // Note: wait 1.5 sec in websocket_experiment_def.txt
  task_config.websocket_idle_ms = kWebSocketIdleSec * 1000;
  task_config.websocket_receive_push_message_deadline_ms =
      kWebSocketPushDeadlineSec * 1000;
  task_config.websocket_bye_message = "Bye";
  task_config.websocket_bye_deadline_ms =
      kWebSocketByeDeadlineSec * 1000;
  task_config.websocket_close_deadline_ms =
      kWebSocketCloseDeadlineSec * 1000;

  config_.ws_config = task_config;
  config_.ws_config.url =
      GURL(StringPrintf("ws://%s/live_exp", kExperimentHost));
  config_.ws_config.ws_location =
      StringPrintf("ws://%s/live_exp", kExperimentHost);
  config_.ws_config.http_url =
      GURL(StringPrintf("http://%s/", kExperimentHost));

  config_.wss_config = task_config;
  config_.wss_config.url =
      GURL(StringPrintf("wss://%s/live_exp", kExperimentHost));
  config_.wss_config.ws_location =
      StringPrintf("wss://%s/live_exp", kExperimentHost);
  config_.wss_config.http_url =
      GURL(StringPrintf("https://%s/", kExperimentHost));

  config_.ws_nondefault_config = task_config;
  config_.ws_nondefault_config.url =
      GURL(StringPrintf("ws://%s:%d/live_exp",
                        kExperimentHost, kAlternativePort));
  config_.ws_nondefault_config.ws_location =
      StringPrintf("ws://%s:%d/live_exp",
                   kExperimentHost, kAlternativePort);
  config_.ws_nondefault_config.http_url =
      GURL(StringPrintf("http://%s:%d/",
                        kExperimentHost, kAlternativePort));
}

void WebSocketExperimentRunner::DoLoop() {
  if (next_state_ == STATE_NONE) {
    if (task_.get()) {
      AddRef();  // Release in OnTaskCompleted.
      task_->Cancel();
    }
    return;
  }

  State state = next_state_;
  task_state_ = STATE_NONE;
  next_state_ = STATE_NONE;

  switch (state) {
    case STATE_IDLE:
      task_.reset();
      next_state_ = STATE_RUN_WS;
      ChromeThread::PostDelayedTask(
          ChromeThread::IO,
          FROM_HERE,
          NewRunnableMethod(this, &WebSocketExperimentRunner::DoLoop),
          config_.next_delay_ms);
      break;
    case STATE_RUN_WS:
      task_.reset(new WebSocketExperimentTask(config_.ws_config,
                                              &task_callback_));
      task_state_ = STATE_RUN_WS;
      next_state_ = STATE_RUN_WSS;
      break;
    case STATE_RUN_WSS:
      task_.reset(new WebSocketExperimentTask(config_.wss_config,
                                              &task_callback_));
      task_state_ = STATE_RUN_WSS;
      next_state_ = STATE_RUN_WS_NODEFAULT_PORT;
      break;
    case STATE_RUN_WS_NODEFAULT_PORT:
      task_.reset(new WebSocketExperimentTask(config_.ws_nondefault_config,
                                              &task_callback_));
      task_state_ = STATE_RUN_WS_NODEFAULT_PORT;
      next_state_ = STATE_IDLE;
      break;
    default:
      NOTREACHED();
      break;
  }
  if (task_.get())
    task_->Run();
}

void WebSocketExperimentRunner::OnTaskCompleted(int result) {
  if (result == net::ERR_ABORTED) {
    task_.reset();
    // Task is Canceled.
    DLOG(INFO) << "WebSocketExperiment Task is canceled.";
    Release();
    return;
  }
  UpdateTaskResultHistogram(task_.get());
  task_.reset();

  DoLoop();
}

void WebSocketExperimentRunner::UpdateTaskResultHistogram(
    const WebSocketExperimentTask* task) {
  DCHECK(task);
  const WebSocketExperimentTask::Result& task_result = task->result();

  UPDATE_HISTOGRAM("LastState", task_result.last_state,
                   1, WebSocketExperimentTask::NUM_STATES,
                   WebSocketExperimentTask::NUM_STATES + 1);

  UPDATE_HISTOGRAM_TIMES("UrlFetch", task_result.url_fetch,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(kUrlFetchDeadlineSec),
                         kTimeBucketCount);

  if (task_result.last_state <
      WebSocketExperimentTask::STATE_WEBSOCKET_CONNECT_COMPLETE)
    return;

  UPDATE_HISTOGRAM_TIMES("WebSocketConnect", task_result.websocket_connect,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(
                             kWebSocketConnectDeadlineSec),
                         kTimeBucketCount);

  if (task_result.last_state <
      WebSocketExperimentTask::STATE_WEBSOCKET_RECV_HELLO)
    return;

  UPDATE_HISTOGRAM_TIMES("WebSocketEcho", task_result.websocket_echo,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(
                             kWebSocketEchoDeadlineSec),
                         kTimeBucketCount);

  if (task_result.last_state <
      WebSocketExperimentTask::STATE_WEBSOCKET_KEEP_IDLE)
    return;

  UPDATE_HISTOGRAM_TIMES("WebSocketIdle", task_result.websocket_idle,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(
                             kWebSocketIdleSec + kWebSocketPushDeadlineSec),
                         kTimeBucketCount);

  if (task_result.last_state <
      WebSocketExperimentTask::STATE_WEBSOCKET_CLOSE_COMPLETE)
    return;

  UPDATE_HISTOGRAM_TIMES("WebSocketTotal", task_result.websocket_total,
                         base::TimeDelta::FromMilliseconds(1),
                         base::TimeDelta::FromSeconds(kWebSocketTimeSec),
                         kTimeBucketCount);
}

}  // namespace chrome_browser_net_websocket_experiment
<|endoftext|>


<commit_before>//-------------------------------------------------------------------------------------------------------
// Copyright (C) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
//-------------------------------------------------------------------------------------------------------
#include "CommonMemoryPch.h"
#ifdef __clang__
#include <cxxabi.h>
#endif

#if ENABLE_MEM_STATS
MemStats::MemStats()
    : objectByteCount(0), totalByteCount(0)
{}

void MemStats::Reset()
{
    objectByteCount = 0;
    totalByteCount = 0;
}

size_t MemStats::FreeBytes() const
{
    return totalByteCount - objectByteCount;
}

double MemStats::UsedRatio() const
{
    return (double)objectByteCount / totalByteCount;
}

void MemStats::Aggregate(const MemStats& other)
{
    objectByteCount += other.objectByteCount;
    totalByteCount += other.totalByteCount;
}

#ifdef DUMP_FRAGMENTATION_STATS
HeapBucketStats::HeapBucketStats()
    : totalBlockCount(0), objectCount(0), finalizeCount(0)
{}

void HeapBucketStats::Reset()
{
    MemStats::Reset();
    totalBlockCount = 0;
    objectCount = 0;
    finalizeCount = 0;
}

void HeapBucketStats::Dump() const
{
    Output::Print(_u("%5d %7d %7d %11lu %11lu %11lu   %6.2f%%\n"),
        totalBlockCount, objectCount, finalizeCount,
        static_cast<ULONG>(objectByteCount),
        static_cast<ULONG>(FreeBytes()),
        static_cast<ULONG>(totalByteCount),
        UsedRatio() * 100);
}
#endif

void HeapBucketStats::PreAggregate()
{
    // When first enter Pre-Aggregate state, clear data and mark state.
    if (!(totalByteCount & 1))
    {
        Reset();
        totalByteCount |= 1;
    }
}

void HeapBucketStats::BeginAggregate()
{
    // If was Pre-Aggregate state, keep data and clear state
    if (totalByteCount & 1)
    {
        totalByteCount &= ~1;
    }
    else
    {
        Reset();
    }
}

void HeapBucketStats::Aggregate(const HeapBucketStats& other)
{
    MemStats::Aggregate(other);
#ifdef DUMP_FRAGMENTATION_STATS
    totalBlockCount += other.totalBlockCount;
    objectCount += other.objectCount;
    finalizeCount += other.finalizeCount;
#endif
}
#endif  // ENABLE_MEM_STATS
<commit_msg>Remove reference to unused header file<commit_after>//-------------------------------------------------------------------------------------------------------
// Copyright (C) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
//-------------------------------------------------------------------------------------------------------
#include "CommonMemoryPch.h"

#if ENABLE_MEM_STATS
MemStats::MemStats()
    : objectByteCount(0), totalByteCount(0)
{}

void MemStats::Reset()
{
    objectByteCount = 0;
    totalByteCount = 0;
}

size_t MemStats::FreeBytes() const
{
    return totalByteCount - objectByteCount;
}

double MemStats::UsedRatio() const
{
    return (double)objectByteCount / totalByteCount;
}

void MemStats::Aggregate(const MemStats& other)
{
    objectByteCount += other.objectByteCount;
    totalByteCount += other.totalByteCount;
}

#ifdef DUMP_FRAGMENTATION_STATS
HeapBucketStats::HeapBucketStats()
    : totalBlockCount(0), objectCount(0), finalizeCount(0)
{}

void HeapBucketStats::Reset()
{
    MemStats::Reset();
    totalBlockCount = 0;
    objectCount = 0;
    finalizeCount = 0;
}

void HeapBucketStats::Dump() const
{
    Output::Print(_u("%5d %7d %7d %11lu %11lu %11lu   %6.2f%%\n"),
        totalBlockCount, objectCount, finalizeCount,
        static_cast<ULONG>(objectByteCount),
        static_cast<ULONG>(FreeBytes()),
        static_cast<ULONG>(totalByteCount),
        UsedRatio() * 100);
}
#endif

void HeapBucketStats::PreAggregate()
{
    // When first enter Pre-Aggregate state, clear data and mark state.
    if (!(totalByteCount & 1))
    {
        Reset();
        totalByteCount |= 1;
    }
}

void HeapBucketStats::BeginAggregate()
{
    // If was Pre-Aggregate state, keep data and clear state
    if (totalByteCount & 1)
    {
        totalByteCount &= ~1;
    }
    else
    {
        Reset();
    }
}

void HeapBucketStats::Aggregate(const HeapBucketStats& other)
{
    MemStats::Aggregate(other);
#ifdef DUMP_FRAGMENTATION_STATS
    totalBlockCount += other.totalBlockCount;
    objectCount += other.objectCount;
    finalizeCount += other.finalizeCount;
#endif
}
#endif  // ENABLE_MEM_STATS
<|endoftext|>


<commit_before>#include "depthai/pipeline/datatype/NNData.hpp"

#include <cassert>
#include <limits>
#include <unordered_map>
#include <vector>

#include "depthai-shared/datatype/RawNNData.hpp"
#include "depthai/pipeline/datatype/ADatatype.hpp"
#include "fp16/fp16.h"

namespace dai {

NNData::NNData() : Buffer(std::make_shared<RawNNData>()), rawNn(*dynamic_cast<RawNNData*>(raw.get())) {}
NNData::NNData(std::shared_ptr<RawNNData> ptr) : Buffer(ptr), rawNn(*ptr.get()) {}

static std::size_t sizeofTensorInfoDataType(TensorInfo::DataType type) {
    switch(type) {
        case TensorInfo::DataType::FP16:
            return sizeof(uint16_t);
        case TensorInfo::DataType::FP32:
            return sizeof(float);
        case TensorInfo::DataType::I8:
            return sizeof(int8_t);
        case TensorInfo::DataType::INT:
            return sizeof(int32_t);
        case TensorInfo::DataType::U8F:
            return sizeof(uint8_t);
        default:
            // invalid type, shouldn't happen
            assert(0);
    }
}

std::shared_ptr<RawBuffer> NNData::serialize() const {
    // get data from u8Data and fp16Data and place properly into the underlying raw buffer
    rawNn.tensors = {};

    // U8 tensors
    for(const auto& kv : u8Data) {
        const auto dataType = TensorInfo::DataType::U8F;
        const auto dataSize = kv.second.size() * sizeofTensorInfoDataType(dataType);

        // First add any required padding bytes
        // calculate how many alignment bytes to add for next tensor
        size_t remainder = (rawNn.data.end() - rawNn.data.begin()) % DATA_ALIGNMENT;
        if(remainder > 0) {
            rawNn.data.insert(rawNn.data.end(), DATA_ALIGNMENT - remainder, 0);
        }

        // Then get offset to beginning of data
        size_t offset = rawNn.data.end() - rawNn.data.begin();

        const auto* data = reinterpret_cast<const std::uint8_t*>(kv.second.data());
        rawNn.data.insert(rawNn.data.end(), data, data + dataSize);

        // Add entry in tensors
        TensorInfo info;
        info.dataType = dataType;
        info.numDimensions = 0;
        info.name = kv.first;
        info.offset = offset;
        rawNn.tensors.push_back(info);
    }

    // FP16 tensors
    for(const auto& kv : fp16Data) {
        const auto dataType = TensorInfo::DataType::FP16;
        const auto dataSize = kv.second.size() * sizeofTensorInfoDataType(dataType);

        // First add any required padding bytes
        // calculate how many alignment bytes to add for next tensor
        size_t remainder = (rawNn.data.end() - rawNn.data.begin()) % DATA_ALIGNMENT;
        if(remainder > 0) {
            rawNn.data.insert(rawNn.data.end(), DATA_ALIGNMENT - remainder, 0);
        }

        // Then get offset to beginning of data
        size_t offset = rawNn.data.end() - rawNn.data.begin();

        const auto* data = reinterpret_cast<const std::uint8_t*>(kv.second.data());
        rawNn.data.insert(rawNn.data.end(), data, data + dataSize);

        // Add entry in tensors
        TensorInfo info;
        info.dataType = dataType;
        info.numDimensions = 0;
        info.name = kv.first;
        info.offset = offset;
        rawNn.tensors.push_back(info);
    }

    return raw;
}

// setters
// uint8_t
void NNData::setLayer(const std::string& name, std::vector<std::uint8_t> data) {
    u8Data[name] = std::move(data);
}
void NNData::setLayer(const std::string& name, const std::vector<int>& data) {
    u8Data[name] = std::vector<std::uint8_t>(data.size());
    for(unsigned i = 0; i < data.size(); i++) {
        u8Data[name][i] = static_cast<std::uint8_t>(data[i]);
    }
}

// fp16
void NNData::setLayer(const std::string& name, std::vector<float> data) {
    fp16Data[name] = std::vector<std::uint16_t>(data.size());
    for(unsigned i = 0; i < data.size(); i++) {
        fp16Data[name][i] = fp16_ieee_from_fp32_value(data[i]);
    }
}
void NNData::setLayer(const std::string& name, std::vector<double> data) {
    fp16Data[name] = std::vector<std::uint16_t>(data.size());
    for(unsigned i = 0; i < data.size(); i++) {
        fp16Data[name][i] = fp16_ieee_from_fp32_value(data[i]);
    }
}

// getters
std::vector<std::string> NNData::getAllLayerNames() const {
    std::vector<std::string> names;
    for(const auto& t : rawNn.tensors) {
        names.push_back(t.name);
    }
    return names;
}

std::vector<TensorInfo> NNData::getAllLayers() const {
    return rawNn.tensors;
}

bool NNData::getLayer(const std::string& name, TensorInfo& tensor) const {
    for(const auto& t : rawNn.tensors) {
        if(t.name == name) {
            tensor = t;
            return true;
        }
    }
    return false;
}

bool NNData::hasLayer(const std::string& name) const {
    for(const auto& tensor : rawNn.tensors) {
        if(tensor.name == name) return true;
    }
    return false;
}

bool NNData::getLayerDatatype(const std::string& name, TensorInfo::DataType& datatype) const {
    TensorInfo tensor;
    if(getLayer(name, tensor)) {
        datatype = tensor.dataType;
        return true;
    }
    return false;
}

// uint8
std::vector<std::uint8_t> NNData::getLayerUInt8(const std::string& name) const {
    // std::vector<std::uint8_t> data;
    // find layer name and its offset
    TensorInfo tensor;
    if(getLayer(name, tensor)) {
        if(tensor.dataType == TensorInfo::DataType::U8F) {
            // Total data size = last dimension * last stride
            if(tensor.numDimensions > 0) {
                size_t size = tensor.dims[tensor.numDimensions - 1] * tensor.strides[tensor.numDimensions - 1];
                auto beg = rawNn.data.begin() + tensor.offset;
                auto end = beg + size;
                return {beg, end};
            }
        }
    }
    return {};
}

// fp16
std::vector<float> NNData::getLayerFp16(const std::string& name) const {
    // find layer name and its offset
    TensorInfo tensor;
    if(getLayer(name, tensor)) {
        if(tensor.dataType == TensorInfo::DataType::FP16) {
            // Total data size = last dimension * last stride
            if(tensor.numDimensions > 0) {
                std::size_t size = tensor.dims[tensor.numDimensions - 1] * tensor.strides[tensor.numDimensions - 1];
                std::size_t numElements = size / 2;  // FP16

                std::vector<float> data;
                auto* pFp16Data = reinterpret_cast<std::uint16_t*>(&rawNn.data[tensor.offset]);
                for(std::size_t i = 0; i < numElements; i++) {
                    data.push_back(fp16_ieee_to_fp32_value(pFp16Data[i]));
                }
                return data;
            }
        }
    }
    return {};
}

// uint8
std::vector<std::uint8_t> NNData::getFirstLayerUInt8() const {
    // find layer name and its offset
    if(!rawNn.tensors.empty()) {
        return getLayerUInt8(rawNn.tensors[0].name);
    }

    return {};
}

// fp16
std::vector<float> NNData::getFirstLayerFp16() const {
    // find layer name and its offset
    if(!rawNn.tensors.empty()) {
        return getLayerFp16(rawNn.tensors[0].name);
    }
    return {};
}

}  // namespace dai
<commit_msg>Compilation error fix<commit_after>#include "depthai/pipeline/datatype/NNData.hpp"

#include <cassert>
#include <limits>
#include <unordered_map>
#include <vector>

#include "depthai-shared/datatype/RawNNData.hpp"
#include "depthai/pipeline/datatype/ADatatype.hpp"
#include "fp16/fp16.h"

namespace dai {

NNData::NNData() : Buffer(std::make_shared<RawNNData>()), rawNn(*dynamic_cast<RawNNData*>(raw.get())) {}
NNData::NNData(std::shared_ptr<RawNNData> ptr) : Buffer(ptr), rawNn(*ptr.get()) {}

static std::size_t sizeofTensorInfoDataType(TensorInfo::DataType type) {
    switch(type) {
        case TensorInfo::DataType::FP16:
            return sizeof(uint16_t);
        case TensorInfo::DataType::FP32:
            return sizeof(float);
        case TensorInfo::DataType::I8:
            return sizeof(int8_t);
        case TensorInfo::DataType::INT:
            return sizeof(int32_t);
        case TensorInfo::DataType::U8F:
            return sizeof(uint8_t);
        default:
            // invalid type, shouldn't happen
            assert(0);
            return 0;
    }
}

std::shared_ptr<RawBuffer> NNData::serialize() const {
    // get data from u8Data and fp16Data and place properly into the underlying raw buffer
    rawNn.tensors = {};

    // U8 tensors
    for(const auto& kv : u8Data) {
        const auto dataType = TensorInfo::DataType::U8F;
        const auto dataSize = kv.second.size() * sizeofTensorInfoDataType(dataType);

        // First add any required padding bytes
        // calculate how many alignment bytes to add for next tensor
        size_t remainder = (rawNn.data.end() - rawNn.data.begin()) % DATA_ALIGNMENT;
        if(remainder > 0) {
            rawNn.data.insert(rawNn.data.end(), DATA_ALIGNMENT - remainder, 0);
        }

        // Then get offset to beginning of data
        size_t offset = rawNn.data.end() - rawNn.data.begin();

        const auto* data = reinterpret_cast<const std::uint8_t*>(kv.second.data());
        rawNn.data.insert(rawNn.data.end(), data, data + dataSize);

        // Add entry in tensors
        TensorInfo info;
        info.dataType = dataType;
        info.numDimensions = 0;
        info.name = kv.first;
        info.offset = offset;
        rawNn.tensors.push_back(info);
    }

    // FP16 tensors
    for(const auto& kv : fp16Data) {
        const auto dataType = TensorInfo::DataType::FP16;
        const auto dataSize = kv.second.size() * sizeofTensorInfoDataType(dataType);

        // First add any required padding bytes
        // calculate how many alignment bytes to add for next tensor
        size_t remainder = (rawNn.data.end() - rawNn.data.begin()) % DATA_ALIGNMENT;
        if(remainder > 0) {
            rawNn.data.insert(rawNn.data.end(), DATA_ALIGNMENT - remainder, 0);
        }

        // Then get offset to beginning of data
        size_t offset = rawNn.data.end() - rawNn.data.begin();

        const auto* data = reinterpret_cast<const std::uint8_t*>(kv.second.data());
        rawNn.data.insert(rawNn.data.end(), data, data + dataSize);

        // Add entry in tensors
        TensorInfo info;
        info.dataType = dataType;
        info.numDimensions = 0;
        info.name = kv.first;
        info.offset = offset;
        rawNn.tensors.push_back(info);
    }

    return raw;
}

// setters
// uint8_t
void NNData::setLayer(const std::string& name, std::vector<std::uint8_t> data) {
    u8Data[name] = std::move(data);
}
void NNData::setLayer(const std::string& name, const std::vector<int>& data) {
    u8Data[name] = std::vector<std::uint8_t>(data.size());
    for(unsigned i = 0; i < data.size(); i++) {
        u8Data[name][i] = static_cast<std::uint8_t>(data[i]);
    }
}

// fp16
void NNData::setLayer(const std::string& name, std::vector<float> data) {
    fp16Data[name] = std::vector<std::uint16_t>(data.size());
    for(unsigned i = 0; i < data.size(); i++) {
        fp16Data[name][i] = fp16_ieee_from_fp32_value(data[i]);
    }
}
void NNData::setLayer(const std::string& name, std::vector<double> data) {
    fp16Data[name] = std::vector<std::uint16_t>(data.size());
    for(unsigned i = 0; i < data.size(); i++) {
        fp16Data[name][i] = fp16_ieee_from_fp32_value(data[i]);
    }
}

// getters
std::vector<std::string> NNData::getAllLayerNames() const {
    std::vector<std::string> names;
    for(const auto& t : rawNn.tensors) {
        names.push_back(t.name);
    }
    return names;
}

std::vector<TensorInfo> NNData::getAllLayers() const {
    return rawNn.tensors;
}

bool NNData::getLayer(const std::string& name, TensorInfo& tensor) const {
    for(const auto& t : rawNn.tensors) {
        if(t.name == name) {
            tensor = t;
            return true;
        }
    }
    return false;
}

bool NNData::hasLayer(const std::string& name) const {
    for(const auto& tensor : rawNn.tensors) {
        if(tensor.name == name) return true;
    }
    return false;
}

bool NNData::getLayerDatatype(const std::string& name, TensorInfo::DataType& datatype) const {
    TensorInfo tensor;
    if(getLayer(name, tensor)) {
        datatype = tensor.dataType;
        return true;
    }
    return false;
}

// uint8
std::vector<std::uint8_t> NNData::getLayerUInt8(const std::string& name) const {
    // std::vector<std::uint8_t> data;
    // find layer name and its offset
    TensorInfo tensor;
    if(getLayer(name, tensor)) {
        if(tensor.dataType == TensorInfo::DataType::U8F) {
            // Total data size = last dimension * last stride
            if(tensor.numDimensions > 0) {
                size_t size = tensor.dims[tensor.numDimensions - 1] * tensor.strides[tensor.numDimensions - 1];
                auto beg = rawNn.data.begin() + tensor.offset;
                auto end = beg + size;
                return {beg, end};
            }
        }
    }
    return {};
}

// fp16
std::vector<float> NNData::getLayerFp16(const std::string& name) const {
    // find layer name and its offset
    TensorInfo tensor;
    if(getLayer(name, tensor)) {
        if(tensor.dataType == TensorInfo::DataType::FP16) {
            // Total data size = last dimension * last stride
            if(tensor.numDimensions > 0) {
                std::size_t size = tensor.dims[tensor.numDimensions - 1] * tensor.strides[tensor.numDimensions - 1];
                std::size_t numElements = size / 2;  // FP16

                std::vector<float> data;
                auto* pFp16Data = reinterpret_cast<std::uint16_t*>(&rawNn.data[tensor.offset]);
                for(std::size_t i = 0; i < numElements; i++) {
                    data.push_back(fp16_ieee_to_fp32_value(pFp16Data[i]));
                }
                return data;
            }
        }
    }
    return {};
}

// uint8
std::vector<std::uint8_t> NNData::getFirstLayerUInt8() const {
    // find layer name and its offset
    if(!rawNn.tensors.empty()) {
        return getLayerUInt8(rawNn.tensors[0].name);
    }

    return {};
}

// fp16
std::vector<float> NNData::getFirstLayerFp16() const {
    // find layer name and its offset
    if(!rawNn.tensors.empty()) {
        return getLayerFp16(rawNn.tensors[0].name);
    }
    return {};
}

}  // namespace dai
<|endoftext|>