<commit_before>#include <v4r/common/miscellaneous.h>  // to extract Pose intrinsically stored in pcd file

#include <v4r/io/eigen.h>
#include <v4r/io/filesystem.h>
#include <v4r/recognition/multi_pipeline_recognizer.h>

#include <pcl/common/time.h>
#include <pcl/filters/passthrough.h>

#include <iostream>
#include <sstream>
#include <time.h>

#include <boost/any.hpp>
#include <boost/program_options.hpp>
#include <glog/logging.h>

namespace po = boost::program_options;

int
main (int argc, char ** argv)
{
    typedef pcl::PointXYZRGB PointT;
    typedef v4r::Model<PointT> ModelT;
    typedef boost::shared_ptr<ModelT> ModelTPtr;

    std::map<std::string, size_t> rec_models_per_id_;
    typedef pcl::PointXYZRGB PointT;
    std::string test_dir, out_dir = "/tmp/sv_object_recognizer_results/";
    bool visualize = false;
    double chop_z = std::numeric_limits<double>::max();

    google::InitGoogleLogging(argv[0]);

    po::options_description desc("Single-View Object Instance Recognizer\n======================================\n**Allowed options");
    desc.add_options()
        ("help,h", "produce help message")
        ("test_dir,t", po::value<std::string>(&test_dir)->required(), "Directory with test scenes stored as point clouds (.pcd). The camera pose is taken directly from the pcd header fields \"sensor_orientation_\" and \"sensor_origin_\" (if the test directory contains subdirectories, each subdirectory is considered as seperate sequence for multiview recognition)")
        ("visualize,v", po::bool_switch(&visualize), "visualize recognition results")
        ("chop_z,z", po::value<double>(&chop_z)->default_value(chop_z, boost::str(boost::format("%.2e") % chop_z) ), "points with z-component higher than chop_z_ will be ignored (low chop_z reduces computation time and false positives (noise increase with z)")
        ("out_dir,o", po::value<std::string>(&out_dir)->default_value(out_dir), "Output directory where recognition results will be stored.")
   ;
    po::variables_map vm;
    po::parsed_options parsed = po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
    po::store(parsed, vm);
    if (vm.count("help")) { std::cout << desc << std::endl; }
    try { po::notify(vm); }
    catch(std::exception& e) { std::cerr << "Error: " << e.what() << std::endl << std::endl << desc << std::endl;  }

    v4r::io::createDirIfNotExist(out_dir);

    // writing parameters to file
    ofstream param_file;
    param_file.open ((out_dir + "/param.nfo").c_str());
    for(const auto& it : vm)
    {
      param_file << "--" << it.first << " ";

      auto& value = it.second.value();
      if (auto v_double = boost::any_cast<double>(&value))
        param_file << std::setprecision(3) << *v_double;
      else if (auto v_string = boost::any_cast<std::string>(&value))
        param_file << *v_string;
      else if (auto v_bool = boost::any_cast<bool>(&value))
        param_file << *v_bool;
      else if (auto v_int = boost::any_cast<int>(&value))
        param_file << *v_int;
      else if (auto v_size_t = boost::any_cast<size_t>(&value))
        param_file << *v_size_t;
      else
        param_file << "error";

      param_file << " ";
    }
    param_file.close();

    v4r::MultiRecognitionPipeline<PointT> r(argc, argv);

    // ----------- TEST ----------
    std::vector< std::string> sub_folder_names = v4r::io::getFoldersInDirectory( test_dir );
    if(sub_folder_names.empty())
        sub_folder_names.push_back("");

    for (const std::string &sub_folder_name : sub_folder_names)
    {
        const std::string sequence_path = test_dir + "/" + sub_folder_name;
        const std::string out_path = out_dir + "/" + sub_folder_name;
        v4r::io::createDirIfNotExist(out_path);

        rec_models_per_id_.clear();     // shouldn't this go inside next for?

        std::vector< std::string > views = v4r::io::getFilesInDirectory(sequence_path, ".*.pcd", false);
        for (size_t v_id=0; v_id<views.size(); v_id++)
        {
            const std::string fn = sequence_path + "/" + views[ v_id ];

            LOG(INFO) << "Recognizing file " << fn;
            typename pcl::PointCloud<PointT>::Ptr cloud(new pcl::PointCloud<PointT>());
            pcl::io::loadPCDFile(fn, *cloud);

            if( chop_z > 0)
            {
                pcl::PassThrough<PointT> pass;
                pass.setFilterLimits ( 0.f, chop_z );
                pass.setFilterFieldName ("z");
                pass.setInputCloud (cloud);
                pass.setKeepOrganized (true);
                pass.filter (*cloud);
            }

            r.setInputCloud (cloud);
            pcl::StopWatch watch;
            r.recognize();
            v4r::io::writeFloatToFile( out_path + "/" + views[v_id].substr(0, views[v_id].length()-4) + "_time.nfo", watch.getTimeSeconds());

            std::vector<ModelTPtr> verified_models = r.getVerifiedModels();
            std::vector<Eigen::Matrix4f, Eigen::aligned_allocator<Eigen::Matrix4f> > transforms_verified;
            transforms_verified = r.getVerifiedTransforms();

            if (visualize)
                r.visualize();

            for(size_t m_id=0; m_id<verified_models.size(); m_id++)
            {
                LOG(INFO) << "********************" << verified_models[m_id]->id_ << std::endl;

                const std::string model_id = verified_models[m_id]->id_;
                const Eigen::Matrix4f tf = transforms_verified[m_id];

                size_t num_models_per_model_id;

                std::map<std::string, size_t>::iterator it_rec_mod;
                it_rec_mod = rec_models_per_id_.find(model_id);
                if(it_rec_mod == rec_models_per_id_.end())
                {
                    rec_models_per_id_.insert(std::pair<std::string, size_t>(model_id, 1));
                    num_models_per_model_id = 0;
                }
                else
                {
                    num_models_per_model_id = it_rec_mod->second;
                    it_rec_mod->second++;
                }

                std::stringstream out_fn;
                out_fn << out_path << "/" << views[v_id].substr(0, views[v_id].length()-4) << "_"
                       << model_id << "_" << num_models_per_model_id << ".txt";

                ofstream or_file (out_fn.str().c_str());
                for (size_t row=0; row <4; row++)
                    for(size_t col=0; col<4; col++)
                        or_file << tf(row, col) << " ";
                or_file.close();
            }
        }
    }
}
<commit_msg>remove parameter output<commit_after>#include <v4r/common/miscellaneous.h>  // to extract Pose intrinsically stored in pcd file

#include <v4r/io/eigen.h>
#include <v4r/io/filesystem.h>
#include <v4r/recognition/multi_pipeline_recognizer.h>

#include <pcl/common/time.h>
#include <pcl/filters/passthrough.h>

#include <iostream>
#include <sstream>
#include <time.h>

#include <boost/program_options.hpp>
#include <glog/logging.h>

namespace po = boost::program_options;

int
main (int argc, char ** argv)
{
    typedef pcl::PointXYZRGB PointT;
    typedef v4r::Model<PointT> ModelT;
    typedef boost::shared_ptr<ModelT> ModelTPtr;

    std::map<std::string, size_t> rec_models_per_id_;
    typedef pcl::PointXYZRGB PointT;
    std::string test_dir, out_dir = "/tmp/sv_object_recognizer_results/";
    bool visualize = false;
    double chop_z = std::numeric_limits<double>::max();

    google::InitGoogleLogging(argv[0]);

    po::options_description desc("Single-View Object Instance Recognizer\n======================================\n**Allowed options");
    desc.add_options()
        ("help,h", "produce help message")
        ("test_dir,t", po::value<std::string>(&test_dir)->required(), "Directory with test scenes stored as point clouds (.pcd). The camera pose is taken directly from the pcd header fields \"sensor_orientation_\" and \"sensor_origin_\" (if the test directory contains subdirectories, each subdirectory is considered as seperate sequence for multiview recognition)")
        ("visualize,v", po::bool_switch(&visualize), "visualize recognition results")
        ("chop_z,z", po::value<double>(&chop_z)->default_value(chop_z, boost::str(boost::format("%.2e") % chop_z) ), "points with z-component higher than chop_z_ will be ignored (low chop_z reduces computation time and false positives (noise increase with z)")
        ("out_dir,o", po::value<std::string>(&out_dir)->default_value(out_dir), "Output directory where recognition results will be stored.")
   ;
    po::variables_map vm;
    po::parsed_options parsed = po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
    po::store(parsed, vm);
    if (vm.count("help")) { std::cout << desc << std::endl; }
    try { po::notify(vm); }
    catch(std::exception& e) { std::cerr << "Error: " << e.what() << std::endl << std::endl << desc << std::endl;  }

    v4r::io::createDirIfNotExist(out_dir);

    v4r::MultiRecognitionPipeline<PointT> r(argc, argv);

    // ----------- TEST ----------
    std::vector< std::string> sub_folder_names = v4r::io::getFoldersInDirectory( test_dir );
    if(sub_folder_names.empty())
        sub_folder_names.push_back("");

    for (const std::string &sub_folder_name : sub_folder_names)
    {
        const std::string sequence_path = test_dir + "/" + sub_folder_name;
        const std::string out_path = out_dir + "/" + sub_folder_name;
        v4r::io::createDirIfNotExist(out_path);

        rec_models_per_id_.clear();     // shouldn't this go inside next for?

        std::vector< std::string > views = v4r::io::getFilesInDirectory(sequence_path, ".*.pcd", false);
        for (size_t v_id=0; v_id<views.size(); v_id++)
        {
            const std::string fn = sequence_path + "/" + views[ v_id ];

            LOG(INFO) << "Recognizing file " << fn;
            typename pcl::PointCloud<PointT>::Ptr cloud(new pcl::PointCloud<PointT>());
            pcl::io::loadPCDFile(fn, *cloud);

            if( chop_z > 0)
            {
                pcl::PassThrough<PointT> pass;
                pass.setFilterLimits ( 0.f, chop_z );
                pass.setFilterFieldName ("z");
                pass.setInputCloud (cloud);
                pass.setKeepOrganized (true);
                pass.filter (*cloud);
            }

            r.setInputCloud (cloud);
            pcl::StopWatch watch;
            r.recognize();
            v4r::io::writeFloatToFile( out_path + "/" + views[v_id].substr(0, views[v_id].length()-4) + "_time.nfo", watch.getTimeSeconds());

            std::vector<ModelTPtr> verified_models = r.getVerifiedModels();
            std::vector<Eigen::Matrix4f, Eigen::aligned_allocator<Eigen::Matrix4f> > transforms_verified;
            transforms_verified = r.getVerifiedTransforms();

            if (visualize)
                r.visualize();

            for(size_t m_id=0; m_id<verified_models.size(); m_id++)
            {
                LOG(INFO) << "********************" << verified_models[m_id]->id_ << std::endl;

                const std::string model_id = verified_models[m_id]->id_;
                const Eigen::Matrix4f tf = transforms_verified[m_id];

                size_t num_models_per_model_id;

                std::map<std::string, size_t>::iterator it_rec_mod;
                it_rec_mod = rec_models_per_id_.find(model_id);
                if(it_rec_mod == rec_models_per_id_.end())
                {
                    rec_models_per_id_.insert(std::pair<std::string, size_t>(model_id, 1));
                    num_models_per_model_id = 0;
                }
                else
                {
                    num_models_per_model_id = it_rec_mod->second;
                    it_rec_mod->second++;
                }

                std::stringstream out_fn;
                out_fn << out_path << "/" << views[v_id].substr(0, views[v_id].length()-4) << "_"
                       << model_id << "_" << num_models_per_model_id << ".txt";

                ofstream or_file (out_fn.str().c_str());
                for (size_t row=0; row <4; row++)
                    for(size_t col=0; col<4; col++)
                        or_file << tf(row, col) << " ";
                or_file.close();
            }
        }
    }
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016, Andrej Kislovskij
 *
 * This is PUBLIC DOMAIN software so use at your own risk as it comes
 * with no warranties. This code is yours to share, use and modify without
 * any restrictions or obligations.
 *
 * For more information see conwrap/LICENSE or refer refer to http://unlicense.org
 *
 * Author: gimesketvirtadieni at gmail dot com (Andrej Kislovskij)
 */

#pragma once

#include <functional>

namespace conwrap
{
	// this is just a 'big-enough' counter
	// assuming increment rate is 1M per sec, it will expire in ~58.5 years!
	// it can be hugely increased by implementing highValue along with lowValue
	class Epoch
	{
		public:
			Epoch(unsigned long long v)
			: lowValue(v) {}

			virtual ~Epoch() {}

			inline Epoch& operator++()
			{
				// incrementing value and returning result
				lowValue++;
				return *this;
			}

			inline Epoch operator++(int)
			{
				// saving value before post-increment
				Epoch tmp(*this);
				operator++();
				return tmp;
			}

		    inline friend bool operator<(const Epoch& l, const Epoch& r)
		    {
		        return std::tie(l.lowValue) < std::tie(r.lowValue);
		    }

		    inline auto getLowValue()
		    {
		        return lowValue;
		    }

		private:
			unsigned long long lowValue;
	};
}
<commit_msg>Minor change<commit_after>/*
 * Copyright 2016, Andrej Kislovskij
 *
 * This is PUBLIC DOMAIN software so use at your own risk as it comes
 * with no warranties. This code is yours to share, use and modify without
 * any restrictions or obligations.
 *
 * For more information see conwrap/LICENSE or refer refer to http://unlicense.org
 *
 * Author: gimesketvirtadieni at gmail dot com (Andrej Kislovskij)
 */

#pragma once

#include <functional>

namespace conwrap
{
	// this is just a 'big-enough' counter
	// assuming increment rate is 1M per sec, it will expire in ~584942 years!
	// it can be hugely increased by implementing highValue along with lowValue
	class Epoch
	{
		public:
			Epoch(unsigned long long v)
			: lowValue(v) {}

			virtual ~Epoch() {}

			inline Epoch& operator++()
			{
				// incrementing value and returning result
				lowValue++;
				return *this;
			}

			inline Epoch operator++(int)
			{
				// saving value before post-increment
				Epoch tmp(*this);
				operator++();
				return tmp;
			}

		    inline friend bool operator<(const Epoch& l, const Epoch& r)
		    {
		        return std::tie(l.lowValue) < std::tie(r.lowValue);
		    }

		    inline auto getLowValue()
		    {
		        return lowValue;
		    }

		private:
			unsigned long long lowValue;
	};
}
<|endoftext|>


<commit_before>/* indivudal.cpp - CS 472 Project #2: Genetic Programming
 * Copyright 2014 Andrew Schwartzmeyer
 *
 * Source file for Individual
 */

#include <algorithm>
#include <cassert>
#include <cmath>
#include <iostream>
#include <memory>
#include <tuple>
#include <vector>

#include "individual.hpp"
#include "../problem/problem.hpp"
#include "../random_generator/random_generator.hpp"

using namespace individual;
using namespace random_generator;
using problem::Problem;

namespace individual {
  using std::vector;
  vector<Function> terminals { constant, input };
  vector<Function> unaries { sqrt, sin, log, exp };
  vector<Function> binaries { add, subtract, divide, multiply, pow };
  vector<Function> quadnaries { cond };
  vector<Function> internals {log, sqrt, sin, add, subtract, divide, multiply, cond};
}

Node::Node(const Problem & problem, const int & depth) {
  using std::find;
  if (depth < problem.max_depth) {
    // assign random internal type
    int_dist dist(0, internals.size() - 1); // closed interval
    type = Function(internals[dist(rg.engine)]);
    int arity = 0;
    if (find(unaries.begin(), unaries.end(), type) != unaries.end()) arity = 1;
    else if (find(binaries.begin(), binaries.end(), type) != binaries.end()) arity = 2;
    else if (find(quadnaries.begin(), quadnaries.end(), type) != quadnaries.end()) arity = 4;
    assert(arity != 0);
    // recursively create subtrees
    for (int i = 0; i < arity; i++)
      children.emplace_back(Node(problem, depth + 1));
  } else {
    // reached max depth, assign random terminal type
    int_dist dist(0, terminals.size() - 1); // closed interval
    type = Function(terminals[dist(rg.engine)]);
    // setup constant type; input is provided on evaluation
    if (type == constant) set_constant(problem);
  }
}

void Node::set_constant(const Problem & problem) {
  // choose a random value between the problem's min and max
  real_dist dist(problem.constant_min, problem.constant_max);
  k = dist(rg.engine);
}

double Node::evaluate(const double & x) const {
  double a, b, c, d;
  if (find(unaries.begin(), unaries.end(), type) != unaries.end())
    a = children[0].evaluate(x);
  if (find(binaries.begin(), binaries.end(), type) != binaries.end()) {
    a = children[0].evaluate(x);
    b = children[1].evaluate(x);
  }
  if (find(quadnaries.begin(), quadnaries.end(), type) != quadnaries.end()) {
    a = children[0].evaluate(x);
    b = children[1].evaluate(x);
    c = children[2].evaluate(x);
    d = children[3].evaluate(x);
  }
  switch(type) {
  case constant:
    return k;
  case input:
    return x;
  case sqrt:
    return std::sqrt(std::abs(a)); // protected
  case sin:
    return std::sin(a);
  case log:
    return (a == 0) ? 0 : std::log(std::abs(a)); // protected
  case exp:
    return std::exp(a);
  case add:
    return a + b;
  case subtract:
    return a - b;
  case multiply:
    return a * b;
  case divide:
    return (b == 0) ? 1 : a / b; // protected
  case pow:
    return std::pow(std::abs(a), std::abs(b));
  case cond:
    return (a < b) ? c : d;
  }
}

const Size Node::size() const {
  // Recursively count children via post-order traversal
  // Keep track of internals and leafs via Size struct
  Size size;
  for (const Node & child : children) {
    Size temp = child.size(); // is this micro-optimizing?
    size.internals += temp.internals;
    size.leafs += temp.leafs;
  }
  if (children.size() == 0) ++size.leafs;
  else ++size.internals;
  return size;
}

std::string Node::represent() const {
  switch(type) {
  case constant:
    return std::to_string(k);
  case input:
    return "x";
  case sqrt:
    return " sqrt";
  case sin:
    return " sin";
  case log:
    return " log";
  case exp:
    return " exp";
  case add:
    return " +";
  case subtract:
    return " -";
  case multiply:
    return " *";
  case divide:
    return " /";
  case pow:
    return " ^";
  case cond:
    return " a < b ? c : d";
  }
}

std::string Node::print() const {
  // Post-order traversal print of expression in RPN/posfix notation
  std::string formula = "(";
  for (auto child : children)
    formula += child.print();
  return formula + represent() + ")";
}

Individual::Individual(const Problem & p): problem(p), root(Node(problem)) {
  size = root.size();
  fitness = evaluate();
}

double Individual::evaluate() const {
  double fitness = 0;
  for (auto pair : problem.values) {
    double output = root.evaluate(std::get<0>(pair));
    fitness += std::pow(output - std::get<1>(pair), 2);
  }
  return std::sqrt(fitness);
}

void Individual::print_formula() const {
  std::cout << "Expression tree of size " << get_total()
	    << " with " << get_internals() << " internals"
	    << " and " << get_leafs() << " leafs"
	    << " has the following formula: " << std::endl
	    << root.print() << std::endl;
}

void Individual::print_calculation() const {
  double fitness = 0;
  for (auto pair : problem.values) {
    double output = root.evaluate(std::get<0>(pair));
    double error = std::pow(output - std::get<1>(pair), 2);
    fitness += error;
    std::cout << "f(" << std::get<0>(pair) << ", " << std::get<1>(pair)
	      << ") = " << output
	      << ", error = " << error << ".\n";
  }
  std::cout << "Total fitness: " << std::sqrt(fitness) << ".\n";
}
<commit_msg>More comments for node.evaluate()<commit_after>/* indivudal.cpp - CS 472 Project #2: Genetic Programming
 * Copyright 2014 Andrew Schwartzmeyer
 *
 * Source file for Individual
 */

#include <algorithm>
#include <cassert>
#include <cmath>
#include <iostream>
#include <memory>
#include <tuple>
#include <vector>

#include "individual.hpp"
#include "../problem/problem.hpp"
#include "../random_generator/random_generator.hpp"

using namespace individual;
using namespace random_generator;
using problem::Problem;

namespace individual {
  using std::vector;
  vector<Function> terminals { constant, input };
  vector<Function> unaries { sqrt, sin, log, exp };
  vector<Function> binaries { add, subtract, divide, multiply, pow };
  vector<Function> quadnaries { cond };
  vector<Function> internals {log, sqrt, sin, add, subtract, divide, multiply, cond};
}

Node::Node(const Problem & problem, const int & depth) {
  using std::find;
  if (depth < problem.max_depth) {
    // assign random internal type
    int_dist dist(0, internals.size() - 1); // closed interval
    type = Function(internals[dist(rg.engine)]);
    int arity = 0;
    if (find(unaries.begin(), unaries.end(), type) != unaries.end()) arity = 1;
    else if (find(binaries.begin(), binaries.end(), type) != binaries.end()) arity = 2;
    else if (find(quadnaries.begin(), quadnaries.end(), type) != quadnaries.end()) arity = 4;
    assert(arity != 0);
    // recursively create subtrees
    for (int i = 0; i < arity; i++)
      children.emplace_back(Node(problem, depth + 1));
  } else {
    // reached max depth, assign random terminal type
    int_dist dist(0, terminals.size() - 1); // closed interval
    type = Function(terminals[dist(rg.engine)]);
    // setup constant type; input is provided on evaluation
    if (type == constant) set_constant(problem);
  }
}

void Node::set_constant(const Problem & problem) {
  // choose a random value between the problem's min and max
  real_dist dist(problem.constant_min, problem.constant_max);
  k = dist(rg.engine);
}

double Node::evaluate(const double & x) const {
  // depth-first post-order recursive evaluation tree
  // slight overhead with find() to make calls a) clearer and b) easy
  // to protect without excess recursive calls
  double a, b, c, d;
  if (find(unaries.begin(), unaries.end(), type) != unaries.end())
    a = children[0].evaluate(x);
  if (find(binaries.begin(), binaries.end(), type) != binaries.end()) {
    a = children[0].evaluate(x);
    b = children[1].evaluate(x);
  }
  if (find(quadnaries.begin(), quadnaries.end(), type) != quadnaries.end()) {
    a = children[0].evaluate(x);
    b = children[1].evaluate(x);
    c = children[2].evaluate(x);
    d = children[3].evaluate(x);
  }
  // calculate the result
  switch(type) {
  case constant:
    return k;
  case input:
    return x;
  case sqrt:
    return std::sqrt(std::abs(a)); // protected
  case sin:
    return std::sin(a);
  case log:
    return (a == 0) ? 0 : std::log(std::abs(a)); // protected
  case exp:
    return std::exp(a);
  case add:
    return a + b;
  case subtract:
    return a - b;
  case multiply:
    return a * b;
  case divide:
    return (b == 0) ? 1 : a / b; // protected
  case pow:
    return std::pow(std::abs(a), std::abs(b)); // protected
  case cond:
    return (a < b) ? c : d;
  }
}

const Size Node::size() const {
  // Recursively count children via post-order traversal
  // Keep track of internals and leafs via Size struct
  Size size;
  for (const Node & child : children) {
    Size temp = child.size(); // is this micro-optimizing?
    size.internals += temp.internals;
    size.leafs += temp.leafs;
  }
  if (children.size() == 0) ++size.leafs;
  else ++size.internals;
  return size;
}

std::string Node::represent() const {
  switch(type) {
  case constant:
    return std::to_string(k);
  case input:
    return "x";
  case sqrt:
    return " sqrt";
  case sin:
    return " sin";
  case log:
    return " log";
  case exp:
    return " exp";
  case add:
    return " +";
  case subtract:
    return " -";
  case multiply:
    return " *";
  case divide:
    return " /";
  case pow:
    return " ^";
  case cond:
    return " a < b ? c : d";
  }
}

std::string Node::print() const {
  // Post-order traversal print of expression in RPN/posfix notation
  std::string formula = "(";
  for (auto child : children)
    formula += child.print();
  return formula + represent() + ")";
}

Individual::Individual(const Problem & p): problem(p), root(Node(problem)) {
  size = root.size();
  fitness = evaluate();
}

double Individual::evaluate() const {
  double fitness = 0;
  for (auto pair : problem.values) {
    double output = root.evaluate(std::get<0>(pair));
    fitness += std::pow(output - std::get<1>(pair), 2);
  }
  return std::sqrt(fitness);
}

void Individual::print_formula() const {
  std::cout << "Expression tree of size " << get_total()
	    << " with " << get_internals() << " internals"
	    << " and " << get_leafs() << " leafs"
	    << " has the following formula: " << std::endl
	    << root.print() << std::endl;
}

void Individual::print_calculation() const {
  double fitness = 0;
  for (auto pair : problem.values) {
    double output = root.evaluate(std::get<0>(pair));
    double error = std::pow(output - std::get<1>(pair), 2);
    fitness += error;
    std::cout << "f(" << std::get<0>(pair) << ", " << std::get<1>(pair)
	      << ") = " << output
	      << ", error = " << error << ".\n";
  }
  std::cout << "Total fitness: " << std::sqrt(fitness) << ".\n";
}
<|endoftext|>


<commit_before>#pragma once

#include <iterator>
#include <stdexcept>
#include <limits>
#include <functional>
#include <numeric>

#include "acmacs-base/throw.hh"
#include "acmacs-base/stream.hh"

// ----------------------------------------------------------------------

namespace acmacs
{
    namespace internal
    {
        template <typename T, typename Increment> class input_iterator : public std::iterator<std::input_iterator_tag, T>
        {
         public:
            constexpr static const T End{std::numeric_limits<T>::max()};

            inline input_iterator() : current(End) {}
            inline input_iterator(T aFirst, const Increment& aIncrement) : current(aFirst), increment(aIncrement) { increment.find_valid(current); }
            inline input_iterator(T aFirst, Increment&& aIncrement) : current(aFirst), increment(std::move(aIncrement)) { increment.find_valid(current); }

            inline input_iterator& operator++() { increment(current); return *this;}
              // inline input_iterator operator++(int) { input_iterator result = *this; ++(*this); return result; }
            inline bool operator==(const input_iterator& other) const { return current == other.current; }
            inline bool operator!=(const input_iterator& other) const { return !(*this == other); }
            inline const T& operator*() const { return current; }
            inline T& operator*() { return current; }

         private:
            T current;
            Increment increment;

        }; // input_iterator<>

    } // namespace internal

// ----------------------------------------------------------------------

// template <typename T> class Range
// {
//  private:
//     class increment
//     {
//      public:
//         inline increment() {}
//         inline increment(T aStep, T aLast) : step(aStep), last(aLast) { if (step == T{0}) THROW_OR_VOID(std::runtime_error("Invalid range with step 0")); }
//         inline void operator()(T& current)
//             {
//                 if (current != internal::input_iterator<T, increment>::End) {
//                     current += step;
//                     if (step > 0) {
//                         if (current >= last)
//                             current = internal::input_iterator<T, increment>::End;
//                     }
//                     else {
//                         if (current <= last)
//                             current = internal::input_iterator<T, increment>::End;
//                     }
//                 }
//             }

//         inline void find_valid(T&) {}

//      private:
//         T step;
//         T last;
//     };

//     class iterator : public internal::input_iterator<T, increment>
//     {
//      public:
//         inline iterator() : internal::input_iterator<T, increment>() {}
//         inline iterator(T aFirst, T aLast, T aStep) : internal::input_iterator<T, increment>(aFirst, {aStep, aLast}) {}
//     };

//  public:
//     static inline iterator begin(T first, T last, T step = T{1})
//         {
//             return {first, last, step};
//         }

//     static inline iterator begin(T last)
//         {
//             return {T{0}, last, T{1}};
//         }

//     static inline iterator end()
//         {
//             return {};
//         }

// }; // class Range

// ----------------------------------------------------------------------

    class IndexGenerator
    {
     public:
        using Validator = std::function<bool (size_t)>;

     private:
        class increment
        {
         public:
            inline increment() {}
            inline increment(size_t aLast, Validator aValidator) : last(aLast), valid(aValidator) {}
            inline bool end(size_t current) const { return current == internal::input_iterator<size_t, increment>::End; }
            inline size_t end() const { return internal::input_iterator<size_t, increment>::End; }

            inline void operator()(size_t& current)
                {
                    if (!end(current)) {
                        ++current;
                        while (current < last && !valid(current))
                            ++current;
                        if (current >= last)
                            current = end();
                    }
                }

            inline void find_valid(size_t& current)
                {
                    while (!end(current) && !valid(current))
                        operator()(current);
                }

            size_t last;
            Validator valid;
        };

     public:
        inline IndexGenerator(size_t aFirst, size_t aLast, Validator aValidator) : mFirst(aFirst), mIncrement(aLast, aValidator) {}
        inline IndexGenerator(size_t aLast, Validator aValidator) : mFirst(0), mIncrement(aLast, aValidator) {}

        using iterator = internal::input_iterator<size_t, IndexGenerator::increment>;

        inline iterator begin() const { return {mFirst, mIncrement}; }
        inline iterator end() const { return {}; }

     private:
        size_t mFirst;
        increment mIncrement;

    }; // class IndexGenerator

// ----------------------------------------------------------------------

    template <typename Index> class index_iterator
    {
     public:
        using difference_type = long;
        using value_type = Index;
        using pointer = Index*;
        using reference = Index&;
        using iterator_category = std::random_access_iterator_tag;

        inline index_iterator(value_type aValue) : value(aValue) {}
        inline index_iterator& operator++() { ++value; return *this;}
          //inline index_iterator operator++(int) { iterator retval = *this; ++(*this); return retval;}
        inline bool operator==(const index_iterator<Index>& other) const { return value == other.value; }
        inline bool operator!=(const index_iterator<Index>& other) const { return !(*this == other); }
        inline value_type operator*() const { return value; }
        inline difference_type operator-(const index_iterator<Index>& other) const { return static_cast<difference_type>(value) - static_cast<difference_type>(other.value); }

     private:
        value_type value;

    }; // class index_iterator<>

    index_iterator(size_t) -> index_iterator<size_t>;
    index_iterator(int) -> index_iterator<int>;

    template <typename Index> class range
    {
     public:
        inline range(Index aBegin, Index aEnd) : mBegin{aBegin}, mEnd{aEnd}
            {
                if (mEnd < mBegin)
                    throw std::runtime_error("acmacs::range: end < begin");
            }

        inline index_iterator<Index> begin() { return mBegin; }
        inline index_iterator<Index> end() { return mEnd; }

     private:
        Index mBegin, mEnd;

    }; // class range<>

    range(size_t, size_t) -> range<size_t>;
    range(int, int) -> range<int>;

// ----------------------------------------------------------------------

    template <typename Index> inline void fill_with_indexes(std::vector<Index>& aToFill, Index aBegin, Index aEnd)
    {
        aToFill.resize(aEnd - aBegin);
        std::iota(aToFill.begin(), aToFill.end(), aBegin);
    }

    template <typename Index> inline void fill_with_indexes(std::vector<Index>& aToFill, Index aSize)
    {
        aToFill.resize(aSize);
        std::iota(aToFill.begin(), aToFill.end(), 0);
    }

    template <typename Index> inline std::vector<Index> filled_with_indexes(Index aSize)
    {
        std::vector<Index> result(aSize);
        std::iota(result.begin(), result.end(), 0);
        return result;
    }

// ----------------------------------------------------------------------

} // namespace acmacs

// ----------------------------------------------------------------------

#ifdef ACMACS_TARGET_OS
inline std::ostream& operator << (std::ostream& out, const acmacs::IndexGenerator& aGen)
{
    return stream_internal::write_to_stream(out, aGen, "<", ">", ", ");
}
#endif

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>acmacs::range improvement<commit_after>#pragma once

#include <iterator>
#include <stdexcept>
#include <limits>
#include <functional>
#include <numeric>

#include "acmacs-base/throw.hh"
#include "acmacs-base/stream.hh"

// ----------------------------------------------------------------------

namespace acmacs
{
    namespace internal
    {
        template <typename T, typename Increment> class input_iterator : public std::iterator<std::input_iterator_tag, T>
        {
         public:
            constexpr static const T End{std::numeric_limits<T>::max()};

            inline input_iterator() : current(End) {}
            inline input_iterator(T aFirst, const Increment& aIncrement) : current(aFirst), increment(aIncrement) { increment.find_valid(current); }
            inline input_iterator(T aFirst, Increment&& aIncrement) : current(aFirst), increment(std::move(aIncrement)) { increment.find_valid(current); }

            inline input_iterator& operator++() { increment(current); return *this;}
              // inline input_iterator operator++(int) { input_iterator result = *this; ++(*this); return result; }
            inline bool operator==(const input_iterator& other) const { return current == other.current; }
            inline bool operator!=(const input_iterator& other) const { return !(*this == other); }
            inline const T& operator*() const { return current; }
            inline T& operator*() { return current; }

         private:
            T current;
            Increment increment;

        }; // input_iterator<>

    } // namespace internal

// ----------------------------------------------------------------------

// template <typename T> class Range
// {
//  private:
//     class increment
//     {
//      public:
//         inline increment() {}
//         inline increment(T aStep, T aLast) : step(aStep), last(aLast) { if (step == T{0}) THROW_OR_VOID(std::runtime_error("Invalid range with step 0")); }
//         inline void operator()(T& current)
//             {
//                 if (current != internal::input_iterator<T, increment>::End) {
//                     current += step;
//                     if (step > 0) {
//                         if (current >= last)
//                             current = internal::input_iterator<T, increment>::End;
//                     }
//                     else {
//                         if (current <= last)
//                             current = internal::input_iterator<T, increment>::End;
//                     }
//                 }
//             }

//         inline void find_valid(T&) {}

//      private:
//         T step;
//         T last;
//     };

//     class iterator : public internal::input_iterator<T, increment>
//     {
//      public:
//         inline iterator() : internal::input_iterator<T, increment>() {}
//         inline iterator(T aFirst, T aLast, T aStep) : internal::input_iterator<T, increment>(aFirst, {aStep, aLast}) {}
//     };

//  public:
//     static inline iterator begin(T first, T last, T step = T{1})
//         {
//             return {first, last, step};
//         }

//     static inline iterator begin(T last)
//         {
//             return {T{0}, last, T{1}};
//         }

//     static inline iterator end()
//         {
//             return {};
//         }

// }; // class Range

// ----------------------------------------------------------------------

    class IndexGenerator
    {
     public:
        using Validator = std::function<bool (size_t)>;

     private:
        class increment
        {
         public:
            inline increment() {}
            inline increment(size_t aLast, Validator aValidator) : last(aLast), valid(aValidator) {}
            inline bool end(size_t current) const { return current == internal::input_iterator<size_t, increment>::End; }
            inline size_t end() const { return internal::input_iterator<size_t, increment>::End; }

            inline void operator()(size_t& current)
                {
                    if (!end(current)) {
                        ++current;
                        while (current < last && !valid(current))
                            ++current;
                        if (current >= last)
                            current = end();
                    }
                }

            inline void find_valid(size_t& current)
                {
                    while (!end(current) && !valid(current))
                        operator()(current);
                }

            size_t last;
            Validator valid;
        };

     public:
        inline IndexGenerator(size_t aFirst, size_t aLast, Validator aValidator) : mFirst(aFirst), mIncrement(aLast, aValidator) {}
        inline IndexGenerator(size_t aLast, Validator aValidator) : mFirst(0), mIncrement(aLast, aValidator) {}

        using iterator = internal::input_iterator<size_t, IndexGenerator::increment>;

        inline iterator begin() const { return {mFirst, mIncrement}; }
        inline iterator end() const { return {}; }

     private:
        size_t mFirst;
        increment mIncrement;

    }; // class IndexGenerator

// ----------------------------------------------------------------------

    template <typename Index> class index_iterator
    {
     public:
        using difference_type = long;
        using value_type = Index;
        using pointer = Index*;
        using reference = Index&;
        using iterator_category = std::random_access_iterator_tag;

        inline index_iterator(value_type aValue) : value(aValue) {}
        inline index_iterator& operator++() { ++value; return *this;}
          //inline index_iterator operator++(int) { iterator retval = *this; ++(*this); return retval;}
        inline bool operator==(const index_iterator<Index>& other) const { return value == other.value; }
        inline bool operator!=(const index_iterator<Index>& other) const { return !(*this == other); }
        inline value_type operator*() const { return value; }
        inline difference_type operator-(const index_iterator<Index>& other) const { return static_cast<difference_type>(value) - static_cast<difference_type>(other.value); }

     private:
        value_type value;

    }; // class index_iterator<>

    index_iterator(size_t) -> index_iterator<size_t>;
    index_iterator(int) -> index_iterator<int>;

    template <typename Index> class range
    {
     public:
        inline range(Index aBegin, Index aEnd) : mBegin{aBegin}, mEnd{aEnd}
            {
                if (mEnd < mBegin)
                    throw std::runtime_error("acmacs::range: end < begin");
            }
        inline range(Index aEnd) : mBegin{0}, mEnd{aEnd} {}

        inline index_iterator<Index> begin() { return mBegin; }
        inline index_iterator<Index> end() { return mEnd; }

     private:
        Index mBegin, mEnd;

    }; // class range<>

    range(size_t, size_t) -> range<size_t>;
    range(int, int) -> range<int>;
    range(size_t) -> range<size_t>;
    range(int) -> range<int>;

// ----------------------------------------------------------------------

    template <typename Index> inline void fill_with_indexes(std::vector<Index>& aToFill, Index aBegin, Index aEnd)
    {
        aToFill.resize(aEnd - aBegin);
        std::iota(aToFill.begin(), aToFill.end(), aBegin);
    }

    template <typename Index> inline void fill_with_indexes(std::vector<Index>& aToFill, Index aSize)
    {
        aToFill.resize(aSize);
        std::iota(aToFill.begin(), aToFill.end(), 0);
    }

    template <typename Index> inline std::vector<Index> filled_with_indexes(Index aSize)
    {
        std::vector<Index> result(aSize);
        std::iota(result.begin(), result.end(), 0);
        return result;
    }

// ----------------------------------------------------------------------

} // namespace acmacs

// ----------------------------------------------------------------------

#ifdef ACMACS_TARGET_OS
inline std::ostream& operator << (std::ostream& out, const acmacs::IndexGenerator& aGen)
{
    return stream_internal::write_to_stream(out, aGen, "<", ">", ", ");
}
#endif

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>


<commit_before>#include "brujin_algorithms.h"

#include <iostream>
#include <sdsl/suffix_arrays.hpp>
#include <sdsl/lcp.hpp>

using namespace std;
using namespace sdsl;

// Algorithm1.
vector<Node *> create_compress_graph(const uint64_t &k, csa_wt<> &csa, lcp_wt<> &lcp) {
    bool open = false;
    uint64_t counter = 1;

    vector<Node *> graph;
    queue<Node *> queue;

    bit_vector B(csa.size(), 0);

    vector<uint64_t> lex_smaller_table(256, 0);
    for (uint64_t i = 0, sum = 0; i < 256; ++i) {
        lex_smaller_table[i] = sum;
        sum += csa.wavelet_tree.rank(csa.wavelet_tree.size(), i);
    }

//    lb will be initialized, I promise. See line below.
    for (uint64_t i = 0, lb; i < csa.size(); ++i) {
        if (lcp[i] < k and open) {
            open = false;
            B[i - 1] = 1;

            Node *node = new Node(counter, lb, i - 1, k); // DRY
            graph.push_back(node);
            queue.push(node);
            counter++;
        } else if (lcp[i] == k and not open) {
            B[lb] = 1;
            open = true;
        }

        if (lcp[i] < k) {
            lb = i;
        }
    }

    bit_vector::rank_1_type bv_rank;
    util::init_support(bv_rank, &B);

    Node *stop_node = new Node(counter, 1, 1, 1);
    graph.push_back(stop_node); // Add stopnode.
    queue.push(stop_node);
    counter++;

//    PASS
    uint64_t quantity;
    vector<uint8_t> cs(csa.wavelet_tree.sigma);
    vector<uint64_t> rank_c_i(csa.wavelet_tree.sigma);
    vector<uint64_t> rank_c_j(csa.wavelet_tree.sigma);
    while (not queue.empty()) {
        Node *node = queue.front();
        queue.pop();
        bool extendable;

        do { // "repeat"
            extendable = false;
            interval_symbols(csa.wavelet_tree, node->lb, node->rb, quantity, cs, rank_c_i, rank_c_j);

            for (uint64_t i = 0; i < quantity; ++i) {
                uint8_t c = cs[i]; // Onaj u lijevo za jedan.
                uint64_t lb = lex_smaller_table[c] + rank_c_i[i];
                uint64_t rb = lex_smaller_table[c] + rank_c_j[i] - 1;
                uint64_t ones = bv_rank(lb + 1);
                uint64_t id;

                if (ones % 2 == 0 and B[i] == 0) {
                    id = ground;
                } else {
                    id = (ones + 1) / 2;
                }

                if (id != ground) {
                    graph[node->id]->successors.push_back(make_pair(node->id, rb - lb + 1)); //DRY
//                    graph[node->id]->lb = node->lb;
//                    graph[node->id]->rb = node->rb;
//                    graph[node->id]->len = node->len;
                } else if (not c <= 1) {
                    if (quantity == 1) {
                        extendable = true;
                        node->len++;
                        node->lb = lb;
                        node->rb = rb;
                    } else {
                        Node *new_node = new Node(counter, lb, rb, k);
                        graph.push_back(new_node);

                        graph[node->id]->successors.push_back(make_pair(node->id, rb - lb + 1)); //DRY
//                        graph[node->id]->lb = node->lb;
//                        graph[node->id]->rb = node->rb;
//                        graph[node->id]->len = node->len;

                        queue.push(new_node);
                        counter++;
                    }
                }
            }
        } while (extendable);
    }
    return graph;
}

// Only for development purposes. When the project is done this may as well be deleted.
int main() {
    string input_string = "ACTACGTACGTACG"; // Notice no explicit dollar!

    csa_wt<> csa;
    construct_im(csa, input_string, 1);

    //region GET_INTERVALS
    lcp_wt<> lcp;
    construct_im(lcp, input_string, 1);

    cout << "Brujin:" << endl;
    vector<Node *> brojin = create_compress_graph(3, csa, lcp);
    for (int i = 0; i < brojin.size(); ++i) {
        cout << brojin[i]->lb << " " << brojin[i]->rb << endl;
    }
    cout << endl;
    //endregion
}
<commit_msg>Commit latest brujin_algorithm.cpp.<commit_after>#include "brujin_algorithms.h"

#include <iostream>
#include <sdsl/suffix_arrays.hpp>
#include <sdsl/lcp.hpp>

using namespace std;
using namespace sdsl;

// Algorithm1.
vector<Node *> create_compress_graph(const uint64_t &k, wt_huff<> &wta, lcp_wt<> &lcp) {
    bool open = false;
    uint64_t counter = 1;

    vector<Node *> graph;
    queue<Node *> queue;

    bit_vector B(wta.size(), 0);

    vector<uint64_t> lex_smaller_table(256, 0);
    for (uint64_t i = 0, sum = 0; i < 256; ++i) {
        lex_smaller_table[i] = sum;
        sum += wta.rank(wta.size(), i);
    }

//    lb will be initialized, I promise. See line below.
    for (uint64_t i = 0, lb; i < wta.size(); ++i) {
        if (lcp[i] < k and open) {
            open = false;
            B[i - 1] = 1;

            Node *node = new Node(counter, lb, i - 1, k); // DRY
            graph.push_back(node);
            queue.push(node);
            counter++;
        } else if (lcp[i] == k and not open) {
            B[lb] = 1;
            open = true;
        }

        if (lcp[i] < k) {
            lb = i;
        }
    }

    bit_vector::rank_1_type bv_rank;
    util::init_support(bv_rank, &B);

    Node *stop_node = new Node(counter, 0, 0, 1);
    graph.push_back(stop_node); // Add stopnode.
    queue.push(stop_node);
    counter++;

//    PASS
    uint64_t quantity;
    vector<uint8_t> cs(wta.sigma);
    vector<uint64_t> rank_c_i(wta.sigma);
    vector<uint64_t> rank_c_j(wta.sigma);
    while (not queue.empty()) {
        Node *node = queue.front();
        queue.pop();
        bool extendable;

        do { // "repeat"
            extendable = false;
            interval_symbols(wta, node->lb, node->rb + 1, quantity, cs, rank_c_i, rank_c_j);

            for (uint64_t i = 0; i < quantity; ++i) {
                uint8_t c = cs[i]; // Onaj u lijevo za jedan.
                uint64_t lb = lex_smaller_table[c] + rank_c_i[i];
                uint64_t rb = lex_smaller_table[c] + rank_c_j[i] - 1;
                uint64_t ones = bv_rank(lb + 1);
                uint64_t id;

                if (ones % 2 == 0 and B[i] == 0) {
                    id = ground;
                } else {
                    id = (ones + 1) / 2;
                }

                if (id != ground) {
//                    graph[node->id]->successors.push_back(make_pair(node->id, rb - lb + 1)); //DRY
//                    graph[node->id]->lb = node->lb;
//                    graph[node->id]->rb = node->rb;
//                    graph[node->id]->len = node->len;
                } else if (not c <= 1) {
                    if (quantity == 1) {
                        extendable = true;
                        node->len++;
                        node->lb = lb;
                        node->rb = rb;
                    } else {
                        Node *new_node = new Node(counter, lb, rb, k);
                        graph.push_back(new_node);

//                        graph[node->id]->successors.push_back(make_pair(node->id, rb - lb + 1)); //DRY
//                        graph[node->id]->lb = node->lb;
//                        graph[node->id]->rb = node->rb;
//                        graph[node->id]->len = node->len;

                        queue.push(new_node);
                        counter++;
                    }
                }
            }
        } while (extendable);
    }
    return graph;
}

// Only for development purposes. When the project is done this may as well be deleted.
int main() {
    string input_string = "ACTACGTACGTACG"; // Notice no explicit dollar!

    wt_huff<> wta;
    construct_im(wta, input_string, 1);

    //region GET_INTERVALS
    lcp_wt<> lcp;
    construct_im(lcp, input_string, 1);

    cout << "Brujin:" << endl;
    vector<Node *> brojin = create_compress_graph(3, wta, lcp);
    for (int i = 0; i < brojin.size(); ++i) {
        cout << brojin[i]->lb << " " << brojin[i]->rb << endl;
    }
    cout << endl;
    //endregion
}
<|endoftext|>


<commit_before>/*
  ==============================================================================

    This file was auto-generated!

    It contains the basic framework code for a JUCE plugin processor.

  ==============================================================================
*/

#include "PluginProcessor.h"
#include "PluginEditor.h"

//==============================================================================
ATKColoredCompressorAudioProcessor::ATKColoredCompressorAudioProcessor()
  :
#ifndef JucePlugin_PreferredChannelConfigurations
    AudioProcessor (BusesProperties()
                     #if ! JucePlugin_IsMidiEffect
                      #if ! JucePlugin_IsSynth
                       .withInput  ("Input",  AudioChannelSet::stereo(), true)
                      #endif
                       .withOutput ("Output", AudioChannelSet::stereo(), true)
                     #endif
                       ),
#endif
  inFilter(nullptr, 1, 0, false), outFilter(nullptr, 1, 0, false), gainCompressorFilter(1, 256*1024), parameters(*this, nullptr), sampleRate(0), old_rms(0), old_attack(0), old_release(0), old_threshold(-1), old_slope(-1), old_softness(-5), old_color(-1), old_quality(-1), old_makeup(-1), old_drywet(-1)
{
  powerFilter.set_input_port(0, &inFilter, 0);
  attackReleaseFilter.set_input_port(0, &powerFilter, 0);
  gainCompressorFilter.set_input_port(0, &attackReleaseFilter, 0);
  applyGainFilter.set_input_port(0, &gainCompressorFilter, 0);
  applyGainFilter.set_input_port(1, &inFilter, 0);
  volumeFilter.set_input_port(0, &applyGainFilter, 0);
  drywetFilter.set_input_port(0, &volumeFilter, 0);
  drywetFilter.set_input_port(1, &inFilter, 0);
  outFilter.set_input_port(0, &drywetFilter, 0);

  parameters.createAndAddParameter("power", "Power", " ms", NormalisableRange<float>(0, 100), 10, nullptr, nullptr);
  parameters.createAndAddParameter("attack", "Attack", " ms", NormalisableRange<float>(1, 100), 10, nullptr, nullptr);
  parameters.createAndAddParameter("release", "Release", " ms",  NormalisableRange<float>(1, 100), 10, nullptr, nullptr);
  parameters.createAndAddParameter("threshold", "Threshold", " dB", NormalisableRange<float>(-40, 0), 0, nullptr, nullptr);
  parameters.createAndAddParameter("slope", "Slope", "", NormalisableRange<float>(1.5, 100), 2, nullptr, nullptr);
  parameters.createAndAddParameter("color", "Color", "", NormalisableRange<float>(-.5, .5), 0, nullptr, nullptr);
  parameters.createAndAddParameter("quality", "Quality", "", NormalisableRange<float>(0.01, .2), .1, nullptr, nullptr);
  parameters.createAndAddParameter("softness", "Softness", "", NormalisableRange<float>(-4, 0), -2, nullptr, nullptr);
  parameters.createAndAddParameter("makeup", "Makeup gain", " dB", NormalisableRange<float>(0, 40), 0, nullptr, nullptr);
  parameters.createAndAddParameter("drywet", "Dry/Wet", "", NormalisableRange<float>(0, 100), 100, nullptr, nullptr);

  parameters.state = ValueTree (Identifier ("ATKColoredCompressor"));;
}

ATKColoredCompressorAudioProcessor::~ATKColoredCompressorAudioProcessor()
{
}

//==============================================================================
const String ATKColoredCompressorAudioProcessor::getName() const
{
    return JucePlugin_Name;
}

bool ATKColoredCompressorAudioProcessor::acceptsMidi() const
{
   #if JucePlugin_WantsMidiInput
    return true;
   #else
    return false;
   #endif
}

bool ATKColoredCompressorAudioProcessor::producesMidi() const
{
   #if JucePlugin_ProducesMidiOutput
    return true;
   #else
    return false;
   #endif
}

double ATKColoredCompressorAudioProcessor::getTailLengthSeconds() const
{
    return 0.0;
}

int ATKColoredCompressorAudioProcessor::getNumPrograms()
{
    return 1;   // NB: some hosts don't cope very well if you tell them there are 0 programs,
                // so this should be at least 1, even if you're not really implementing programs.
}

int ATKColoredCompressorAudioProcessor::getCurrentProgram()
{
    return 0;
}

void ATKColoredCompressorAudioProcessor::setCurrentProgram (int index)
{
}

const String ATKColoredCompressorAudioProcessor::getProgramName (int index)
{
    return {};
}

void ATKColoredCompressorAudioProcessor::changeProgramName (int index, const String& newName)
{
}

//==============================================================================
void ATKColoredCompressorAudioProcessor::prepareToPlay (double dbSampleRate, int samplesPerBlock)
{
	auto sampleRate = std::lround(dbSampleRate);

  if(sampleRate != inFilter.get_output_sampling_rate())
  {
    inFilter.set_input_sampling_rate(sampleRate);
    inFilter.set_output_sampling_rate(sampleRate);
    powerFilter.set_input_sampling_rate(sampleRate);
    powerFilter.set_output_sampling_rate(sampleRate);
    attackReleaseFilter.set_input_sampling_rate(sampleRate);
    attackReleaseFilter.set_output_sampling_rate(sampleRate);
    gainCompressorFilter.set_input_sampling_rate(sampleRate);
    gainCompressorFilter.set_output_sampling_rate(sampleRate);
    applyGainFilter.set_input_sampling_rate(sampleRate);
    applyGainFilter.set_output_sampling_rate(sampleRate);
    volumeFilter.set_input_sampling_rate(sampleRate);
    volumeFilter.set_output_sampling_rate(sampleRate);
    drywetFilter.set_input_sampling_rate(sampleRate);
    drywetFilter.set_output_sampling_rate(sampleRate);
    outFilter.set_input_sampling_rate(sampleRate);
    outFilter.set_output_sampling_rate(sampleRate);
    outFilter.set_input_sampling_rate(sampleRate);
    outFilter.set_output_sampling_rate(sampleRate);

    if (old_rms == 0)
    {
      powerFilter.set_memory(0);
    }
    else
    {
      powerFilter.set_memory(std::exp(-1e3 / (old_rms * sampleRate)));
    }
    attackReleaseFilter.set_release(std::exp(-1e3/(old_release * sampleRate))); // in ms
    attackReleaseFilter.set_attack(std::exp(-1e3/(old_attack * sampleRate))); // in ms

    powerFilter.full_setup();
    attackReleaseFilter.full_setup();
  }
}

void ATKColoredCompressorAudioProcessor::releaseResources()
{
    // When playback stops, you can use this as an opportunity to free up any
    // spare memory, etc.
}

#ifndef JucePlugin_PreferredChannelConfigurations
bool ATKColoredCompressorAudioProcessor::isBusesLayoutSupported (const BusesLayout& layouts) const
{
  #if JucePlugin_IsMidiEffect
    ignoreUnused (layouts);
    return true;
  #else
    // This is the place where you check if the layout is supported.
    if (layouts.getMainOutputChannelSet() != AudioChannelSet::stereo())
        return false;

    // This checks if the input layout matches the output layout
   #if ! JucePlugin_IsSynth
    if (layouts.getMainOutputChannelSet() != layouts.getMainInputChannelSet())
        return false;
   #endif

    return true;
  #endif
}
#endif

void ATKColoredCompressorAudioProcessor::processBlock (AudioSampleBuffer& buffer, MidiBuffer& midiMessages)
{
  if (*parameters.getRawParameterValue ("power") != old_rms)
  {
    old_rms = *parameters.getRawParameterValue ("power");
    if (old_rms == 0)
    {
      powerFilter.set_memory(0);
    }
    else
    {
      powerFilter.set_memory(std::exp(-1e3 / (old_rms * sampleRate)));
    }
  }
  if(*parameters.getRawParameterValue ("release") != old_release)
  {
    old_release = *parameters.getRawParameterValue ("release");
    attackReleaseFilter.set_release(std::exp(-1e3/(old_release * sampleRate))); // in ms
  }
  if(*parameters.getRawParameterValue ("attack") != old_attack)
  {
    old_attack = *parameters.getRawParameterValue ("attack");
    attackReleaseFilter.set_attack(std::exp(-1e3/(old_attack * sampleRate))); // in ms
  }
  if(*parameters.getRawParameterValue ("threshold") != old_threshold)
  {
    old_threshold = *parameters.getRawParameterValue ("threshold");
    gainCompressorFilter.set_threshold(std::pow(10, old_threshold / 10));
  }
  if(*parameters.getRawParameterValue ("slope") != old_slope)
  {
    old_slope = *parameters.getRawParameterValue ("slope");
    gainCompressorFilter.set_ratio(old_slope);
  }
  if(*parameters.getRawParameterValue ("softness") != old_softness)
  {
    old_softness = *parameters.getRawParameterValue ("softness");
    gainCompressorFilter.set_softness(std::pow(10, old_softness));
  }
  if(*parameters.getRawParameterValue ("color") != old_color)
  {
    old_color = *parameters.getRawParameterValue ("color");
    gainCompressorFilter.set_color(old_color);
  }
  if(*parameters.getRawParameterValue ("quality") != old_quality)
  {
    old_quality = *parameters.getRawParameterValue ("quality");
    gainCompressorFilter.set_quality(old_quality);
  }
  if(*parameters.getRawParameterValue ("makeup") != old_makeup)
  {
    old_makeup = *parameters.getRawParameterValue ("makeup");
    volumeFilter.set_volume_db(old_makeup);
  }

  if(*parameters.getRawParameterValue ("drywet") != old_drywet)
  {
    old_drywet = *parameters.getRawParameterValue ("drywet");
    drywetFilter.set_dry(old_drywet / 100);
  }

  const int totalNumInputChannels  = getTotalNumInputChannels();
  const int totalNumOutputChannels = getTotalNumOutputChannels();

  assert(totalNumInputChannels == totalNumOutputChannels);
  assert(totalNumOutputChannels == 1);

  inFilter.set_pointer(buffer.getReadPointer(0), buffer.getNumSamples());
  outFilter.set_pointer(buffer.getWritePointer(0), buffer.getNumSamples());

  outFilter.process(buffer.getNumSamples());
}

//==============================================================================
bool ATKColoredCompressorAudioProcessor::hasEditor() const
{
    return true; // (change this to false if you choose to not supply an editor)
}

AudioProcessorEditor* ATKColoredCompressorAudioProcessor::createEditor()
{
    return new ATKColoredCompressorAudioProcessorEditor (*this, parameters);
}

//==============================================================================
void ATKColoredCompressorAudioProcessor::getStateInformation (MemoryBlock& destData)
{
  MemoryOutputStream store(destData, true);
  //store.writeInt(0); // version ID
  store.writeFloat(*parameters.getRawParameterValue ("power"));
  store.writeFloat(*parameters.getRawParameterValue ("attack"));
  store.writeFloat(*parameters.getRawParameterValue ("release"));
  store.writeFloat(*parameters.getRawParameterValue ("threshold"));
  store.writeFloat(*parameters.getRawParameterValue ("slope"));
  store.writeFloat(*parameters.getRawParameterValue ("softness"));
  store.writeFloat(*parameters.getRawParameterValue ("color"));
  store.writeFloat(*parameters.getRawParameterValue ("quality"));
  store.writeFloat(*parameters.getRawParameterValue ("makeup"));
  store.writeFloat(*parameters.getRawParameterValue ("drywet"));
}

void ATKColoredCompressorAudioProcessor::setStateInformation (const void* data, int sizeInBytes)
{
  MemoryInputStream store(data, static_cast<size_t> (sizeInBytes), false);
  //int version = store.readInt(); // version ID
  *parameters.getRawParameterValue ("power") = store.readFloat();
  *parameters.getRawParameterValue ("attack") = store.readFloat();
  *parameters.getRawParameterValue ("release") = store.readFloat();
  *parameters.getRawParameterValue ("threshold") = store.readFloat();
  *parameters.getRawParameterValue ("slope") = store.readFloat();
  *parameters.getRawParameterValue ("softness") = store.readFloat();
  *parameters.getRawParameterValue ("color") = store.readFloat();
  *parameters.getRawParameterValue ("quality") = store.readFloat();
  *parameters.getRawParameterValue ("makeup") = store.readFloat();
  *parameters.getRawParameterValue ("drywet") = store.readFloat();
}

//==============================================================================
// This creates new instances of the plugin..
AudioProcessor* JUCE_CALLTYPE createPluginFilter()
{
    return new ATKColoredCompressorAudioProcessor();
}
<commit_msg>Fixed full_setup calls<commit_after>/*
  ==============================================================================

    This file was auto-generated!

    It contains the basic framework code for a JUCE plugin processor.

  ==============================================================================
*/

#include "PluginProcessor.h"
#include "PluginEditor.h"

//==============================================================================
ATKColoredCompressorAudioProcessor::ATKColoredCompressorAudioProcessor()
  :
#ifndef JucePlugin_PreferredChannelConfigurations
    AudioProcessor (BusesProperties()
                     #if ! JucePlugin_IsMidiEffect
                      #if ! JucePlugin_IsSynth
                       .withInput  ("Input",  AudioChannelSet::stereo(), true)
                      #endif
                       .withOutput ("Output", AudioChannelSet::stereo(), true)
                     #endif
                       ),
#endif
  inFilter(nullptr, 1, 0, false), outFilter(nullptr, 1, 0, false), gainCompressorFilter(1, 256*1024), parameters(*this, nullptr), sampleRate(0), old_rms(0), old_attack(0), old_release(0), old_threshold(-1), old_slope(-1), old_softness(-5), old_color(-1), old_quality(-1), old_makeup(-1), old_drywet(-1)
{
  powerFilter.set_input_port(0, &inFilter, 0);
  attackReleaseFilter.set_input_port(0, &powerFilter, 0);
  gainCompressorFilter.set_input_port(0, &attackReleaseFilter, 0);
  applyGainFilter.set_input_port(0, &gainCompressorFilter, 0);
  applyGainFilter.set_input_port(1, &inFilter, 0);
  volumeFilter.set_input_port(0, &applyGainFilter, 0);
  drywetFilter.set_input_port(0, &volumeFilter, 0);
  drywetFilter.set_input_port(1, &inFilter, 0);
  outFilter.set_input_port(0, &drywetFilter, 0);

  parameters.createAndAddParameter("power", "Power", " ms", NormalisableRange<float>(0, 100), 10, nullptr, nullptr);
  parameters.createAndAddParameter("attack", "Attack", " ms", NormalisableRange<float>(1, 100), 10, nullptr, nullptr);
  parameters.createAndAddParameter("release", "Release", " ms",  NormalisableRange<float>(1, 100), 10, nullptr, nullptr);
  parameters.createAndAddParameter("threshold", "Threshold", " dB", NormalisableRange<float>(-40, 0), 0, nullptr, nullptr);
  parameters.createAndAddParameter("slope", "Slope", "", NormalisableRange<float>(1.5, 100), 2, nullptr, nullptr);
  parameters.createAndAddParameter("color", "Color", "", NormalisableRange<float>(-.5, .5), 0, nullptr, nullptr);
  parameters.createAndAddParameter("quality", "Quality", "", NormalisableRange<float>(0.01, .2), .1, nullptr, nullptr);
  parameters.createAndAddParameter("softness", "Softness", "", NormalisableRange<float>(-4, 0), -2, nullptr, nullptr);
  parameters.createAndAddParameter("makeup", "Makeup gain", " dB", NormalisableRange<float>(0, 40), 0, nullptr, nullptr);
  parameters.createAndAddParameter("drywet", "Dry/Wet", "", NormalisableRange<float>(0, 100), 100, nullptr, nullptr);

  parameters.state = ValueTree (Identifier ("ATKColoredCompressor"));;
}

ATKColoredCompressorAudioProcessor::~ATKColoredCompressorAudioProcessor()
{
}

//==============================================================================
const String ATKColoredCompressorAudioProcessor::getName() const
{
    return JucePlugin_Name;
}

bool ATKColoredCompressorAudioProcessor::acceptsMidi() const
{
   #if JucePlugin_WantsMidiInput
    return true;
   #else
    return false;
   #endif
}

bool ATKColoredCompressorAudioProcessor::producesMidi() const
{
   #if JucePlugin_ProducesMidiOutput
    return true;
   #else
    return false;
   #endif
}

double ATKColoredCompressorAudioProcessor::getTailLengthSeconds() const
{
    return 0.0;
}

int ATKColoredCompressorAudioProcessor::getNumPrograms()
{
    return 1;   // NB: some hosts don't cope very well if you tell them there are 0 programs,
                // so this should be at least 1, even if you're not really implementing programs.
}

int ATKColoredCompressorAudioProcessor::getCurrentProgram()
{
    return 0;
}

void ATKColoredCompressorAudioProcessor::setCurrentProgram (int index)
{
}

const String ATKColoredCompressorAudioProcessor::getProgramName (int index)
{
    return {};
}

void ATKColoredCompressorAudioProcessor::changeProgramName (int index, const String& newName)
{
}

//==============================================================================
void ATKColoredCompressorAudioProcessor::prepareToPlay (double dbSampleRate, int samplesPerBlock)
{
	auto sampleRate = std::lround(dbSampleRate);

  if(sampleRate != inFilter.get_output_sampling_rate())
  {
    inFilter.set_input_sampling_rate(sampleRate);
    inFilter.set_output_sampling_rate(sampleRate);
    powerFilter.set_input_sampling_rate(sampleRate);
    powerFilter.set_output_sampling_rate(sampleRate);
    attackReleaseFilter.set_input_sampling_rate(sampleRate);
    attackReleaseFilter.set_output_sampling_rate(sampleRate);
    gainCompressorFilter.set_input_sampling_rate(sampleRate);
    gainCompressorFilter.set_output_sampling_rate(sampleRate);
    applyGainFilter.set_input_sampling_rate(sampleRate);
    applyGainFilter.set_output_sampling_rate(sampleRate);
    volumeFilter.set_input_sampling_rate(sampleRate);
    volumeFilter.set_output_sampling_rate(sampleRate);
    drywetFilter.set_input_sampling_rate(sampleRate);
    drywetFilter.set_output_sampling_rate(sampleRate);
    outFilter.set_input_sampling_rate(sampleRate);
    outFilter.set_output_sampling_rate(sampleRate);
    outFilter.set_input_sampling_rate(sampleRate);
    outFilter.set_output_sampling_rate(sampleRate);

    if (old_rms == 0)
    {
      powerFilter.set_memory(0);
    }
    else
    {
      powerFilter.set_memory(std::exp(-1e3 / (old_rms * sampleRate)));
    }
    attackReleaseFilter.set_release(std::exp(-1e3/(old_release * sampleRate))); // in ms
    attackReleaseFilter.set_attack(std::exp(-1e3/(old_attack * sampleRate))); // in ms
  }
  powerFilter.full_setup();
  attackReleaseFilter.full_setup();
}

void ATKColoredCompressorAudioProcessor::releaseResources()
{
    // When playback stops, you can use this as an opportunity to free up any
    // spare memory, etc.
}

#ifndef JucePlugin_PreferredChannelConfigurations
bool ATKColoredCompressorAudioProcessor::isBusesLayoutSupported (const BusesLayout& layouts) const
{
  #if JucePlugin_IsMidiEffect
    ignoreUnused (layouts);
    return true;
  #else
    // This is the place where you check if the layout is supported.
    if (layouts.getMainOutputChannelSet() != AudioChannelSet::stereo())
        return false;

    // This checks if the input layout matches the output layout
   #if ! JucePlugin_IsSynth
    if (layouts.getMainOutputChannelSet() != layouts.getMainInputChannelSet())
        return false;
   #endif

    return true;
  #endif
}
#endif

void ATKColoredCompressorAudioProcessor::processBlock (AudioSampleBuffer& buffer, MidiBuffer& midiMessages)
{
  if (*parameters.getRawParameterValue ("power") != old_rms)
  {
    old_rms = *parameters.getRawParameterValue ("power");
    if (old_rms == 0)
    {
      powerFilter.set_memory(0);
    }
    else
    {
      powerFilter.set_memory(std::exp(-1e3 / (old_rms * sampleRate)));
    }
  }
  if(*parameters.getRawParameterValue ("release") != old_release)
  {
    old_release = *parameters.getRawParameterValue ("release");
    attackReleaseFilter.set_release(std::exp(-1e3/(old_release * sampleRate))); // in ms
  }
  if(*parameters.getRawParameterValue ("attack") != old_attack)
  {
    old_attack = *parameters.getRawParameterValue ("attack");
    attackReleaseFilter.set_attack(std::exp(-1e3/(old_attack * sampleRate))); // in ms
  }
  if(*parameters.getRawParameterValue ("threshold") != old_threshold)
  {
    old_threshold = *parameters.getRawParameterValue ("threshold");
    gainCompressorFilter.set_threshold(std::pow(10, old_threshold / 10));
  }
  if(*parameters.getRawParameterValue ("slope") != old_slope)
  {
    old_slope = *parameters.getRawParameterValue ("slope");
    gainCompressorFilter.set_ratio(old_slope);
  }
  if(*parameters.getRawParameterValue ("softness") != old_softness)
  {
    old_softness = *parameters.getRawParameterValue ("softness");
    gainCompressorFilter.set_softness(std::pow(10, old_softness));
  }
  if(*parameters.getRawParameterValue ("color") != old_color)
  {
    old_color = *parameters.getRawParameterValue ("color");
    gainCompressorFilter.set_color(old_color);
  }
  if(*parameters.getRawParameterValue ("quality") != old_quality)
  {
    old_quality = *parameters.getRawParameterValue ("quality");
    gainCompressorFilter.set_quality(old_quality);
  }
  if(*parameters.getRawParameterValue ("makeup") != old_makeup)
  {
    old_makeup = *parameters.getRawParameterValue ("makeup");
    volumeFilter.set_volume_db(old_makeup);
  }

  if(*parameters.getRawParameterValue ("drywet") != old_drywet)
  {
    old_drywet = *parameters.getRawParameterValue ("drywet");
    drywetFilter.set_dry(old_drywet / 100);
  }

  const int totalNumInputChannels  = getTotalNumInputChannels();
  const int totalNumOutputChannels = getTotalNumOutputChannels();

  assert(totalNumInputChannels == totalNumOutputChannels);
  assert(totalNumOutputChannels == 1);

  inFilter.set_pointer(buffer.getReadPointer(0), buffer.getNumSamples());
  outFilter.set_pointer(buffer.getWritePointer(0), buffer.getNumSamples());

  outFilter.process(buffer.getNumSamples());
}

//==============================================================================
bool ATKColoredCompressorAudioProcessor::hasEditor() const
{
    return true; // (change this to false if you choose to not supply an editor)
}

AudioProcessorEditor* ATKColoredCompressorAudioProcessor::createEditor()
{
    return new ATKColoredCompressorAudioProcessorEditor (*this, parameters);
}

//==============================================================================
void ATKColoredCompressorAudioProcessor::getStateInformation (MemoryBlock& destData)
{
  MemoryOutputStream store(destData, true);
  //store.writeInt(0); // version ID
  store.writeFloat(*parameters.getRawParameterValue ("power"));
  store.writeFloat(*parameters.getRawParameterValue ("attack"));
  store.writeFloat(*parameters.getRawParameterValue ("release"));
  store.writeFloat(*parameters.getRawParameterValue ("threshold"));
  store.writeFloat(*parameters.getRawParameterValue ("slope"));
  store.writeFloat(*parameters.getRawParameterValue ("softness"));
  store.writeFloat(*parameters.getRawParameterValue ("color"));
  store.writeFloat(*parameters.getRawParameterValue ("quality"));
  store.writeFloat(*parameters.getRawParameterValue ("makeup"));
  store.writeFloat(*parameters.getRawParameterValue ("drywet"));
}

void ATKColoredCompressorAudioProcessor::setStateInformation (const void* data, int sizeInBytes)
{
  MemoryInputStream store(data, static_cast<size_t> (sizeInBytes), false);
  //int version = store.readInt(); // version ID
  *parameters.getRawParameterValue ("power") = store.readFloat();
  *parameters.getRawParameterValue ("attack") = store.readFloat();
  *parameters.getRawParameterValue ("release") = store.readFloat();
  *parameters.getRawParameterValue ("threshold") = store.readFloat();
  *parameters.getRawParameterValue ("slope") = store.readFloat();
  *parameters.getRawParameterValue ("softness") = store.readFloat();
  *parameters.getRawParameterValue ("color") = store.readFloat();
  *parameters.getRawParameterValue ("quality") = store.readFloat();
  *parameters.getRawParameterValue ("makeup") = store.readFloat();
  *parameters.getRawParameterValue ("drywet") = store.readFloat();
}

//==============================================================================
// This creates new instances of the plugin..
AudioProcessor* JUCE_CALLTYPE createPluginFilter()
{
    return new ATKColoredCompressorAudioProcessor();
}
<|endoftext|>


<commit_before>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.h"

#include <random>

#include "tensorflow/core/platform/path.h"
#include "tensorflow/core/platform/stacktrace_handler.h"
#include "tensorflow/core/platform/test.h"

#define ASSERT_TF_OK(x) ASSERT_EQ(TF_OK, TF_GetCode(x)) << TF_Message(x)
#define EXPECT_TF_OK(x) EXPECT_EQ(TF_OK, TF_GetCode(x)) << TF_Message(x)

static std::string InitializeTmpDir() {
  // This env should be something like `s3://bucket/path`
  const char* test_dir = getenv("S3_TEST_TMPDIR");
  if (test_dir != nullptr) {
    Aws::String bucket, object;
    TF_Status* status = TF_NewStatus();
    ParseS3Path(test_dir, true, &bucket, &object, status);
    if (TF_GetCode(status) != TF_OK) {
      TF_DeleteStatus(status);
      return "";
    }
    TF_DeleteStatus(status);

    // We add a random value into `test_dir` to ensures that two consecutive
    // runs are unlikely to clash.
    std::random_device rd;
    std::mt19937 gen(rd());
    std::uniform_int_distribution<> distribution;
    std::string rng_val = std::to_string(distribution(gen));
    return tensorflow::io::JoinPath(std::string(test_dir), rng_val);
  } else {
    return "";
  }
}

static std::string* GetTmpDir() {
  static std::string tmp_dir = InitializeTmpDir();
  if (tmp_dir == "")
    return nullptr;
  else
    return &tmp_dir;
}

namespace tensorflow {
namespace {

class S3FilesystemTest : public ::testing::Test {
 public:
  void SetUp() override {
    root_dir_ = io::JoinPath(
        *GetTmpDir(),
        ::testing::UnitTest::GetInstance()->current_test_info()->name());
    status_ = TF_NewStatus();
    filesystem_ = new TF_Filesystem;
    tf_s3_filesystem::Init(filesystem_, status_);
    ASSERT_TF_OK(status_) << "Could not initialize filesystem. "
                          << TF_Message(status_);
  }
  void TearDown() override {
    TF_DeleteStatus(status_);
    tf_s3_filesystem::Cleanup(filesystem_);
    delete filesystem_;
  }

  std::string GetURIForPath(const std::string& path) {
    const std::string translated_name =
        tensorflow::io::JoinPath(root_dir_, path);
    return translated_name;
  }

  std::unique_ptr<TF_WritableFile, void (*)(TF_WritableFile* file)>
  GetWriter() {
    std::unique_ptr<TF_WritableFile, void (*)(TF_WritableFile * file)> writer(
        new TF_WritableFile, [](TF_WritableFile* file) {
          if (file != nullptr) {
            if (file->plugin_file != nullptr) tf_writable_file::Cleanup(file);
            delete file;
          }
        });
    writer->plugin_file = nullptr;
    return writer;
  }

  std::unique_ptr<TF_RandomAccessFile, void (*)(TF_RandomAccessFile* file)>
  GetReader() {
    std::unique_ptr<TF_RandomAccessFile, void (*)(TF_RandomAccessFile * file)>
        reader(new TF_RandomAccessFile, [](TF_RandomAccessFile* file) {
          if (file != nullptr) {
            if (file->plugin_file != nullptr)
              tf_random_access_file::Cleanup(file);
            delete file;
          }
        });
    reader->plugin_file = nullptr;
    return reader;
  }

  void WriteString(const std::string& path, const std::string& content) {
    auto writer = GetWriter();
    tf_s3_filesystem::NewWritableFile(filesystem_, path.c_str(), writer.get(),
                                      status_);
    if (TF_GetCode(status_) != TF_OK) return;
    tf_writable_file::Append(writer.get(), content.c_str(), content.length(),
                             status_);
    if (TF_GetCode(status_) != TF_OK) return;
    tf_writable_file::Close(writer.get(), status_);
    if (TF_GetCode(status_) != TF_OK) return;
  }

  std::string ReadAll(const string& path) {
    auto reader = GetReader();
    tf_s3_filesystem::NewRandomAccessFile(filesystem_, path.c_str(),
                                          reader.get(), status_);
    if (TF_GetCode(status_) != TF_OK) return "";

    auto file_size =
        tf_s3_filesystem::GetFileSize(filesystem_, path.c_str(), status_);
    if (TF_GetCode(status_) != TF_OK) return "";

    std::string content;
    content.resize(file_size);
    auto read = tf_random_access_file::Read(reader.get(), 0, file_size,
                                            &content[0], status_);
    if (TF_GetCode(status_) != TF_OK) return "";
    if (read >= 0) content.resize(read);
    if (file_size != content.size())
      TF_SetStatus(
          status_, TF_DATA_LOSS,
          std::string("expected " + std::to_string(file_size) + " got " +
                      std::to_string(content.size()) + " bytes")
              .c_str());
    return content;
  }

 protected:
  TF_Filesystem* filesystem_;
  TF_Status* status_;

 private:
  std::string root_dir_;
};

TEST_F(S3FilesystemTest, NewRandomAccessFile) {
  const std::string path = GetURIForPath("RandomAccessFile");
  const std::string content = "abcdefghijklmn";

  WriteString(path, content);
  ASSERT_TF_OK(status_);

  auto reader = GetReader();
  tf_s3_filesystem::NewRandomAccessFile(filesystem_, path.c_str(), reader.get(),
                                        status_);
  EXPECT_TF_OK(status_);

  std::string result;
  result.resize(content.size());
  auto read = tf_random_access_file::Read(reader.get(), 0, content.size(),
                                          &result[0], status_);
  result.resize(read);
  EXPECT_TF_OK(status_);
  EXPECT_EQ(content.size(), result.size());
  EXPECT_EQ(content, result);

  result.clear();
  result.resize(4);
  read = tf_random_access_file::Read(reader.get(), 2, 4, &result[0], status_);
  result.resize(read);
  EXPECT_TF_OK(status_);
  EXPECT_EQ(4, result.size());
  EXPECT_EQ(content.substr(2, 4), result);
}

TEST_F(S3FilesystemTest, NewWritableFile) {
  auto writer = GetWriter();
  const std::string path = GetURIForPath("WritableFile");
  tf_s3_filesystem::NewWritableFile(filesystem_, path.c_str(), writer.get(),
                                    status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Append(writer.get(), "content1,", strlen("content1,"),
                           status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Append(writer.get(), "content2", strlen("content2"),
                           status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Flush(writer.get(), status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Sync(writer.get(), status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Close(writer.get(), status_);
  EXPECT_TF_OK(status_);

  auto content = ReadAll(path);
  EXPECT_TF_OK(status_);
  EXPECT_EQ("content1,content2", content);
}

TEST_F(S3FilesystemTest, NewAppendableFile) {
  const std::string path = GetURIForPath("AppendableFile");
  WriteString(path, "test");
  ASSERT_TF_OK(status_);

  auto writer = GetWriter();
  tf_s3_filesystem::NewAppendableFile(filesystem_, path.c_str(), writer.get(),
                                      status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Append(writer.get(), "content", strlen("content"), status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Close(writer.get(), status_);
  EXPECT_TF_OK(status_);
}

TEST_F(S3FilesystemTest, NewReadOnlyMemoryRegionFromFile) {
  const std::string path = GetURIForPath("MemoryFile");
  const std::string content = "content";
  WriteString(path, content);
  ASSERT_TF_OK(status_);

  std::unique_ptr<TF_ReadOnlyMemoryRegion,
                  void (*)(TF_ReadOnlyMemoryRegion * file)>
      region(new TF_ReadOnlyMemoryRegion, [](TF_ReadOnlyMemoryRegion* file) {
        if (file != nullptr) {
          if (file->plugin_memory_region != nullptr)
            tf_read_only_memory_region::Cleanup(file);
          delete file;
        }
      });
  region->plugin_memory_region = nullptr;
  tf_s3_filesystem::NewReadOnlyMemoryRegionFromFile(filesystem_, path.c_str(),
                                                    region.get(), status_);
  EXPECT_TF_OK(status_);
  std::string result(reinterpret_cast<const char*>(
                         tf_read_only_memory_region::Data(region.get())),
                     tf_read_only_memory_region::Length(region.get()));
  EXPECT_EQ(content, result);
}

TEST_F(S3FilesystemTest, PathExists) {
  const std::string path = GetURIForPath("PathExists");
  tf_s3_filesystem::PathExists(filesystem_, path.c_str(), status_);
  EXPECT_EQ(TF_NOT_FOUND, TF_GetCode(status_)) << TF_Message(status_);
  TF_SetStatus(status_, TF_OK, "");
  WriteString(path, "test");
  ASSERT_TF_OK(status_);
  tf_s3_filesystem::PathExists(filesystem_, path.c_str(), status_);
  EXPECT_TF_OK(status_);
}

TEST_F(S3FilesystemTest, GetChildren) {
  const std::string base = GetURIForPath("GetChildren");
  tf_s3_filesystem::CreateDir(filesystem_, base.c_str(), status_);
  EXPECT_TF_OK(status_);

  const std::string file = io::JoinPath(base, "TestFile.csv");
  WriteString(file, "test");
  EXPECT_TF_OK(status_);

  const std::string subdir = io::JoinPath(base, "SubDir");
  tf_s3_filesystem::CreateDir(filesystem_, subdir.c_str(), status_);
  EXPECT_TF_OK(status_);
  const std::string subfile = io::JoinPath(subdir, "TestSubFile.csv");
  WriteString(subfile, "test");
  EXPECT_TF_OK(status_);

  char** entries;
  auto num_entries = tf_s3_filesystem::GetChildren(filesystem_, base.c_str(),
                                                   &entries, status_);
  EXPECT_TF_OK(status_);

  std::vector<std::string> childrens;
  for (int i = 0; i < num_entries; ++i) {
    childrens.push_back(entries[i]);
  }
  std::sort(childrens.begin(), childrens.end());
  EXPECT_EQ(std::vector<string>({"SubDir", "TestFile.csv"}), childrens);
}

TEST_F(S3FilesystemTest, DeleteFile) {
  const std::string path = GetURIForPath("DeleteFile");
  WriteString(path, "test");
  ASSERT_TF_OK(status_);
  tf_s3_filesystem::DeleteFile(filesystem_, path.c_str(), status_);
  EXPECT_TF_OK(status_);
}

TEST_F(S3FilesystemTest, CreateDir) {
  // s3 object storage doesn't support empty directory, we create file in the
  // directory
  const std::string dir = GetURIForPath("CreateDir");
  tf_s3_filesystem::CreateDir(filesystem_, dir.c_str(), status_);
  EXPECT_TF_OK(status_);

  const std::string file = io::JoinPath(dir, "CreateDirFile.csv");
  WriteString(file, "test");
  ASSERT_TF_OK(status_);

  TF_FileStatistics stat;
  tf_s3_filesystem::Stat(filesystem_, dir.c_str(), &stat, status_);
  EXPECT_TF_OK(status_);
  EXPECT_TRUE(stat.is_directory);
}

TEST_F(S3FilesystemTest, DeleteDir) {
  // s3 object storage doesn't support empty directory, we create file in the
  // directory
  const std::string dir = GetURIForPath("DeleteDir");
  const std::string file = io::JoinPath(dir, "DeleteDirFile.csv");
  WriteString(file, "test");
  ASSERT_TF_OK(status_);
  tf_s3_filesystem::DeleteDir(filesystem_, dir.c_str(), status_);
  EXPECT_NE(TF_GetCode(status_), TF_OK);

  TF_SetStatus(status_, TF_OK, "");
  tf_s3_filesystem::DeleteFile(filesystem_, file.c_str(), status_);
  EXPECT_TF_OK(status_);
  tf_s3_filesystem::DeleteDir(filesystem_, dir.c_str(), status_);
  EXPECT_TF_OK(status_);
  TF_FileStatistics stat;
  tf_s3_filesystem::Stat(filesystem_, dir.c_str(), &stat, status_);
  EXPECT_EQ(TF_GetCode(status_), TF_NOT_FOUND) << TF_Message(status_);
}

}  // namespace
}  // namespace tensorflow

GTEST_API_ int main(int argc, char** argv) {
  tensorflow::testing::InstallStacktraceHandler();
  if (!GetTmpDir()) {
    std::cerr << "Could not read S3_TEST_TMPDIR env";
    return -1;
  }
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<commit_msg>Add test StatFile<commit_after>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/c/experimental/filesystem/plugins/s3/s3_filesystem.h"

#include <random>

#include "tensorflow/core/platform/path.h"
#include "tensorflow/core/platform/stacktrace_handler.h"
#include "tensorflow/core/platform/test.h"

#define ASSERT_TF_OK(x) ASSERT_EQ(TF_OK, TF_GetCode(x)) << TF_Message(x)
#define EXPECT_TF_OK(x) EXPECT_EQ(TF_OK, TF_GetCode(x)) << TF_Message(x)

static std::string InitializeTmpDir() {
  // This env should be something like `s3://bucket/path`
  const char* test_dir = getenv("S3_TEST_TMPDIR");
  if (test_dir != nullptr) {
    Aws::String bucket, object;
    TF_Status* status = TF_NewStatus();
    ParseS3Path(test_dir, true, &bucket, &object, status);
    if (TF_GetCode(status) != TF_OK) {
      TF_DeleteStatus(status);
      return "";
    }
    TF_DeleteStatus(status);

    // We add a random value into `test_dir` to ensures that two consecutive
    // runs are unlikely to clash.
    std::random_device rd;
    std::mt19937 gen(rd());
    std::uniform_int_distribution<> distribution;
    std::string rng_val = std::to_string(distribution(gen));
    return tensorflow::io::JoinPath(std::string(test_dir), rng_val);
  } else {
    return "";
  }
}

static std::string* GetTmpDir() {
  static std::string tmp_dir = InitializeTmpDir();
  if (tmp_dir == "")
    return nullptr;
  else
    return &tmp_dir;
}

namespace tensorflow {
namespace {

class S3FilesystemTest : public ::testing::Test {
 public:
  void SetUp() override {
    root_dir_ = io::JoinPath(
        *GetTmpDir(),
        ::testing::UnitTest::GetInstance()->current_test_info()->name());
    status_ = TF_NewStatus();
    filesystem_ = new TF_Filesystem;
    tf_s3_filesystem::Init(filesystem_, status_);
    ASSERT_TF_OK(status_) << "Could not initialize filesystem. "
                          << TF_Message(status_);
  }
  void TearDown() override {
    TF_DeleteStatus(status_);
    tf_s3_filesystem::Cleanup(filesystem_);
    delete filesystem_;
  }

  std::string GetURIForPath(const std::string& path) {
    const std::string translated_name =
        tensorflow::io::JoinPath(root_dir_, path);
    return translated_name;
  }

  std::unique_ptr<TF_WritableFile, void (*)(TF_WritableFile* file)>
  GetWriter() {
    std::unique_ptr<TF_WritableFile, void (*)(TF_WritableFile * file)> writer(
        new TF_WritableFile, [](TF_WritableFile* file) {
          if (file != nullptr) {
            if (file->plugin_file != nullptr) tf_writable_file::Cleanup(file);
            delete file;
          }
        });
    writer->plugin_file = nullptr;
    return writer;
  }

  std::unique_ptr<TF_RandomAccessFile, void (*)(TF_RandomAccessFile* file)>
  GetReader() {
    std::unique_ptr<TF_RandomAccessFile, void (*)(TF_RandomAccessFile * file)>
        reader(new TF_RandomAccessFile, [](TF_RandomAccessFile* file) {
          if (file != nullptr) {
            if (file->plugin_file != nullptr)
              tf_random_access_file::Cleanup(file);
            delete file;
          }
        });
    reader->plugin_file = nullptr;
    return reader;
  }

  void WriteString(const std::string& path, const std::string& content) {
    auto writer = GetWriter();
    tf_s3_filesystem::NewWritableFile(filesystem_, path.c_str(), writer.get(),
                                      status_);
    if (TF_GetCode(status_) != TF_OK) return;
    tf_writable_file::Append(writer.get(), content.c_str(), content.length(),
                             status_);
    if (TF_GetCode(status_) != TF_OK) return;
    tf_writable_file::Close(writer.get(), status_);
    if (TF_GetCode(status_) != TF_OK) return;
  }

  std::string ReadAll(const string& path) {
    auto reader = GetReader();
    tf_s3_filesystem::NewRandomAccessFile(filesystem_, path.c_str(),
                                          reader.get(), status_);
    if (TF_GetCode(status_) != TF_OK) return "";

    auto file_size =
        tf_s3_filesystem::GetFileSize(filesystem_, path.c_str(), status_);
    if (TF_GetCode(status_) != TF_OK) return "";

    std::string content;
    content.resize(file_size);
    auto read = tf_random_access_file::Read(reader.get(), 0, file_size,
                                            &content[0], status_);
    if (TF_GetCode(status_) != TF_OK) return "";
    if (read >= 0) content.resize(read);
    if (file_size != content.size())
      TF_SetStatus(
          status_, TF_DATA_LOSS,
          std::string("expected " + std::to_string(file_size) + " got " +
                      std::to_string(content.size()) + " bytes")
              .c_str());
    return content;
  }

 protected:
  TF_Filesystem* filesystem_;
  TF_Status* status_;

 private:
  std::string root_dir_;
};

TEST_F(S3FilesystemTest, NewRandomAccessFile) {
  const std::string path = GetURIForPath("RandomAccessFile");
  const std::string content = "abcdefghijklmn";

  WriteString(path, content);
  ASSERT_TF_OK(status_);

  auto reader = GetReader();
  tf_s3_filesystem::NewRandomAccessFile(filesystem_, path.c_str(), reader.get(),
                                        status_);
  EXPECT_TF_OK(status_);

  std::string result;
  result.resize(content.size());
  auto read = tf_random_access_file::Read(reader.get(), 0, content.size(),
                                          &result[0], status_);
  result.resize(read);
  EXPECT_TF_OK(status_);
  EXPECT_EQ(content.size(), result.size());
  EXPECT_EQ(content, result);

  result.clear();
  result.resize(4);
  read = tf_random_access_file::Read(reader.get(), 2, 4, &result[0], status_);
  result.resize(read);
  EXPECT_TF_OK(status_);
  EXPECT_EQ(4, result.size());
  EXPECT_EQ(content.substr(2, 4), result);
}

TEST_F(S3FilesystemTest, NewWritableFile) {
  auto writer = GetWriter();
  const std::string path = GetURIForPath("WritableFile");
  tf_s3_filesystem::NewWritableFile(filesystem_, path.c_str(), writer.get(),
                                    status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Append(writer.get(), "content1,", strlen("content1,"),
                           status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Append(writer.get(), "content2", strlen("content2"),
                           status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Flush(writer.get(), status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Sync(writer.get(), status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Close(writer.get(), status_);
  EXPECT_TF_OK(status_);

  auto content = ReadAll(path);
  EXPECT_TF_OK(status_);
  EXPECT_EQ("content1,content2", content);
}

TEST_F(S3FilesystemTest, NewAppendableFile) {
  const std::string path = GetURIForPath("AppendableFile");
  WriteString(path, "test");
  ASSERT_TF_OK(status_);

  auto writer = GetWriter();
  tf_s3_filesystem::NewAppendableFile(filesystem_, path.c_str(), writer.get(),
                                      status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Append(writer.get(), "content", strlen("content"), status_);
  EXPECT_TF_OK(status_);
  tf_writable_file::Close(writer.get(), status_);
  EXPECT_TF_OK(status_);
}

TEST_F(S3FilesystemTest, NewReadOnlyMemoryRegionFromFile) {
  const std::string path = GetURIForPath("MemoryFile");
  const std::string content = "content";
  WriteString(path, content);
  ASSERT_TF_OK(status_);

  std::unique_ptr<TF_ReadOnlyMemoryRegion,
                  void (*)(TF_ReadOnlyMemoryRegion * file)>
      region(new TF_ReadOnlyMemoryRegion, [](TF_ReadOnlyMemoryRegion* file) {
        if (file != nullptr) {
          if (file->plugin_memory_region != nullptr)
            tf_read_only_memory_region::Cleanup(file);
          delete file;
        }
      });
  region->plugin_memory_region = nullptr;
  tf_s3_filesystem::NewReadOnlyMemoryRegionFromFile(filesystem_, path.c_str(),
                                                    region.get(), status_);
  EXPECT_TF_OK(status_);
  std::string result(reinterpret_cast<const char*>(
                         tf_read_only_memory_region::Data(region.get())),
                     tf_read_only_memory_region::Length(region.get()));
  EXPECT_EQ(content, result);
}

TEST_F(S3FilesystemTest, PathExists) {
  const std::string path = GetURIForPath("PathExists");
  tf_s3_filesystem::PathExists(filesystem_, path.c_str(), status_);
  EXPECT_EQ(TF_NOT_FOUND, TF_GetCode(status_)) << TF_Message(status_);
  TF_SetStatus(status_, TF_OK, "");
  WriteString(path, "test");
  ASSERT_TF_OK(status_);
  tf_s3_filesystem::PathExists(filesystem_, path.c_str(), status_);
  EXPECT_TF_OK(status_);
}

TEST_F(S3FilesystemTest, GetChildren) {
  const std::string base = GetURIForPath("GetChildren");
  tf_s3_filesystem::CreateDir(filesystem_, base.c_str(), status_);
  EXPECT_TF_OK(status_);

  const std::string file = io::JoinPath(base, "TestFile.csv");
  WriteString(file, "test");
  EXPECT_TF_OK(status_);

  const std::string subdir = io::JoinPath(base, "SubDir");
  tf_s3_filesystem::CreateDir(filesystem_, subdir.c_str(), status_);
  EXPECT_TF_OK(status_);
  const std::string subfile = io::JoinPath(subdir, "TestSubFile.csv");
  WriteString(subfile, "test");
  EXPECT_TF_OK(status_);

  char** entries;
  auto num_entries = tf_s3_filesystem::GetChildren(filesystem_, base.c_str(),
                                                   &entries, status_);
  EXPECT_TF_OK(status_);

  std::vector<std::string> childrens;
  for (int i = 0; i < num_entries; ++i) {
    childrens.push_back(entries[i]);
  }
  std::sort(childrens.begin(), childrens.end());
  EXPECT_EQ(std::vector<string>({"SubDir", "TestFile.csv"}), childrens);
}

TEST_F(S3FilesystemTest, DeleteFile) {
  const std::string path = GetURIForPath("DeleteFile");
  WriteString(path, "test");
  ASSERT_TF_OK(status_);
  tf_s3_filesystem::DeleteFile(filesystem_, path.c_str(), status_);
  EXPECT_TF_OK(status_);
}

TEST_F(S3FilesystemTest, CreateDir) {
  // s3 object storage doesn't support empty directory, we create file in the
  // directory
  const std::string dir = GetURIForPath("CreateDir");
  tf_s3_filesystem::CreateDir(filesystem_, dir.c_str(), status_);
  EXPECT_TF_OK(status_);

  const std::string file = io::JoinPath(dir, "CreateDirFile.csv");
  WriteString(file, "test");
  ASSERT_TF_OK(status_);

  TF_FileStatistics stat;
  tf_s3_filesystem::Stat(filesystem_, dir.c_str(), &stat, status_);
  EXPECT_TF_OK(status_);
  EXPECT_TRUE(stat.is_directory);
}

TEST_F(S3FilesystemTest, DeleteDir) {
  // s3 object storage doesn't support empty directory, we create file in the
  // directory
  const std::string dir = GetURIForPath("DeleteDir");
  const std::string file = io::JoinPath(dir, "DeleteDirFile.csv");
  WriteString(file, "test");
  ASSERT_TF_OK(status_);
  tf_s3_filesystem::DeleteDir(filesystem_, dir.c_str(), status_);
  EXPECT_NE(TF_GetCode(status_), TF_OK);

  TF_SetStatus(status_, TF_OK, "");
  tf_s3_filesystem::DeleteFile(filesystem_, file.c_str(), status_);
  EXPECT_TF_OK(status_);
  tf_s3_filesystem::DeleteDir(filesystem_, dir.c_str(), status_);
  EXPECT_TF_OK(status_);
  TF_FileStatistics stat;
  tf_s3_filesystem::Stat(filesystem_, dir.c_str(), &stat, status_);
  EXPECT_EQ(TF_GetCode(status_), TF_NOT_FOUND) << TF_Message(status_);
}

TEST_F(S3FilesystemTest, StatFile) {
  const std::string path = GetURIForPath("StatFile");
  WriteString(path, "test");
  ASSERT_TF_OK(status_);

  TF_FileStatistics stat;
  tf_s3_filesystem::Stat(filesystem_, path.c_str(), &stat, status_);
  EXPECT_TF_OK(status_);
  EXPECT_EQ(4, stat.length);
  EXPECT_FALSE(stat.is_directory);
}

}  // namespace
}  // namespace tensorflow

GTEST_API_ int main(int argc, char** argv) {
  tensorflow::testing::InstallStacktraceHandler();
  if (!GetTmpDir()) {
    std::cerr << "Could not read S3_TEST_TMPDIR env";
    return -1;
  }
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<|endoftext|>