<commit_before>/*
 *  Copyright 2008-2009 NVIDIA Corporation
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

/*! \file fill.inl
 *  \brief Inline file for fill.h.
 */

#ifdef __CUDACC__

/////////////////////
// NVCC definition //
/////////////////////

#include <thrust/detail/util/align.h>
#include <thrust/detail/device/generate.h>
#include <thrust/iterator/iterator_traits.h>
#include <thrust/detail/type_traits.h>
#include <thrust/extrema.h>

namespace thrust
{
namespace detail
{
namespace device
{
namespace cuda
{
namespace detail
{

template <typename T>
struct fill_functor
{
  T exemplar;

  fill_functor(T _exemplar)
    : exemplar(_exemplar) {}

  __device__
  T operator()(void)
  {
    return exemplar;
  }
}; // end fill_functor

template<typename Pointer, typename T>
  void wide_fill(Pointer first,
                 Pointer last,
                 const T &exemplar)
{
  typedef typename thrust::iterator_value<Pointer>::type OutputType;

  typedef unsigned long long WideType; // type used to pack the Ts

  size_t ALIGNMENT_BOUNDARY = 128; // begin copying blocks at this byte boundary

  size_t n = last - first;

  WideType wide_exemplar;
  for(size_t i = 0; i < sizeof(WideType)/sizeof(T); i++)
      reinterpret_cast<T *>(&wide_exemplar)[i] = exemplar;

  OutputType *first_raw = thrust::raw_pointer_cast(first);
  OutputType *last_raw  = thrust::raw_pointer_cast(last);

  OutputType *block_first_raw = thrust::min(first_raw + n,   thrust::detail::util::align_up(first_raw, ALIGNMENT_BOUNDARY));
  OutputType *block_last_raw  = thrust::max(block_first_raw, thrust::detail::util::align_down(last_raw, sizeof(WideType)));

  thrust::device_ptr<WideType> block_first_wide = thrust::device_pointer_cast(reinterpret_cast<WideType*>(block_first_raw));
  thrust::device_ptr<WideType> block_last_wide  = thrust::device_pointer_cast(reinterpret_cast<WideType*>(block_last_raw));

  thrust::detail::device::generate(first, thrust::device_pointer_cast(block_first_raw), fill_functor<OutputType>(exemplar));
  thrust::detail::device::generate(block_first_wide, block_last_wide,
                                   fill_functor<WideType>(wide_exemplar));
  thrust::detail::device::generate(thrust::device_pointer_cast(block_last_raw), last, fill_functor<OutputType>(exemplar));
}

template<typename ForwardIterator, typename T>
  void fill(ForwardIterator first,
            ForwardIterator last,
            const T &exemplar,
            thrust::detail::false_type)
{
  fill_functor<T> func(exemplar);
  thrust::detail::device::generate(first, last, func);
}

template<typename ForwardIterator, typename T>
  void fill(ForwardIterator first,
            ForwardIterator last,
            const T &exemplar,
            thrust::detail::true_type)
{
    typedef typename thrust::iterator_traits<ForwardIterator>::value_type OutputType;

    if ( thrust::detail::util::is_aligned<OutputType>(thrust::raw_pointer_cast(&*first)) )
    {
        wide_fill(&*first, &*last, exemplar);
    }
    else
    {
        fill(first, last, exemplar, thrust::detail::false_type());
    }
}

} // end detail

template<typename ForwardIterator, typename T>
  void fill(ForwardIterator first,
            ForwardIterator last,
            const T &exemplar)
{
  typedef typename thrust::iterator_traits<ForwardIterator>::value_type      OutputType;
  typedef typename thrust::iterator_traits<ForwardIterator>::difference_type IndexType;

  // we're compiling with nvcc, launch a kernel
  const bool use_wide_fill = thrust::detail::is_trivial_iterator<ForwardIterator>::value
      && thrust::detail::has_trivial_copy<OutputType>::value
      && (sizeof(OutputType) == 1 || sizeof(OutputType) == 2 || sizeof(OutputType) == 4);

  // XXX WAR nvcc 3.0 usused variable warning
  (void)use_wide_fill;

  detail::fill(first, last, exemplar, thrust::detail::integral_constant<bool, use_wide_fill>());
}

} // end namespace cuda
} // end namespace device
} // end namespace detail
} // end namespace thrust

#else

///////////////////////////
// C++ (only) definition //
///////////////////////////

#include <thrust/copy.h>
#include <thrust/device_ptr.h>
#include <thrust/distance.h>

namespace thrust
{
namespace detail
{
namespace device
{
namespace cuda
{

template<typename ForwardIterator, typename T>
  void fill(ForwardIterator first,
            ForwardIterator last,
            const T &exemplar)
{
  typedef typename thrust::iterator_traits<ForwardIterator>::value_type      OutputType;
  typedef typename thrust::iterator_traits<ForwardIterator>::difference_type IndexType;

  // we can't launch a kernel, implement this with a copy
  IndexType n = thrust::distance(first,last);
  raw_host_buffer<OutputType> temp(n);
  thrust::fill(temp.begin(), temp.end(), exemplar);
  thrust::copy(temp.begin(), temp.end(), first);
}

} // end namespace cuda
} // end namespace device
} // end namespace detail
} // end namespace thrust

#endif // __CUDACC__

<commit_msg>fixed bug exposed by TestFillMixedTypesDevice<commit_after>/*
 *  Copyright 2008-2009 NVIDIA Corporation
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

/*! \file fill.inl
 *  \brief Inline file for fill.h.
 */

#ifdef __CUDACC__

/////////////////////
// NVCC definition //
/////////////////////

#include <thrust/detail/util/align.h>
#include <thrust/detail/device/generate.h>
#include <thrust/iterator/iterator_traits.h>
#include <thrust/detail/type_traits.h>
#include <thrust/extrema.h>

namespace thrust
{
namespace detail
{
namespace device
{
namespace cuda
{
namespace detail
{

template <typename T>
struct fill_functor
{
  T exemplar;

  fill_functor(T _exemplar)
    : exemplar(_exemplar) {}

  __device__
  T operator()(void)
  {
    return exemplar;
  }
}; // end fill_functor

template<typename Pointer, typename T>
  void wide_fill(Pointer first,
                 Pointer last,
                 const T &exemplar)
{
  typedef typename thrust::iterator_value<Pointer>::type OutputType;

  typedef unsigned long long WideType; // type used to pack the Ts

  size_t ALIGNMENT_BOUNDARY = 128; // begin copying blocks at this byte boundary

  size_t n = last - first;

  WideType wide_exemplar;
  for(size_t i = 0; i < sizeof(WideType)/sizeof(OutputType); i++)
      reinterpret_cast<OutputType *>(&wide_exemplar)[i] = static_cast<OutputType>(exemplar);

  OutputType *first_raw = thrust::raw_pointer_cast(first);
  OutputType *last_raw  = thrust::raw_pointer_cast(last);

  OutputType *block_first_raw = thrust::min(first_raw + n,   thrust::detail::util::align_up(first_raw, ALIGNMENT_BOUNDARY));
  OutputType *block_last_raw  = thrust::max(block_first_raw, thrust::detail::util::align_down(last_raw, sizeof(WideType)));

  thrust::device_ptr<WideType> block_first_wide = thrust::device_pointer_cast(reinterpret_cast<WideType*>(block_first_raw));
  thrust::device_ptr<WideType> block_last_wide  = thrust::device_pointer_cast(reinterpret_cast<WideType*>(block_last_raw));

  thrust::detail::device::generate(first, thrust::device_pointer_cast(block_first_raw), fill_functor<OutputType>(exemplar));
  thrust::detail::device::generate(block_first_wide, block_last_wide,
                                   fill_functor<WideType>(wide_exemplar));
  thrust::detail::device::generate(thrust::device_pointer_cast(block_last_raw), last, fill_functor<OutputType>(exemplar));
}

template<typename ForwardIterator, typename T>
  void fill(ForwardIterator first,
            ForwardIterator last,
            const T &exemplar,
            thrust::detail::false_type)
{
  fill_functor<T> func(exemplar);
  thrust::detail::device::generate(first, last, func);
}

template<typename ForwardIterator, typename T>
  void fill(ForwardIterator first,
            ForwardIterator last,
            const T &exemplar,
            thrust::detail::true_type)
{
    typedef typename thrust::iterator_traits<ForwardIterator>::value_type OutputType;

    if ( thrust::detail::util::is_aligned<OutputType>(thrust::raw_pointer_cast(&*first)) )
    {
        wide_fill(&*first, &*last, exemplar);
    }
    else
    {
        fill(first, last, exemplar, thrust::detail::false_type());
    }
}

} // end detail

template<typename ForwardIterator, typename T>
  void fill(ForwardIterator first,
            ForwardIterator last,
            const T &exemplar)
{
  typedef typename thrust::iterator_traits<ForwardIterator>::value_type      OutputType;
  typedef typename thrust::iterator_traits<ForwardIterator>::difference_type IndexType;

  // we're compiling with nvcc, launch a kernel
  const bool use_wide_fill = thrust::detail::is_trivial_iterator<ForwardIterator>::value
      && thrust::detail::has_trivial_copy<OutputType>::value
      && (sizeof(OutputType) == 1 || sizeof(OutputType) == 2 || sizeof(OutputType) == 4);

  // XXX WAR nvcc 3.0 usused variable warning
  (void)use_wide_fill;

  detail::fill(first, last, exemplar, thrust::detail::integral_constant<bool, use_wide_fill>());
}

} // end namespace cuda
} // end namespace device
} // end namespace detail
} // end namespace thrust

#else

///////////////////////////
// C++ (only) definition //
///////////////////////////

#include <thrust/copy.h>
#include <thrust/device_ptr.h>
#include <thrust/distance.h>

namespace thrust
{
namespace detail
{
namespace device
{
namespace cuda
{

template<typename ForwardIterator, typename T>
  void fill(ForwardIterator first,
            ForwardIterator last,
            const T &exemplar)
{
  typedef typename thrust::iterator_traits<ForwardIterator>::value_type      OutputType;
  typedef typename thrust::iterator_traits<ForwardIterator>::difference_type IndexType;

  // we can't launch a kernel, implement this with a copy
  IndexType n = thrust::distance(first,last);
  raw_host_buffer<OutputType> temp(n);
  thrust::fill(temp.begin(), temp.end(), exemplar);
  thrust::copy(temp.begin(), temp.end(), first);
}

} // end namespace cuda
} // end namespace device
} // end namespace detail
} // end namespace thrust

#endif // __CUDACC__

<|endoftext|>


<commit_before>#include <stdlib.h>
#include <algorithm>
#include <iostream>
#include <vector>
#include <unistd.h>
#include <dirent.h>
#include <fstream>
#include <queue>
#include "omp.h"

// Contains the various options the user can pass ptgz.
struct Settings {
	Settings(): extract(),
				compress(),
   				verbose(),
				output() {}
	bool extract;
	bool compress;
	bool verbose;
	bool output;
	std::string name;
};

// Checks if the user asks for help.
// Provides usage information to the user.
// Parameters: argc (int) number of cli arguments.
// 			   argv (char *[]) user provided arguments.
void helpCheck(int argc, char *argv[]) {
	if (argc == 1) {
		std::cout << "ERROR: ptgz was passed no parameters. \"ptgz -h\" for help." << std::endl;
		exit(0);
	}

	if (argv[1] == std::string("-h") || argv[1] == std::string("--help")) {
		std::cout << "\nptgz - Parallel Tar GZ by Ryan Chui (2017)\n" << std::endl;
		exit(0);
	}
}

// Gets the parameters passed by the user and stores them.
// Parameters: argc (int) number of cli arguments.
// 			   argv (char *[]) user provided arguments.
// 			   instance (Settings *) user argument storage.
void getSettings(int argc, char *argv[], Settings *instance) {
	// Get all passed arguments
	std::queue<std::string> settings;
	for (int i = 1; i < argc; ++i) {
		settings.push(argv[i]);
	}

	// Continue to check until there are no more passed arguments.
	while (!settings.empty()) {
		std::string arg = settings.front();

		if (arg == "-x") {
			if ((*instance).compress) {
				std::cout << "ERROR: ptgz cannot both compress and extract. \"ptgz -h\" for help." << std::endl;
				exit(0);
			}
			(*instance).extract = true;
		} else if (arg == "-c") {
			if ((*instance).extract) {
				std::cout << "ERROR: ptgz cannot both compress and extract. \"ptgz -h\" for help." << std::endl;
				exit(0);
			}
			(*instance).compress = true;
		} else if (arg == "-v"){
			(*instance).verbose = true;
		} else if (arg == "-o") {
			(*instance).output = true;
		} else {
			if (settings.size() > 1) {
				std::cout << "ERROR: ptgz was called incorrectly. \"ptgz -h\" for help." << std::endl;
				exit(0);
			}
			(*instance).output = true;
			(*instance).name = arg;
		}

		settings.pop();
	}

	if (!(*instance).output) {
		std::cout << "ERROR: No output file name given. \"ptgz -h\" for help." << std::endl;
		exit(0);
	}
}

// Finds the number of files in the space to store.
// Parameters: numFiles (int *) number of files.
// 			   cwd (const char *) current working directory.
void findAll(int *numFiles, const char *cwd) {
	DIR *dir;
	struct dirent *ent;

	// Check if cwd is a directory
	if ((dir = opendir(cwd)) != NULL) {
		// Get all file paths within directory.
		while ((ent = readdir (dir)) != NULL) {
			std::string fileBuff = std::string(ent -> d_name);
			if (fileBuff != "." && fileBuff != "..") {
				DIR *dir2;
				std::string filePath = std::string(cwd) + "/" + fileBuff;
				// Check if file path is a directory.
				if ((dir2 = opendir(filePath.c_str())) != NULL) {
					closedir(dir2);
					findAll(numFiles, filePath.c_str());
				} else {
					*numFiles += 1;
				}
			}
		}
		closedir(dir);
	}
}

// Gets the paths for all files in the space to store.
// Parameters: filePaths (std::vector<std::string> *) holder for all file paths.
// 			   cwd (const char *) current working directory.
// 			   rootPath (std::string) path from the root of the directory to be stored.
void getPaths(std::vector<std::string> *filePaths, const char *cwd, std::string rootPath) {
	DIR *dir;
	struct dirent *ent;

	// Check if cwd is a directory
	if ((dir = opendir(cwd)) != NULL) {
		// Get all file paths within directory.
		while ((ent = readdir (dir)) != NULL) {
			std::string fileBuff = std::string(ent -> d_name);
			if (fileBuff != "." && fileBuff != "..") {
				DIR *dir2;
				std::string filePath = std::string(cwd) + "/" + fileBuff;
				// Check if file path is a directory.
				if ((dir2 = opendir(filePath.c_str())) != NULL) {
					closedir(dir2);
					getPaths(filePaths, filePath.c_str(), rootPath + fileBuff + "/");
				} else {
					filePaths->push_back(rootPath + fileBuff);
				}
			}
		}
		closedir(dir);
	}
}

// Divides files into blocks.
// Compresses each block into a single file.
// Combines all compressed blocks into a single file.
// Removes temporary blocks and header files.
// Parameters: filePaths (std::vector<std::string> *) holder for all file paths.
// 			   name (std::string) user given name for storage file.
// 			   verbose (bool) user option for verbose output.
void compression(std::vector<std::string> *filePaths, std::string name, bool verbose) {
	std::random_shuffle(filePaths->begin(), filePaths->end());
	unsigned long long int filePathSize = filePaths->size();
	unsigned long long int blockSize = (filePathSize / (omp_get_max_threads() * 10)) + 1;
	std::vector<std::string> *tarNames = new std::vector<std::string>(filePaths->size());

	// Gzips the blocks of files into a single compressed file
	std::cout << "3.1 Gzipping Blocks" << std::endl;
	#pragma omp parallel for schedule(dynamic)
	for (int i = 0; i < omp_get_max_threads() * 10; ++i) {
		unsigned long long int start = blockSize * i;
		if (start < filePathSize) {
			// Store the name of each file for a block owned by each thread.
			// Each thread will use the file to tar and gzip compress their block.
			std::ofstream tmp;
			tmp.open(std::to_string(i) + "." + name + ".ptgz.tmp", std::ios_base::app);
			std::string gzCommand = "GZIP=-1 tar -cz -T " + std::to_string(i) + "." + name + ".ptgz.tmp -f " + std::to_string(i) + "." + name + ".tar.gz";
			for (unsigned long long int j = start; j < std::min(start + blockSize, filePathSize); ++j) {
				tmp << filePaths->at(j) + "\n";
			}

			if (verbose) {
				std::cout << gzCommand + "\n";
			}

			tmp.close();
			system(gzCommand.c_str());
			tarNames->at(i) = std::to_string(i) + "." + name + ".tar.gz";
		}
	}

	// Combines gzipped blocks together into a single tarball.
	// Write tarball names into an idx file for extraction.
	std::ofstream idx, tmp;
	idx.open(name + ".ptgz.idx", std::ios_base::app);
	std::cout << "3.2 Combining Blocks Together" << std::endl;
	std::string tarCommand = "tar -c -T " + name + ".ptgz.idx -f " + name + ".ptgz.tar";
	for (unsigned long long int i = 0; i < tarNames->size(); ++i) {
		idx << tarNames->at(i) + "\n";
	}
	idx << name + ".ptgz.idx" + "\n";
	idx.close();

	if (verbose) {
		std::cout << tarCommand + "\n";
	}

	system(tarCommand.c_str());

	// Removes all temporary blocks and idx file.
	std::cout << "3.3 Removing Temporary Blocks" << std::endl;
	#pragma omp parallel for schedule(static)
	for (unsigned long long int i = 0; i < tarNames->size(); ++i) {
		std::string rmCommand = "rm " + tarNames->at(i);
		if (verbose) {
			std::cout << rmCommand + "\n";
		}
		system(rmCommand.c_str());

		rmCommand = "rm " + std::to_string(i) + "." + name + ".ptgz.tmp";
		if (verbose) {
			std::cout << rmCommand + "\n";
		}
		system(rmCommand.c_str());
	}

	std::string rmCommand;
	if (verbose) {
		std::cout << "rm " + name + ".ptgz.idx\n";
	}
	rmCommand = "rm " + name + ".ptgz.idx";
	system(rmCommand.c_str());

	tarNames->clear();
	delete(tarNames);
}

void extraction() {

}

char cwd [PATH_MAX];

// Checks to see if the user asks for help.
// Gathers the user provided settings for ptgz.
// Finds the number of files that need to be stored.
// Gathers the file paths of all files to be stored.
// Either compresses the files or extracts the ptgz.tar archive.
int main(int argc, char *argv[]) {
	Settings *instance = new Settings;
	int *numFiles = new int(0);
	std::vector<std::string> *filePaths = new std::vector<std::string>();

	helpCheck(argc, argv);
	getSettings(argc, argv, instance);
	getcwd(cwd, PATH_MAX);

	if ((*instance).compress) {
		std::cout << "1.  Searching File Tree" << std::endl;
		findAll(numFiles, cwd);
		std::cout << "2.  Gathering Files" << std::endl;
		getPaths(filePaths, cwd, "");
		std::cout << "3.  Starting File Compression" << std::endl;
		compression(filePaths, (*instance).name, (*instance).verbose);
	} else {
		extraction();
	}

	std::cout << "4.  Cleaning Up" << std::endl;
	delete(instance);
	delete(numFiles);
	filePaths->clear();
	delete(filePaths);
	return 0;
}
<commit_msg>Moved from system remove to c style remove<commit_after>#include <stdlib.h>
#include <algorithm>
#include <iostream>
#include <vector>
#include <unistd.h>
#include <dirent.h>
#include <fstream>
#include <queue>
#include "omp.h"

// Contains the various options the user can pass ptgz.
struct Settings {
	Settings(): extract(),
				compress(),
   				verbose(),
				output() {}
	bool extract;
	bool compress;
	bool verbose;
	bool output;
	std::string name;
};

// Checks if the user asks for help.
// Provides usage information to the user.
// Parameters: argc (int) number of cli arguments.
// 			   argv (char *[]) user provided arguments.
void helpCheck(int argc, char *argv[]) {
	if (argc == 1) {
		std::cout << "ERROR: ptgz was passed no parameters. \"ptgz -h\" for help." << std::endl;
		exit(0);
	}

	if (argv[1] == std::string("-h") || argv[1] == std::string("--help")) {
		std::cout << "\nptgz - Parallel Tar GZ by Ryan Chui (2017)\n" << std::endl;
		exit(0);
	}
}

// Gets the parameters passed by the user and stores them.
// Parameters: argc (int) number of cli arguments.
// 			   argv (char *[]) user provided arguments.
// 			   instance (Settings *) user argument storage.
void getSettings(int argc, char *argv[], Settings *instance) {
	// Get all passed arguments
	std::queue<std::string> settings;
	for (int i = 1; i < argc; ++i) {
		settings.push(argv[i]);
	}

	// Continue to check until there are no more passed arguments.
	while (!settings.empty()) {
		std::string arg = settings.front();

		if (arg == "-x") {
			if ((*instance).compress) {
				std::cout << "ERROR: ptgz cannot both compress and extract. \"ptgz -h\" for help." << std::endl;
				exit(0);
			}
			(*instance).extract = true;
		} else if (arg == "-c") {
			if ((*instance).extract) {
				std::cout << "ERROR: ptgz cannot both compress and extract. \"ptgz -h\" for help." << std::endl;
				exit(0);
			}
			(*instance).compress = true;
		} else if (arg == "-v"){
			(*instance).verbose = true;
		} else if (arg == "-o") {
			(*instance).output = true;
		} else {
			if (settings.size() > 1) {
				std::cout << "ERROR: ptgz was called incorrectly. \"ptgz -h\" for help." << std::endl;
				exit(0);
			}
			(*instance).output = true;
			(*instance).name = arg;
		}

		settings.pop();
	}

	if (!(*instance).output) {
		std::cout << "ERROR: No output file name given. \"ptgz -h\" for help." << std::endl;
		exit(0);
	}
}

// Finds the number of files in the space to store.
// Parameters: numFiles (int *) number of files.
// 			   cwd (const char *) current working directory.
void findAll(int *numFiles, const char *cwd) {
	DIR *dir;
	struct dirent *ent;

	// Check if cwd is a directory
	if ((dir = opendir(cwd)) != NULL) {
		// Get all file paths within directory.
		while ((ent = readdir (dir)) != NULL) {
			std::string fileBuff = std::string(ent -> d_name);
			if (fileBuff != "." && fileBuff != "..") {
				DIR *dir2;
				std::string filePath = std::string(cwd) + "/" + fileBuff;
				// Check if file path is a directory.
				if ((dir2 = opendir(filePath.c_str())) != NULL) {
					closedir(dir2);
					findAll(numFiles, filePath.c_str());
				} else {
					*numFiles += 1;
				}
			}
		}
		closedir(dir);
	}
}

// Gets the paths for all files in the space to store.
// Parameters: filePaths (std::vector<std::string> *) holder for all file paths.
// 			   cwd (const char *) current working directory.
// 			   rootPath (std::string) path from the root of the directory to be stored.
void getPaths(std::vector<std::string> *filePaths, const char *cwd, std::string rootPath) {
	DIR *dir;
	struct dirent *ent;

	// Check if cwd is a directory
	if ((dir = opendir(cwd)) != NULL) {
		// Get all file paths within directory.
		while ((ent = readdir (dir)) != NULL) {
			std::string fileBuff = std::string(ent -> d_name);
			if (fileBuff != "." && fileBuff != "..") {
				DIR *dir2;
				std::string filePath = std::string(cwd) + "/" + fileBuff;
				// Check if file path is a directory.
				if ((dir2 = opendir(filePath.c_str())) != NULL) {
					closedir(dir2);
					getPaths(filePaths, filePath.c_str(), rootPath + fileBuff + "/");
				} else {
					filePaths->push_back(rootPath + fileBuff);
				}
			}
		}
		closedir(dir);
	}
}

// Divides files into blocks.
// Compresses each block into a single file.
// Combines all compressed blocks into a single file.
// Removes temporary blocks and header files.
// Parameters: filePaths (std::vector<std::string> *) holder for all file paths.
// 			   name (std::string) user given name for storage file.
// 			   verbose (bool) user option for verbose output.
void compression(std::vector<std::string> *filePaths, std::string name, bool verbose) {
	std::random_shuffle(filePaths->begin(), filePaths->end());
	unsigned long long int filePathSize = filePaths->size();
	unsigned long long int blockSize = (filePathSize / (omp_get_max_threads() * 10)) + 1;
	std::vector<std::string> *tarNames = new std::vector<std::string>(filePaths->size());

	// Gzips the blocks of files into a single compressed file
	std::cout << "3.1 Gzipping Blocks" << std::endl;
	#pragma omp parallel for schedule(dynamic)
	for (int i = 0; i < omp_get_max_threads() * 10; ++i) {
		unsigned long long int start = blockSize * i;
		if (start < filePathSize) {
			// Store the name of each file for a block owned by each thread.
			// Each thread will use the file to tar and gzip compress their block.
			std::ofstream tmp;
			tmp.open(std::to_string(i) + "." + name + ".ptgz.tmp", std::ios_base::app);
			std::string gzCommand = "GZIP=-1 tar -cz -T " + std::to_string(i) + "." + name + ".ptgz.tmp -f " + std::to_string(i) + "." + name + ".tar.gz";
			for (unsigned long long int j = start; j < std::min(start + blockSize, filePathSize); ++j) {
				tmp << filePaths->at(j) + "\n";
			}

			if (verbose) {
				std::cout << gzCommand + "\n";
			}

			tmp.close();
			system(gzCommand.c_str());
			tarNames->at(i) = std::to_string(i) + "." + name + ".tar.gz";
		}
	}

	// Combines gzipped blocks together into a single tarball.
	// Write tarball names into an idx file for extraction.
	std::ofstream idx, tmp;
	idx.open(name + ".ptgz.idx", std::ios_base::app);
	std::cout << "3.2 Combining Blocks Together" << std::endl;
	std::string tarCommand = "tar -c -T " + name + ".ptgz.idx -f " + name + ".ptgz.tar";
	for (unsigned long long int i = 0; i < tarNames->size(); ++i) {
		idx << tarNames->at(i) + "\n";
	}
	idx << name + ".ptgz.idx" + "\n";
	idx.close();

	if (verbose) {
		std::cout << tarCommand + "\n";
	}

	system(tarCommand.c_str());

	// Removes all temporary blocks and idx file.
	std::cout << "3.3 Removing Temporary Blocks" << std::endl;
	#pragma omp parallel for schedule(static)
	for (unsigned long long int i = 0; i < tarNames->size(); ++i) {
		std::string rmCommand = tarNames->at(i);
		if (verbose) {
			std::cout << "remove("rmCommand + ")\n";
		}
		remove(rmCommand.c_str());

		rmCommand = std::to_string(i) + "." + name + ".ptgz.tmp";
		if (verbose) {
			std::cout << "remove("rmCommand + ")\n";
		}
		remove(rmCommand.c_str());
	}

	std::string rmCommand;
	if (verbose) {
		std::cout << "remove(" + name + ".ptgz.idx)\n";
	}
	rmCommand = name + ".ptgz.idx";
	remove(rmCommand.c_str());

	tarNames->clear();
	delete(tarNames);
}

void extraction() {

}

char cwd [PATH_MAX];

// Checks to see if the user asks for help.
// Gathers the user provided settings for ptgz.
// Finds the number of files that need to be stored.
// Gathers the file paths of all files to be stored.
// Either compresses the files or extracts the ptgz.tar archive.
int main(int argc, char *argv[]) {
	Settings *instance = new Settings;
	int *numFiles = new int(0);
	std::vector<std::string> *filePaths = new std::vector<std::string>();

	helpCheck(argc, argv);
	getSettings(argc, argv, instance);
	getcwd(cwd, PATH_MAX);

	if ((*instance).compress) {
		std::cout << "1.  Searching File Tree" << std::endl;
		findAll(numFiles, cwd);
		std::cout << "2.  Gathering Files" << std::endl;
		getPaths(filePaths, cwd, "");
		std::cout << "3.  Starting File Compression" << std::endl;
		compression(filePaths, (*instance).name, (*instance).verbose);
	} else {
		extraction();
	}

	std::cout << "4.  Cleaning Up" << std::endl;
	delete(instance);
	delete(numFiles);
	filePaths->clear();
	delete(filePaths);
	return 0;
}
<|endoftext|>


<commit_before>#include "CustomRankDocumentIterator.h"
#include "FilterDocumentIterator.h"
#include <index-manager/IndexManager.h>
#include <algorithm> // sort

namespace
{

using namespace sf1r;

FilterDocumentIterator* createFilterDocIterator(CustomRankDocId::DocIdList& docIdList)
{
    boost::shared_ptr<IndexManager::FilterBitmapT> filterBitmap(
        new IndexManager::FilterBitmapT);

    std::sort(docIdList.begin(), docIdList.end());
    for (CustomRankDocId::DocIdList::const_iterator it = docIdList.begin();
            it != docIdList.end(); ++it)
    {
        filterBitmap->set(*it);
    }

    IndexManager::FilterTermDocFreqsT* filterTermDocFreqs =
        new IndexManager::FilterTermDocFreqsT(filterBitmap);

    return new FilterDocumentIterator(filterTermDocFreqs);
}

} // namespace

namespace sf1r
{

CustomRankDocumentIterator::CustomRankDocumentIterator(CustomRankDocId& customRankDocId)
    : defaultScoreDocIterator_(NULL)
{
    if (! customRankDocId.topIds.empty())
    {
        FilterDocumentIterator* includeDocIter =
            createFilterDocIterator(customRankDocId.topIds);

        ORDocumentIterator::add(includeDocIter);
    }

    if (! customRankDocId.excludeIds.empty())
    {
        FilterDocumentIterator* excludeDocIter =
            createFilterDocIterator(customRankDocId.excludeIds);

        excludeDocIter->setNot(true);
        ORDocumentIterator::add(excludeDocIter);
    }
}

void CustomRankDocumentIterator::add(DocumentIterator* pDocIterator)
{
    ORDocumentIterator::add(pDocIterator);

    defaultScoreDocIterator_ = pDocIterator;
}

double CustomRankDocumentIterator::score(
    const std::vector<RankQueryProperty>& rankQueryProperties,
    const std::vector<boost::shared_ptr<PropertyRanker> >& propertyRankers
)
{
    double result = 0;

    if (defaultScoreDocIterator_)
    {
        result = defaultScoreDocIterator_->score(rankQueryProperties,
                 propertyRankers);
    }

    return result;
}

} // namespace sf1r
<commit_msg>in calling EWAHBoolArray::set(), we need to ensure the docids are sorted without duplication, so CustomRankDocumentIterator is revised to ensure it.<commit_after>#include "CustomRankDocumentIterator.h"
#include "FilterDocumentIterator.h"
#include <index-manager/IndexManager.h>
#include <algorithm> // sort

namespace
{

using namespace sf1r;

FilterDocumentIterator* createFilterDocIterator(CustomRankDocId::DocIdList& docIdList)
{
    boost::shared_ptr<IndexManager::FilterBitmapT> filterBitmap(
        new IndexManager::FilterBitmapT);

    std::sort(docIdList.begin(), docIdList.end());

    CustomRankDocId::DocIdList::const_iterator uniqueEnd =
        std::unique(docIdList.begin(), docIdList.end());

    for (CustomRankDocId::DocIdList::const_iterator it = docIdList.begin();
         it != uniqueEnd; ++it)
    {
        filterBitmap->set(*it);
    }

    IndexManager::FilterTermDocFreqsT* filterTermDocFreqs =
        new IndexManager::FilterTermDocFreqsT(filterBitmap);

    return new FilterDocumentIterator(filterTermDocFreqs);
}

} // namespace

namespace sf1r
{

CustomRankDocumentIterator::CustomRankDocumentIterator(CustomRankDocId& customRankDocId)
    : defaultScoreDocIterator_(NULL)
{
    if (! customRankDocId.topIds.empty())
    {
        FilterDocumentIterator* includeDocIter =
            createFilterDocIterator(customRankDocId.topIds);

        ORDocumentIterator::add(includeDocIter);
    }

    if (! customRankDocId.excludeIds.empty())
    {
        FilterDocumentIterator* excludeDocIter =
            createFilterDocIterator(customRankDocId.excludeIds);

        excludeDocIter->setNot(true);
        ORDocumentIterator::add(excludeDocIter);
    }
}

void CustomRankDocumentIterator::add(DocumentIterator* pDocIterator)
{
    ORDocumentIterator::add(pDocIterator);

    defaultScoreDocIterator_ = pDocIterator;
}

double CustomRankDocumentIterator::score(
    const std::vector<RankQueryProperty>& rankQueryProperties,
    const std::vector<boost::shared_ptr<PropertyRanker> >& propertyRankers
)
{
    double result = 0;

    if (defaultScoreDocIterator_)
    {
        result = defaultScoreDocIterator_->score(rankQueryProperties,
                 propertyRankers);
    }

    return result;
}

} // namespace sf1r
<|endoftext|>


<commit_before>#pragma once

// ----------------------------------------------------------------------
// std::vector<std::string> split(const std::string& s, const std::string& delim, bool keep_empty = Split::KeepEmpty)
// Split string by delimiter into array of substrings
// std::vector<std::string> split(const std::string& s, const std::string& delim, bool keep_empty = Split::KeepEmpty)

// std::string strip(std::string source)
// Removes leading and trailing spaces from string
//
// ----------------------------------------------------------------------

#include <string>
#include <vector>
#include <cctype>
#include <algorithm>
#include <cstring>
#include <iterator>
#include <utility>

// ----------------------------------------------------------------------

namespace string
{

// ----------------------------------------------------------------------

    enum class Split { RemoveEmpty, KeepEmpty };

      // http://stackoverflow.com/questions/236129/split-a-string-in-c
    inline std::vector<std::string> split(std::string s, std::string delim, Split keep_empty = Split::KeepEmpty)
    {
        std::vector<std::string> result;
        if (! delim.empty()) {
            for (std::string::iterator substart = s.begin(), subend = substart; substart <= s.end(); substart = subend + static_cast<std::string::difference_type>(delim.size())) {
                subend = std::search(substart, s.end(), delim.begin(), delim.end());
                if (substart != subend || keep_empty == Split::KeepEmpty) {
                    result.push_back(std::string(substart, subend));
                }
            }
        }
        else {
            result.push_back(s);
        }
        return result;
    }

      // ----------------------------------------------------------------------

    namespace _internal {
        template <typename InputIterator, typename Source> inline std::pair<InputIterator, InputIterator> strip_begin_end(Source& source)
        {
            auto predicate = [](auto c) { return std::isspace(c); }; // have to use lambda, other compiler cannot infer Predicate type from isspace
            auto e = std::find_if_not(source.rbegin(), source.rend(), predicate);
            auto b = std::find_if_not(source.begin(), e.base(), predicate);
            return std::make_pair(b, e.base());
        }
    } // namespace _internal

    inline std::string& strip(std::string& source)
    {
        auto be = _internal::strip_begin_end<std::string::iterator>(source);
        source.erase(be.second, source.end()); // erase at the end first
        source.erase(source.begin(), be.first); // invalidates be.second!
        return source;
    }

    inline std::string strip(std::string&& source)
    {
        auto be = _internal::strip_begin_end<std::string::iterator>(source);
        source.erase(be.second, source.end()); // erase at the end first
        source.erase(source.begin(), be.first); // invalidates be.second!
        return source;
    }

    inline std::string strip(const std::string& source)
    {
        auto be = _internal::strip_begin_end<std::string::const_iterator>(source);
        return std::string(be.first, be.second);
    }

      // ----------------------------------------------------------------------

    inline std::string& lower(std::string& source)
    {
        std::transform(source.begin(), source.end(), source.begin(), ::tolower);
        return source;
    }

    inline std::string lower(const std::string& source)
    {
        std::string result;
        std::transform(source.begin(), source.end(), std::back_inserter(result), ::tolower);
        return result;
    }

    inline std::string& upper(std::string& source)
    {
        std::transform(source.begin(), source.end(), source.begin(), ::toupper);
        return source;
    }

    inline std::string upper(const std::string& source)
    {
        std::string result;
        std::transform(source.begin(), source.end(), std::back_inserter(result), ::toupper);
        return result;
    }

    inline std::string& capitalize(std::string& source)
    {
        if (!source.empty()) {
            std::transform(source.begin(), source.begin() + 1, source.begin(), ::toupper);
            std::transform(source.begin() + 1, source.end(), source.begin() + 1, ::tolower);
        }
        return source;
    }

    inline std::string capitalize(const std::string& source)
    {
        std::string result;
        if (!source.empty()) {
            std::transform(source.begin(), source.begin() + 1, std::back_inserter(result), ::toupper);
            std::transform(source.begin() + 1, source.end(), std::back_inserter(result), ::tolower);
        }
        return result;
    }

// ----------------------------------------------------------------------

} // namespace string

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>string::replace added<commit_after>#pragma once

// ----------------------------------------------------------------------
// std::vector<std::string> split(const std::string& s, const std::string& delim, bool keep_empty = Split::KeepEmpty)
// Split string by delimiter into array of substrings
// std::vector<std::string> split(const std::string& s, const std::string& delim, bool keep_empty = Split::KeepEmpty)

// std::string strip(std::string source)
// Removes leading and trailing spaces from string
//
// ----------------------------------------------------------------------

#include <string>
#include <vector>
#include <cctype>
#include <algorithm>
#include <cstring>
#include <iterator>
#include <utility>

// ----------------------------------------------------------------------

namespace string
{

// ----------------------------------------------------------------------

    enum class Split { RemoveEmpty, KeepEmpty };

      // http://stackoverflow.com/questions/236129/split-a-string-in-c
    inline std::vector<std::string> split(std::string s, std::string delim, Split keep_empty = Split::KeepEmpty)
    {
        std::vector<std::string> result;
        if (! delim.empty()) {
            for (std::string::iterator substart = s.begin(), subend = substart; substart <= s.end(); substart = subend + static_cast<std::string::difference_type>(delim.size())) {
                subend = std::search(substart, s.end(), delim.begin(), delim.end());
                if (substart != subend || keep_empty == Split::KeepEmpty) {
                    result.push_back(std::string(substart, subend));
                }
            }
        }
        else {
            result.push_back(s);
        }
        return result;
    }

      // ----------------------------------------------------------------------

    namespace _internal {
        template <typename InputIterator, typename Source> inline std::pair<InputIterator, InputIterator> strip_begin_end(Source& source)
        {
            auto predicate = [](auto c) { return std::isspace(c); }; // have to use lambda, other compiler cannot infer Predicate type from isspace
            auto e = std::find_if_not(source.rbegin(), source.rend(), predicate);
            auto b = std::find_if_not(source.begin(), e.base(), predicate);
            return std::make_pair(b, e.base());
        }
    } // namespace _internal

    inline std::string& strip(std::string& source)
    {
        auto be = _internal::strip_begin_end<std::string::iterator>(source);
        source.erase(be.second, source.end()); // erase at the end first
        source.erase(source.begin(), be.first); // invalidates be.second!
        return source;
    }

    inline std::string strip(std::string&& source)
    {
        auto be = _internal::strip_begin_end<std::string::iterator>(source);
        source.erase(be.second, source.end()); // erase at the end first
        source.erase(source.begin(), be.first); // invalidates be.second!
        return source;
    }

    inline std::string strip(const std::string& source)
    {
        auto be = _internal::strip_begin_end<std::string::const_iterator>(source);
        return std::string(be.first, be.second);
    }

      // ----------------------------------------------------------------------

    inline std::string replace(std::string source, std::string look_for, std::string replace_with)
    {
        std::string result;
        std::string::size_type start = 0;
        while (true) {
            const auto pos = source.find(look_for, start);
            if (pos != std::string::npos) {
                result.append(source.begin() + static_cast<std::string::difference_type>(start), source.begin() + static_cast<std::string::difference_type>(pos));
                result.append(replace_with);
                start = pos + look_for.size();
            }
            else {
                result.append(source.begin() + static_cast<std::string::difference_type>(start), source.end());
                break;
            }
        }
        return result;
    }

      // ----------------------------------------------------------------------

    inline std::string& lower(std::string& source)
    {
        std::transform(source.begin(), source.end(), source.begin(), ::tolower);
        return source;
    }

    inline std::string lower(const std::string& source)
    {
        std::string result;
        std::transform(source.begin(), source.end(), std::back_inserter(result), ::tolower);
        return result;
    }

    inline std::string& upper(std::string& source)
    {
        std::transform(source.begin(), source.end(), source.begin(), ::toupper);
        return source;
    }

    inline std::string upper(const std::string& source)
    {
        std::string result;
        std::transform(source.begin(), source.end(), std::back_inserter(result), ::toupper);
        return result;
    }

    inline std::string& capitalize(std::string& source)
    {
        if (!source.empty()) {
            std::transform(source.begin(), source.begin() + 1, source.begin(), ::toupper);
            std::transform(source.begin() + 1, source.end(), source.begin() + 1, ::tolower);
        }
        return source;
    }

    inline std::string capitalize(const std::string& source)
    {
        std::string result;
        if (!source.empty()) {
            std::transform(source.begin(), source.begin() + 1, std::back_inserter(result), ::toupper);
            std::transform(source.begin() + 1, source.end(), std::back_inserter(result), ::tolower);
        }
        return result;
    }

// ----------------------------------------------------------------------

} // namespace string

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>


<commit_before>// Copyright 2011-2012 Jean Charles Régin
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "base/hash.h"
#include "base/map-util.h"
#include "base/stl_util.h"
#include "base/random.h"
#include "constraint_solver/constraint_solveri.h"
#include "constraint_solver/constraint_solver.h"

DEFINE_int32(arity, 3, "Arity of tuples");
DEFINE_int32(upper, 10, "Upper bound of variables, lower is always 0");
DEFINE_int32(tuples, 1000, "Number of tuples");
DEFINE_int32(bucket, 64, "Size of buckets");

namespace operations_research {
// Enums.
#define TABLECT_RESTART 0
#define TABLECT_CONTINUE 1
#define TABLECT_INVERSE 2
#define TABLECT_ORIGINAL 3
// External build functions.
extern Constraint* BuildTableCt(Solver* const solver,
                                const IntTupleSet& tuples,
                                const std::vector<IntVar*>& vars,
                                int size_bucket);

void RandomFillTable(int num_tuples,
                     int64 lower,
                     int64 upper,
                     IntTupleSet* const tuples) {
  ACMRandom rgen(0); // defines a random generator
  std::vector<int64> vals(tuples->Arity());

  for(int t = 0; t < num_tuples; ++t){
    for(int i = 0; i < tuples->Arity(); i++){
      int64 rnumber = rgen.Next64(); // generates a 64bits number
      vals[i] = (rnumber % (upper - lower + 1)) + lower;
    }
    tuples->Insert(vals);
  }
}

void test_table_in_bk(int n,
                      unsigned num_tuples,
                      unsigned upper,
                      unsigned size_bucket,
                      bool use_bucket_table) {
  // var array creation
  Solver solver("SolverInBk");
  std::vector<IntVar*> vars;
  solver.MakeIntVarArray(n, 0, upper, &vars);

  IntTupleSet table(n);

  if (use_bucket_table) {
    LOG(INFO) <<  "Creation of a Bucketed tuple Table ("
              << size_bucket << ") with :";
  } else {
    LOG(INFO) << "Creation of a Allowed Assignment Table with :";
  }
  LOG(INFO) << " - " << n <<  " variables";
  LOG(INFO) << " - " << upper + 1 <<  " values per domain";
  LOG(INFO) << " - " << num_tuples << " tuples";

  RandomFillTable(num_tuples, 0, upper, &table);

  LOG(INFO) << "Table is created";

  Constraint* const ct = use_bucket_table ?
      BuildTableCt(&solver, table, vars, size_bucket) :
      solver.MakeAllowedAssignments(vars, table);
  solver.AddConstraint(ct);

  LOG(INFO) << "The constraint has been added";

  DecisionBuilder* const db = solver.MakePhase(vars,
                                               Solver::CHOOSE_FIRST_UNBOUND,
                                               Solver::ASSIGN_MIN_VALUE);

  solver.NewSearch(db);

  CycleTimer t;
  t.Start();
  int counter = 0;
  while(solver.NextSolution()) {
    counter++;
  }
  t.Stop();
  solver.EndSearch();
  LOG(INFO) << "test time : " << t.GetInUsec() << " micro seconds";
  CHECK_EQ(counter, table.NumTuples());
}
}  // namespace operations_research

int main(int argc, char** argv) {
  google::ParseCommandLineFlags(&argc, &argv, true);

  operations_research::test_table_in_bk(FLAGS_arity,
                                        FLAGS_tuples,
                                        FLAGS_upper,
                                        FLAGS_bucket,
                                        false);

  operations_research::test_table_in_bk(FLAGS_arity,
                                        FLAGS_tuples,
                                        FLAGS_upper,
                                        FLAGS_bucket,
                                        true);
  return 0;
}

<commit_msg>improve bucket_table_test.cc<commit_after>// Copyright 2011-2012 Jean Charles Régin
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "base/hash.h"
#include "base/map-util.h"
#include "base/stl_util.h"
#include "base/random.h"
#include "constraint_solver/constraint_solveri.h"
#include "constraint_solver/constraint_solver.h"

DEFINE_int32(arity, 3, "Arity of tuples");
DEFINE_int32(upper, 10, "Upper bound of variables, lower is always 0");
DEFINE_int32(tuples, 1000, "Number of tuples");
DEFINE_int32(bucket, 64, "Size of buckets");

namespace operations_research {
extern Constraint* BuildTableCt(Solver* const solver,
                                const IntTupleSet& tuples,
                                const std::vector<IntVar*>& vars,
                                int size_bucket);

void RandomFillTable(int num_tuples,
                     int64 lower,
                     int64 upper,
                     IntTupleSet* const tuples) {
  ACMRandom rgen(0); // defines a random generator
  std::vector<int64> vals(tuples->Arity());

  for(int t = 0; t < num_tuples; ++t){
    for(int i = 0; i < tuples->Arity(); i++){
      int64 rnumber = rgen.Next64(); // generates a 64bits number
      vals[i] = (rnumber % (upper - lower + 1)) + lower;
    }
    tuples->Insert(vals);
  }
}

void TestTable(int arity, int num_tuples, int upper, int size_bucket) {
  if (size_bucket > 0) {
    LOG(INFO) <<  "Creation of a Bucketed tuple Table ("
              << size_bucket << ") with :";
  } else {
    LOG(INFO) << "Creation of a Allowed Assignment Table with :";
  }
  LOG(INFO) << " - " << arity <<  " variables";
  LOG(INFO) << " - " << upper + 1 <<  " values per domain";
  LOG(INFO) << " - " << num_tuples << " tuples";

  Solver solver("SolverInBk");
  std::vector<IntVar*> vars;
  solver.MakeIntVarArray(arity, 0, upper, &vars);

  IntTupleSet table(arity);
  RandomFillTable(num_tuples, 0, upper, &table);
  LOG(INFO) << "Table is created";

  Constraint* const ct = size_bucket > 0 ?
      BuildTableCt(&solver, table, vars, size_bucket) :
      solver.MakeAllowedAssignments(vars, table);
  solver.AddConstraint(ct);

  DecisionBuilder* const db = solver.MakePhase(vars,
                                               Solver::CHOOSE_FIRST_UNBOUND,
                                               Solver::ASSIGN_MIN_VALUE);

  LOG(INFO) << "Start search";
  CycleTimer t;
  t.Start();
  solver.NewSearch(db);
  int counter = 0;
  while(solver.NextSolution()) {
    counter++;
  }
  solver.EndSearch();
  t.Stop();

  LOG(INFO) << "test time : " << t.GetInUsec() << " micro seconds";
  CHECK_EQ(counter, table.NumTuples());
}
}  // namespace operations_research

int main(int argc, char** argv) {
  google::ParseCommandLineFlags(&argc, &argv, true);
  operations_research::TestTable(FLAGS_arity, FLAGS_tuples, FLAGS_upper, 0);
  operations_research::TestTable(FLAGS_arity,
                                 FLAGS_tuples,
                                 FLAGS_upper,
                                 FLAGS_bucket);
  return 0;
}

<|endoftext|>


<commit_before>//   Copyright 2016-2017 Jean-Francois Poilpret
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

/*
 * Hardware UART example.
 * This program demonstrates usage of FastArduino Hardware UART support (on target supporting it) and formatted
 * input streams.
 *
 * Wiring:
 * - on Arduino UNO, Arduino NANO and Arduino MEGA:
 *   - Use standard TX/RX
 * - on ATmega328P based boards:
 *   - Use standard TX/RX connected to an Serial-USB converter
 * - on ATtinyX4 based boards:
 *   - NOT SUPPORTED
 */

#include <fastarduino/time.h>
#include <fastarduino/uart.h>

// Define vectors we need in the example
REGISTER_UART_ISR(0)

// Buffers for UART
static const uint8_t INPUT_BUFFER_SIZE = 64;
static const uint8_t OUTPUT_BUFFER_SIZE = 64;
static char input_buffer[INPUT_BUFFER_SIZE];
static char output_buffer[OUTPUT_BUFFER_SIZE];

using namespace streams;

using INPUT = FormattedInput;
using OUTPUT = FormattedOutput;

template<typename T>
static void handle(OUTPUT& out, INPUT& in, const flash::FlashStorage* type)
{
	out << type << F(": ") << flush;
	T value{};
	in >> skipws >> value;
	out << value << endl;
}

template<typename T>
static void display_num(OUTPUT& out, T value)
{
	out << bin << value << endl;
	out << dec << value << endl;
	out << oct << value << endl;
	out << hex << value << endl;
}

template<typename T>
static void handle_num(OUTPUT& out, T value, const flash::FlashStorage* type)
{
	out << F("testing output of ") << type << F(" (") << dec << value << ')' << endl;
	display_num<T>(out, value);

	out << showbase;
	display_num<T>(out, value);
	out << noshowbase;

	out << uppercase;
	display_num<T>(out, value);
	out << nouppercase;

	out << uppercase << showbase;
	display_num<T>(out, value);
	out << nouppercase << noshowbase;

	out << showpos;
	display_num<T>(out, value);
	out << noshowpos;
}

static void handle_float(OUTPUT& out, double value)
{
	out << F("testing output of double (") << defaultfloat << setprecision(6) << value << ')' << endl;
	display_num<double>(out, value);

	out << showbase;
	display_num<double>(out, value);
	out << noshowbase;

	out << fixed << value << endl;
	out << scientific << value << endl;

	out << uppercase;
	out << fixed << value << endl;
	out << scientific << value << endl;
	out << nouppercase;

	out << showpos;
	out << fixed << value << endl;
	out << scientific << value << endl;
	out << noshowpos;

	// check precision too
	out << setprecision(12) << fixed << value << endl;
	out << setprecision(12) << scientific << value << endl;

	out << setprecision(3) << fixed << value << endl;
	out << setprecision(3) << scientific << value << endl;

	out << setprecision(0) << fixed << value << endl;
	out << setprecision(0) << scientific << value << endl;
}

static void handle_alignments(OUTPUT& out, uint8_t width, char filler, bool is_left = true)
{
	out << F("testing alignments") << endl;
	out << setfill(filler);
	if (is_left) out << left; else out << right;

	out << setw(width) << 'a' << endl;
	out << setw(width) << "abcdefghij" << endl;
	out << setw(width) << F("abcdefghij") << endl;
	out << setw(width) << 1234 << endl;
	out << setw(width) << 1234U << endl;
	out << setw(width) << 123456L << endl;
	out << setw(width) << 123456UL << endl;
	out << setw(width) << 123.456 << endl;
	out << setw(width) << true << endl;
	out << setw(width) << false << endl;
	out << boolalpha;
	out << setw(width) << true << endl;
	out << setw(width) << false << endl;
	out << noboolalpha;

	out << setfill(' ');
}

int main() __attribute__((OS_main));
int main()
{
	// Enable interrupts at startup time
	sei();

	// Start UART
	serial::hard::UART<board::USART::USART0> uart{input_buffer, output_buffer};
	uart.register_handler();
	uart.begin(115200);
	INPUT in = uart.fin();
	OUTPUT out = uart.fout();

	// Check all output manipulators
	handle_num<uint16_t>(out, 1234, F("uint16_t"));
	handle_num<int16_t>(out, 1234, F("int16_t"));
	handle_num<int16_t>(out, -1234, F("int16_t"));

	handle_num<uint32_t>(out, 123456, F("uint32_t"));
	handle_num<int32_t>(out, 123456, F("int32_t"));
	handle_num<int32_t>(out, -123456, F("int32_t"));

	// check floats
	handle_float(out, 123.456);
	handle_float(out, -123.456);
	handle_float(out, -12345678901234567890.12345);

	// check justification: setw(), setfill(), left, right...
	handle_alignments(out, 5, ' ', false);
	handle_alignments(out, 5, ' ', true);
	handle_alignments(out, 5, '~', false);
	handle_alignments(out, 5, '~', true);

	handle_alignments(out, 10, ' ', false);
	handle_alignments(out, 10, ' ', true);
	handle_alignments(out, 10, '~', false);
	handle_alignments(out, 10, '~', true);

	handle_alignments(out, 30, ' ', false);
	handle_alignments(out, 30, ' ', true);
	handle_alignments(out, 30, '~', false);
	handle_alignments(out, 30, '~', true);

	// Event Loop
	while (true)
	{
		handle<char>(out, in, F("char"));
		handle<uint16_t>(out, in, F("uint16_t"));
		handle<int16_t>(out, in, F("int16_t"));
		handle<uint32_t>(out, in, F("uint32_t"));
		handle<int32_t>(out, in, F("int32_t"));
		handle<bool>(out, in, F("bool"));

		// check formatted inputs: bool
		bool v1;
		out << F("bool as alpha: ") << flush;
		in >> boolalpha >> skipws >> v1;
		out << v1 << endl;
		out << F("bool as num: ") << flush;
		in >> noboolalpha >> skipws >> v1;
		out << v1 << endl;

		// check formatted inputs: numeric with base
		uint16_t v2;
		out << F("num as dec: ") << flush;
		in >> dec >> skipws >> v2;
		out << v2 << endl;
		out << F("num as hex: ") << flush;
		in >> hex >> skipws >> v2;
		out << v2 << endl;
		out << F("num as bin: ") << flush;
		in >> bin >> skipws >> v2;
		out << v2 << endl;
		out << F("num as oct: ") << flush;
		in >> oct >> skipws >> v2;
		out << v2 << endl;

		time::delay_ms(1000);
	}
}
<commit_msg>Streams refactoring: improve example to test char* extraction<commit_after>//   Copyright 2016-2017 Jean-Francois Poilpret
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

/*
 * Hardware UART example.
 * This program demonstrates usage of FastArduino Hardware UART support (on target supporting it) and formatted
 * input streams.
 *
 * Wiring:
 * - on Arduino UNO, Arduino NANO and Arduino MEGA:
 *   - Use standard TX/RX
 * - on ATmega328P based boards:
 *   - Use standard TX/RX connected to an Serial-USB converter
 * - on ATtinyX4 based boards:
 *   - NOT SUPPORTED
 */

#include <fastarduino/time.h>
#include <fastarduino/uart.h>

// Define vectors we need in the example
REGISTER_UART_ISR(0)

// Buffers for UART
static const uint8_t INPUT_BUFFER_SIZE = 64;
static const uint8_t OUTPUT_BUFFER_SIZE = 64;
static char input_buffer[INPUT_BUFFER_SIZE];
static char output_buffer[OUTPUT_BUFFER_SIZE];

using namespace streams;

using INPUT = FormattedInput;
using OUTPUT = FormattedOutput;

template<typename T>
static void handle(OUTPUT& out, INPUT& in, const flash::FlashStorage* type)
{
	out << type << F(": ") << flush;
	T value{};
	in >> skipws >> value;
	out << value << endl;
}

template<typename T>
static void display_num(OUTPUT& out, T value)
{
	out << bin << value << endl;
	out << dec << value << endl;
	out << oct << value << endl;
	out << hex << value << endl;
}

template<typename T>
static void handle_num(OUTPUT& out, T value, const flash::FlashStorage* type)
{
	out << F("testing output of ") << type << F(" (") << dec << value << ')' << endl;
	display_num<T>(out, value);

	out << showbase;
	display_num<T>(out, value);
	out << noshowbase;

	out << uppercase;
	display_num<T>(out, value);
	out << nouppercase;

	out << uppercase << showbase;
	display_num<T>(out, value);
	out << nouppercase << noshowbase;

	out << showpos;
	display_num<T>(out, value);
	out << noshowpos;
}

static void handle_float(OUTPUT& out, double value)
{
	out << F("testing output of double (") << defaultfloat << setprecision(6) << value << ')' << endl;
	display_num<double>(out, value);

	out << showbase;
	display_num<double>(out, value);
	out << noshowbase;

	out << fixed << value << endl;
	out << scientific << value << endl;

	out << uppercase;
	out << fixed << value << endl;
	out << scientific << value << endl;
	out << nouppercase;

	out << showpos;
	out << fixed << value << endl;
	out << scientific << value << endl;
	out << noshowpos;

	// check precision too
	out << setprecision(12) << fixed << value << endl;
	out << setprecision(12) << scientific << value << endl;

	out << setprecision(3) << fixed << value << endl;
	out << setprecision(3) << scientific << value << endl;

	out << setprecision(0) << fixed << value << endl;
	out << setprecision(0) << scientific << value << endl;
}

static void handle_alignments(OUTPUT& out, uint8_t width, char filler, bool is_left = true)
{
	out << F("testing alignments") << endl;
	out << setfill(filler);
	if (is_left) out << left; else out << right;

	out << setw(width) << 'a' << endl;
	out << setw(width) << "abcdefghij" << endl;
	out << setw(width) << F("abcdefghij") << endl;
	out << setw(width) << 1234 << endl;
	out << setw(width) << 1234U << endl;
	out << setw(width) << 123456L << endl;
	out << setw(width) << 123456UL << endl;
	out << setw(width) << 123.456 << endl;
	out << setw(width) << true << endl;
	out << setw(width) << false << endl;
	out << boolalpha;
	out << setw(width) << true << endl;
	out << setw(width) << false << endl;
	out << noboolalpha;

	out << setfill(' ');
}

template<uint8_t WIDTH>
void input_string(OUTPUT& out, INPUT& in)
{
	out << F("string of length ") << dec << WIDTH << F(": ") << flush;
	char buffer[WIDTH + 1];
	in >> noskipws >> setw(WIDTH + 1) >> buffer;
	out << buffer << endl;
	in >> skipws;
}

int main() __attribute__((OS_main));
int main()
{
	// Enable interrupts at startup time
	sei();

	// Start UART
	serial::hard::UART<board::USART::USART0> uart{input_buffer, output_buffer};
	uart.register_handler();
	uart.begin(115200);
	INPUT in = uart.fin();
	OUTPUT out = uart.fout();

	// // Check all output manipulators
	// handle_num<uint16_t>(out, 1234, F("uint16_t"));
	// handle_num<int16_t>(out, 1234, F("int16_t"));
	// handle_num<int16_t>(out, -1234, F("int16_t"));

	// handle_num<uint32_t>(out, 123456, F("uint32_t"));
	// handle_num<int32_t>(out, 123456, F("int32_t"));
	// handle_num<int32_t>(out, -123456, F("int32_t"));

	// // check floats
	// handle_float(out, 123.456);
	// handle_float(out, -123.456);
	// handle_float(out, -12345678901234567890.12345);

	// // check justification: setw(), setfill(), left, right...
	// handle_alignments(out, 5, ' ', false);
	// handle_alignments(out, 5, ' ', true);
	// handle_alignments(out, 5, '~', false);
	// handle_alignments(out, 5, '~', true);

	// handle_alignments(out, 10, ' ', false);
	// handle_alignments(out, 10, ' ', true);
	// handle_alignments(out, 10, '~', false);
	// handle_alignments(out, 10, '~', true);

	// handle_alignments(out, 30, ' ', false);
	// handle_alignments(out, 30, ' ', true);
	// handle_alignments(out, 30, '~', false);
	// handle_alignments(out, 30, '~', true);

	// Event Loop
	while (true)
	{
		// handle<char>(out, in, F("char"));
		// handle<uint16_t>(out, in, F("uint16_t"));
		// handle<int16_t>(out, in, F("int16_t"));
		// handle<uint32_t>(out, in, F("uint32_t"));
		// handle<int32_t>(out, in, F("int32_t"));
		// handle<bool>(out, in, F("bool"));

		// check formatted inputs: bool
		// bool v1;
		// out << F("bool as alpha: ") << flush;
		// in >> boolalpha >> skipws >> v1;
		// out << v1 << endl;
		// out << F("bool as num: ") << flush;
		// in >> noboolalpha >> skipws >> v1;
		// out << v1 << endl;

		// check formatted inputs: numeric with base
		// uint16_t v2;
		// out << F("num as dec: ") << flush;
		// in >> dec >> skipws >> v2;
		// out << v2 << endl;
		// out << F("num as hex: ") << flush;
		// in >> hex >> skipws >> v2;
		// out << v2 << endl;
		// out << F("num as bin: ") << flush;
		// in >> bin >> skipws >> v2;
		// out << v2 << endl;
		// out << F("num as oct: ") << flush;
		// in >> oct >> skipws >> v2;
		// out << v2 << endl;

		// check input to char*
		// input_string<10>(out, in);
		// input_string<50>(out, in);
		// input_string<200>(out, in);

		time::delay_ms(1000);
	}

	// check unitbuf: with unitbuf the program should not exit until every character has been output
	// out << unitbuf << F("abcdefghijklmnopqrstuvwxyz\n");
	// out << F("ABCDEFGHIJKLMNOPQRSTUVWXYZ\n");
	// out << F("1234567890\n");

	// check nounitbuf: with nounitbuf the program will exit before all characters have been output
	// out << nounitbuf << F("abcdefghijklmnopqrstuvwxyz\n");
	// out << F("ABCDEFGHIJKLMNOPQRSTUVWXYZ\n");
	// out << F("1234567890\n");
}
<|endoftext|>


<commit_before>// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/fluid/framework/io/shell.h"

namespace paddle {
namespace framework {

std::shared_ptr<FILE> shell_fopen(const std::string& path,
                                  const std::string& mode) {
#ifndef _WIN32
  if (shell_verbose()) {
    LOG(INFO) << "Opening file[" << path << "] with mode[" << mode << "]";
  }
  FILE* fp;
  if (!(fp = fopen(path.c_str(), mode.c_str()))) {
    LOG(FATAL) << "fopen fail, path[" << path << "], mode[" << mode << "]";
  }
  return {fp, [path](FILE* fp) {
            if (shell_verbose()) {
              LOG(INFO) << "Closing file[" << path << "]";
            }
            if (0 != fclose(fp)) {
              LOG(FATAL) << "fclose fail, path[" << path << "]";
            }
          }};
#else
  return nullptr;
#endif
}

// Close all open file descriptors
// The implementation is async signal safe
// Mostly copy from CPython code
static int close_open_fds_internal() {
#ifndef _WIN32
  struct linux_dirent {
    long d_ino = 0;  // NOLINT
    off_t d_off;
    unsigned short d_reclen = 0;  // NOLINT
    char d_name[256];
  };

  int dir_fd = -1;
  if ((dir_fd = open("/proc/self/fd", O_RDONLY)) < 0) {
    LOG(FATAL) << "proc/self/fd open fail";
    return -1;
  }
  char buffer[sizeof(linux_dirent)];

  for (;;) {
    int bytes = 0;
    if ((bytes = syscall(SYS_getdents, dir_fd,
                         reinterpret_cast<linux_dirent*>(buffer),
                         sizeof(buffer))) < 0) {
      LOG(FATAL) << "syscall fail";
      return -1;
    }

    if (bytes == 0) {
      break;
    }

    linux_dirent* entry = NULL;

    for (int offset = 0; offset < bytes; offset += entry->d_reclen) {
      entry = reinterpret_cast<linux_dirent*>(buffer + offset);
      int fd = 0;
      const char* s = entry->d_name;

      while (*s >= '0' && *s <= '9') {
        fd = fd * 10 + (*s - '0');
        s++;
      }

      if (s != entry->d_name && fd != dir_fd && fd >= 3) {
        close(fd);
      }
    }
  }

  close(dir_fd);
#endif
  return 0;
}

static int shell_popen_fork_internal(const char* real_cmd, bool do_read,
                                     int parent_end, int child_end) {
#ifndef _WIN32
  int child_pid = -1;
  // Too frequent calls to fork() makes openmpi very slow. Use vfork() instead.
  // But vfork() is very dangerous. Be careful.
  if ((child_pid = vfork()) < 0) {
    return -1;
  }

  // The following code is async signal safe (No memory allocation, no access to
  // global data, etc.)
  if (child_pid != 0) {
    return child_pid;
  }

  int child_std_end = do_read ? 1 : 0;
  close(parent_end);

  if (child_end != child_std_end) {
    if (dup2(child_end, child_std_end) != child_std_end) {
      return -1;
    }
    close(child_end);
  }

  close_open_fds_internal();
  if (execl("/bin/sh", "sh", "-c", real_cmd, NULL) < 0) {
    return -1;
  }
  exit(127);
#endif
  return 0;
}

std::shared_ptr<FILE> shell_popen(const std::string& cmd,
                                  const std::string& mode, int* err_no) {
#ifndef _WIN32
  bool do_read = mode == "r";
  bool do_write = mode == "w";
  if (!(do_read || do_write)) {
    *err_no = -1;
    return NULL;
  }

  if (shell_verbose()) {
    LOG(INFO) << "Opening pipe[" << cmd << "] with mode[" << mode << "]";
  }

  std::string real_cmd = "set -o pipefail; " + cmd;

  int pipe_fds[2];
  if (pipe(pipe_fds) != 0) {
    *err_no = -1;
    return NULL;
  }
  int parent_end = 0;
  int child_end = 0;

  if (do_read) {
    parent_end = pipe_fds[0];
    child_end = pipe_fds[1];
  } else if (do_write) {
    parent_end = pipe_fds[1];
    child_end = pipe_fds[0];
  }

  int child_pid = shell_popen_fork_internal(real_cmd.c_str(), do_read,
                                            parent_end, child_end);
  close(child_end);
  fcntl(parent_end, F_SETFD, FD_CLOEXEC);
  FILE* fp;
  if ((fp = fdopen(parent_end, mode.c_str())) == NULL) {
    *err_no = -1;
    return NULL;
  }
  return {fp, [child_pid, cmd, err_no](FILE* fp) {
            if (shell_verbose()) {
              LOG(INFO) << "Closing pipe[" << cmd << "]";
            }

            if (fclose(fp) != 0) {
              *err_no = -1;
            }
            int wstatus = -1;
            waitpid(child_pid, &wstatus, 0);
            if (wstatus == 0 || wstatus == (128 + SIGPIPE) * 256 ||
                (wstatus == -1 && errno == ECHILD)) {
            } else {
              *err_no = -1;
              LOG(WARNING) << "status[" << wstatus << "], cmd[" << cmd << "]"
                           << ", err_no[" << *err_no << "]";
            }
            if (wstatus == -1 && errno == ECHILD) {
              LOG(WARNING) << "errno is ECHILD";
            }
          }};
#endif
}

static int shell_p2open_fork_internal(const char* real_cmd, int pipein_fds[2],
                                      int pipeout_fds[2]) {
#ifndef _WIN32
  int child_pid = -1;
  if ((child_pid = fork()) < 0) {
    return -1;
  }

  if (child_pid != 0) {
    return child_pid;
  }

  close(pipein_fds[0]);
  close(pipeout_fds[1]);

  if (pipein_fds[1] != 1) {
    if (dup2(pipein_fds[1], 1) != 1) {
      return -1;
    }
    close(pipein_fds[1]);
  }

  if (pipeout_fds[0] != 0) {
    if (dup2(pipeout_fds[0], 0) != 0) {
      return -1;
    }
    close(pipeout_fds[0]);
  }

  close_open_fds_internal();
  if (execl("/bin/sh", "sh", "-c", real_cmd, NULL) < 0) {
    return -1;
  }
  exit(127);
#endif
  return 0;
}

std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open(
    const std::string& cmd) {
#ifndef _WIN32
  if (shell_verbose()) {
    LOG(INFO) << "Opening bidirectional pipe[" << cmd << "]";
  }

  std::string real_cmd = "set -o pipefail; " + cmd;

  int pipein_fds[2];
  int pipeout_fds[2];
  if (pipe(pipein_fds) != 0) {
    return {NULL, NULL};
  }
  if (pipe(pipeout_fds) != 0) {
    return {NULL, NULL};
  }

  int child_pid =
      shell_p2open_fork_internal(real_cmd.c_str(), pipein_fds, pipeout_fds);

  close(pipein_fds[1]);
  close(pipeout_fds[0]);
  fcntl(pipein_fds[0], F_SETFD, FD_CLOEXEC);
  fcntl(pipeout_fds[1], F_SETFD, FD_CLOEXEC);

  std::shared_ptr<int> child_life = {
      NULL, [child_pid, cmd](void*) {
        if (shell_verbose()) {
          LOG(INFO) << "Closing bidirectional pipe[" << cmd << "]";
        }

        int wstatus, ret;

        do {
          PCHECK((ret = waitpid(child_pid, &wstatus, 0)) >= 0 ||
                 (ret == -1 && errno == EINTR));
        } while (ret == -1 && errno == EINTR);

        PCHECK(wstatus == 0 || wstatus == (128 + SIGPIPE) * 256 ||
               (wstatus == -1 && errno == ECHILD))
            << "status[" << wstatus << "], cmd[" << cmd << "]";

        if (wstatus == -1 && errno == ECHILD) {
          LOG(WARNING) << "errno is ECHILD";
        }
      }};

  FILE* in_fp;
  PCHECK((in_fp = fdopen(pipein_fds[0], "r")) != NULL);
  FILE* out_fp;
  PCHECK((out_fp = fdopen(pipeout_fds[1], "w")) != NULL);
  return {{in_fp, [child_life](FILE* fp) { PCHECK(fclose(fp) == 0); }},
          {out_fp, [child_life](FILE* fp) { PCHECK(fclose(fp) == 0); }}};
#else
  return nullptr;
#endif
}

std::string shell_get_command_output(const std::string& cmd) {
#ifndef _WIN32
  int err_no = 0;
  do {
    err_no = 0;
    std::shared_ptr<FILE> pipe = shell_popen(cmd, "r", &err_no);
    string::LineFileReader reader;

    if (reader.getdelim(&*pipe, 0)) {
      pipe = nullptr;
      if (err_no == 0) {
        return reader.get();
      }
    }
  } while (err_no == -1);
#endif
  return "";
}
}  // end namespace framework
}  // end namespace paddle
<commit_msg>fix io and fs compile on mac test=develop<commit_after>// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/fluid/framework/io/shell.h"

namespace paddle {
namespace framework {

std::shared_ptr<FILE> shell_fopen(const std::string& path,
                                  const std::string& mode) {
#if defined _WIN32 || defined __APPLE__
  return nullptr;
#else
  if (shell_verbose()) {
    LOG(INFO) << "Opening file[" << path << "] with mode[" << mode << "]";
  }
  FILE* fp;
  if (!(fp = fopen(path.c_str(), mode.c_str()))) {
    LOG(FATAL) << "fopen fail, path[" << path << "], mode[" << mode << "]";
  }
  return {fp, [path](FILE* fp) {
            if (shell_verbose()) {
              LOG(INFO) << "Closing file[" << path << "]";
            }
            if (0 != fclose(fp)) {
              LOG(FATAL) << "fclose fail, path[" << path << "]";
            }
          }};
#endif
}

// Close all open file descriptors
// The implementation is async signal safe
// Mostly copy from CPython code
static int close_open_fds_internal() {
#if defined _WIN32 || defined __APPLE__
  return 0;
#else
  struct linux_dirent {
    long d_ino = 0;  // NOLINT
    off_t d_off;
    unsigned short d_reclen = 0;  // NOLINT
    char d_name[256];
  };

  int dir_fd = -1;
  if ((dir_fd = open("/proc/self/fd", O_RDONLY)) < 0) {
    LOG(FATAL) << "proc/self/fd open fail";
    return -1;
  }
  char buffer[sizeof(linux_dirent)];

  for (;;) {
    int bytes = 0;
    if ((bytes = syscall(SYS_getdents, dir_fd,
                         reinterpret_cast<linux_dirent*>(buffer),
                         sizeof(buffer))) < 0) {
      LOG(FATAL) << "syscall fail";
      return -1;
    }

    if (bytes == 0) {
      break;
    }

    linux_dirent* entry = NULL;

    for (int offset = 0; offset < bytes; offset += entry->d_reclen) {
      entry = reinterpret_cast<linux_dirent*>(buffer + offset);
      int fd = 0;
      const char* s = entry->d_name;

      while (*s >= '0' && *s <= '9') {
        fd = fd * 10 + (*s - '0');
        s++;
      }

      if (s != entry->d_name && fd != dir_fd && fd >= 3) {
        close(fd);
      }
    }
  }

  close(dir_fd);
  return 0;
#endif
}

static int shell_popen_fork_internal(const char* real_cmd, bool do_read,
                                     int parent_end, int child_end) {
#if defined _WIN32 || defined __APPLE__
  return 0;
#else
  int child_pid = -1;
  // Too frequent calls to fork() makes openmpi very slow. Use vfork() instead.
  // But vfork() is very dangerous. Be careful.
  if ((child_pid = vfork()) < 0) {
    return -1;
  }

  // The following code is async signal safe (No memory allocation, no access to
  // global data, etc.)
  if (child_pid != 0) {
    return child_pid;
  }

  int child_std_end = do_read ? 1 : 0;
  close(parent_end);

  if (child_end != child_std_end) {
    if (dup2(child_end, child_std_end) != child_std_end) {
      return -1;
    }
    close(child_end);
  }

  close_open_fds_internal();
  if (execl("/bin/sh", "sh", "-c", real_cmd, NULL) < 0) {
    return -1;
  }
  exit(127);
#endif
}

std::shared_ptr<FILE> shell_popen(const std::string& cmd,
                                  const std::string& mode, int* err_no) {
#if defined _WIN32 || defined __APPLE__
  return nullptr;
#else
  bool do_read = mode == "r";
  bool do_write = mode == "w";
  if (!(do_read || do_write)) {
    *err_no = -1;
    return NULL;
  }

  if (shell_verbose()) {
    LOG(INFO) << "Opening pipe[" << cmd << "] with mode[" << mode << "]";
  }

  std::string real_cmd = "set -o pipefail; " + cmd;

  int pipe_fds[2];
  if (pipe(pipe_fds) != 0) {
    *err_no = -1;
    return NULL;
  }
  int parent_end = 0;
  int child_end = 0;

  if (do_read) {
    parent_end = pipe_fds[0];
    child_end = pipe_fds[1];
  } else if (do_write) {
    parent_end = pipe_fds[1];
    child_end = pipe_fds[0];
  }

  int child_pid = shell_popen_fork_internal(real_cmd.c_str(), do_read,
                                            parent_end, child_end);
  close(child_end);
  fcntl(parent_end, F_SETFD, FD_CLOEXEC);
  FILE* fp;
  if ((fp = fdopen(parent_end, mode.c_str())) == NULL) {
    *err_no = -1;
    return NULL;
  }
  return {fp, [child_pid, cmd, err_no](FILE* fp) {
            if (shell_verbose()) {
              LOG(INFO) << "Closing pipe[" << cmd << "]";
            }

            if (fclose(fp) != 0) {
              *err_no = -1;
            }
            int wstatus = -1;
            waitpid(child_pid, &wstatus, 0);
            if (wstatus == 0 || wstatus == (128 + SIGPIPE) * 256 ||
                (wstatus == -1 && errno == ECHILD)) {
            } else {
              *err_no = -1;
              LOG(WARNING) << "status[" << wstatus << "], cmd[" << cmd << "]"
                           << ", err_no[" << *err_no << "]";
            }
            if (wstatus == -1 && errno == ECHILD) {
              LOG(WARNING) << "errno is ECHILD";
            }
          }};
#endif
}

static int shell_p2open_fork_internal(const char* real_cmd, int pipein_fds[2],
                                      int pipeout_fds[2]) {
#if defined _WIN32 || defined __APPLE__
  return 0;
#else
  int child_pid = -1;
  if ((child_pid = fork()) < 0) {
    return -1;
  }

  if (child_pid != 0) {
    return child_pid;
  }

  close(pipein_fds[0]);
  close(pipeout_fds[1]);

  if (pipein_fds[1] != 1) {
    if (dup2(pipein_fds[1], 1) != 1) {
      return -1;
    }
    close(pipein_fds[1]);
  }

  if (pipeout_fds[0] != 0) {
    if (dup2(pipeout_fds[0], 0) != 0) {
      return -1;
    }
    close(pipeout_fds[0]);
  }

  close_open_fds_internal();
  if (execl("/bin/sh", "sh", "-c", real_cmd, NULL) < 0) {
    return -1;
  }
  exit(127);
#endif
}

std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open(
    const std::string& cmd) {
#if defined _WIN32 || defined __APPLE__
  return nullptr;
#else
  if (shell_verbose()) {
    LOG(INFO) << "Opening bidirectional pipe[" << cmd << "]";
  }

  std::string real_cmd = "set -o pipefail; " + cmd;

  int pipein_fds[2];
  int pipeout_fds[2];
  if (pipe(pipein_fds) != 0) {
    return {NULL, NULL};
  }
  if (pipe(pipeout_fds) != 0) {
    return {NULL, NULL};
  }

  int child_pid =
      shell_p2open_fork_internal(real_cmd.c_str(), pipein_fds, pipeout_fds);

  close(pipein_fds[1]);
  close(pipeout_fds[0]);
  fcntl(pipein_fds[0], F_SETFD, FD_CLOEXEC);
  fcntl(pipeout_fds[1], F_SETFD, FD_CLOEXEC);

  std::shared_ptr<int> child_life = {
      NULL, [child_pid, cmd](void*) {
        if (shell_verbose()) {
          LOG(INFO) << "Closing bidirectional pipe[" << cmd << "]";
        }

        int wstatus, ret;

        do {
          PCHECK((ret = waitpid(child_pid, &wstatus, 0)) >= 0 ||
                 (ret == -1 && errno == EINTR));
        } while (ret == -1 && errno == EINTR);

        PCHECK(wstatus == 0 || wstatus == (128 + SIGPIPE) * 256 ||
               (wstatus == -1 && errno == ECHILD))
            << "status[" << wstatus << "], cmd[" << cmd << "]";

        if (wstatus == -1 && errno == ECHILD) {
          LOG(WARNING) << "errno is ECHILD";
        }
      }};

  FILE* in_fp;
  PCHECK((in_fp = fdopen(pipein_fds[0], "r")) != NULL);
  FILE* out_fp;
  PCHECK((out_fp = fdopen(pipeout_fds[1], "w")) != NULL);
  return {{in_fp, [child_life](FILE* fp) { PCHECK(fclose(fp) == 0); }},
          {out_fp, [child_life](FILE* fp) { PCHECK(fclose(fp) == 0); }}};
#endif
}

std::string shell_get_command_output(const std::string& cmd) {
#if defined _WIN32 || defined __APPLE__
  return "";
#else
  int err_no = 0;
  do {
    err_no = 0;
    std::shared_ptr<FILE> pipe = shell_popen(cmd, "r", &err_no);
    string::LineFileReader reader;

    if (reader.getdelim(&*pipe, 0)) {
      pipe = nullptr;
      if (err_no == 0) {
        return reader.get();
      }
    }
  } while (err_no == -1);
#endif
}
}  // end namespace framework
}  // end namespace paddle
<|endoftext|>