<commit_before>/**
 * Copyright 2013 Da Zheng
 *
 * This file is part of SAFSlib.
 *
 * SAFSlib is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * SAFSlib is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with SAFSlib.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#include <string>

#include "io_interface.h"
#include "native_file.h"
#include "safs_file.h"

const int BUF_SIZE = 1024 * 64 * PAGE_SIZE;
const size_t DATA_SIZE = 10L * 1024 * 1024 * 1024;

ssize_t complete_read(int fd, char *buf, size_t count)
{
	ssize_t bytes = 0;
	do {
		ssize_t ret = read(fd, buf, count);
		if (ret < 0)
			return ret;
		if (ret == 0)
			return bytes;
		bytes += ret;
		count -= ret;
		buf += ret;
	} while (count > 0);
	return bytes;
}

int verify_bytes(const char *bs1, const char *bs2, int size)
{
	for (int i = 0; i < size; i++) {
		assert(bs1[i] == bs2[i]);
	}
	return 0;
}

class data_source
{
public:
	virtual ssize_t get_data(off_t off, size_t size, char *buf) const = 0;
	virtual size_t get_size() const = 0;
};

class file_data_source: public data_source
{
	int fd;
	size_t file_size;
public:
	file_data_source(const std::string &ext_file) {
		fd = open(ext_file.c_str(), O_RDONLY);
		if (fd < 0) {
			perror("open");
			exit(-1);
		}
		native_file f(ext_file);
		file_size = f.get_size();
	}

	virtual ssize_t get_data(off_t off, size_t size, char *buf) const {
		long new_off = lseek(fd, off, SEEK_SET);
		assert(new_off == off);
		ssize_t ret = complete_read(fd, buf, size);
		if (ret < 0) {
			perror("complete_read");
			exit(-1);
		}
		return ret;
	}

	virtual size_t get_size() const {
		return file_size;
	}
};

class synthetic_data_source: public data_source
{
	size_t size;
public:
	synthetic_data_source(size_t size) {
		this->size = size;
	}

	virtual ssize_t get_data(off_t off, size_t size, char *buf) const {
		off_t *long_pointer = (off_t *) buf;
		int num_longs = size / sizeof(off_t);
		long value = off / sizeof(off_t);
		for (int i = 0; i < num_longs; i++, value++) {
			long_pointer[i] = value;
		}
		return size;
	}

	virtual size_t get_size() const {
		return size;
	}
};

class verify_callback: public callback
{
	char *orig_buf;
	data_source *source;
	size_t verified_bytes;
public:
	verify_callback(data_source *source) {
		this->source = source;
		orig_buf = (char *) malloc(BUF_SIZE);
		verified_bytes = 0;
	}

	~verify_callback() {
		free(orig_buf);
	}

	int invoke(io_request *rqs[], int num) {
		size_t read_bytes = min<size_t>(rqs[0]->get_size(),
				source->get_size() - rqs[0]->get_offset());
		assert(read_bytes > 0);
		size_t ret = source->get_data(rqs[0]->get_offset(), read_bytes, orig_buf);
		fprintf(stderr, "verify block %lx of %ld bytes\n", rqs[0]->get_offset(), read_bytes);
		assert(ret == read_bytes);
		verified_bytes += read_bytes;
		verify_bytes(rqs[0]->get_buf(), orig_buf, read_bytes);
	}

	size_t get_verified_bytes() const {
		return verified_bytes;
	}
};

void comm_verify_file(int argc, char *argv[])
{
	if (argc < 1) {
		fprintf(stderr, "verify file_name [ext_file]");
		fprintf(stderr, "file_name is the file name in the SA-FS file system\n");
		fprintf(stderr, "ext_file is the file in the external file system\n");
		exit(-1);
	}

	std::string int_file_name = argv[0];
	std::string ext_file;
	if (argc >= 2)
		ext_file = argv[1];

	file_io_factory *factory = create_io_factory(int_file_name, REMOTE_ACCESS);
	thread *curr_thread = thread::represent_thread(0);
	io_interface *io = factory->create_io(curr_thread);
	data_source *source;
	if (ext_file.empty())
		source = new synthetic_data_source(DATA_SIZE);
	else
		source = new file_data_source(ext_file);
	verify_callback *cb = new verify_callback(source);
	io->set_callback(cb);

	size_t file_size = source->get_size();
	assert(factory->get_file_size() >= file_size);
	file_size = ROUNDUP(file_size, BUF_SIZE);
	char *buf = (char *) valloc(BUF_SIZE);
	for (off_t off = 0; off < file_size; off += BUF_SIZE) {
		io_request req(buf, off, BUF_SIZE, READ, io, 0);
		io->access(&req, 1);
		io->wait4complete(1);
	}
	printf("verify %ld bytes\n", cb->get_verified_bytes());

	io->cleanup();
	delete io->get_callback();
	factory->destroy_io(io);
}

void comm_load_file2fs(int argc, char *argv[])
{
	if (argc < 1) {
		fprintf(stderr, "load file_name [ext_file]\n");
		fprintf(stderr, "file_name is the file name in the SA-FS file system\n");
		fprintf(stderr, "ext_file is the file in the external file system\n");
		exit(-1);
	}

	std::string int_file_name = argv[0];
	std::string ext_file;
	if (argc >= 2)
		ext_file = argv[1];

	file_io_factory *factory = create_io_factory(int_file_name, REMOTE_ACCESS);
	thread *curr_thread = thread::represent_thread(0);
	io_interface *io = factory->create_io(curr_thread);

	data_source *source;
	if (ext_file.empty()) {
		printf("use synthetic data\n");
		source = new synthetic_data_source(DATA_SIZE);
	}
	else {
		printf("use file %s\n", ext_file.c_str());
		source = new file_data_source(ext_file);
	}
	assert(factory->get_file_size() >= source->get_size());
	printf("source size: %ld\n", source->get_size());

	char *buf = (char *) valloc(BUF_SIZE);
	off_t off = 0;

	while (off < source->get_size()) {
		size_t size = min<size_t>(BUF_SIZE, source->get_size() - off);
		size_t ret = source->get_data(off, size, buf);
		assert(ret == size);
		ssize_t write_bytes = ROUNDUP(ret, 512);
		io_request req(buf, off, write_bytes, WRITE, io, 0);
		io->access(&req, 1);
		io->wait4complete(1);
		off += write_bytes;
	}
	printf("write all data\n");

	io->cleanup();
	factory->destroy_io(io);
}

void comm_create_file(int argc, char *argv[])
{
	if (argc < 2) {
		fprintf(stderr, "create file_name size\n");
		fprintf(stderr, "file_name is the file name in the SA-FS file system\n");
		exit(-1);
	}

	std::string file_name = argv[0];
	size_t file_size = str2size(argv[1]);
	safs_file file(get_sys_RAID_conf(), file_name);
	file.create_file(file_size);
	printf("create file %s of %ld bytes\n", file_name.c_str(),
			file.get_file_size());
}

typedef void (*command_func_t)(int argc, char *argv[]);

struct command
{
	std::string name;
	command_func_t func;
	std::string help_info;
};

struct command commands[] = {
	{"create", comm_create_file,
		"create file_name size: create a file with the specified size"},
	{"load", comm_load_file2fs,
		"load file_name [ext_file]: load data to the file"},
	{"verify", comm_verify_file,
		"verify file_name [ext_file]: verify data in the file"},
};

int get_num_commands()
{
	return sizeof(commands) / sizeof(commands[0]);
}

const command *get_command(const std::string &name)
{
	int num_comms = get_num_commands();
	for (int i = 0; i < num_comms; i++) {
		if (commands[i].name == name)
			return &commands[i];
	}
	return NULL;
}

void print_help()
{
	printf("SAFS-util conf_file command ...\n");
	printf("The supported commands are\n");
	int num_commands = get_num_commands();
	for (int i =0; i < num_commands; i++) {
		printf("\t%s\n", commands[i].help_info.c_str());
	}
}

/**
 * This is a utility tool for the SA-FS.
 */

int main(int argc, char *argv[])
{
	if (argc < 3) {
		print_help();
		exit(-1);
	}

	std::string conf_file = argv[1];
	std::string command = argv[2];

	config_map configs(conf_file);
	init_io_system(configs);

	const struct command *comm = get_command(command);
	if (comm == NULL) {
		fprintf(stderr, "wrong command %s\n", comm->name.c_str());
		print_help();
		return -1;
	}
	comm->func(argc - 3, argv + 3);
}
<commit_msg>[SAFS]: add command help and list in SAFS-util.<commit_after>/**
 * Copyright 2013 Da Zheng
 *
 * This file is part of SAFSlib.
 *
 * SAFSlib is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * SAFSlib is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with SAFSlib.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#include <string>

#include "io_interface.h"
#include "native_file.h"
#include "safs_file.h"

const int BUF_SIZE = 1024 * 64 * PAGE_SIZE;
const size_t DATA_SIZE = 10L * 1024 * 1024 * 1024;

ssize_t complete_read(int fd, char *buf, size_t count)
{
	ssize_t bytes = 0;
	do {
		ssize_t ret = read(fd, buf, count);
		if (ret < 0)
			return ret;
		if (ret == 0)
			return bytes;
		bytes += ret;
		count -= ret;
		buf += ret;
	} while (count > 0);
	return bytes;
}

int verify_bytes(const char *bs1, const char *bs2, int size)
{
	for (int i = 0; i < size; i++) {
		assert(bs1[i] == bs2[i]);
	}
	return 0;
}

class data_source
{
public:
	virtual ssize_t get_data(off_t off, size_t size, char *buf) const = 0;
	virtual size_t get_size() const = 0;
};

class file_data_source: public data_source
{
	int fd;
	size_t file_size;
public:
	file_data_source(const std::string &ext_file) {
		fd = open(ext_file.c_str(), O_RDONLY);
		if (fd < 0) {
			perror("open");
			exit(-1);
		}
		native_file f(ext_file);
		file_size = f.get_size();
	}

	virtual ssize_t get_data(off_t off, size_t size, char *buf) const {
		long new_off = lseek(fd, off, SEEK_SET);
		assert(new_off == off);
		ssize_t ret = complete_read(fd, buf, size);
		if (ret < 0) {
			perror("complete_read");
			exit(-1);
		}
		return ret;
	}

	virtual size_t get_size() const {
		return file_size;
	}
};

class synthetic_data_source: public data_source
{
	size_t size;
public:
	synthetic_data_source(size_t size) {
		this->size = size;
	}

	virtual ssize_t get_data(off_t off, size_t size, char *buf) const {
		off_t *long_pointer = (off_t *) buf;
		int num_longs = size / sizeof(off_t);
		long value = off / sizeof(off_t);
		for (int i = 0; i < num_longs; i++, value++) {
			long_pointer[i] = value;
		}
		return size;
	}

	virtual size_t get_size() const {
		return size;
	}
};

class verify_callback: public callback
{
	char *orig_buf;
	data_source *source;
	size_t verified_bytes;
public:
	verify_callback(data_source *source) {
		this->source = source;
		orig_buf = (char *) malloc(BUF_SIZE);
		verified_bytes = 0;
	}

	~verify_callback() {
		free(orig_buf);
	}

	int invoke(io_request *rqs[], int num) {
		size_t read_bytes = min<size_t>(rqs[0]->get_size(),
				source->get_size() - rqs[0]->get_offset());
		assert(read_bytes > 0);
		size_t ret = source->get_data(rqs[0]->get_offset(), read_bytes, orig_buf);
		fprintf(stderr, "verify block %lx of %ld bytes\n", rqs[0]->get_offset(), read_bytes);
		assert(ret == read_bytes);
		verified_bytes += read_bytes;
		verify_bytes(rqs[0]->get_buf(), orig_buf, read_bytes);
	}

	size_t get_verified_bytes() const {
		return verified_bytes;
	}
};

void comm_verify_file(int argc, char *argv[])
{
	if (argc < 1) {
		fprintf(stderr, "verify file_name [ext_file]");
		fprintf(stderr, "file_name is the file name in the SA-FS file system\n");
		fprintf(stderr, "ext_file is the file in the external file system\n");
		exit(-1);
	}

	std::string int_file_name = argv[0];
	std::string ext_file;
	if (argc >= 2)
		ext_file = argv[1];

	file_io_factory *factory = create_io_factory(int_file_name, REMOTE_ACCESS);
	thread *curr_thread = thread::represent_thread(0);
	io_interface *io = factory->create_io(curr_thread);
	data_source *source;
	if (ext_file.empty())
		source = new synthetic_data_source(DATA_SIZE);
	else
		source = new file_data_source(ext_file);
	verify_callback *cb = new verify_callback(source);
	io->set_callback(cb);

	size_t file_size = source->get_size();
	assert(factory->get_file_size() >= file_size);
	file_size = ROUNDUP(file_size, BUF_SIZE);
	char *buf = (char *) valloc(BUF_SIZE);
	for (off_t off = 0; off < file_size; off += BUF_SIZE) {
		io_request req(buf, off, BUF_SIZE, READ, io, 0);
		io->access(&req, 1);
		io->wait4complete(1);
	}
	printf("verify %ld bytes\n", cb->get_verified_bytes());

	io->cleanup();
	delete io->get_callback();
	factory->destroy_io(io);
}

void comm_load_file2fs(int argc, char *argv[])
{
	if (argc < 1) {
		fprintf(stderr, "load file_name [ext_file]\n");
		fprintf(stderr, "file_name is the file name in the SA-FS file system\n");
		fprintf(stderr, "ext_file is the file in the external file system\n");
		exit(-1);
	}

	std::string int_file_name = argv[0];
	std::string ext_file;
	if (argc >= 2)
		ext_file = argv[1];

	file_io_factory *factory = create_io_factory(int_file_name, REMOTE_ACCESS);
	thread *curr_thread = thread::represent_thread(0);
	io_interface *io = factory->create_io(curr_thread);

	data_source *source;
	if (ext_file.empty()) {
		printf("use synthetic data\n");
		source = new synthetic_data_source(DATA_SIZE);
	}
	else {
		printf("use file %s\n", ext_file.c_str());
		source = new file_data_source(ext_file);
	}
	assert(factory->get_file_size() >= source->get_size());
	printf("source size: %ld\n", source->get_size());

	char *buf = (char *) valloc(BUF_SIZE);
	off_t off = 0;

	while (off < source->get_size()) {
		size_t size = min<size_t>(BUF_SIZE, source->get_size() - off);
		size_t ret = source->get_data(off, size, buf);
		assert(ret == size);
		ssize_t write_bytes = ROUNDUP(ret, 512);
		io_request req(buf, off, write_bytes, WRITE, io, 0);
		io->access(&req, 1);
		io->wait4complete(1);
		off += write_bytes;
	}
	printf("write all data\n");

	io->cleanup();
	factory->destroy_io(io);
}

void comm_create_file(int argc, char *argv[])
{
	if (argc < 2) {
		fprintf(stderr, "create file_name size\n");
		fprintf(stderr, "file_name is the file name in the SA-FS file system\n");
		exit(-1);
	}

	std::string file_name = argv[0];
	size_t file_size = str2size(argv[1]);
	safs_file file(get_sys_RAID_conf(), file_name);
	file.create_file(file_size);
	printf("create file %s of %ld bytes\n", file_name.c_str(),
			file.get_file_size());
}

void print_help();

void comm_help(int argc, char *argv[])
{
	print_help();
}

void comm_list(int argc, char *argv[])
{
	std::set<std::string> files;
	const RAID_config &conf = get_sys_RAID_conf();

	// First find all individual file names in the root directories.
	for (int i = 0; i < conf.get_num_disks(); i++) {
		std::string dir_name = conf.get_disk(i).name;
		native_dir dir(dir_name);
		std::vector<std::string> file_names;
		dir.read_all_files(file_names);
		files.insert(file_names.begin(), file_names.end());
	}

	for (std::set<std::string>::const_iterator it = files.begin();
			it != files.end(); it++) {
		safs_file file(conf, *it);
		if (file.exist()) {
			printf("%s: %ld bytes\n", file.get_name().c_str(),
					file.get_file_size());
		}
		else {
			printf("%s is corrupted\n", file.get_name().c_str());
		}
	}
}

typedef void (*command_func_t)(int argc, char *argv[]);

struct command
{
	std::string name;
	command_func_t func;
	std::string help_info;
};

struct command commands[] = {
	{"create", comm_create_file,
		"create file_name size: create a file with the specified size"},
	{"help", comm_help,
		"help: print the help info"},
	{"list", comm_list, "list: list existing files in SAFS"},
	{"load", comm_load_file2fs,
		"load file_name [ext_file]: load data to the file"},
	{"verify", comm_verify_file,
		"verify file_name [ext_file]: verify data in the file"},
};

int get_num_commands()
{
	return sizeof(commands) / sizeof(commands[0]);
}

const command *get_command(const std::string &name)
{
	int num_comms = get_num_commands();
	for (int i = 0; i < num_comms; i++) {
		if (commands[i].name == name)
			return &commands[i];
	}
	return NULL;
}

void print_help()
{
	printf("SAFS-util conf_file command ...\n");
	printf("The supported commands are\n");
	int num_commands = get_num_commands();
	for (int i =0; i < num_commands; i++) {
		printf("\t%s\n", commands[i].help_info.c_str());
	}
}

/**
 * This is a utility tool for the SA-FS.
 */

int main(int argc, char *argv[])
{
	if (argc < 3) {
		print_help();
		exit(-1);
	}

	std::string conf_file = argv[1];
	std::string command = argv[2];

	config_map configs(conf_file);
	init_io_system(configs);

	const struct command *comm = get_command(command);
	if (comm == NULL) {
		fprintf(stderr, "wrong command %s\n", comm->name.c_str());
		print_help();
		return -1;
	}
	comm->func(argc - 3, argv + 3);
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2018 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "platform.h"

#include "base_swapchain.h"

namespace swapchain {

namespace {
VkResult createSemaphore(const DeviceData *device_functions, VkDevice device,
                         const VkAllocationCallbacks *pAllocator,
                         VkSemaphore *pSem) {
  VkSemaphoreCreateInfo createInfo = {
      VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,  // sType
      nullptr,                                  // pNext
      0,                                        // flags
  };
  return device_functions->vkCreateSemaphore(device, &createInfo, pAllocator,
                                             pSem);
}
VkResult createSemaphores(const DeviceData *device_functions, VkDevice device,
                          const VkAllocationCallbacks *pAllocator, size_t count,
                          std::vector<VkSemaphore> &sems) {
  sems.resize(count);
  for (VkSemaphore &sem : sems) {
    VkResult res;
    if ((res = createSemaphore(device_functions, device, pAllocator, &sem)) !=
        VK_SUCCESS) {
      return res;
    }
  }
  return VK_SUCCESS;
}

void destroySemaphores(const DeviceData *device_functions, VkDevice device,
                       const VkAllocationCallbacks *pAllocator,
                       std::vector<VkSemaphore> &sems) {
  for (VkSemaphore sem : sems) {
    device_functions->vkDestroySemaphore(device, sem, pAllocator);
  }
  sems.clear();
}
}  // namespace

BaseSwapchain::BaseSwapchain(VkInstance instance, VkDevice device,
                             uint32_t queue, VkCommandPool command_pool,
                             uint32_t num_images,
                             const InstanceData *instance_functions,
                             const DeviceData *device_functions,
                             const VkSwapchainCreateInfoKHR *swapchain_info,
                             const VkAllocationCallbacks *pAllocator,
                             const void *platform_info)
    : instance_(instance),
      device_(device),
      instance_functions_(instance_functions),
      device_functions_(device_functions),
      swapchain_info_(*swapchain_info) {
  if (platform_info == nullptr) {
    return;
  }

  CreateSurface(instance_functions, instance, platform_info, pAllocator,
                &surface_);
  if (surface_ == 0) {
    // Surface creation failed
    return;
  }

  {
    // Create the swapchain
    VkSwapchainCreateInfoKHR createInfo = {
        VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR,  // sType
        nullptr,                                      // pNext
        0,                                            // flags
        surface_,                                     // surface
        num_images,                                   // minImageCount
        swapchain_info_.imageFormat,                  // imageFormat
        swapchain_info_.imageColorSpace,              // imageColorSpace
        swapchain_info_.imageExtent,                  // imageExtent
        swapchain_info_.imageArrayLayers,             // arrayLayers
        VK_IMAGE_USAGE_TRANSFER_DST_BIT,              // imageUsage
        VK_SHARING_MODE_EXCLUSIVE,                    // imageSharingMode,
        0,                                            // queueFamilyIndexCount
        nullptr,                                      // pQueueFamilyIndices
        swapchain_info_.preTransform,                 // preTransform
        swapchain_info_.compositeAlpha,               // compositeAlpha
        VK_PRESENT_MODE_FIFO_KHR,                     // presentMode
        VK_TRUE,                                      // clipped
        0,                                            // oldSwapchain
    };
    if (device_functions_->vkCreateSwapchainKHR(
            device_, &createInfo, pAllocator, &swapchain_) != VK_SUCCESS) {
      // Creating swapchain failed
      swapchain_ = 0;
      return;
    }
  }

  uint32_t num_base_images = 0;
  device_functions_->vkGetSwapchainImagesKHR(device, swapchain_,
                                             &num_base_images, nullptr);
  images_.resize(num_base_images);
  device_functions_->vkGetSwapchainImagesKHR(device, swapchain_,
                                             &num_base_images, images_.data());

  // Create a command buffer for each virtual image to blit from
  VkCommandBufferAllocateInfo command_buffer_info = {
      VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,  // sType
      nullptr,                                         // pNext
      command_pool,                                    // commandPool
      VK_COMMAND_BUFFER_LEVEL_PRIMARY,                 // level
      num_images,                                      // commandBufferCount
  };
  command_buffers_.resize(num_images);
  if (device_functions_->vkAllocateCommandBuffers(device, &command_buffer_info,
                                                  command_buffers_.data()) !=
      VK_SUCCESS) {
    return;
  }
  for (VkCommandBuffer cmdbuf : command_buffers_) {
    set_dispatch_from_parent(cmdbuf, device);
  }

  if (createSemaphore(device_functions_, device_, pAllocator,
                      &acquire_semaphore_) != VK_SUCCESS) {
    return;
  }
  if (createSemaphores(device_functions_, device_, pAllocator, num_images,
                       blit_semaphores_) != VK_SUCCESS) {
    return;
  }
  if (createSemaphores(device_functions_, device_, pAllocator, num_images,
                       present_semaphores_) != VK_SUCCESS) {
    return;
  }

  is_pending_.resize(num_images);
}

void BaseSwapchain::Destroy(const VkAllocationCallbacks *pAllocator) {
  device_functions_->vkDestroySemaphore(device_, acquire_semaphore_,
                                        pAllocator);
  acquire_semaphore_ = VK_NULL_HANDLE;

  destroySemaphores(device_functions_, device_, pAllocator, blit_semaphores_);
  destroySemaphores(device_functions_, device_, pAllocator,
                    present_semaphores_);

  device_functions_->vkDestroySwapchainKHR(device_, swapchain_, pAllocator);
  swapchain_ = VK_NULL_HANDLE;

  instance_functions_->vkDestroySurfaceKHR(instance_, surface_, pAllocator);
  surface_ = VK_NULL_HANDLE;

  images_.clear();

  is_pending_.clear();
  command_buffers_.clear();
}

VkResult BaseSwapchain::PresentFrom(VkQueue queue, size_t index,
                                    VkImage image) {
  std::unique_lock<threading::mutex> guard(present_lock_);
  VkResult res;
  uint32_t base_index = 0;
  // TODO: the error return values here aren't necessarily valid return values
  // for VkQueueSubmit
  if ((res = device_functions_->vkAcquireNextImageKHR(
           device_, swapchain_, 0, acquire_semaphore_, VK_NULL_HANDLE,
           &base_index)) != VK_SUCCESS) {
    return res;
  }

  VkCommandBuffer cmdbuf = command_buffers_[index];
  if ((res = device_functions_->vkResetCommandBuffer(cmdbuf, 0)) !=
      VK_SUCCESS) {
    return res;
  }
  VkCommandBufferBeginInfo beginInfo = {
      VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,  // sType
      0,                                            // pNext
      VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,  // flags
      nullptr,                                      // pInheritanceInfo
  };
  if ((res = device_functions_->vkBeginCommandBuffer(cmdbuf, &beginInfo)) !=
      VK_SUCCESS) {
    return res;
  }

  // The source image is already in VK_IMAGE_LAYOUT_TRANSFER_SRC, we need to
  // transition our image between VK_IMAGE_LAYOUT_TRANSFER_DST and
  // VK_IMAGE_LAYOUT_SHADER_PRESENT_KHR
  VkImageMemoryBarrier initialBarrier = {
      VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,  // sType
      nullptr,                                 // pNext
      0,                                       // srcAccessFlags
      VK_ACCESS_TRANSFER_WRITE_BIT,            // dstAccessFlags
      VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,         // oldLayout
      VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,    // newLayout
      VK_QUEUE_FAMILY_IGNORED,                 // srcQueueFamilyIndex
      VK_QUEUE_FAMILY_IGNORED,                 // dstQueueFamilyIndex
      images_[base_index],                     // image
      VkImageSubresourceRange{
          VK_IMAGE_ASPECT_COLOR_BIT,         // aspectMask
          0,                                 // baseMipLevel
          1,                                 // levelCount
          0,                                 // baseArrayLayer
          swapchain_info_.imageArrayLayers,  // layerCount
      },                                     // subresourceRange
  };

  device_functions_->vkCmdPipelineBarrier(
      cmdbuf,
      0,                               // srcStageMask
      VK_PIPELINE_STAGE_TRANSFER_BIT,  // dstStageMask
      0,                               // dependencyFlags
      0,                               // memoryBarrierCount
      nullptr,                         // pMemoryBarriers
      0,                               // bufferMemoryBarrierCount
      nullptr,                         // pBufferMemoryBarriers
      1,                               // imageMemoryBarrierCount
      &initialBarrier                  // pImageMemoryBarriers
  );

  VkImageSubresourceLayers subresource = {
      VK_IMAGE_ASPECT_COLOR_BIT,         // aspectMask
      0,                                 // mipLevel
      0,                                 // baseArrayLayer
      swapchain_info_.imageArrayLayers,  // layerCount
  };
  VkOffset3D offsets[2] = {
      {
          0,
          0,
          0,
      },
      {
          (int32_t)swapchain_info_.imageExtent.width,
          (int32_t)swapchain_info_.imageExtent.height,
          1,
      },
  };
  VkImageBlit blit = {
      subresource,
      {offsets[0], offsets[1]},
      subresource,
      {offsets[0], offsets[1]},
  };
  device_functions_->vkCmdBlitImage(
      cmdbuf,
      image,                                 // srcImage
      VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,  // srcImageLayout
      images_[base_index],                   // dstImage
      VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,       // dstImageLayout
      1,                                     // regionCount
      &blit,                                 // pRegions
      VK_FILTER_NEAREST                      // filter
  );

  VkImageMemoryBarrier finalBarrier = initialBarrier;
  finalBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  finalBarrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  finalBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  finalBarrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;

  device_functions_->vkCmdPipelineBarrier(
      cmdbuf,
      VK_PIPELINE_STAGE_TRANSFER_BIT,  // srcStageMask
      0,                               // dstStageMask
      0,                               // dependencyFlags
      0,                               // memoryBarrierCount
      nullptr,                         // pMemoryBarriers
      0,                               // bufferMemoryBarrierCount
      nullptr,                         // pBufferMemoryBarriers
      1,                               // imageMemoryBarrierCount
      &finalBarrier                    // pImageMemoryBarriers
  );

  if ((res = device_functions_->vkEndCommandBuffer(cmdbuf)) != VK_SUCCESS) {
    return res;
  }

  VkSemaphore signal_semaphores[] = {blit_semaphores_[index],
                                     present_semaphores_[index]};
  VkPipelineStageFlags waitStage = VK_PIPELINE_STAGE_TRANSFER_BIT;
  VkSubmitInfo submitInfo = {
      VK_STRUCTURE_TYPE_SUBMIT_INFO,  // sType
      0,                              // pNext
      1,                              // waitSemaphoreCount
      &acquire_semaphore_,            // pWaitSemaphores
      &waitStage,                     // pWaitDstStageMask
      1,                              // commandBufferCount
      &cmdbuf,                        // pCommandBuffers
      2,                              // signalSemaphoreCount
      signal_semaphores,              // pSignalSemaphores
  };
  auto queue_functions = GetGlobalContext().GetQueueData(queue);
  if ((res = queue_functions->vkQueueSubmit(queue, 1, &submitInfo,
                                            VK_NULL_HANDLE)) != VK_SUCCESS) {
    return res;
  }
  is_pending_[index] = true;

  VkPresentInfoKHR presentInfo = {
      VK_STRUCTURE_TYPE_PRESENT_INFO_KHR,  // sType
      0,                                   // pNext
      1,                                   // waitSemaphoreCount
      &present_semaphores_[index],         // waitSemaphores
      1,                                   // swapchainCount
      &swapchain_,                         // pSwapchains,
      &base_index,                         // pImageIndices
      nullptr,                             // pResults
  };
  if ((res = queue_functions->vkQueuePresentKHR(queue, &presentInfo)) !=
      VK_SUCCESS) {
    return res;
  }
  return VK_SUCCESS;
}

VkSemaphore BaseSwapchain::BlitWaitSemaphore(size_t index) {
  if (!is_pending_[index]) {
    return VK_NULL_HANDLE;
  }
  is_pending_[index] = false;
  return blit_semaphores_[index];
}
}  // namespace swapchain
<commit_msg>Fix incorrect image blitting layout<commit_after>/*
 * Copyright (C) 2018 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "platform.h"

#include "base_swapchain.h"

namespace swapchain {

namespace {
VkResult createSemaphore(const DeviceData *device_functions, VkDevice device,
                         const VkAllocationCallbacks *pAllocator,
                         VkSemaphore *pSem) {
  VkSemaphoreCreateInfo createInfo = {
      VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,  // sType
      nullptr,                                  // pNext
      0,                                        // flags
  };
  return device_functions->vkCreateSemaphore(device, &createInfo, pAllocator,
                                             pSem);
}
VkResult createSemaphores(const DeviceData *device_functions, VkDevice device,
                          const VkAllocationCallbacks *pAllocator, size_t count,
                          std::vector<VkSemaphore> &sems) {
  sems.resize(count);
  for (VkSemaphore &sem : sems) {
    VkResult res;
    if ((res = createSemaphore(device_functions, device, pAllocator, &sem)) !=
        VK_SUCCESS) {
      return res;
    }
  }
  return VK_SUCCESS;
}

void destroySemaphores(const DeviceData *device_functions, VkDevice device,
                       const VkAllocationCallbacks *pAllocator,
                       std::vector<VkSemaphore> &sems) {
  for (VkSemaphore sem : sems) {
    device_functions->vkDestroySemaphore(device, sem, pAllocator);
  }
  sems.clear();
}
}  // namespace

BaseSwapchain::BaseSwapchain(VkInstance instance, VkDevice device,
                             uint32_t queue, VkCommandPool command_pool,
                             uint32_t num_images,
                             const InstanceData *instance_functions,
                             const DeviceData *device_functions,
                             const VkSwapchainCreateInfoKHR *swapchain_info,
                             const VkAllocationCallbacks *pAllocator,
                             const void *platform_info)
    : instance_(instance),
      device_(device),
      instance_functions_(instance_functions),
      device_functions_(device_functions),
      swapchain_info_(*swapchain_info) {
  if (platform_info == nullptr) {
    return;
  }

  CreateSurface(instance_functions, instance, platform_info, pAllocator,
                &surface_);
  if (surface_ == 0) {
    // Surface creation failed
    return;
  }

  {
    // Create the swapchain
    VkSwapchainCreateInfoKHR createInfo = {
        VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR,  // sType
        nullptr,                                      // pNext
        0,                                            // flags
        surface_,                                     // surface
        num_images,                                   // minImageCount
        swapchain_info_.imageFormat,                  // imageFormat
        swapchain_info_.imageColorSpace,              // imageColorSpace
        swapchain_info_.imageExtent,                  // imageExtent
        swapchain_info_.imageArrayLayers,             // arrayLayers
        VK_IMAGE_USAGE_TRANSFER_DST_BIT,              // imageUsage
        VK_SHARING_MODE_EXCLUSIVE,                    // imageSharingMode,
        0,                                            // queueFamilyIndexCount
        nullptr,                                      // pQueueFamilyIndices
        swapchain_info_.preTransform,                 // preTransform
        swapchain_info_.compositeAlpha,               // compositeAlpha
        VK_PRESENT_MODE_FIFO_KHR,                     // presentMode
        VK_TRUE,                                      // clipped
        0,                                            // oldSwapchain
    };
    if (device_functions_->vkCreateSwapchainKHR(
            device_, &createInfo, pAllocator, &swapchain_) != VK_SUCCESS) {
      // Creating swapchain failed
      swapchain_ = 0;
      return;
    }
  }

  uint32_t num_base_images = 0;
  device_functions_->vkGetSwapchainImagesKHR(device, swapchain_,
                                             &num_base_images, nullptr);
  images_.resize(num_base_images);
  device_functions_->vkGetSwapchainImagesKHR(device, swapchain_,
                                             &num_base_images, images_.data());

  // Create a command buffer for each virtual image to blit from
  VkCommandBufferAllocateInfo command_buffer_info = {
      VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,  // sType
      nullptr,                                         // pNext
      command_pool,                                    // commandPool
      VK_COMMAND_BUFFER_LEVEL_PRIMARY,                 // level
      num_images,                                      // commandBufferCount
  };
  command_buffers_.resize(num_images);
  if (device_functions_->vkAllocateCommandBuffers(device, &command_buffer_info,
                                                  command_buffers_.data()) !=
      VK_SUCCESS) {
    return;
  }
  for (VkCommandBuffer cmdbuf : command_buffers_) {
    set_dispatch_from_parent(cmdbuf, device);
  }

  if (createSemaphore(device_functions_, device_, pAllocator,
                      &acquire_semaphore_) != VK_SUCCESS) {
    return;
  }
  if (createSemaphores(device_functions_, device_, pAllocator, num_images,
                       blit_semaphores_) != VK_SUCCESS) {
    return;
  }
  if (createSemaphores(device_functions_, device_, pAllocator, num_images,
                       present_semaphores_) != VK_SUCCESS) {
    return;
  }

  is_pending_.resize(num_images);
}

void BaseSwapchain::Destroy(const VkAllocationCallbacks *pAllocator) {
  device_functions_->vkDestroySemaphore(device_, acquire_semaphore_,
                                        pAllocator);
  acquire_semaphore_ = VK_NULL_HANDLE;

  destroySemaphores(device_functions_, device_, pAllocator, blit_semaphores_);
  destroySemaphores(device_functions_, device_, pAllocator,
                    present_semaphores_);

  device_functions_->vkDestroySwapchainKHR(device_, swapchain_, pAllocator);
  swapchain_ = VK_NULL_HANDLE;

  instance_functions_->vkDestroySurfaceKHR(instance_, surface_, pAllocator);
  surface_ = VK_NULL_HANDLE;

  images_.clear();

  is_pending_.clear();
  command_buffers_.clear();
}

VkResult BaseSwapchain::PresentFrom(VkQueue queue, size_t index,
                                    VkImage image) {
  std::unique_lock<threading::mutex> guard(present_lock_);
  VkResult res;
  uint32_t base_index = 0;
  // TODO: the error return values here aren't necessarily valid return values
  // for VkQueueSubmit
  if ((res = device_functions_->vkAcquireNextImageKHR(
           device_, swapchain_, 0, acquire_semaphore_, VK_NULL_HANDLE,
           &base_index)) != VK_SUCCESS) {
    return res;
  }

  VkCommandBuffer cmdbuf = command_buffers_[index];
  if ((res = device_functions_->vkResetCommandBuffer(cmdbuf, 0)) !=
      VK_SUCCESS) {
    return res;
  }
  VkCommandBufferBeginInfo beginInfo = {
      VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,  // sType
      0,                                            // pNext
      VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,  // flags
      nullptr,                                      // pInheritanceInfo
  };
  if ((res = device_functions_->vkBeginCommandBuffer(cmdbuf, &beginInfo)) !=
      VK_SUCCESS) {
    return res;
  }

  // The source image is already in VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, we
  // need to transition our image between VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
  // and VK_IMAGE_LAYOUT_PRESENT_SRC_KHR
  VkImageMemoryBarrier initialBarrier = {
      VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,  // sType
      nullptr,                                 // pNext
      0,                                       // srcAccessFlags
      VK_ACCESS_TRANSFER_WRITE_BIT,            // dstAccessFlags
      VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,         // oldLayout
      VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,    // newLayout
      VK_QUEUE_FAMILY_IGNORED,                 // srcQueueFamilyIndex
      VK_QUEUE_FAMILY_IGNORED,                 // dstQueueFamilyIndex
      images_[base_index],                     // image
      VkImageSubresourceRange{
          VK_IMAGE_ASPECT_COLOR_BIT,         // aspectMask
          0,                                 // baseMipLevel
          1,                                 // levelCount
          0,                                 // baseArrayLayer
          swapchain_info_.imageArrayLayers,  // layerCount
      },                                     // subresourceRange
  };

  device_functions_->vkCmdPipelineBarrier(
      cmdbuf,
      0,                               // srcStageMask
      VK_PIPELINE_STAGE_TRANSFER_BIT,  // dstStageMask
      0,                               // dependencyFlags
      0,                               // memoryBarrierCount
      nullptr,                         // pMemoryBarriers
      0,                               // bufferMemoryBarrierCount
      nullptr,                         // pBufferMemoryBarriers
      1,                               // imageMemoryBarrierCount
      &initialBarrier                  // pImageMemoryBarriers
  );

  VkImageSubresourceLayers subresource = {
      VK_IMAGE_ASPECT_COLOR_BIT,         // aspectMask
      0,                                 // mipLevel
      0,                                 // baseArrayLayer
      swapchain_info_.imageArrayLayers,  // layerCount
  };
  VkOffset3D offsets[2] = {
      {
          0,
          0,
          0,
      },
      {
          (int32_t)swapchain_info_.imageExtent.width,
          (int32_t)swapchain_info_.imageExtent.height,
          1,
      },
  };
  VkImageBlit blit = {
      subresource,
      {offsets[0], offsets[1]},
      subresource,
      {offsets[0], offsets[1]},
  };
  device_functions_->vkCmdBlitImage(
      cmdbuf,
      image,                                 // srcImage
      VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,  // srcImageLayout
      images_[base_index],                   // dstImage
      VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,  // dstImageLayout
      1,                                     // regionCount
      &blit,                                 // pRegions
      VK_FILTER_NEAREST                      // filter
  );

  VkImageMemoryBarrier finalBarrier = initialBarrier;
  finalBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
  finalBarrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
  finalBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
  finalBarrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR;

  device_functions_->vkCmdPipelineBarrier(
      cmdbuf,
      VK_PIPELINE_STAGE_TRANSFER_BIT,  // srcStageMask
      0,                               // dstStageMask
      0,                               // dependencyFlags
      0,                               // memoryBarrierCount
      nullptr,                         // pMemoryBarriers
      0,                               // bufferMemoryBarrierCount
      nullptr,                         // pBufferMemoryBarriers
      1,                               // imageMemoryBarrierCount
      &finalBarrier                    // pImageMemoryBarriers
  );

  if ((res = device_functions_->vkEndCommandBuffer(cmdbuf)) != VK_SUCCESS) {
    return res;
  }

  VkSemaphore signal_semaphores[] = {blit_semaphores_[index],
                                     present_semaphores_[index]};
  VkPipelineStageFlags waitStage = VK_PIPELINE_STAGE_TRANSFER_BIT;
  VkSubmitInfo submitInfo = {
      VK_STRUCTURE_TYPE_SUBMIT_INFO,  // sType
      0,                              // pNext
      1,                              // waitSemaphoreCount
      &acquire_semaphore_,            // pWaitSemaphores
      &waitStage,                     // pWaitDstStageMask
      1,                              // commandBufferCount
      &cmdbuf,                        // pCommandBuffers
      2,                              // signalSemaphoreCount
      signal_semaphores,              // pSignalSemaphores
  };
  auto queue_functions = GetGlobalContext().GetQueueData(queue);
  if ((res = queue_functions->vkQueueSubmit(queue, 1, &submitInfo,
                                            VK_NULL_HANDLE)) != VK_SUCCESS) {
    return res;
  }
  is_pending_[index] = true;

  VkPresentInfoKHR presentInfo = {
      VK_STRUCTURE_TYPE_PRESENT_INFO_KHR,  // sType
      0,                                   // pNext
      1,                                   // waitSemaphoreCount
      &present_semaphores_[index],         // waitSemaphores
      1,                                   // swapchainCount
      &swapchain_,                         // pSwapchains,
      &base_index,                         // pImageIndices
      nullptr,                             // pResults
  };
  if ((res = queue_functions->vkQueuePresentKHR(queue, &presentInfo)) !=
      VK_SUCCESS) {
    return res;
  }
  return VK_SUCCESS;
}

VkSemaphore BaseSwapchain::BlitWaitSemaphore(size_t index) {
  if (!is_pending_[index]) {
    return VK_NULL_HANDLE;
  }
  is_pending_[index] = false;
  return blit_semaphores_[index];
}
}  // namespace swapchain
<|endoftext|>


<commit_before><commit_msg>SingleSelectQueryComposer: the name of a non-SELECT column is its "real" name<commit_after><|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License

#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <unistd.h>

#include <string>

#include <glog/logging.h>

#include <process/future.hpp>
#include <process/reap.hpp>
#include <process/subprocess.hpp>

#include <stout/error.hpp>
#include <stout/lambda.hpp>
#include <stout/foreach.hpp>
#include <stout/option.hpp>
#include <stout/os.hpp>
#include <stout/os/strerror.hpp>
#include <stout/strings.hpp>
#include <stout/try.hpp>

using std::map;
using std::string;
using std::vector;

namespace process {

using InputFileDescriptors = Subprocess::IO::InputFileDescriptors;
using OutputFileDescriptors = Subprocess::IO::OutputFileDescriptors;

Subprocess::IO Subprocess::PIPE()
{
  return Subprocess::IO(
      []() -> Try<InputFileDescriptors> {
        int pipefd[2];
        if (::pipe(pipefd) == -1) {
          return ErrnoError("Failed to create pipe");
        }

        InputFileDescriptors fds;
        fds.read = pipefd[0];
        fds.write = pipefd[1];
        return fds;
      },
      []() -> Try<OutputFileDescriptors> {
        int pipefd[2];
        if (::pipe(pipefd) == -1) {
          return ErrnoError("Failed to create pipe");
        }

        OutputFileDescriptors fds;
        fds.read = pipefd[0];
        fds.write = pipefd[1];
        return fds;
      });
}

Subprocess::IO Subprocess::PATH(const string& path)
{
  return Subprocess::IO(
      [path]() -> Try<InputFileDescriptors> {
        Try<int> open = os::open(path, O_RDONLY | O_CLOEXEC);
        if (open.isError()) {
          return Error("Failed to open '" + path + "': " + open.error());
        }

        InputFileDescriptors fds;
        fds.read = open.get();
        return fds;
      },
      [path]() -> Try<OutputFileDescriptors> {
        Try<int> open = os::open(
            path,
            O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC,
            S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);

        if (open.isError()) {
          return Error("Failed to open '" + path + "': " + open.error());
        }

        OutputFileDescriptors fds;
        fds.write = open.get();
        return fds;
      });
}

}  // namespace process {
<commit_msg>Updated the ::pipe() system calls to pipe2 in posix subprocess.<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License

#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <unistd.h>

#include <string>

#include <glog/logging.h>

#include <process/future.hpp>
#include <process/reap.hpp>
#include <process/subprocess.hpp>

#include <stout/error.hpp>
#include <stout/lambda.hpp>
#include <stout/foreach.hpp>
#include <stout/option.hpp>
#include <stout/os.hpp>
#include <stout/os/pipe.hpp>
#include <stout/os/strerror.hpp>
#include <stout/strings.hpp>
#include <stout/try.hpp>

using std::array;
using std::map;
using std::string;
using std::vector;

namespace process {

using InputFileDescriptors = Subprocess::IO::InputFileDescriptors;
using OutputFileDescriptors = Subprocess::IO::OutputFileDescriptors;

Subprocess::IO Subprocess::PIPE()
{
  return Subprocess::IO(
      []() -> Try<InputFileDescriptors> {
        Try<array<int, 2>> pipefd = os::pipe();
        if (pipefd.isError()) {
          return Error(pipefd.error());
        }

        InputFileDescriptors fds;
        fds.read = pipefd->at(0);
        fds.write = pipefd->at(1);
        return fds;
      },
      []() -> Try<OutputFileDescriptors> {
        Try<array<int, 2>> pipefd = os::pipe();
        if (pipefd.isError()) {
          return Error(pipefd.error());
        }

        OutputFileDescriptors fds;
        fds.read = pipefd->at(0);
        fds.write = pipefd->at(1);
        return fds;
      });
}

Subprocess::IO Subprocess::PATH(const string& path)
{
  return Subprocess::IO(
      [path]() -> Try<InputFileDescriptors> {
        Try<int> open = os::open(path, O_RDONLY | O_CLOEXEC);
        if (open.isError()) {
          return Error("Failed to open '" + path + "': " + open.error());
        }

        InputFileDescriptors fds;
        fds.read = open.get();
        return fds;
      },
      [path]() -> Try<OutputFileDescriptors> {
        Try<int> open = os::open(
            path,
            O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC,
            S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);

        if (open.isError()) {
          return Error("Failed to open '" + path + "': " + open.error());
        }

        OutputFileDescriptors fds;
        fds.write = open.get();
        return fds;
      });
}

}  // namespace process {
<|endoftext|>


<commit_before><commit_msg>BZOJ 2629<commit_after><|endoftext|>


<commit_before>/* MicroFlo - Flow-Based Programming for microcontrollers
 * Copyright (c) 2016 Jon Nordby <jononor@gmail.com>
 * MicroFlo may be freely distributed under the MIT license
 */

// MQTT and MsgFlo.org support, at least for embedded Linux

#include "mosquitto.h"

#include <string>
#include <vector>

#include <stdlib.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <assert.h>
#include <err.h>

#ifdef DEBUG
#define LOG(...) do { printf(__VA_ARGS__); } while (0)
#else
#define LOG(...)
#endif

class MqttMount;

struct Port {
    MicroFlo::NodeId node;
    MicroFlo::PortId port;
    std::string topic;
    std::string name;

    Port(std::string t, MicroFlo::NodeId n, MicroFlo::PortId p, std::string na)
        : node(n)
        , port(p)
        , topic(t)
        , name(na)
    {

    }
};

std::string
toTopic(std::string role, std::string portName) {
    return "/" + role + "/" + portName;
}

struct ParticipantInfo {
    std::vector<Port> inports;
    std::vector<Port> outports;
    std::string role;
    std::string id;
    std::string component;

    ParticipantInfo *addInport(std::string name, MicroFlo::NodeId n, MicroFlo::PortId p) {
        Port port(toTopic(role, name), n, p, name);
        inports.push_back(port);
        return this;
    }
    ParticipantInfo *addOutport(std::string name, MicroFlo::NodeId n, MicroFlo::PortId p) {
        Port port(toTopic(role, name), n, p, name);
        outports.push_back(port);
        return this;
    }
};

#define JSON_ATTR_STRING(name, value) \
    +std::string("    \"") +name+std::string("\": \"") + value + std::string("\",\n")
#define JSON_ATTR_ARRAY(name, value) \
    +std::string("    \"") +name+std::string("\": [") + value + std::string("],\n")
#define JSON_ATTR_ENDNULL(name) \
    +std::string("    \"") +name+std::string("\": ") + "null" + std::string("\n")

std::string msgfloPorts(const std::vector<Port> &ports) {
    std::string str;
    // TODO: specify datatype on ports

    for (int i=0; i<(int)ports.size(); i++) {
        const Port &port = ports[i];
        str += "\n    {\n"
            JSON_ATTR_STRING("id", port.name)
            JSON_ATTR_STRING("queue", port.topic)
            JSON_ATTR_STRING("type", "any")
            JSON_ATTR_ENDNULL("_")
        + "    }";
        if (i < (int)ports.size()-1) {
            str =+ ",";
        }
    }
    return str;
}

std::string msgfloDiscoveryMessage(const ParticipantInfo *info) {
    // TODO: add label
    // TODO: add icon
    std::string inports = msgfloPorts(info->inports);
    std::string outports = msgfloPorts(info->outports);

    std::string msg = "{\n"
        JSON_ATTR_STRING("id", info->id)
        JSON_ATTR_STRING("role", info->role)
        JSON_ATTR_STRING("component", info->component)
        JSON_ATTR_ARRAY("inports", inports)
        JSON_ATTR_ARRAY("outports", outports)
        JSON_ATTR_ENDNULL("label")
    + "}";
    return msg;
}

struct MqttOptions {
    int brokerPort;
    char * brokerHostname;
    int keepaliveSeconds;
    char * clientId;
    ParticipantInfo info;
};

const Port *
findPortByTopic(const std::vector<Port> &ports, char *topic) {
    for (std::vector<Port>::const_iterator it = ports.begin() ; it != ports.end(); ++it) {
        const Port *port = &(*it);
        if (port->topic == std::string(topic)) {
            return port;
        }
    }
    return NULL;
}

const Port *
findPortByEdge(const std::vector<Port> &ports, MicroFlo::NodeId node, MicroFlo::PortId portId) {
    for (std::vector<Port>::const_iterator it = ports.begin() ; it != ports.end(); ++it) {
        const Port *port = &(*it);
        if (port->node == node && port->port == portId) {
            return port;
        }
    }
    return NULL;
}

// FIXME: write automated test
class MqttMount : public NetworkNotificationHandler {

public:
    static bool runForever(MqttMount *mount) {
        const int timeoutMs = 10;

        while(1){
            const int status = mosquitto_loop(mount->connection, timeoutMs, 1);
            if (status == MOSQ_ERR_CONN_LOST) {
                mount->connect();
            } else if (status == MOSQ_ERR_SUCCESS) {
                // Run MicroFlo network
                for (int i=0; i<20; i++) {
                    mount->network->runTick();
                }
            } else {
                LOG("mosquitto loop error: %s\n", mosquitto_strerror(status));
            }
        }
    }

public:
    MqttMount(Network *net, const MqttOptions &o)
        : network(net)
        , options(o)
        , connection(NULL)
    {
        network->setNotificationHandler(this);
    }

    ~MqttMount() {
        mosquitto_destroy(this->connection);
        this->connection = NULL;
        (void)mosquitto_lib_cleanup();
    }

    bool connect() {
        struct mosquitto *m = mosquitto_new(options.clientId, true, this);
        this->connection = m;

        mosquitto_connect_callback_set(m, on_connect);
        mosquitto_publish_callback_set(m, on_publish);
        mosquitto_subscribe_callback_set(m, on_subscribe);
        mosquitto_message_callback_set(m, on_message);

        const int res = mosquitto_connect(m, options.brokerHostname, options.brokerPort,
                                          options.keepaliveSeconds);
        return res == MOSQ_ERR_SUCCESS;
    }
    void disconnect() {} // TODO: implement

public:
    // Not really public, used by C trampolines
    void onConnect(int status) {
        const bool connected = status == 0;
        if (connected) {
            subscribePorts();
            sendDiscovery();
        }
    }

    void onMessage(const struct mosquitto_message *msg) {

        const Port *port = findPortByTopic(options.info.inports, msg->topic);
        if (port) {
            LOG("processing, sending to %d %d \n", port->node, port->port);

            // FIXME: parse out data from input message
            Packet pkg = Packet((long)msg->payloadlen);
            network->sendMessageTo(port->node, port->port, pkg);

            LOG("processing done\n");
        } else {
            LOG("Failed to find port for MQTT topic: %s", msg->topic);
        }
    }

    // implements NetworkNotificationHandler
    virtual void nodeAdded(Component *c, MicroFlo::NodeId parentId) {}
    virtual void nodesConnected(Component *src, MicroFlo::PortId srcPort,
                                Component *target, MicroFlo::PortId targetPort) {}
    virtual void networkStateChanged(Network::State s) {}
    virtual void emitDebug(DebugLevel level, DebugId id) {}
    virtual void debugChanged(DebugLevel level) {}
    virtual void portSubscriptionChanged(MicroFlo::NodeId nodeId, MicroFlo::PortId portId, bool enable) {}
    virtual void subgraphConnected(bool isOutput, MicroFlo::NodeId subgraphNode,
                                   MicroFlo::PortId subgraphPort, MicroFlo::NodeId childNode, MicroFlo::PortId childPort) {}

    virtual void packetSent(const Message &m, const Component *sender, MicroFlo::PortId senderPort) {
        const MicroFlo::NodeId senderId = sender->id();
        LOG("packet sent %d\n", senderId);

        const Port * port = findPortByEdge(options.info.outports, senderId, senderPort);
        if (port) {
            const char *outTopic = port->topic.c_str();
            LOG("sending on MQTT topic %s\n", outTopic);

            // FIXME: determine sane serialization
            size_t payload_sz = 32;
            char payload[payload_sz];
            size_t payloadlen = 0;
            payloadlen = snprintf(payload, payload_sz, "tock %d", (int)m.pkg.asInteger());
            if (payload_sz < payloadlen) {
                //die("snprintf\n");
            }

            const int res = mosquitto_publish(this->connection, NULL, outTopic, payloadlen, payload, 0, false);
            if (res != MOSQ_ERR_SUCCESS) {
                //die("publish\n");
            }
        } else {
            LOG("no MQTT topic associated\n");
        }
    }

private:
    void subscribePorts() {
        for (std::vector<Port>::iterator it = options.info.inports.begin() ; it != options.info.inports.end(); ++it) {
            const Port &port = *it;
            const char *pattern = port.topic.c_str();
            mosquitto_subscribe(this->connection, NULL, pattern, 0);
            LOG("subscribed inport to MQTT topic: %s\n", pattern);
        }
        for (std::vector<Port>::iterator it = options.info.outports.begin() ; it != options.info.outports.end(); ++it) {
            const Port &port = *it;
            network->subscribeToPort(port.node, port.port, true);
            LOG("setup outport to MQTT topic: %s\n", port.topic.c_str());
        }
    }

    void sendDiscovery() {
        const std::string msgfloDiscoveryTopic = "fbp";
        publish(msgfloDiscoveryTopic, msgfloDiscoveryMessage(&options.info));
        LOG("sent MsgFlo discovery message\n");
    }

    void publish(std::string topic, std::string payload) {
        const int res = mosquitto_publish(this->connection, NULL,
                                          topic.c_str(), payload.size(), payload.c_str(), 0, false);
        if (res != MOSQ_ERR_SUCCESS) {
            //die("publish\n");
        }
    }

private:
    // mosquitto callback trampolines
    static void on_connect(struct mosquitto *m, void *udata, int res) {
        MqttMount *self = (MqttMount *)udata;
        self->onConnect(res);
    }

    static void on_message(struct mosquitto *m, void *udata,
                           const struct mosquitto_message *msg) {
        if (msg == NULL) {
            return;
        }
        LOG("-- got message @ %s: (%d, QoS %d, %s) '%s'\n",
            msg->topic, msg->payloadlen, msg->qos, msg->retain ? "R" : "!r", (char *)msg->payload);
        MqttMount *self = (MqttMount *)udata;
        self->onMessage(msg);
    }

    static void on_subscribe(struct mosquitto *m, void *udata, int mid,
                             int qos_count, const int *granted_qos) {
        LOG("-- subscribed successfully\n");
    }

    static void on_publish(struct mosquitto *m, void *udata, int m_id) {
        LOG("-- published successfully\n");
    }

private:
    Network *network;
    MqttOptions options;
    struct mosquitto *connection;
};

bool parse_brokerurl(MqttOptions *options, const char *url) {
    if (!url) {
        return true;
    }

    int *ip = &options->brokerPort;
    char *host = options->brokerHostname;
    if (sscanf(url, "mqtt://%99[^:]:%i[^\n]", host, ip) == 2) {
        return true;
    } else if (sscanf(url, "mqtt://%99[^\n]", host) == 1) {
        return true;
    } else {
        return false;
    }
}

bool mqttParseOptions(MqttOptions *options, int argc, char **argv) {

    // defaults
    options->brokerHostname = strndup("localhost", 99);
    options->brokerPort = 1883;
    options->keepaliveSeconds = 60;
    options->clientId = NULL; // MQTT will autogenerate
    options->info.role = "micro";
    options->info.component = "microflo/Component"; // FIXME: take from graph

    if (argc > 1) {
        options->info.role = std::string(argv[1]);
    }

    options->info.id = options->info.role + std::to_string(rand());

    char* broker = getenv("MSGFLO_BROKER");
    return parse_brokerurl(options, broker);
}
<commit_msg>mqtt: Always return value from runForever()<commit_after>/* MicroFlo - Flow-Based Programming for microcontrollers
 * Copyright (c) 2016 Jon Nordby <jononor@gmail.com>
 * MicroFlo may be freely distributed under the MIT license
 */

// MQTT and MsgFlo.org support, at least for embedded Linux

#include "mosquitto.h"

#include <string>
#include <vector>

#include <stdlib.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <assert.h>
#include <err.h>

#ifdef DEBUG
#define LOG(...) do { printf(__VA_ARGS__); } while (0)
#else
#define LOG(...)
#endif

class MqttMount;

struct Port {
    MicroFlo::NodeId node;
    MicroFlo::PortId port;
    std::string topic;
    std::string name;

    Port(std::string t, MicroFlo::NodeId n, MicroFlo::PortId p, std::string na)
        : node(n)
        , port(p)
        , topic(t)
        , name(na)
    {

    }
};

std::string
toTopic(std::string role, std::string portName) {
    return "/" + role + "/" + portName;
}

struct ParticipantInfo {
    std::vector<Port> inports;
    std::vector<Port> outports;
    std::string role;
    std::string id;
    std::string component;

    ParticipantInfo *addInport(std::string name, MicroFlo::NodeId n, MicroFlo::PortId p) {
        Port port(toTopic(role, name), n, p, name);
        inports.push_back(port);
        return this;
    }
    ParticipantInfo *addOutport(std::string name, MicroFlo::NodeId n, MicroFlo::PortId p) {
        Port port(toTopic(role, name), n, p, name);
        outports.push_back(port);
        return this;
    }
};

#define JSON_ATTR_STRING(name, value) \
    +std::string("    \"") +name+std::string("\": \"") + value + std::string("\",\n")
#define JSON_ATTR_ARRAY(name, value) \
    +std::string("    \"") +name+std::string("\": [") + value + std::string("],\n")
#define JSON_ATTR_ENDNULL(name) \
    +std::string("    \"") +name+std::string("\": ") + "null" + std::string("\n")

std::string msgfloPorts(const std::vector<Port> &ports) {
    std::string str;
    // TODO: specify datatype on ports

    for (int i=0; i<(int)ports.size(); i++) {
        const Port &port = ports[i];
        str += "\n    {\n"
            JSON_ATTR_STRING("id", port.name)
            JSON_ATTR_STRING("queue", port.topic)
            JSON_ATTR_STRING("type", "any")
            JSON_ATTR_ENDNULL("_")
        + "    }";
        if (i < (int)ports.size()-1) {
            str =+ ",";
        }
    }
    return str;
}

std::string msgfloDiscoveryMessage(const ParticipantInfo *info) {
    // TODO: add label
    // TODO: add icon
    std::string inports = msgfloPorts(info->inports);
    std::string outports = msgfloPorts(info->outports);

    std::string msg = "{\n"
        JSON_ATTR_STRING("id", info->id)
        JSON_ATTR_STRING("role", info->role)
        JSON_ATTR_STRING("component", info->component)
        JSON_ATTR_ARRAY("inports", inports)
        JSON_ATTR_ARRAY("outports", outports)
        JSON_ATTR_ENDNULL("label")
    + "}";
    return msg;
}

struct MqttOptions {
    int brokerPort;
    char * brokerHostname;
    int keepaliveSeconds;
    char * clientId;
    ParticipantInfo info;
};

const Port *
findPortByTopic(const std::vector<Port> &ports, char *topic) {
    for (std::vector<Port>::const_iterator it = ports.begin() ; it != ports.end(); ++it) {
        const Port *port = &(*it);
        if (port->topic == std::string(topic)) {
            return port;
        }
    }
    return NULL;
}

const Port *
findPortByEdge(const std::vector<Port> &ports, MicroFlo::NodeId node, MicroFlo::PortId portId) {
    for (std::vector<Port>::const_iterator it = ports.begin() ; it != ports.end(); ++it) {
        const Port *port = &(*it);
        if (port->node == node && port->port == portId) {
            return port;
        }
    }
    return NULL;
}

// FIXME: write automated test
class MqttMount : public NetworkNotificationHandler {

public:
    static bool runForever(MqttMount *mount) {
        const int timeoutMs = 10;

        while(1){
            const int status = mosquitto_loop(mount->connection, timeoutMs, 1);
            if (status == MOSQ_ERR_CONN_LOST) {
                mount->connect();
            } else if (status == MOSQ_ERR_SUCCESS) {
                // Run MicroFlo network
                for (int i=0; i<20; i++) {
                    mount->network->runTick();
                }
            } else {
                LOG("mosquitto loop error: %s\n", mosquitto_strerror(status));
                return false;
            }
        }
        return true;
    }

public:
    MqttMount(Network *net, const MqttOptions &o)
        : network(net)
        , options(o)
        , connection(NULL)
    {
        network->setNotificationHandler(this);
    }

    ~MqttMount() {
        mosquitto_destroy(this->connection);
        this->connection = NULL;
        (void)mosquitto_lib_cleanup();
    }

    bool connect() {
        struct mosquitto *m = mosquitto_new(options.clientId, true, this);
        this->connection = m;

        mosquitto_connect_callback_set(m, on_connect);
        mosquitto_publish_callback_set(m, on_publish);
        mosquitto_subscribe_callback_set(m, on_subscribe);
        mosquitto_message_callback_set(m, on_message);

        const int res = mosquitto_connect(m, options.brokerHostname, options.brokerPort,
                                          options.keepaliveSeconds);
        return res == MOSQ_ERR_SUCCESS;
    }
    void disconnect() {} // TODO: implement

public:
    // Not really public, used by C trampolines
    void onConnect(int status) {
        const bool connected = status == 0;
        if (connected) {
            subscribePorts();
            sendDiscovery();
        }
    }

    void onMessage(const struct mosquitto_message *msg) {

        const Port *port = findPortByTopic(options.info.inports, msg->topic);
        if (port) {
            LOG("processing, sending to %d %d \n", port->node, port->port);

            // FIXME: parse out data from input message
            Packet pkg = Packet((long)msg->payloadlen);
            network->sendMessageTo(port->node, port->port, pkg);

            LOG("processing done\n");
        } else {
            LOG("Failed to find port for MQTT topic: %s", msg->topic);
        }
    }

    // implements NetworkNotificationHandler
    virtual void nodeAdded(Component *c, MicroFlo::NodeId parentId) {}
    virtual void nodesConnected(Component *src, MicroFlo::PortId srcPort,
                                Component *target, MicroFlo::PortId targetPort) {}
    virtual void networkStateChanged(Network::State s) {}
    virtual void emitDebug(DebugLevel level, DebugId id) {}
    virtual void debugChanged(DebugLevel level) {}
    virtual void portSubscriptionChanged(MicroFlo::NodeId nodeId, MicroFlo::PortId portId, bool enable) {}
    virtual void subgraphConnected(bool isOutput, MicroFlo::NodeId subgraphNode,
                                   MicroFlo::PortId subgraphPort, MicroFlo::NodeId childNode, MicroFlo::PortId childPort) {}

    virtual void packetSent(const Message &m, const Component *sender, MicroFlo::PortId senderPort) {
        const MicroFlo::NodeId senderId = sender->id();
        LOG("packet sent %d\n", senderId);

        const Port * port = findPortByEdge(options.info.outports, senderId, senderPort);
        if (port) {
            const char *outTopic = port->topic.c_str();
            LOG("sending on MQTT topic %s\n", outTopic);

            // FIXME: determine sane serialization
            size_t payload_sz = 32;
            char payload[payload_sz];
            size_t payloadlen = 0;
            payloadlen = snprintf(payload, payload_sz, "tock %d", (int)m.pkg.asInteger());
            if (payload_sz < payloadlen) {
                //die("snprintf\n");
            }

            const int res = mosquitto_publish(this->connection, NULL, outTopic, payloadlen, payload, 0, false);
            if (res != MOSQ_ERR_SUCCESS) {
                //die("publish\n");
            }
        } else {
            LOG("no MQTT topic associated\n");
        }
    }

private:
    void subscribePorts() {
        for (std::vector<Port>::iterator it = options.info.inports.begin() ; it != options.info.inports.end(); ++it) {
            const Port &port = *it;
            const char *pattern = port.topic.c_str();
            mosquitto_subscribe(this->connection, NULL, pattern, 0);
            LOG("subscribed inport to MQTT topic: %s\n", pattern);
        }
        for (std::vector<Port>::iterator it = options.info.outports.begin() ; it != options.info.outports.end(); ++it) {
            const Port &port = *it;
            network->subscribeToPort(port.node, port.port, true);
            LOG("setup outport to MQTT topic: %s\n", port.topic.c_str());
        }
    }

    void sendDiscovery() {
        const std::string msgfloDiscoveryTopic = "fbp";
        publish(msgfloDiscoveryTopic, msgfloDiscoveryMessage(&options.info));
        LOG("sent MsgFlo discovery message\n");
    }

    void publish(std::string topic, std::string payload) {
        const int res = mosquitto_publish(this->connection, NULL,
                                          topic.c_str(), payload.size(), payload.c_str(), 0, false);
        if (res != MOSQ_ERR_SUCCESS) {
            //die("publish\n");
        }
    }

private:
    // mosquitto callback trampolines
    static void on_connect(struct mosquitto *m, void *udata, int res) {
        MqttMount *self = (MqttMount *)udata;
        self->onConnect(res);
    }

    static void on_message(struct mosquitto *m, void *udata,
                           const struct mosquitto_message *msg) {
        if (msg == NULL) {
            return;
        }
        LOG("-- got message @ %s: (%d, QoS %d, %s) '%s'\n",
            msg->topic, msg->payloadlen, msg->qos, msg->retain ? "R" : "!r", (char *)msg->payload);
        MqttMount *self = (MqttMount *)udata;
        self->onMessage(msg);
    }

    static void on_subscribe(struct mosquitto *m, void *udata, int mid,
                             int qos_count, const int *granted_qos) {
        LOG("-- subscribed successfully\n");
    }

    static void on_publish(struct mosquitto *m, void *udata, int m_id) {
        LOG("-- published successfully\n");
    }

private:
    Network *network;
    MqttOptions options;
    struct mosquitto *connection;
};

bool parse_brokerurl(MqttOptions *options, const char *url) {
    if (!url) {
        return true;
    }

    int *ip = &options->brokerPort;
    char *host = options->brokerHostname;
    if (sscanf(url, "mqtt://%99[^:]:%i[^\n]", host, ip) == 2) {
        return true;
    } else if (sscanf(url, "mqtt://%99[^\n]", host) == 1) {
        return true;
    } else {
        return false;
    }
}

bool mqttParseOptions(MqttOptions *options, int argc, char **argv) {

    // defaults
    options->brokerHostname = strndup("localhost", 99);
    options->brokerPort = 1883;
    options->keepaliveSeconds = 60;
    options->clientId = NULL; // MQTT will autogenerate
    options->info.role = "micro";
    options->info.component = "microflo/Component"; // FIXME: take from graph

    if (argc > 1) {
        options->info.role = std::string(argv[1]);
    }

    options->info.id = options->info.role + std::to_string(rand());

    char* broker = getenv("MSGFLO_BROKER");
    return parse_brokerurl(options, broker);
}
<|endoftext|>


<commit_before>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <chartsql/CSTableScan.h>
#include <chartsql/qtree/FieldReferenceNode.h>
#include <chartsql/runtime/defaultruntime.h>
#include <chartsql/runtime/compile.h>
#include <fnord/inspect.h>

using namespace fnord;

namespace csql {

CSTableScan::CSTableScan(
    RefPtr<SelectProjectAggregateNode> stmt,
    cstable::CSTableReader&& cstable) :
    cstable_(std::move(cstable)),
    colindex_(0),
    flat_(false) {

  Set<String> column_names;
  for (const auto& expr : stmt->selectList()) {
    findColumns(expr, &column_names);
  }

  for (const auto& col : column_names) {
    columns_.emplace(col, ColumnRef(cstable_.getColumnReader(col), colindex_++));
  }

  for (auto& expr : stmt->selectList()) {
    resolveColumns(expr);
  }

  //fnord::iputs("all cols: $0", column_names);

  DefaultRuntime runtime;

  for (const auto& expr : stmt->selectList()) {
    select_list_.emplace_back(
        findMaxRepetitionLevel(expr),
        runtime.compiler()->compile(expr),
        &scratch_);
  }
}

void CSTableScan::execute(Function<bool (int argc, const SValue* argv)> fn) {
  uint64_t select_level = 0;
  uint64_t fetch_level = 0;

  Vector<SValue> in_row(colindex_, SValue{});
  Vector<SValue> out_row(select_list_.size(), SValue{});

  size_t num_records = 0;
  size_t total_records = cstable_.numRecords();
  while (num_records < total_records) {
    uint64_t next_level = 0;

    if (fetch_level == 0) {
      ++num_records;
    }

    for (auto& col : columns_) {
      if (col.second.reader->nextRepetitionLevel() >= fetch_level) {
        auto& reader = col.second.reader;

        uint64_t r;
        uint64_t d;
        void* data;
        size_t size;
        reader->next(&r, &d, &data, &size);

        switch (col.second.reader->type()) {

          case msg::FieldType::STRING:
            in_row[col.second.index] =
                SValue(SValue::StringType((char*) data, size));
            break;

          case msg::FieldType::UINT32:
          case msg::FieldType::UINT64:
            switch (size) {
              case sizeof(uint32_t):
                in_row[col.second.index] =
                    SValue(SValue::IntegerType(*((uint32_t*) data)));
                break;
              case sizeof(uint64_t):
                in_row[col.second.index] =
                    SValue(SValue::IntegerType(*((uint64_t*) data)));
                break;
              case 0:
                in_row[col.second.index] = SValue(SValue::IntegerType(0));
                break;
            }
            break;

          case msg::FieldType::BOOLEAN:
            switch (size) {
              case sizeof(uint32_t):
                in_row[col.second.index] =
                    SValue(SValue::BoolType(*((uint32_t*) data) > 0));
                break;
              case sizeof(uint64_t):
                in_row[col.second.index] =
                    SValue(SValue::BoolType(*((uint64_t*) data) > 0));
                break;
              case 0:
                in_row[col.second.index] = SValue(SValue::BoolType(false));
                break;
            }
            break;

          case msg::FieldType::OBJECT:
            RAISE(kIllegalStateError);

        }

      }

      next_level = std::max(next_level, col.second.reader->nextRepetitionLevel());
    }

    fetch_level = next_level;

    if (true) { // where clause
      for (int i = 0; i < select_list_.size(); ++i) {
        if (select_list_[i].rep_level >= select_level) {
          //select_level_[i].compiled.accumulate();
        }
      }

      if (!flat_ || next_level == 0) {
        for (int i = 0; i < select_list_.size(); ++i) {
          select_list_[i].compiled->evaluate(
              &select_list_[i].instance,
              in_row.size(),
              in_row.data(),
              &out_row[i]);
        }
      }

      if (!fn(out_row.size(), out_row.data())) {
        return;
      }

      select_level = fetch_level;
    } else {
      select_level = std::min(select_level, fetch_level);
    }
  }
}

void CSTableScan::findColumns(
    RefPtr<ScalarExpressionNode> expr,
    Set<String>* column_names) const {
  auto fieldref = dynamic_cast<FieldReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    column_names->emplace(fieldref->fieldName());
  }

  for (const auto& e : expr->arguments()) {
    findColumns(e, column_names);
  }
}

void CSTableScan::resolveColumns(RefPtr<ScalarExpressionNode> expr) const {
  auto fieldref = dynamic_cast<FieldReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      RAISE(kIllegalStateError);
    }

    fieldref->setColumnIndex(col->second.index);
  }

  for (const auto& e : expr->arguments()) {
    resolveColumns(e);
  }
}

uint64_t CSTableScan::findMaxRepetitionLevel(
    RefPtr<ScalarExpressionNode> expr) const {
  uint64_t max_level = 0;

  auto fieldref = dynamic_cast<FieldReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      RAISE(kIllegalStateError);
    }

    auto col_level = col->second.reader->maxRepetitionLevel();
    if (col_level > max_level) {
      max_level = col_level;
    }
  }

  for (const auto& e : expr->arguments()) {
    auto e_level = findMaxRepetitionLevel(e);
    if (e_level > max_level) {
      max_level = e_level;
    }
  }

  return max_level;
}

CSTableScan::ColumnRef::ColumnRef(
    RefPtr<cstable::ColumnReader> r,
    size_t i) :
    reader(r),
    index(i) {}

CSTableScan::ExpressionRef::ExpressionRef(
    size_t _rep_level,
    ScopedPtr<CompiledProgram> _compiled,
    ScratchMemory* smem) :
    rep_level(_rep_level),
    compiled(std::move(_compiled)),
    instance(compiled->allocInstance(smem)) {}

CSTableScan::ExpressionRef::ExpressionRef(
    ExpressionRef&& other) :
    rep_level(other.rep_level),
    compiled(std::move(other.compiled)),
    instance(other.instance) {
  other.instance.scratch = nullptr;
}

CSTableScan::ExpressionRef::~ExpressionRef() {
  if (instance.scratch) {
    compiled->freeInstance(&instance);
  }
}

} // namespace csql
<commit_msg>call accumulate/reset<commit_after>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <chartsql/CSTableScan.h>
#include <chartsql/qtree/FieldReferenceNode.h>
#include <chartsql/runtime/defaultruntime.h>
#include <chartsql/runtime/compile.h>
#include <fnord/inspect.h>

using namespace fnord;

namespace csql {

CSTableScan::CSTableScan(
    RefPtr<SelectProjectAggregateNode> stmt,
    cstable::CSTableReader&& cstable) :
    cstable_(std::move(cstable)),
    colindex_(0),
    flat_(false) {

  Set<String> column_names;
  for (const auto& expr : stmt->selectList()) {
    findColumns(expr, &column_names);
  }

  for (const auto& col : column_names) {
    columns_.emplace(col, ColumnRef(cstable_.getColumnReader(col), colindex_++));
  }

  for (auto& expr : stmt->selectList()) {
    resolveColumns(expr);
  }

  //fnord::iputs("all cols: $0", column_names);

  DefaultRuntime runtime;

  for (const auto& expr : stmt->selectList()) {
    select_list_.emplace_back(
        findMaxRepetitionLevel(expr),
        runtime.compiler()->compile(expr),
        &scratch_);
  }
}

void CSTableScan::execute(Function<bool (int argc, const SValue* argv)> fn) {
  uint64_t select_level = 0;
  uint64_t fetch_level = 0;

  Vector<SValue> in_row(colindex_, SValue{});
  Vector<SValue> out_row(select_list_.size(), SValue{});

  size_t num_records = 0;
  size_t total_records = cstable_.numRecords();
  while (num_records < total_records) {
    uint64_t next_level = 0;

    if (fetch_level == 0) {
      ++num_records;
    }

    for (auto& col : columns_) {
      if (col.second.reader->nextRepetitionLevel() >= fetch_level) {
        auto& reader = col.second.reader;

        uint64_t r;
        uint64_t d;
        void* data;
        size_t size;
        reader->next(&r, &d, &data, &size);

        switch (col.second.reader->type()) {

          case msg::FieldType::STRING:
            in_row[col.second.index] =
                SValue(SValue::StringType((char*) data, size));
            break;

          case msg::FieldType::UINT32:
          case msg::FieldType::UINT64:
            switch (size) {
              case sizeof(uint32_t):
                in_row[col.second.index] =
                    SValue(SValue::IntegerType(*((uint32_t*) data)));
                break;
              case sizeof(uint64_t):
                in_row[col.second.index] =
                    SValue(SValue::IntegerType(*((uint64_t*) data)));
                break;
              case 0:
                in_row[col.second.index] = SValue(SValue::IntegerType(0));
                break;
            }
            break;

          case msg::FieldType::BOOLEAN:
            switch (size) {
              case sizeof(uint32_t):
                in_row[col.second.index] =
                    SValue(SValue::BoolType(*((uint32_t*) data) > 0));
                break;
              case sizeof(uint64_t):
                in_row[col.second.index] =
                    SValue(SValue::BoolType(*((uint64_t*) data) > 0));
                break;
              case 0:
                in_row[col.second.index] = SValue(SValue::BoolType(false));
                break;
            }
            break;

          case msg::FieldType::OBJECT:
            RAISE(kIllegalStateError);

        }

      }

      next_level = std::max(next_level, col.second.reader->nextRepetitionLevel());
    }

    fetch_level = next_level;

    if (true) { // where clause
      for (int i = 0; i < select_list_.size(); ++i) {
        if (select_list_[i].rep_level >= select_level) {
          select_list_[i].compiled->accumulate(
              &select_list_[i].instance,
              in_row.size(),
              in_row.data());
        }
      }

      if (!flat_ || next_level == 0) {
        for (int i = 0; i < select_list_.size(); ++i) {
          select_list_[i].compiled->evaluate(
              &select_list_[i].instance,
              in_row.size(),
              in_row.data(),
              &out_row[i]);

          select_list_[i].compiled->reset(&select_list_[i].instance);
        }
      }

      if (!fn(out_row.size(), out_row.data())) {
        return;
      }

      select_level = fetch_level;
    } else {
      select_level = std::min(select_level, fetch_level);
    }
  }
}

void CSTableScan::findColumns(
    RefPtr<ScalarExpressionNode> expr,
    Set<String>* column_names) const {
  auto fieldref = dynamic_cast<FieldReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    column_names->emplace(fieldref->fieldName());
  }

  for (const auto& e : expr->arguments()) {
    findColumns(e, column_names);
  }
}

void CSTableScan::resolveColumns(RefPtr<ScalarExpressionNode> expr) const {
  auto fieldref = dynamic_cast<FieldReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      RAISE(kIllegalStateError);
    }

    fieldref->setColumnIndex(col->second.index);
  }

  for (const auto& e : expr->arguments()) {
    resolveColumns(e);
  }
}

uint64_t CSTableScan::findMaxRepetitionLevel(
    RefPtr<ScalarExpressionNode> expr) const {
  uint64_t max_level = 0;

  auto fieldref = dynamic_cast<FieldReferenceNode*>(expr.get());
  if (fieldref != nullptr) {
    auto col = columns_.find(fieldref->fieldName());
    if (col == columns_.end()) {
      RAISE(kIllegalStateError);
    }

    auto col_level = col->second.reader->maxRepetitionLevel();
    if (col_level > max_level) {
      max_level = col_level;
    }
  }

  for (const auto& e : expr->arguments()) {
    auto e_level = findMaxRepetitionLevel(e);
    if (e_level > max_level) {
      max_level = e_level;
    }
  }

  return max_level;
}

CSTableScan::ColumnRef::ColumnRef(
    RefPtr<cstable::ColumnReader> r,
    size_t i) :
    reader(r),
    index(i) {}

CSTableScan::ExpressionRef::ExpressionRef(
    size_t _rep_level,
    ScopedPtr<CompiledProgram> _compiled,
    ScratchMemory* smem) :
    rep_level(_rep_level),
    compiled(std::move(_compiled)),
    instance(compiled->allocInstance(smem)) {}

CSTableScan::ExpressionRef::ExpressionRef(
    ExpressionRef&& other) :
    rep_level(other.rep_level),
    compiled(std::move(other.compiled)),
    instance(other.instance) {
  other.instance.scratch = nullptr;
}

CSTableScan::ExpressionRef::~ExpressionRef() {
  if (instance.scratch) {
    compiled->freeInstance(&instance);
  }
}

} // namespace csql
<|endoftext|>


<commit_before>#pragma once

template <typename T>
void allocate2D(const int Nx,
                const int Ny,
                T**& ptr) {
  ptr = new T* [Nx];
  ptr[0] = new T[Nx * Ny];
  for (int i = 1; i < Nx; i++) ptr[i] = ptr[0] + i * Ny;
}

template <typename T>
void deallocate2D(T**& ptr) {
  delete [] ptr[0];
  delete [] ptr;
  ptr = nullptr;
}
<commit_msg>アラインメントを指定する関数を追加<commit_after>#pragma once

#include <iostream>
#include <cstdlib>

template <typename T, int alignment>
void allocate2D_aligend(const int Nx,
                        const int Ny,
                        T**& ptr) {
  ptr = static_cast<T**>(malloc(Nx * sizeof(T*)));
  const auto stat = posix_memalign(reinterpret_cast<void**>(ptr),
                                   alignment,
                                   sizeof(T) * Nx * Ny);
  if (stat) {
    std::cerr << "error occurs at allocate2D_aligend\n";
    std::exit(1);
  }
  for (int i = 1; i < Nx; i++) ptr[i] = ptr[0] + i * Ny;
}

template <typename T>
void deallocate2D_aligend(T**& ptr) {
  free(ptr[0]);
  free(ptr);
  ptr = nullptr;
}

template <typename T>
void allocate2D(const int Nx,
                const int Ny,
                T**& ptr) {
  ptr = new T* [Nx];
  ptr[0] = new T[Nx * Ny];
  for (int i = 1; i < Nx; i++) ptr[i] = ptr[0] + i * Ny;
}

template <typename T>
void deallocate2D(T**& ptr) {
  delete [] ptr[0];
  delete [] ptr;
  ptr = nullptr;
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "circuit.h"

#include "../benchmark_util.h"

BENCHMARK(circuit_parse) {
    Circuit c;
    benchmark_go([&]() {
        c = Circuit::from_text(R"input(
H 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
CNOT 4 5 6 7
M 1 2 3 4 5 6 7 8 9 10 11
            )input");
    }).goal_nanos(950);
    if (c.num_qubits == 0) {
        std::cerr << "impossible";
    }
}

BENCHMARK(circuit_parse_sparse) {
    Circuit c;
    for (auto k = 0; k < 1000; k++) {
        c.append_op("H", {0});
        c.append_op("CNOT", {1, 2});
        c.append_op("M", {0});
    }
    auto text = c.str();
    benchmark_go([&]() {
        c = Circuit::from_text(text);
    }).goal_micros(150);
    if (c.num_qubits == 0) {
        std::cerr << "impossible";
    }
}
<commit_msg>Restore CI checks<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "circuit.h"

#include "../benchmark_util.h"

BENCHMARK(circuit_parse) {
    Circuit c;
    benchmark_go([&]() {
        c = Circuit::from_text(R"input(
H 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
CNOT 4 5 6 7
M 1 2 3 4 5 6 7 8 9 10 11
            )input");
    }).goal_nanos(950);
    if (c.num_qubits == 0) {
        std::cerr << "impossible";
    }
}

BENCHMARK(circuit_parse_sparse) {
    Circuit c;
    for (auto k = 0; k < 1000; k++) {
        c.append_op("H", {0});
        c.append_op("CNOT", {1, 2});
        c.append_op("M", {0});
    }
    auto text = c.str();
    benchmark_go([&]() {
        c = Circuit::from_text(text.data());
    }).goal_micros(150);
    if (c.num_qubits == 0) {
        std::cerr << "impossible";
    }
}
<|endoftext|>