<commit_before>/*
 *  Copyright (c) 2014-present, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */

#include <gtest/gtest.h>

#include <boost/filesystem.hpp>

#include <osquery/core.h>

#include "osquery/filesystem/fileops.h"
#include "osquery/tests/test_util.h"

namespace fs = boost::filesystem;

namespace osquery {

class FileOpsTests : public testing::Test {
 protected:
  void SetUp() override {
    createMockFileStructure();
  }

  void TearDown() override {
    tearDownMockFileStructure();
  }

  bool globResultsMatch(const std::vector<std::string>& results,
                        std::vector<fs::path>& expected) {
    if (results.size() == expected.size()) {
      size_t i = 0;
      for (auto const& path : results) {
        if (path != expected[i].make_preferred().string()) {
          return false;
        }
        i++;
      }

      return true;
    }

    return false;
  }
};

class TempFile {
 public:
  TempFile()
      : path_((fs::temp_directory_path() / fs::unique_path())
                  .make_preferred()
                  .string()) {}

  ~TempFile() {
    if (fs::exists(path_)) {
      fs::remove(path_);
    }
  }

  const std::string& path() const {
    return path_;
  }

 private:
  std::string path_;
};

TEST_F(FileOpsTests, test_openFile) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_FALSE(fd.isValid());
  }

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_WRITE);
    EXPECT_TRUE(fd.isValid());
  }

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_READ);
    EXPECT_FALSE(fd.isValid());
  }

  fs::remove(path);

  {
    PlatformFile fd(path, PF_CREATE_ALWAYS | PF_READ);
    EXPECT_TRUE(fd.isValid());
  }

  {
    PlatformFile fd(path, PF_CREATE_ALWAYS | PF_READ);
    EXPECT_TRUE(fd.isValid());
  }

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(fd.isValid());
  }
}

/*
 * This is a special function for testing file share operations on Windows. Our
 * PlatformFile as of now will only set FILE_SHARE_READ to play nicely with log
 * reading tools. However, we need to create one with FILE_SHARE_READ and
 * FILE_SHARE_WRITE for testing.
 */
std::unique_ptr<PlatformFile> openRWSharedFile(const std::string& path,
                                               int mode) {
#ifdef WIN32
  DWORD access_mask = -1;
  DWORD creation_disposition = -1;

  if (mode == (PF_OPEN_EXISTING | PF_READ)) {
    access_mask = PF_READ;
    creation_disposition = OPEN_EXISTING;
  } else if (mode == (PF_OPEN_ALWAYS | PF_WRITE)) {
    access_mask = PF_WRITE;
    creation_disposition = OPEN_ALWAYS;
  }

  HANDLE handle = ::CreateFileA(path.c_str(),
                                access_mask,
                                FILE_SHARE_READ | FILE_SHARE_WRITE,
                                nullptr,
                                creation_disposition,
                                0,
                                nullptr);
  return std::unique_ptr<PlatformFile>(new PlatformFile(handle));
#else
  return std::unique_ptr<PlatformFile>(new PlatformFile(path, mode));
#endif
}

TEST_F(FileOpsTests, test_shareRead) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* test1_data = "AAAABBBB";
  const ssize_t test1_size = ::strlen(test1_data);

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_WRITE);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(test1_size, fd.write(test1_data, test1_size));
  }

  {
    auto reader_fd = openRWSharedFile(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(reader_fd->isValid());

    std::vector<char> buf;
    buf.assign(test1_size, '\0');

    EXPECT_EQ(test1_size, reader_fd->read(buf.data(), test1_size));
    EXPECT_EQ(test1_size, buf.size());

    for (ssize_t i = 0; i < test1_size; i++) {
      EXPECT_EQ(test1_data[i], buf[i]);
    }

    PlatformFile fd(path, PF_OPEN_ALWAYS | PF_WRITE | PF_APPEND);
    EXPECT_TRUE(fd.isValid());
  }
}

TEST_F(FileOpsTests, test_fileIo) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* expected_read = "AAAABBBBCCCCDDDD";
  const ssize_t expected_read_len = ::strlen(expected_read);
  const ssize_t expected_write_len = ::strlen(expected_read);
  const size_t expected_buf_size = ::strlen(expected_read);

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_WRITE);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(expected_write_len, fd.write(expected_read, expected_read_len));
  }

  {
    std::vector<char> buf(expected_read_len);
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(fd.isValid());
    EXPECT_FALSE(fd.isSpecialFile());
    EXPECT_EQ(expected_read_len, fd.read(buf.data(), expected_read_len));
    EXPECT_EQ(expected_buf_size, buf.size());
    for (ssize_t i = 0; i < expected_read_len; i++) {
      EXPECT_EQ(expected_read[i], buf[i]);
    }
  }
}

TEST_F(FileOpsTests, test_append) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* test_data = "AAAABBBBCCCCDDDDD";
  const ssize_t test_size = ::strlen(test_data);
  const ssize_t test1_size = 7;
  const ssize_t test2_size = test_size - test1_size;

  {
    PlatformFile fd(path, PF_OPEN_ALWAYS | PF_WRITE | PF_APPEND);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(test1_size, fd.write(test_data, test1_size));
  }

  {
    PlatformFile fd(path, PF_OPEN_ALWAYS | PF_WRITE | PF_APPEND);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(test2_size, fd.write(&test_data[7], test2_size));
  }

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(fd.isValid());

    std::vector<char> buf;
    buf.assign(test_size, '\0');
    EXPECT_EQ(test_size, fd.read(buf.data(), test_size));
    EXPECT_EQ(test_size, buf.size());

    for (ssize_t i = 0; i < test_size; i++) {
      EXPECT_EQ(test_data[i], buf[i]);
    }
  }
}

TEST_F(FileOpsTests, test_asyncIo) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* expected = "AAAABBBBCCCCDDDDEEEEFFFFGGGG";
  const ssize_t expected_len = ::strlen(expected);

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_WRITE | PF_NONBLOCK);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(expected_len, fd.write(expected, expected_len));
  }

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ | PF_NONBLOCK);
    EXPECT_TRUE(fd.isValid());
    EXPECT_FALSE(fd.isSpecialFile());

    std::vector<char> buf(expected_len);
    EXPECT_EQ(expected_len, fd.read(buf.data(), expected_len));
    EXPECT_EQ(0, ::memcmp(expected, buf.data(), expected_len));
  }

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ | PF_NONBLOCK);
    EXPECT_TRUE(fd.isValid());
    EXPECT_FALSE(fd.isSpecialFile());

    std::vector<char> buf(expected_len);
    char* ptr = buf.data();
    ssize_t part_bytes = 0;
    int iterations = 0;
    do {
      part_bytes = fd.read(ptr, 4);
      if (part_bytes > 0) {
        ptr += part_bytes;
        iterations++;
      }
    } while (part_bytes > 0);

    EXPECT_EQ(7, iterations);
    EXPECT_EQ(0, ::memcmp(expected, buf.data(), expected_len));
  }
}

TEST_F(FileOpsTests, test_seekFile) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* expected = "AABBBBAACCCAAAAADDDDAAAAAAAA";
  const ssize_t expected_len = ::strlen(expected);
  ssize_t expected_offs;

  {
    PlatformFile fd(path, PF_CREATE_ALWAYS | PF_WRITE);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(expected_len,
              fd.write("AAAAAAAAAAAAAAAAAAAAAAAAAAAA", expected_len));
  }

  // Cast to the proper type, off_t
  expected_offs = expected_len - 12;

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_WRITE);
    EXPECT_TRUE(fd.isValid());

    EXPECT_EQ(expected_offs, fd.seek(-12, PF_SEEK_END));
    EXPECT_EQ(4, fd.write("DDDD", 4));

    EXPECT_EQ(2, fd.seek(2, PF_SEEK_BEGIN));
    EXPECT_EQ(4, fd.write("BBBB", 4));

    EXPECT_EQ(8, fd.seek(2, PF_SEEK_CURRENT));
    EXPECT_EQ(3, fd.write("CCC", 3));
  }

  {
    std::vector<char> buffer(expected_len);

    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(fd.isValid());

    EXPECT_EQ(expected_len, fd.read(buffer.data(), expected_len));
    EXPECT_EQ(0, ::memcmp(buffer.data(), expected, expected_len));
  }
}

TEST_F(FileOpsTests, test_glob) {
  {
    std::vector<fs::path> expected{kFakeDirectory + "/door.txt",
                                   kFakeDirectory + "/root.txt",
                                   kFakeDirectory + "/root2.txt",
                                   kFakeDirectory + "/roto.txt"};
    auto result = platformGlob(kFakeDirectory + "/*.txt");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep1/",
                                   kFakeDirectory + "/deep11/",
                                   kFakeDirectory + "/door.txt",
                                   kFakeDirectory + "/root.txt",
                                   kFakeDirectory + "/root2.txt",
                                   kFakeDirectory + "/roto.txt",
                                   kFakeDirectory + "/toplevel/"};
    auto result = platformGlob(kFakeDirectory + "/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep1/deep2/",
                                   kFakeDirectory + "/deep1/level1.txt",
                                   kFakeDirectory + "/deep11/deep2/",
                                   kFakeDirectory + "/deep11/level1.txt",
                                   kFakeDirectory + "/deep11/not_bash",
                                   kFakeDirectory + "/toplevel/secondlevel1/",
                                   kFakeDirectory + "/toplevel/secondlevel2/",
                                   kFakeDirectory + "/toplevel/secondlevel3/"};
    auto result = platformGlob(kFakeDirectory + "/*/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{
        kFakeDirectory + "/deep1/deep2/level2.txt",
        kFakeDirectory + "/deep11/deep2/deep3/",
        kFakeDirectory + "/deep11/deep2/level2.txt",
        kFakeDirectory + "/toplevel/secondlevel3/thirdlevel1/",
    };
    auto result = platformGlob(kFakeDirectory + "/*/*/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep11/deep2/deep3/",
                                   kFakeDirectory + "/deep11/deep2/level2.txt"};
    auto result = platformGlob(kFakeDirectory + "/*11/*/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep1/",
                                   kFakeDirectory + "/root.txt"};
    auto result = platformGlob(kFakeDirectory + "/{deep,root}{1,.txt}");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep1/deep2/level2.txt",
                                   kFakeDirectory + "/deep11/deep2/deep3/",
                                   kFakeDirectory + "/deep11/deep2/level2.txt"};
    auto result = platformGlob(kFakeDirectory + "/*/deep2/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected;
    if (isPlatform(PlatformType::TYPE_WINDOWS)) {
      expected = {kFakeDirectory + "/deep1/deep2/",
                  kFakeDirectory + "/deep1/level1.txt",
                  kFakeDirectory + "/deep11/deep2/",
                  kFakeDirectory + "/deep11/level1.txt",
                  kFakeDirectory + "/deep11/not_bash"};
    } else {
      expected = {kFakeDirectory + "/deep1/deep2/",
                  kFakeDirectory + "/deep11/deep2/",
                  kFakeDirectory + "/deep1/level1.txt",
                  kFakeDirectory + "/deep11/level1.txt",
                  kFakeDirectory + "/deep11/not_bash"};
    }

    auto result =
        platformGlob(kFakeDirectory + "/*/{deep2,level1,not_bash}{,.txt}");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }
}
}
<commit_msg>Adding large file write/read test for fileops (#2706)<commit_after>/*
 *  Copyright (c) 2014-present, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */

#include <gtest/gtest.h>

#include <boost/filesystem.hpp>

#include <osquery/core.h>

#include "osquery/filesystem/fileops.h"
#include "osquery/tests/test_util.h"

namespace fs = boost::filesystem;

namespace osquery {

class FileOpsTests : public testing::Test {
 protected:
  void SetUp() override {
    createMockFileStructure();
  }

  void TearDown() override {
    tearDownMockFileStructure();
  }

  bool globResultsMatch(const std::vector<std::string>& results,
                        std::vector<fs::path>& expected) {
    if (results.size() == expected.size()) {
      size_t i = 0;
      for (auto const& path : results) {
        if (path != expected[i].make_preferred().string()) {
          return false;
        }
        i++;
      }

      return true;
    }

    return false;
  }
};

class TempFile {
 public:
  TempFile()
      : path_((fs::temp_directory_path() / fs::unique_path())
                  .make_preferred()
                  .string()) {}

  ~TempFile() {
    if (fs::exists(path_)) {
      fs::remove(path_);
    }
  }

  const std::string& path() const {
    return path_;
  }

 private:
  std::string path_;
};

TEST_F(FileOpsTests, test_openFile) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_FALSE(fd.isValid());
  }

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_WRITE);
    EXPECT_TRUE(fd.isValid());
  }

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_READ);
    EXPECT_FALSE(fd.isValid());
  }

  fs::remove(path);

  {
    PlatformFile fd(path, PF_CREATE_ALWAYS | PF_READ);
    EXPECT_TRUE(fd.isValid());
  }

  {
    PlatformFile fd(path, PF_CREATE_ALWAYS | PF_READ);
    EXPECT_TRUE(fd.isValid());
  }

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(fd.isValid());
  }
}

/*
 * This is a special function for testing file share operations on Windows. Our
 * PlatformFile as of now will only set FILE_SHARE_READ to play nicely with log
 * reading tools. However, we need to create one with FILE_SHARE_READ and
 * FILE_SHARE_WRITE for testing.
 */
std::unique_ptr<PlatformFile> openRWSharedFile(const std::string& path,
                                               int mode) {
#ifdef WIN32
  DWORD access_mask = -1;
  DWORD creation_disposition = -1;

  if (mode == (PF_OPEN_EXISTING | PF_READ)) {
    access_mask = PF_READ;
    creation_disposition = OPEN_EXISTING;
  } else if (mode == (PF_OPEN_ALWAYS | PF_WRITE)) {
    access_mask = PF_WRITE;
    creation_disposition = OPEN_ALWAYS;
  }

  HANDLE handle = ::CreateFileA(path.c_str(),
                                access_mask,
                                FILE_SHARE_READ | FILE_SHARE_WRITE,
                                nullptr,
                                creation_disposition,
                                0,
                                nullptr);
  return std::unique_ptr<PlatformFile>(new PlatformFile(handle));
#else
  return std::unique_ptr<PlatformFile>(new PlatformFile(path, mode));
#endif
}

TEST_F(FileOpsTests, test_shareRead) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* test1_data = "AAAABBBB";
  const ssize_t test1_size = ::strlen(test1_data);

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_WRITE);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(test1_size, fd.write(test1_data, test1_size));
  }

  {
    auto reader_fd = openRWSharedFile(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(reader_fd->isValid());

    std::vector<char> buf;
    buf.assign(test1_size, '\0');

    EXPECT_EQ(test1_size, reader_fd->read(buf.data(), test1_size));
    EXPECT_EQ(test1_size, buf.size());

    for (ssize_t i = 0; i < test1_size; i++) {
      EXPECT_EQ(test1_data[i], buf[i]);
    }

    PlatformFile fd(path, PF_OPEN_ALWAYS | PF_WRITE | PF_APPEND);
    EXPECT_TRUE(fd.isValid());
  }
}

TEST_F(FileOpsTests, test_fileIo) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* expected_read = "AAAABBBBCCCCDDDD";
  const ssize_t expected_read_len = ::strlen(expected_read);
  const ssize_t expected_write_len = ::strlen(expected_read);
  const size_t expected_buf_size = ::strlen(expected_read);

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_WRITE);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(expected_write_len, fd.write(expected_read, expected_read_len));
  }

  {
    std::vector<char> buf(expected_read_len);
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(fd.isValid());
    EXPECT_FALSE(fd.isSpecialFile());
    EXPECT_EQ(expected_read_len, fd.read(buf.data(), expected_read_len));
    EXPECT_EQ(expected_buf_size, buf.size());
    for (ssize_t i = 0; i < expected_read_len; i++) {
      EXPECT_EQ(expected_read[i], buf[i]);
    }
  }
}

TEST_F(FileOpsTests, test_append) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* test_data = "AAAABBBBCCCCDDDDD";
  const ssize_t test_size = ::strlen(test_data);
  const ssize_t test1_size = 7;
  const ssize_t test2_size = test_size - test1_size;

  {
    PlatformFile fd(path, PF_OPEN_ALWAYS | PF_WRITE | PF_APPEND);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(test1_size, fd.write(test_data, test1_size));
  }

  {
    PlatformFile fd(path, PF_OPEN_ALWAYS | PF_WRITE | PF_APPEND);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(test2_size, fd.write(&test_data[7], test2_size));
  }

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(fd.isValid());

    std::vector<char> buf;
    buf.assign(test_size, '\0');
    EXPECT_EQ(test_size, fd.read(buf.data(), test_size));
    EXPECT_EQ(test_size, buf.size());

    for (ssize_t i = 0; i < test_size; i++) {
      EXPECT_EQ(test_data[i], buf[i]);
    }
  }
}

TEST_F(FileOpsTests, test_asyncIo) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* expected = "AAAABBBBCCCCDDDDEEEEFFFFGGGG";
  const ssize_t expected_len = ::strlen(expected);

  {
    PlatformFile fd(path, PF_CREATE_NEW | PF_WRITE | PF_NONBLOCK);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(expected_len, fd.write(expected, expected_len));
  }

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ | PF_NONBLOCK);
    EXPECT_TRUE(fd.isValid());
    EXPECT_FALSE(fd.isSpecialFile());

    std::vector<char> buf(expected_len);
    EXPECT_EQ(expected_len, fd.read(buf.data(), expected_len));
    EXPECT_EQ(0, ::memcmp(expected, buf.data(), expected_len));
  }

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ | PF_NONBLOCK);
    EXPECT_TRUE(fd.isValid());
    EXPECT_FALSE(fd.isSpecialFile());

    std::vector<char> buf(expected_len);
    char* ptr = buf.data();
    ssize_t part_bytes = 0;
    int iterations = 0;
    do {
      part_bytes = fd.read(ptr, 4);
      if (part_bytes > 0) {
        ptr += part_bytes;
        iterations++;
      }
    } while (part_bytes > 0);

    EXPECT_EQ(7, iterations);
    EXPECT_EQ(0, ::memcmp(expected, buf.data(), expected_len));
  }
}

TEST_F(FileOpsTests, test_seekFile) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const char* expected = "AABBBBAACCCAAAAADDDDAAAAAAAA";
  const ssize_t expected_len = ::strlen(expected);
  ssize_t expected_offs;

  {
    PlatformFile fd(path, PF_CREATE_ALWAYS | PF_WRITE);
    EXPECT_TRUE(fd.isValid());
    EXPECT_EQ(expected_len,
              fd.write("AAAAAAAAAAAAAAAAAAAAAAAAAAAA", expected_len));
  }

  // Cast to the proper type, off_t
  expected_offs = expected_len - 12;

  {
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_WRITE);
    EXPECT_TRUE(fd.isValid());

    EXPECT_EQ(expected_offs, fd.seek(-12, PF_SEEK_END));
    EXPECT_EQ(4, fd.write("DDDD", 4));

    EXPECT_EQ(2, fd.seek(2, PF_SEEK_BEGIN));
    EXPECT_EQ(4, fd.write("BBBB", 4));

    EXPECT_EQ(8, fd.seek(2, PF_SEEK_CURRENT));
    EXPECT_EQ(3, fd.write("CCC", 3));
  }

  {
    std::vector<char> buffer(expected_len);

    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(fd.isValid());

    EXPECT_EQ(expected_len, fd.read(buffer.data(), expected_len));
    EXPECT_EQ(0, ::memcmp(buffer.data(), expected, expected_len));
  }
}

TEST_F(FileOpsTests, test_large_read_write) {
  TempFile tmp_file;
  std::string path = tmp_file.path();

  const std::string expected(600000000, 'A');
  const ssize_t expected_len = expected.size();

  {
    PlatformFile fd(path, PF_CREATE_ALWAYS | PF_WRITE);
    EXPECT_TRUE(fd.isValid());
    auto write_len = fd.write(expected.c_str(), expected_len);
    EXPECT_EQ(expected_len, write_len);
  }

  {
    std::vector<char> buffer(expected_len);
    PlatformFile fd(path, PF_OPEN_EXISTING | PF_READ);
    EXPECT_TRUE(fd.isValid());
    auto read_len = fd.read(buffer.data(), expected_len);
    EXPECT_EQ(expected_len, read_len);
    EXPECT_EQ(expected, std::string(buffer.data()));
  }
}

TEST_F(FileOpsTests, test_glob) {
  {
    std::vector<fs::path> expected{kFakeDirectory + "/door.txt",
                                   kFakeDirectory + "/root.txt",
                                   kFakeDirectory + "/root2.txt",
                                   kFakeDirectory + "/roto.txt"};
    auto result = platformGlob(kFakeDirectory + "/*.txt");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep1/",
                                   kFakeDirectory + "/deep11/",
                                   kFakeDirectory + "/door.txt",
                                   kFakeDirectory + "/root.txt",
                                   kFakeDirectory + "/root2.txt",
                                   kFakeDirectory + "/roto.txt",
                                   kFakeDirectory + "/toplevel/"};
    auto result = platformGlob(kFakeDirectory + "/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep1/deep2/",
                                   kFakeDirectory + "/deep1/level1.txt",
                                   kFakeDirectory + "/deep11/deep2/",
                                   kFakeDirectory + "/deep11/level1.txt",
                                   kFakeDirectory + "/deep11/not_bash",
                                   kFakeDirectory + "/toplevel/secondlevel1/",
                                   kFakeDirectory + "/toplevel/secondlevel2/",
                                   kFakeDirectory + "/toplevel/secondlevel3/"};
    auto result = platformGlob(kFakeDirectory + "/*/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{
        kFakeDirectory + "/deep1/deep2/level2.txt",
        kFakeDirectory + "/deep11/deep2/deep3/",
        kFakeDirectory + "/deep11/deep2/level2.txt",
        kFakeDirectory + "/toplevel/secondlevel3/thirdlevel1/",
    };
    auto result = platformGlob(kFakeDirectory + "/*/*/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep11/deep2/deep3/",
                                   kFakeDirectory + "/deep11/deep2/level2.txt"};
    auto result = platformGlob(kFakeDirectory + "/*11/*/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep1/",
                                   kFakeDirectory + "/root.txt"};
    auto result = platformGlob(kFakeDirectory + "/{deep,root}{1,.txt}");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected{kFakeDirectory + "/deep1/deep2/level2.txt",
                                   kFakeDirectory + "/deep11/deep2/deep3/",
                                   kFakeDirectory + "/deep11/deep2/level2.txt"};
    auto result = platformGlob(kFakeDirectory + "/*/deep2/*");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }

  {
    std::vector<fs::path> expected;
    if (isPlatform(PlatformType::TYPE_WINDOWS)) {
      expected = {kFakeDirectory + "/deep1/deep2/",
                  kFakeDirectory + "/deep1/level1.txt",
                  kFakeDirectory + "/deep11/deep2/",
                  kFakeDirectory + "/deep11/level1.txt",
                  kFakeDirectory + "/deep11/not_bash"};
    } else {
      expected = {kFakeDirectory + "/deep1/deep2/",
                  kFakeDirectory + "/deep11/deep2/",
                  kFakeDirectory + "/deep1/level1.txt",
                  kFakeDirectory + "/deep11/level1.txt",
                  kFakeDirectory + "/deep11/not_bash"};
    }

    auto result =
        platformGlob(kFakeDirectory + "/*/{deep2,level1,not_bash}{,.txt}");
    EXPECT_TRUE(globResultsMatch(result, expected));
  }
}
}
<|endoftext|>


<commit_before>#include <bits/stdc++.h>

using namespace std;

typedef pair<long long, long long> point;

long long cross(const point &a, const point &b, const point &c) {
	return (b.first - a.first) * (c.second - a.second) - (b.second - a.second) * (c.first - a.first);
}

vector<point> convexHull(vector<point> points) {
	if (points.size() <= 1)
		return points;
	sort(points.begin(), points.end());
	vector<point> h;
	for (auto p: points) {
		while (h.size() >= 2 && cross(h.end()[-2], h.back(), p) >= 0)
			h.pop_back();
		h.push_back(p);
	}
	reverse(points.begin(), points.end());
	int upper = h.size();
	for (auto p: points) {
		while (h.size() > upper && cross(h.end()[-2], h.back(), p) >= 0)
			h.pop_back();
		h.push_back(p);
	}
	h.resize(h.size() - 1 - (h[0] == h[1]));
	return h;
}

// usage example
int main() {
	vector<point> hull1 = convexHull((vector<point>) {point(0, 0), point(3, 0), point(0, 3), point(1, 1)});
	cout << (3 == hull1.size()) << endl;
	vector<point> hull2 = convexHull((vector<point>) {point(0, 0), point(0, 0)});
	cout << (1 == hull2.size()) << endl;
}
<commit_msg>update<commit_after>#include <bits/stdc++.h>

using namespace std;

typedef pair<long long, long long> point;

long long cross(const point &a, const point &b, const point &c) {
    return (b.first - a.first) * (c.second - a.second) - (b.second - a.second) * (c.first - a.first);
}

vector<point> convexHull(vector<point> points) {
    if (points.size() <= 1)
        return points;
    sort(points.begin(), points.end());
    vector<point> h;
    for (auto &p: points) {
        while (h.size() >= 2 && cross(h.end()[-2], h.back(), p) >= 0)
            h.pop_back();
        h.emplace_back(p);
    }
    reverse(points.begin(), points.end());
    int upper = h.size();
    for (auto &p: points) {
        while (h.size() > upper && cross(h.end()[-2], h.back(), p) >= 0)
            h.pop_back();
        h.emplace_back(p);
    }
    h.resize(h.size() - 1 - (h[0] == h[1]));
    return h;
}

// usage example
int main() {
    vector<point> hull1 = convexHull({{0, 0},
                                      {3, 0},
                                      {0, 3},
                                      {1, 1}});
    cout << (3 == hull1.size()) << endl;

    vector<point> hull2 = convexHull({{0, 0},
                                      {0, 0}});
    cout << (1 == hull2.size()) << endl;
}
<|endoftext|>


<commit_before>/** \file   FullTextCache.cc
 *  \brief  Implementation of functions relating to our full-text cache.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2017-2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "FullTextCache.h"
#include <algorithm>
#include <tuple>
#include <ctime>
#include "Compiler.h"
#include "DbRow.h"
#include "Random.h"
#include "StringUtil.h"
#include "TimeUtil.h"
#include "UrlUtil.h"
#include "util.h"
#include "VuFind.h"

constexpr unsigned MIN_CACHE_EXPIRE_TIME_ON_ERROR(42300 * 60); // About 1 month in seconds.
constexpr unsigned MAX_CACHE_EXPIRE_TIME_ON_ERROR(42300 * 60 * 2); // About 2 months in seconds.

bool FullTextCache::getDomainFromUrl(const std::string &url, std::string * const domain) const {
    std::string scheme, username_password, authority, port, path, params, query, fragment, relative_url;
    const bool result(UrlUtil::ParseUrl(url, &scheme, &username_password, &authority, &port, &path, &params, &query,
                                        &fragment, &relative_url));
    if (result)
        *domain = authority;

    return result;
}

bool FullTextCache::entryExpired(const std::string &id, std::vector<std::string> urls) {
    Entry entry;
    if (not getEntry(id, &entry))
        return true;

    const time_t now(std::time(nullptr));
    if (entry.expiration_ == TimeUtil::BAD_TIME_T or now < entry.expiration_) {
        std::vector<std::string> existing_urls(FullTextCache::getEntryUrlsAsStrings(id));
        std::sort(existing_urls.begin(), existing_urls.end());
        std::sort(urls.begin(), urls.end());
        if (urls == existing_urls)
            return false;
    }

    return full_text_cache_urls_.deleteDocument(id);
}

void FullTextCache::expireEntries() {
    full_text_cache_urls_.deleteRange("expiration", Elasticsearch::RO_LTE, "now");
}

bool FullTextCache::getEntry(const std::string &id, Entry * const entry) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "expiration" }, "id", id));
    if (results.empty())
        return false;

    entry->id_ = id;
    if (results.front().find("expiration") == results.front().cend())
        entry->expiration_ = TimeUtil::BAD_TIME_T;
    else
        entry->expiration_ = TimeUtil::Iso8601StringToTimeT(results.front().find("expiration")->second);
    return true;
}

inline std::string GetValueOrEmptyString(const std::map<std::string, std::string> &map, const std::string &key) {
    const auto pair(map.find(key));
    return pair == map.cend() ? "" : pair->second;
}

std::vector<FullTextCache::EntryUrl> FullTextCache::getEntryUrls(const std::string &id) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message" }, "id", id));

    std::vector<EntryUrl> entry_urls;
    entry_urls.reserve(results.size());
    for (const auto &map : results) {
        EntryUrl entry_url;
        entry_url.id_            = id;
        entry_url.url_           = GetValueOrEmptyString(map, "url");
        entry_url.domain_        = GetValueOrEmptyString(map, "domain");
        entry_url.error_message_ = GetValueOrEmptyString(map, "error_message");
        entry_urls.emplace_back(entry_url);
    }
    return entry_urls;
}

std::vector<std::string> FullTextCache::getEntryUrlsAsStrings(const std::string &id) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url" }, "id", id));

    std::vector<std::string> urls;
    for (const auto &map : results) {
        const auto url(GetValueOrEmptyString(map, "url"));
        if (not url.empty())
            urls.emplace_back(url);
    }
    return urls;
}

unsigned FullTextCache::getErrorCount() const {
    const auto results(full_text_cache_urls_.simpleSelect({ "id", "error_message" }));

    std::unordered_set<std::string> unique_ids;
    for (const auto &map : results) {
        if (map.find("error_message") != map.cend())
            unique_ids.emplace(map.find("id")->second);
    }

    return static_cast<unsigned>(unique_ids.size());
}

bool FullTextCache::getFullText(const std::string &id, std::string * const full_text) const {
    const auto results(full_text_cache_.simpleSelect({ "full_text" }, "id", id));

    if (results.empty())
        return false;

    *full_text = GetValueOrEmptyString(results.front(), "full_text");
    return true;
}

const std::string US("\x1F"); // ASCII unit separator

std::vector<FullTextCache::EntryGroup> FullTextCache::getEntryGroupsByDomainAndErrorMessage() const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message", "id" }));

    std::unordered_map<std::string, std::tuple<std::string, unsigned>> urls_and_domains_to_ids_and_counts_map;
    for (const auto &map : results) {
        const auto url_pair(map.find("url"));
        const auto domain_pair(map.find("domain"));
        const auto error_message_pair(map.find("error_message"));
        if (url_pair != map.cend() or domain_pair != map.cend() or error_message_pair != map.cend())
            continue;

        const auto id_pair(map.find("id"));
        const auto key(url_pair->second + US + domain_pair->second + US + error_message_pair->second);
        auto url_and_domain_and_id_and_count(urls_and_domains_to_ids_and_counts_map.find(key));
        if (url_and_domain_and_id_and_count != urls_and_domains_to_ids_and_counts_map.end())
            ++std::get<1>(url_and_domain_and_id_and_count->second);
        else
            urls_and_domains_to_ids_and_counts_map[key] = std::make_tuple(id_pair->second, 1);
    }

    std::vector<EntryGroup> groups;
    groups.reserve(urls_and_domains_to_ids_and_counts_map.size());
    for (const auto &url_and_domain_and_id_and_count : urls_and_domains_to_ids_and_counts_map) {
        std::vector<std::string> parts;
        StringUtil::Split(url_and_domain_and_id_and_count.first, US, &parts);
        const std::string &url(parts[0]);
        const std::string &domain(parts[1]);
        const std::string &error_message(parts[2]);

        groups.emplace_back(EntryGroup(std::get<1>(url_and_domain_and_id_and_count.second), domain, error_message,
                                       std::get<0>(url_and_domain_and_id_and_count.second), url));
    }

    std::sort(groups.begin(), groups.end(), [](const EntryGroup &eg1, const EntryGroup &eg2){ return eg1.count_ < eg2.count_;});
    return groups;
}

std::vector<FullTextCache::EntryUrl> FullTextCache::getJoinedEntriesByDomainAndErrorMessage(const std::string &domain_,
                                                                                const std::string &error_message_) const
{
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message", "id" },
                                                          { { "domain", domain_ }, { "error_message", error_message_ } }));

    std::vector<EntryUrl> entries;
    for (const auto &map : results) {
        const std::string url(GetValueOrEmptyString(map, "url"));
        const std::string domain(GetValueOrEmptyString(map, "domain"));
        const std::string error_message(GetValueOrEmptyString(map, "error_message"));
        const std::string id(GetValueOrEmptyString(map, "id"));

        entries.emplace_back(EntryUrl(id, url, domain, error_message));
    }

    return entries;
}

FullTextCache::EntryUrl FullTextCache::getJoinedEntryByDomainAndErrorMessage(const std::string &domain_, const std::string &error_message_)
    const
{
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message", "id" },
                                                          { { "domain", domain_ }, { "error_message", error_message_ } },
                                                          /* max_count = */1));
    if (unlikely(results.size() != 1))
        LOG_ERROR("failed to get one entry!");

    const std::string url(GetValueOrEmptyString(results.front(), "url"));
    const std::string domain(GetValueOrEmptyString(results.front(), "domain"));
    const std::string error_message(GetValueOrEmptyString(results.front(), "error_message"));
    const std::string id(GetValueOrEmptyString(results.front(), "id"));

    return EntryUrl(id, url, domain, error_message);
}

unsigned FullTextCache::getSize() const {
    return full_text_cache_.size();
}

void FullTextCache::insertEntry(const std::string &id, const std::string &full_text, const std::vector<EntryUrl> &entry_urls) {
    const time_t now(std::time(nullptr));
    Random::Rand rand(now);
    time_t expiration(TimeUtil::BAD_TIME_T);
    for (const auto &entry_url : entry_urls) {
        if (full_text.empty() and entry_url.error_message_.empty())
            LOG_ERROR("you must provide either data to be cached or a non-empty error message! (id " + id + ")");

        if (not entry_url.error_message_.empty())
            expiration = now + MIN_CACHE_EXPIRE_TIME_ON_ERROR + rand(MAX_CACHE_EXPIRE_TIME_ON_ERROR - MIN_CACHE_EXPIRE_TIME_ON_ERROR);
    }

    std::string expiration_string;
    if (expiration == TimeUtil::BAD_TIME_T)
        full_text_cache_.simpleInsert({ { "id", id }, { "full_text", full_text } });
    else {
        expiration_string = "\"" + TimeUtil::TimeTToString(expiration, TimeUtil::ISO_8601_FORMAT) + "\"";
        full_text_cache_.simpleInsert({ { "id", id }, { "expiration", expiration_string }, { "full_text", full_text } });
    }

    for (const auto &entry_url : entry_urls) {
        if (entry_url.error_message_.empty())
            full_text_cache_urls_.simpleInsert({ { "id", id }, { "url", entry_url.url_ }, { "domain", entry_url.domain_ } });
        else
            full_text_cache_urls_.simpleInsert({ { "id", id }, { "url", entry_url.url_ }, { "domain", entry_url.domain_ },
                                                 { "error_message", entry_url.error_message_ } });
    }
}
<commit_msg>Skip upload if ID already exists<commit_after>/** \file   FullTextCache.cc
 *  \brief  Implementation of functions relating to our full-text cache.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2017-2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "FullTextCache.h"
#include <algorithm>
#include <tuple>
#include <ctime>
#include "Compiler.h"
#include "DbRow.h"
#include "Random.h"
#include "StringUtil.h"
#include "TimeUtil.h"
#include "UrlUtil.h"
#include "util.h"
#include "VuFind.h"

#include <iostream>

constexpr unsigned MIN_CACHE_EXPIRE_TIME_ON_ERROR(42300 * 60); // About 1 month in seconds.
constexpr unsigned MAX_CACHE_EXPIRE_TIME_ON_ERROR(42300 * 60 * 2); // About 2 months in seconds.

bool FullTextCache::getDomainFromUrl(const std::string &url, std::string * const domain) const {
    std::string scheme, username_password, authority, port, path, params, query, fragment, relative_url;
    const bool result(UrlUtil::ParseUrl(url, &scheme, &username_password, &authority, &port, &path, &params, &query,
                                        &fragment, &relative_url));
    if (result)
        *domain = authority;

    return result;
}

bool FullTextCache::entryExpired(const std::string &id, std::vector<std::string> urls) {
    Entry entry;
    if (not getEntry(id, &entry))
        return true;

    const time_t now(std::time(nullptr));
    if (entry.expiration_ == TimeUtil::BAD_TIME_T or now < entry.expiration_) {
        std::vector<std::string> existing_urls(FullTextCache::getEntryUrlsAsStrings(id));
        std::sort(existing_urls.begin(), existing_urls.end());
        std::sort(urls.begin(), urls.end());
        if (urls == existing_urls)
            return false;
    }

    return full_text_cache_urls_.deleteDocument(id);
}

void FullTextCache::expireEntries() {
    full_text_cache_urls_.deleteRange("expiration", Elasticsearch::RO_LTE, "now");
}

bool FullTextCache::getEntry(const std::string &id, Entry * const entry) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "expiration" }, "id", id));
    if (results.empty())
        return false;

    entry->id_ = id;
    if (results.front().find("expiration") == results.front().cend())
        entry->expiration_ = TimeUtil::BAD_TIME_T;
    else
        entry->expiration_ = TimeUtil::Iso8601StringToTimeT(results.front().find("expiration")->second);
    return true;
}

inline std::string GetValueOrEmptyString(const std::map<std::string, std::string> &map, const std::string &key) {
    const auto pair(map.find(key));
    return pair == map.cend() ? "" : pair->second;
}

std::vector<FullTextCache::EntryUrl> FullTextCache::getEntryUrls(const std::string &id) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message" }, "id", id));

    std::vector<EntryUrl> entry_urls;
    entry_urls.reserve(results.size());
    for (const auto &map : results) {
        EntryUrl entry_url;
        entry_url.id_            = id;
        entry_url.url_           = GetValueOrEmptyString(map, "url");
        entry_url.domain_        = GetValueOrEmptyString(map, "domain");
        entry_url.error_message_ = GetValueOrEmptyString(map, "error_message");
        entry_urls.emplace_back(entry_url);
    }
    return entry_urls;
}

std::vector<std::string> FullTextCache::getEntryUrlsAsStrings(const std::string &id) const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url" }, "id", id));

    std::vector<std::string> urls;
    for (const auto &map : results) {
        const auto url(GetValueOrEmptyString(map, "url"));
        if (not url.empty())
            urls.emplace_back(url);
    }
    return urls;
}

unsigned FullTextCache::getErrorCount() const {
    const auto results(full_text_cache_urls_.simpleSelect({ "id", "error_message" }));

    std::unordered_set<std::string> unique_ids;
    for (const auto &map : results) {
        if (map.find("error_message") != map.cend())
            unique_ids.emplace(map.find("id")->second);
    }

    return static_cast<unsigned>(unique_ids.size());
}

bool FullTextCache::getFullText(const std::string &id, std::string * const full_text) const {
    const auto results(full_text_cache_.simpleSelect({ "full_text" }, "id", id));

    if (results.empty())
        return false;

    *full_text = GetValueOrEmptyString(results.front(), "full_text");
    return true;
}

const std::string US("\x1F"); // ASCII unit separator

std::vector<FullTextCache::EntryGroup> FullTextCache::getEntryGroupsByDomainAndErrorMessage() const {
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message", "id" }));

    std::unordered_map<std::string, std::tuple<std::string, unsigned>> urls_and_domains_to_ids_and_counts_map;
    for (const auto &map : results) {
        const auto url_pair(map.find("url"));
        const auto domain_pair(map.find("domain"));
        const auto error_message_pair(map.find("error_message"));
        if (url_pair != map.cend() or domain_pair != map.cend() or error_message_pair != map.cend())
            continue;

        const auto id_pair(map.find("id"));
        const auto key(url_pair->second + US + domain_pair->second + US + error_message_pair->second);
        auto url_and_domain_and_id_and_count(urls_and_domains_to_ids_and_counts_map.find(key));
        if (url_and_domain_and_id_and_count != urls_and_domains_to_ids_and_counts_map.end())
            ++std::get<1>(url_and_domain_and_id_and_count->second);
        else
            urls_and_domains_to_ids_and_counts_map[key] = std::make_tuple(id_pair->second, 1);
    }

    std::vector<EntryGroup> groups;
    groups.reserve(urls_and_domains_to_ids_and_counts_map.size());
    for (const auto &url_and_domain_and_id_and_count : urls_and_domains_to_ids_and_counts_map) {
        std::vector<std::string> parts;
        StringUtil::Split(url_and_domain_and_id_and_count.first, US, &parts);
        const std::string &url(parts[0]);
        const std::string &domain(parts[1]);
        const std::string &error_message(parts[2]);

        groups.emplace_back(EntryGroup(std::get<1>(url_and_domain_and_id_and_count.second), domain, error_message,
                                       std::get<0>(url_and_domain_and_id_and_count.second), url));
    }

    std::sort(groups.begin(), groups.end(), [](const EntryGroup &eg1, const EntryGroup &eg2){ return eg1.count_ < eg2.count_;});
    return groups;
}

std::vector<FullTextCache::EntryUrl> FullTextCache::getJoinedEntriesByDomainAndErrorMessage(const std::string &domain_,
                                                                                const std::string &error_message_) const
{
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message", "id" },
                                                          { { "domain", domain_ }, { "error_message", error_message_ } }));

    std::vector<EntryUrl> entries;
    for (const auto &map : results) {
        const std::string url(GetValueOrEmptyString(map, "url"));
        const std::string domain(GetValueOrEmptyString(map, "domain"));
        const std::string error_message(GetValueOrEmptyString(map, "error_message"));
        const std::string id(GetValueOrEmptyString(map, "id"));

        entries.emplace_back(EntryUrl(id, url, domain, error_message));
    }

    return entries;
}

FullTextCache::EntryUrl FullTextCache::getJoinedEntryByDomainAndErrorMessage(const std::string &domain_, const std::string &error_message_)
    const
{
    const auto results(full_text_cache_urls_.simpleSelect({ "url", "domain", "error_message", "id" },
                                                          { { "domain", domain_ }, { "error_message", error_message_ } },
                                                          /* max_count = */1));
    if (unlikely(results.size() != 1))
        LOG_ERROR("failed to get one entry!");

    const std::string url(GetValueOrEmptyString(results.front(), "url"));
    const std::string domain(GetValueOrEmptyString(results.front(), "domain"));
    const std::string error_message(GetValueOrEmptyString(results.front(), "error_message"));
    const std::string id(GetValueOrEmptyString(results.front(), "id"));

    return EntryUrl(id, url, domain, error_message);
}

unsigned FullTextCache::getSize() const {
    return full_text_cache_.size();
}

void FullTextCache::insertEntry(const std::string &id, const std::string &full_text, const std::vector<EntryUrl> &entry_urls) {
    const time_t now(std::time(nullptr));
    Random::Rand rand(now);
    time_t expiration(TimeUtil::BAD_TIME_T);
    for (const auto &entry_url : entry_urls) {
        if (full_text.empty() and entry_url.error_message_.empty())
            LOG_ERROR("you must provide either data to be cached or a non-empty error message! (id " + id + ")");

        if (not entry_url.error_message_.empty())
            expiration = now + MIN_CACHE_EXPIRE_TIME_ON_ERROR + rand(MAX_CACHE_EXPIRE_TIME_ON_ERROR - MIN_CACHE_EXPIRE_TIME_ON_ERROR);
    }

    std::string expiration_string;
    if (expiration == TimeUtil::BAD_TIME_T) {
        if (!full_text_cache_.fieldWithValueExists("id", id))
            full_text_cache_.simpleInsert({ { "id", id }, { "full_text", full_text } });
        else
            LOG_INFO("ID \"" + id + "\" already present, skipping insertion...");
    }
    else {
        expiration_string = "\"" + TimeUtil::TimeTToString(expiration, TimeUtil::ISO_8601_FORMAT) + "\"";
        full_text_cache_.simpleInsert({ { "id", id }, { "expiration", expiration_string }, { "full_text", full_text } });
    }

    for (const auto &entry_url : entry_urls) {
        if (entry_url.error_message_.empty())
            full_text_cache_urls_.simpleInsert({ { "id", id }, { "url", entry_url.url_ }, { "domain", entry_url.domain_ } });
        else
            full_text_cache_urls_.simpleInsert({ { "id", id }, { "url", entry_url.url_ }, { "domain", entry_url.domain_ },
                                                 { "error_message", entry_url.error_message_ } });
    }
}
<|endoftext|>


<commit_before>/** \file    add_oa_urls.cc
 *  \brief   Add additional urls for oa_access of items
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2018-2019, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <memory>
#include <unordered_map>
#include "Compiler.h"
#include "FileUtil.h"
#include "JSON.h"
#include "MARC.h"
#include "util.h"

namespace {

struct oadoi_info {
    std::string url_;
    std::string evidence_;
    std::string host_type_;

public:
    oadoi_info(const std::string url, const std::string evidence, const std::string host_type)
        : url_(url), evidence_(evidence), host_type_(host_type) { }
};

[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " doi_to_url_map.json marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

void CreateDoiToUrlMap(const std::string &map_filename, std::unordered_map<std::string, oadoi_info> * const doi_to_oainfo) {
    std::string json_document;
    if (not FileUtil::ReadString(map_filename, &json_document))
        LOG_ERROR("Could not read in " + map_filename);

    JSON::Parser json_parser(json_document);
    std::shared_ptr<JSON::JSONNode> entries;
    if (not json_parser.parse(&entries))
        LOG_ERROR("Could not properly parse \"" + map_filename + ": " + json_parser.getErrorMessage());

    std::shared_ptr<JSON::ArrayNode> entry_array(JSON::JSONNode::CastToArrayNodeOrDie("", entries));
    for (const auto &entry : *entry_array) {
        const std::string doi(LookupString("/doi", entry));
        const std::string url(LookupString("/best_oa_location/url", entry));
        const std::string evidence(LookupString("/best_oa_location/evidence", entry));
        const std::string host_type(LookupString("/best_oa_location/host_type", entry));
        if (not(doi.empty() or url.empty()))
            doi_to_oainfo->emplace(doi, oadoi_info(url, evidence, host_type));
        else
            LOG_ERROR("Either doi or url missing");
    }
}

bool AlreadyHasIdenticalUrl(const MARC::Record &record, const std::string &url) {
    for (const auto &field : record.getTagRange("856")) {
        if (field.hasSubfieldWithValue('u', url))
            return true;
    }
    return false;
}

void Augment856(MARC::Reader * const marc_reader, MARC::Writer * const marc_writer,
                const std::unordered_map<std::string, oadoi_info> &doi_to_oainfo) {
    while (MARC::Record record = marc_reader->read()) {
        bool flag_as_open_access_resource(false);
        for (const auto &field : record.getTagRange("024")) {
            if (field.hasSubfieldWithValue('2', "doi")) {
                const std::string doi(field.getFirstSubfieldWithCode('a'));
                const auto doi_and_info(doi_to_oainfo.find(doi));
                if (doi_and_info != doi_to_oainfo.cend()) {
                    const auto oainfo(doi_and_info->second);
                    const std::string url(oainfo.url_);
                    const std::string evidence(oainfo.evidence_);
                    const std::string host_type(oainfo.host_type_);
                    if (not AlreadyHasIdenticalUrl(record, url))
                        record.insertField("856", { { 'u', url },
                                                    { 'x', "unpaywall" },
                                                    { 'z', "Vermutlich kostenfreier Zugang" },
                                                    { 'h', host_type + " [" + evidence + "]" } });
                    flag_as_open_access_resource = true;
                }
            }

            if (flag_as_open_access_resource and record.getFirstField("OAS") == record.end())
                record.insertField("OAS", { MARC::Subfield('a', "1") });
        }
        marc_writer->write(record);
    }
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc != 4)
        Usage();

    std::unordered_map<std::string, oadoi_info> doi_to_oainfo;
    CreateDoiToUrlMap(argv[1], &doi_to_oainfo);
    std::unique_ptr<MARC::Reader> marc_reader(MARC::Reader::Factory(argv[2]));
    std::unique_ptr<MARC::Writer> marc_writer(MARC::Writer::Factory(argv[3]));
    Augment856(marc_reader.get(), marc_writer.get(), doi_to_oainfo);

    return EXIT_SUCCESS;
}
<commit_msg>Address https://github.com/ubtue/tuefind/issues/2326<commit_after>/** \file    add_oa_urls.cc
 *  \brief   Add additional urls for oa_access of items
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2018-2019, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <memory>
#include <unordered_map>
#include "Compiler.h"
#include "FileUtil.h"
#include "JSON.h"
#include "MARC.h"
#include "util.h"

namespace {

struct oadoi_info {
    std::string url_;
    std::string evidence_;
    std::string host_type_;

public:
    oadoi_info(const std::string url, const std::string evidence, const std::string host_type)
        : url_(url), evidence_(evidence), host_type_(host_type) { }
};

[[noreturn]] void Usage() {
    std::cerr << "Usage: " << ::progname << " doi_to_url_map.json marc_input marc_output\n";
    std::exit(EXIT_FAILURE);
}

void CreateDoiToUrlMap(const std::string &map_filename, std::unordered_map<std::string, oadoi_info> * const doi_to_oainfo) {
    std::string json_document;
    if (not FileUtil::ReadString(map_filename, &json_document))
        LOG_ERROR("Could not read in " + map_filename);

    JSON::Parser json_parser(json_document);
    std::shared_ptr<JSON::JSONNode> entries;
    if (not json_parser.parse(&entries))
        LOG_ERROR("Could not properly parse \"" + map_filename + ": " + json_parser.getErrorMessage());

    std::shared_ptr<JSON::ArrayNode> entry_array(JSON::JSONNode::CastToArrayNodeOrDie("", entries));
    for (const auto &entry : *entry_array) {
        const std::string doi(LookupString("/doi", entry));
        const std::string url(LookupString("/best_oa_location/url", entry));
        const std::string evidence(LookupString("/best_oa_location/evidence", entry));
        const std::string host_type(LookupString("/best_oa_location/host_type", entry));

        // Workaround for erroneous SWBplus entries that are no full texts
        static const auto swbplus_matcher(ThreadSafeRegexMatcher("^https?://swbplus.bsz-bw.de/.*"));
        if (swbplus_matcher.match(url))
            continue;

        if (not(doi.empty() or url.empty()))
            doi_to_oainfo->emplace(doi, oadoi_info(url, evidence, host_type));
        else
            LOG_ERROR("Either doi or url missing");
    }
}

bool AlreadyHasIdenticalUrl(const MARC::Record &record, const std::string &url) {
    for (const auto &field : record.getTagRange("856")) {
        if (field.hasSubfieldWithValue('u', url))
            return true;
    }
    return false;
}

void Augment856(MARC::Reader * const marc_reader, MARC::Writer * const marc_writer,
                const std::unordered_map<std::string, oadoi_info> &doi_to_oainfo) {
    while (MARC::Record record = marc_reader->read()) {
        bool flag_as_open_access_resource(false);
        for (const auto &field : record.getTagRange("024")) {
            if (field.hasSubfieldWithValue('2', "doi")) {
                const std::string doi(field.getFirstSubfieldWithCode('a'));
                const auto doi_and_info(doi_to_oainfo.find(doi));
                if (doi_and_info != doi_to_oainfo.cend()) {
                    const auto oainfo(doi_and_info->second);
                    const std::string url(oainfo.url_);
                    const std::string evidence(oainfo.evidence_);
                    const std::string host_type(oainfo.host_type_);
                    if (not AlreadyHasIdenticalUrl(record, url))
                        record.insertField("856", { { 'u', url },
                                                    { 'x', "unpaywall" },
                                                    { 'z', "Vermutlich kostenfreier Zugang" },
                                                    { 'h', host_type + " [" + evidence + "]" } });
                    flag_as_open_access_resource = true;
                }
            }

            if (flag_as_open_access_resource and record.getFirstField("OAS") == record.end())
                record.insertField("OAS", { MARC::Subfield('a', "1") });
        }
        marc_writer->write(record);
    }
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc != 4)
        Usage();

    std::unordered_map<std::string, oadoi_info> doi_to_oainfo;
    CreateDoiToUrlMap(argv[1], &doi_to_oainfo);
    std::unique_ptr<MARC::Reader> marc_reader(MARC::Reader::Factory(argv[2]));
    std::unique_ptr<MARC::Writer> marc_writer(MARC::Writer::Factory(argv[3]));
    Augment856(marc_reader.get(), marc_writer.get(), doi_to_oainfo);

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2009 Digital Bazaar, Inc. All rights reserved.
 */
#include "db/test/Test.h"
#include "db/test/Tester.h"
#include "db/test/TestRunner.h"
#include "db/rt/HashTable.h"
#include "db/rt/Runnable.h"
#include "db/rt/RunnableDelegate.h"
#include "db/rt/Thread.h"
#include "db/util/Timer.h"

#include <math.h>

using namespace std;
using namespace db::test;
using namespace db::rt;

struct KeyAsHash
{
   int operator()(int key) const
   {
      return key;
   }
};

struct AddressAsHash
{
   int operator()(const char* key) const
   {
      return (int)key;
   }
};

struct JavaHashCodeAsHash
{
   int operator()(const char* key) const
   {
      // s[0]*31^(n-1) + s[1]*31^(n-2) + ... + s[n-1]
      int n = strlen(key) - 1;
      int hash = 0;
      for(const char* ptr = key; ptr[0] != '\0'; ptr++)
      {
         hash += (int)ptr[0] * pow(31, n);
         n--;
      }
      return hash;
   }
};

struct StringEquals
{
   bool operator()(const char* key1, const char* key2)
   {
      return strcmp(key1, key2) == 0;
   }
};

void runHashTableTests(TestRunner& tr)
{
   tr.group("HashTable");

   tr.test("complex");
   {
      HashTable<int, int, KeyAsHash> table(1);

      table.put(1, 7);
      table.put(2, 13);

      int num;

      assert(table.get(1, num));
      assert(num == 7);
      assert(table.get(2, num));
      assert(num == 13);
      assert(!table.put(2, 10, false));
      assert(table.put(2, 10));
      assert(table.get(2, num));
      assert(num == 10);
      assert(table.put(2, 11));
      assert(table.get(2, num));
      assert(num == 11);

      assert(!table.get(99, num));

      HashTable<int, int, KeyAsHash> table2 = table;
      num = 0;
      assert(table.get(1, num));
      assert(num == 7);
      assert(table.get(2, num));
      assert(num == 11);

      HashTable<int, int, KeyAsHash> table3;
      table3 = table;
      num = 0;
      assert(table.get(1, num));
      assert(num == 7);
      assert(table.get(2, num));
      assert(num == 11);
   }
   tr.passIfNoException();

   tr.test("static string");
   {
      HashTable<const char*, int, AddressAsHash> table;

      table.put("foo", 7);
      table.put("bar", 13);

      int num;
      assert(table.get("foo", num));
      assert(num == 7);
      assert(table.get("bar", num));
      assert(num == 13);
   }
   tr.passIfNoException();

   tr.test("dynamic string");
   {
      HashTable<const char*, int, JavaHashCodeAsHash, StringEquals> table;

      table.put("foo", 7);
      table.put("bar", 13);

      string key1 = "foo";
      string key2 = "bar";

      int num;
      assert(table.get(key1.c_str(), num));
      assert(num == 7);
      assert(table.get(key2.c_str(), num));
      assert(num == 13);
   }
   tr.passIfNoException();

   /*
   HashTable<int, int, KeyAsHash> table(1);
   int hit = 1;
   int miss = 2;
   int value = 7;

   tr.test("put");
   {
      table.put(hit, value);
   }
   tr.passIfNoException();

   tr.test("get hit");
   {
      int num;
      if(table.get(hit, num))
      {
         printf("GOT: %d => %d\n", hit, num);
      }
      else
      {
         printf("VALUE FOR %d NOT FOUND\n", hit);
      }
   }
   tr.passIfNoException();

   tr.test("get miss");
   {
      int num;
      if(table.get(miss, num))
      {
         printf("GOT: %d => %d\n", miss, num);
      }
      else
      {
         printf("VALUE FOR %d NOT FOUND\n", miss);
      }
   }
   tr.passIfNoException();
   */

   tr.ungroup();
}

void runHashTableConcurrencyTest(TestRunner& tr)
{
   tr.group("HashTable concurrency");

   tr.test("single thread");
   {
   }
   tr.passIfNoException();

   tr.test("many threads");
   {
   }
   tr.passIfNoException();

   tr.ungroup();
}

class DbHashTableTester : public db::test::Tester
{
public:
   DbHashTableTester()
   {
      setName("HashTable");
   }

   /**
    * Run automatic unit tests.
    */
   virtual int runAutomaticTests(TestRunner& tr)
   {
      runHashTableTests(tr);
      return 0;
   }

   /**
    * Runs interactive unit tests.
    */
   virtual int runInteractiveTests(TestRunner& tr)
   {
      runHashTableConcurrencyTest(tr);
      return 0;
   }
};

db::test::Tester* getDbHashTableTester() { return new DbHashTableTester(); }

DB_TEST_MAIN(DbHashTableTester)
<commit_msg>Added first pass of perf test of ht vs map.<commit_after>/*
 * Copyright (c) 2009 Digital Bazaar, Inc. All rights reserved.
 */
#define __STDC_CONSTANT_MACROS
#define __STDC_FORMAT_MACROS

#include "db/test/Test.h"
#include "db/test/Tester.h"
#include "db/test/TestRunner.h"
#include "db/rt/ExclusiveLock.h"
#include "db/rt/HashTable.h"
#include "db/rt/Runnable.h"
#include "db/rt/RunnableDelegate.h"
#include "db/rt/SharedLock.h"
#include "db/rt/Thread.h"
#include "db/util/Timer.h"

#include <cmath>

using namespace std;
using namespace db::config;
using namespace db::rt;
using namespace db::test;
using namespace db::util;

struct KeyAsHash
{
   int operator()(int key) const
   {
      return key;
   }
};

struct AddressAsHash
{
   int operator()(const char* key) const
   {
      return (int)key;
   }
};

struct JavaHashCodeAsHash
{
   int operator()(const char* key) const
   {
      // s[0]*31^(n-1) + s[1]*31^(n-2) + ... + s[n-1]
      int n = strlen(key) - 1;
      int hash = 0;
      for(const char* ptr = key; ptr[0] != '\0'; ptr++)
      {
         hash += (int)ptr[0] * pow(31, n);
         n--;
      }
      return hash;
   }
};

struct StringEquals
{
   bool operator()(const char* key1, const char* key2)
   {
      return strcmp(key1, key2) == 0;
   }
};

void runHashTableTests(TestRunner& tr)
{
   tr.group("HashTable");

   tr.test("complex");
   {
      HashTable<int, int, KeyAsHash> table(1);

      table.put(1, 7);
      table.put(2, 13);

      int num;

      assert(table.get(1, num));
      assert(num == 7);
      assert(table.get(2, num));
      assert(num == 13);
      assert(!table.put(2, 10, false));
      assert(table.put(2, 10));
      assert(table.get(2, num));
      assert(num == 10);
      assert(table.put(2, 11));
      assert(table.get(2, num));
      assert(num == 11);

      assert(!table.get(99, num));

      HashTable<int, int, KeyAsHash> table2 = table;
      num = 0;
      assert(table.get(1, num));
      assert(num == 7);
      assert(table.get(2, num));
      assert(num == 11);

      HashTable<int, int, KeyAsHash> table3;
      table3 = table;
      num = 0;
      assert(table.get(1, num));
      assert(num == 7);
      assert(table.get(2, num));
      assert(num == 11);
   }
   tr.passIfNoException();

   tr.test("static string");
   {
      HashTable<const char*, int, AddressAsHash> table;

      table.put("foo", 7);
      table.put("bar", 13);

      int num;
      assert(table.get("foo", num));
      assert(num == 7);
      assert(table.get("bar", num));
      assert(num == 13);
   }
   tr.passIfNoException();

   tr.test("dynamic string");
   {
      HashTable<const char*, int, JavaHashCodeAsHash, StringEquals> table;

      table.put("foo", 7);
      table.put("bar", 13);

      string key1 = "foo";
      string key2 = "bar";

      int num;
      assert(table.get(key1.c_str(), num));
      assert(num == 7);
      assert(table.get(key2.c_str(), num));
      assert(num == 13);
   }
   tr.passIfNoException();

   /*
   HashTable<int, int, KeyAsHash> table(1);
   int hit = 1;
   int miss = 2;
   int value = 7;

   tr.test("put");
   {
      table.put(hit, value);
   }
   tr.passIfNoException();

   tr.test("get hit");
   {
      int num;
      if(table.get(hit, num))
      {
         printf("GOT: %d => %d\n", hit, num);
      }
      else
      {
         printf("VALUE FOR %d NOT FOUND\n", hit);
      }
   }
   tr.passIfNoException();

   tr.test("get miss");
   {
      int num;
      if(table.get(miss, num))
      {
         printf("GOT: %d => %d\n", miss, num);
      }
      else
      {
         printf("VALUE FOR %d NOT FOUND\n", miss);
      }
   }
   tr.passIfNoException();
   */

   tr.ungroup();
}

void runHashTableConcurrencyTest(
   TestRunner& tr, uint32_t threads, uint32_t reads, uint32_t writes)
{
   tr.group("HashTable concurrency");

   char name[100];
   snprintf(name, 100,
      "RW threads:%" PRIu32
      " reads:%" PRIu32
      " writes:%" PRIu32,
      threads, reads, writes);
   tr.test(name);
   {
   }
   tr.passIfNoException();

   tr.ungroup();
}

void runHashTableVsMapTest(
   TestRunner& tr, uint32_t reads, uint32_t writes)
{
   int threads = 1;
   char name[100];
   tr.group("HashTable vs Map Single Thread");

   bool csv = true;
   const char* comment = csv ? "#" : "";
   const char* sep = csv ? "," : " ";
   uint32_t wr = writes + reads;

   printf("%s w:%" PRIu32 " r:%" PRIu32 "\n", comment, reads, writes);
   uint64_t mrdt, mwdt;
   uint64_t hrdt, hwdt;

   snprintf(name, 100,
      "map RW"
      " reads:%" PRIu32
      " writes:%" PRIu32,
      reads, writes);
   tr.test(name);
   {
      uint64_t rstart, wstart;

      map<int, uint32_t> m;

      printf("%s map\n", comment);
      printf(
         "%1s%8s%s"
         "%9s%s%9s%s"
         "%9s%s%9s%s"
         "%9s%s%9s\n",
         comment, "threads", sep,
         "total (s)", sep, "rw/s", sep,
         "write (s)", sep, "w/s", sep,
         "read (s)", sep, "r/s");

      wstart = Timer::startTiming();
      for(uint32_t i = 0; i < writes; i++)
      {
         m[0] = i;
      }
      mwdt = Timer::getMilliseconds(wstart);

      uint32_t v;
      rstart = Timer::startTiming();
      for(uint32_t i = 0; i < reads; i++)
      {
         v = m[0];
      }
      mrdt = Timer::getMilliseconds(rstart);

      uint64_t dt = mwdt + mrdt;
      printf(
         "%9d%s"
         "%9.3f%s%9.0f%s"
         "%9.3f%s%9.0f%s"
         "%9.3f%s%9.0f\n",
         threads, sep,
         dt/1000.0, sep, wr/(double)dt, sep,
         mwdt/1000.0, sep, writes/(double)mwdt, sep,
         mrdt/1000.0, sep, reads/(double)mrdt);
   }
   tr.passIfNoException();

   snprintf(name, 100,
      "ht RW"
      " reads:%" PRIu32
      " writes:%" PRIu32,
      reads, writes);
   tr.test(name);
   {
      uint64_t rstart, wstart;

      HashTable<int, uint32_t, KeyAsHash> h;

      printf("%s ht\n", comment);
      printf(
         "%1s%8s%s"
         "%9s%s%9s%s"
         "%9s%s%9s%s"
         "%9s%s%9s\n",
         comment, "threads", sep,
         "total (s)", sep, "rw/s", sep,
         "write (s)", sep, "w/s", sep,
         "read (s)", sep, "r/s");

      wstart = Timer::startTiming();
      for(uint32_t i = 0; i < writes; i++)
      {
         h.put(0, i);
      }
      hwdt = Timer::getMilliseconds(wstart);

      uint32_t v;
      rstart = Timer::startTiming();
      for(uint32_t i = 0; i < reads; i++)
      {
         h.get(0, v);
      }
      hrdt = Timer::getMilliseconds(rstart);

      uint64_t dt = hwdt + hrdt;
      printf(
         "%9d%s"
         "%9.3f%s%9.0f%s"
         "%9.3f%s%9.0f%s"
         "%9.3f%s%9.0f\n",
         threads, sep,
         dt/1000.0, sep, wr/(double)dt, sep,
         hwdt/1000.0, sep, writes/(double)hwdt, sep,
         hrdt/1000.0, sep, reads/(double)hrdt);
   }
   tr.passIfNoException();

   printf("%s ratio ht/m\n", comment);
   printf(
      "%9d%s"
      "%9.3f%s%9s%s"
      "%9.3f%s%9s%s"
      "%9.3f%s%9s\n",
      threads, sep,
      ((double)hwdt+hrdt)/(mwdt+mrdt), sep, "", sep,
      ((double)hwdt/mwdt), sep, "", sep,
      ((double)hrdt/mrdt), sep, "");

   snprintf(name, 100,
      "map excl lock RW"
      " reads:%" PRIu32
      " writes:%" PRIu32,
      reads, writes);
   tr.test(name);
   {
      uint64_t rstart, wstart;

      map<int, uint32_t> m;
      ExclusiveLock lock;

      printf("%s map exclusive lock\n", comment);
      printf(
         "%1s%8s%s"
         "%9s%s%9s%s"
         "%9s%s%9s%s"
         "%9s%s%9s\n",
         comment, "threads", sep,
         "total (s)", sep, "rw/s", sep,
         "write (s)", sep, "w/s", sep,
         "read (s)", sep, "r/s");

      wstart = Timer::startTiming();
      for(uint32_t i = 0; i < writes; i++)
      {
         lock.lock();
         m[0] = i;
         lock.unlock();
      }
      mwdt = Timer::getMilliseconds(wstart);

      uint32_t v;
      rstart = Timer::startTiming();
      for(uint32_t i = 0; i < reads; i++)
      {
         lock.lock();
         v = m[0];
         lock.unlock();
      }
      mrdt = Timer::getMilliseconds(rstart);

      uint64_t dt = mwdt + mrdt;
      printf(
         "%9d%s"
         "%9.3f%s%9.0f%s"
         "%9.3f%s%9.0f%s"
         "%9.3f%s%9.0f\n",
         threads, sep,
         dt/1000.0, sep, wr/(double)dt, sep,
         mwdt/1000.0, sep, writes/(double)mwdt, sep,
         mrdt/1000.0, sep, reads/(double)mrdt);
   }
   tr.passIfNoException();

   snprintf(name, 100,
      "map shared lock RW"
      " reads:%" PRIu32
      " writes:%" PRIu32,
      reads, writes);
   tr.test(name);
   {
      uint64_t rstart, wstart;

      map<int, uint32_t> m;
      SharedLock lock;

      printf("%s map shared lock\n", comment);
      printf(
         "%1s%8s%s"
         "%9s%s%9s%s"
         "%9s%s%9s%s"
         "%9s%s%9s\n",
         comment, "threads", sep,
         "total (s)", sep, "rw/s", sep,
         "write (s)", sep, "w/s", sep,
         "read (s)", sep, "r/s");

      wstart = Timer::startTiming();
      for(uint32_t i = 0; i < writes; i++)
      {
         lock.lockExclusive();
         m[0] = i;
         lock.unlockExclusive();
      }
      mwdt = Timer::getMilliseconds(wstart);

      uint32_t v;
      rstart = Timer::startTiming();
      for(uint32_t i = 0; i < reads; i++)
      {
         lock.lockShared();
         v = m[0];
         lock.unlockShared();
      }
      mrdt = Timer::getMilliseconds(rstart);

      uint64_t dt = mwdt + mrdt;
      printf(
         "%9d%s"
         "%9.3f%s%9.0f%s"
         "%9.3f%s%9.0f%s"
         "%9.3f%s%9.0f\n",
         threads, sep,
         dt/1000.0, sep, wr/(double)dt, sep,
         mwdt/1000.0, sep, writes/(double)mwdt, sep,
         mrdt/1000.0, sep, reads/(double)mrdt);
   }
   tr.passIfNoException();

   tr.ungroup();
}

void runHashTableVsMapThreadsTest(
   TestRunner& tr, uint32_t threads, uint32_t reads, uint32_t writes)
{
   tr.group("HashTable vs Map Multiple Threads");

   char name[100];
   snprintf(name, 100,
      "RW threads:%" PRIu32
      " reads:%" PRIu32
      " writes:%" PRIu32,
      threads, reads, writes);
   tr.test(name);
   {
   }
   tr.passIfNoException();

   tr.ungroup();
}

class DbHashTableTester : public db::test::Tester
{
public:
   DbHashTableTester()
   {
      setName("HashTable");
   }

   /**
    * Run automatic unit tests.
    */
   virtual int runAutomaticTests(TestRunner& tr)
   {
      runHashTableTests(tr);
      return 0;
   }

   /**
    * Runs interactive unit tests.
    *
    * Options:
    * --test all - run all tests
    * --test threads - test thread concurrency
    * --test map - test speed vs map (single threaded)
    * --test mapthreads - test speed vs map (multiple threads w/ locked map)
    * --option threads <n> - number of threads
    * --option reads <n> - number of read operations
    * --option writes <n> - number of write operations
    */
   virtual int runInteractiveTests(TestRunner& tr)
   {
      Config cfg = tr.getApp()->getConfig();
      const char* test = cfg["db.test.Tester"]["test"]->getString();
      bool all = (strcmp(test, "all") == 0);

      uint32_t threads =
         cfg->hasMember("threads") ? cfg["threads"]->getUInt32() : 1;
      uint32_t reads =
         cfg->hasMember("reads") ? cfg["reads"]->getUInt32() : 1;
      uint32_t writes =
         cfg->hasMember("writes") ? cfg["writes"]->getUInt32() : 1;
      //uint32_t loops =
      //   cfg->hasMember("loops") ? cfg["loops"]->getUInt32() : 1;
      //uint32_t ops =
      //   cfg->hasMember("ops") ? cfg["ops"]->getUInt32() : 1;

      if(all || (strcmp(test, "threads") == 0))
      {
         runHashTableConcurrencyTest(tr, threads, reads, writes);
      }
      if(all || (strcmp(test, "map") == 0))
      {
         runHashTableVsMapTest(tr, reads, writes);
      }
      if(all || (strcmp(test, "mapthreads") == 0))
      {
         runHashTableVsMapThreadsTest(tr, threads, reads, writes);
      }
      return 0;
   }
};

db::test::Tester* getDbHashTableTester() { return new DbHashTableTester(); }

DB_TEST_MAIN(DbHashTableTester)
<|endoftext|>


<commit_before>#include "node.hpp"
#include <set>
#include <fstream>
#include <queue>
#include <functional>

#ifndef HUFFMAN_TREE_HPP
#define HUFFMAN_TREE_HPP

namespace cpr
{
	namespace huffman
	{
		template<typename Char, typename Freq>
		class HuffmanTree
		{
		public:
			using SharedNode = Node<Char, Freq>::SharedNode;

			template<typename Container>
			explicit HuffmanTree(Container const& nodes)
				: root_{ make_tree(nodes) }
			{	}

		private:
			SharedNode root_;

			template<typename Container>
			SharedNode make_tree(Container const& nodes) const
			{
				auto greater = [](SharedNode lhs, SharedNode rhs){return lhs->freq_ > rhs->freq_; };
				using MinPriorityQueue = std::priority_queue < SharedNode, std::vector<SharedNode>>, decltype(greater) > ;

				MinPriorityQueue queue(greater);
				for (auto const& n : node)
					queue.push(make_new_node(n));

				for (int _ = 1; _ != container.size(); ++_)
				{
					auto z = make_new_node();
					z->left_ = make_new_node(queue.top());
					queue.pop();
					z->right_ = make_new_node(queue.top());
					queue.pop();
					z.freq_ = z.left_->freq_ + z.right_->freq_;
					queue.push(z);
				}
			}
		};
	}
}

#endif // !HUFFMAN_TREE_HPP<commit_msg>use std::map for frequency table instead<commit_after>#include "node.hpp"
#include <set>
#include <fstream>
#include <queue>
#include <functional>

#ifndef HUFFMAN_TREE_HPP
#define HUFFMAN_TREE_HPP

namespace cpr
{
	namespace huffman
	{
		template<typename Char, typename Freq>
		class HuffmanTree
		{
		public:
			using SharedNode = Node<Char, Freq>::SharedNode;

			//template<typename Container>
			//explicit HuffmanTree(Container const& nodes)
			//	: root_{ make_tree(nodes) }
			//{	}

		private:
			SharedNode root_;

			//template<typename Container>
			//SharedNode make_tree(Container const& nodes) const
			//{
			//	auto greater = [](SharedNode lhs, SharedNode rhs){return lhs->freq_ > rhs->freq_; };
			//	using MinPriorityQueue = std::priority_queue < SharedNode, std::vector<SharedNode>>, decltype(greater) > ;

			//	MinPriorityQueue queue(greater);
			//	for (auto const& n : node)
			//		queue.push(make_new_node(n));

			//	for (int _ = 1; _ != container.size(); ++_)
			//	{
			//		auto z = make_new_node();
			//		z->left_ = make_new_node(queue.top());
			//		queue.pop();
			//		z->right_ = make_new_node(queue.top());
			//		queue.pop();
			//		z.freq_ = z.left_->freq_ + z.right_->freq_;
			//		queue.push(z);
			//	}
			//}
		};
	}
}

#endif // !HUFFMAN_TREE_HPP<|endoftext|>


<commit_before>/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "selectable.hh"
#include "selectable_with_field_selection.hh"
#include "field_selector.hh"
#include "writetime_or_ttl.hh"
#include "selector_factories.hh"
#include "cql3/functions/functions.hh"
#include "cql3/functions/castas_fcts.hh"
#include "cql3/functions/aggregate_fcts.hh"
#include "abstract_function_selector.hh"
#include "writetime_or_ttl_selector.hh"

namespace cql3 {

namespace selection {

shared_ptr<selector::factory>
selectable::writetime_or_ttl::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    auto&& def = s->get_column_definition(_id->name());
    if (!def) {
        throw exceptions::invalid_request_exception(format("Undefined name {} in selection clause", _id));
    }
    if (def->is_primary_key()) {
        throw exceptions::invalid_request_exception(
                format("Cannot use selection function {} on PRIMARY KEY part {}",
                              _is_writetime ? "writeTime" : "ttl",
                              def->name()));
    }
    if (def->type->is_multi_cell()) {
        throw exceptions::invalid_request_exception(format("Cannot use selection function {} on non-frozen collections",
                                                        _is_writetime ? "writeTime" : "ttl"));
    }

    return writetime_or_ttl_selector::new_factory(def->name_as_text(), add_and_get_index(*def, defs), _is_writetime);
}

sstring
selectable::writetime_or_ttl::to_string() const {
    return format("{}({})", _is_writetime ? "writetime" : "ttl", _id->to_string());
}

shared_ptr<selectable>
selectable::writetime_or_ttl::raw::prepare(schema_ptr s) {
    return make_shared<writetime_or_ttl>(_id->prepare_column_identifier(s), _is_writetime);
}

bool
selectable::writetime_or_ttl::raw::processes_selection() const {
    return true;
}

shared_ptr<selector::factory>
selectable::with_function::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    auto&& factories = selector_factories::create_factories_and_collect_column_definitions(_args, db, s, defs);

    // resolve built-in functions before user defined functions
    auto&& fun = functions::functions::get(db, s->ks_name(), _function_name, factories->new_instances(), s->ks_name(), s->cf_name());
    if (!fun) {
        throw exceptions::invalid_request_exception(format("Unknown function '{}'", _function_name));
    }
    if (!fun->return_type()) {
        throw exceptions::invalid_request_exception(format("Unknown function {} called in selection clause", _function_name));
    }

    return abstract_function_selector::new_factory(std::move(fun), std::move(factories));
}

sstring
selectable::with_function::to_string() const {
    return format("{}({})", _function_name.name, join(", ", _args));
}

shared_ptr<selectable>
selectable::with_function::raw::prepare(schema_ptr s) {
        std::vector<shared_ptr<selectable>> prepared_args;
        prepared_args.reserve(_args.size());
        for (auto&& arg : _args) {
            prepared_args.push_back(arg->prepare(s));
        }
        return ::make_shared<with_function>(_function_name, std::move(prepared_args));
    }

bool
selectable::with_function::raw::processes_selection() const {
    return true;
}

shared_ptr<selectable::with_function::raw>
selectable::with_function::raw::make_count_rows_function() {
    return ::make_shared<cql3::selection::selectable::with_function::raw>(
            cql3::functions::function_name::native_function(cql3::functions::aggregate_fcts::COUNT_ROWS_FUNCTION_NAME),
                    std::vector<shared_ptr<cql3::selection::selectable::raw>>());
}

shared_ptr<selector::factory>
selectable::with_anonymous_function::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    auto&& factories = selector_factories::create_factories_and_collect_column_definitions(_args, db, s, defs);
    return abstract_function_selector::new_factory(_function, std::move(factories));
}

sstring
selectable::with_anonymous_function::to_string() const {
    return format("{}({})", _function->name().name, join(", ", _args));
}

shared_ptr<selectable>
selectable::with_anonymous_function::raw::prepare(schema_ptr s) {
        std::vector<shared_ptr<selectable>> prepared_args;
        prepared_args.reserve(_args.size());
        for (auto&& arg : _args) {
            prepared_args.push_back(arg->prepare(s));
        }
        return ::make_shared<with_anonymous_function>(_function, std::move(prepared_args));
    }

bool
selectable::with_anonymous_function::raw::processes_selection() const {
    return true;
}

shared_ptr<selector::factory>
selectable::with_field_selection::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    auto&& factory = _selected->new_selector_factory(db, s, defs);
    auto&& type = factory->new_instance()->get_type();
    auto&& ut = dynamic_pointer_cast<const user_type_impl>(std::move(type));
    if (!ut) {
        throw exceptions::invalid_request_exception(
                format("Invalid field selection: {} of type {} is not a user type",
                       _selected->to_string(), factory->new_instance()->get_type()->as_cql3_type()));
    }
    for (size_t i = 0; i < ut->size(); ++i) {
        if (ut->field_name(i) != _field->bytes_) {
            continue;
        }
        return field_selector::new_factory(std::move(ut), i, std::move(factory));
    }
    throw exceptions::invalid_request_exception(format("{} of type {} has no field {}",
                                                       _selected->to_string(), ut->as_cql3_type(), _field));
}

sstring
selectable::with_field_selection::to_string() const {
    return format("{}.{}", _selected->to_string(), _field->to_string());
}

shared_ptr<selectable>
selectable::with_field_selection::raw::prepare(schema_ptr s) {
    // static_pointer_cast<> needed due to lack of covariant return type
    // support with smart pointers
    return make_shared<with_field_selection>(_selected->prepare(s),
            static_pointer_cast<column_identifier>(_field->prepare(s)));
}

bool
selectable::with_field_selection::raw::processes_selection() const {
    return true;
}

shared_ptr<selector::factory>
selectable::with_cast::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    std::vector<shared_ptr<selectable>> args{_arg};
    auto&& factories = selector_factories::create_factories_and_collect_column_definitions(args, db, s, defs);
    auto&& fun = functions::castas_functions::get(_type->get_type(), factories->new_instances(), s);

    return abstract_function_selector::new_factory(std::move(fun), std::move(factories));
}

sstring
selectable::with_cast::to_string() const {
    return format("cast({} as {})", _arg->to_string(), _type->to_string());
}

shared_ptr<selectable>
selectable::with_cast::raw::prepare(schema_ptr s) {
    return ::make_shared<selectable::with_cast>(_arg->prepare(s), _type);
}

bool
selectable::with_cast::raw::processes_selection() const {
    return true;
}

std::ostream & operator<<(std::ostream &os, const selectable& s) {
    return os << s.to_string();
}

}

}
<commit_msg>cql3: hide virtual columns from WRITETIME() and TTL()<commit_after>/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "selectable.hh"
#include "selectable_with_field_selection.hh"
#include "field_selector.hh"
#include "writetime_or_ttl.hh"
#include "selector_factories.hh"
#include "cql3/functions/functions.hh"
#include "cql3/functions/castas_fcts.hh"
#include "cql3/functions/aggregate_fcts.hh"
#include "abstract_function_selector.hh"
#include "writetime_or_ttl_selector.hh"

namespace cql3 {

namespace selection {

shared_ptr<selector::factory>
selectable::writetime_or_ttl::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    auto&& def = s->get_column_definition(_id->name());
    if (!def || def->is_hidden_from_cql()) {
        throw exceptions::invalid_request_exception(format("Undefined name {} in selection clause", _id));
    }
    if (def->is_primary_key()) {
        throw exceptions::invalid_request_exception(
                format("Cannot use selection function {} on PRIMARY KEY part {}",
                              _is_writetime ? "writeTime" : "ttl",
                              def->name()));
    }
    if (def->type->is_multi_cell()) {
        throw exceptions::invalid_request_exception(format("Cannot use selection function {} on non-frozen collections",
                                                        _is_writetime ? "writeTime" : "ttl"));
    }

    return writetime_or_ttl_selector::new_factory(def->name_as_text(), add_and_get_index(*def, defs), _is_writetime);
}

sstring
selectable::writetime_or_ttl::to_string() const {
    return format("{}({})", _is_writetime ? "writetime" : "ttl", _id->to_string());
}

shared_ptr<selectable>
selectable::writetime_or_ttl::raw::prepare(schema_ptr s) {
    return make_shared<writetime_or_ttl>(_id->prepare_column_identifier(s), _is_writetime);
}

bool
selectable::writetime_or_ttl::raw::processes_selection() const {
    return true;
}

shared_ptr<selector::factory>
selectable::with_function::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    auto&& factories = selector_factories::create_factories_and_collect_column_definitions(_args, db, s, defs);

    // resolve built-in functions before user defined functions
    auto&& fun = functions::functions::get(db, s->ks_name(), _function_name, factories->new_instances(), s->ks_name(), s->cf_name());
    if (!fun) {
        throw exceptions::invalid_request_exception(format("Unknown function '{}'", _function_name));
    }
    if (!fun->return_type()) {
        throw exceptions::invalid_request_exception(format("Unknown function {} called in selection clause", _function_name));
    }

    return abstract_function_selector::new_factory(std::move(fun), std::move(factories));
}

sstring
selectable::with_function::to_string() const {
    return format("{}({})", _function_name.name, join(", ", _args));
}

shared_ptr<selectable>
selectable::with_function::raw::prepare(schema_ptr s) {
        std::vector<shared_ptr<selectable>> prepared_args;
        prepared_args.reserve(_args.size());
        for (auto&& arg : _args) {
            prepared_args.push_back(arg->prepare(s));
        }
        return ::make_shared<with_function>(_function_name, std::move(prepared_args));
    }

bool
selectable::with_function::raw::processes_selection() const {
    return true;
}

shared_ptr<selectable::with_function::raw>
selectable::with_function::raw::make_count_rows_function() {
    return ::make_shared<cql3::selection::selectable::with_function::raw>(
            cql3::functions::function_name::native_function(cql3::functions::aggregate_fcts::COUNT_ROWS_FUNCTION_NAME),
                    std::vector<shared_ptr<cql3::selection::selectable::raw>>());
}

shared_ptr<selector::factory>
selectable::with_anonymous_function::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    auto&& factories = selector_factories::create_factories_and_collect_column_definitions(_args, db, s, defs);
    return abstract_function_selector::new_factory(_function, std::move(factories));
}

sstring
selectable::with_anonymous_function::to_string() const {
    return format("{}({})", _function->name().name, join(", ", _args));
}

shared_ptr<selectable>
selectable::with_anonymous_function::raw::prepare(schema_ptr s) {
        std::vector<shared_ptr<selectable>> prepared_args;
        prepared_args.reserve(_args.size());
        for (auto&& arg : _args) {
            prepared_args.push_back(arg->prepare(s));
        }
        return ::make_shared<with_anonymous_function>(_function, std::move(prepared_args));
    }

bool
selectable::with_anonymous_function::raw::processes_selection() const {
    return true;
}

shared_ptr<selector::factory>
selectable::with_field_selection::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    auto&& factory = _selected->new_selector_factory(db, s, defs);
    auto&& type = factory->new_instance()->get_type();
    auto&& ut = dynamic_pointer_cast<const user_type_impl>(std::move(type));
    if (!ut) {
        throw exceptions::invalid_request_exception(
                format("Invalid field selection: {} of type {} is not a user type",
                       _selected->to_string(), factory->new_instance()->get_type()->as_cql3_type()));
    }
    for (size_t i = 0; i < ut->size(); ++i) {
        if (ut->field_name(i) != _field->bytes_) {
            continue;
        }
        return field_selector::new_factory(std::move(ut), i, std::move(factory));
    }
    throw exceptions::invalid_request_exception(format("{} of type {} has no field {}",
                                                       _selected->to_string(), ut->as_cql3_type(), _field));
}

sstring
selectable::with_field_selection::to_string() const {
    return format("{}.{}", _selected->to_string(), _field->to_string());
}

shared_ptr<selectable>
selectable::with_field_selection::raw::prepare(schema_ptr s) {
    // static_pointer_cast<> needed due to lack of covariant return type
    // support with smart pointers
    return make_shared<with_field_selection>(_selected->prepare(s),
            static_pointer_cast<column_identifier>(_field->prepare(s)));
}

bool
selectable::with_field_selection::raw::processes_selection() const {
    return true;
}

shared_ptr<selector::factory>
selectable::with_cast::new_selector_factory(database& db, schema_ptr s, std::vector<const column_definition*>& defs) {
    std::vector<shared_ptr<selectable>> args{_arg};
    auto&& factories = selector_factories::create_factories_and_collect_column_definitions(args, db, s, defs);
    auto&& fun = functions::castas_functions::get(_type->get_type(), factories->new_instances(), s);

    return abstract_function_selector::new_factory(std::move(fun), std::move(factories));
}

sstring
selectable::with_cast::to_string() const {
    return format("cast({} as {})", _arg->to_string(), _type->to_string());
}

shared_ptr<selectable>
selectable::with_cast::raw::prepare(schema_ptr s) {
    return ::make_shared<selectable::with_cast>(_arg->prepare(s), _type);
}

bool
selectable::with_cast::raw::processes_selection() const {
    return true;
}

std::ostream & operator<<(std::ostream &os, const selectable& s) {
    return os << s.to_string();
}

}

}
<|endoftext|>