<commit_before>/**
 *  @copyright Copyright 2016 The J-PET Framework Authors. All rights reserved.
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may find a copy of the License in the LICENCE file.
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *  @file main.cpp
 */

#include <DBHandler/HeaderFiles/DBHandler.h>
#include <JPetManager/JPetManager.h>
#include <JPetTaskLoader/JPetTaskLoader.h>
#include "TaskA.h"
#include "TaskB1.h"
#include "TaskC.h"
#include "TaskD.h"
#include "TaskE.h"

using namespace std;

int main(int argc, char* argv[]) {
	DB::SERVICES::DBHandler::createDBConnection("../DBConfig/configDB.cfg");
  JPetManager& manager = JPetManager::getManager();
  manager.parseCmdLine(argc, argv);

  // Here create all analysis modules to be used:

  // manager.registerTask([](){
  //     return new JPetTaskLoader("hld", "tslot.raw",
  // 				  new TaskA("Module A: Unp to TSlot Raw",
  // 					    "Process unpacked HLD file into a tree of JPetTSlot objects"));
  //   });

  // manager.registerTask([](){
  //     return new JPetTaskLoader("tslot.raw", "raw.sig",
  // 				new TaskB1("Module B1: Make TOT histos and assemble signals",
  // 					   "Assemble signals and create TOT historgrams"));
  //   });

   manager.registerTask([](){
       return new JPetTaskLoader("raw.sig", "phys.hit",
				 new TaskC("Module C: Pair signals",
					   "Create hits from pairs of signals"));
     });

  // manager.registerTask([](){
  //     return new JPetTaskLoader("phys.hit", "phys.hit.means",
  // 				new TaskD("Module D: Make histograms for hits",
  // 					  "Only make timeDiff histos and produce mean timeDiff value for each threshold and slot to be used by the next module"));
  //   });

  // manager.registerTask([](){
  //     return new JPetTaskLoader("phys.hit.means", "phys.hit.coincplots",
  // 				new TaskE("Module E: Filter hits",
  // 					  "Pass only hits with time diffrerence close to the peak"));
  //   });

  manager.run();
}
<commit_msg>Uncomment modules in the example<commit_after>/**
 *  @copyright Copyright 2016 The J-PET Framework Authors. All rights reserved.
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may find a copy of the License in the LICENCE file.
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *  @file main.cpp
 */

#include <DBHandler/HeaderFiles/DBHandler.h>
#include <JPetManager/JPetManager.h>
#include <JPetTaskLoader/JPetTaskLoader.h>
#include "TaskA.h"
#include "TaskB1.h"
#include "TaskC.h"
#include "TaskD.h"
#include "TaskE.h"

using namespace std;

int main(int argc, char* argv[]) {
	DB::SERVICES::DBHandler::createDBConnection("../DBConfig/configDB.cfg");
  JPetManager& manager = JPetManager::getManager();
  manager.parseCmdLine(argc, argv);

  // Here create all analysis modules to be used:

  manager.registerTask([](){
      return new JPetTaskLoader("hld", "tslot.raw",
  				  new TaskA("Module A: Unp to TSlot Raw",
  					    "Process unpacked HLD file into a tree of JPetTSlot objects"));
    });

  manager.registerTask([](){
      return new JPetTaskLoader("tslot.raw", "raw.sig",
  				new TaskB1("Module B1: Make TOT histos and assemble signals",
					   "Assemble signals and create TOT historgrams"));
    });

  manager.registerTask([](){
      return new JPetTaskLoader("raw.sig", "phys.hit",
  				new TaskC("Module C: Pair signals",
  					  "Create hits from pairs of signals"));
    });

  manager.registerTask([](){
      return new JPetTaskLoader("phys.hit", "phys.hit.means",
				new TaskD("Module D: Make histograms for hits",
					  "Only make timeDiff histos and produce mean timeDiff value for each threshold and slot to be used by the next module"));
    });

  manager.registerTask([](){
      return new JPetTaskLoader("phys.hit.means", "phys.hit.coincplots",
  				new TaskE("Module E: Filter hits",
  					  "Pass only hits with time diffrerence close to the peak"));
    });

  manager.run();
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2017, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include "ExternalCarbonConnectionImpl.h"

#include <gflags/gflags.h>

#include <folly/Baton.h>
#include <folly/Memory.h>
#include <folly/ScopeGuard.h>
#include <folly/Singleton.h>
#include <folly/ThreadName.h>
#include <folly/fibers/EventBaseLoopController.h>
#include <folly/fibers/FiberManager.h>
#include <folly/io/async/EventBaseManager.h>

#include "mcrouter/lib/McResUtil.h"

DEFINE_int32(
    cacheclient_external_connection_threads,
    4,
    "Thread count for ExternalCarbonConnectionImpl");

namespace carbon {
namespace detail {

Client::Client(
    facebook::memcache::ConnectionOptions connectionOptions,
    ExternalCarbonConnectionImpl::Options options)
    : connectionOptions_(connectionOptions),
      options_(options),
      client_(
          *folly::EventBaseManager::get()->getEventBase(),
          connectionOptions_) {
  if (options_.maxOutstanding > 0) {
    counting_sem_init(&outstandingReqsSem_, options_.maxOutstanding);
  }
}

Client::~Client() {
  closeNow();
}

size_t Client::limitRequests(size_t requestsCount) {
  if (options_.maxOutstanding == 0) {
    return requestsCount;
  }

  if (options_.maxOutstandingError) {
    return counting_sem_lazy_nonblocking(&outstandingReqsSem_, requestsCount);
  } else {
    return counting_sem_lazy_wait(&outstandingReqsSem_, requestsCount);
  }
}

void Client::closeNow() {
  client_.closeNow();
}

ThreadInfo::ThreadInfo()
    : fiberManager_(
          folly::make_unique<folly::fibers::EventBaseLoopController>()) {
  folly::Baton<> baton;

  thread_ = std::thread([this, &baton] {
    folly::setThreadName("mc-eccc-pool");

    folly::EventBase* evb = folly::EventBaseManager::get()->getEventBase();
    dynamic_cast<folly::fibers::EventBaseLoopController&>(
        fiberManager_.loopController())
        .attachEventBase(*evb);
    // At this point it is safe to use fiberManager.
    baton.post();
    evb->loopForever();

    // Close all connections.
    for (auto& client : clients_) {
      client->closeNow();
    }
    clients_.clear();

    // Drain fiber manager.
    while (fiberManager_.hasTasks()) {
      evb->loop();
    }

    // Drain all remaining events.
    evb->loop();
  });

  // Wait until the thread is properly initialized.
  baton.wait();
}

std::weak_ptr<Client> ThreadInfo::createClient(
    facebook::memcache::ConnectionOptions connectionOptions,
    ExternalCarbonConnectionImpl::Options options) {
  return folly::fibers::await(
      [&](folly::fibers::Promise<std::weak_ptr<Client>> p) {
        fiberManager_.addTaskRemote([
          this,
          promise = std::move(p),
          connectionOptions = std::move(connectionOptions),
          options
        ]() mutable {
          auto client = std::make_shared<Client>(connectionOptions, options);
          clients_.insert(client);
          promise.setValue(client);
        });
      });
}

void ThreadInfo::releaseClient(std::weak_ptr<Client> clientWeak) {
  fiberManager_.addTaskRemote([this, clientWeak] {
    if (auto client = clientWeak.lock()) {
      clients_.erase(client);
      client->closeNow();
      assert(client.unique());
    }
  });
}

ThreadInfo::~ThreadInfo() {
  fiberManager_.addTaskRemote([] {
    folly::EventBaseManager::get()->getEventBase()->terminateLoopSoon();
  });
  thread_.join();
}

} // detail

namespace {

// Pool of threads, each of them has its own EventBase and FiberManager.
class ThreadPool : public std::enable_shared_from_this<ThreadPool> {
 public:
  ThreadPool() {
    threads_.reserve(FLAGS_cacheclient_external_connection_threads);
  }

  std::pair<std::weak_ptr<detail::Client>, std::weak_ptr<detail::ThreadInfo>>
  createClient(
      facebook::memcache::ConnectionOptions connectionOptions,
      ExternalCarbonConnectionImpl::Options options) {
    auto& threadInfo = [&]() -> detail::ThreadInfo& {
      std::lock_guard<std::mutex> lck(mutex_);
      // Select a thread in round robin fashion.
      size_t threadId =
          (nextThreadId_++) % FLAGS_cacheclient_external_connection_threads;

      // If it's not running yet, then we need to start it.
      if (threads_.size() <= threadId) {
        threads_.emplace_back(folly::make_unique<detail::ThreadInfo>());
      }
      return *threads_[threadId];
    }();

    std::weak_ptr<detail::Client> clientPtr =
        threadInfo.createClient(connectionOptions, options);

    std::shared_ptr<detail::ThreadInfo> threadInfoShared(
        shared_from_this(), &threadInfo);

    return std::make_pair(
        std::move(clientPtr),
        std::weak_ptr<detail::ThreadInfo>(std::move(threadInfoShared)));
  }

 private:
  std::mutex mutex_;
  size_t nextThreadId_{0};
  std::vector<std::unique_ptr<detail::ThreadInfo>> threads_;
};

folly::Singleton<ThreadPool> threadPool;

} // anonymous namespace

ExternalCarbonConnectionImpl::Impl::Impl(
    facebook::memcache::ConnectionOptions connectionOptions,
    ExternalCarbonConnectionImpl::Options options) {
  auto pool = threadPool.try_get();

  auto info = pool->createClient(connectionOptions, options);
  client_ = info.first;
  threadInfo_ = info.second;
}

ExternalCarbonConnectionImpl::Impl::~Impl() {
  if (auto threadInfo = threadInfo_.lock()) {
    threadInfo->releaseClient(client_);
  }
}

bool ExternalCarbonConnectionImpl::Impl::healthCheck() {
  folly::fibers::Baton baton;
  bool ret = false;

  auto clientWeak = client_;
  auto threadInfo = threadInfo_.lock();
  if (!threadInfo) {
    throw CarbonConnectionRecreateException(
        "Singleton<ThreadPool> was destroyed!");
  }

  threadInfo->addTaskRemote([clientWeak, &baton, &ret]() {
    auto client = clientWeak.lock();
    if (!client) {
      baton.post();
      return;
    }

    auto reply = client->sendRequest(facebook::memcache::McVersionRequest());
    ret = !facebook::memcache::isErrorResult(reply.result());
    baton.post();
  });

  baton.wait();
  return ret;
}

ExternalCarbonConnectionImpl::ExternalCarbonConnectionImpl(
    facebook::memcache::ConnectionOptions connectionOptions,
    Options options)
    : connectionOptions_(std::move(connectionOptions)),
      options_(std::move(options)),
      impl_(folly::make_unique<Impl>(connectionOptions_, options_)) {}

bool ExternalCarbonConnectionImpl::healthCheck() {
  try {
    return impl_->healthCheck();
  } catch (const CarbonConnectionRecreateException&) {
    impl_ = folly::make_unique<Impl>(connectionOptions_, options_);
    return impl_->healthCheck();
  }
}
} // carbon
<commit_msg>Correctly applying fix to race condition in external connection<commit_after>/*
 *  Copyright (c) 2017, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include "ExternalCarbonConnectionImpl.h"

#include <gflags/gflags.h>

#include <folly/Baton.h>
#include <folly/Memory.h>
#include <folly/ScopeGuard.h>
#include <folly/Singleton.h>
#include <folly/ThreadName.h>
#include <folly/fibers/EventBaseLoopController.h>
#include <folly/fibers/FiberManager.h>
#include <folly/io/async/EventBaseManager.h>

#include "mcrouter/lib/McResUtil.h"

DEFINE_int32(
    cacheclient_external_connection_threads,
    4,
    "Thread count for ExternalCarbonConnectionImpl");

namespace carbon {
namespace detail {

Client::Client(
    facebook::memcache::ConnectionOptions connectionOptions,
    ExternalCarbonConnectionImpl::Options options)
    : connectionOptions_(connectionOptions),
      options_(options),
      client_(
          *folly::EventBaseManager::get()->getEventBase(),
          connectionOptions_) {
  if (options_.maxOutstanding > 0) {
    counting_sem_init(&outstandingReqsSem_, options_.maxOutstanding);
  }
}

Client::~Client() {
  closeNow();
}

size_t Client::limitRequests(size_t requestsCount) {
  if (options_.maxOutstanding == 0) {
    return requestsCount;
  }

  if (options_.maxOutstandingError) {
    return counting_sem_lazy_nonblocking(&outstandingReqsSem_, requestsCount);
  } else {
    return counting_sem_lazy_wait(&outstandingReqsSem_, requestsCount);
  }
}

void Client::closeNow() {
  client_.closeNow();
}

ThreadInfo::ThreadInfo()
    : fiberManager_(
          folly::make_unique<folly::fibers::EventBaseLoopController>()) {
  folly::Baton<> baton;

  thread_ = std::thread([this, &baton] {
    folly::setThreadName("mc-eccc-pool");

    folly::EventBase* evb = folly::EventBaseManager::get()->getEventBase();
    dynamic_cast<folly::fibers::EventBaseLoopController&>(
        fiberManager_.loopController())
        .attachEventBase(*evb);
    // At this point it is safe to use fiberManager.
    baton.post();
    evb->loopForever();

    // Close all connections.
    for (auto& client : clients_) {
      client->closeNow();
    }
    clients_.clear();

    // Drain fiber manager.
    while (fiberManager_.hasTasks()) {
      evb->loop();
    }

    // Drain all remaining events.
    evb->loop();
  });

  // Wait until the thread is properly initialized.
  baton.wait();
}

std::weak_ptr<Client> ThreadInfo::createClient(
    facebook::memcache::ConnectionOptions connectionOptions,
    ExternalCarbonConnectionImpl::Options options) {
  return folly::fibers::await(
      [&](folly::fibers::Promise<std::weak_ptr<Client>> p) {
        fiberManager_.addTaskRemote([
          this,
          promise = std::move(p),
          connectionOptions = std::move(connectionOptions),
          options
        ]() mutable {
          auto client = std::make_shared<Client>(connectionOptions, options);
          clients_.insert(client);
          promise.setValue(client);
        });
      });
}

void ThreadInfo::releaseClient(std::weak_ptr<Client> clientWeak) {
  fiberManager_.addTaskRemote([this, clientWeak] {
    if (auto client = clientWeak.lock()) {
      clients_.erase(client);
      client->closeNow();
    }
  });
}

ThreadInfo::~ThreadInfo() {
  fiberManager_.addTaskRemote([] {
    folly::EventBaseManager::get()->getEventBase()->terminateLoopSoon();
  });
  thread_.join();
}

} // detail

namespace {

// Pool of threads, each of them has its own EventBase and FiberManager.
class ThreadPool : public std::enable_shared_from_this<ThreadPool> {
 public:
  ThreadPool() {
    threads_.reserve(FLAGS_cacheclient_external_connection_threads);
  }

  std::pair<std::weak_ptr<detail::Client>, std::weak_ptr<detail::ThreadInfo>>
  createClient(
      facebook::memcache::ConnectionOptions connectionOptions,
      ExternalCarbonConnectionImpl::Options options) {
    auto& threadInfo = [&]() -> detail::ThreadInfo& {
      std::lock_guard<std::mutex> lck(mutex_);
      // Select a thread in round robin fashion.
      size_t threadId =
          (nextThreadId_++) % FLAGS_cacheclient_external_connection_threads;

      // If it's not running yet, then we need to start it.
      if (threads_.size() <= threadId) {
        threads_.emplace_back(folly::make_unique<detail::ThreadInfo>());
      }
      return *threads_[threadId];
    }();

    std::weak_ptr<detail::Client> clientPtr =
        threadInfo.createClient(connectionOptions, options);

    std::shared_ptr<detail::ThreadInfo> threadInfoShared(
        shared_from_this(), &threadInfo);

    return std::make_pair(
        std::move(clientPtr),
        std::weak_ptr<detail::ThreadInfo>(std::move(threadInfoShared)));
  }

 private:
  std::mutex mutex_;
  size_t nextThreadId_{0};
  std::vector<std::unique_ptr<detail::ThreadInfo>> threads_;
};

folly::Singleton<ThreadPool> threadPool;

} // anonymous namespace

ExternalCarbonConnectionImpl::Impl::Impl(
    facebook::memcache::ConnectionOptions connectionOptions,
    ExternalCarbonConnectionImpl::Options options) {
  auto pool = threadPool.try_get();

  auto info = pool->createClient(connectionOptions, options);
  client_ = info.first;
  threadInfo_ = info.second;
}

ExternalCarbonConnectionImpl::Impl::~Impl() {
  if (auto threadInfo = threadInfo_.lock()) {
    threadInfo->releaseClient(client_);
  }
}

bool ExternalCarbonConnectionImpl::Impl::healthCheck() {
  folly::fibers::Baton baton;
  bool ret = false;

  auto clientWeak = client_;
  auto threadInfo = threadInfo_.lock();
  if (!threadInfo) {
    throw CarbonConnectionRecreateException(
        "Singleton<ThreadPool> was destroyed!");
  }

  threadInfo->addTaskRemote([clientWeak, &baton, &ret]() {
    auto client = clientWeak.lock();
    if (!client) {
      baton.post();
      return;
    }

    auto reply = client->sendRequest(facebook::memcache::McVersionRequest());
    ret = !facebook::memcache::isErrorResult(reply.result());
    baton.post();
  });

  baton.wait();
  return ret;
}

ExternalCarbonConnectionImpl::ExternalCarbonConnectionImpl(
    facebook::memcache::ConnectionOptions connectionOptions,
    Options options)
    : connectionOptions_(std::move(connectionOptions)),
      options_(std::move(options)),
      impl_(folly::make_unique<Impl>(connectionOptions_, options_)) {}

bool ExternalCarbonConnectionImpl::healthCheck() {
  try {
    return impl_->healthCheck();
  } catch (const CarbonConnectionRecreateException&) {
    impl_ = folly::make_unique<Impl>(connectionOptions_, options_);
    return impl_->healthCheck();
  }
}
} // carbon
<|endoftext|>


<commit_before>/** \brief Utility for creating Debian/Ubunto AMD64 packages.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *  \documentation See https://ubuntuforums.org/showthread.php?t=910717
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <vector>
#include <sys/stat.h>
#include "ExecUtil.h"
#include "FileUtil.h"
#include "MiscUtil.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("(--deb|--rpm) [--output-directory=directory] path_to_binary description [blacklisted_library1 .. blacklisted_libraryN]");
}

struct Library {
    std::string full_name_;
    std::string name_;
    std::string version_;
public:
    Library() = default;
    Library(const Library &other) = default;
    Library(const std::string &full_name, const std::string &name, const std::string &version)
        : full_name_(full_name), name_(name), version_(version) { }

    inline std::string toString() const { return name_ + " (>= " + version_ + ")"; }
};

// Parses lines of the form "libkyotocabinet.so.16 => /lib64/libkyotocabinet.so.16 (0x00007f2334ecc000)".
void ExtractLibrary(const std::string &line, std::string * const full_name, std::string * const path, std::string * const simplified_name) {
    const auto first_space_pos(line.find(' '));
    if (first_space_pos == std::string::npos)
        LOG_ERROR("no space found in \"" + line + "\"!");

    *full_name = line.substr(0, first_space_pos);
    const auto first_dot_pos(full_name->find('.'));
    *simplified_name = (first_dot_pos == std::string::npos) ? *full_name : full_name->substr(0, first_dot_pos);

    const auto arrow_pos(line.find(" => "));
    if (arrow_pos == std::string::npos)
        LOG_ERROR("no => found in \"" + line + "\"!");

    const auto space_pos(line.find(' ', arrow_pos + 4));
    if (space_pos == std::string::npos)
        LOG_ERROR("no space found after library path in \"" + line + "\"!");

    *path = line.substr(arrow_pos + 4, space_pos - arrow_pos - 4);
}

std::string GetVersionHelper(const std::string &library_name) {
    std::string dpkg_output;
    if (not ExecUtil::ExecSubcommandAndCaptureStdout("dpkg -s " + library_name, &dpkg_output))
        LOG_ERROR("failed to execute dpkg!");

    std::vector<std::string> lines;
    StringUtil::SplitThenTrimWhite(dpkg_output, '\n', &lines);
    for (const auto &line : lines) {
        if (StringUtil::StartsWith(line, "Version: ")) {
            const auto version(line.substr(__builtin_strlen("Version: ")));
            const auto first_plus_pos(version.find('+'));
            return (first_plus_pos == std::string::npos) ? version : version.substr(0, first_plus_pos);
        }
    }

    return "";
}

inline std::set<std::string> FilterPackages(const std::set<std::string> &unfiltered_set, const std::set<std::string> &filter) {
    std::set<std::string> filtered_set;
    for (const auto &element : unfiltered_set) {
        if (filter.find(element) == filter.cend())
            filtered_set.emplace(element);
    }

    return filtered_set;
}

bool GetPackageAndVersion(const std::string &full_library_name, const std::set<std::string> &blacklist,
                          std::string * const package_name, std::string * const package_version)
{
    std::string dpkg_output;
    if (not ExecUtil::ExecSubcommandAndCaptureStdout("dpkg -S " + full_library_name, &dpkg_output))
        LOG_ERROR("failed to execute dpkg!");

    std::vector<std::string> lines;
    StringUtil::SplitThenTrimWhite(dpkg_output, '\n', &lines);
    std::set<std::string> packages;
    for (const auto &line : lines) {
        const auto first_colon_pos(line.find(':'));
        if (first_colon_pos == std::string::npos or first_colon_pos == 0)
            LOG_ERROR("weird output line of \"dpkg -S\": \"" + line + "\"!");
        const auto package(line.substr(0, first_colon_pos));
        if (not StringUtil::EndsWith(package, "-dev"))
            packages.emplace(package);
    }

    if (packages.empty())
        LOG_ERROR("no packages found for library \"" + full_library_name + "\"!");

    if (packages.size() > 1) {
        packages = FilterPackages(packages, blacklist);
        if (packages.size() > 1)
            LOG_ERROR("multiple packages for \"" + full_library_name + "\": " + StringUtil::Join(packages, ", "));
    }
    if (packages.empty())
        return false;

    *package_version = GetVersionHelper(*(packages.cbegin()));
    if (package_version->empty())
        LOG_ERROR("library \"" + full_library_name + "\" not found!");

    *package_name = *(packages.cbegin());
    return true;
}

void GetLibraries(const bool build_deb, const std::string &binary_path, const std::set<std::string> &blacklist,
                  std::vector<Library> * const libraries)
{
    std::string ldd_output;
    if (not ExecUtil::ExecSubcommandAndCaptureStdout("ldd " + binary_path, &ldd_output))
        LOG_ERROR("failed to execute ldd!");

    std::vector<std::string> lines;
    StringUtil::SplitThenTrimWhite(ldd_output, '\n', &lines);
    for (auto line(lines.cbegin() + 1); line != lines.cend(); ++line) {
        if (StringUtil::StartsWith(*line, "linux-vdso.so") or StringUtil::StartsWith(*line, "/lib64/ld-linux-x86-64.so") or line->empty())
            continue;

        std::string full_name, path, simplified_name;
        ExtractLibrary(*line, &full_name, &path, &simplified_name);

        if (build_deb) {
            std::string package_name, package_version;
            if (GetPackageAndVersion(full_name, blacklist, &package_name, &package_version))
                libraries->emplace_back(full_name, package_name, package_version);
        } else {
            std::string rpm_output;
            const std::string COMMAND("rpm --query --whatprovides " + path);
            if (not ExecUtil::ExecSubcommandAndCaptureStdout(COMMAND, &rpm_output))
                LOG_ERROR("failed to execute \"" + COMMAND + "\"!");
            StringUtil::SplitThenTrimWhite(rpm_output, '\n', &lines);
            if (lines.size() != 1)
                LOG_ERROR("found multiple packages for \"" + path + "\": " + StringUtil::Join(lines, ','));
        }
    }
}

void GenerateControl(File * const output, const std::string &package, const std::string &version, const std::string &description,
                     const std::vector<Library> &libraries)
{
    (*output) << "Package: " << StringUtil::Map(package, '_', '-') << '\n';
    (*output) << "Version: " << version << '\n';
    (*output) << "Section: ub_tools\n";
    (*output) << "Priority: optional\n";
    (*output) << "Architecture: amd64\n";

    (*output) << "Depends: locales, ";
    bool first(true);
    for (const auto &library : libraries) {
        if (first)
            first = false;
        else
            (*output) << ", ";
        (*output) << library.toString();
    }
    (*output) << '\n';

    (*output) << "Maintainer: johannes.ruscheinski@uni-tuebingen.de\n";
    (*output) << "Description:";
    std::vector<std::string> description_lines;
    StringUtil::SplitThenTrimWhite(description, "\\n", &description_lines);
    for (const auto &line : description_lines)
        (*output) << ' ' << line << '\n';
}

void GeneratePostInst(const std::string &path) {
    FileUtil::WriteStringOrDie("#!/bin/bash\nocale-gen en_US.UTF-8\n");
    if (::chmod(path.c_str(), 0700) == -1)
        LOG_ERROR("chmod(3) on \"" + path + "\" failed!");
}

void BuildDebPackage(const std::string &binary_path, const std::string &package_version, const std::string &description,
                     const std::vector<Library> &libraries, const std::string &output_directory)
{
    const std::string PACKAGE_NAME(FileUtil::GetBasename(binary_path));
    const std::string WORKING_DIR(PACKAGE_NAME + "_" + package_version);

    const std::string TARGET_DIRECTORY(WORKING_DIR + "/usr/local/bin");
    FileUtil::MakeDirectoryOrDie(TARGET_DIRECTORY, /* recursive = */true);
    const std::string target_binary(TARGET_DIRECTORY + "/" + PACKAGE_NAME);
    FileUtil::CopyOrDie(binary_path, target_binary);
    if (::chmod(target_binary.c_str(), 0755) == -1)
        LOG_ERROR("chmod(3) on \"" + target_binary + "\" failed!");
    ExecUtil::ExecOrDie(ExecUtil::Which("strip"), { target_binary });

    FileUtil::MakeDirectoryOrDie(WORKING_DIR + "/DEBIAN");
    const auto control(FileUtil::OpenOutputFileOrDie(WORKING_DIR + "/DEBIAN/control"));
    GenerateControl(control.get(), FileUtil::GetBasename(binary_path), package_version, description, libraries);
    control->close();
    GeneratePostInst(WORKING_DIR + "/DEBIAN/postinst");

    ExecUtil::ExecOrDie(ExecUtil::Which("dpkg-deb"), { "--build", PACKAGE_NAME + "_" + package_version });

    if (not FileUtil::RemoveDirectory(WORKING_DIR))
        LOG_ERROR("failed to recursively delete \"" + WORKING_DIR + "\"!");

    if (not output_directory.empty()) {
        const std::string DEB_NAME(PACKAGE_NAME + "_" + package_version + ".deb");
        FileUtil::RenameFileOrDie(DEB_NAME, output_directory + "/" + DEB_NAME, /* remove_target = */true);
    }
}

void GenerateSpecs(File * const output, const std::string &package, const std::string &version, const std::string &description,
                   const std::vector<Library> &libraries)
{
    (*output) << "Name:           " << package << '\n';
    (*output) << "Version:        " << version << '\n';
    (*output) << "License:        AGPL 3";
    for (const auto &library : libraries)
        (*output) << "Requires:       " << library.full_name_ << '\n';
    (*output) << "BuildArch:      x86_64\n";

    (*output) << "%description\n";
    std::vector<std::string> description_lines;
    StringUtil::SplitThenTrimWhite(description, "\\n", &description_lines);
    for (const auto &line : description_lines)
        (*output) << line << '\n';
    (*output) << '\n';
}

void BuildRPMPackage(const std::string &binary_path, const std::string &package_version, const std::string &description,
                     const std::vector<Library> &libraries, const std::string &/*output_directory*/)
{
    // Create rpmbuild directory tree in our home directory:
    ExecUtil::ExecOrDie(ExecUtil::Which("rpmdev-setuptree"));

    const std::string PACKAGE_NAME(FileUtil::GetBasename(binary_path));
    const std::string WORKING_DIR(MiscUtil::GetEnv("HOME") + "/rpmbuild");
    const auto specs(FileUtil::OpenOutputFileOrDie(WORKING_DIR + "/SPECS/" + PACKAGE_NAME + ".specs"));
    GenerateSpecs(specs.get(), PACKAGE_NAME, package_version, description, libraries);
    specs->close();

    ExecUtil::ExecOrDie("/bin/rm", { "--recursive", WORKING_DIR});
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 4)
        Usage();

    bool build_deb;
    if (std::strcmp(argv[1], "--deb") == 0)
        build_deb = true;
    else if (std::strcmp(argv[1], "--rpm") == 0)
        build_deb = false;
    else
        LOG_ERROR("first argument must be --deb or --rpm!");

    std::string output_directory;
    if (StringUtil::StartsWith(argv[2], "--output-directory=")) {
        output_directory = argv[2] + __builtin_strlen("--output-directory=");
        --argc, ++argv;
    }
    if (argc < 4)
        Usage();

    const std::string binary_path(argv[2]);
    if (not FileUtil::Exists(binary_path))
        LOG_ERROR("file not found: " + binary_path);

    const std::string description(argv[3]);

    std::set<std::string> blacklist;
    for (int arg_no(4); arg_no < argc; ++arg_no)
        blacklist.emplace(argv[arg_no]);

    std::vector<Library> libraries;
    GetLibraries(build_deb, binary_path, blacklist, &libraries);

    const auto package_version(TimeUtil::GetCurrentDateAndTime("%Y.%m.%d"));

    if (build_deb)
        BuildDebPackage(binary_path, package_version, description, libraries, output_directory);
    else
        BuildRPMPackage(binary_path, package_version, description, libraries, output_directory);

    return EXIT_SUCCESS;
}
<commit_msg>Fixed a typo.<commit_after>/** \brief Utility for creating Debian/Ubunto AMD64 packages.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *  \documentation See https://ubuntuforums.org/showthread.php?t=910717
 *
 *  \copyright 2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <vector>
#include <sys/stat.h>
#include "ExecUtil.h"
#include "FileUtil.h"
#include "MiscUtil.h"
#include "StringUtil.h"
#include "util.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("(--deb|--rpm) [--output-directory=directory] path_to_binary description [blacklisted_library1 .. blacklisted_libraryN]");
}

struct Library {
    std::string full_name_;
    std::string name_;
    std::string version_;
public:
    Library() = default;
    Library(const Library &other) = default;
    Library(const std::string &full_name, const std::string &name, const std::string &version)
        : full_name_(full_name), name_(name), version_(version) { }

    inline std::string toString() const { return name_ + " (>= " + version_ + ")"; }
};

// Parses lines of the form "libkyotocabinet.so.16 => /lib64/libkyotocabinet.so.16 (0x00007f2334ecc000)".
void ExtractLibrary(const std::string &line, std::string * const full_name, std::string * const path, std::string * const simplified_name) {
    const auto first_space_pos(line.find(' '));
    if (first_space_pos == std::string::npos)
        LOG_ERROR("no space found in \"" + line + "\"!");

    *full_name = line.substr(0, first_space_pos);
    const auto first_dot_pos(full_name->find('.'));
    *simplified_name = (first_dot_pos == std::string::npos) ? *full_name : full_name->substr(0, first_dot_pos);

    const auto arrow_pos(line.find(" => "));
    if (arrow_pos == std::string::npos)
        LOG_ERROR("no => found in \"" + line + "\"!");

    const auto space_pos(line.find(' ', arrow_pos + 4));
    if (space_pos == std::string::npos)
        LOG_ERROR("no space found after library path in \"" + line + "\"!");

    *path = line.substr(arrow_pos + 4, space_pos - arrow_pos - 4);
}

std::string GetVersionHelper(const std::string &library_name) {
    std::string dpkg_output;
    if (not ExecUtil::ExecSubcommandAndCaptureStdout("dpkg -s " + library_name, &dpkg_output))
        LOG_ERROR("failed to execute dpkg!");

    std::vector<std::string> lines;
    StringUtil::SplitThenTrimWhite(dpkg_output, '\n', &lines);
    for (const auto &line : lines) {
        if (StringUtil::StartsWith(line, "Version: ")) {
            const auto version(line.substr(__builtin_strlen("Version: ")));
            const auto first_plus_pos(version.find('+'));
            return (first_plus_pos == std::string::npos) ? version : version.substr(0, first_plus_pos);
        }
    }

    return "";
}

inline std::set<std::string> FilterPackages(const std::set<std::string> &unfiltered_set, const std::set<std::string> &filter) {
    std::set<std::string> filtered_set;
    for (const auto &element : unfiltered_set) {
        if (filter.find(element) == filter.cend())
            filtered_set.emplace(element);
    }

    return filtered_set;
}

bool GetPackageAndVersion(const std::string &full_library_name, const std::set<std::string> &blacklist,
                          std::string * const package_name, std::string * const package_version)
{
    std::string dpkg_output;
    if (not ExecUtil::ExecSubcommandAndCaptureStdout("dpkg -S " + full_library_name, &dpkg_output))
        LOG_ERROR("failed to execute dpkg!");

    std::vector<std::string> lines;
    StringUtil::SplitThenTrimWhite(dpkg_output, '\n', &lines);
    std::set<std::string> packages;
    for (const auto &line : lines) {
        const auto first_colon_pos(line.find(':'));
        if (first_colon_pos == std::string::npos or first_colon_pos == 0)
            LOG_ERROR("weird output line of \"dpkg -S\": \"" + line + "\"!");
        const auto package(line.substr(0, first_colon_pos));
        if (not StringUtil::EndsWith(package, "-dev"))
            packages.emplace(package);
    }

    if (packages.empty())
        LOG_ERROR("no packages found for library \"" + full_library_name + "\"!");

    if (packages.size() > 1) {
        packages = FilterPackages(packages, blacklist);
        if (packages.size() > 1)
            LOG_ERROR("multiple packages for \"" + full_library_name + "\": " + StringUtil::Join(packages, ", "));
    }
    if (packages.empty())
        return false;

    *package_version = GetVersionHelper(*(packages.cbegin()));
    if (package_version->empty())
        LOG_ERROR("library \"" + full_library_name + "\" not found!");

    *package_name = *(packages.cbegin());
    return true;
}

void GetLibraries(const bool build_deb, const std::string &binary_path, const std::set<std::string> &blacklist,
                  std::vector<Library> * const libraries)
{
    std::string ldd_output;
    if (not ExecUtil::ExecSubcommandAndCaptureStdout("ldd " + binary_path, &ldd_output))
        LOG_ERROR("failed to execute ldd!");

    std::vector<std::string> lines;
    StringUtil::SplitThenTrimWhite(ldd_output, '\n', &lines);
    for (auto line(lines.cbegin() + 1); line != lines.cend(); ++line) {
        if (StringUtil::StartsWith(*line, "linux-vdso.so") or StringUtil::StartsWith(*line, "/lib64/ld-linux-x86-64.so") or line->empty())
            continue;

        std::string full_name, path, simplified_name;
        ExtractLibrary(*line, &full_name, &path, &simplified_name);

        if (build_deb) {
            std::string package_name, package_version;
            if (GetPackageAndVersion(full_name, blacklist, &package_name, &package_version))
                libraries->emplace_back(full_name, package_name, package_version);
        } else {
            std::string rpm_output;
            const std::string COMMAND("rpm --query --whatprovides " + path);
            if (not ExecUtil::ExecSubcommandAndCaptureStdout(COMMAND, &rpm_output))
                LOG_ERROR("failed to execute \"" + COMMAND + "\"!");
            StringUtil::SplitThenTrimWhite(rpm_output, '\n', &lines);
            if (lines.size() != 1)
                LOG_ERROR("found multiple packages for \"" + path + "\": " + StringUtil::Join(lines, ','));
        }
    }
}

void GenerateControl(File * const output, const std::string &package, const std::string &version, const std::string &description,
                     const std::vector<Library> &libraries)
{
    (*output) << "Package: " << StringUtil::Map(package, '_', '-') << '\n';
    (*output) << "Version: " << version << '\n';
    (*output) << "Section: ub_tools\n";
    (*output) << "Priority: optional\n";
    (*output) << "Architecture: amd64\n";

    (*output) << "Depends: locales, ";
    bool first(true);
    for (const auto &library : libraries) {
        if (first)
            first = false;
        else
            (*output) << ", ";
        (*output) << library.toString();
    }
    (*output) << '\n';

    (*output) << "Maintainer: johannes.ruscheinski@uni-tuebingen.de\n";
    (*output) << "Description:";
    std::vector<std::string> description_lines;
    StringUtil::SplitThenTrimWhite(description, "\\n", &description_lines);
    for (const auto &line : description_lines)
        (*output) << ' ' << line << '\n';
}

void GeneratePostInst(const std::string &path) {
    FileUtil::WriteStringOrDie("#!/bin/bash\nlocale-gen en_US.UTF-8\n");
    if (::chmod(path.c_str(), 0700) == -1)
        LOG_ERROR("chmod(3) on \"" + path + "\" failed!");
}

void BuildDebPackage(const std::string &binary_path, const std::string &package_version, const std::string &description,
                     const std::vector<Library> &libraries, const std::string &output_directory)
{
    const std::string PACKAGE_NAME(FileUtil::GetBasename(binary_path));
    const std::string WORKING_DIR(PACKAGE_NAME + "_" + package_version);

    const std::string TARGET_DIRECTORY(WORKING_DIR + "/usr/local/bin");
    FileUtil::MakeDirectoryOrDie(TARGET_DIRECTORY, /* recursive = */true);
    const std::string target_binary(TARGET_DIRECTORY + "/" + PACKAGE_NAME);
    FileUtil::CopyOrDie(binary_path, target_binary);
    if (::chmod(target_binary.c_str(), 0755) == -1)
        LOG_ERROR("chmod(3) on \"" + target_binary + "\" failed!");
    ExecUtil::ExecOrDie(ExecUtil::Which("strip"), { target_binary });

    FileUtil::MakeDirectoryOrDie(WORKING_DIR + "/DEBIAN");
    const auto control(FileUtil::OpenOutputFileOrDie(WORKING_DIR + "/DEBIAN/control"));
    GenerateControl(control.get(), FileUtil::GetBasename(binary_path), package_version, description, libraries);
    control->close();
    GeneratePostInst(WORKING_DIR + "/DEBIAN/postinst");

    ExecUtil::ExecOrDie(ExecUtil::Which("dpkg-deb"), { "--build", PACKAGE_NAME + "_" + package_version });

    if (not FileUtil::RemoveDirectory(WORKING_DIR))
        LOG_ERROR("failed to recursively delete \"" + WORKING_DIR + "\"!");

    if (not output_directory.empty()) {
        const std::string DEB_NAME(PACKAGE_NAME + "_" + package_version + ".deb");
        FileUtil::RenameFileOrDie(DEB_NAME, output_directory + "/" + DEB_NAME, /* remove_target = */true);
    }
}

void GenerateSpecs(File * const output, const std::string &package, const std::string &version, const std::string &description,
                   const std::vector<Library> &libraries)
{
    (*output) << "Name:           " << package << '\n';
    (*output) << "Version:        " << version << '\n';
    (*output) << "License:        AGPL 3";
    for (const auto &library : libraries)
        (*output) << "Requires:       " << library.full_name_ << '\n';
    (*output) << "BuildArch:      x86_64\n";

    (*output) << "%description\n";
    std::vector<std::string> description_lines;
    StringUtil::SplitThenTrimWhite(description, "\\n", &description_lines);
    for (const auto &line : description_lines)
        (*output) << line << '\n';
    (*output) << '\n';
}

void BuildRPMPackage(const std::string &binary_path, const std::string &package_version, const std::string &description,
                     const std::vector<Library> &libraries, const std::string &/*output_directory*/)
{
    // Create rpmbuild directory tree in our home directory:
    ExecUtil::ExecOrDie(ExecUtil::Which("rpmdev-setuptree"));

    const std::string PACKAGE_NAME(FileUtil::GetBasename(binary_path));
    const std::string WORKING_DIR(MiscUtil::GetEnv("HOME") + "/rpmbuild");
    const auto specs(FileUtil::OpenOutputFileOrDie(WORKING_DIR + "/SPECS/" + PACKAGE_NAME + ".specs"));
    GenerateSpecs(specs.get(), PACKAGE_NAME, package_version, description, libraries);
    specs->close();

    ExecUtil::ExecOrDie("/bin/rm", { "--recursive", WORKING_DIR});
}

} // unnamed namespace

int Main(int argc, char *argv[]) {
    if (argc < 4)
        Usage();

    bool build_deb;
    if (std::strcmp(argv[1], "--deb") == 0)
        build_deb = true;
    else if (std::strcmp(argv[1], "--rpm") == 0)
        build_deb = false;
    else
        LOG_ERROR("first argument must be --deb or --rpm!");

    std::string output_directory;
    if (StringUtil::StartsWith(argv[2], "--output-directory=")) {
        output_directory = argv[2] + __builtin_strlen("--output-directory=");
        --argc, ++argv;
    }
    if (argc < 4)
        Usage();

    const std::string binary_path(argv[2]);
    if (not FileUtil::Exists(binary_path))
        LOG_ERROR("file not found: " + binary_path);

    const std::string description(argv[3]);

    std::set<std::string> blacklist;
    for (int arg_no(4); arg_no < argc; ++arg_no)
        blacklist.emplace(argv[arg_no]);

    std::vector<Library> libraries;
    GetLibraries(build_deb, binary_path, blacklist, &libraries);

    const auto package_version(TimeUtil::GetCurrentDateAndTime("%Y.%m.%d"));

    if (build_deb)
        BuildDebPackage(binary_path, package_version, description, libraries, output_directory);
    else
        BuildRPMPackage(binary_path, package_version, description, libraries, output_directory);

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/** \file    find_tue_dups.cc
 *  \brief   Find duplicates amongst the libraries of institutions associated w/ the University of Tübingen.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2017, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <set>
#include <cstring>
#include "Compiler.h"
#include "JSON.h"
#include "TextUtil.h"
#include "MarcRecord.h"
#include "RegexMatcher.h"
#include "util.h"

void Usage() {
    std::cerr << "Usage: " << ::progname << " --output-set=(MONOGRAPHS|SERIALS) marc_input\n";
    std::cerr << "       Please note that this program requires an input MARC format as provided by\n";
    std::cerr << "       the team at the University of Freiburg!\n\n";
    std::exit(EXIT_FAILURE);
}

void ExtractSubfield(const MarcRecord &record, const std::string &tag, const char subfield_code,
                     std::set<std::string> * const extracted_values)
{
    std::vector<size_t> field_indices;
    record.getFieldIndices(tag, &field_indices);
    for (const size_t field_index : field_indices) {
        const std::string subfield_a(record.extractFirstSubfield(field_index, subfield_code));
        if (likely(not subfield_a.empty()))
            extracted_values->emplace(subfield_a);
    }
}

std::string ExtractZDBNumber(const MarcRecord &record) {
    // First we try our luck w/ 016...
    std::vector<size_t> field_indices;
    record.getFieldIndices("016", &field_indices);
    for (const size_t field_index : field_indices) {
        const std::string field_contents(record.getFieldData(field_index));
        const Subfields subfields(field_contents);
        if (subfields.getFirstSubfieldValue('2') == "DE-600")
            return subfields.getFirstSubfieldValue('a');
    }

    // ...and then we try our luck w/ 035:
    record.getFieldIndices("035", &field_indices);
    for (const size_t field_index : field_indices) {
        const std::string field_contents(record.getFieldData(field_index));
        const Subfields subfields(field_contents);
        const std::string subfield_a(subfields.getFirstSubfieldValue('a'));
        if (StringUtil::StartsWith(subfield_a, "(DE-599)ZDB"))
            return subfield_a.substr(11);
    }

    return "";
}

void ExtractISSNs(const MarcRecord &record, std::set<std::string> * const issns_and_isbns) {
    ExtractSubfield(record, "022", 'a', issns_and_isbns);
    ExtractSubfield(record, "440", 'x', issns_and_isbns);
    ExtractSubfield(record, "490", 'x', issns_and_isbns);
    ExtractSubfield(record, "730", 'x', issns_and_isbns);
    ExtractSubfield(record, "773", 'x', issns_and_isbns);
    ExtractSubfield(record, "776", 'x', issns_and_isbns);
    ExtractSubfield(record, "780", 'x', issns_and_isbns);
    ExtractSubfield(record, "785", 'x', issns_and_isbns);
}

void ExtractISBNs(const MarcRecord &record, std::set<std::string> * const issns_and_isbns) {
    ExtractSubfield(record, "020", 'a', issns_and_isbns);
    ExtractSubfield(record, "773", 'a', issns_and_isbns);
}

std::string ExtractInventory(const std::string &_910_subfield_a) {
    if (_910_subfield_a.empty())
        return "";

    JSON::JSONNode *tree_root(nullptr);
    try {
        JSON::Parser json_parser(_910_subfield_a);
        if (not (json_parser.parse(&tree_root)))
            Error("in ExtractInventory: failed to parse returned JSON: " + json_parser.getErrorMessage()
                  + "(input was: " + StringUtil::CStyleEscape(_910_subfield_a) + ")");

        if (tree_root->getType() != JSON::JSONNode::OBJECT_NODE)
            Error("in ExtractInventory: expected an object node!");
        const JSON::ObjectNode * const object(reinterpret_cast<const JSON::ObjectNode * const>(tree_root));

        const JSON::JSONNode * const inventory_node(object->getValue("bestand8032"));
        if (inventory_node == nullptr)
            return "";
        if (inventory_node->getType() != JSON::JSONNode::STRING_NODE)
            Error("in ExtractInventory: expected a string node!");
        return reinterpret_cast<const JSON::StringNode * const>(inventory_node)->getValue();

        delete tree_root;
    } catch (...) {
        delete tree_root;
        throw;
    }
}

enum SupplementaryInfoType { SIGNATURE, INVENTORY };

// \return The number of occurrences of an object in Tübingen.
unsigned FindTueSigilsAndSignaturesOrInventory(const MarcRecord &record,
                                               const SupplementaryInfoType supplementary_info_type,
                                               std::string * const ub_signatures_or_inventory,
                                               std::string * const non_ub_sigils_and_inventory)
{
    unsigned occurrence_count(0);
    std::vector<size_t> _910_indices;
    record.getFieldIndices("910", &_910_indices);
    for (const size_t _910_index : _910_indices) {
        const std::string _910_field_contents(record.getFieldData(_910_index));
        if (_910_field_contents.empty())
            continue;
        const Subfields subfields(_910_field_contents);
        const std::string sigil(subfields.getFirstSubfieldValue('c'));

        std::vector<std::string> signatures;
        std::string inventory;
        if (supplementary_info_type == SIGNATURE)
            subfields.extractSubfields('d', &signatures);
        else
            inventory = ExtractInventory(subfields.getFirstSubfieldValue('a'));

        if (not sigil.empty() and sigil != "LFER") {
            ++occurrence_count;
            if (sigil == "21") { // UB
                if (supplementary_info_type == SIGNATURE) {
                    if (not ub_signatures_or_inventory->empty())
                        ub_signatures_or_inventory->append(",");
                    ub_signatures_or_inventory->append(StringUtil::Join(signatures, ", "));
                } else if (not inventory.empty()) { // Assume supplementary_info_type == INVENTORY.
                    if (not ub_signatures_or_inventory->empty())
                        ub_signatures_or_inventory->append(",");
                    ub_signatures_or_inventory->append(inventory);
                }
            } else if (not inventory.empty()) { // Assume supplementary_info_type == INVENTORY.
                if (not non_ub_sigils_and_inventory->empty())
                    non_ub_sigils_and_inventory->append(", ");
                non_ub_sigils_and_inventory->append(sigil + ':' + inventory);
            } else { // Non-UB and we don't have inventory information.
                if (not non_ub_sigils_and_inventory->empty())
                    non_ub_sigils_and_inventory->append(", ");
                non_ub_sigils_and_inventory->append(sigil);
            }
        }
    }

    return occurrence_count;
}

// Apperently we have "PPN's" that look like "103571965_256955174", which are actually two PPN's.  In this case
// we only want the first one.  (The second one is presumably the PPN of the superior work.)
std::string NormalisePPN(const std::string &ppn_or_double_ppn) {
    const size_t first_underscore_pos(ppn_or_double_ppn.find('_'));
    return first_underscore_pos == std::string::npos ? ppn_or_double_ppn
                                                     : ppn_or_double_ppn.substr(0, first_underscore_pos);
}

enum OutputSet { MONOGRAPHS, SERIALS };

bool FindTueDups(const OutputSet output_set, const MarcRecord &record) {
    std::string ub_signatures_or_inventory, non_ub_sigils_and_inventory;
    const unsigned occurrence_count(
        FindTueSigilsAndSignaturesOrInventory(record, (output_set == MONOGRAPHS ? SIGNATURE : INVENTORY),
                                              &ub_signatures_or_inventory,
                                              &non_ub_sigils_and_inventory));

    // We only keep dups and only those that occur at least once in the Tübingen University's main library:
    if (occurrence_count < 2 or ub_signatures_or_inventory.empty())
        return false;

    const std::string _008_contents(record.getFieldData("008"));
    std::string publication_year;
    if (likely(_008_contents.length() >= 11))
        publication_year = _008_contents.substr(7, 4);

    std::string area_or_zdb_number;
    if (output_set == MONOGRAPHS) {
        const std::string _910_contents(record.getFieldData("910"));
        if (not _910_contents.empty()) {
            const Subfields subfields(_910_contents);
            area_or_zdb_number = subfields.getFirstSubfieldValue('j');
        }
    } else // SERIALS
        area_or_zdb_number = ExtractZDBNumber(record);

    const std::string _245_contents(record.getFieldData("245"));
    std::string main_title;
    if (not _245_contents.empty()) {
        const Subfields subfields(_245_contents);
        main_title = subfields.getFirstSubfieldValue('a');
    }

    std::set<std::string> issns_and_isbns;
    if (output_set == MONOGRAPHS)
        ExtractISBNs(record, &issns_and_isbns);
    else
        ExtractISSNs(record, &issns_and_isbns);

    std::cout << '"' << NormalisePPN(record.getControlNumber()) << "\",\"" << TextUtil::CSVEscape(main_title)
              << "\",\"" << TextUtil::CSVEscape(StringUtil::Join(issns_and_isbns, ','))
              << (output_set == MONOGRAPHS ? "\",\"" + TextUtil::CSVEscape(publication_year) : "")
              << "\",\"" << TextUtil::CSVEscape(area_or_zdb_number)
              <<"\",\"" << TextUtil::CSVEscape(ub_signatures_or_inventory) <<  "\",\""
              << TextUtil::CSVEscape(non_ub_sigils_and_inventory) << "\"\n";

    return true;
}

void FindTueDups(const OutputSet output_set, MarcReader * const marc_reader) {
    // Write a header:
    std::cout << "\"PPN\"" << ",\"Titel\"" << (output_set == MONOGRAPHS ? ",\"ISBN\"" : ",\"ISSN\"")
              << (output_set == MONOGRAPHS ? ",\"Erscheinungsjahr\"" : "")
              << (output_set == MONOGRAPHS ? ",\"Fachgebiet\"" : ",\"ZDB-ID-Nummer\"")
              << (output_set == MONOGRAPHS ? ",\"UB - Signatur\"" : ",\"UB - Bestandsangabe\"")
              << ",\"Sigel der anderen besitzenden Bibliotheken"
              << (output_set == SERIALS ? " mit Bestandsangaben\"" : "\"") << '\n';

    unsigned count(0), dups_count(0), monograph_count(0), serial_count(0);
    while (MarcRecord record = marc_reader->read()) {
        ++count;

        // Only consider monographs and serials:
        const Leader &leader(record.getLeader());
        if (not (leader.isMonograph() or leader.isSerial())
            or (leader.isMonograph() and output_set == SERIALS)
            or (leader.isSerial() and output_set == MONOGRAPHS))
            continue;

        if (FindTueDups(output_set, record)) {
            ++dups_count;
            if (leader.isMonograph())
                ++monograph_count;
            else
                ++serial_count;
        }
    }
    std::cerr << "Processed " << count << " records and found " << dups_count << " dups (" << monograph_count
              << " monographs and " << serial_count << " serials).\n";
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    OutputSet output_set;
    if (std::strcmp(argv[1], "--output-set=MONOGRAPHS") == 0)
        output_set = MONOGRAPHS;
    else if (std::strcmp(argv[1], "--output-set=SERIALS") == 0)
        output_set = SERIALS;
    else
        Error("invalid input format \"" + std::string(argv[1]) + "\"!  (Must be MONOGRAPHS or SERIALS)");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(argv[2], MarcReader::BINARY));

    try {
        FindTueDups(output_set, marc_reader.get());
    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<commit_msg>Update find_tue_dups.cc<commit_after>/** \file    find_tue_dups.cc
 *  \brief   Find duplicates amongst the libraries of institutions associated w/ the University of Tübingen.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2017, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <algorithm>
#include <iostream>
#include <set>
#include <cstring>
#include "Compiler.h"
#include "JSON.h"
#include "TextUtil.h"
#include "MarcRecord.h"
#include "RegexMatcher.h"
#include "util.h"

void Usage() {
    std::cerr << "Usage: " << ::progname << " --output-set=(MONOGRAPHS|SERIALS) marc_input\n";
    std::cerr << "       Please note that this program requires an input MARC format as provided by\n";
    std::cerr << "       the team at the University of Freiburg!\n\n";
    std::exit(EXIT_FAILURE);
}

void ExtractSubfield(const MarcRecord &record, const std::string &tag, const char subfield_code,
                     std::set<std::string> * const extracted_values)
{
    std::vector<size_t> field_indices;
    record.getFieldIndices(tag, &field_indices);
    for (const size_t field_index : field_indices) {
        const std::string subfield_a(record.extractFirstSubfield(field_index, subfield_code));
        if (likely(not subfield_a.empty()))
            extracted_values->emplace(subfield_a);
    }
}

std::string ExtractZDBNumber(const MarcRecord &record) {
    // First we try our luck w/ 016...
    std::vector<size_t> field_indices;
    record.getFieldIndices("016", &field_indices);
    for (const size_t field_index : field_indices) {
        const std::string field_contents(record.getFieldData(field_index));
        const Subfields subfields(field_contents);
        if (subfields.getFirstSubfieldValue('2') == "DE-600")
            return subfields.getFirstSubfieldValue('a');
    }

    // ...and then we try our luck w/ 035:
    record.getFieldIndices("035", &field_indices);
    for (const size_t field_index : field_indices) {
        const std::string field_contents(record.getFieldData(field_index));
        const Subfields subfields(field_contents);
        const std::string subfield_a(subfields.getFirstSubfieldValue('a'));
        if (StringUtil::StartsWith(subfield_a, "(DE-599)ZDB"))
            return subfield_a.substr(11);
    }

    return "";
}

void ExtractISSNs(const MarcRecord &record, std::set<std::string> * const issns_and_isbns) {
    ExtractSubfield(record, "022", 'a', issns_and_isbns);
    ExtractSubfield(record, "440", 'x', issns_and_isbns);
    ExtractSubfield(record, "490", 'x', issns_and_isbns);
    ExtractSubfield(record, "730", 'x', issns_and_isbns);
    ExtractSubfield(record, "773", 'x', issns_and_isbns);
    ExtractSubfield(record, "776", 'x', issns_and_isbns);
    ExtractSubfield(record, "780", 'x', issns_and_isbns);
    ExtractSubfield(record, "785", 'x', issns_and_isbns);
}

void ExtractISBNs(const MarcRecord &record, std::set<std::string> * const issns_and_isbns) {
    ExtractSubfield(record, "020", 'a', issns_and_isbns);
    ExtractSubfield(record, "773", 'a', issns_and_isbns);
}

std::string ExtractInventory(const std::string &_910_subfield_a) {
    if (_910_subfield_a.empty())
        return "";

    JSON::JSONNode *tree_root(nullptr);
    try {
        JSON::Parser json_parser(_910_subfield_a);
        if (not (json_parser.parse(&tree_root)))
            Error("in ExtractInventory: failed to parse returned JSON: " + json_parser.getErrorMessage()
                  + "(input was: " + StringUtil::CStyleEscape(_910_subfield_a) + ")");

        if (tree_root->getType() != JSON::JSONNode::OBJECT_NODE)
            Error("in ExtractInventory: expected an object node!");
        const JSON::ObjectNode * const object(reinterpret_cast<const JSON::ObjectNode * const>(tree_root));

        const JSON::JSONNode * const inventory_node(object->getValue("bestand8032"));
        if (inventory_node == nullptr)
            return "";
        if (inventory_node->getType() != JSON::JSONNode::STRING_NODE)
            Error("in ExtractInventory: expected a string node!");
        return reinterpret_cast<const JSON::StringNode * const>(inventory_node)->getValue();

        delete tree_root;
    } catch (...) {
        delete tree_root;
        throw;
    }
}

enum SupplementaryInfoType { SIGNATURE, INVENTORY };

// \return The number of occurrences of an object in Tübingen.
unsigned FindTueSigilsAndSignaturesOrInventory(const MarcRecord &record,
                                               const SupplementaryInfoType supplementary_info_type,
                                               std::string * const ub_signatures_or_inventory,
                                               std::string * const non_ub_sigils_and_inventory)
{
    unsigned occurrence_count(0);
    std::vector<size_t> _910_indices;
    record.getFieldIndices("910", &_910_indices);
    for (const size_t _910_index : _910_indices) {
        const std::string _910_field_contents(record.getFieldData(_910_index));
        if (_910_field_contents.empty())
            continue;
        const Subfields subfields(_910_field_contents);
        const std::string sigil(subfields.getFirstSubfieldValue('c'));

        std::vector<std::string> signatures;
        std::string inventory;
        if (supplementary_info_type == SIGNATURE)
            subfields.extractSubfields('d', &signatures);
        else
            inventory = ExtractInventory(subfields.getFirstSubfieldValue('a'));

        if (not sigil.empty() and sigil != "LFER") {
            ++occurrence_count;
            if (sigil == "21") { // UB
                if (supplementary_info_type == SIGNATURE) {
                    if (not ub_signatures_or_inventory->empty())
                        ub_signatures_or_inventory->append(",");
                    ub_signatures_or_inventory->append(StringUtil::Join(signatures, ", "));
                } else if (not inventory.empty()) { // Assume supplementary_info_type == INVENTORY.
                    if (not ub_signatures_or_inventory->empty())
                        ub_signatures_or_inventory->append(",");
                    ub_signatures_or_inventory->append(inventory);
                }
            } else if (not inventory.empty()) { // Assume supplementary_info_type == INVENTORY.
                if (not non_ub_sigils_and_inventory->empty())
                    non_ub_sigils_and_inventory->append(", ");
                non_ub_sigils_and_inventory->append(sigil + ':' + inventory);
            } else { // Non-UB and we don't have inventory information.
                if (not non_ub_sigils_and_inventory->empty())
                    non_ub_sigils_and_inventory->append(", ");
                non_ub_sigils_and_inventory->append(sigil);
            }
        }
    }

    return occurrence_count;
}

// Apperently we have "PPN's" that look like "103571965_256955174", which are actually two PPN's.  In this case
// we only want the first one.  (The second one is presumably the PPN of the superior work.)
std::string NormalisePPN(const std::string &ppn_or_double_ppn) {
    const size_t first_underscore_pos(ppn_or_double_ppn.find('_'));
    return first_underscore_pos == std::string::npos ? ppn_or_double_ppn
                                                     : ppn_or_double_ppn.substr(0, first_underscore_pos);
}

enum OutputSet { MONOGRAPHS, SERIALS };

bool FindTueDups(const OutputSet output_set, const MarcRecord &record) {
    std::string ub_signatures_or_inventory, non_ub_sigils_and_inventory;
    const unsigned occurrence_count(
        FindTueSigilsAndSignaturesOrInventory(record, (output_set == MONOGRAPHS ? SIGNATURE : INVENTORY),
                                              &ub_signatures_or_inventory,
                                              &non_ub_sigils_and_inventory));

    // We only keep dups and only those that occur at least once in the Tübingen University's main library:
    if (occurrence_count < 2 or ub_signatures_or_inventory.empty())
        return false;

    const std::string _008_contents(record.getFieldData("008"));
    std::string publication_year;
    if (likely(_008_contents.length() >= 11))
        publication_year = _008_contents.substr(7, 4);

    std::string area_or_zdb_number;
    if (output_set == MONOGRAPHS) {
        const std::string _910_contents(record.getFieldData("910"));
        if (not _910_contents.empty()) {
            const Subfields subfields(_910_contents);
            area_or_zdb_number = subfields.getFirstSubfieldValue('j');
        }
    } else // SERIALS
        area_or_zdb_number = ExtractZDBNumber(record);

    const std::string _245_contents(record.getFieldData("245"));
    std::string main_title;
    if (not _245_contents.empty()) {
        const Subfields subfields(_245_contents);
        main_title = subfields.getFirstSubfieldValue('a');
    }

    std::set<std::string> issns_and_isbns;
    if (output_set == MONOGRAPHS)
        ExtractISBNs(record, &issns_and_isbns);
    else
        ExtractISSNs(record, &issns_and_isbns);

    std::cout << '"' << NormalisePPN(record.getControlNumber()) << "\",\"" << TextUtil::CSVEscape(main_title)
              << "\",\"" << TextUtil::CSVEscape(StringUtil::Join(issns_and_isbns, ','))
              << (output_set == MONOGRAPHS ? "\",\"" + TextUtil::CSVEscape(publication_year) : "")
              << "\",\"" << TextUtil::CSVEscape(area_or_zdb_number)
              << "\",\"" << TextUtil::CSVEscape(ub_signatures_or_inventory) <<  "\",\""
              << TextUtil::CSVEscape(non_ub_sigils_and_inventory) << "\"\n";

    return true;
}

void FindTueDups(const OutputSet output_set, MarcReader * const marc_reader) {
    // Write a header:
    std::cout << "\"PPN\"" << ",\"Titel\"" << (output_set == MONOGRAPHS ? ",\"ISBN\"" : ",\"ISSN\"")
              << (output_set == MONOGRAPHS ? ",\"Erscheinungsjahr\"" : "")
              << (output_set == MONOGRAPHS ? ",\"Fachgebiet\"" : ",\"ZDB-ID-Nummer\"")
              << (output_set == MONOGRAPHS ? ",\"UB - Signatur\"" : ",\"UB - Bestandsangabe\"")
              << ",\"Sigel der anderen besitzenden Bibliotheken"
              << (output_set == SERIALS ? " mit Bestandsangaben\"" : "\"") << '\n';

    unsigned count(0), dups_count(0), monograph_count(0), serial_count(0);
    while (MarcRecord record = marc_reader->read()) {
        ++count;

        // Only consider monographs and serials:
        const Leader &leader(record.getLeader());
        if (not (leader.isMonograph() or leader.isSerial())
            or (leader.isMonograph() and output_set == SERIALS)
            or (leader.isSerial() and output_set == MONOGRAPHS))
            continue;

        if (FindTueDups(output_set, record)) {
            ++dups_count;
            if (leader.isMonograph())
                ++monograph_count;
            else
                ++serial_count;
        }
    }
    std::cerr << "Processed " << count << " records and found " << dups_count << " dups (" << monograph_count
              << " monographs and " << serial_count << " serials).\n";
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    OutputSet output_set;
    if (std::strcmp(argv[1], "--output-set=MONOGRAPHS") == 0)
        output_set = MONOGRAPHS;
    else if (std::strcmp(argv[1], "--output-set=SERIALS") == 0)
        output_set = SERIALS;
    else
        Error("invalid input format \"" + std::string(argv[1]) + "\"!  (Must be MONOGRAPHS or SERIALS)");

    std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(argv[2], MarcReader::BINARY));

    try {
        FindTueDups(output_set, marc_reader.get());
    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}
<|endoftext|>


<commit_before>/***************************************************************************
 *   Copyright (C) 2006 by Till Adam <adam@kde.org>                        *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU Library General Public License as       *
 *   published by the Free Software Foundation; either version 2 of the    *
 *   License, or (at your option) any later version.                       *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU Library General Public     *
 *   License along with this program; if not, write to the                 *
 *   Free Software Foundation, Inc.,                                       *
 *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.             *
 ***************************************************************************/

#include "akonadi.h"
#include "akonadiconnection.h"
#include "serveradaptor.h"
#include "akdebug.h"

#include "cachecleaner.h"
#include "intervalcheck.h"
#include "storage/dbconfig.h"
#include "storage/datastore.h"
#include "notificationmanager.h"
#include "resourcemanager.h"
#include "tracer.h"
#include "xesammanager.h"
#include "nepomukmanager.h"
#include "debuginterface.h"

#include "../libs/xdgbasedirs_p.h"
#include "../libs/protocol_p.h"

#include <QtCore/QCoreApplication>
#include <QtCore/QDir>
#include <QtCore/QProcess>
#include <QtCore/QSettings>
#include <QtCore/QTimer>

#include <config-akonadi.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#include <stdlib.h>

using namespace Akonadi;

static AkonadiServer *s_instance = 0;

AkonadiServer::AkonadiServer( QObject* parent )
    : QLocalServer( parent )
    , mCacheCleaner( 0 )
    , mIntervalChecker( 0 )
    , mDatabaseProcess( 0 )
    , mAlreadyShutdown( false )
{
    DbConfig::init();
    startDatabaseProcess();

    s_instance = this;

    const QString serverConfigFile = XdgBaseDirs::akonadiServerConfigFile( XdgBaseDirs::ReadWrite );
    QSettings settings( serverConfigFile, QSettings::IniFormat );

    const QString connectionSettingsFile = XdgBaseDirs::akonadiConnectionConfigFile( XdgBaseDirs::WriteOnly );
    QSettings connectionSettings( connectionSettingsFile, QSettings::IniFormat );

#ifdef Q_OS_WIN
    QString namedPipe = settings.value( QLatin1String( "Connection/NamedPipe" ), QLatin1String( "Akonadi" ) ).toString();
    if ( !listen( namedPipe ) )
      akFatal() << "Unable to listen on Named Pipe" << namedPipe;

    connectionSettings.setValue( QLatin1String( "Data/Method" ), QLatin1String( "NamedPipe" ) );
    connectionSettings.setValue( QLatin1String( "Data/NamedPipe" ), namedPipe );
#else
    const QString defaultSocketDir = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi" ) );
    QString socketDir = settings.value( QLatin1String( "Connection/SocketDirectory" ), defaultSocketDir ).toString();
    if ( socketDir[0] != QLatin1Char( '/' ) ) {
      QDir::home().mkdir( socketDir );
      socketDir = QDir::homePath() + QLatin1Char( '/' ) + socketDir;
    }

    const QString socketFile = socketDir + QLatin1String( "/akonadiserver.socket" );
    unlink( socketFile.toUtf8().constData() );
    if ( !listen( socketFile ) )
      akFatal() << "Unable to listen on Unix socket" << socketFile;

    connectionSettings.setValue( QLatin1String( "Data/Method" ), QLatin1String( "UnixPath" ) );
    connectionSettings.setValue( QLatin1String( "Data/UnixPath" ), socketFile );
#endif

    // initialize the database
    DataStore *db = DataStore::self();
    if ( !db->database().isOpen() )
      akFatal() << "Unable to open database" << db->database().lastError().text();
    if ( !db->init() )
      akFatal() << "Unable to initialize database.";

    NotificationManager::self();
    Tracer::self();
    new DebugInterface( this );
    ResourceManager::self();
    if ( settings.value( QLatin1String( "Cache/EnableCleaner" ), true ).toBool() ) {
      mCacheCleaner = new CacheCleaner( this );
      mCacheCleaner->start( QThread::IdlePriority );
    }

    mIntervalChecker = new IntervalCheck( this );
    mIntervalChecker->start( QThread::IdlePriority );

    mSearchManager = new DummySearchManager;

    new ServerAdaptor( this );
    QDBusConnection::sessionBus().registerObject( QLatin1String( "/Server" ), this );

    const QByteArray dbusAddress = qgetenv( "DBUS_SESSION_BUS_ADDRESS" );
    if ( !dbusAddress.isEmpty() ) {
      connectionSettings.setValue( QLatin1String( "DBUS/Address" ), QLatin1String( dbusAddress ) );
    }

    connect( QDBusConnection::sessionBus().interface(), SIGNAL(serviceOwnerChanged(QString,QString,QString)),
             SLOT(serviceOwnerChanged(QString,QString,QString)) );
}

AkonadiServer::~AkonadiServer()
{
}

void AkonadiServer::quit()
{
    if ( mAlreadyShutdown )
      return;

    mAlreadyShutdown = true;

    if ( mCacheCleaner )
      QMetaObject::invokeMethod( mCacheCleaner, "quit", Qt::QueuedConnection );
    if ( mIntervalChecker )
      QMetaObject::invokeMethod( mIntervalChecker, "quit", Qt::QueuedConnection );
    QCoreApplication::instance()->processEvents();

    if ( mCacheCleaner )
      mCacheCleaner->wait();
    if ( mIntervalChecker )
      mIntervalChecker->wait();

    delete mSearchManager;
    mSearchManager = 0;

    for ( int i = 0; i < mConnections.count(); ++i ) {
      if ( mConnections[ i ] ) {
        mConnections[ i ]->quit();
        mConnections[ i ]->wait();
      }
    }

    DataStore::self()->close();

    // execute the deleteLater() calls for the threads so they free their db connections
    // and the following db termination will work
    QCoreApplication::instance()->processEvents();

    stopDatabaseProcess();

    QSettings settings( XdgBaseDirs::akonadiServerConfigFile(), QSettings::IniFormat );
    const QString connectionSettingsFile = XdgBaseDirs::akonadiConnectionConfigFile( XdgBaseDirs::WriteOnly );

#ifndef Q_OS_WIN
    QSettings connectionSettings( connectionSettingsFile, QSettings::IniFormat );
    const QString defaultSocketDir = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi" ) );
    const QString socketDir = settings.value( QLatin1String( "Connection/SocketDirectory" ), defaultSocketDir ).toString();

    if ( !QDir::home().remove( socketDir + QLatin1String( "/akonadiserver.socket" ) ) )
        akError() << "Failed to remove Unix socket";
#endif
    if ( !QDir::home().remove( connectionSettingsFile ) )
        akError() << "Failed to remove runtime connection config file";

    QTimer::singleShot( 0, this, SLOT( doQuit() ) );
}

void AkonadiServer::doQuit()
{
    QCoreApplication::exit();
}

void AkonadiServer::incomingConnection( quintptr socketDescriptor )
{
    QPointer<AkonadiConnection> thread = new AkonadiConnection( socketDescriptor, this );
    connect( thread, SIGNAL( finished() ), thread, SLOT( deleteLater() ) );
    mConnections.append( thread );
    thread->start();
}

AkonadiServer * AkonadiServer::instance()
{
    if ( !s_instance )
        s_instance = new AkonadiServer();
    return s_instance;
}

void AkonadiServer::startDatabaseProcess()
{
  if ( !DbConfig::useInternalServer() )
    return;

  const QString mysqldPath = DbConfig::serverPath();
  if ( mysqldPath.isEmpty() )
    akFatal() << "No path to mysqld set in server configuration!";

  // create the database directories if they don't exists
  const QString dataDir = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi/db_data" ) );
  const QString akDir   = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi/" ) );
  const QString miscDir = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi/db_misc" ) );

  // generate config file
  const QString globalConfig = XdgBaseDirs::findResourceFile( "config", QLatin1String( "akonadi/mysql-global.conf" ) );
  const QString localConfig  = XdgBaseDirs::findResourceFile( "config", QLatin1String( "akonadi/mysql-local.conf" ) );
  const QString actualConfig = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi" ) ) + QLatin1String("/mysql.conf");
  if ( globalConfig.isEmpty() )
    akFatal() << "Did not find MySQL server default configuration (mysql-global.conf)";
  QFile globalFile( globalConfig );
  QFile actualFile( actualConfig );
  if ( globalFile.open( QFile::ReadOnly ) && actualFile.open( QFile::WriteOnly ) ) {
    actualFile.write( globalFile.readAll() );
    if ( !localConfig.isEmpty() ) {
      QFile localFile( localConfig );
      if ( localFile.open( QFile::ReadOnly ) ) {
        actualFile.write( localFile.readAll() );
        localFile.close();
      }
    }
    actualFile.close();
    globalFile.close();
  } else {
    akError() << "Unable to create MySQL server configuration file.";
    akError() << "This means that either the default configuration file (mysql-global.conf) was not readable";
    akFatal() << "or the target file (mysql.conf) could not be written.";
  }

  // MySQL doesn't like world writeable config files (which makes sense), but
  // our config file somehow ends up being world-writable on some systems for no
  // apparent reason nevertheless, so fix that
  const QFile::Permissions allowedPerms = actualFile.permissions()
      & ( QFile::ReadOwner | QFile::WriteOwner | QFile::ReadGroup | QFile::WriteGroup | QFile::ReadOther );
  if ( allowedPerms != actualFile.permissions() )
    actualFile.setPermissions( allowedPerms );

  if ( dataDir.isEmpty() )
    akFatal() << "Akonadi server was not able not create database data directory";

  if ( akDir.isEmpty() )
    akFatal() << "Akonadi server was not able not create database log directory";

  if ( miscDir.isEmpty() )
    akFatal() << "Akonadi server was not able not create database misc directory";

  // move mysql error log file out of the way
  const QFileInfo errorLog( dataDir + QDir::separator() + QString::fromLatin1( "mysql.err" ) );
  if ( errorLog.exists() ) {
    QFile logFile( errorLog.absoluteFilePath() );
    QFile oldLogFile( dataDir + QDir::separator() + QString::fromLatin1( "mysql.err.old" ) );
    if ( logFile.open( QFile::ReadOnly ) && oldLogFile.open( QFile::Append ) ) {
      oldLogFile.write( logFile.readAll() );
      oldLogFile.close();
      logFile.close();
      logFile.remove();
    } else {
      akError() << "Failed to open MySQL error log.";
    }
  }

  // synthesize the mysqld command
  QStringList arguments;
  arguments << QString::fromLatin1( "--defaults-file=%1/mysql.conf" ).arg( akDir );
  arguments << QString::fromLatin1( "--datadir=%1/" ).arg( dataDir );
  arguments << QString::fromLatin1( "--socket=%1/mysql.socket" ).arg( miscDir );

  mDatabaseProcess = new QProcess( this );
  mDatabaseProcess->start( mysqldPath, arguments );
  if ( !mDatabaseProcess->waitForStarted() ) {
    akError() << "Could not start database server!";
    akError() << "executable:" << mysqldPath;
    akError() << "arguments:" << arguments;
    akFatal() << "process error:" << mDatabaseProcess->errorString();
  }

  const QLatin1String initCon( "initConnection" );
  {
    QSqlDatabase db = QSqlDatabase::addDatabase( DbConfig::driverName(), initCon );
    DbConfig::configure( db );
    db.setDatabaseName( QString() ); // might not exist yet, then connecting to the actual db will fail
    if ( !db.isValid() )
      akFatal() << "Invalid database object during database server startup";

    bool opened = false;
    for ( int i = 0; i < 120; ++i ) {
      opened = db.open();
      if ( opened )
        break;
      if ( mDatabaseProcess->waitForFinished( 500 ) ) {
        akError() << "Database process existed unexpectedly during intial connection!";
        akError() << "executable:" << mysqldPath;
        akError() << "arguments:" << arguments;
        akError() << "stdout:" << mDatabaseProcess->readAllStandardOutput();
        akError() << "stderr:" << mDatabaseProcess->readAllStandardError();
        akError() << "exit code:" << mDatabaseProcess->exitCode();
        akFatal() << "process error:" << mDatabaseProcess->errorString();
      }
    }

    if ( opened ) {
      QSqlQuery query( db );
      if ( !query.exec( QString::fromLatin1( "USE %1" ).arg( DbConfig::databaseName() ) ) ) {
        akDebug() << "Failed to use database" << DbConfig::databaseName();
        akDebug() << "Query error:" << query.lastError().text();
        akDebug() << "Database error:" << db.lastError().text();
        akDebug() << "Trying to create database now...";
        if ( !query.exec( QLatin1String( "CREATE DATABASE akonadi" ) ) ) {
          akError() << "Failed to create database";
          akError() << "Query error:" << query.lastError().text();
          akFatal() << "Database error:" << db.lastError().text();
        }
      }
      db.close();
    }
  }

  QSqlDatabase::removeDatabase( initCon );
}

void AkonadiServer::stopDatabaseProcess()
{
  if ( !mDatabaseProcess )
    return;
  mDatabaseProcess->terminate();
  mDatabaseProcess->waitForFinished();
}

void AkonadiServer::serviceOwnerChanged(const QString & name, const QString & oldOwner, const QString & newOwner)
{
  Q_UNUSED( oldOwner );
  if ( name == QLatin1String( AKONADI_DBUS_CONTROL_SERVICE ) && newOwner.isEmpty() ) {
    akError() << "Control process died, committing suicide!";
    quit();
  }
}

#include "akonadi.moc"
<commit_msg>Make sure the query object is destroyed before we close the database connection. Fixes a bunch of invalid reads found by valgrind.<commit_after>/***************************************************************************
 *   Copyright (C) 2006 by Till Adam <adam@kde.org>                        *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU Library General Public License as       *
 *   published by the Free Software Foundation; either version 2 of the    *
 *   License, or (at your option) any later version.                       *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU Library General Public     *
 *   License along with this program; if not, write to the                 *
 *   Free Software Foundation, Inc.,                                       *
 *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.             *
 ***************************************************************************/

#include "akonadi.h"
#include "akonadiconnection.h"
#include "serveradaptor.h"
#include "akdebug.h"

#include "cachecleaner.h"
#include "intervalcheck.h"
#include "storage/dbconfig.h"
#include "storage/datastore.h"
#include "notificationmanager.h"
#include "resourcemanager.h"
#include "tracer.h"
#include "xesammanager.h"
#include "nepomukmanager.h"
#include "debuginterface.h"

#include "../libs/xdgbasedirs_p.h"
#include "../libs/protocol_p.h"

#include <QtCore/QCoreApplication>
#include <QtCore/QDir>
#include <QtCore/QProcess>
#include <QtCore/QSettings>
#include <QtCore/QTimer>

#include <config-akonadi.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#include <stdlib.h>

using namespace Akonadi;

static AkonadiServer *s_instance = 0;

AkonadiServer::AkonadiServer( QObject* parent )
    : QLocalServer( parent )
    , mCacheCleaner( 0 )
    , mIntervalChecker( 0 )
    , mDatabaseProcess( 0 )
    , mAlreadyShutdown( false )
{
    DbConfig::init();
    startDatabaseProcess();

    s_instance = this;

    const QString serverConfigFile = XdgBaseDirs::akonadiServerConfigFile( XdgBaseDirs::ReadWrite );
    QSettings settings( serverConfigFile, QSettings::IniFormat );

    const QString connectionSettingsFile = XdgBaseDirs::akonadiConnectionConfigFile( XdgBaseDirs::WriteOnly );
    QSettings connectionSettings( connectionSettingsFile, QSettings::IniFormat );

#ifdef Q_OS_WIN
    QString namedPipe = settings.value( QLatin1String( "Connection/NamedPipe" ), QLatin1String( "Akonadi" ) ).toString();
    if ( !listen( namedPipe ) )
      akFatal() << "Unable to listen on Named Pipe" << namedPipe;

    connectionSettings.setValue( QLatin1String( "Data/Method" ), QLatin1String( "NamedPipe" ) );
    connectionSettings.setValue( QLatin1String( "Data/NamedPipe" ), namedPipe );
#else
    const QString defaultSocketDir = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi" ) );
    QString socketDir = settings.value( QLatin1String( "Connection/SocketDirectory" ), defaultSocketDir ).toString();
    if ( socketDir[0] != QLatin1Char( '/' ) ) {
      QDir::home().mkdir( socketDir );
      socketDir = QDir::homePath() + QLatin1Char( '/' ) + socketDir;
    }

    const QString socketFile = socketDir + QLatin1String( "/akonadiserver.socket" );
    unlink( socketFile.toUtf8().constData() );
    if ( !listen( socketFile ) )
      akFatal() << "Unable to listen on Unix socket" << socketFile;

    connectionSettings.setValue( QLatin1String( "Data/Method" ), QLatin1String( "UnixPath" ) );
    connectionSettings.setValue( QLatin1String( "Data/UnixPath" ), socketFile );
#endif

    // initialize the database
    DataStore *db = DataStore::self();
    if ( !db->database().isOpen() )
      akFatal() << "Unable to open database" << db->database().lastError().text();
    if ( !db->init() )
      akFatal() << "Unable to initialize database.";

    NotificationManager::self();
    Tracer::self();
    new DebugInterface( this );
    ResourceManager::self();
    if ( settings.value( QLatin1String( "Cache/EnableCleaner" ), true ).toBool() ) {
      mCacheCleaner = new CacheCleaner( this );
      mCacheCleaner->start( QThread::IdlePriority );
    }

    mIntervalChecker = new IntervalCheck( this );
    mIntervalChecker->start( QThread::IdlePriority );

    mSearchManager = new DummySearchManager;

    new ServerAdaptor( this );
    QDBusConnection::sessionBus().registerObject( QLatin1String( "/Server" ), this );

    const QByteArray dbusAddress = qgetenv( "DBUS_SESSION_BUS_ADDRESS" );
    if ( !dbusAddress.isEmpty() ) {
      connectionSettings.setValue( QLatin1String( "DBUS/Address" ), QLatin1String( dbusAddress ) );
    }

    connect( QDBusConnection::sessionBus().interface(), SIGNAL(serviceOwnerChanged(QString,QString,QString)),
             SLOT(serviceOwnerChanged(QString,QString,QString)) );
}

AkonadiServer::~AkonadiServer()
{
}

void AkonadiServer::quit()
{
    if ( mAlreadyShutdown )
      return;

    mAlreadyShutdown = true;

    if ( mCacheCleaner )
      QMetaObject::invokeMethod( mCacheCleaner, "quit", Qt::QueuedConnection );
    if ( mIntervalChecker )
      QMetaObject::invokeMethod( mIntervalChecker, "quit", Qt::QueuedConnection );
    QCoreApplication::instance()->processEvents();

    if ( mCacheCleaner )
      mCacheCleaner->wait();
    if ( mIntervalChecker )
      mIntervalChecker->wait();

    delete mSearchManager;
    mSearchManager = 0;

    for ( int i = 0; i < mConnections.count(); ++i ) {
      if ( mConnections[ i ] ) {
        mConnections[ i ]->quit();
        mConnections[ i ]->wait();
      }
    }

    DataStore::self()->close();

    // execute the deleteLater() calls for the threads so they free their db connections
    // and the following db termination will work
    QCoreApplication::instance()->processEvents();

    stopDatabaseProcess();

    QSettings settings( XdgBaseDirs::akonadiServerConfigFile(), QSettings::IniFormat );
    const QString connectionSettingsFile = XdgBaseDirs::akonadiConnectionConfigFile( XdgBaseDirs::WriteOnly );

#ifndef Q_OS_WIN
    QSettings connectionSettings( connectionSettingsFile, QSettings::IniFormat );
    const QString defaultSocketDir = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi" ) );
    const QString socketDir = settings.value( QLatin1String( "Connection/SocketDirectory" ), defaultSocketDir ).toString();

    if ( !QDir::home().remove( socketDir + QLatin1String( "/akonadiserver.socket" ) ) )
        akError() << "Failed to remove Unix socket";
#endif
    if ( !QDir::home().remove( connectionSettingsFile ) )
        akError() << "Failed to remove runtime connection config file";

    QTimer::singleShot( 0, this, SLOT( doQuit() ) );
}

void AkonadiServer::doQuit()
{
    QCoreApplication::exit();
}

void AkonadiServer::incomingConnection( quintptr socketDescriptor )
{
    QPointer<AkonadiConnection> thread = new AkonadiConnection( socketDescriptor, this );
    connect( thread, SIGNAL( finished() ), thread, SLOT( deleteLater() ) );
    mConnections.append( thread );
    thread->start();
}

AkonadiServer * AkonadiServer::instance()
{
    if ( !s_instance )
        s_instance = new AkonadiServer();
    return s_instance;
}

void AkonadiServer::startDatabaseProcess()
{
  if ( !DbConfig::useInternalServer() )
    return;

  const QString mysqldPath = DbConfig::serverPath();
  if ( mysqldPath.isEmpty() )
    akFatal() << "No path to mysqld set in server configuration!";

  // create the database directories if they don't exists
  const QString dataDir = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi/db_data" ) );
  const QString akDir   = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi/" ) );
  const QString miscDir = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi/db_misc" ) );

  // generate config file
  const QString globalConfig = XdgBaseDirs::findResourceFile( "config", QLatin1String( "akonadi/mysql-global.conf" ) );
  const QString localConfig  = XdgBaseDirs::findResourceFile( "config", QLatin1String( "akonadi/mysql-local.conf" ) );
  const QString actualConfig = XdgBaseDirs::saveDir( "data", QLatin1String( "akonadi" ) ) + QLatin1String("/mysql.conf");
  if ( globalConfig.isEmpty() )
    akFatal() << "Did not find MySQL server default configuration (mysql-global.conf)";
  QFile globalFile( globalConfig );
  QFile actualFile( actualConfig );
  if ( globalFile.open( QFile::ReadOnly ) && actualFile.open( QFile::WriteOnly ) ) {
    actualFile.write( globalFile.readAll() );
    if ( !localConfig.isEmpty() ) {
      QFile localFile( localConfig );
      if ( localFile.open( QFile::ReadOnly ) ) {
        actualFile.write( localFile.readAll() );
        localFile.close();
      }
    }
    actualFile.close();
    globalFile.close();
  } else {
    akError() << "Unable to create MySQL server configuration file.";
    akError() << "This means that either the default configuration file (mysql-global.conf) was not readable";
    akFatal() << "or the target file (mysql.conf) could not be written.";
  }

  // MySQL doesn't like world writeable config files (which makes sense), but
  // our config file somehow ends up being world-writable on some systems for no
  // apparent reason nevertheless, so fix that
  const QFile::Permissions allowedPerms = actualFile.permissions()
      & ( QFile::ReadOwner | QFile::WriteOwner | QFile::ReadGroup | QFile::WriteGroup | QFile::ReadOther );
  if ( allowedPerms != actualFile.permissions() )
    actualFile.setPermissions( allowedPerms );

  if ( dataDir.isEmpty() )
    akFatal() << "Akonadi server was not able not create database data directory";

  if ( akDir.isEmpty() )
    akFatal() << "Akonadi server was not able not create database log directory";

  if ( miscDir.isEmpty() )
    akFatal() << "Akonadi server was not able not create database misc directory";

  // move mysql error log file out of the way
  const QFileInfo errorLog( dataDir + QDir::separator() + QString::fromLatin1( "mysql.err" ) );
  if ( errorLog.exists() ) {
    QFile logFile( errorLog.absoluteFilePath() );
    QFile oldLogFile( dataDir + QDir::separator() + QString::fromLatin1( "mysql.err.old" ) );
    if ( logFile.open( QFile::ReadOnly ) && oldLogFile.open( QFile::Append ) ) {
      oldLogFile.write( logFile.readAll() );
      oldLogFile.close();
      logFile.close();
      logFile.remove();
    } else {
      akError() << "Failed to open MySQL error log.";
    }
  }

  // synthesize the mysqld command
  QStringList arguments;
  arguments << QString::fromLatin1( "--defaults-file=%1/mysql.conf" ).arg( akDir );
  arguments << QString::fromLatin1( "--datadir=%1/" ).arg( dataDir );
  arguments << QString::fromLatin1( "--socket=%1/mysql.socket" ).arg( miscDir );

  mDatabaseProcess = new QProcess( this );
  mDatabaseProcess->start( mysqldPath, arguments );
  if ( !mDatabaseProcess->waitForStarted() ) {
    akError() << "Could not start database server!";
    akError() << "executable:" << mysqldPath;
    akError() << "arguments:" << arguments;
    akFatal() << "process error:" << mDatabaseProcess->errorString();
  }

  const QLatin1String initCon( "initConnection" );
  {
    QSqlDatabase db = QSqlDatabase::addDatabase( DbConfig::driverName(), initCon );
    DbConfig::configure( db );
    db.setDatabaseName( QString() ); // might not exist yet, then connecting to the actual db will fail
    if ( !db.isValid() )
      akFatal() << "Invalid database object during database server startup";

    bool opened = false;
    for ( int i = 0; i < 120; ++i ) {
      opened = db.open();
      if ( opened )
        break;
      if ( mDatabaseProcess->waitForFinished( 500 ) ) {
        akError() << "Database process existed unexpectedly during intial connection!";
        akError() << "executable:" << mysqldPath;
        akError() << "arguments:" << arguments;
        akError() << "stdout:" << mDatabaseProcess->readAllStandardOutput();
        akError() << "stderr:" << mDatabaseProcess->readAllStandardError();
        akError() << "exit code:" << mDatabaseProcess->exitCode();
        akFatal() << "process error:" << mDatabaseProcess->errorString();
      }
    }

    if ( opened ) {
      {
        QSqlQuery query( db );
        if ( !query.exec( QString::fromLatin1( "USE %1" ).arg( DbConfig::databaseName() ) ) ) {
          akDebug() << "Failed to use database" << DbConfig::databaseName();
          akDebug() << "Query error:" << query.lastError().text();
          akDebug() << "Database error:" << db.lastError().text();
          akDebug() << "Trying to create database now...";
          if ( !query.exec( QLatin1String( "CREATE DATABASE akonadi" ) ) ) {
            akError() << "Failed to create database";
            akError() << "Query error:" << query.lastError().text();
            akFatal() << "Database error:" << db.lastError().text();
          }
        }
      } // make sure query is destroyed before we close the db
      db.close();
    }
  }

  QSqlDatabase::removeDatabase( initCon );
}

void AkonadiServer::stopDatabaseProcess()
{
  if ( !mDatabaseProcess )
    return;
  mDatabaseProcess->terminate();
  mDatabaseProcess->waitForFinished();
}

void AkonadiServer::serviceOwnerChanged(const QString & name, const QString & oldOwner, const QString & newOwner)
{
  Q_UNUSED( oldOwner );
  if ( name == QLatin1String( AKONADI_DBUS_CONTROL_SERVICE ) && newOwner.isEmpty() ) {
    akError() << "Control process died, committing suicide!";
    quit();
  }
}

#include "akonadi.moc"
<|endoftext|>