<commit_before>// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
// Copyright (c) 2005, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// ---
// Author: Sanjay Ghemawat
//
// Produce stack trace.
//
// There are three different ways we can try to get the stack trace:
//
// 1) Our hand-coded stack-unwinder.  This depends on a certain stack
//    layout, which is used by gcc (and those systems using a
//    gcc-compatible ABI) on x86 systems, at least since gcc 2.95.
//    It uses the frame pointer to do its work.
//
// 2) The libunwind library.  This is still in development, and as a
//    separate library adds a new dependency, abut doesn't need a frame
//    pointer.  It also doesn't call malloc.
//
// 3) The gdb unwinder -- also the one used by the c++ exception code.
//    It's obviously well-tested, but has a fatal flaw: it can call
//    malloc() from the unwinder.  This is a problem because we're
//    trying to use the unwinder to instrument malloc().
//
// Note: if you add a new implementation here, make sure it works
// correctly when GetStackTrace() is called with max_depth == 0.
// Some code may do that.

#include <config.h>
#include <stdlib.h> // for getenv
#include <string.h> // for strcmp
#include <stdio.h> // for fprintf
#include "gperftools/stacktrace.h"
#include "base/commandlineflags.h"
#include "base/googleinit.h"

// we're using plain struct and not class to avoid any possible issues
// during initialization. Struct of pointers is easy to init at
// link-time.
struct GetStackImplementation {
  int (*GetStackFramesPtr)(void** result, int* sizes, int max_depth,
                           int skip_count);

  int (*GetStackFramesWithContextPtr)(void** result, int* sizes, int max_depth,
                                      int skip_count, const void *uc);

  int (*GetStackTracePtr)(void** result, int max_depth,
                          int skip_count);

  int (*GetStackTraceWithContextPtr)(void** result, int max_depth,
                                  int skip_count, const void *uc);

  const char *name;
};

#if HAVE_DECL_BACKTRACE
#define STACKTRACE_INL_HEADER "stacktrace_generic-inl.h"
#define GST_SUFFIX generic
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_generic
#endif

#if HAVE_LIBUNWIND_H
#define STACKTRACE_INL_HEADER "stacktrace_libunwind-inl.h"
#define GST_SUFFIX libunwind
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_libunwind
#endif // HAVE_LIBUNWIND_H

#if defined(__i386__) || defined(__x86_64__)
#define STACKTRACE_INL_HEADER "stacktrace_x86-inl.h"
#define GST_SUFFIX x86
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_x86
#endif // i386 || x86_64

#if defined(__ppc__) || defined(__PPC__)
#if defined(__linux__)
#define STACKTRACE_INL_HEADER "stacktrace_powerpc-linux-inl.h"
#else
#define STACKTRACE_INL_HEADER "stacktrace_powerpc-darwin-inl.h"
#endif
#define GST_SUFFIX ppc
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_ppc
#endif

#if defined(__arm__)
#define STACKTRACE_INL_HEADER "stacktrace_arm-inl.h"
#define GST_SUFFIX arm
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_arm
#endif

#ifdef TCMALLOC_ENABLE_INSTRUMENT_STACKTRACE
#define STACKTRACE_INL_HEADER "stacktrace_instrument-inl.h"
#define GST_SUFFIX instrument
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_instrument
#endif

// The Windows case -- probably cygwin and mingw will use one of the
// x86-includes above, but if not, we can fall back to windows intrinsics.
#if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
#define STACKTRACE_INL_HEADER "stacktrace_win32-inl.h"
#define GST_SUFFIX win32
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_win32
#endif

static GetStackImplementation *all_impls[] = {
#ifdef HAVE_GST_generic
  &impl__generic,
#endif
#ifdef HAVE_GST_libunwind
  &impl__libunwind,
#endif
#ifdef HAVE_GST_x86
  &impl__x86,
#endif
#ifdef HAVE_GST_arm
  &impl__arm,
#endif
#ifdef HAVE_GST_ppc
  &impl__ppc,
#endif
#ifdef HAVE_GST_instrument
  &impl__instrument,
#endif
#ifdef HAVE_GST_win32
  &impl__win32,
#endif
  NULL
};

// ppc and i386 implementations prefer arch-specific asm implementations.
// arm's asm implementation is broken
#if defined(__i386__) || defined(__x86_64__) || defined(__ppc__) || defined(__PPC__)
#if !defined(NO_FRAME_POINTER)
#define TCMALLOC_DONT_PREFER_LIBUNWIND
#endif
#endif

#if defined(HAVE_GST_instrument)
static GetStackImplementation *get_stack_impl = &impl__instrument;
#elif defined(HAVE_GST_win32)
static GetStackImplementation *get_stack_impl = &impl__win32;
#elif defined(HAVE_GST_x86) && defined(TCMALLOC_DONT_PREFER_LIBUNWIND)
static GetStackImplementation *get_stack_impl = &impl__x86;
#elif defined(HAVE_GST_ppc) && defined(TCMALLOC_DONT_PREFER_LIBUNWIND)
static GetStackImplementation *get_stack_impl = &impl__ppc;
#elif defined(HAVE_GST_libunwind)
static GetStackImplementation *get_stack_impl = &impl__libunwind;
#elif defined(HAVE_GST_arm)
static GetStackImplementation *get_stack_impl = &impl__arm;
#elif defined(HAVE_GST_generic)
static GetStackImplementation *get_stack_impl = &impl__generic;
#elif 0
// This is for the benefit of code analysis tools that may have
// trouble with the computed #include above.
# include "stacktrace_x86-inl.h"
# include "stacktrace_libunwind-inl.h"
# include "stacktrace_generic-inl.h"
# include "stacktrace_powerpc-inl.h"
# include "stacktrace_win32-inl.h"
# include "stacktrace_arm-inl.h"
# include "stacktrace_instrument-inl.h"
#else
#error Cannot calculate stack trace: will need to write for your environment
#endif

static int ATTRIBUTE_NOINLINE frame_forcer(int rv) {
  return rv;
}

PERFTOOLS_DLL_DECL int GetStackFrames(void** result, int* sizes, int max_depth,
                                      int skip_count) {
  return frame_forcer(get_stack_impl->GetStackFramesPtr(result, sizes, max_depth, skip_count));
}

PERFTOOLS_DLL_DECL int GetStackFramesWithContext(void** result, int* sizes, int max_depth,
                                                 int skip_count, const void *uc) {
  return frame_forcer(get_stack_impl->GetStackFramesWithContextPtr(
                        result, sizes, max_depth,
                        skip_count, uc));
}

PERFTOOLS_DLL_DECL int GetStackTrace(void** result, int max_depth,
                                     int skip_count) {
  return frame_forcer(get_stack_impl->GetStackTracePtr(result, max_depth, skip_count));
}

PERFTOOLS_DLL_DECL int GetStackTraceWithContext(void** result, int max_depth,
                                                int skip_count, const void *uc) {
  return frame_forcer(get_stack_impl->GetStackTraceWithContextPtr(
                        result, max_depth, skip_count, uc));
}

static void init_default_stack_impl_inner(void) {
  char *val = getenv("TCMALLOC_STACKTRACE_METHOD");
  if (!val || !*val) {
    return;
  }
  for (GetStackImplementation **p = all_impls; *p; p++) {
    GetStackImplementation *c = *p;
    if (strcmp(c->name, val) == 0) {
      get_stack_impl = c;
      return;
    }
  }
  fprintf(stderr, "Unknown or unsupported stacktrace method requested: %s. Ignoring it\n", val);
}

static void init_default_stack_impl(void) {
  init_default_stack_impl_inner();
  if (EnvToBool("TCMALLOC_STACKTRACE_METHOD_VERBOSE", false)) {
    fprintf(stderr, "Chosen stacktrace method is %s\nSupported methods:\n", get_stack_impl->name);
    for (GetStackImplementation **p = all_impls; *p; p++) {
      GetStackImplementation *c = *p;
      fprintf(stderr, "* %s\n", c->name);
    }
    fputs("\n", stderr);
  }
}

REGISTER_MODULE_INITIALIZER(stacktrace_init_default_stack_impl, init_default_stack_impl());
<commit_msg>compile libunwind unwinder only of __thread is supported<commit_after>// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
// Copyright (c) 2005, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// ---
// Author: Sanjay Ghemawat
//
// Produce stack trace.
//
// There are three different ways we can try to get the stack trace:
//
// 1) Our hand-coded stack-unwinder.  This depends on a certain stack
//    layout, which is used by gcc (and those systems using a
//    gcc-compatible ABI) on x86 systems, at least since gcc 2.95.
//    It uses the frame pointer to do its work.
//
// 2) The libunwind library.  This is still in development, and as a
//    separate library adds a new dependency, abut doesn't need a frame
//    pointer.  It also doesn't call malloc.
//
// 3) The gdb unwinder -- also the one used by the c++ exception code.
//    It's obviously well-tested, but has a fatal flaw: it can call
//    malloc() from the unwinder.  This is a problem because we're
//    trying to use the unwinder to instrument malloc().
//
// Note: if you add a new implementation here, make sure it works
// correctly when GetStackTrace() is called with max_depth == 0.
// Some code may do that.

#include <config.h>
#include <stdlib.h> // for getenv
#include <string.h> // for strcmp
#include <stdio.h> // for fprintf
#include "gperftools/stacktrace.h"
#include "base/commandlineflags.h"
#include "base/googleinit.h"

// we're using plain struct and not class to avoid any possible issues
// during initialization. Struct of pointers is easy to init at
// link-time.
struct GetStackImplementation {
  int (*GetStackFramesPtr)(void** result, int* sizes, int max_depth,
                           int skip_count);

  int (*GetStackFramesWithContextPtr)(void** result, int* sizes, int max_depth,
                                      int skip_count, const void *uc);

  int (*GetStackTracePtr)(void** result, int max_depth,
                          int skip_count);

  int (*GetStackTraceWithContextPtr)(void** result, int max_depth,
                                  int skip_count, const void *uc);

  const char *name;
};

#if HAVE_DECL_BACKTRACE
#define STACKTRACE_INL_HEADER "stacktrace_generic-inl.h"
#define GST_SUFFIX generic
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_generic
#endif

// libunwind uses __thread so we check for both libunwind.h and
// __thread support
#if defined(HAVE_LIBUNWIND_H) && defined(HAVE_TLS)
#define STACKTRACE_INL_HEADER "stacktrace_libunwind-inl.h"
#define GST_SUFFIX libunwind
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_libunwind
#endif // HAVE_LIBUNWIND_H

#if defined(__i386__) || defined(__x86_64__)
#define STACKTRACE_INL_HEADER "stacktrace_x86-inl.h"
#define GST_SUFFIX x86
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_x86
#endif // i386 || x86_64

#if defined(__ppc__) || defined(__PPC__)
#if defined(__linux__)
#define STACKTRACE_INL_HEADER "stacktrace_powerpc-linux-inl.h"
#else
#define STACKTRACE_INL_HEADER "stacktrace_powerpc-darwin-inl.h"
#endif
#define GST_SUFFIX ppc
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_ppc
#endif

#if defined(__arm__)
#define STACKTRACE_INL_HEADER "stacktrace_arm-inl.h"
#define GST_SUFFIX arm
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_arm
#endif

#ifdef TCMALLOC_ENABLE_INSTRUMENT_STACKTRACE
#define STACKTRACE_INL_HEADER "stacktrace_instrument-inl.h"
#define GST_SUFFIX instrument
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_instrument
#endif

// The Windows case -- probably cygwin and mingw will use one of the
// x86-includes above, but if not, we can fall back to windows intrinsics.
#if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
#define STACKTRACE_INL_HEADER "stacktrace_win32-inl.h"
#define GST_SUFFIX win32
#include "stacktrace_impl_setup-inl.h"
#undef GST_SUFFIX
#undef STACKTRACE_INL_HEADER
#define HAVE_GST_win32
#endif

static GetStackImplementation *all_impls[] = {
#ifdef HAVE_GST_generic
  &impl__generic,
#endif
#ifdef HAVE_GST_libunwind
  &impl__libunwind,
#endif
#ifdef HAVE_GST_x86
  &impl__x86,
#endif
#ifdef HAVE_GST_arm
  &impl__arm,
#endif
#ifdef HAVE_GST_ppc
  &impl__ppc,
#endif
#ifdef HAVE_GST_instrument
  &impl__instrument,
#endif
#ifdef HAVE_GST_win32
  &impl__win32,
#endif
  NULL
};

// ppc and i386 implementations prefer arch-specific asm implementations.
// arm's asm implementation is broken
#if defined(__i386__) || defined(__x86_64__) || defined(__ppc__) || defined(__PPC__)
#if !defined(NO_FRAME_POINTER)
#define TCMALLOC_DONT_PREFER_LIBUNWIND
#endif
#endif

#if defined(HAVE_GST_instrument)
static GetStackImplementation *get_stack_impl = &impl__instrument;
#elif defined(HAVE_GST_win32)
static GetStackImplementation *get_stack_impl = &impl__win32;
#elif defined(HAVE_GST_x86) && defined(TCMALLOC_DONT_PREFER_LIBUNWIND)
static GetStackImplementation *get_stack_impl = &impl__x86;
#elif defined(HAVE_GST_ppc) && defined(TCMALLOC_DONT_PREFER_LIBUNWIND)
static GetStackImplementation *get_stack_impl = &impl__ppc;
#elif defined(HAVE_GST_libunwind)
static GetStackImplementation *get_stack_impl = &impl__libunwind;
#elif defined(HAVE_GST_arm)
static GetStackImplementation *get_stack_impl = &impl__arm;
#elif defined(HAVE_GST_generic)
static GetStackImplementation *get_stack_impl = &impl__generic;
#elif 0
// This is for the benefit of code analysis tools that may have
// trouble with the computed #include above.
# include "stacktrace_x86-inl.h"
# include "stacktrace_libunwind-inl.h"
# include "stacktrace_generic-inl.h"
# include "stacktrace_powerpc-inl.h"
# include "stacktrace_win32-inl.h"
# include "stacktrace_arm-inl.h"
# include "stacktrace_instrument-inl.h"
#else
#error Cannot calculate stack trace: will need to write for your environment
#endif

static int ATTRIBUTE_NOINLINE frame_forcer(int rv) {
  return rv;
}

PERFTOOLS_DLL_DECL int GetStackFrames(void** result, int* sizes, int max_depth,
                                      int skip_count) {
  return frame_forcer(get_stack_impl->GetStackFramesPtr(result, sizes, max_depth, skip_count));
}

PERFTOOLS_DLL_DECL int GetStackFramesWithContext(void** result, int* sizes, int max_depth,
                                                 int skip_count, const void *uc) {
  return frame_forcer(get_stack_impl->GetStackFramesWithContextPtr(
                        result, sizes, max_depth,
                        skip_count, uc));
}

PERFTOOLS_DLL_DECL int GetStackTrace(void** result, int max_depth,
                                     int skip_count) {
  return frame_forcer(get_stack_impl->GetStackTracePtr(result, max_depth, skip_count));
}

PERFTOOLS_DLL_DECL int GetStackTraceWithContext(void** result, int max_depth,
                                                int skip_count, const void *uc) {
  return frame_forcer(get_stack_impl->GetStackTraceWithContextPtr(
                        result, max_depth, skip_count, uc));
}

static void init_default_stack_impl_inner(void) {
  char *val = getenv("TCMALLOC_STACKTRACE_METHOD");
  if (!val || !*val) {
    return;
  }
  for (GetStackImplementation **p = all_impls; *p; p++) {
    GetStackImplementation *c = *p;
    if (strcmp(c->name, val) == 0) {
      get_stack_impl = c;
      return;
    }
  }
  fprintf(stderr, "Unknown or unsupported stacktrace method requested: %s. Ignoring it\n", val);
}

static void init_default_stack_impl(void) {
  init_default_stack_impl_inner();
  if (EnvToBool("TCMALLOC_STACKTRACE_METHOD_VERBOSE", false)) {
    fprintf(stderr, "Chosen stacktrace method is %s\nSupported methods:\n", get_stack_impl->name);
    for (GetStackImplementation **p = all_impls; *p; p++) {
      GetStackImplementation *c = *p;
      fprintf(stderr, "* %s\n", c->name);
    }
    fputs("\n", stderr);
  }
}

REGISTER_MODULE_INITIALIZER(stacktrace_init_default_stack_impl, init_default_stack_impl());
<|endoftext|>


<commit_before>/** \file    patch_up_ppns_for_k10plus.cc
 *  \brief   Swaps out all persistent old PPN's with new PPN's.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2019, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <memory>
#include <unordered_map>
#include <vector>
#include <cstdlib>
#include <cstring>
#include <kchashdb.h>
#include "BSZUtil.h"
#include "Compiler.h"
#include "DbConnection.h"
#include "DbResultSet.h"
#include "DbRow.h"
#include "FileUtil.h"
#include "MapUtil.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "UBTools.h"
#include "util.h"
#include "VuFind.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("[--store-only] marc_input1 [marc_input2 .. marc_inputN] [-- deletion_list1 deletion_list2 .. deletion_listN]\n"
            "If --store-only has been specified, no swapping will be performed and only the persistent map file will be overwritten.\n"
            "If deletion lists should be processed, they need to be specified after a double-hyphen to indicate the end of the MARC files.");
}

struct PPNsAndSigil {
    std::string old_ppn_, old_sigil_, new_ppn_;
public:
    PPNsAndSigil(const std::string &old_ppn, const std::string &old_sigil, const std::string &new_ppn)
        : old_ppn_(old_ppn), old_sigil_(old_sigil), new_ppn_(new_ppn) { }
    PPNsAndSigil() = default;
    PPNsAndSigil(const PPNsAndSigil &other) = default;
};

void LoadMapping(MARC::Reader * const marc_reader,
                 const std::unordered_multimap<std::string, std::string> &already_processed_ppns_and_sigils,
                 std::vector<PPNsAndSigil> * const old_ppns_sigils_and_new_ppns)
{
    // We need to consider the sigil of the old BSZ as well as the K10+ sigil because in future there may also be merges of
    // K10+ entities which will lead to old PPN's being K10+ PPN's.
    auto matcher(RegexMatcher::RegexMatcherFactoryOrDie("^\\((DE-576|DE-627)\\)(.+)"));
    while (const auto record = marc_reader->read()) {
        for (const auto &field : record.getTagRange("035")) {
            const auto subfield_a(field.getFirstSubfieldWithCode('a'));
            if (matcher->matched(subfield_a)) {
                const std::string old_sigil((*matcher)[1]);
                const std::string old_ppn((*matcher)[2]);
                if (not MapUtil::Contains(already_processed_ppns_and_sigils, old_ppn, old_sigil))
                    old_ppns_sigils_and_new_ppns->emplace_back(old_ppn, old_sigil, record.getControlNumber());
            }
        }
    }

    LOG_INFO("Found " + std::to_string(old_ppns_sigils_and_new_ppns->size()) + " new mappings of old PPN's to new PPN's in \""
             + marc_reader->getPath() + "\".\n");
}

void PatchTable(DbConnection * const db_connection, const std::string &table, const std::string &column,
                const std::vector<PPNsAndSigil> &old_ppns_sigils_and_new_ppns)
{
    const unsigned MAX_BATCH_SIZE(100);

    db_connection->queryOrDie("BEGIN");

    unsigned replacement_count(0), batch_size(0);
    for (const auto &old_ppn_sigil_and_new_ppn : old_ppns_sigils_and_new_ppns) {
        ++batch_size;
        db_connection->queryOrDie("UPDATE IGNORE " + table + " SET " + column + "='" + old_ppn_sigil_and_new_ppn.new_ppn_
                                  + "' WHERE " + column + "='" + old_ppn_sigil_and_new_ppn.old_ppn_ + "'");
        replacement_count += db_connection->getNoOfAffectedRows();
        if (batch_size >= MAX_BATCH_SIZE) {
            db_connection->queryOrDie("COMMIT");
            db_connection->queryOrDie("BEGIN");
        }
    }

    db_connection->queryOrDie("COMMIT");

    LOG_INFO("Replaced " + std::to_string(replacement_count) + " rows in " + table + ".");
}

void DeleteFromTable(DbConnection * const db_connection, const std::string &table, const std::string &column,
                     const std::unordered_set<std::string> &deletion_ppns)
{
    const unsigned MAX_BATCH_SIZE(100);

    db_connection->queryOrDie("BEGIN");

    unsigned deletion_count(0), batch_size(0);
    for (const auto &deletion_ppn : deletion_ppns) {
        ++batch_size;
        db_connection->queryOrDie("DELETE FROM '" + table + "' WHERE " + column + "='" + deletion_ppn + "'");
        deletion_count += db_connection->getNoOfAffectedRows();
        if (batch_size >= MAX_BATCH_SIZE) {
            db_connection->queryOrDie("COMMIT");
            db_connection->queryOrDie("BEGIN");
        }
    }

    db_connection->queryOrDie("COMMIT");

    LOG_INFO("Deleted " + std::to_string(deletion_count) + " rows from " + table + ".");
}

void PatchNotifiedDB(const std::string &user_type, const std::vector<PPNsAndSigil> &old_ppns_sigils_and_new_ppns) {
    const std::string DB_FILENAME(UBTools::GetTuelibPath() + user_type + "_notified.db");
    std::unique_ptr<kyotocabinet::HashDB> db(new kyotocabinet::HashDB());
    if (not (db->open(DB_FILENAME, kyotocabinet::HashDB::OWRITER | kyotocabinet::HashDB::OREADER))) {
        LOG_INFO("\"" + DB_FILENAME + "\" not found!");
        return;
    }

    unsigned updated_count(0);
    for (const auto &ppns_and_sigil : old_ppns_sigils_and_new_ppns) {
        std::string value;
        if (db->get(ppns_and_sigil.old_ppn_, &value)) {
            if (unlikely(not db->remove(ppns_and_sigil.old_ppn_)))
                LOG_ERROR("failed to remove key \"" + ppns_and_sigil.old_ppn_ + "\" from \"" + DB_FILENAME + "\"!");
            if (unlikely(not db->add(ppns_and_sigil.old_ppn_, value)))
                LOG_ERROR("failed to add key \"" + ppns_and_sigil.old_ppn_ + "\" from \"" + DB_FILENAME + "\"!");
            ++updated_count;
        }
    }

    LOG_INFO("Updated " + std::to_string(updated_count) + " entries in \"" + DB_FILENAME + "\".");
}

void DeleteFromNotifiedDB(const std::string &user_type, const std::unordered_set<std::string> &deletion_ppns) {
    const std::string DB_FILENAME(UBTools::GetTuelibPath() + user_type + "_notified.db");
    std::unique_ptr<kyotocabinet::HashDB> db(new kyotocabinet::HashDB());
    if (not (db->open(DB_FILENAME, kyotocabinet::HashDB::OWRITER | kyotocabinet::HashDB::OREADER))) {
        LOG_INFO("\"" + DB_FILENAME + "\" not found!");
        return;
    }

    unsigned deletion_count(0);
    for (const auto &deletion_ppn : deletion_ppns) {
        if (db->remove(deletion_ppn))
            ++deletion_count;
    }

    LOG_INFO("Deleted " + std::to_string(deletion_count) + " entries from \"" + DB_FILENAME + "\".");
}

bool HaveAllPermissions(DbConnection * const db_connection, const std::string &database) {
    const std::string QUERY("SHOW GRANTS FOR '" + db_connection->getUser() + "'@'" + db_connection->getHost() + "'");
    if (not db_connection->query(QUERY)) {
        if (db_connection->getLastErrorCode() == 1141)
            return false;
        LOG_ERROR(QUERY + " failed: " + db_connection->getLastErrorMessage());
    }

    DbResultSet result_set(db_connection->getLastResultSet());
    while (const auto row = result_set.getNextRow()) {
        if (row[0]
            == "GRANT ALL PRIVILEGES ON `" + database + "`.* TO '" + db_connection->getUser() + "'@'" + db_connection->getHost() + "'")
            return true;
    }
    return false;
}

void CheckMySQLPermissions(DbConnection * const db_connection) {
    if (not HaveAllPermissions(db_connection, "vufind"))
        LOG_ERROR("'" + db_connection->getUser() + "'@'" + db_connection->getHost() + "' needs all permissions on the vufind database!");
    if (VuFind::GetTueFindFlavour() == "ixtheo") {
        if (not HaveAllPermissions(db_connection, "ixtheo"))
            LOG_ERROR("'" + db_connection->getUser() + "'@' " + db_connection->getHost()
                      + "' needs all permissions on the ixtheo database!");
    }
}

void AddPPNsAndSigilsToMultiMap(const std::vector<PPNsAndSigil> &old_ppns_sigils_and_new_ppns,
                                std::unordered_multimap<std::string, std::string> * const already_processed_ppns_and_sigils)
{
    for (const auto &old_ppn_sigil_and_new_ppn : old_ppns_sigils_and_new_ppns)
        already_processed_ppns_and_sigils->emplace(std::make_pair(old_ppn_sigil_and_new_ppn.old_ppn_, old_ppn_sigil_and_new_ppn.old_sigil_));
}

template<class SetOrMap, typename ProcessNotifieldDBFunc, typename ProcessTableFunc>
void ProcessAllDatabases(DbConnection * const db_connection, const SetOrMap &set_or_map, const ProcessNotifieldDBFunc notified_db_func,
                         const ProcessTableFunc table_func)
{
    notified_db_func("ixtheo", set_or_map);
    notified_db_func("relbib", set_or_map);

    table_func(db_connection, "vufind.resource", "record_id", set_or_map);
    table_func(db_connection, "vufind.record", "record_id", set_or_map);
    table_func(db_connection, "vufind.change_tracker", "id", set_or_map);
    if (VuFind::GetTueFindFlavour() == "ixtheo") {
        table_func(db_connection, "ixtheo.keyword_translations", "ppn", set_or_map);
        table_func(db_connection, "vufind.ixtheo_journal_subscriptions", "journal_control_number_or_bundle_name",
                   set_or_map);
        table_func(db_connection, "vufind.ixtheo_pda_subscriptions", "book_ppn", set_or_map);
        table_func(db_connection, "vufind.relbib_ids", "record_id", set_or_map);
        table_func(db_connection, "vufind.bibstudies_ids", "record_id", set_or_map);
    }
}

} // unnamed namespace

static const std::string ALREADY_SWAPPED_PPNS_MAP_FILE(UBTools::GetTuelibPath() + "k10+_ppn_map.map");

int Main(int argc, char **argv) {
    if (argc < 2)
        Usage();

    bool store_only(false);
    if (std::strcmp(argv[1], "--store-only") == 0) {
        store_only = true;
        --argc, ++argv;
        if (argc < 2)
            Usage();
    }

    DbConnection db_connection; // ub_tools user

    CheckMySQLPermissions(&db_connection);

    std::unordered_multimap<std::string, std::string> already_processed_ppns_and_sigils;
    if (not FileUtil::Exists(ALREADY_SWAPPED_PPNS_MAP_FILE))
        FileUtil::WriteStringOrDie(ALREADY_SWAPPED_PPNS_MAP_FILE, "");
    if (not store_only)
        MapUtil::DeserialiseMap(ALREADY_SWAPPED_PPNS_MAP_FILE, &already_processed_ppns_and_sigils);

    std::vector<PPNsAndSigil> old_ppns_sigils_and_new_ppns;
    int arg_no(1);
    for (/* Intentionally empty! */; arg_no < argc; ++arg_no) {
        if (__builtin_strcmp(argv[arg_no], "--")) {
            ++arg_no;
            break;
        }
        const auto marc_reader(MARC::Reader::Factory(argv[arg_no]));
        LoadMapping(marc_reader.get(), already_processed_ppns_and_sigils, &old_ppns_sigils_and_new_ppns);
    }

    std::unordered_set <std::string> title_deletion_ppns;
    for (/* Intentionally empty! */; arg_no < argc; ++arg_no) {
        const auto input(FileUtil::OpenInputFileOrDie(argv[arg_no]));
        std::unordered_set <std::string> local_deletion_ids;
        BSZUtil::ExtractDeletionIds(input.get(), &title_deletion_ppns, &local_deletion_ids);
    }

    if (old_ppns_sigils_and_new_ppns.empty() and title_deletion_ppns.empty()) {
        LOG_INFO("nothing to do!");
        return EXIT_SUCCESS;
    }
    if (old_ppns_sigils_and_new_ppns.empty())
        goto clean_up_deleted_ppns;

    if (store_only) {
        AddPPNsAndSigilsToMultiMap(old_ppns_sigils_and_new_ppns, &already_processed_ppns_and_sigils);
        MapUtil::SerialiseMap(ALREADY_SWAPPED_PPNS_MAP_FILE, already_processed_ppns_and_sigils);
        return EXIT_SUCCESS;
    }

    ProcessAllDatabases(&db_connection, old_ppns_sigils_and_new_ppns, PatchNotifiedDB, PatchTable);
    AddPPNsAndSigilsToMultiMap(old_ppns_sigils_and_new_ppns, &already_processed_ppns_and_sigils);
    MapUtil::SerialiseMap(ALREADY_SWAPPED_PPNS_MAP_FILE, already_processed_ppns_and_sigils);

clean_up_deleted_ppns:
    ProcessAllDatabases(&db_connection, title_deletion_ppns, DeleteFromNotifiedDB, DeleteFromTable);

    return EXIT_SUCCESS;
}
<commit_msg>And another bug fix.<commit_after>/** \file    patch_up_ppns_for_k10plus.cc
 *  \brief   Swaps out all persistent old PPN's with new PPN's.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2019, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <memory>
#include <unordered_map>
#include <vector>
#include <cstdlib>
#include <cstring>
#include <kchashdb.h>
#include "BSZUtil.h"
#include "Compiler.h"
#include "DbConnection.h"
#include "DbResultSet.h"
#include "DbRow.h"
#include "FileUtil.h"
#include "MapUtil.h"
#include "MARC.h"
#include "RegexMatcher.h"
#include "StringUtil.h"
#include "UBTools.h"
#include "util.h"
#include "VuFind.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("[--store-only] marc_input1 [marc_input2 .. marc_inputN] [-- deletion_list1 deletion_list2 .. deletion_listN]\n"
            "If --store-only has been specified, no swapping will be performed and only the persistent map file will be overwritten.\n"
            "If deletion lists should be processed, they need to be specified after a double-hyphen to indicate the end of the MARC files.");
}

struct PPNsAndSigil {
    std::string old_ppn_, old_sigil_, new_ppn_;
public:
    PPNsAndSigil(const std::string &old_ppn, const std::string &old_sigil, const std::string &new_ppn)
        : old_ppn_(old_ppn), old_sigil_(old_sigil), new_ppn_(new_ppn) { }
    PPNsAndSigil() = default;
    PPNsAndSigil(const PPNsAndSigil &other) = default;
};

void LoadMapping(MARC::Reader * const marc_reader,
                 const std::unordered_multimap<std::string, std::string> &already_processed_ppns_and_sigils,
                 std::vector<PPNsAndSigil> * const old_ppns_sigils_and_new_ppns)
{
    // We need to consider the sigil of the old BSZ as well as the K10+ sigil because in future there may also be merges of
    // K10+ entities which will lead to old PPN's being K10+ PPN's.
    auto matcher(RegexMatcher::RegexMatcherFactoryOrDie("^\\((DE-576|DE-627)\\)(.+)"));
    while (const auto record = marc_reader->read()) {
        for (const auto &field : record.getTagRange("035")) {
            const auto subfield_a(field.getFirstSubfieldWithCode('a'));
            if (matcher->matched(subfield_a)) {
                const std::string old_sigil((*matcher)[1]);
                const std::string old_ppn((*matcher)[2]);
                if (not MapUtil::Contains(already_processed_ppns_and_sigils, old_ppn, old_sigil))
                    old_ppns_sigils_and_new_ppns->emplace_back(old_ppn, old_sigil, record.getControlNumber());
            }
        }
    }

    LOG_INFO("Found " + std::to_string(old_ppns_sigils_and_new_ppns->size()) + " new mappings of old PPN's to new PPN's in \""
             + marc_reader->getPath() + "\".\n");
}

void PatchTable(DbConnection * const db_connection, const std::string &table, const std::string &column,
                const std::vector<PPNsAndSigil> &old_ppns_sigils_and_new_ppns)
{
    const unsigned MAX_BATCH_SIZE(100);

    db_connection->queryOrDie("BEGIN");

    unsigned replacement_count(0), batch_size(0);
    for (const auto &old_ppn_sigil_and_new_ppn : old_ppns_sigils_and_new_ppns) {
        ++batch_size;
        db_connection->queryOrDie("UPDATE IGNORE " + table + " SET " + column + "='" + old_ppn_sigil_and_new_ppn.new_ppn_
                                  + "' WHERE " + column + "='" + old_ppn_sigil_and_new_ppn.old_ppn_ + "'");
        replacement_count += db_connection->getNoOfAffectedRows();
        if (batch_size >= MAX_BATCH_SIZE) {
            db_connection->queryOrDie("COMMIT");
            db_connection->queryOrDie("BEGIN");
        }
    }

    db_connection->queryOrDie("COMMIT");

    LOG_INFO("Replaced " + std::to_string(replacement_count) + " rows in " + table + ".");
}

void DeleteFromTable(DbConnection * const db_connection, const std::string &table, const std::string &column,
                     const std::unordered_set<std::string> &deletion_ppns)
{
    const unsigned MAX_BATCH_SIZE(100);

    db_connection->queryOrDie("BEGIN");

    unsigned deletion_count(0), batch_size(0);
    for (const auto &deletion_ppn : deletion_ppns) {
        ++batch_size;
        db_connection->queryOrDie("DELETE FROM '" + table + "' WHERE " + column + "='" + deletion_ppn + "'");
        deletion_count += db_connection->getNoOfAffectedRows();
        if (batch_size >= MAX_BATCH_SIZE) {
            db_connection->queryOrDie("COMMIT");
            db_connection->queryOrDie("BEGIN");
        }
    }

    db_connection->queryOrDie("COMMIT");

    LOG_INFO("Deleted " + std::to_string(deletion_count) + " rows from " + table + ".");
}

void PatchNotifiedDB(const std::string &user_type, const std::vector<PPNsAndSigil> &old_ppns_sigils_and_new_ppns) {
    const std::string DB_FILENAME(UBTools::GetTuelibPath() + user_type + "_notified.db");
    std::unique_ptr<kyotocabinet::HashDB> db(new kyotocabinet::HashDB());
    if (not (db->open(DB_FILENAME, kyotocabinet::HashDB::OWRITER | kyotocabinet::HashDB::OREADER))) {
        LOG_INFO("\"" + DB_FILENAME + "\" not found!");
        return;
    }

    unsigned updated_count(0);
    for (const auto &ppns_and_sigil : old_ppns_sigils_and_new_ppns) {
        std::string value;
        if (db->get(ppns_and_sigil.old_ppn_, &value)) {
            if (unlikely(not db->remove(ppns_and_sigil.old_ppn_)))
                LOG_ERROR("failed to remove key \"" + ppns_and_sigil.old_ppn_ + "\" from \"" + DB_FILENAME + "\"!");
            if (unlikely(not db->add(ppns_and_sigil.old_ppn_, value)))
                LOG_ERROR("failed to add key \"" + ppns_and_sigil.old_ppn_ + "\" from \"" + DB_FILENAME + "\"!");
            ++updated_count;
        }
    }

    LOG_INFO("Updated " + std::to_string(updated_count) + " entries in \"" + DB_FILENAME + "\".");
}

void DeleteFromNotifiedDB(const std::string &user_type, const std::unordered_set<std::string> &deletion_ppns) {
    const std::string DB_FILENAME(UBTools::GetTuelibPath() + user_type + "_notified.db");
    std::unique_ptr<kyotocabinet::HashDB> db(new kyotocabinet::HashDB());
    if (not (db->open(DB_FILENAME, kyotocabinet::HashDB::OWRITER | kyotocabinet::HashDB::OREADER))) {
        LOG_INFO("\"" + DB_FILENAME + "\" not found!");
        return;
    }

    unsigned deletion_count(0);
    for (const auto &deletion_ppn : deletion_ppns) {
        if (db->remove(deletion_ppn))
            ++deletion_count;
    }

    LOG_INFO("Deleted " + std::to_string(deletion_count) + " entries from \"" + DB_FILENAME + "\".");
}

bool HaveAllPermissions(DbConnection * const db_connection, const std::string &database) {
    const std::string QUERY("SHOW GRANTS FOR '" + db_connection->getUser() + "'@'" + db_connection->getHost() + "'");
    if (not db_connection->query(QUERY)) {
        if (db_connection->getLastErrorCode() == 1141)
            return false;
        LOG_ERROR(QUERY + " failed: " + db_connection->getLastErrorMessage());
    }

    DbResultSet result_set(db_connection->getLastResultSet());
    while (const auto row = result_set.getNextRow()) {
        if (row[0]
            == "GRANT ALL PRIVILEGES ON `" + database + "`.* TO '" + db_connection->getUser() + "'@'" + db_connection->getHost() + "'")
            return true;
    }
    return false;
}

void CheckMySQLPermissions(DbConnection * const db_connection) {
    if (not HaveAllPermissions(db_connection, "vufind"))
        LOG_ERROR("'" + db_connection->getUser() + "'@'" + db_connection->getHost() + "' needs all permissions on the vufind database!");
    if (VuFind::GetTueFindFlavour() == "ixtheo") {
        if (not HaveAllPermissions(db_connection, "ixtheo"))
            LOG_ERROR("'" + db_connection->getUser() + "'@' " + db_connection->getHost()
                      + "' needs all permissions on the ixtheo database!");
    }
}

void AddPPNsAndSigilsToMultiMap(const std::vector<PPNsAndSigil> &old_ppns_sigils_and_new_ppns,
                                std::unordered_multimap<std::string, std::string> * const already_processed_ppns_and_sigils)
{
    for (const auto &old_ppn_sigil_and_new_ppn : old_ppns_sigils_and_new_ppns)
        already_processed_ppns_and_sigils->emplace(std::make_pair(old_ppn_sigil_and_new_ppn.old_ppn_, old_ppn_sigil_and_new_ppn.old_sigil_));
}

template<class SetOrMap, typename ProcessNotifieldDBFunc, typename ProcessTableFunc>
void ProcessAllDatabases(DbConnection * const db_connection, const SetOrMap &set_or_map, const ProcessNotifieldDBFunc notified_db_func,
                         const ProcessTableFunc table_func)
{
    notified_db_func("ixtheo", set_or_map);
    notified_db_func("relbib", set_or_map);

    table_func(db_connection, "vufind.resource", "record_id", set_or_map);
    table_func(db_connection, "vufind.record", "record_id", set_or_map);
    table_func(db_connection, "vufind.change_tracker", "id", set_or_map);
    if (VuFind::GetTueFindFlavour() == "ixtheo") {
        table_func(db_connection, "ixtheo.keyword_translations", "ppn", set_or_map);
        table_func(db_connection, "vufind.ixtheo_journal_subscriptions", "journal_control_number_or_bundle_name",
                   set_or_map);
        table_func(db_connection, "vufind.ixtheo_pda_subscriptions", "book_ppn", set_or_map);
        table_func(db_connection, "vufind.relbib_ids", "record_id", set_or_map);
        table_func(db_connection, "vufind.bibstudies_ids", "record_id", set_or_map);
    }
}

} // unnamed namespace

static const std::string ALREADY_SWAPPED_PPNS_MAP_FILE(UBTools::GetTuelibPath() + "k10+_ppn_map.map");

int Main(int argc, char **argv) {
    if (argc < 2)
        Usage();

    bool store_only(false);
    if (std::strcmp(argv[1], "--store-only") == 0) {
        store_only = true;
        --argc, ++argv;
        if (argc < 2)
            Usage();
    }

    DbConnection db_connection; // ub_tools user

    CheckMySQLPermissions(&db_connection);

    std::unordered_multimap<std::string, std::string> already_processed_ppns_and_sigils;
    if (not FileUtil::Exists(ALREADY_SWAPPED_PPNS_MAP_FILE))
        FileUtil::WriteStringOrDie(ALREADY_SWAPPED_PPNS_MAP_FILE, "");
    if (not store_only)
        MapUtil::DeserialiseMap(ALREADY_SWAPPED_PPNS_MAP_FILE, &already_processed_ppns_and_sigils);

    std::vector<PPNsAndSigil> old_ppns_sigils_and_new_ppns;
    int arg_no(1);
    for (/* Intentionally empty! */; arg_no < argc; ++arg_no) {
        if (__builtin_strcmp(argv[arg_no], "--") == 0) {
            ++arg_no;
            break;
        }
        const auto marc_reader(MARC::Reader::Factory(argv[arg_no]));
        LoadMapping(marc_reader.get(), already_processed_ppns_and_sigils, &old_ppns_sigils_and_new_ppns);
    }

    std::unordered_set <std::string> title_deletion_ppns;
    for (/* Intentionally empty! */; arg_no < argc; ++arg_no) {
        const auto input(FileUtil::OpenInputFileOrDie(argv[arg_no]));
        std::unordered_set <std::string> local_deletion_ids;
        BSZUtil::ExtractDeletionIds(input.get(), &title_deletion_ppns, &local_deletion_ids);
    }

    if (old_ppns_sigils_and_new_ppns.empty() and title_deletion_ppns.empty()) {
        LOG_INFO("nothing to do!");
        return EXIT_SUCCESS;
    }
    if (old_ppns_sigils_and_new_ppns.empty())
        goto clean_up_deleted_ppns;

    if (store_only) {
        AddPPNsAndSigilsToMultiMap(old_ppns_sigils_and_new_ppns, &already_processed_ppns_and_sigils);
        MapUtil::SerialiseMap(ALREADY_SWAPPED_PPNS_MAP_FILE, already_processed_ppns_and_sigils);
        return EXIT_SUCCESS;
    }

    ProcessAllDatabases(&db_connection, old_ppns_sigils_and_new_ppns, PatchNotifiedDB, PatchTable);
    AddPPNsAndSigilsToMultiMap(old_ppns_sigils_and_new_ppns, &already_processed_ppns_and_sigils);
    MapUtil::SerialiseMap(ALREADY_SWAPPED_PPNS_MAP_FILE, already_processed_ppns_and_sigils);

clean_up_deleted_ppns:
    ProcessAllDatabases(&db_connection, title_deletion_ppns, DeleteFromNotifiedDB, DeleteFromTable);

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/*
 * The MIT License
 *
 * Copyright 2017-2018 Norwegian University of Technology
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING  FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <string>
#include <memory>
#include <vector>
#include <iostream>

#include <fmi4cpp/fmi2/fmi4cpp.hpp>
#include <fmi4cpp/tools/os_util.hpp>

using namespace std;
using namespace fmi4cpp::fmi2;
using namespace fmi4cpp::solver;

namespace logger = fmi4cpp::logger;

const double stop = 1.0;
const double microStep = 1E-3;
const double macroStep = 1.0/10;

const string fmuPath = string(getenv("TEST_FMUs"))
                       + "/2.0/me/" + getOs() +
                       "/OpenModelica/v1.11.0/FmuExportCrossCompile/FmuExportCrossCompile.fmu";

int main() {

    auto fmu = Fmu(fmuPath).asModelExchangeFmu();

    unique_ptr<ModelExchangeSolver> solver = make_solver<RK4ClassicSolver>(microStep);
    auto slave = fmu->newInstance(solver);

    slave->setupExperiment();
    slave->enterInitializationMode();
    slave->exitInitializationMode();

    double t = 0;
    double ref = 0;
    auto hVar = slave->getModelDescription()->getVariableByName("h").asReal();

    while ( ( t = slave->getSimulationTime()) <= stop) {

        if (!slave->doStep(macroStep)) {
            logger::error("Error! doStep returned with status: {}", to_string(slave->getLastStatus()));
            break;
        }

        if (!hVar.read(*slave, ref)) {
            logger::error("Error! readReal returned with status: {}", to_string(slave->getLastStatus()));
            break;
        }

        logger::info("t={}, h={}",  t, ref);
    }

    slave->terminate();

}<commit_msg>refactor<commit_after>/*
 * The MIT License
 *
 * Copyright 2017-2018 Norwegian University of Technology
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING  FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <string>
#include <memory>
#include <vector>
#include <iostream>

#include <fmi4cpp/fmi2/fmi4cpp.hpp>
#include <fmi4cpp/tools/os_util.hpp>

using namespace std;
using namespace fmi4cpp::fmi2;
using namespace fmi4cpp::solver;

namespace logger = fmi4cpp::logger;

const double stop = 1.0;
const double microStep = 1E-3;
const double macroStep = 1.0/10;

const string fmuPath = string(getenv("TEST_FMUs"))
                       + "/2.0/me/" + getOs() +
                       "/OpenModelica/v1.11.0/FmuExportCrossCompile/FmuExportCrossCompile.fmu";

int main() {

    auto fmu = Fmu(fmuPath).asModelExchangeFmu();

    auto solver = make_solver<RK4ClassicSolver>(microStep);
    auto slave = fmu->newInstance(solver);

    slave->setupExperiment();
    slave->enterInitializationMode();
    slave->exitInitializationMode();

    double t = 0;
    double ref = 0;
    auto hVar = slave->getModelDescription()->getVariableByName("h").asReal();

    while ( ( t = slave->getSimulationTime()) <= stop) {

        if (!slave->doStep(macroStep)) {
            logger::error("Error! doStep returned with status: {}", to_string(slave->getLastStatus()));
            break;
        }

        if (!hVar.read(*slave, ref)) {
            logger::error("Error! readReal returned with status: {}", to_string(slave->getLastStatus()));
            break;
        }

        logger::info("t={}, h={}",  t, ref);
    }

    slave->terminate();

}<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*
 * Modified by Cloudius Systems
 * Copyright 2015 Cloudius Systems
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <memory>
#include <vector>
#include <algorithm>
#include <unordered_map>
#include <boost/range/adaptor/map.hpp>

#include <core/future.hh>
#include <core/sharded.hh>

#include "commitlog.hh"
#include "commitlog_replayer.hh"
#include "database.hh"
#include "sstables/sstables.hh"
#include "db/system_keyspace.hh"
#include "db/serializer.hh"
#include "cql3/query_processor.hh"
#include "log.hh"

static logging::logger logger("commitlog_replayer");

class db::commitlog_replayer::impl {
public:
    impl(seastar::sharded<cql3::query_processor>& db);

    future<> init();

    struct stats {
        uint64_t invalid_mutations = 0;
        uint64_t skipped_mutations = 0;
        uint64_t applied_mutations = 0;
    };

    future<> process(stats*, temporary_buffer<char> buf, replay_position rp);
    future<stats> recover(sstring file);

    typedef std::unordered_map<utils::UUID, replay_position> rp_map;
    typedef std::unordered_map<unsigned, rp_map> shard_rpm_map;
    typedef std::unordered_map<unsigned, replay_position> shard_rp_map;

    seastar::sharded<cql3::query_processor>&
        _qp;
    shard_rpm_map
        _rpm;
    shard_rp_map
        _min_pos;
};

db::commitlog_replayer::impl::impl(seastar::sharded<cql3::query_processor>& qp)
    : _qp(qp)
{}

future<> db::commitlog_replayer::impl::init() {
    return _qp.map_reduce([this](shard_rpm_map map) {
        for (auto& p1 : map) {
            for (auto& p2 : p1.second) {
                auto& pp = _rpm[p1.first][p2.first];
                pp = std::max(pp, p2.second);

                auto& min = _min_pos[p1.first];
                min = (min == replay_position()) ? p2.second : std::min(p2.second, min);
            }
        }
    }, [this](cql3::query_processor& qp) {
        return do_with(shard_rpm_map{}, [this, &qp](shard_rpm_map& map) {
            return parallel_for_each(qp.db().local().get_column_families(), [&map, &qp](auto& cfp) {
                auto uuid = cfp.first;
                for (auto& sst : *cfp.second->get_sstables() | boost::adaptors::map_values) {
                    try {
                        auto p = sst->get_stats_metadata().position;
                        logger.trace("sstable {} -> rp {}", sst->get_filename(), p);
                        if (p != replay_position()) {
                            auto& pp = map[p.shard_id()][uuid];
                            pp = std::max(pp, p);
                        }
                    } catch (...) {
                        logger.warn("Could not read sstable metadata {}", std::current_exception());
                    }
                }
                // TODO: this is not correct. Truncation does not fully take sharding into consideration
                return db::system_keyspace::get_truncated_position(uuid).then([&map, uuid](auto truncated_rp) {
                    if (truncated_rp != replay_position()) {
                        auto& pp = map[engine().cpu_id()][uuid];
                        pp = std::max(pp, truncated_rp);
                    }
                });
            }).then([&map] {
                return make_ready_future<shard_rpm_map>(map);
            });
        });
    }).finally([this] {
        for (auto&p : _min_pos) {
            logger.debug("minimum position for shard {}: {}", p.first, p.second);
        }
        for (auto&p1 : _rpm) {
            for (auto& p2 : p1.second) {
                logger.debug("replay position for shard/uuid {}/{}: {}", p1.first, p2.first, p2.second);
            }
        }
    });
}

future<db::commitlog_replayer::impl::stats>
db::commitlog_replayer::impl::recover(sstring file) {
    logger.info("Replaying {}", file);

    replay_position rp{commitlog::descriptor(file)};
    auto gp = _min_pos[rp.shard_id()];

    if (rp.id < gp.id) {
        logger.debug("skipping replay of fully-flushed {}", file);
        return make_ready_future<stats>();
    }
    position_type p = 0;
    if (rp.id == gp.id) {
        p = gp.pos;
    }

    auto s = make_lw_shared<stats>();

    return db::commitlog::read_log_file(file,
            std::bind(&impl::process, this, s.get(), std::placeholders::_1,
                    std::placeholders::_2), p).then([](auto s) {
        auto f = s.done();
        return f.finally([s = std::move(s)] {});
    }).then([s] {
        return make_ready_future<stats>(*s);
    });
}

future<> db::commitlog_replayer::impl::process(stats* s, temporary_buffer<char> buf, replay_position rp) {
    auto shard = rp.shard_id();
    if (rp < _min_pos[shard]) {
        logger.trace("entry {} is less than global min position. skipping", rp);
        s->skipped_mutations++;
        return make_ready_future<>();
    }

    try {

        frozen_mutation fm(bytes(reinterpret_cast<const int8_t *>(buf.get()), buf.size()));

        auto uuid = fm.column_family_id();
        auto& map = _rpm[shard];
        auto i = map.find(uuid);
        if (i != map.end() && rp <= i->second) {
            logger.trace("entry {} at {} is younger than recorded replay position {}. skipping", fm.column_family_id(), rp, i->second);
            s->skipped_mutations++;
            return make_ready_future<>();
        }

        auto shard = _qp.local().db().local().shard_of(fm);
        return _qp.local().db().invoke_on(shard, [fm = std::move(fm), rp, shard, s] (database& db) -> future<> {
            // TODO: might need better verification that the deserialized mutation
            // is schema compatible. My guess is that just applying the mutation
            // will not do this.
            auto& cf = db.find_column_family(fm.column_family_id());

            if (logger.is_enabled(logging::log_level::debug)) {
                logger.debug("replaying at {} {}:{} at {}", fm.column_family_id(),
                        cf.schema()->ks_name(), cf.schema()->cf_name(), rp);
            }
            // Removed forwarding "new" RP. Instead give none/empty.
            // This is what origin does, and it should be fine.
            // The end result should be that once sstables are flushed out
            // their "replay_position" attribute will be empty, which is
            // lower than anything the new session will produce.
            cf.apply(fm);
            s->applied_mutations++;
            return make_ready_future<>();
        }).handle_exception([s](auto ep) {
            s->invalid_mutations++;
            // TODO: write mutation to file like origin.
            logger.warn("error replaying: {}", ep);
        });
    } catch (no_such_column_family&) {
        // No such CF now? Origin just ignores this.
    } catch (...) {
        s->invalid_mutations++;
        // TODO: write mutation to file like origin.
        logger.warn("error replaying: {}", std::current_exception());
    }

    return make_ready_future<>();
}

db::commitlog_replayer::commitlog_replayer(seastar::sharded<cql3::query_processor>& qp)
    : _impl(std::make_unique<impl>(qp))
{}

db::commitlog_replayer::commitlog_replayer(commitlog_replayer&& r)
    : _impl(std::move(r._impl))
{}

db::commitlog_replayer::~commitlog_replayer()
{}

future<db::commitlog_replayer> db::commitlog_replayer::create_replayer(seastar::sharded<cql3::query_processor>& qp) {
    return do_with(commitlog_replayer(qp), [](auto&& rp) {
        auto f = rp._impl->init();
        return f.then([rp = std::move(rp)]() mutable {
            return make_ready_future<commitlog_replayer>(std::move(rp));
        });
    });
}

future<> db::commitlog_replayer::recover(std::vector<sstring> files) {
    logger.info("Replaying {}", files);

    return parallel_for_each(files, [this](auto f) {
        return this->recover(f).handle_exception([f](auto ep) {
            logger.error("Error recovering {}: {}", f, ep);
            std::rethrow_exception(ep);
        });
    });
}

future<> db::commitlog_replayer::recover(sstring file) {
    return _impl->recover(file).then([file](impl::stats stats) {
        logger.info("Log replay of {} complete, {} replayed mutations ({} invalid, {} skipped)"
                , file
                , stats.applied_mutations
                , stats.invalid_mutations
                , stats.skipped_mutations
                );
    });
}

<commit_msg>commitlog_replayer: Acquire truncation RP:s per replayed shard<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*
 * Modified by Cloudius Systems
 * Copyright 2015 Cloudius Systems
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <memory>
#include <vector>
#include <algorithm>
#include <unordered_map>
#include <boost/range/adaptor/map.hpp>

#include <core/future.hh>
#include <core/sharded.hh>

#include "commitlog.hh"
#include "commitlog_replayer.hh"
#include "database.hh"
#include "sstables/sstables.hh"
#include "db/system_keyspace.hh"
#include "db/serializer.hh"
#include "cql3/query_processor.hh"
#include "log.hh"

static logging::logger logger("commitlog_replayer");

class db::commitlog_replayer::impl {
public:
    impl(seastar::sharded<cql3::query_processor>& db);

    future<> init();

    struct stats {
        uint64_t invalid_mutations = 0;
        uint64_t skipped_mutations = 0;
        uint64_t applied_mutations = 0;
    };

    future<> process(stats*, temporary_buffer<char> buf, replay_position rp);
    future<stats> recover(sstring file);

    typedef std::unordered_map<utils::UUID, replay_position> rp_map;
    typedef std::unordered_map<unsigned, rp_map> shard_rpm_map;
    typedef std::unordered_map<unsigned, replay_position> shard_rp_map;

    seastar::sharded<cql3::query_processor>&
        _qp;
    shard_rpm_map
        _rpm;
    shard_rp_map
        _min_pos;
};

db::commitlog_replayer::impl::impl(seastar::sharded<cql3::query_processor>& qp)
    : _qp(qp)
{}

future<> db::commitlog_replayer::impl::init() {
    return _qp.map_reduce([this](shard_rpm_map map) {
        for (auto& p1 : map) {
            for (auto& p2 : p1.second) {
                auto& pp = _rpm[p1.first][p2.first];
                pp = std::max(pp, p2.second);

                auto& min = _min_pos[p1.first];
                min = (min == replay_position()) ? p2.second : std::min(p2.second, min);
            }
        }
    }, [this](cql3::query_processor& qp) {
        return do_with(shard_rpm_map{}, [this, &qp](shard_rpm_map& map) {
            return parallel_for_each(qp.db().local().get_column_families(), [&map, &qp](auto& cfp) {
                auto uuid = cfp.first;
                for (auto& sst : *cfp.second->get_sstables() | boost::adaptors::map_values) {
                    try {
                        auto p = sst->get_stats_metadata().position;
                        logger.trace("sstable {} -> rp {}", sst->get_filename(), p);
                        if (p != replay_position()) {
                            auto& pp = map[p.shard_id()][uuid];
                            pp = std::max(pp, p);
                        }
                    } catch (...) {
                        logger.warn("Could not read sstable metadata {}", std::current_exception());
                    }
                }
                // We do this on each cpu, for each CF, which technically is a little wasteful, but the values are
                // cached, this is only startup, and it makes the code easier.
                // Get all truncation records for the CF and initialize max rps if
                // present. Cannot do this on demand, as there may be no sstables to
                // mark the CF as "needed".
                return db::system_keyspace::get_truncated_position(uuid).then([&map, &uuid](std::vector<db::replay_position> tpps) {
                    for (auto& p : tpps) {
                        logger.trace("CF {} truncated at {}", uuid, p);
                        auto& pp = map[p.shard_id()][uuid];
                        pp = std::max(pp, p);
                    }
                });
            }).then([&map] {
                return make_ready_future<shard_rpm_map>(map);
            });
        });
    }).finally([this] {
        for (auto&p : _min_pos) {
            logger.debug("minimum position for shard {}: {}", p.first, p.second);
        }
        for (auto&p1 : _rpm) {
            for (auto& p2 : p1.second) {
                logger.debug("replay position for shard/uuid {}/{}: {}", p1.first, p2.first, p2.second);
            }
        }
    });
}

future<db::commitlog_replayer::impl::stats>
db::commitlog_replayer::impl::recover(sstring file) {
    logger.info("Replaying {}", file);

    replay_position rp{commitlog::descriptor(file)};
    auto gp = _min_pos[rp.shard_id()];

    if (rp.id < gp.id) {
        logger.debug("skipping replay of fully-flushed {}", file);
        return make_ready_future<stats>();
    }
    position_type p = 0;
    if (rp.id == gp.id) {
        p = gp.pos;
    }

    auto s = make_lw_shared<stats>();

    return db::commitlog::read_log_file(file,
            std::bind(&impl::process, this, s.get(), std::placeholders::_1,
                    std::placeholders::_2), p).then([](auto s) {
        auto f = s.done();
        return f.finally([s = std::move(s)] {});
    }).then([s] {
        return make_ready_future<stats>(*s);
    });
}

future<> db::commitlog_replayer::impl::process(stats* s, temporary_buffer<char> buf, replay_position rp) {
    auto shard = rp.shard_id();
    if (rp < _min_pos[shard]) {
        logger.trace("entry {} is less than global min position. skipping", rp);
        s->skipped_mutations++;
        return make_ready_future<>();
    }

    try {

        frozen_mutation fm(bytes(reinterpret_cast<const int8_t *>(buf.get()), buf.size()));

        auto uuid = fm.column_family_id();
        auto& map = _rpm[shard];
        auto i = map.find(uuid);
        if (i != map.end() && rp <= i->second) {
            logger.trace("entry {} at {} is younger than recorded replay position {}. skipping", fm.column_family_id(), rp, i->second);
            s->skipped_mutations++;
            return make_ready_future<>();
        }

        auto shard = _qp.local().db().local().shard_of(fm);
        return _qp.local().db().invoke_on(shard, [fm = std::move(fm), rp, shard, s] (database& db) -> future<> {
            // TODO: might need better verification that the deserialized mutation
            // is schema compatible. My guess is that just applying the mutation
            // will not do this.
            auto& cf = db.find_column_family(fm.column_family_id());

            if (logger.is_enabled(logging::log_level::debug)) {
                logger.debug("replaying at {} {}:{} at {}", fm.column_family_id(),
                        cf.schema()->ks_name(), cf.schema()->cf_name(), rp);
            }
            // Removed forwarding "new" RP. Instead give none/empty.
            // This is what origin does, and it should be fine.
            // The end result should be that once sstables are flushed out
            // their "replay_position" attribute will be empty, which is
            // lower than anything the new session will produce.
            cf.apply(fm);
            s->applied_mutations++;
            return make_ready_future<>();
        }).handle_exception([s](auto ep) {
            s->invalid_mutations++;
            // TODO: write mutation to file like origin.
            logger.warn("error replaying: {}", ep);
        });
    } catch (no_such_column_family&) {
        // No such CF now? Origin just ignores this.
    } catch (...) {
        s->invalid_mutations++;
        // TODO: write mutation to file like origin.
        logger.warn("error replaying: {}", std::current_exception());
    }

    return make_ready_future<>();
}

db::commitlog_replayer::commitlog_replayer(seastar::sharded<cql3::query_processor>& qp)
    : _impl(std::make_unique<impl>(qp))
{}

db::commitlog_replayer::commitlog_replayer(commitlog_replayer&& r)
    : _impl(std::move(r._impl))
{}

db::commitlog_replayer::~commitlog_replayer()
{}

future<db::commitlog_replayer> db::commitlog_replayer::create_replayer(seastar::sharded<cql3::query_processor>& qp) {
    return do_with(commitlog_replayer(qp), [](auto&& rp) {
        auto f = rp._impl->init();
        return f.then([rp = std::move(rp)]() mutable {
            return make_ready_future<commitlog_replayer>(std::move(rp));
        });
    });
}

future<> db::commitlog_replayer::recover(std::vector<sstring> files) {
    logger.info("Replaying {}", files);

    return parallel_for_each(files, [this](auto f) {
        return this->recover(f).handle_exception([f](auto ep) {
            logger.error("Error recovering {}: {}", f, ep);
            std::rethrow_exception(ep);
        });
    });
}

future<> db::commitlog_replayer::recover(sstring file) {
    return _impl->recover(file).then([file](impl::stats stats) {
        logger.info("Log replay of {} complete, {} replayed mutations ({} invalid, {} skipped)"
                , file
                , stats.applied_mutations
                , stats.invalid_mutations
                , stats.skipped_mutations
                );
    });
}

<|endoftext|>


<commit_before>/******************************************************************************
 * This file is part of the Gluon Development Platform
 * Copyright (C) 2010 Arjen Hiemstra <ahiemstra@heimr.nl>
 * Copyright (C) 2010 Keith Rusler <xzekecomax@gmail.com>
 * Copyright (C) 2011 Laszlo Papp <djszapi@archlinux.us>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "projectselectiondialog.h"

#include "newprojectdialogpage.h"
#include "recentprojectsdialogpage.h"
#include "openprojectdialogpage.h"

#include <KDE/KLocalizedString>
#include <KDE/KConfig>

using namespace GluonCreator;

class ProjectSelectionDialog::ProjectSelectionDialogPrivate
{
    public:
        explicit ProjectSelectionDialogPrivate( ProjectSelectionDialog* qq )
            : q( qq )
        {
            pages.clear();
        }

        void okClicked()
        {
            if( pages.key( q->currentPage() ) == ProjectSelectionDialog::NewProjectPage )
            {
                NewProjectDialogPage* page = static_cast<NewProjectDialogPage*>( q->currentPage() );
                if( page )
                    fileName = page->createProject();
            }

            if( pages.key( q->currentPage() ) == ProjectSelectionDialog::RecentProjectPage )
            {
                RecentProjectsDialogPage* page = static_cast<RecentProjectsDialogPage*>( q->currentPage() );
                if( page )
                    fileName = page->selectedItem();
            }

        }

        void projectRequested( const QString& project )
        {
            fileName = project;
            q->accept();
        }
    public:
        QHash<ProjectPage, KPageWidgetItem*> pages;
        QString fileName;
    private:
        ProjectSelectionDialog* q;
};

ProjectSelectionDialog::ProjectSelectionDialog( QWidget* parent, Qt::WFlags flags )
    : KPageDialog( parent, flags ),
      d( new ProjectSelectionDialogPrivate( this ) )
{
    setFaceType( List );
    setButtons( Ok | Close );

    NewProjectDialogPage *npdp = new NewProjectDialogPage;
    addPage( npdp, NewProjectPage );
    addPage( new RecentProjectsDialogPage, RecentProjectPage );
    addPage( new OpenProjectDialogPage, OpenProjectPage );

    restoreDialogSize( KGlobal::config()->group( "ProjectSelectionDialog" ) );

    connect( npdp, SIGNAL( validationFinished( bool ) ), SLOT( enableButtonOk( bool ) ) );
    connect( this, SIGNAL( okClicked() ), SLOT( okClicked() ) );
}

ProjectSelectionDialog::~ProjectSelectionDialog()
{
    KConfigGroup group = KGlobal::config()->group( "ProjectSelectionDialog" );
    saveDialogSize( group );
    delete d;
}

void ProjectSelectionDialog::addPage( KPageWidgetItem* item, ProjectSelectionDialog::ProjectPage page )
{
    DEBUG_FUNC_NAME
    switch( page )
    {
        case NewProjectPage:
            DEBUG_TEXT( "New" );
            break;
        case OpenProjectPage:
        {
            DEBUG_TEXT( "Open" );
            connect( item, SIGNAL( projectRequested( QString ) ),
                     SLOT( projectRequested( QString ) ) );
            break;
        }
        case RecentProjectPage:
            DEBUG_TEXT( "Recent" );
            connect( item, SIGNAL( projectRequested( QString ) ),
                     SLOT( projectRequested( QString ) ) );
            break;
        default:
            DEBUG_TEXT( "Unknown Project Page" );
            break;
    };

    d->pages.insert( page, item );
    KPageDialog::addPage( item );
}

QString ProjectSelectionDialog::fileName() const
{
    return d->fileName;
}

void ProjectSelectionDialog::setPage( ProjectSelectionDialog::ProjectPage page )
{
    setCurrentPage( d->pages[page] );
}

#include "projectselectiondialog.moc"
<commit_msg>Put back one header because that is neccesary there.<commit_after>/******************************************************************************
 * This file is part of the Gluon Development Platform
 * Copyright (C) 2010 Arjen Hiemstra <ahiemstra@heimr.nl>
 * Copyright (C) 2010 Keith Rusler <xzekecomax@gmail.com>
 * Copyright (C) 2011 Laszlo Papp <djszapi@archlinux.us>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "projectselectiondialog.h"

#include "newprojectdialogpage.h"
#include "recentprojectsdialogpage.h"
#include "openprojectdialogpage.h"

#include "core/debughelper.h"

#include <KDE/KLocalizedString>
#include <KDE/KConfig>

using namespace GluonCreator;

class ProjectSelectionDialog::ProjectSelectionDialogPrivate
{
    public:
        explicit ProjectSelectionDialogPrivate( ProjectSelectionDialog* qq )
            : q( qq )
        {
            pages.clear();
        }

        void okClicked()
        {
            if( pages.key( q->currentPage() ) == ProjectSelectionDialog::NewProjectPage )
            {
                NewProjectDialogPage* page = static_cast<NewProjectDialogPage*>( q->currentPage() );
                if( page )
                    fileName = page->createProject();
            }

            if( pages.key( q->currentPage() ) == ProjectSelectionDialog::RecentProjectPage )
            {
                RecentProjectsDialogPage* page = static_cast<RecentProjectsDialogPage*>( q->currentPage() );
                if( page )
                    fileName = page->selectedItem();
            }

        }

        void projectRequested( const QString& project )
        {
            fileName = project;
            q->accept();
        }
    public:
        QHash<ProjectPage, KPageWidgetItem*> pages;
        QString fileName;
    private:
        ProjectSelectionDialog* q;
};

ProjectSelectionDialog::ProjectSelectionDialog( QWidget* parent, Qt::WFlags flags )
    : KPageDialog( parent, flags ),
      d( new ProjectSelectionDialogPrivate( this ) )
{
    setFaceType( List );
    setButtons( Ok | Close );

    NewProjectDialogPage *npdp = new NewProjectDialogPage;
    addPage( npdp, NewProjectPage );
    addPage( new RecentProjectsDialogPage, RecentProjectPage );
    addPage( new OpenProjectDialogPage, OpenProjectPage );

    restoreDialogSize( KGlobal::config()->group( "ProjectSelectionDialog" ) );

    connect( npdp, SIGNAL( validationFinished( bool ) ), SLOT( enableButtonOk( bool ) ) );
    connect( this, SIGNAL( okClicked() ), SLOT( okClicked() ) );
}

ProjectSelectionDialog::~ProjectSelectionDialog()
{
    KConfigGroup group = KGlobal::config()->group( "ProjectSelectionDialog" );
    saveDialogSize( group );
    delete d;
}

void ProjectSelectionDialog::addPage( KPageWidgetItem* item, ProjectSelectionDialog::ProjectPage page )
{
    DEBUG_FUNC_NAME
    switch( page )
    {
        case NewProjectPage:
            DEBUG_TEXT( "New" );
            break;
        case OpenProjectPage:
        {
            DEBUG_TEXT( "Open" );
            connect( item, SIGNAL( projectRequested( QString ) ),
                     SLOT( projectRequested( QString ) ) );
            break;
        }
        case RecentProjectPage:
            DEBUG_TEXT( "Recent" );
            connect( item, SIGNAL( projectRequested( QString ) ),
                     SLOT( projectRequested( QString ) ) );
            break;
        default:
            DEBUG_TEXT( "Unknown Project Page" );
            break;
    };

    d->pages.insert( page, item );
    KPageDialog::addPage( item );
}

QString ProjectSelectionDialog::fileName() const
{
    return d->fileName;
}

void ProjectSelectionDialog::setPage( ProjectSelectionDialog::ProjectPage page )
{
    setCurrentPage( d->pages[page] );
}

#include "projectselectiondialog.moc"
<|endoftext|>


<commit_before>#include <cstdint>
#include <cstdio>
#include <iostream>
#include <defer.h>
#include <cstring>
#include "utils.h"
#include "models.h"
#include "fileMapping.h"
#include "../assimp/include/assimp/Importer.hpp"
#include "../assimp/include/assimp/postprocess.h"
#include "../assimp/include/assimp/scene.h"

#pragma pack(push, 1)

struct EaxmodHeader {
  char signature[7];
  unsigned char version;
  uint16_t headerSize;
  uint32_t verticesDataSize;
  uint32_t indicesDataSize;
  unsigned char indexSize;
};

#pragma pack(pop)

static const char eaxmodSignature[] = "EAXMOD";
static const char eaxmodVersion = 1;

bool checkFileSizeAndHeader(const char* fname, const EaxmodHeader * header, unsigned int fileSize) {
  if(fileSize < sizeof(EaxmodHeader)) {
    std::cout << "modelLoad - file is too small, fname = " << fname << std::endl;
    return false;
  }

  if(strncmp(header->signature, eaxmodSignature, sizeof(eaxmodSignature)) != 0) {
    std::cout << "modelLoad - invalid signature, fname = " << fname << std::endl;
    return false;
  }

  if(header->version > eaxmodVersion) {
    std::cout << "modelLoad - unsupported version " << header->version << ", fname = " << fname << std::endl;
    return false;
  }

  if(sizeof(eaxmodSignature) > header->headerSize) {
    std::cout << "modelLoad - invalid header size, actual: " << header->headerSize << ", expected at least: " << sizeof(eaxmodSignature) << ", fname = " << fname << std::endl;
    return false;
  }

  uint32_t expectedSize = header->headerSize + header->verticesDataSize + header->indicesDataSize;
  if(fileSize != expectedSize) {
    std::cout << "modelLoad - invalid size, actual: " << fileSize << ", expected: " << expectedSize << ", fname = " << fname << std::endl;
    return false;
  }

  return true;
}

bool modelSave(const char *fname, const void *verticesData, size_t verticesDataSize, const void *indicesData,
               size_t indicesDataSize, unsigned char indexSize) {
  if(indexSize != 1 && indexSize != 2 && indexSize != 4) {
    std::cout << "modelSave - invalid index size " << indexSize << std::endl;
    return false;
  }

  FILE* fd = fopen(fname, "wb");
  if(fd == nullptr) {
    std::cout << "modelSave - failed to open file, fname = " << fname << std::endl;
    return false;
  }
  defer(fclose(fd));

  EaxmodHeader header;
  strcpy(&(header.signature[0]), eaxmodSignature);
  header.version = eaxmodVersion;
  header.headerSize = sizeof(header);
  header.verticesDataSize = (uint32_t)verticesDataSize;
  header.indicesDataSize = (uint32_t)indicesDataSize;
  header.indexSize = indexSize;

  if(fwrite(&header, sizeof(header), 1, fd) != 1) {
    std::cout << "modelSave - failed to write header, fname = " << fname << std::endl;
    return false;
  }

  if(fwrite(verticesData, verticesDataSize, 1, fd) != 1) {
    std::cout << "modelSave - failed to write verticesData, fname = " << fname << std::endl;
    return false;
  }

  if(fwrite(indicesData, indicesDataSize, 1, fd) != 1) {
    std::cout << "modelSave - failed to write indicesData, fname = " << fname << std::endl;
    return false;
  }

  return true;
}

bool modelLoad(const char *fname, GLuint modelVAO, GLuint modelVBO, GLuint indicesVBO, GLsizei* outIndicesNumber, GLenum* outIndicesType) {
  *outIndicesNumber = 0;
  *outIndicesType = GL_UNSIGNED_BYTE;
  unsigned char indexSize = 1; // default, for v0 format

  FileMapping* mapping = fileMappingCreate(fname);
  if(mapping == nullptr) return false;
  defer(fileMappingClose(mapping));

  unsigned char* dataPtr = fileMappingGetPointer(mapping);
  unsigned int dataSize = fileMappingGetSize(mapping);

  EaxmodHeader * header = (EaxmodHeader *)dataPtr;
  if(!checkFileSizeAndHeader(fname, header, dataSize)) return false;

  if(header->version > 0) {
    indexSize = header->indexSize;
    if(indexSize == 1) {
      *outIndicesType = GL_UNSIGNED_BYTE;
    } else if(indexSize == 2) {
      *outIndicesType = GL_UNSIGNED_SHORT;
    } else if(indexSize == 4) {
      *outIndicesType = GL_UNSIGNED_INT;
    } else {
      std::cout << "modelLoad - unsupported indexSize: " << indexSize << std::endl;
      return false;
    }
  }

  *outIndicesNumber = header->indicesDataSize / indexSize;

  unsigned char* verticesPtr = dataPtr + header->headerSize;
  unsigned char* indicesPtr = verticesPtr + header->verticesDataSize;

  glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, indicesVBO);
  glBufferData(GL_ELEMENT_ARRAY_BUFFER, header->indicesDataSize, indicesPtr, GL_STATIC_DRAW);

  glBindVertexArray(modelVAO);
  glEnableVertexAttribArray(0);
  glEnableVertexAttribArray(1);

  glBindBuffer(GL_ARRAY_BUFFER, modelVBO);
  glBufferData(GL_ARRAY_BUFFER, header->verticesDataSize, verticesPtr, GL_STATIC_DRAW);

  glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5*sizeof(GLfloat), nullptr);
  glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5*sizeof(GLfloat), (const void*)(3*sizeof(GLfloat)));

  return true;
}

GLfloat* importedModelCreate(const char* fname, unsigned int meshNumber, size_t* outVerticesBufferSize, unsigned int* outVerticesNumber) { // TODO: optimize + indices
  *outVerticesBufferSize = 0;
  *outVerticesNumber = 0;
  Assimp::Importer importer;
  const aiScene* scene = importer.ReadFile(fname, aiProcess_CalcTangentSpace | aiProcess_Triangulate | aiProcess_JoinIdenticalVertices | aiProcess_SortByPType);

  if(scene == nullptr) {
    std::cerr << "Failed to load model " << fname << std::endl;
    return nullptr;
  }

  if(scene->mNumMeshes <= meshNumber) {
    std::cerr << "There is no mesh #" << meshNumber << " in model (" << scene->mNumMeshes << " only)" << fname << std::endl;
    return nullptr;
  }

  aiMesh* mesh = scene->mMeshes[meshNumber];
  unsigned int facesNum = mesh->mNumFaces;
//  unsigned int verticesNum = mesh->mNumVertices;

  *outVerticesNumber = facesNum*3;

  if(mesh->mTextureCoords == nullptr) {
    std::cerr << "mesh->mTextureCoords == nullptr, fname = " << fname << std::endl;
    return nullptr;
  }

  if(mesh->mTextureCoords[0] == nullptr) {
    std::cerr << "mesh->mTextureCoords[0] == nullptr, fname = " << fname << std::endl;
    return nullptr;
  }

  *outVerticesBufferSize = facesNum*sizeof(GLfloat)* 5 /* coordinates per vertex */ * 3 /* 3 vertices per face */;
  GLfloat* verticesBuffer = (GLfloat*)malloc(*outVerticesBufferSize);

  unsigned int verticesBufferIndex = 0;

  for(unsigned int i = 0; i < facesNum; ++i) {
    const aiFace& face = mesh->mFaces[i];
    if(face.mNumIndices != 3) {
      std::cerr << "face.numIndices = " << face.mNumIndices << " (3 expected), i = " << i << ", fname = " << fname << std::endl;
      free(verticesBuffer);
      return nullptr;
    }

    for(unsigned int j = 0; j < face.mNumIndices; ++j) {
      unsigned int index = face.mIndices[j];
      aiVector3D pos = mesh->mVertices[index];
      aiVector3D uv = mesh->mTextureCoords[0][index];
//      aiVector3D normal = mesh->mNormals[index];

      verticesBuffer[verticesBufferIndex++] = pos.x;
      verticesBuffer[verticesBufferIndex++] = pos.y;
      verticesBuffer[verticesBufferIndex++] = pos.z;
      verticesBuffer[verticesBufferIndex++] = uv.x;
      verticesBuffer[verticesBufferIndex++] = 1.0f - uv.y;
    }
  }

  return verticesBuffer;
}

bool importedModelSave(const char* fname, GLfloat* verticesBuffer, unsigned int verticesNumber) {
  std::vector<unsigned int> indices;
  unsigned int usedIndices = 0;
  unsigned const int offsetStepSize = 5; // 3 coordinates + UV

  const GLfloat eps = 0.00001f;

  for(unsigned int vtx = 0; vtx < verticesNumber; ++vtx) {
    GLfloat currentX = verticesBuffer[vtx*offsetStepSize+0];
    GLfloat currentY = verticesBuffer[vtx*offsetStepSize+1];
    GLfloat currentZ = verticesBuffer[vtx*offsetStepSize+2];
    GLfloat currentU = verticesBuffer[vtx*offsetStepSize+3];
    GLfloat currentV = verticesBuffer[vtx*offsetStepSize+4];

    unsigned int foundIndex = 0;
    bool indexFound = false;
    for(unsigned int idx = 0; !indexFound && idx < verticesNumber; ++idx) {
      GLfloat idxX = verticesBuffer[idx * offsetStepSize + 0];
      GLfloat idxY = verticesBuffer[idx * offsetStepSize + 1];
      GLfloat idxZ = verticesBuffer[idx * offsetStepSize + 2];
      GLfloat idxU = verticesBuffer[idx * offsetStepSize + 3];
      GLfloat idxV = verticesBuffer[idx * offsetStepSize + 4];

      if((fabs(currentX - idxX) < eps) && (fabs(currentY - idxY) < eps) && (fabs(currentZ - idxZ) < eps) &&
            (fabs(currentU - idxU) < eps) && (fabs(currentV - idxV) < eps)) {
        foundIndex = idx;
        indexFound = true;
      }
    }

//    std::cout << "vtx = " << vtx << ", idx = " << foundIndex << ", indexFound = " << indexFound << std::endl;

    usedIndices = std::max(usedIndices, foundIndex + 1);
    indices.push_back(foundIndex);
  }

//  indices.data()

  std::cout << "importedModelSave - fname = " << fname << ", verticesNumber = " << verticesNumber << ", usedIndices = " << usedIndices << std::endl;

//  return modelSave(fname, verticesBuffer, )
  return true;
}

void importedModelFree(GLfloat* model) {
  free(model);
}
<commit_msg>importedModelSave - work in progress...<commit_after>#include <cstdint>
#include <cstdio>
#include <iostream>
#include <defer.h>
#include <cstring>
#include "utils.h"
#include "models.h"
#include "fileMapping.h"
#include "../assimp/include/assimp/Importer.hpp"
#include "../assimp/include/assimp/postprocess.h"
#include "../assimp/include/assimp/scene.h"

#pragma pack(push, 1)

struct EaxmodHeader {
  char signature[7];
  unsigned char version;
  uint16_t headerSize;
  uint32_t verticesDataSize;
  uint32_t indicesDataSize;
  unsigned char indexSize;
};

#pragma pack(pop)

static const char eaxmodSignature[] = "EAXMOD";
static const char eaxmodVersion = 1;

bool checkFileSizeAndHeader(const char* fname, const EaxmodHeader * header, unsigned int fileSize) {
  if(fileSize < sizeof(EaxmodHeader)) {
    std::cout << "modelLoad - file is too small, fname = " << fname << std::endl;
    return false;
  }

  if(strncmp(header->signature, eaxmodSignature, sizeof(eaxmodSignature)) != 0) {
    std::cout << "modelLoad - invalid signature, fname = " << fname << std::endl;
    return false;
  }

  if(header->version > eaxmodVersion) {
    std::cout << "modelLoad - unsupported version " << header->version << ", fname = " << fname << std::endl;
    return false;
  }

  if(sizeof(eaxmodSignature) > header->headerSize) {
    std::cout << "modelLoad - invalid header size, actual: " << header->headerSize << ", expected at least: " << sizeof(eaxmodSignature) << ", fname = " << fname << std::endl;
    return false;
  }

  uint32_t expectedSize = header->headerSize + header->verticesDataSize + header->indicesDataSize;
  if(fileSize != expectedSize) {
    std::cout << "modelLoad - invalid size, actual: " << fileSize << ", expected: " << expectedSize << ", fname = " << fname << std::endl;
    return false;
  }

  return true;
}

bool modelSave(const char *fname, const void *verticesData, size_t verticesDataSize, const void *indicesData,
               size_t indicesDataSize, unsigned char indexSize) {
  if(indexSize != 1 && indexSize != 2 && indexSize != 4) {
    std::cout << "modelSave - invalid index size " << indexSize << std::endl;
    return false;
  }

  FILE* fd = fopen(fname, "wb");
  if(fd == nullptr) {
    std::cout << "modelSave - failed to open file, fname = " << fname << std::endl;
    return false;
  }
  defer(fclose(fd));

  EaxmodHeader header;
  strcpy(&(header.signature[0]), eaxmodSignature);
  header.version = eaxmodVersion;
  header.headerSize = sizeof(header);
  header.verticesDataSize = (uint32_t)verticesDataSize;
  header.indicesDataSize = (uint32_t)indicesDataSize;
  header.indexSize = indexSize;

  if(fwrite(&header, sizeof(header), 1, fd) != 1) {
    std::cout << "modelSave - failed to write header, fname = " << fname << std::endl;
    return false;
  }

  if(fwrite(verticesData, verticesDataSize, 1, fd) != 1) {
    std::cout << "modelSave - failed to write verticesData, fname = " << fname << std::endl;
    return false;
  }

  if(fwrite(indicesData, indicesDataSize, 1, fd) != 1) {
    std::cout << "modelSave - failed to write indicesData, fname = " << fname << std::endl;
    return false;
  }

  return true;
}

bool modelLoad(const char *fname, GLuint modelVAO, GLuint modelVBO, GLuint indicesVBO, GLsizei* outIndicesNumber, GLenum* outIndicesType) {
  *outIndicesNumber = 0;
  *outIndicesType = GL_UNSIGNED_BYTE;
  unsigned char indexSize = 1; // default, for v0 format

  FileMapping* mapping = fileMappingCreate(fname);
  if(mapping == nullptr) return false;
  defer(fileMappingClose(mapping));

  unsigned char* dataPtr = fileMappingGetPointer(mapping);
  unsigned int dataSize = fileMappingGetSize(mapping);

  EaxmodHeader * header = (EaxmodHeader *)dataPtr;
  if(!checkFileSizeAndHeader(fname, header, dataSize)) return false;

  if(header->version > 0) {
    indexSize = header->indexSize;
    if(indexSize == 1) {
      *outIndicesType = GL_UNSIGNED_BYTE;
    } else if(indexSize == 2) {
      *outIndicesType = GL_UNSIGNED_SHORT;
    } else if(indexSize == 4) {
      *outIndicesType = GL_UNSIGNED_INT;
    } else {
      std::cout << "modelLoad - unsupported indexSize: " << indexSize << std::endl;
      return false;
    }
  }

  *outIndicesNumber = header->indicesDataSize / indexSize;

  unsigned char* verticesPtr = dataPtr + header->headerSize;
  unsigned char* indicesPtr = verticesPtr + header->verticesDataSize;

  glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, indicesVBO);
  glBufferData(GL_ELEMENT_ARRAY_BUFFER, header->indicesDataSize, indicesPtr, GL_STATIC_DRAW);

  glBindVertexArray(modelVAO);
  glEnableVertexAttribArray(0);
  glEnableVertexAttribArray(1);

  glBindBuffer(GL_ARRAY_BUFFER, modelVBO);
  glBufferData(GL_ARRAY_BUFFER, header->verticesDataSize, verticesPtr, GL_STATIC_DRAW);

  glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5*sizeof(GLfloat), nullptr);
  glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5*sizeof(GLfloat), (const void*)(3*sizeof(GLfloat)));

  return true;
}

GLfloat* importedModelCreate(const char* fname, unsigned int meshNumber, size_t* outVerticesBufferSize, unsigned int* outVerticesNumber) { // TODO: optimize + indices
  *outVerticesBufferSize = 0;
  *outVerticesNumber = 0;
  Assimp::Importer importer;
  const aiScene* scene = importer.ReadFile(fname, aiProcess_CalcTangentSpace | aiProcess_Triangulate | aiProcess_JoinIdenticalVertices | aiProcess_SortByPType);

  if(scene == nullptr) {
    std::cerr << "Failed to load model " << fname << std::endl;
    return nullptr;
  }

  if(scene->mNumMeshes <= meshNumber) {
    std::cerr << "There is no mesh #" << meshNumber << " in model (" << scene->mNumMeshes << " only)" << fname << std::endl;
    return nullptr;
  }

  aiMesh* mesh = scene->mMeshes[meshNumber];
  unsigned int facesNum = mesh->mNumFaces;
//  unsigned int verticesNum = mesh->mNumVertices;

  *outVerticesNumber = facesNum*3;

  if(mesh->mTextureCoords == nullptr) {
    std::cerr << "mesh->mTextureCoords == nullptr, fname = " << fname << std::endl;
    return nullptr;
  }

  if(mesh->mTextureCoords[0] == nullptr) {
    std::cerr << "mesh->mTextureCoords[0] == nullptr, fname = " << fname << std::endl;
    return nullptr;
  }

  *outVerticesBufferSize = facesNum*sizeof(GLfloat)* 5 /* coordinates per vertex */ * 3 /* 3 vertices per face */;
  GLfloat* verticesBuffer = (GLfloat*)malloc(*outVerticesBufferSize);

  unsigned int verticesBufferIndex = 0;

  for(unsigned int i = 0; i < facesNum; ++i) {
    const aiFace& face = mesh->mFaces[i];
    if(face.mNumIndices != 3) {
      std::cerr << "face.numIndices = " << face.mNumIndices << " (3 expected), i = " << i << ", fname = " << fname << std::endl;
      free(verticesBuffer);
      return nullptr;
    }

    for(unsigned int j = 0; j < face.mNumIndices; ++j) {
      unsigned int index = face.mIndices[j];
      aiVector3D pos = mesh->mVertices[index];
      aiVector3D uv = mesh->mTextureCoords[0][index];
//      aiVector3D normal = mesh->mNormals[index];

      verticesBuffer[verticesBufferIndex++] = pos.x;
      verticesBuffer[verticesBufferIndex++] = pos.y;
      verticesBuffer[verticesBufferIndex++] = pos.z;
      verticesBuffer[verticesBufferIndex++] = uv.x;
      verticesBuffer[verticesBufferIndex++] = 1.0f - uv.y;
    }
  }

  return verticesBuffer;
}

bool importedModelSave(const char* fname, GLfloat* verticesBuffer, unsigned int verticesNumber) {
  std::vector<GLfloat> vertices;
  std::vector<unsigned int> indices;
  unsigned int usedIndices = 0;
  unsigned const int offsetStepSize = 5; // 3 coordinates + UV

  const GLfloat eps = 0.00001f;

  for(unsigned int vtx = 0; vtx < verticesNumber; ++vtx) {
    GLfloat currentX = verticesBuffer[vtx*offsetStepSize+0];
    GLfloat currentY = verticesBuffer[vtx*offsetStepSize+1];
    GLfloat currentZ = verticesBuffer[vtx*offsetStepSize+2];
    GLfloat currentU = verticesBuffer[vtx*offsetStepSize+3];
    GLfloat currentV = verticesBuffer[vtx*offsetStepSize+4];

    unsigned int foundIndex = 0;
    bool indexFound = false;
    for(unsigned int idx = 0; !indexFound && idx < usedIndices; ++idx) {
      GLfloat idxX = vertices[idx * offsetStepSize + 0];
      GLfloat idxY = vertices[idx * offsetStepSize + 1];
      GLfloat idxZ = vertices[idx * offsetStepSize + 2];
      GLfloat idxU = vertices[idx * offsetStepSize + 3];
      GLfloat idxV = vertices[idx * offsetStepSize + 4];

      if((fabs(currentX - idxX) < eps) && (fabs(currentY - idxY) < eps) && (fabs(currentZ - idxZ) < eps) &&
            (fabs(currentU - idxU) < eps) && (fabs(currentV - idxV) < eps)) {
        foundIndex = idx;
        indexFound = true;
      }
    }

    if(!indexFound) {
      vertices.push_back(currentX);
      vertices.push_back(currentY);
      vertices.push_back(currentZ);
      vertices.push_back(currentU);
      vertices.push_back(currentV);

      foundIndex = usedIndices;
      usedIndices++;
    }

//    std::cout << "vtx = " << vtx << ", foundIndex = " << foundIndex << ", indexFound = " << indexFound << std::endl;
    indices.push_back(foundIndex);
  }

//  indices.data()

  std::cout << "importedModelSave - fname = " << fname << ", verticesNumber = " << verticesNumber << ", usedIndices = " << usedIndices << std::endl;

//  return modelSave(fname, verticesBuffer, )
  return true;
}

void importedModelFree(GLfloat* model) {
  free(model);
}
<|endoftext|>