<commit_before>/** \file    util.cc
 *  \brief   Implementation of various utility functions.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2015-2020 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#include "util.h"
#include <iostream>
#include <iterator>
#include <stdexcept>
#include <cctype>
#include <cerrno>
#include <cstdlib>
#include <execinfo.h>
#include <signal.h>
#include "Compiler.h"
#include "MiscUtil.h"
#include "StringUtil.h"
#include "TimeUtil.h"

// Macro to determine the number of entries in a one-dimensional array:
#define DIM(array) (sizeof(array) / sizeof(array[0]))

char *progname; // Must be set in main() with "progname = argv[0];";

const std::string Logger::FUNCTION_NAME_SEPARATOR(" --> ");

Logger::Logger()
    : fd_(STDERR_FILENO), log_process_pids_(false), log_no_decorations_(false), log_strip_call_site_(false),
      min_log_level_(LL_INFO)
{
    const char * const min_log_level(::getenv("MIN_LOG_LEVEL"));
    if (min_log_level != nullptr)
        min_log_level_ = Logger::StringToLogLevel(min_log_level);
    const char * const logger_format(::getenv("LOGGER_FORMAT"));
    if (logger_format != nullptr) {
        if (std::strstr(logger_format, "process_pids") != nullptr)
            log_process_pids_ = true;
        if (std::strstr(logger_format, "no_decorations") != nullptr)
            log_no_decorations_ = true;
        if (std::strstr(logger_format, "strip_call_site") != nullptr)
            log_strip_call_site_ = true;
    }
}

void Logger::error(const std::string &msg) {
    std::lock_guard<std::mutex> mutex_locker(mutex_);

    std::string error_message_string;
    if (errno != 0)
        error_message_string = " (last error code: " + std::string(std::strerror(errno)) + ")";

    writeString("SEVERE", msg + error_message_string);
    if (::getenv("BACKTRACE") != nullptr) {
        const bool saved_log_no_decorations(log_no_decorations_);
        log_no_decorations_ = true;
        writeString("", "Backtrace:");
        for (const auto &stack_entry : MiscUtil::GetCallStack())
            writeString("", "  " + stack_entry);
        log_no_decorations_ = saved_log_no_decorations;
    }

    std::exit(EXIT_FAILURE);
}

void Logger::warning(const std::string &msg) {
    if (min_log_level_ < LL_WARNING)
        return;

    std::lock_guard<std::mutex> mutex_locker(mutex_);
    writeString("WARN", msg);
}

void Logger::info(const std::string &msg) {
    if (min_log_level_ < LL_INFO)
        return;

    std::lock_guard<std::mutex> mutex_locker(mutex_);
    writeString("INFO", msg);
}

void Logger::debug(const std::string &msg) {
    if ((min_log_level_ < LL_DEBUG) and (MiscUtil::SafeGetEnv("UTIL_LOG_DEBUG") != "true"))
        return;

    std::lock_guard<std::mutex> mutex_locker(mutex_);
    writeString("DEBUG", msg);
}

inline Logger *LoggerInstantiator() {
    return new Logger();
}

Logger *logger(LoggerInstantiator());

Logger::LogLevel Logger::StringToLogLevel(const std::string &level_candidate) {
    if (level_candidate == "ERROR")
        return Logger::LL_ERROR;
    if (level_candidate == "WARNING")
        return Logger::LL_WARNING;
    if (level_candidate == "INFO")
        return Logger::LL_INFO;
    if (level_candidate == "DEBUG")
        return Logger::LL_DEBUG;
    LOG_ERROR("not a valid minimum log level: \"" + level_candidate + "\"! (Use ERROR, WARNING, INFO or DEBUG)");
}

std::string Logger::LogLevelToString(const LogLevel log_level) {
    if (log_level == Logger::LL_ERROR)
        return "ERROR";
    if (log_level == Logger::LL_WARNING)
        return "WARNING";
    if (log_level == Logger::LL_INFO)
        return "INFO";
    if (log_level == Logger::LL_DEBUG)
        return "DEBUG";
    LOG_ERROR("unsupported log level, we should *never* get here!");
}

void Logger::formatMessage(const std::string &level, std::string * const msg) {
    if (not log_no_decorations_) {
        *msg = TimeUtil::GetCurrentDateAndTime(TimeUtil::ISO_8601_FORMAT) + " " + level + " " + std::string(::program_invocation_name) + ": "
              + *msg;
        if (log_process_pids_)
            *msg += " (PID: " + std::to_string(::getpid()) + ")";
    }

    if (log_strip_call_site_) {
        const auto END_OF_CALL_SITE_PREFIX(msg->find(FUNCTION_NAME_SEPARATOR));
        if (END_OF_CALL_SITE_PREFIX != std::string::npos)
            *msg = msg->substr(END_OF_CALL_SITE_PREFIX + FUNCTION_NAME_SEPARATOR.length());
    }

    *msg += '\n';
}

void Logger::writeString(const std::string &level, std::string msg, const bool format_message) {
    if (format_message)
        formatMessage(level, &msg);

    if (unlikely(::write(fd_, reinterpret_cast<const void *>(msg.data()), msg.size()) == -1)) {
        const std::string error_message("in Logger::writeString(util.cc): write to file descriptor " + std::to_string(fd_)
                                        + " failed! (errno = " + std::to_string(errno) + ")");
        #pragma GCC diagnostic ignored "-Wunused-result"
        ::write(STDERR_FILENO, error_message.data(), error_message.size());
        #pragma GCC diagnostic warning "-Wunused-result"
        _exit(EXIT_FAILURE);
    }
}

DSVReader::DSVReader(const std::string &filename, const char field_separator, const char field_delimiter)
    : field_separator_(field_separator), field_delimiter_(field_delimiter), line_no_(0), filename_(filename)
{
    input_ = std::fopen(filename.c_str(), "rm");
    if (input_ == nullptr)
        throw std::runtime_error("in DSVReader::DSVReader: can't open \"" + filename + "\" for reading!");
}

DSVReader::~DSVReader() {
    if (input_ != nullptr)
        std::fclose(input_);
}

namespace {

void SkipFieldPadding(FILE * const input) {
    int ch = std::fgetc(input);
    while (isblank(ch))
        ch = std::fgetc(input);
    std::ungetc(ch, input);
}

/** \brief Remove trailing spaces and tabs from "s". */
std::string TrimBlanks(std::string * s) {
    std::string::const_reverse_iterator it(s->crbegin());
    for (/* Empty! */; it != s->crend() and std::isblank(*it); ++it)
        /* Intentionally Empty! */;
    if (it != s->crbegin())
        *s = s->substr(0, std::distance(it, s->crend()));

    return *s;
}

std::string ReadQuotedValue(FILE * const input, const char field_delimiter, const char field_separator) {
    std::string value;
    bool delimiter_seen(false);
    for (;;) {
        const int ch(std::fgetc(input));
        if (ch == EOF)
            throw std::runtime_error("unexpected EOF while reading a quoted DSV value!");
        if (ch == field_delimiter) {
            if (delimiter_seen) {
                // ignore if it's not the outermost delimiter
                const int next(std::fgetc(input));
                std::ungetc(next, input);

                if (next == field_separator or next == '\n' or next == EOF)
                    return value;
            } else
                delimiter_seen = true;
        } else {
            if (ch == '\n' or ch == EOF) {
                std::ungetc(ch, input);
                return TrimBlanks(&value);
            }
            value += static_cast<char>(ch);
        }
    }
}

std::string ReadNonQuotedValue(FILE * const input, const char field_separator) {
    std::string value;
    for (;;) {
        const int ch(std::fgetc(input));
        if (ch == EOF or ch == '\n' or ch == field_separator) {
            std::ungetc(ch, input);
            return TrimBlanks(&value);
        }
        value += static_cast<char>(ch);
    }
}

void BacktraceSignalHandler(int signal_no) {
    void *stack_return_addresses[20];
    const size_t number_of_addresses(::backtrace(stack_return_addresses, DIM(stack_return_addresses)));
    char err_msg[1024] = "Caught signal ";
    char *cp = err_msg + std::strlen(err_msg);
    if (signal_no > 10)
        *cp++ = '0' + (signal_no / 10);
    *cp++ = '0' + (signal_no % 10);
    *cp++ = '.';
    *cp++ = '\n';
    *cp = '\0';
    ssize_t unused(::write(STDERR_FILENO, err_msg, std::strlen(err_msg)));
    (void)unused;
    ::backtrace_symbols_fd(stack_return_addresses, number_of_addresses, STDERR_FILENO);
    ::_exit(EXIT_FAILURE);
}

int InstallSegvSignalHandler(void handler(int)) {
    ::signal(SIGSEGV, handler);
    return 0;
}

volatile int dummy = InstallSegvSignalHandler(BacktraceSignalHandler);

} // unnamed namespace

bool DSVReader::readLine(std::vector<std::string> * const values) {
    values->clear();
    ++line_no_;

    int ch;
    for (;;) {
        if (not values->empty()) {
            SkipFieldPadding(input_);
            ch = std::fgetc(input_);
            if (ch == EOF)
                return not values->empty();
            if (ch == '\n')
                return true;
            if (ch != field_separator_)
                throw std::runtime_error("in DSVReader::readLine: on line " + std::to_string(line_no_)
                                         + ": field separator expected, found '"
                                         + std::string(1, static_cast<char>(ch)) + "' instead!");
        }

        SkipFieldPadding(input_);
        ch = std::fgetc(input_);
        if (ch == '\n')
            return true;
        if (ch == EOF)
            return not values->empty();
        if (ch == field_separator_) {
            std::ungetc(ch, input_);
            values->emplace_back("");
        } else if (ch == field_delimiter_) {
            std::ungetc(ch, input_);
            values->emplace_back(ReadQuotedValue(input_, field_delimiter_, field_separator_));
        } else {
            std::ungetc(ch, input_);
            values->emplace_back(ReadNonQuotedValue(input_, field_separator_));
        }
    }
}

[[noreturn]] void Usage(const std::string &usage_message) {
    std::vector<std::string> lines;
    StringUtil::SplitThenTrimWhite(usage_message, '\n', &lines, /* suppress_empty_words */ false);
    auto line(lines.begin());
    if (unlikely(line == lines.cend()))
        LOG_ERROR("missing usage message!");

    std::cerr << "Usage: " << ::program_invocation_name << " [--min-log-level] " << *line << '\n';
    const std::string padding(__builtin_strlen("Usage: ") + __builtin_strlen(::program_invocation_name) + 1, ' ');
    for (++line; line != lines.cend(); ++line)
        std::cerr << padding << *line << '\n';

    std::exit(EXIT_FAILURE);
}
<commit_msg>Improved usage message.<commit_after>/** \file    util.cc
 *  \brief   Implementation of various utility functions.
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2015-2020 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#include "util.h"
#include <iostream>
#include <iterator>
#include <stdexcept>
#include <cctype>
#include <cerrno>
#include <cstdlib>
#include <execinfo.h>
#include <signal.h>
#include "Compiler.h"
#include "MiscUtil.h"
#include "StringUtil.h"
#include "TimeUtil.h"

// Macro to determine the number of entries in a one-dimensional array:
#define DIM(array) (sizeof(array) / sizeof(array[0]))

char *progname; // Must be set in main() with "progname = argv[0];";

const std::string Logger::FUNCTION_NAME_SEPARATOR(" --> ");

Logger::Logger()
    : fd_(STDERR_FILENO), log_process_pids_(false), log_no_decorations_(false), log_strip_call_site_(false),
      min_log_level_(LL_INFO)
{
    const char * const min_log_level(::getenv("MIN_LOG_LEVEL"));
    if (min_log_level != nullptr)
        min_log_level_ = Logger::StringToLogLevel(min_log_level);
    const char * const logger_format(::getenv("LOGGER_FORMAT"));
    if (logger_format != nullptr) {
        if (std::strstr(logger_format, "process_pids") != nullptr)
            log_process_pids_ = true;
        if (std::strstr(logger_format, "no_decorations") != nullptr)
            log_no_decorations_ = true;
        if (std::strstr(logger_format, "strip_call_site") != nullptr)
            log_strip_call_site_ = true;
    }
}

void Logger::error(const std::string &msg) {
    std::lock_guard<std::mutex> mutex_locker(mutex_);

    std::string error_message_string;
    if (errno != 0)
        error_message_string = " (last error code: " + std::string(std::strerror(errno)) + ")";

    writeString("SEVERE", msg + error_message_string);
    if (::getenv("BACKTRACE") != nullptr) {
        const bool saved_log_no_decorations(log_no_decorations_);
        log_no_decorations_ = true;
        writeString("", "Backtrace:");
        for (const auto &stack_entry : MiscUtil::GetCallStack())
            writeString("", "  " + stack_entry);
        log_no_decorations_ = saved_log_no_decorations;
    }

    std::exit(EXIT_FAILURE);
}

void Logger::warning(const std::string &msg) {
    if (min_log_level_ < LL_WARNING)
        return;

    std::lock_guard<std::mutex> mutex_locker(mutex_);
    writeString("WARN", msg);
}

void Logger::info(const std::string &msg) {
    if (min_log_level_ < LL_INFO)
        return;

    std::lock_guard<std::mutex> mutex_locker(mutex_);
    writeString("INFO", msg);
}

void Logger::debug(const std::string &msg) {
    if ((min_log_level_ < LL_DEBUG) and (MiscUtil::SafeGetEnv("UTIL_LOG_DEBUG") != "true"))
        return;

    std::lock_guard<std::mutex> mutex_locker(mutex_);
    writeString("DEBUG", msg);
}

inline Logger *LoggerInstantiator() {
    return new Logger();
}

Logger *logger(LoggerInstantiator());

Logger::LogLevel Logger::StringToLogLevel(const std::string &level_candidate) {
    if (level_candidate == "ERROR")
        return Logger::LL_ERROR;
    if (level_candidate == "WARNING")
        return Logger::LL_WARNING;
    if (level_candidate == "INFO")
        return Logger::LL_INFO;
    if (level_candidate == "DEBUG")
        return Logger::LL_DEBUG;
    LOG_ERROR("not a valid minimum log level: \"" + level_candidate + "\"! (Use ERROR, WARNING, INFO or DEBUG)");
}

std::string Logger::LogLevelToString(const LogLevel log_level) {
    if (log_level == Logger::LL_ERROR)
        return "ERROR";
    if (log_level == Logger::LL_WARNING)
        return "WARNING";
    if (log_level == Logger::LL_INFO)
        return "INFO";
    if (log_level == Logger::LL_DEBUG)
        return "DEBUG";
    LOG_ERROR("unsupported log level, we should *never* get here!");
}

void Logger::formatMessage(const std::string &level, std::string * const msg) {
    if (not log_no_decorations_) {
        *msg = TimeUtil::GetCurrentDateAndTime(TimeUtil::ISO_8601_FORMAT) + " " + level + " " + std::string(::program_invocation_name) + ": "
              + *msg;
        if (log_process_pids_)
            *msg += " (PID: " + std::to_string(::getpid()) + ")";
    }

    if (log_strip_call_site_) {
        const auto END_OF_CALL_SITE_PREFIX(msg->find(FUNCTION_NAME_SEPARATOR));
        if (END_OF_CALL_SITE_PREFIX != std::string::npos)
            *msg = msg->substr(END_OF_CALL_SITE_PREFIX + FUNCTION_NAME_SEPARATOR.length());
    }

    *msg += '\n';
}

void Logger::writeString(const std::string &level, std::string msg, const bool format_message) {
    if (format_message)
        formatMessage(level, &msg);

    if (unlikely(::write(fd_, reinterpret_cast<const void *>(msg.data()), msg.size()) == -1)) {
        const std::string error_message("in Logger::writeString(util.cc): write to file descriptor " + std::to_string(fd_)
                                        + " failed! (errno = " + std::to_string(errno) + ")");
        #pragma GCC diagnostic ignored "-Wunused-result"
        ::write(STDERR_FILENO, error_message.data(), error_message.size());
        #pragma GCC diagnostic warning "-Wunused-result"
        _exit(EXIT_FAILURE);
    }
}

DSVReader::DSVReader(const std::string &filename, const char field_separator, const char field_delimiter)
    : field_separator_(field_separator), field_delimiter_(field_delimiter), line_no_(0), filename_(filename)
{
    input_ = std::fopen(filename.c_str(), "rm");
    if (input_ == nullptr)
        throw std::runtime_error("in DSVReader::DSVReader: can't open \"" + filename + "\" for reading!");
}

DSVReader::~DSVReader() {
    if (input_ != nullptr)
        std::fclose(input_);
}

namespace {

void SkipFieldPadding(FILE * const input) {
    int ch = std::fgetc(input);
    while (isblank(ch))
        ch = std::fgetc(input);
    std::ungetc(ch, input);
}

/** \brief Remove trailing spaces and tabs from "s". */
std::string TrimBlanks(std::string * s) {
    std::string::const_reverse_iterator it(s->crbegin());
    for (/* Empty! */; it != s->crend() and std::isblank(*it); ++it)
        /* Intentionally Empty! */;
    if (it != s->crbegin())
        *s = s->substr(0, std::distance(it, s->crend()));

    return *s;
}

std::string ReadQuotedValue(FILE * const input, const char field_delimiter, const char field_separator) {
    std::string value;
    bool delimiter_seen(false);
    for (;;) {
        const int ch(std::fgetc(input));
        if (ch == EOF)
            throw std::runtime_error("unexpected EOF while reading a quoted DSV value!");
        if (ch == field_delimiter) {
            if (delimiter_seen) {
                // ignore if it's not the outermost delimiter
                const int next(std::fgetc(input));
                std::ungetc(next, input);

                if (next == field_separator or next == '\n' or next == EOF)
                    return value;
            } else
                delimiter_seen = true;
        } else {
            if (ch == '\n' or ch == EOF) {
                std::ungetc(ch, input);
                return TrimBlanks(&value);
            }
            value += static_cast<char>(ch);
        }
    }
}

std::string ReadNonQuotedValue(FILE * const input, const char field_separator) {
    std::string value;
    for (;;) {
        const int ch(std::fgetc(input));
        if (ch == EOF or ch == '\n' or ch == field_separator) {
            std::ungetc(ch, input);
            return TrimBlanks(&value);
        }
        value += static_cast<char>(ch);
    }
}

void BacktraceSignalHandler(int signal_no) {
    void *stack_return_addresses[20];
    const size_t number_of_addresses(::backtrace(stack_return_addresses, DIM(stack_return_addresses)));
    char err_msg[1024] = "Caught signal ";
    char *cp = err_msg + std::strlen(err_msg);
    if (signal_no > 10)
        *cp++ = '0' + (signal_no / 10);
    *cp++ = '0' + (signal_no % 10);
    *cp++ = '.';
    *cp++ = '\n';
    *cp = '\0';
    ssize_t unused(::write(STDERR_FILENO, err_msg, std::strlen(err_msg)));
    (void)unused;
    ::backtrace_symbols_fd(stack_return_addresses, number_of_addresses, STDERR_FILENO);
    ::_exit(EXIT_FAILURE);
}

int InstallSegvSignalHandler(void handler(int)) {
    ::signal(SIGSEGV, handler);
    return 0;
}

volatile int dummy = InstallSegvSignalHandler(BacktraceSignalHandler);

} // unnamed namespace

bool DSVReader::readLine(std::vector<std::string> * const values) {
    values->clear();
    ++line_no_;

    int ch;
    for (;;) {
        if (not values->empty()) {
            SkipFieldPadding(input_);
            ch = std::fgetc(input_);
            if (ch == EOF)
                return not values->empty();
            if (ch == '\n')
                return true;
            if (ch != field_separator_)
                throw std::runtime_error("in DSVReader::readLine: on line " + std::to_string(line_no_)
                                         + ": field separator expected, found '"
                                         + std::string(1, static_cast<char>(ch)) + "' instead!");
        }

        SkipFieldPadding(input_);
        ch = std::fgetc(input_);
        if (ch == '\n')
            return true;
        if (ch == EOF)
            return not values->empty();
        if (ch == field_separator_) {
            std::ungetc(ch, input_);
            values->emplace_back("");
        } else if (ch == field_delimiter_) {
            std::ungetc(ch, input_);
            values->emplace_back(ReadQuotedValue(input_, field_delimiter_, field_separator_));
        } else {
            std::ungetc(ch, input_);
            values->emplace_back(ReadNonQuotedValue(input_, field_separator_));
        }
    }
}

[[noreturn]] void Usage(const std::string &usage_message) {
    std::vector<std::string> lines;
    StringUtil::SplitThenTrimWhite(usage_message, '\n', &lines, /* suppress_empty_words */ false);
    auto line(lines.begin());
    if (unlikely(line == lines.cend()))
        LOG_ERROR("missing usage message!");

    std::cerr << "Usage: " << ::program_invocation_name << " [--min-log-level=(ERROR|WARNING|INFO|DEBUG)] " << *line << '\n';
    const std::string padding(__builtin_strlen("Usage: ") + __builtin_strlen(::program_invocation_name) + 1, ' ');
    for (++line; line != lines.cend(); ++line)
        std::cerr << padding << *line << '\n';

    std::exit(EXIT_FAILURE);
}
<|endoftext|>


<commit_before>// Copyright (c) 2009-2012 Bitcoin Developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <string>

#include "init.h" // for pwalletMain
#include "base58.h"
#include "rpcmain.h"
#include "ui_interface.h"

#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>

#define printf OutputDebugStringF

using namespace json_spirit;
using namespace std;

class CTxDump
{
public:
    CBlockIndex *pindex;
    int64 nValue;
    bool fSpent;
    CWalletTx* ptx;
    int nOut;
    CTxDump(CWalletTx* ptx = NULL, int nOut = -1)
    {
        pindex = NULL;
        nValue = 0;
        fSpent = false;
        this->ptx = ptx;
        this->nOut = nOut;
    }
};

Value importprivkey(const Array &params, bool fHelp) {

    if(fHelp || (params.size() < 1) || (params.size() > 3)) {
        string msg = "importprivkey <key> [label] [rescan]\n"
          "Adds a private <key> to your wallet in the format of RPC dumpprivkey.\n"
          "Block chain re-scanning is on (true) by default.\n";
        throw(runtime_error(msg));
    }

    string strSecret = params[0].get_str();
    string strLabel = "";
    if(params.size() > 1)
      strLabel = params[1].get_str();

    bool fRescan = true;
    if(params.size() > 2)
      fRescan = params[2].get_bool();

    CCoinSecret vchSecret;
    bool fGood = vchSecret.SetString(strSecret);

    if (!fGood) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key");

    CKey key;
    bool fCompressed;
    CSecret secret = vchSecret.GetSecret(fCompressed);
    key.SetSecret(secret, fCompressed);
    CKeyID vchAddress = key.GetPubKey().GetID();
    {
        LOCK2(cs_main, pwalletMain->cs_wallet);

        pwalletMain->MarkDirty();
        pwalletMain->SetAddressBookName(vchAddress, strLabel);

        if (!pwalletMain->AddKey(key))
            throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet");

        pwalletMain->UpdateTimeFirstKey();

        if(fRescan) {
            pwalletMain->ScanForWalletTransactions(pindexGenesisBlock, true);
            pwalletMain->ReacceptWalletTransactions();
        }
    }

    return(Value::null);
}

Value importaddress(const Array &params, bool fHelp) {

    if(fHelp || (params.size() < 1) || (params.size() > 3)) {
        string msg =  "importaddress <address> [label] [rescan]\n"
        "Adds a watch only (unspendable) address to your wallet.\n"
        "P2PKH pubkey script in hex may be specified instead of the address.\n"
        "Block chain re-scanning is off (false) by default.\n";
        throw(runtime_error(msg));
    }

    string strLabel = "";
    if(params.size() > 1)
      strLabel = params[1].get_str();

    bool fRescan = false;
    if(params.size() > 2)
      fRescan = params[2].get_bool();

    CScript script;
    CCoinAddress addr;

    if(IsHex(params[0].get_str())) {
        std::vector<uchar> vchScriptPubKey(ParseHex(params[0].get_str()));
        if(vchScriptPubKey.size() != 25)
          throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid P2PKH pubkey script"));
        script = CScript(vchScriptPubKey.begin(), vchScriptPubKey.end());
        /* Copy the public key hash */
        std::string strTemp = params[0].get_str().substr(6, 40);
        /* Insert the Base58 prefix */
        char prefix[1];
        sprintf(prefix, "%x", fTestNet ? PUBKEY_ADDRESS_TEST_PREFIX : PUBKEY_ADDRESS_PREFIX);
        strTemp.insert(0, prefix);
        /* Convert and encode */
        std::vector<uchar> vchTemp(ParseHex(strTemp));
        addr = CCoinAddress(EncodeBase58Check(vchTemp));
    } else {
        CKeyID keyID;
        addr = CCoinAddress(params[0].get_str());
        if(!addr.GetKeyID(keyID))
          throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"));
        script = GetScriptForPubKeyHash(keyID);
    }

    {
        LOCK2(cs_main, pwalletMain->cs_wallet);

        if(::IsMine(*pwalletMain, script) == MINE_SPENDABLE)
          throw(JSONRPCError(RPC_WALLET_ERROR, "The private key is already in the wallet"));

        if(pwalletMain->HaveWatchOnly(script))
          throw(JSONRPCError(RPC_WALLET_ERROR, "The address is being watched already"));

        pwalletMain->MarkDirty();

        if(addr.IsValid())
          pwalletMain->SetAddressBookName(addr.Get(), strLabel);

        if(!pwalletMain->AddWatchOnly(script))
          throw(JSONRPCError(RPC_WALLET_ERROR, "Failed to add the address to the wallet"));

        if(fRescan) {
            pwalletMain->ScanForWalletTransactions(pindexGenesisBlock, true);
            pwalletMain->ReacceptWalletTransactions();
        }

    }

    return(Value::null);
}

Value importpubkey(const Array &params, bool fHelp) {

    if(fHelp || (params.size() < 1) || (params.size() > 3)) {
        string msg =  "importpubkey <key> [label] [rescan]\n"
        "Adds a watch only (unspendable) public key in hex to your wallet.\n"
        "Block chain re-scanning is off (false) by default.\n";
        throw(runtime_error(msg));
    }

    string strLabel = "";
    if(params.size() > 1)
      strLabel = params[1].get_str();

    bool fRescan = false;
    if(params.size() > 2)
      fRescan = params[2].get_bool();

    CScript script;
    CCoinAddress addr(params[0].get_str());

    if(!IsHex(params[0].get_str()))
      throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Hex string expected for public key"));

    CPubKey pubKey(std::vector<uchar> (ParseHex(params[0].get_str())));
    if(!pubKey.IsValid())
      throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid public key"));

    CKeyID keyID = pubKey.GetID();
    addr = CCoinAddress(keyID);
    script = GetScriptForPubKeyHash(keyID);

    {
        LOCK2(cs_main, pwalletMain->cs_wallet);

        if(::IsMine(*pwalletMain, script) == MINE_SPENDABLE)
          throw(JSONRPCError(RPC_WALLET_ERROR, "The private key is already in the wallet"));

        if(pwalletMain->HaveWatchOnly(script))
          throw(JSONRPCError(RPC_WALLET_ERROR, "The public key is being watched already"));

        pwalletMain->MarkDirty();

        if(addr.IsValid())
          pwalletMain->SetAddressBookName(addr.Get(), strLabel);

        if(!pwalletMain->AddWatchOnly(script))
          throw(JSONRPCError(RPC_WALLET_ERROR, "Failed to add the public key to the wallet"));

        if(fRescan) {
            pwalletMain->ScanForWalletTransactions(pindexGenesisBlock, true);
            pwalletMain->ReacceptWalletTransactions();
        }

    }

    return(Value::null);
}

Value importwallet(const Array &params, bool fHelp) {

    if(fHelp || (params.size() != 1)) {
        string msg = "importwallet <file>\n"
          "Imports keys from a <file> generated by RPC dumpwallet.\n"
          "The file name may be specified with a directory path.";
        throw(runtime_error(msg));
    }

    EnsureWalletIsUnlocked();

    boost::filesystem::ifstream file;
    boost::filesystem::path pathImportFile = params[0].get_str().c_str();
    if(!pathImportFile.is_complete()) pathImportFile = GetDataDir(true) / pathImportFile;
    if(!boost::filesystem::exists(pathImportFile))
      throw(JSONRPCError(RPC_INVALID_PARAMETER, "The file with wallet keys doesn't exist"));
    file.open(pathImportFile, std::ios_base::in);
    if(!file.good())
      throw(JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open the file with wallet keys"));
    file.close();

    if(!ImportWallet(pwalletMain, pathImportFile.string().c_str()))
      throw(JSONRPCError(RPC_WALLET_ERROR, "Failed while importing keys to the wallet"));

    return(Value::null);
}

Value dumpprivkey(const Array &params, bool fHelp) {

    if(fHelp || (params.size() != 1)) {
        string msg = "dumpprivkey <address>\n"
          "Reveals the private key corresponding to an <address>";
        throw(runtime_error(msg));
    }

    string strAddress = params[0].get_str();
    CCoinAddress address;
    if(!address.SetString(strAddress))
      throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Phoenixcoin address"));
    CKeyID keyID;
    if(!address.GetKeyID(keyID))
      throw(JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to a key"));
    CSecret vchSecret;
    bool fCompressed;
    if(!pwalletMain->GetSecret(keyID, vchSecret, fCompressed)) {
        throw(JSONRPCError(RPC_WALLET_ERROR,
          "Private key for address " + strAddress + " is not known"));
    }
    return(CCoinSecret(vchSecret, fCompressed).ToString());
}

Value dumpwallet(const Array &params, bool fHelp) {

    if(fHelp || (params.size() != 1)) {
        string msg = "dumpwallet <file>\n"
          "Dumps all wallet keys to a <file> in a human readable format.\n"
          "The file name may be specified with a directory path.";
        throw(runtime_error(msg));
    }

    EnsureWalletIsUnlocked();

    boost::filesystem::ofstream file;
    boost::filesystem::path pathDumpFile = params[0].get_str().c_str();
    if(!pathDumpFile.is_complete()) pathDumpFile = GetDataDir(true) / pathDumpFile;
    if(boost::filesystem::exists(pathDumpFile))
      throw(JSONRPCError(RPC_INVALID_PARAMETER, "The file for wallet keys exists already"));
    file.open(pathDumpFile, std::ios_base::out);
    if(!file.good())
      throw(JSONRPCError(RPC_INVALID_PARAMETER, "Cannot create the file for wallet keys"));
    file.close();

    if(!ExportWallet(pwalletMain, pathDumpFile.string().c_str()))
      throw(JSONRPCError(RPC_WALLET_ERROR, "Failed while exporting keys from the wallet"));

    return(Value::null);
}
<commit_msg>RPC importaddress Improved<commit_after>// Copyright (c) 2009-2012 Bitcoin Developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <string>

#include "init.h" // for pwalletMain
#include "base58.h"
#include "rpcmain.h"
#include "ui_interface.h"

#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>

#define printf OutputDebugStringF

using namespace json_spirit;
using namespace std;

class CTxDump
{
public:
    CBlockIndex *pindex;
    int64 nValue;
    bool fSpent;
    CWalletTx* ptx;
    int nOut;
    CTxDump(CWalletTx* ptx = NULL, int nOut = -1)
    {
        pindex = NULL;
        nValue = 0;
        fSpent = false;
        this->ptx = ptx;
        this->nOut = nOut;
    }
};

Value importprivkey(const Array &params, bool fHelp) {

    if(fHelp || (params.size() < 1) || (params.size() > 3)) {
        string msg = "importprivkey <key> [label] [rescan]\n"
          "Adds a private <key> to your wallet in the format of RPC dumpprivkey.\n"
          "Block chain re-scanning is on (true) by default.\n";
        throw(runtime_error(msg));
    }

    string strSecret = params[0].get_str();
    string strLabel = "";
    if(params.size() > 1)
      strLabel = params[1].get_str();

    bool fRescan = true;
    if(params.size() > 2)
      fRescan = params[2].get_bool();

    CCoinSecret vchSecret;
    bool fGood = vchSecret.SetString(strSecret);

    if (!fGood) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key");

    CKey key;
    bool fCompressed;
    CSecret secret = vchSecret.GetSecret(fCompressed);
    key.SetSecret(secret, fCompressed);
    CKeyID vchAddress = key.GetPubKey().GetID();
    {
        LOCK2(cs_main, pwalletMain->cs_wallet);

        pwalletMain->MarkDirty();
        pwalletMain->SetAddressBookName(vchAddress, strLabel);

        if (!pwalletMain->AddKey(key))
            throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet");

        pwalletMain->UpdateTimeFirstKey();

        if(fRescan) {
            pwalletMain->ScanForWalletTransactions(pindexGenesisBlock, true);
            pwalletMain->ReacceptWalletTransactions();
        }
    }

    return(Value::null);
}

Value importaddress(const Array &params, bool fHelp) {

    if(fHelp || (params.size() < 1) || (params.size() > 3)) {
        string msg =  "importaddress <address> [label] [rescan]\n"
        "Adds a watch only (unspendable) P2PKH address to your wallet.\n"
        "Pubkey hash or script in hex may be specified instead of the address.\n"
        "Block chain re-scanning is off (false) by default.\n";
        throw(runtime_error(msg));
    }

    string strLabel = "";
    if(params.size() > 1)
      strLabel = params[1].get_str();

    bool fRescan = false;
    if(params.size() > 2)
      fRescan = params[2].get_bool();

    CScript script;
    CCoinAddress addr;

    if(IsHex(params[0].get_str())) {
        std::vector<uchar> vchScriptPubKey(ParseHex(params[0].get_str()));
        std::string strTemp;
        if(vchScriptPubKey.size() == 20) {
            /* 20-byte pubkey hash assumed;
             * <pubKeyHash> = RIPEMD160(SHA256(pubKey))
             * Example using OpenSSL:
             * echo -n <pubKey> | xxd -r -p | openssl dgst -sha256 -binary | openssl dgst -rmd160
             * Convert to 25-byte script:
             * OP_DUP OP_HASH160 <pubKeyHash> OP_EQUALVERIFY OP_CHECKSIG */
            strTemp = std::string(vchScriptPubKey.begin(), vchScriptPubKey.end());
            const uchar begin[] = { 0x76, 0xA9, 0x14 };
            const uchar end[] = { 0x88, 0xAC };
            vchScriptPubKey.insert(vchScriptPubKey.begin(), begin, begin + 3);
            vchScriptPubKey.insert(vchScriptPubKey.end(), end, end + 2);
        } else if(vchScriptPubKey.size() == 25) {
            /* Copy the public key hash */
            strTemp = params[0].get_str().substr(6, 40);
        } else {
            throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid pubkey hash or script"));
        }
        script = CScript(vchScriptPubKey.begin(), vchScriptPubKey.end());
        /* Insert the Base58 prefix */
        char prefix[2];
        if(fTestNet) prefix[0] = PUBKEY_ADDRESS_TEST_PREFIX;
        else prefix[0] = PUBKEY_ADDRESS_PREFIX;
        prefix[1] = 0x00;
        strTemp.insert(0, prefix);
        /* Convert and encode */
        std::vector<uchar> vchTemp(ParseHex(strTemp));
        addr = CCoinAddress(EncodeBase58Check(vchTemp));
    } else {
        CKeyID keyID;
        addr = CCoinAddress(params[0].get_str());
        if(!addr.GetKeyID(keyID))
          throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"));
        script = GetScriptForPubKeyHash(keyID);
    }

    {
        LOCK2(cs_main, pwalletMain->cs_wallet);

        if(::IsMine(*pwalletMain, script) == MINE_SPENDABLE)
          throw(JSONRPCError(RPC_WALLET_ERROR, "The private key is already in the wallet"));

        if(pwalletMain->HaveWatchOnly(script))
          throw(JSONRPCError(RPC_WALLET_ERROR, "The address is being watched already"));

        pwalletMain->MarkDirty();

        if(addr.IsValid())
          pwalletMain->SetAddressBookName(addr.Get(), strLabel);

        if(!pwalletMain->AddWatchOnly(script))
          throw(JSONRPCError(RPC_WALLET_ERROR, "Failed to add the address to the wallet"));

        if(fRescan) {
            pwalletMain->ScanForWalletTransactions(pindexGenesisBlock, true);
            pwalletMain->ReacceptWalletTransactions();
        }

    }

    return(Value::null);
}

Value importpubkey(const Array &params, bool fHelp) {

    if(fHelp || (params.size() < 1) || (params.size() > 3)) {
        string msg =  "importpubkey <key> [label] [rescan]\n"
        "Adds a watch only (unspendable) public key in hex to your wallet.\n"
        "Block chain re-scanning is off (false) by default.\n";
        throw(runtime_error(msg));
    }

    string strLabel = "";
    if(params.size() > 1)
      strLabel = params[1].get_str();

    bool fRescan = false;
    if(params.size() > 2)
      fRescan = params[2].get_bool();

    CScript script;
    CCoinAddress addr(params[0].get_str());

    if(!IsHex(params[0].get_str()))
      throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Hex string expected for public key"));

    CPubKey pubKey(std::vector<uchar> (ParseHex(params[0].get_str())));
    if(!pubKey.IsValid())
      throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid public key"));

    CKeyID keyID = pubKey.GetID();
    addr = CCoinAddress(keyID);
    script = GetScriptForPubKeyHash(keyID);

    {
        LOCK2(cs_main, pwalletMain->cs_wallet);

        if(::IsMine(*pwalletMain, script) == MINE_SPENDABLE)
          throw(JSONRPCError(RPC_WALLET_ERROR, "The private key is already in the wallet"));

        if(pwalletMain->HaveWatchOnly(script))
          throw(JSONRPCError(RPC_WALLET_ERROR, "The public key is being watched already"));

        pwalletMain->MarkDirty();

        if(addr.IsValid())
          pwalletMain->SetAddressBookName(addr.Get(), strLabel);

        if(!pwalletMain->AddWatchOnly(script))
          throw(JSONRPCError(RPC_WALLET_ERROR, "Failed to add the public key to the wallet"));

        if(fRescan) {
            pwalletMain->ScanForWalletTransactions(pindexGenesisBlock, true);
            pwalletMain->ReacceptWalletTransactions();
        }

    }

    return(Value::null);
}

Value importwallet(const Array &params, bool fHelp) {

    if(fHelp || (params.size() != 1)) {
        string msg = "importwallet <file>\n"
          "Imports keys from a <file> generated by RPC dumpwallet.\n"
          "The file name may be specified with a directory path.";
        throw(runtime_error(msg));
    }

    EnsureWalletIsUnlocked();

    boost::filesystem::ifstream file;
    boost::filesystem::path pathImportFile = params[0].get_str().c_str();
    if(!pathImportFile.is_complete()) pathImportFile = GetDataDir(true) / pathImportFile;
    if(!boost::filesystem::exists(pathImportFile))
      throw(JSONRPCError(RPC_INVALID_PARAMETER, "The file with wallet keys doesn't exist"));
    file.open(pathImportFile, std::ios_base::in);
    if(!file.good())
      throw(JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open the file with wallet keys"));
    file.close();

    if(!ImportWallet(pwalletMain, pathImportFile.string().c_str()))
      throw(JSONRPCError(RPC_WALLET_ERROR, "Failed while importing keys to the wallet"));

    return(Value::null);
}

Value dumpprivkey(const Array &params, bool fHelp) {

    if(fHelp || (params.size() != 1)) {
        string msg = "dumpprivkey <address>\n"
          "Reveals the private key corresponding to an <address>";
        throw(runtime_error(msg));
    }

    string strAddress = params[0].get_str();
    CCoinAddress address;
    if(!address.SetString(strAddress))
      throw(JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Phoenixcoin address"));
    CKeyID keyID;
    if(!address.GetKeyID(keyID))
      throw(JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to a key"));
    CSecret vchSecret;
    bool fCompressed;
    if(!pwalletMain->GetSecret(keyID, vchSecret, fCompressed)) {
        throw(JSONRPCError(RPC_WALLET_ERROR,
          "Private key for address " + strAddress + " is not known"));
    }
    return(CCoinSecret(vchSecret, fCompressed).ToString());
}

Value dumpwallet(const Array &params, bool fHelp) {

    if(fHelp || (params.size() != 1)) {
        string msg = "dumpwallet <file>\n"
          "Dumps all wallet keys to a <file> in a human readable format.\n"
          "The file name may be specified with a directory path.";
        throw(runtime_error(msg));
    }

    EnsureWalletIsUnlocked();

    boost::filesystem::ofstream file;
    boost::filesystem::path pathDumpFile = params[0].get_str().c_str();
    if(!pathDumpFile.is_complete()) pathDumpFile = GetDataDir(true) / pathDumpFile;
    if(boost::filesystem::exists(pathDumpFile))
      throw(JSONRPCError(RPC_INVALID_PARAMETER, "The file for wallet keys exists already"));
    file.open(pathDumpFile, std::ios_base::out);
    if(!file.good())
      throw(JSONRPCError(RPC_INVALID_PARAMETER, "Cannot create the file for wallet keys"));
    file.close();

    if(!ExportWallet(pwalletMain, pathDumpFile.string().c_str()))
      throw(JSONRPCError(RPC_WALLET_ERROR, "Failed while exporting keys from the wallet"));

    return(Value::null);
}
<|endoftext|>


<commit_before>#include <albert/scraper.hpp>

#include <boost/program_options.hpp>

#include <vector>
#include <iostream>

#include <supermarx/api/client.hpp>

struct cli_options
{
	std::string api_host;
	bool dry_run;
	unsigned int ratelimit;
	bool silent;
};

int read_options(cli_options& opt, int argc, char** argv)
{
	boost::program_options::options_description o_general("General options");
	o_general.add_options()
			("help,h", "display this message")
			("api,a", boost::program_options::value(&opt.api_host), "api host to send results to")
			("dry-run,d", "does not send products to api when set")
			("ratelimit,r", boost::program_options::value(&opt.ratelimit), "minimal time between each request in milliseconds (default: 5000)")
			("silent,s", "do not write status reports to cerr");

	boost::program_options::variables_map vm;
	boost::program_options::positional_options_description pos;

	boost::program_options::options_description options("Allowed options");
	options.add(o_general);

	try
	{
		boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(options).positional(pos).run(), vm);
	} catch(boost::program_options::unknown_option &e)
	{
		std::cerr << "Unknown option --" << e.get_option_name() << ", see --help." << std::endl;
		return EXIT_FAILURE;
	}

	try
	{
		boost::program_options::notify(vm);
	} catch(const boost::program_options::required_option &e)
	{
		std::cerr << "You forgot this: " << e.what() << std::endl;
		return EXIT_FAILURE;
	}

	if(vm.count("help"))
	{
		std::cout
				<< "SuperMarx scraper for the Albert Heijn inventory. [https://github.com/SuperMarx/albert]" << std::endl
				<< "Usage: ./albert [options]" << std::endl
				<< std::endl
				<< o_general;

		return EXIT_FAILURE;
	}

	if(!vm.count("api"))
		opt.api_host = "https://api.supermarx.nl";

	opt.dry_run = vm.count("dry-run");

	if(!vm.count("ratelimit"))
		opt.ratelimit = 5000;

	opt.silent = vm.count("silent");

	return EXIT_SUCCESS;
}

int main(int argc, char** argv)
{
	cli_options opt;

	int result = read_options(opt, argc, argv);
	if(result != EXIT_SUCCESS)
		return result;

	supermarx::api::client api(opt.api_host, "albert (libsupermarx-api)");

	supermarx::scraper s([&](supermarx::product const& product, supermarx::datetime retrieved_on, supermarx::confidence c, supermarx::scraper::problems_t problems) {
		if(!opt.silent)
		{
			std::cerr << "Product '" << product.name << "' [" << product.identifier << "] ";

			if(product.price == product.orig_price)
				std::cerr << product.price/100.0f << " EUR";
			else
				std::cerr << product.orig_price/100.0f << " EUR -> " << product.price/100.0f << " EUR";

			if(product.discount_amount > 1)
				std::cerr << " (at " << product.discount_amount << ')';

			std::cerr << std::endl;
		}

		if(!opt.dry_run)
			api.add_product(product, 1, retrieved_on, c, problems);
	}, opt.ratelimit);

	s.scrape();

	return EXIT_SUCCESS;
}
<commit_msg>Increased default ratelimit<commit_after>#include <albert/scraper.hpp>

#include <boost/program_options.hpp>

#include <vector>
#include <iostream>

#include <supermarx/api/client.hpp>

struct cli_options
{
	std::string api_host;
	bool dry_run;
	unsigned int ratelimit;
	bool silent;
};

int read_options(cli_options& opt, int argc, char** argv)
{
	boost::program_options::options_description o_general("General options");
	o_general.add_options()
			("help,h", "display this message")
			("api,a", boost::program_options::value(&opt.api_host), "api host to send results to")
			("dry-run,d", "does not send products to api when set")
			("ratelimit,r", boost::program_options::value(&opt.ratelimit), "minimal time between each request in milliseconds (default: 30000)")
			("silent,s", "do not write status reports to cerr");

	boost::program_options::variables_map vm;
	boost::program_options::positional_options_description pos;

	boost::program_options::options_description options("Allowed options");
	options.add(o_general);

	try
	{
		boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(options).positional(pos).run(), vm);
	} catch(boost::program_options::unknown_option &e)
	{
		std::cerr << "Unknown option --" << e.get_option_name() << ", see --help." << std::endl;
		return EXIT_FAILURE;
	}

	try
	{
		boost::program_options::notify(vm);
	} catch(const boost::program_options::required_option &e)
	{
		std::cerr << "You forgot this: " << e.what() << std::endl;
		return EXIT_FAILURE;
	}

	if(vm.count("help"))
	{
		std::cout
				<< "SuperMarx scraper for the Albert Heijn inventory. [https://github.com/SuperMarx/albert]" << std::endl
				<< "Usage: ./albert [options]" << std::endl
				<< std::endl
				<< o_general;

		return EXIT_FAILURE;
	}

	if(!vm.count("api"))
		opt.api_host = "https://api.supermarx.nl";

	opt.dry_run = vm.count("dry-run");

	if(!vm.count("ratelimit"))
		opt.ratelimit = 30000;

	opt.silent = vm.count("silent");

	return EXIT_SUCCESS;
}

int main(int argc, char** argv)
{
	cli_options opt;

	int result = read_options(opt, argc, argv);
	if(result != EXIT_SUCCESS)
		return result;

	supermarx::api::client api(opt.api_host, "albert (libsupermarx-api)");

	supermarx::scraper s([&](supermarx::product const& product, supermarx::datetime retrieved_on, supermarx::confidence c, supermarx::scraper::problems_t problems) {
		if(!opt.silent)
		{
			std::cerr << "Product '" << product.name << "' [" << product.identifier << "] ";

			if(product.price == product.orig_price)
				std::cerr << product.price/100.0f << " EUR";
			else
				std::cerr << product.orig_price/100.0f << " EUR -> " << product.price/100.0f << " EUR";

			if(product.discount_amount > 1)
				std::cerr << " (at " << product.discount_amount << ')';

			std::cerr << std::endl;
		}

		if(!opt.dry_run)
			api.add_product(product, 1, retrieved_on, c, problems);
	}, opt.ratelimit);

	s.scrape();

	return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>///////////////////////////////////////////////////////////////////////
// File:        svutil.cpp
// Description: ScrollView Utilities
// Author:      Joern Wanke
// Created:     Thu Nov 29 2007
//
// (C) Copyright 2007, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////
//
// SVUtil contains the SVSync and SVNetwork classes, which are used for
// thread/process creation & synchronization and network connection.

#include <cstdio>
#ifdef _WIN32
#include <windows.h>
struct addrinfo {
  struct sockaddr* ai_addr;
  int ai_addrlen;
  int ai_family;
  int ai_socktype;
  int ai_protocol;
};
#else
#include <arpa/inet.h>
#include <netdb.h>
#include <netinet/in.h>
#include <pthread.h>
#include <semaphore.h>
#include <csignal>
#include <sys/select.h>
#include <sys/socket.h>
#ifdef __linux__
#include <sys/prctl.h>
#endif
#include <unistd.h>
#endif

#include <cstdlib>
#include <cstring>
#include <iostream>
#include <memory>
#include <string>

// Include automatically generated configuration file if running autoconf.
#ifdef HAVE_CONFIG_H
#include "config_auto.h"
#endif

#include "svutil.h"

SVMutex::SVMutex() {
#ifdef _WIN32
  mutex_ = CreateMutex(0, FALSE, 0);
#else
  pthread_mutex_init(&mutex_, nullptr);
#endif
}

void SVMutex::Lock() {
#ifdef _WIN32
  WaitForSingleObject(mutex_, INFINITE);
#else
  pthread_mutex_lock(&mutex_);
#endif
}

void SVMutex::Unlock() {
#ifdef _WIN32
  ReleaseMutex(mutex_);
#else
  pthread_mutex_unlock(&mutex_);
#endif
}

// Create new thread.
void SVSync::StartThread(void* (*func)(void*), void* arg) {
#ifdef _WIN32
  LPTHREAD_START_ROUTINE f = (LPTHREAD_START_ROUTINE)func;
  DWORD threadid;
  HANDLE newthread = CreateThread(nullptr,        // default security attributes
                                  0,           // use default stack size
                                  f,           // thread function
                                  arg,         // argument to thread function
                                  0,           // use default creation flags
                                  &threadid);  // returns the thread identifier
#else
  pthread_t helper;
  pthread_attr_t attr;
  pthread_attr_init(&attr);
  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
  pthread_create(&helper, &attr, func, arg);
#endif
}

#ifndef GRAPHICS_DISABLED

const int kMaxMsgSize = 4096;

// Signals a thread to exit.
void SVSync::ExitThread() {
#ifdef _WIN32
  // ExitThread(0);
#else
  pthread_exit(nullptr);
#endif
}

// Starts a new process.
void SVSync::StartProcess(const char* executable, const char* args) {
  std::string proc;
  proc.append(executable);
  proc.append(" ");
  proc.append(args);
  std::cout << "Starting " << proc << std::endl;
#ifdef _WIN32
  STARTUPINFO start_info;
  PROCESS_INFORMATION proc_info;
  GetStartupInfo(&start_info);
  if (!CreateProcess(nullptr, const_cast<char*>(proc.c_str()), nullptr, nullptr, FALSE,
                CREATE_NO_WINDOW | DETACHED_PROCESS, nullptr, nullptr,
                &start_info, &proc_info))
    return;
#else
  int pid = fork();
  if (pid != 0) {   // The father process returns
  } else {
#ifdef __linux__
    // Make sure the java process terminates on exit, since its
    // broken socket detection seems to be useless.
    prctl(PR_SET_PDEATHSIG, 2, 0, 0, 0);
#endif
    char* mutable_args = strdup(args);
    int argc = 1;
    for (int i = 0; mutable_args[i]; ++i) {
      if (mutable_args[i] == ' ') {
        ++argc;
      }
    }
    std::unique_ptr<char*[]> argv(new char*[argc + 2]);
    argv[0] = strdup(executable);
    argv[1] = mutable_args;
    argc = 2;
    bool inquote = false;
    for (int i = 0; mutable_args[i]; ++i) {
      if (!inquote && mutable_args[i] == ' ') {
        mutable_args[i] = '\0';
        argv[argc++] = mutable_args + i + 1;
      } else if (mutable_args[i] == '"') {
        inquote = !inquote;
        mutable_args[i] = ' ';
      }
    }
    argv[argc] = nullptr;
    execvp(executable, argv.get());
    free(argv[0]);
    free(argv[1]);
  }
#endif
}

SVSemaphore::SVSemaphore() {
#ifdef _WIN32
  semaphore_ = CreateSemaphore(0, 0, 10, 0);
#elif defined(__APPLE__)
  char name[50];
  snprintf(name, sizeof(name), "%ld", random());
  sem_unlink(name);
  semaphore_ = sem_open(name, O_CREAT , S_IWUSR, 0);
  if (semaphore_ == SEM_FAILED) {
    perror("sem_open");
  }
#else
  sem_init(&semaphore_, 0, 0);
#endif
}

void SVSemaphore::Signal() {
#ifdef _WIN32
  ReleaseSemaphore(semaphore_, 1, nullptr);
#elif defined(__APPLE__)
  sem_post(semaphore_);
#else
  sem_post(&semaphore_);
#endif
}

void SVSemaphore::Wait() {
#ifdef _WIN32
  WaitForSingleObject(semaphore_, INFINITE);
#elif defined(__APPLE__)
  sem_wait(semaphore_);
#else
  sem_wait(&semaphore_);
#endif
}

// Place a message in the message buffer (and flush it).
void SVNetwork::Send(const char* msg) {
  mutex_send_.Lock();
  msg_buffer_out_.append(msg);
  mutex_send_.Unlock();
}

// Send the whole buffer.
void SVNetwork::Flush() {
  mutex_send_.Lock();
  while (!msg_buffer_out_.empty()) {
    int i = send(stream_, msg_buffer_out_.c_str(), msg_buffer_out_.length(), 0);
    msg_buffer_out_.erase(0, i);
  }
  mutex_send_.Unlock();
}

// Receive a message from the server.
// This will always return one line of char* (denoted by \n).
char* SVNetwork::Receive() {
  char* result = nullptr;
#if defined(_WIN32) || defined(__CYGWIN__)
  if (has_content) { result = strtok (nullptr, "\n"); }
#else
  if (buffer_ptr_ != nullptr) { result = strtok_r(nullptr, "\n", &buffer_ptr_); }
#endif

  // This means there is something left in the buffer and we return it.
  if (result != nullptr) { return result;
  // Otherwise, we read from the stream_.
  } else {
    buffer_ptr_ = nullptr;
    has_content = false;

    // The timeout length is not really important since we are looping anyway
    // until a new message is delivered.
    struct timeval tv;
    tv.tv_sec = 10;
    tv.tv_usec = 0;

    // Set the flags to return when the stream_ is ready to be read.
    fd_set readfds;
    FD_ZERO(&readfds);
    FD_SET(stream_, &readfds);

    int i = select(stream_+1, &readfds, nullptr, nullptr, &tv);

    // The stream_ died.
    if (i == 0) { return nullptr; }

    // Read the message buffer.
    i = recv(stream_, msg_buffer_in_, kMaxMsgSize, 0);

    // Server quit (0) or error (-1).
    if (i <= 0) { return nullptr; }
    msg_buffer_in_[i] = '\0';
    has_content = true;
#ifdef _WIN32
    return strtok(msg_buffer_in_, "\n");
#else
    // Setup a new string tokenizer.
    return strtok_r(msg_buffer_in_, "\n", &buffer_ptr_);
#endif
  }
}

// Close the connection to the server.
void SVNetwork::Close() {
#ifdef _WIN32
  closesocket(stream_);
#else
  close(stream_);
#endif
  // Mark stream_ as invalid.
  stream_ = -1;
}

// The program to invoke to start ScrollView
static const char* ScrollViewProg() {
#ifdef _WIN32
  const char* prog = "java -Xms512m -Xmx1024m";
#else
  const char* prog = "sh";
#endif
  return prog;
}

// The arguments to the program to invoke to start ScrollView
static std::string ScrollViewCommand(std::string scrollview_path) {
  // The following ugly ifdef is to enable the output of the java runtime
  // to be sent down a black hole on non-windows to ignore all the
  // exceptions in piccolo. Ideally piccolo would be debugged to make
  // this unnecessary.
  // Also the path has to be separated by ; on windows and : otherwise.
#ifdef _WIN32
  const char* cmd_template = "-Djava.library.path=%s -jar %s/ScrollView.jar";

#else
  const char* cmd_template =
      "-c \"trap 'kill %%1' 0 1 2 ; java "
      "-Xms1024m -Xmx2048m -jar %s/ScrollView.jar"
      " & wait\"";
#endif
  size_t cmdlen = strlen(cmd_template) + 4 * strlen(scrollview_path.c_str()) + 1;
  std::unique_ptr<char[]> cmd(new char[cmdlen]);
  const char* sv_path = scrollview_path.c_str();
  snprintf(cmd.get(), cmdlen, cmd_template, sv_path, sv_path, sv_path, sv_path);
  std::string command(cmd.get());
  return command;
}

// Platform-independent freeaddrinfo()
static void FreeAddrInfo(struct addrinfo* addr_info) {
  #if defined(__linux__)
  freeaddrinfo(addr_info);
  #else
  delete addr_info->ai_addr;
  delete addr_info;
  #endif
}

// Non-linux version of getaddrinfo()
#if !defined(__linux__)
static int GetAddrInfoNonLinux(const char* hostname, int port,
                               struct addrinfo** addr_info) {
// Get the host data depending on the OS.
  struct sockaddr_in* address;
  *addr_info = new struct addrinfo;
  memset(*addr_info, 0, sizeof(struct addrinfo));
  address = new struct sockaddr_in;
  memset(address, 0, sizeof(struct sockaddr_in));

  (*addr_info)->ai_addr = (struct sockaddr*) address;
  (*addr_info)->ai_addrlen = sizeof(struct sockaddr);
  (*addr_info)->ai_family = AF_INET;
  (*addr_info)->ai_socktype = SOCK_STREAM;

  struct hostent *name;
#ifdef _WIN32
  WSADATA wsaData;
  WSAStartup(MAKEWORD(1, 1), &wsaData);
  name = gethostbyname(hostname);
#else
  name = gethostbyname(hostname);
#endif

  if (name == nullptr) {
    FreeAddrInfo(*addr_info);
    *addr_info = nullptr;
    return -1;
  }

  // Fill in the appropriate variables to be able to connect to the server.
  address->sin_family = name->h_addrtype;
  memcpy(&address->sin_addr.s_addr, name->h_addr_list[0], name->h_length);
  address->sin_port = htons(port);
  return 0;
}
#endif

// Platform independent version of getaddrinfo()
//   Given a hostname:port, produce an addrinfo struct
static int GetAddrInfo(const char* hostname, int port,
                       struct addrinfo** address) {
#if defined(__linux__)
  char port_str[40];
  snprintf(port_str, 40, "%d", port);
  return getaddrinfo(hostname, port_str, nullptr, address);
#else
  return GetAddrInfoNonLinux(hostname, port, address);
#endif
}

// Set up a connection to a ScrollView on hostname:port.
SVNetwork::SVNetwork(const char* hostname, int port) {
  msg_buffer_in_ = new char[kMaxMsgSize + 1];
  msg_buffer_in_[0] = '\0';

  has_content = false;
  buffer_ptr_ = nullptr;

  struct addrinfo *addr_info = nullptr;

  if (GetAddrInfo(hostname, port, &addr_info) != 0) {
    std::cerr << "Error resolving name for ScrollView host "
              << std::string(hostname) << ":" << port << std::endl;
  }

  stream_ = socket(addr_info->ai_family, addr_info->ai_socktype,
                   addr_info->ai_protocol);

  if (stream_ < 0) {
    std::cerr << "Failed to open socket" << std::endl;
  } else if (connect(stream_, addr_info->ai_addr, addr_info->ai_addrlen) < 0) {
    // If server is not there, we will start a new server as local child process.
    const char* scrollview_path = getenv("SCROLLVIEW_PATH");
    if (scrollview_path == nullptr) {
#ifdef SCROLLVIEW_PATH
#define _STR(a) #a
#define _XSTR(a) _STR(a)
      scrollview_path = _XSTR(SCROLLVIEW_PATH);
#undef _XSTR
#undef _STR
#else
      scrollview_path = ".";
#endif
    }
    const char *prog = ScrollViewProg();
    std::string command = ScrollViewCommand(scrollview_path);
    SVSync::StartProcess(prog, command.c_str());

    // Wait for server to show up.
    // Note: There is no exception handling in case the server never turns up.

    Close();
    for (;;) {
      stream_ = socket(addr_info->ai_family, addr_info->ai_socktype,
                       addr_info->ai_protocol);
      if (stream_ >= 0) {
        if (connect(stream_, addr_info->ai_addr, addr_info->ai_addrlen) == 0) {
          break;
        }

        Close();

        std::cout << "ScrollView: Waiting for server...\n";
#ifdef _WIN32
        Sleep(1000);
#else
        sleep(1);
#endif
      }
    }
  }
  FreeAddrInfo(addr_info);
}

SVNetwork::~SVNetwork() {
  Close();
  delete[] msg_buffer_in_;
}

#endif  // GRAPHICS_DISABLED
<commit_msg>Fix function ScrollViewCommand<commit_after>///////////////////////////////////////////////////////////////////////
// File:        svutil.cpp
// Description: ScrollView Utilities
// Author:      Joern Wanke
// Created:     Thu Nov 29 2007
//
// (C) Copyright 2007, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////
//
// SVUtil contains the SVSync and SVNetwork classes, which are used for
// thread/process creation & synchronization and network connection.

#include <cstdio>
#ifdef _WIN32
#include <windows.h>
struct addrinfo {
  struct sockaddr* ai_addr;
  int ai_addrlen;
  int ai_family;
  int ai_socktype;
  int ai_protocol;
};
#else
#include <arpa/inet.h>
#include <netdb.h>
#include <netinet/in.h>
#include <pthread.h>
#include <semaphore.h>
#include <csignal>
#include <sys/select.h>
#include <sys/socket.h>
#ifdef __linux__
#include <sys/prctl.h>
#endif
#include <unistd.h>
#endif

#include <cstdlib>
#include <cstring>
#include <iostream>
#include <memory>
#include <string>
#include <vector>

// Include automatically generated configuration file if running autoconf.
#ifdef HAVE_CONFIG_H
#include "config_auto.h"
#endif

#include "svutil.h"

SVMutex::SVMutex() {
#ifdef _WIN32
  mutex_ = CreateMutex(0, FALSE, 0);
#else
  pthread_mutex_init(&mutex_, nullptr);
#endif
}

void SVMutex::Lock() {
#ifdef _WIN32
  WaitForSingleObject(mutex_, INFINITE);
#else
  pthread_mutex_lock(&mutex_);
#endif
}

void SVMutex::Unlock() {
#ifdef _WIN32
  ReleaseMutex(mutex_);
#else
  pthread_mutex_unlock(&mutex_);
#endif
}

// Create new thread.
void SVSync::StartThread(void* (*func)(void*), void* arg) {
#ifdef _WIN32
  LPTHREAD_START_ROUTINE f = (LPTHREAD_START_ROUTINE)func;
  DWORD threadid;
  HANDLE newthread = CreateThread(nullptr,        // default security attributes
                                  0,           // use default stack size
                                  f,           // thread function
                                  arg,         // argument to thread function
                                  0,           // use default creation flags
                                  &threadid);  // returns the thread identifier
#else
  pthread_t helper;
  pthread_attr_t attr;
  pthread_attr_init(&attr);
  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
  pthread_create(&helper, &attr, func, arg);
#endif
}

#ifndef GRAPHICS_DISABLED

const int kMaxMsgSize = 4096;

// Signals a thread to exit.
void SVSync::ExitThread() {
#ifdef _WIN32
  // ExitThread(0);
#else
  pthread_exit(nullptr);
#endif
}

// Starts a new process.
void SVSync::StartProcess(const char* executable, const char* args) {
  std::string proc;
  proc.append(executable);
  proc.append(" ");
  proc.append(args);
  std::cout << "Starting " << proc << std::endl;
#ifdef _WIN32
  STARTUPINFO start_info;
  PROCESS_INFORMATION proc_info;
  GetStartupInfo(&start_info);
  if (!CreateProcess(nullptr, const_cast<char*>(proc.c_str()), nullptr, nullptr, FALSE,
                CREATE_NO_WINDOW | DETACHED_PROCESS, nullptr, nullptr,
                &start_info, &proc_info))
    return;
#else
  int pid = fork();
  if (pid != 0) {   // The father process returns
  } else {
#ifdef __linux__
    // Make sure the java process terminates on exit, since its
    // broken socket detection seems to be useless.
    prctl(PR_SET_PDEATHSIG, 2, 0, 0, 0);
#endif
    char* mutable_args = strdup(args);
    int argc = 1;
    for (int i = 0; mutable_args[i]; ++i) {
      if (mutable_args[i] == ' ') {
        ++argc;
      }
    }
    std::unique_ptr<char*[]> argv(new char*[argc + 2]);
    argv[0] = strdup(executable);
    argv[1] = mutable_args;
    argc = 2;
    bool inquote = false;
    for (int i = 0; mutable_args[i]; ++i) {
      if (!inquote && mutable_args[i] == ' ') {
        mutable_args[i] = '\0';
        argv[argc++] = mutable_args + i + 1;
      } else if (mutable_args[i] == '"') {
        inquote = !inquote;
        mutable_args[i] = ' ';
      }
    }
    argv[argc] = nullptr;
    execvp(executable, argv.get());
    free(argv[0]);
    free(argv[1]);
  }
#endif
}

SVSemaphore::SVSemaphore() {
#ifdef _WIN32
  semaphore_ = CreateSemaphore(0, 0, 10, 0);
#elif defined(__APPLE__)
  char name[50];
  snprintf(name, sizeof(name), "%ld", random());
  sem_unlink(name);
  semaphore_ = sem_open(name, O_CREAT , S_IWUSR, 0);
  if (semaphore_ == SEM_FAILED) {
    perror("sem_open");
  }
#else
  sem_init(&semaphore_, 0, 0);
#endif
}

void SVSemaphore::Signal() {
#ifdef _WIN32
  ReleaseSemaphore(semaphore_, 1, nullptr);
#elif defined(__APPLE__)
  sem_post(semaphore_);
#else
  sem_post(&semaphore_);
#endif
}

void SVSemaphore::Wait() {
#ifdef _WIN32
  WaitForSingleObject(semaphore_, INFINITE);
#elif defined(__APPLE__)
  sem_wait(semaphore_);
#else
  sem_wait(&semaphore_);
#endif
}

// Place a message in the message buffer (and flush it).
void SVNetwork::Send(const char* msg) {
  mutex_send_.Lock();
  msg_buffer_out_.append(msg);
  mutex_send_.Unlock();
}

// Send the whole buffer.
void SVNetwork::Flush() {
  mutex_send_.Lock();
  while (!msg_buffer_out_.empty()) {
    int i = send(stream_, msg_buffer_out_.c_str(), msg_buffer_out_.length(), 0);
    msg_buffer_out_.erase(0, i);
  }
  mutex_send_.Unlock();
}

// Receive a message from the server.
// This will always return one line of char* (denoted by \n).
char* SVNetwork::Receive() {
  char* result = nullptr;
#if defined(_WIN32) || defined(__CYGWIN__)
  if (has_content) { result = strtok (nullptr, "\n"); }
#else
  if (buffer_ptr_ != nullptr) { result = strtok_r(nullptr, "\n", &buffer_ptr_); }
#endif

  // This means there is something left in the buffer and we return it.
  if (result != nullptr) { return result;
  // Otherwise, we read from the stream_.
  } else {
    buffer_ptr_ = nullptr;
    has_content = false;

    // The timeout length is not really important since we are looping anyway
    // until a new message is delivered.
    struct timeval tv;
    tv.tv_sec = 10;
    tv.tv_usec = 0;

    // Set the flags to return when the stream_ is ready to be read.
    fd_set readfds;
    FD_ZERO(&readfds);
    FD_SET(stream_, &readfds);

    int i = select(stream_+1, &readfds, nullptr, nullptr, &tv);

    // The stream_ died.
    if (i == 0) { return nullptr; }

    // Read the message buffer.
    i = recv(stream_, msg_buffer_in_, kMaxMsgSize, 0);

    // Server quit (0) or error (-1).
    if (i <= 0) { return nullptr; }
    msg_buffer_in_[i] = '\0';
    has_content = true;
#ifdef _WIN32
    return strtok(msg_buffer_in_, "\n");
#else
    // Setup a new string tokenizer.
    return strtok_r(msg_buffer_in_, "\n", &buffer_ptr_);
#endif
  }
}

// Close the connection to the server.
void SVNetwork::Close() {
#ifdef _WIN32
  closesocket(stream_);
#else
  close(stream_);
#endif
  // Mark stream_ as invalid.
  stream_ = -1;
}

// The program to invoke to start ScrollView
static const char* ScrollViewProg() {
#ifdef _WIN32
  const char* prog = "java -Xms512m -Xmx1024m";
#else
  const char* prog = "sh";
#endif
  return prog;
}

// The arguments to the program to invoke to start ScrollView
static std::string ScrollViewCommand(std::string scrollview_path) {
  // The following ugly ifdef is to enable the output of the java runtime
  // to be sent down a black hole on non-windows to ignore all the
  // exceptions in piccolo. Ideally piccolo would be debugged to make
  // this unnecessary.
  // Also the path has to be separated by ; on windows and : otherwise.
#ifdef _WIN32
  const char cmd_template[] = "-Djava.library.path=%s -jar %s/ScrollView.jar";

#else
  const char cmd_template[] =
      "-c \"trap 'kill %%1' 0 1 2 ; java "
      "-Xms1024m -Xmx2048m -jar %s/ScrollView.jar"
      " & wait\"";
#endif
  size_t cmdlen = sizeof(cmd_template) + 2 * scrollview_path.size() + 1;
  std::vector<char> cmd(cmdlen);
  const char* sv_path = scrollview_path.c_str();
#ifdef _WIN32
  snprintf(&cmd[0], cmdlen, cmd_template, sv_path, sv_path);
#else
  snprintf(&cmd[0], cmdlen, cmd_template, sv_path);
#endif
  std::string command(&cmd[0]);
  return command;
}

// Platform-independent freeaddrinfo()
static void FreeAddrInfo(struct addrinfo* addr_info) {
  #if defined(__linux__)
  freeaddrinfo(addr_info);
  #else
  delete addr_info->ai_addr;
  delete addr_info;
  #endif
}

// Non-linux version of getaddrinfo()
#if !defined(__linux__)
static int GetAddrInfoNonLinux(const char* hostname, int port,
                               struct addrinfo** addr_info) {
// Get the host data depending on the OS.
  struct sockaddr_in* address;
  *addr_info = new struct addrinfo;
  memset(*addr_info, 0, sizeof(struct addrinfo));
  address = new struct sockaddr_in;
  memset(address, 0, sizeof(struct sockaddr_in));

  (*addr_info)->ai_addr = (struct sockaddr*) address;
  (*addr_info)->ai_addrlen = sizeof(struct sockaddr);
  (*addr_info)->ai_family = AF_INET;
  (*addr_info)->ai_socktype = SOCK_STREAM;

  struct hostent *name;
#ifdef _WIN32
  WSADATA wsaData;
  WSAStartup(MAKEWORD(1, 1), &wsaData);
  name = gethostbyname(hostname);
#else
  name = gethostbyname(hostname);
#endif

  if (name == nullptr) {
    FreeAddrInfo(*addr_info);
    *addr_info = nullptr;
    return -1;
  }

  // Fill in the appropriate variables to be able to connect to the server.
  address->sin_family = name->h_addrtype;
  memcpy(&address->sin_addr.s_addr, name->h_addr_list[0], name->h_length);
  address->sin_port = htons(port);
  return 0;
}
#endif

// Platform independent version of getaddrinfo()
//   Given a hostname:port, produce an addrinfo struct
static int GetAddrInfo(const char* hostname, int port,
                       struct addrinfo** address) {
#if defined(__linux__)
  char port_str[40];
  snprintf(port_str, 40, "%d", port);
  return getaddrinfo(hostname, port_str, nullptr, address);
#else
  return GetAddrInfoNonLinux(hostname, port, address);
#endif
}

// Set up a connection to a ScrollView on hostname:port.
SVNetwork::SVNetwork(const char* hostname, int port) {
  msg_buffer_in_ = new char[kMaxMsgSize + 1];
  msg_buffer_in_[0] = '\0';

  has_content = false;
  buffer_ptr_ = nullptr;

  struct addrinfo *addr_info = nullptr;

  if (GetAddrInfo(hostname, port, &addr_info) != 0) {
    std::cerr << "Error resolving name for ScrollView host "
              << std::string(hostname) << ":" << port << std::endl;
  }

  stream_ = socket(addr_info->ai_family, addr_info->ai_socktype,
                   addr_info->ai_protocol);

  if (stream_ < 0) {
    std::cerr << "Failed to open socket" << std::endl;
  } else if (connect(stream_, addr_info->ai_addr, addr_info->ai_addrlen) < 0) {
    // If server is not there, we will start a new server as local child process.
    const char* scrollview_path = getenv("SCROLLVIEW_PATH");
    if (scrollview_path == nullptr) {
#ifdef SCROLLVIEW_PATH
#define _STR(a) #a
#define _XSTR(a) _STR(a)
      scrollview_path = _XSTR(SCROLLVIEW_PATH);
#undef _XSTR
#undef _STR
#else
      scrollview_path = ".";
#endif
    }
    const char *prog = ScrollViewProg();
    std::string command = ScrollViewCommand(scrollview_path);
    SVSync::StartProcess(prog, command.c_str());

    // Wait for server to show up.
    // Note: There is no exception handling in case the server never turns up.

    Close();
    for (;;) {
      stream_ = socket(addr_info->ai_family, addr_info->ai_socktype,
                       addr_info->ai_protocol);
      if (stream_ >= 0) {
        if (connect(stream_, addr_info->ai_addr, addr_info->ai_addrlen) == 0) {
          break;
        }

        Close();

        std::cout << "ScrollView: Waiting for server...\n";
#ifdef _WIN32
        Sleep(1000);
#else
        sleep(1);
#endif
      }
    }
  }
  FreeAddrInfo(addr_info);
}

SVNetwork::~SVNetwork() {
  Close();
  delete[] msg_buffer_in_;
}

#endif  // GRAPHICS_DISABLED
<|endoftext|>


<commit_before>#include "get_new_points.hxx"

#include "eval_weighted.hxx"
#include "poles_prefactor.hxx"
#include "power_prefactor.hxx"

#include "../sdp_solve.hxx"
#include "../ostream_set.hxx"

std::vector<El::BigFloat>
compute_optimal(const std::vector<Positive_Matrix_With_Prefactor> &matrices,
                const std::vector<El::BigFloat> &objectives,
                const std::vector<El::BigFloat> &normalization,
                const SDP_Solver_Parameters &parameters)
{
  size_t num_weights(normalization.size());

  const size_t num_blocks(matrices.size());
  std::vector<El::BigFloat> weights(num_weights, 0);
  std::vector<std::set<El::BigFloat>> points(num_blocks);
  std::vector<std::vector<El::BigFloat>> new_points(num_blocks);

  // Need to have a point at zero and infinity
  // GMP does not have a special infinity value, so we use max double.
  const El::BigFloat infinity(std::numeric_limits<double>::max());
  const El::BigFloat min_x(0), max_x(infinity);
  for(size_t block(0); block < num_blocks; ++block)
    {
      points.at(block).emplace(min_x);
      points.at(block).emplace(0.1);
      points.at(block).emplace(1);
      // for(double x(min_x); x < max_x; x *= 4)
      //   points.at(block).emplace(x);

      new_points.at(block).emplace_back(max_x);
    }

  bool has_new_points(true);

  while(has_new_points)
    {
      has_new_points = false;
      size_t num_constraints(0);
      std::vector<size_t> matrix_dimensions;
      for(size_t block(0); block != num_blocks; ++block)
        {
          for(auto &point : new_points.at(block))
            {
              points.at(block).emplace(point);
            }
          num_constraints += points.at(block).size();
          matrix_dimensions.insert(matrix_dimensions.end(),
                                   points.at(block).size(),
                                   matrices[block].polynomials.size());
          if(El::mpi::Rank() == 0)
            {
              std::cout << "points: " << block << " " << points.at(block)
                        << "\n";
            }
        }

      if(El::mpi::Rank() == 0)
        {
          std::cout << "num_constraints: " << num_constraints << "\n";
        }
      // std::cout << "matrix_dimensions: " << matrix_dimensions << "\n";

      Block_Info block_info(matrix_dimensions, parameters.procs_per_node,
                            parameters.proc_granularity, parameters.verbosity);
      El::Grid grid(block_info.mpi_comm.value);
      std::vector<El::BigFloat> prefactors;
      prefactors.reserve(num_constraints);
      std::vector<std::vector<El::BigFloat>> primal_objective_c;
      primal_objective_c.reserve(num_constraints);
      std::vector<El::Matrix<El::BigFloat>> free_var_matrix;
      free_var_matrix.reserve(num_constraints);

      // TODO: This is duplicated from sdp2input/write_output/write_output.cxx
      auto max_normalization(normalization.begin());
      for(auto n(normalization.begin()); n != normalization.end(); ++n)
        {
          if(Abs(*n) > Abs(*max_normalization))
            {
              max_normalization = n;
            }
        }
      const int64_t max_index(
        std::distance(normalization.begin(), max_normalization));

      for(size_t block(0); block != num_blocks; ++block)
        {
          for(auto &x : points.at(block))
            {
              if(x == infinity)
                {
                  prefactors.push_back(1);
                }
              else
                {
                  prefactors.push_back(
                    power_prefactor(matrices[block].damped_rational.base, x)
                    * poles_prefactor(matrices[block].damped_rational.poles,
                                      x));
                }
              auto &prefactor(prefactors.back());
              const size_t dim(matrices[block].polynomials.size());
              free_var_matrix.emplace_back(
                dim * (dim + 1) / 2,
                matrices[block].polynomials.at(0).at(0).size() - 1);
              auto &free_var(free_var_matrix.back());

              primal_objective_c.emplace_back();
              auto &primal(primal_objective_c.back());

              int64_t max_degree(0);
              for(auto &row: matrices[block].polynomials)
                for(auto &column: row)
                  for(auto &poly: column)
                    {
                      max_degree = std::max(max_degree, poly.degree());
                    }

              size_t flattened_matrix_row(0);
              for(size_t matrix_row(0); matrix_row != dim; ++matrix_row)
                for(size_t matrix_column(0); matrix_column <= matrix_row;
                    ++matrix_column)
                  {
                    if(x == infinity)
                      {
                        if(matrices[block]
                             .polynomials.at(matrix_row)
                             .at(matrix_column)
                             .at(max_index)
                             .degree()
                           < max_degree)
                          {
                            primal.push_back(0);
                          }
                        else
                          {
                            primal.push_back(matrices[block]
                                               .polynomials.at(matrix_row)
                                               .at(matrix_column)
                                               .at(max_index)
                                               .coefficients.at(max_degree)
                                             / normalization.at(max_index));
                          }
                        auto &primal_constant(primal.back());
                        for(int64_t column(0); column != free_var.Width();
                            ++column)
                          {
                            const int64_t index(
                              column + (column < max_index ? 0 : 1));
                            if(matrices[block]
                                 .polynomials.at(matrix_row)
                                 .at(matrix_column)
                                 .at(index)
                                 .degree()
                               < max_degree)
                              {
                                free_var(flattened_matrix_row, column)
                                  = primal_constant * normalization.at(index);
                              }
                            else
                              {
                                free_var(flattened_matrix_row, column)
                                  = primal_constant * normalization.at(index)
                                    - matrices[block]
                                        .polynomials.at(matrix_row)
                                        .at(matrix_column)
                                        .at(index)
                                        .coefficients.at(max_degree);
                              }
                          }
                      }
                    else
                      {
                        primal.push_back(prefactor
                                         * matrices[block]
                                             .polynomials.at(matrix_row)
                                             .at(matrix_column)
                                             .at(max_index)(x)
                                         / normalization.at(max_index));

                        auto &primal_constant(primal.back());
                        for(int64_t column(0); column != free_var.Width();
                            ++column)
                          {
                            const int64_t index(
                              column + (column < max_index ? 0 : 1));
                            free_var(flattened_matrix_row, column)
                              = primal_constant * normalization.at(index)
                                - prefactor
                                    * matrices[block]
                                        .polynomials.at(matrix_row)
                                        .at(matrix_column)
                                        .at(index)(x);
                          }
                      }
                    ++flattened_matrix_row;
                  }
            }
        }

      SDP sdp(objectives, normalization, prefactors, primal_objective_c,
              free_var_matrix, block_info, grid);

      SDP_Solver solver(parameters, block_info, grid,
                        sdp.dual_objective_b.Height());

      for(auto &block : solver.y.blocks)
        {
          if(block.GlobalCol(0) == 0)
            {
              for(int64_t row(0); row != block.LocalHeight(); ++row)
                {
                  int64_t global_row(block.GlobalRow(row));
                  const int64_t index(global_row
                                      + (global_row < max_index ? 0 : 1));
                  block.SetLocal(row, 0, weights.at(index));
                }
            }
        }

      Timers timers(parameters.verbosity >= Verbosity::debug);
      SDP_Solver_Terminate_Reason reason
        = solver.run(parameters, block_info, sdp, grid, timers);
      if(reason != SDP_Solver_Terminate_Reason::PrimalDualOptimal)
        {
          std::stringstream ss;
          ss << "Can not find solution: " << reason;
          throw std::runtime_error(ss.str());
        }

      // y is duplicated among cores, so only need to print out copy on
      // the root node.
      // THe weight at max_index is determined by the normalization condition
      // dot(norm,weights)=1
      weights.at(max_index) = 1;
      for(int64_t block_row(0); block_row != solver.y.blocks.at(0).Height();
          ++block_row)
        {
          const int64_t index(block_row + (block_row < max_index ? 0 : 1));
          weights.at(index) = solver.y.blocks.at(0).Get(block_row, 0);
          weights.at(max_index) -= weights.at(index) * normalization.at(index);
        }
      weights.at(max_index) /= normalization.at(max_index);

      if(El::mpi::Rank() == 0)
        {
          std::cout.precision(10);
          std::cout << "weight: " << weights << "\n";

          El::BigFloat optimal(0);
          for(size_t index(0); index < objectives.size(); ++index)
            {
              optimal += objectives[index] * weights[index];
            }
          std::cout << "optimal: " << optimal << "\n";
        }
      for(size_t block(0); block != num_blocks; ++block)
        {
          // 0.01 should be a small enough relative error so that we are
          // in the regime of convergence.  Then the error estimates will
          // work
          // Mesh mesh(*(points.at(block).begin()), *(points.at(block).rbegin()),
          Mesh mesh(*(points.at(block).begin()), El::BigFloat(100),
                    [&](const El::BigFloat &x) {
                      return eval_weighted(matrices[block], x, weights);
                    },
                    0.01);
          new_points.at(block) = get_new_points(mesh);
          for(auto &point : new_points.at(block))
            {
              has_new_points
                = has_new_points || (points.at(block).count(point) == 0);
            }
        }
    }
  // if(El::mpi::Rank() == 0)
  //   {
  //     std::cout << "weights: " << weights << "\n";
  //   }
  return weights;
}
<commit_msg>Move max_degree out of the coordinate loop<commit_after>#include "get_new_points.hxx"

#include "eval_weighted.hxx"
#include "poles_prefactor.hxx"
#include "power_prefactor.hxx"

#include "../sdp_solve.hxx"
#include "../ostream_set.hxx"

std::vector<El::BigFloat>
compute_optimal(const std::vector<Positive_Matrix_With_Prefactor> &matrices,
                const std::vector<El::BigFloat> &objectives,
                const std::vector<El::BigFloat> &normalization,
                const SDP_Solver_Parameters &parameters)
{
  size_t num_weights(normalization.size());

  const size_t num_blocks(matrices.size());
  std::vector<El::BigFloat> weights(num_weights, 0);
  std::vector<std::set<El::BigFloat>> points(num_blocks);
  std::vector<std::vector<El::BigFloat>> new_points(num_blocks);

  // Need to have a point at zero and infinity
  // GMP does not have a special infinity value, so we use max double.
  const El::BigFloat infinity(std::numeric_limits<double>::max());
  const El::BigFloat min_x(0), max_x(infinity);
  for(size_t block(0); block < num_blocks; ++block)
    {
      points.at(block).emplace(min_x);
      points.at(block).emplace(0.1);
      points.at(block).emplace(1);
      // for(double x(min_x); x < max_x; x *= 4)
      //   points.at(block).emplace(x);

      new_points.at(block).emplace_back(max_x);
    }

  bool has_new_points(true);

  while(has_new_points)
    {
      has_new_points = false;
      size_t num_constraints(0);
      std::vector<size_t> matrix_dimensions;
      for(size_t block(0); block != num_blocks; ++block)
        {
          for(auto &point : new_points.at(block))
            {
              points.at(block).emplace(point);
            }
          num_constraints += points.at(block).size();
          matrix_dimensions.insert(matrix_dimensions.end(),
                                   points.at(block).size(),
                                   matrices[block].polynomials.size());
          if(El::mpi::Rank() == 0)
            {
              std::cout << "points: " << block << " " << points.at(block)
                        << "\n";
            }
        }

      if(El::mpi::Rank() == 0)
        {
          std::cout << "num_constraints: " << num_constraints << "\n";
        }
      // std::cout << "matrix_dimensions: " << matrix_dimensions << "\n";

      Block_Info block_info(matrix_dimensions, parameters.procs_per_node,
                            parameters.proc_granularity, parameters.verbosity);
      El::Grid grid(block_info.mpi_comm.value);
      std::vector<El::BigFloat> prefactors;
      prefactors.reserve(num_constraints);
      std::vector<std::vector<El::BigFloat>> primal_objective_c;
      primal_objective_c.reserve(num_constraints);
      std::vector<El::Matrix<El::BigFloat>> free_var_matrix;
      free_var_matrix.reserve(num_constraints);

      // TODO: This is duplicated from sdp2input/write_output/write_output.cxx
      auto max_normalization(normalization.begin());
      for(auto n(normalization.begin()); n != normalization.end(); ++n)
        {
          if(Abs(*n) > Abs(*max_normalization))
            {
              max_normalization = n;
            }
        }
      const int64_t max_index(
        std::distance(normalization.begin(), max_normalization));

      for(size_t block(0); block != num_blocks; ++block)
        {
          const int64_t max_degree([&]() {
            int64_t result(0);
            for(auto &row : matrices[block].polynomials)
              for(auto &column : row)
                for(auto &poly : column)
                  {
                    result = std::max(result, poly.degree());
                  }
            return result;
          }());

          for(auto &x : points.at(block))
            {
              if(x == infinity)
                {
                  prefactors.push_back(1);
                }
              else
                {
                  prefactors.push_back(
                    power_prefactor(matrices[block].damped_rational.base, x)
                    * poles_prefactor(matrices[block].damped_rational.poles,
                                      x));
                }
              auto &prefactor(prefactors.back());
              const size_t dim(matrices[block].polynomials.size());
              free_var_matrix.emplace_back(
                dim * (dim + 1) / 2,
                matrices[block].polynomials.at(0).at(0).size() - 1);
              auto &free_var(free_var_matrix.back());

              primal_objective_c.emplace_back();
              auto &primal(primal_objective_c.back());

              size_t flattened_matrix_row(0);
              for(size_t matrix_row(0); matrix_row != dim; ++matrix_row)
                for(size_t matrix_column(0); matrix_column <= matrix_row;
                    ++matrix_column)
                  {
                    if(x == infinity)
                      {
                        if(matrices[block]
                             .polynomials.at(matrix_row)
                             .at(matrix_column)
                             .at(max_index)
                             .degree()
                           < max_degree)
                          {
                            primal.push_back(0);
                          }
                        else
                          {
                            primal.push_back(matrices[block]
                                               .polynomials.at(matrix_row)
                                               .at(matrix_column)
                                               .at(max_index)
                                               .coefficients.at(max_degree)
                                             / normalization.at(max_index));
                          }
                        auto &primal_constant(primal.back());
                        for(int64_t column(0); column != free_var.Width();
                            ++column)
                          {
                            const int64_t index(
                              column + (column < max_index ? 0 : 1));
                            if(matrices[block]
                                 .polynomials.at(matrix_row)
                                 .at(matrix_column)
                                 .at(index)
                                 .degree()
                               < max_degree)
                              {
                                free_var(flattened_matrix_row, column)
                                  = primal_constant * normalization.at(index);
                              }
                            else
                              {
                                free_var(flattened_matrix_row, column)
                                  = primal_constant * normalization.at(index)
                                    - matrices[block]
                                        .polynomials.at(matrix_row)
                                        .at(matrix_column)
                                        .at(index)
                                        .coefficients.at(max_degree);
                              }
                          }
                      }
                    else
                      {
                        primal.push_back(prefactor
                                         * matrices[block]
                                             .polynomials.at(matrix_row)
                                             .at(matrix_column)
                                             .at(max_index)(x)
                                         / normalization.at(max_index));

                        auto &primal_constant(primal.back());
                        for(int64_t column(0); column != free_var.Width();
                            ++column)
                          {
                            const int64_t index(
                              column + (column < max_index ? 0 : 1));
                            free_var(flattened_matrix_row, column)
                              = primal_constant * normalization.at(index)
                                - prefactor
                                    * matrices[block]
                                        .polynomials.at(matrix_row)
                                        .at(matrix_column)
                                        .at(index)(x);
                          }
                      }
                    ++flattened_matrix_row;
                  }
            }
        }

      SDP sdp(objectives, normalization, prefactors, primal_objective_c,
              free_var_matrix, block_info, grid);

      SDP_Solver solver(parameters, block_info, grid,
                        sdp.dual_objective_b.Height());

      for(auto &block : solver.y.blocks)
        {
          if(block.GlobalCol(0) == 0)
            {
              for(int64_t row(0); row != block.LocalHeight(); ++row)
                {
                  int64_t global_row(block.GlobalRow(row));
                  const int64_t index(global_row
                                      + (global_row < max_index ? 0 : 1));
                  block.SetLocal(row, 0, weights.at(index));
                }
            }
        }

      Timers timers(parameters.verbosity >= Verbosity::debug);
      SDP_Solver_Terminate_Reason reason
        = solver.run(parameters, block_info, sdp, grid, timers);
      if(reason != SDP_Solver_Terminate_Reason::PrimalDualOptimal)
        {
          std::stringstream ss;
          ss << "Can not find solution: " << reason;
          throw std::runtime_error(ss.str());
        }

      // y is duplicated among cores, so only need to print out copy on
      // the root node.
      // THe weight at max_index is determined by the normalization condition
      // dot(norm,weights)=1
      weights.at(max_index) = 1;
      for(int64_t block_row(0); block_row != solver.y.blocks.at(0).Height();
          ++block_row)
        {
          const int64_t index(block_row + (block_row < max_index ? 0 : 1));
          weights.at(index) = solver.y.blocks.at(0).Get(block_row, 0);
          weights.at(max_index) -= weights.at(index) * normalization.at(index);
        }
      weights.at(max_index) /= normalization.at(max_index);

      if(El::mpi::Rank() == 0)
        {
          std::cout.precision(10);
          std::cout << "weight: " << weights << "\n";

          El::BigFloat optimal(0);
          for(size_t index(0); index < objectives.size(); ++index)
            {
              optimal += objectives[index] * weights[index];
            }
          std::cout << "optimal: " << optimal << "\n";
        }
      for(size_t block(0); block != num_blocks; ++block)
        {
          // 0.01 should be a small enough relative error so that we are
          // in the regime of convergence.  Then the error estimates will
          // work
          // Mesh mesh(*(points.at(block).begin()), *(points.at(block).rbegin()),
          Mesh mesh(*(points.at(block).begin()), El::BigFloat(100),
                    [&](const El::BigFloat &x) {
                      return eval_weighted(matrices[block], x, weights);
                    },
                    0.01);
          new_points.at(block) = get_new_points(mesh);
          for(auto &point : new_points.at(block))
            {
              has_new_points
                = has_new_points || (points.at(block).count(point) == 0);
            }
        }
    }
  // if(El::mpi::Rank() == 0)
  //   {
  //     std::cout << "weights: " << weights << "\n";
  //   }
  return weights;
}
<|endoftext|>