<commit_before>// This file is part of the HörTech Open Master Hearing Aid (openMHA)
// Copyright © 2020 2021 HörTech gGmbH
//
// openMHA is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, version 3 of the License.
//
// openMHA is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License, version 3 for more details.
//
// You should have received a copy of the GNU Affero General Public License,
// version 3 along with openMHA.  If not, see <http://www.gnu.org/licenses/>.

#include "lsl2ac.hh"
#include <algorithm>

lsl2ac::lsl2ac_t::lsl2ac_t(algo_comm_t iac, const std::string &)
    : MHAPlugin::plugin_t<lsl2ac::cfg_t>("Receive LSL streams and copy"
                                         " them to AC variables.",iac)
{
    //Nota bene: The configuration variables are not connected to the patchbay because
    // we either lock them anyway during prepare or they are not used within the context
    // of the cfg_t class.
    insert_member(streams);
    insert_member(activate);
    insert_member(overrun_behavior);
    insert_member(nchannels);
    insert_member(buffersize);
    insert_member(chunksize);
    insert_member(nsamples);
    insert_item("available_streams", &available_streams);
    patchbay.connect(&available_streams.prereadaccess,this,&lsl2ac_t::get_all_stream_names);
}

void lsl2ac::lsl2ac_t::prepare(mhaconfig_t&)
{
    update();
    setlock(true);
}

void lsl2ac::lsl2ac_t::release()
{
    setlock(false);
}

void lsl2ac::lsl2ac_t::process()
{
    poll_config();
    if(activate.data)
        cfg->process();
}

void lsl2ac::lsl2ac_t::update(){
    if(is_prepared()){
        auto c=new cfg_t(ac, static_cast<lsl2ac::overrun_behavior>(overrun_behavior.data.get_index()),
                         buffersize.data, chunksize.data, streams.data, nchannels.data, nsamples.data);
        push_config(c);
    }
}

void lsl2ac::lsl2ac_t::setlock(bool lock_){
    overrun_behavior.setlock(lock_);
    buffersize.setlock(lock_);
    chunksize.setlock(lock_);
    streams.setlock(lock_);
    nchannels.setlock(lock_);
    nsamples.setlock(lock_);
}

void lsl2ac::lsl2ac_t::get_all_stream_names()
{
    available_streams.data.clear();
    std::vector<lsl::stream_info> streams = lsl::resolve_streams();
    available_streams.data.resize(streams.size());
    for(const auto& istream : streams)
        available_streams.data.push_back(istream.name());
}

lsl2ac::cfg_t::cfg_t(const algo_comm_t& ac_,
                     overrun_behavior ob_,
                     int bufsize_,
                     int chunksize_,
                     const std::vector<std::string>& streamnames_,
                     int nchannels_,
                     int nsamples_)
{
    for(auto& name : streamnames_) {
        //Find all streams with matching name and take the first one, throw if none found
        auto matching_streams=lsl::resolve_stream("name",name,/*minimum=*/1,/*timeout=*/1.0);
        if(!matching_streams.size())
            throw MHA_Error(__FILE__,__LINE__,"No stream with name %s found!",name.c_str());

        switch(matching_streams[0].channel_format()){
        case lsl::channel_format_t::cf_float32:
            varlist.emplace(name,std::make_unique<save_var_t<mha_real_t>>(matching_streams[0],
                                                                          ac_,
                                                                          ob_,
                                                                          MHA_AC_MHAREAL,
                                                                          bufsize_,
                                                                          chunksize_,
                                                                          nchannels_,
                                                                          nsamples_));
            break;
        case lsl::channel_format_t::cf_double64:
            varlist.emplace(name,std::make_unique<save_var_t<double>>(matching_streams[0],
                                                                      ac_,
                                                                      ob_,
                                                                      MHA_AC_DOUBLE,
                                                                      bufsize_,
                                                                      chunksize_,
                                                                      nchannels_,
                                                                      nsamples_));
            break;
            // LSL takes care of type conversion from short integers to ints
        case lsl::channel_format_t::cf_int8:
        case lsl::channel_format_t::cf_int16:
        case lsl::channel_format_t::cf_int32:
            varlist.emplace(name,std::make_unique<save_var_t<int>>(matching_streams[0],
                                                                   ac_,
                                                                   ob_,
                                                                   MHA_AC_INT,
                                                                   bufsize_,
                                                                   chunksize_,
                                                                   nchannels_,
                                                                   nsamples_));
            break;
        case lsl::channel_format_t::cf_string:
            // char arrays and marker streams both use cf_string - distinguish by type meta data
            if(matching_streams[0].type()=="Marker")
                varlist.emplace(name,std::make_unique<save_var_t<std::string>>(matching_streams[0],
                                                                        ac_,
                                                                        ob_,
                                                                        bufsize_,
                                                                        chunksize_,
                                                                        nsamples_));
            else
                varlist.emplace(name,std::make_unique<save_var_t<char>>(matching_streams[0],
                                                                        ac_,
                                                                        ob_,
                                                                        MHA_AC_CHAR,
                                                                        bufsize_,
                                                                        chunksize_,
                                                                        nchannels_,
                                                                        nsamples_));
            break;
        case lsl::channel_format_t::cf_int64:
            throw MHA_Error(__FILE__,__LINE__,"Stream %s: Channel format int64 is not supported.",matching_streams[0].name());
        case lsl::channel_format_t::cf_undefined:
            throw MHA_Error(__FILE__,__LINE__,"Stream %s:Can not store undefined channel format in AC variable",matching_streams[0].name());
        default:
            throw MHA_Error(__FILE__,__LINE__,"Stream %s:Unknown lsl channel format: %i",matching_streams[0].name(),matching_streams[0].name());
        }
    }
}

void lsl2ac::cfg_t::process(){
    for(auto& var : varlist){
        var.second->receive_frame();
    }
}

MHAPLUGIN_CALLBACKS(lsl2ac,lsl2ac::lsl2ac_t,wave,wave)
MHAPLUGIN_PROC_CALLBACK(lsl2ac,lsl2ac::lsl2ac_t,spec,spec)
MHAPLUGIN_DOCUMENTATION(
                        lsl2ac, "data-import network-communication",
                        " This plugin provides a mechanism to receive lsl streams and"
                        " convert them to ac variables."
                        " It currently only supports MHA\\_AC\\_FLOAT-type variables. Type "
                        "conversions from other"
                        " types of stream should be handled in the background."
                        "\n"
                        "This is a beta version of the plugin. It is probably real-time safe.\n"
                        " An LSL stream named NAME results in the following AC variables: NAME containing the data,"
                        " NAME\\_ts containing the time stamps, NAME\\_ts containing the offset between receiver and sender clocks,"
                        " and NAME\\_new containing the number of new samples per channel since the last process callback.\n "
                        " The size of the AC variables is configurable via the nchannels and "
                        "nsamples configuration variables."
                        " nchannels controls the stride of the AC variable and must be equal to the number of channels of the"
                        " AC variables or be left on default to accept"
                        " any number of channels of the LSL stream. nsamples corresponds to the number of samples per channel of the AC"
                        " variable. Leaving nsamples on default means that the AC variable will be resized according to the number of"
                        " samples received, up to a maximum of chunksize samples."
                        " If the size of the AC variable is fixed and there are less samples available in the LSL buffers than are needed"
                        " to fill the AC variable, the oldest samples are overwritten and the contents of the AC variable are reordered so"
                        " that the oldest samples come first."
                        " On overrun, i.e. there are more samples available than fit in the AC variable, the user can decide if all samples "
                        " but the newest"
                        " should be discarded or if the overrun should be ignored and only the oldest samples should be saved to AC,"
                        " leaving newer samples in the LSL buffers. Warning: If the overrun behavior is set to discard, the plugin pulls"
                        " new samples as long as samples are ready for pickup in the LSL buffers. If the sender is considerably faster than the receiver"
                        " this may cause the plugin to hang indefinitely."
                        " The buffer length and chunk size of the LSL inlet are configurable. For more details on the meaning of these"
                        " variables please consult the LSL documentation.\n"
                        " In the case of string type streams e.g. marker streams, only one entry is saved into the AC variable. The nchannels"
                        " configuration variable is ignored for string streams, marker streams with more than one channel are not supported."
                        " To nsamples configuration variable is used to determine the size of the character buffer used to store the received strings"
                        " Longer marker strings are cut off memory safe, they may however cause memory allocations within lsl, so the size should not be chosen"
                        " too small.\n"
                        " The configuration regarding the AC variable size and the LSL stream inlet applies plugin wide. To use per-stream"
                        " configuration this plugin must be instantiated multiple times.\n")
/*
 * Local variables:
 * c-basic-offset: 4
 * compile-command: "make"
 * End:
 */
<commit_msg>lsl2ac: Hotfix error messages<commit_after>// This file is part of the HörTech Open Master Hearing Aid (openMHA)
// Copyright © 2020 2021 HörTech gGmbH
//
// openMHA is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, version 3 of the License.
//
// openMHA is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License, version 3 for more details.
//
// You should have received a copy of the GNU Affero General Public License,
// version 3 along with openMHA.  If not, see <http://www.gnu.org/licenses/>.

#include "lsl2ac.hh"
#include <algorithm>

lsl2ac::lsl2ac_t::lsl2ac_t(algo_comm_t iac, const std::string &)
    : MHAPlugin::plugin_t<lsl2ac::cfg_t>("Receive LSL streams and copy"
                                         " them to AC variables.",iac)
{
    //Nota bene: The configuration variables are not connected to the patchbay because
    // we either lock them anyway during prepare or they are not used within the context
    // of the cfg_t class.
    insert_member(streams);
    insert_member(activate);
    insert_member(overrun_behavior);
    insert_member(nchannels);
    insert_member(buffersize);
    insert_member(chunksize);
    insert_member(nsamples);
    insert_item("available_streams", &available_streams);
    patchbay.connect(&available_streams.prereadaccess,this,&lsl2ac_t::get_all_stream_names);
}

void lsl2ac::lsl2ac_t::prepare(mhaconfig_t&)
{
    update();
    setlock(true);
}

void lsl2ac::lsl2ac_t::release()
{
    setlock(false);
}

void lsl2ac::lsl2ac_t::process()
{
    poll_config();
    if(activate.data)
        cfg->process();
}

void lsl2ac::lsl2ac_t::update(){
    if(is_prepared()){
        auto c=new cfg_t(ac, static_cast<lsl2ac::overrun_behavior>(overrun_behavior.data.get_index()),
                         buffersize.data, chunksize.data, streams.data, nchannels.data, nsamples.data);
        push_config(c);
    }
}

void lsl2ac::lsl2ac_t::setlock(bool lock_){
    overrun_behavior.setlock(lock_);
    buffersize.setlock(lock_);
    chunksize.setlock(lock_);
    streams.setlock(lock_);
    nchannels.setlock(lock_);
    nsamples.setlock(lock_);
}

void lsl2ac::lsl2ac_t::get_all_stream_names()
{
    available_streams.data.clear();
    std::vector<lsl::stream_info> streams = lsl::resolve_streams();
    available_streams.data.resize(streams.size());
    for(const auto& istream : streams)
        available_streams.data.push_back(istream.name());
}

lsl2ac::cfg_t::cfg_t(const algo_comm_t& ac_,
                     overrun_behavior ob_,
                     int bufsize_,
                     int chunksize_,
                     const std::vector<std::string>& streamnames_,
                     int nchannels_,
                     int nsamples_)
{
    for(auto& name : streamnames_) {
        //Find all streams with matching name and take the first one, throw if none found
        auto matching_streams=lsl::resolve_stream("name",name,/*minimum=*/1,/*timeout=*/1.0);
        if(!matching_streams.size())
            throw MHA_Error(__FILE__,__LINE__,"No stream with name %s found!",name.c_str());

        switch(matching_streams[0].channel_format()){
        case lsl::channel_format_t::cf_float32:
            varlist.emplace(name,std::make_unique<save_var_t<mha_real_t>>(matching_streams[0],
                                                                          ac_,
                                                                          ob_,
                                                                          MHA_AC_MHAREAL,
                                                                          bufsize_,
                                                                          chunksize_,
                                                                          nchannels_,
                                                                          nsamples_));
            break;
        case lsl::channel_format_t::cf_double64:
            varlist.emplace(name,std::make_unique<save_var_t<double>>(matching_streams[0],
                                                                      ac_,
                                                                      ob_,
                                                                      MHA_AC_DOUBLE,
                                                                      bufsize_,
                                                                      chunksize_,
                                                                      nchannels_,
                                                                      nsamples_));
            break;
            // LSL takes care of type conversion from short integers to ints
        case lsl::channel_format_t::cf_int8:
        case lsl::channel_format_t::cf_int16:
        case lsl::channel_format_t::cf_int32:
            varlist.emplace(name,std::make_unique<save_var_t<int>>(matching_streams[0],
                                                                   ac_,
                                                                   ob_,
                                                                   MHA_AC_INT,
                                                                   bufsize_,
                                                                   chunksize_,
                                                                   nchannels_,
                                                                   nsamples_));
            break;
        case lsl::channel_format_t::cf_string:
            // char arrays and marker streams both use cf_string - distinguish by type meta data
            if(matching_streams[0].type()=="Marker")
                varlist.emplace(name,std::make_unique<save_var_t<std::string>>(matching_streams[0],
                                                                        ac_,
                                                                        ob_,
                                                                        bufsize_,
                                                                        chunksize_,
                                                                        nsamples_));
            else
                varlist.emplace(name,std::make_unique<save_var_t<char>>(matching_streams[0],
                                                                        ac_,
                                                                        ob_,
                                                                        MHA_AC_CHAR,
                                                                        bufsize_,
                                                                        chunksize_,
                                                                        nchannels_,
                                                                        nsamples_));
            break;
        case lsl::channel_format_t::cf_int64:
            throw MHA_Error(__FILE__,__LINE__,"Stream %s: Channel format int64 is not supported.",matching_streams[0].name().c_str());
        case lsl::channel_format_t::cf_undefined:
                            throw MHA_Error(__FILE__,__LINE__,"Stream %s:Can not store undefined channel format in AC variable",matching_streams[0].name().c_str());
        default:
                            throw MHA_Error(__FILE__,__LINE__,"Stream %s:Unknown lsl channel format: %i",matching_streams[0].name().c_str(),matching_streams[0].channel_format());
        }
    }
}

void lsl2ac::cfg_t::process(){
    for(auto& var : varlist){
        var.second->receive_frame();
    }
}

MHAPLUGIN_CALLBACKS(lsl2ac,lsl2ac::lsl2ac_t,wave,wave)
MHAPLUGIN_PROC_CALLBACK(lsl2ac,lsl2ac::lsl2ac_t,spec,spec)
MHAPLUGIN_DOCUMENTATION(
                        lsl2ac, "data-import network-communication",
                        " This plugin provides a mechanism to receive lsl streams and"
                        " convert them to ac variables."
                        " It currently only supports MHA\\_AC\\_FLOAT-type variables. Type "
                        "conversions from other"
                        " types of stream should be handled in the background."
                        "\n"
                        "This is a beta version of the plugin. It is probably real-time safe.\n"
                        " An LSL stream named NAME results in the following AC variables: NAME containing the data,"
                        " NAME\\_ts containing the time stamps, NAME\\_ts containing the offset between receiver and sender clocks,"
                        " and NAME\\_new containing the number of new samples per channel since the last process callback.\n "
                        " The size of the AC variables is configurable via the nchannels and "
                        "nsamples configuration variables."
                        " nchannels controls the stride of the AC variable and must be equal to the number of channels of the"
                        " AC variables or be left on default to accept"
                        " any number of channels of the LSL stream. nsamples corresponds to the number of samples per channel of the AC"
                        " variable. Leaving nsamples on default means that the AC variable will be resized according to the number of"
                        " samples received, up to a maximum of chunksize samples."
                        " If the size of the AC variable is fixed and there are less samples available in the LSL buffers than are needed"
                        " to fill the AC variable, the oldest samples are overwritten and the contents of the AC variable are reordered so"
                        " that the oldest samples come first."
                        " On overrun, i.e. there are more samples available than fit in the AC variable, the user can decide if all samples "
                        " but the newest"
                        " should be discarded or if the overrun should be ignored and only the oldest samples should be saved to AC,"
                        " leaving newer samples in the LSL buffers. Warning: If the overrun behavior is set to discard, the plugin pulls"
                        " new samples as long as samples are ready for pickup in the LSL buffers. If the sender is considerably faster than the receiver"
                        " this may cause the plugin to hang indefinitely."
                        " The buffer length and chunk size of the LSL inlet are configurable. For more details on the meaning of these"
                        " variables please consult the LSL documentation.\n"
                        " In the case of string type streams e.g. marker streams, only one entry is saved into the AC variable. The nchannels"
                        " configuration variable is ignored for string streams, marker streams with more than one channel are not supported."
                        " To nsamples configuration variable is used to determine the size of the character buffer used to store the received strings"
                        " Longer marker strings are cut off memory safe, they may however cause memory allocations within lsl, so the size should not be chosen"
                        " too small.\n"
                        " The configuration regarding the AC variable size and the LSL stream inlet applies plugin wide. To use per-stream"
                        " configuration this plugin must be instantiated multiple times.\n")
/*
 * Local variables:
 * c-basic-offset: 4
 * compile-command: "make"
 * End:
 */
<|endoftext|>


<commit_before>#include "product_price_trend.h"

#include <common/Utilities.h>
#include <log-manager/PriceHistory.h>
#include <document-manager/DocumentManager.h>

#include <am/range/AmIterator.h>
#include <libcassandra/util_functions.h>
#include <glog/logging.h>

#include <boost/filesystem.hpp>
#include <algorithm>

using namespace std;
using namespace libcassandra;
using izenelib::util::UString;

namespace sf1r
{

ProductPriceTrend::ProductPriceTrend(
        const CassandraStorageConfig& cassandraConfig,
        const string& data_dir,
        const vector<string>& group_prop_vec,
        const vector<uint32_t>& time_int_vec)
    : cassandraConfig_(cassandraConfig)
    , data_dir_(data_dir)
    , group_prop_vec_(group_prop_vec)
    , time_int_vec_(time_int_vec)
    , buffer_size_(0)
    , enable_tpc_(!group_prop_vec_.empty() && !time_int_vec_.empty())
{
}

ProductPriceTrend::~ProductPriceTrend()
{
    for (TPCStorage::iterator it = tpc_storage_.begin();
            it != tpc_storage_.end(); ++it)
    {
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
            delete it->second[i];
    }
}

bool ProductPriceTrend::Init()
{
    if (!cassandraConfig_.enable)
        return false;

    price_history_.reset(new PriceHistory(cassandraConfig_.keyspace));
    price_history_->createColumnFamily();

    if (!price_history_->is_enabled)
        return false;

    for (vector<string>::const_iterator it = group_prop_vec_.begin();
            it != group_prop_vec_.end(); ++it)
    {
        vector<TPCBTree *>& prop_tpc = tpc_storage_[*it];
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
        {
            string db_path = data_dir_ + "/" + *it + "." + boost::lexical_cast<string>(time_int_vec_[i]) + ".tpc";
            prop_tpc.push_back(new TPCBTree(db_path));
            if (!prop_tpc.back()->open())
            {
                boost::filesystem::remove_all(db_path);
                prop_tpc.back()->open();
            }
        }
    }
    price_history_buffer_.resize(2000);

    return true;
}

bool ProductPriceTrend::Insert(
        const uint128_t& docid,
        const ProductPrice& price,
        time_t timestamp)
{
    price_history_buffer_[buffer_size_].resetKey(docid);
    price_history_buffer_[buffer_size_].resetHistory(0, timestamp, price);
    ++buffer_size_;

    if (IsBufferFull_())
    {
        return Flush(timestamp);
    }

    return true;
}

bool ProductPriceTrend::Update(
        const uint128_t& docid,
        const ProductPrice& price,
        time_t timestamp,
        map<string, string>& group_prop_map)
{
    price_history_buffer_[buffer_size_].resetKey(docid);
    price_history_buffer_[buffer_size_].resetHistory(0, timestamp, price);
    ++buffer_size_;

    if (enable_tpc_ && !group_prop_map.empty() && price.value.first > 0)
    {
        PropItemType& prop_item = prop_map_[docid];
        prop_item.first = price.value.first;
        prop_item.second.swap(group_prop_map);
    }

    if (IsBufferFull_())
    {
        return Flush(timestamp);
    }

    return true;
}

bool ProductPriceTrend::IsBufferFull_() const
{
    return buffer_size_ >= 2000;
}

bool ProductPriceTrend::Flush(time_t timestamp)
{
    bool ret = true;
    //time_t now = Utilities::createTimeStamp();

    if (!prop_map_.empty())
    {
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
        {
            if (!UpdateTPC_(i, timestamp))
                ret = false;
        }

        PropMapType().swap(prop_map_);
    }

    if (!price_history_->updateMultiRow(price_history_buffer_))
        ret = false;

    buffer_size_ = 0;

    return ret;
}

bool ProductPriceTrend::CronJob()
{
    if (!enable_tpc_) return true;

    //TODO
    return true;
}

bool ProductPriceTrend::UpdateTPC_(uint32_t time_int, time_t timestamp)
{
    vector<uint128_t> key_list;
    Utilities::getKeyList(key_list, prop_map_);
    if (timestamp == -1)
        timestamp = Utilities::createTimeStamp();
    timestamp -= 86400000000LL * time_int_vec_[time_int];

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, key_list, serializeLong(timestamp), "", 1))
        return false;

    map<string, map<string, TPCQueue> > tpc_cache;
    for (uint32_t i = 0; i < row_list.size(); i++)
    {
        PropMapType::const_iterator prop_map_it = prop_map_.find(row_list[i].getDocId());
        if(prop_map_it == prop_map_.end()) continue;
        const PropItemType& prop_item = prop_map_it->second;
        const pair<time_t, ProductPrice>& price_record = row_list[i].getPriceHistory()[0];
        const ProductPriceType& old_price = price_record.second.value.first;
        float price_cut = old_price == 0 ? 0 : 1 - prop_item.first / old_price;

        for (map<string, string>::const_iterator it = prop_item.second.begin();
                it != prop_item.second.end(); ++it)
        {
            TPCQueue& tpc_queue = tpc_cache[it->first][it->second];
            if (tpc_queue.empty())
            {
                tpc_queue.reserve(1000);
                tpc_storage_[it->first][time_int]->get(it->second, tpc_queue);
            }

            if (tpc_queue.size() < 1000)
            {
                tpc_queue.push_back(make_pair(price_cut, row_list[i].getDocId()));
                push_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
            }
            else if (price_cut > tpc_queue[0].first)
            {
                pop_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
                tpc_queue.back().first = price_cut;
                tpc_queue.back().second = row_list[i].getDocId();
                push_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
            }
        }
    }

    for (map<string, map<string, TPCQueue> >::const_iterator it = tpc_cache.begin();
            it != tpc_cache.end(); ++it)
    {
        TPCBTree* tpc_btree = tpc_storage_[it->first][time_int];
        for (map<string, TPCQueue>::const_iterator mit = it->second.begin();
                mit != it->second.end(); ++mit)
        {
            tpc_btree->update(mit->first, mit->second);
        }
        tpc_btree->flush();
    }

    return true;
}

bool ProductPriceTrend::GetMultiPriceHistory(
        PriceHistoryList& history_list,
        const vector<uint128_t>& docid_list,
        time_t from_tt,
        time_t to_tt,
        string& error_msg)
{
    string from_str(from_tt == -1 ? "" : serializeLong(from_tt));
    string to_str(to_tt == -1 ? "" : serializeLong(to_tt));

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, docid_list, from_str, to_str))
    {
        error_msg = "Failed retrieving price histories from Cassandra";
        return false;
    }

    history_list.reserve(history_list.size() + row_list.size());
    for (vector<PriceHistoryRow>::const_iterator it = row_list.begin();
            it != row_list.end(); ++it)
    {
        const PriceHistory::PriceHistoryType& price_history = it->getPriceHistory();
        PriceHistoryItem history_item;
        history_item.reserve(price_history.size());
        for (PriceHistory::PriceHistoryType::const_iterator hit = price_history.begin();
                hit != price_history.end(); ++hit)
        {
            history_item.push_back(make_pair(hit->first, hit->second.value));
        }
        history_list.push_back(make_pair(it->getDocId(), PriceHistoryItem()));
        history_list.back().second.swap(history_item);
    }

    if (history_list.empty())
    {
        error_msg = "Have not got any valid docid";
        return false;
    }

    return true;
}

bool ProductPriceTrend::GetMultiPriceRange(
        PriceRangeList& range_list,
        const vector<uint128_t>& docid_list,
        time_t from_tt,
        time_t to_tt,
        string& error_msg)
{
    string from_str(from_tt == -1 ? "" : serializeLong(from_tt));
    string to_str(to_tt == -1 ? "" : serializeLong(to_tt));

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, docid_list, from_str, to_str))
    {
        error_msg = "Failed retrieving price histories from Cassandra";
        return false;
    }

    range_list.reserve(range_list.size() + row_list.size());
    for (vector<PriceHistoryRow>::const_iterator it = row_list.begin();
            it != row_list.end(); ++it)
    {
        const PriceHistory::PriceHistoryType& price_history = it->getPriceHistory();
        ProductPrice range_item;
        for (PriceHistory::PriceHistoryType::const_iterator hit = price_history.begin();
                hit != price_history.end(); ++hit)
        {
            range_item += hit->second;
        }
        range_list.push_back(make_pair(it->getDocId(), range_item.value));
    }

    if (range_list.empty())
    {
        error_msg = "Have not got any valid docid";
        return false;
    }

    return true;
}

bool ProductPriceTrend::GetTopPriceCutList(
        TPCQueue& tpc_queue,
        const string& prop_name,
        const string& prop_value,
        uint32_t days,
        uint32_t count,
        string& error_msg)
{
    if (!enable_tpc_)
    {
        error_msg = "Top price-cut list is not enabled for this collection.";
        return false;
    }

    TPCStorage::const_iterator tpc_it = tpc_storage_.find(prop_name);
    if (tpc_it == tpc_storage_.end())
    {
        error_msg = "Don't find data for this property name: " + prop_name;
        return false;
    }

    uint32_t time_int = 0;
    while (days > time_int_vec_[time_int++] && time_int < time_int_vec_.size() - 1);
    if (!tpc_it->second[time_int]->get(prop_value, tpc_queue))
    {
        error_msg = "Don't find price-cut record for this property value: " + prop_value;
        return false;
    }

    sort_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator><TPCQueue::value_type>);
    if (tpc_queue.size() > count)
        tpc_queue.resize(count);

    return true;
}

bool ProductPriceTrend::MigratePriceHistory(
        const boost::shared_ptr<DocumentManager>& document_manager,
        const string& new_keyspace,
        uint32_t start,
        string& error_msg)
{
    if (new_keyspace == cassandraConfig_.keyspace)
        return true;

    vector<uint128_t> docid_list;
    docid_list.reserve(1000);
    boost::shared_ptr<PriceHistory> new_price_history(new PriceHistory(new_keyspace));
    new_price_history->createColumnFamily();

    uint32_t count = 0;
    for (uint32_t i = start; i <= document_manager->getMaxDocId(); i++)
    {
        Document doc;
        if (!document_manager->getDocument(i, doc))
            continue;

        Document::property_const_iterator kit = doc.findProperty("DOCID");
        if (kit == doc.propertyEnd())
            continue;

        const UString& key = kit->second.get<UString>();
        string str_docid;
        key.convertString(str_docid, UString::UTF_8);
        docid_list.push_back(Utilities::md5ToUint128(str_docid));
        if (docid_list.size() == 1000)
        {
            vector<PriceHistoryRow> row_list;
            if (!price_history_->getMultiSlice(row_list, docid_list))
            {
                error_msg = "Can't get price history from Cassandra.";
                return false;
            }

            if (!new_price_history->updateMultiRow(row_list))
            {
                error_msg = "Can't update price history to Cassandra.";
                return false;
            }
            docid_list.clear();
        }

        if (++count % 100000 == 0)
        {
            LOG(INFO) << "Migrating price history: " << count;
        }
    }

    if (!docid_list.empty())
    {
        vector<PriceHistoryRow> row_list;
        if (!price_history_->getMultiSlice(row_list, docid_list))
        {
            error_msg = "Can't get price history from Cassandra.";
            return false;
        }

        if (!new_price_history->updateMultiRow(row_list))
        {
            error_msg = "Can't update price history to Cassandra.";
            return false;
        }
    }
    LOG(INFO) << "Successfully migrated price history for " << count << " products.";

    return true;
}

}
<commit_msg>add end timestamp while get price history<commit_after>#include "product_price_trend.h"

#include <common/Utilities.h>
#include <log-manager/PriceHistory.h>
#include <document-manager/DocumentManager.h>

#include <am/range/AmIterator.h>
#include <libcassandra/util_functions.h>
#include <glog/logging.h>

#include <boost/filesystem.hpp>
#include <algorithm>

using namespace std;
using namespace libcassandra;
using izenelib::util::UString;

namespace sf1r
{

ProductPriceTrend::ProductPriceTrend(
        const CassandraStorageConfig& cassandraConfig,
        const string& data_dir,
        const vector<string>& group_prop_vec,
        const vector<uint32_t>& time_int_vec)
    : cassandraConfig_(cassandraConfig)
    , data_dir_(data_dir)
    , group_prop_vec_(group_prop_vec)
    , time_int_vec_(time_int_vec)
    , buffer_size_(0)
    , enable_tpc_(!group_prop_vec_.empty() && !time_int_vec_.empty())
{
}

ProductPriceTrend::~ProductPriceTrend()
{
    for (TPCStorage::iterator it = tpc_storage_.begin();
            it != tpc_storage_.end(); ++it)
    {
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
            delete it->second[i];
    }
}

bool ProductPriceTrend::Init()
{
    if (!cassandraConfig_.enable)
        return false;

    price_history_.reset(new PriceHistory(cassandraConfig_.keyspace));
    price_history_->createColumnFamily();

    if (!price_history_->is_enabled)
        return false;

    for (vector<string>::const_iterator it = group_prop_vec_.begin();
            it != group_prop_vec_.end(); ++it)
    {
        vector<TPCBTree *>& prop_tpc = tpc_storage_[*it];
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
        {
            string db_path = data_dir_ + "/" + *it + "." + boost::lexical_cast<string>(time_int_vec_[i]) + ".tpc";
            prop_tpc.push_back(new TPCBTree(db_path));
            if (!prop_tpc.back()->open())
            {
                boost::filesystem::remove_all(db_path);
                prop_tpc.back()->open();
            }
        }
    }
    price_history_buffer_.resize(2000);

    return true;
}

bool ProductPriceTrend::Insert(
        const uint128_t& docid,
        const ProductPrice& price,
        time_t timestamp)
{
    price_history_buffer_[buffer_size_].resetKey(docid);
    price_history_buffer_[buffer_size_].resetHistory(0, timestamp, price);
    ++buffer_size_;

    if (IsBufferFull_())
    {
        return Flush(timestamp);
    }

    return true;
}

bool ProductPriceTrend::Update(
        const uint128_t& docid,
        const ProductPrice& price,
        time_t timestamp,
        map<string, string>& group_prop_map)
{
    price_history_buffer_[buffer_size_].resetKey(docid);
    price_history_buffer_[buffer_size_].resetHistory(0, timestamp, price);
    ++buffer_size_;

    if (enable_tpc_ && !group_prop_map.empty() && price.value.first > 0)
    {
        PropItemType& prop_item = prop_map_[docid];
        prop_item.first = price.value.first;
        prop_item.second.swap(group_prop_map);
    }

    if (IsBufferFull_())
    {
        return Flush(timestamp);
    }

    return true;
}

bool ProductPriceTrend::IsBufferFull_() const
{
    return buffer_size_ >= 2000;
}

bool ProductPriceTrend::Flush(time_t timestamp)
{
    bool ret = true;
    //time_t now = Utilities::createTimeStamp();

    if (!prop_map_.empty())
    {
        for (uint32_t i = 0; i < time_int_vec_.size(); i++)
        {
            if (!UpdateTPC_(i, timestamp))
                ret = false;
        }

        PropMapType().swap(prop_map_);
    }

    if (!price_history_->updateMultiRow(price_history_buffer_))
        ret = false;

    buffer_size_ = 0;

    return ret;
}

bool ProductPriceTrend::CronJob()
{
    if (!enable_tpc_) return true;

    //TODO
    return true;
}

bool ProductPriceTrend::UpdateTPC_(uint32_t time_int, time_t timestamp)
{
    vector<uint128_t> key_list;
    Utilities::getKeyList(key_list, prop_map_);
    if (timestamp == -1)
        timestamp = Utilities::createTimeStamp();
    time_t end_time = timestamp;
    LOG(INFO) << "UpdateTPC at : " << timestamp;
    timestamp -= 86400000000LL * time_int_vec_[time_int];

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, key_list, serializeLong(timestamp), serializeLong(end_time), 1))
        return false;

    map<string, map<string, TPCQueue> > tpc_cache;
    for (uint32_t i = 0; i < row_list.size(); i++)
    {
        PropMapType::const_iterator prop_map_it = prop_map_.find(row_list[i].getDocId());
        if(prop_map_it == prop_map_.end()) continue;
        const PropItemType& prop_item = prop_map_it->second;
        const pair<time_t, ProductPrice>& price_record = row_list[i].getPriceHistory()[0];
        const ProductPriceType& old_price = price_record.second.value.first;
        float price_cut = old_price == 0 ? 0 : 1 - prop_item.first / old_price;

        for (map<string, string>::const_iterator it = prop_item.second.begin();
                it != prop_item.second.end(); ++it)
        {
            TPCQueue& tpc_queue = tpc_cache[it->first][it->second];
            if (tpc_queue.empty())
            {
                tpc_queue.reserve(1000);
                tpc_storage_[it->first][time_int]->get(it->second, tpc_queue);
            }

            if (tpc_queue.size() < 1000)
            {
                tpc_queue.push_back(make_pair(price_cut, row_list[i].getDocId()));
                push_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
            }
            else if (price_cut > tpc_queue[0].first)
            {
                pop_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
                tpc_queue.back().first = price_cut;
                tpc_queue.back().second = row_list[i].getDocId();
                push_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator> <TPCQueue::value_type>);
            }
        }
    }

    for (map<string, map<string, TPCQueue> >::const_iterator it = tpc_cache.begin();
            it != tpc_cache.end(); ++it)
    {
        TPCBTree* tpc_btree = tpc_storage_[it->first][time_int];
        for (map<string, TPCQueue>::const_iterator mit = it->second.begin();
                mit != it->second.end(); ++mit)
        {
            tpc_btree->update(mit->first, mit->second);
        }
        tpc_btree->flush();
    }

    return true;
}

bool ProductPriceTrend::GetMultiPriceHistory(
        PriceHistoryList& history_list,
        const vector<uint128_t>& docid_list,
        time_t from_tt,
        time_t to_tt,
        string& error_msg)
{
    string from_str(from_tt == -1 ? "" : serializeLong(from_tt));
    string to_str(to_tt == -1 ? "" : serializeLong(to_tt));

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, docid_list, from_str, to_str))
    {
        error_msg = "Failed retrieving price histories from Cassandra";
        return false;
    }

    history_list.reserve(history_list.size() + row_list.size());
    for (vector<PriceHistoryRow>::const_iterator it = row_list.begin();
            it != row_list.end(); ++it)
    {
        const PriceHistory::PriceHistoryType& price_history = it->getPriceHistory();
        PriceHistoryItem history_item;
        history_item.reserve(price_history.size());
        for (PriceHistory::PriceHistoryType::const_iterator hit = price_history.begin();
                hit != price_history.end(); ++hit)
        {
            history_item.push_back(make_pair(hit->first, hit->second.value));
        }
        history_list.push_back(make_pair(it->getDocId(), PriceHistoryItem()));
        history_list.back().second.swap(history_item);
    }

    if (history_list.empty())
    {
        error_msg = "Have not got any valid docid";
        return false;
    }

    return true;
}

bool ProductPriceTrend::GetMultiPriceRange(
        PriceRangeList& range_list,
        const vector<uint128_t>& docid_list,
        time_t from_tt,
        time_t to_tt,
        string& error_msg)
{
    string from_str(from_tt == -1 ? "" : serializeLong(from_tt));
    string to_str(to_tt == -1 ? "" : serializeLong(to_tt));

    vector<PriceHistoryRow> row_list;
    if (!price_history_->getMultiSlice(row_list, docid_list, from_str, to_str))
    {
        error_msg = "Failed retrieving price histories from Cassandra";
        return false;
    }

    range_list.reserve(range_list.size() + row_list.size());
    for (vector<PriceHistoryRow>::const_iterator it = row_list.begin();
            it != row_list.end(); ++it)
    {
        const PriceHistory::PriceHistoryType& price_history = it->getPriceHistory();
        ProductPrice range_item;
        for (PriceHistory::PriceHistoryType::const_iterator hit = price_history.begin();
                hit != price_history.end(); ++hit)
        {
            range_item += hit->second;
        }
        range_list.push_back(make_pair(it->getDocId(), range_item.value));
    }

    if (range_list.empty())
    {
        error_msg = "Have not got any valid docid";
        return false;
    }

    return true;
}

bool ProductPriceTrend::GetTopPriceCutList(
        TPCQueue& tpc_queue,
        const string& prop_name,
        const string& prop_value,
        uint32_t days,
        uint32_t count,
        string& error_msg)
{
    if (!enable_tpc_)
    {
        error_msg = "Top price-cut list is not enabled for this collection.";
        return false;
    }

    TPCStorage::const_iterator tpc_it = tpc_storage_.find(prop_name);
    if (tpc_it == tpc_storage_.end())
    {
        error_msg = "Don't find data for this property name: " + prop_name;
        return false;
    }

    uint32_t time_int = 0;
    while (days > time_int_vec_[time_int++] && time_int < time_int_vec_.size() - 1);
    if (!tpc_it->second[time_int]->get(prop_value, tpc_queue))
    {
        error_msg = "Don't find price-cut record for this property value: " + prop_value;
        return false;
    }

    sort_heap(tpc_queue.begin(), tpc_queue.end(), rel_ops::operator><TPCQueue::value_type>);
    if (tpc_queue.size() > count)
        tpc_queue.resize(count);

    return true;
}

bool ProductPriceTrend::MigratePriceHistory(
        const boost::shared_ptr<DocumentManager>& document_manager,
        const string& new_keyspace,
        uint32_t start,
        string& error_msg)
{
    if (new_keyspace == cassandraConfig_.keyspace)
        return true;

    vector<uint128_t> docid_list;
    docid_list.reserve(1000);
    boost::shared_ptr<PriceHistory> new_price_history(new PriceHistory(new_keyspace));
    new_price_history->createColumnFamily();

    uint32_t count = 0;
    for (uint32_t i = start; i <= document_manager->getMaxDocId(); i++)
    {
        Document doc;
        if (!document_manager->getDocument(i, doc))
            continue;

        Document::property_const_iterator kit = doc.findProperty("DOCID");
        if (kit == doc.propertyEnd())
            continue;

        const UString& key = kit->second.get<UString>();
        string str_docid;
        key.convertString(str_docid, UString::UTF_8);
        docid_list.push_back(Utilities::md5ToUint128(str_docid));
        if (docid_list.size() == 1000)
        {
            vector<PriceHistoryRow> row_list;
            if (!price_history_->getMultiSlice(row_list, docid_list))
            {
                error_msg = "Can't get price history from Cassandra.";
                return false;
            }

            if (!new_price_history->updateMultiRow(row_list))
            {
                error_msg = "Can't update price history to Cassandra.";
                return false;
            }
            docid_list.clear();
        }

        if (++count % 100000 == 0)
        {
            LOG(INFO) << "Migrating price history: " << count;
        }
    }

    if (!docid_list.empty())
    {
        vector<PriceHistoryRow> row_list;
        if (!price_history_->getMultiSlice(row_list, docid_list))
        {
            error_msg = "Can't get price history from Cassandra.";
            return false;
        }

        if (!new_price_history->updateMultiRow(row_list))
        {
            error_msg = "Can't update price history to Cassandra.";
            return false;
        }
    }
    LOG(INFO) << "Successfully migrated price history for " << count << " products.";

    return true;
}

}
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
#include <stdlib.h>
#include <inttypes.h>
#include "Crypto.h"
#include "Identity.h"
#include "Timestamp.h"
#include "common/key.hpp"

int main (int argc, char * argv[])
{
	if (argc < 3)
	{
		std::cout << "Usage: offlinekeys <output file> <keys file> <signature type> <days>" << std::endl;
		return -1;
	}
	i2p::crypto::InitCrypto (false, true, true, false);

	std::string fname(argv[2]);
	i2p::data::PrivateKeys keys;
	{
		std::vector<uint8_t> buff;
		std::ifstream inf;
		inf.open(fname);
		if (!inf.is_open()) {
			std::cout << "cannot open keys file " << fname << std::endl;
			return 2;
		}
		inf.seekg(0, std::ios::end);
		const std::size_t len = inf.tellg();
		inf.seekg(0, std::ios::beg);
		buff.resize(len);
		inf.read((char*)buff.data(), buff.size());
		if (!keys.FromBuffer(buff.data(), buff.size())) {
			std::cout << "bad keys file format" << std::endl;
			return 3;
		}
	}

	i2p::data::SigningKeyType type = i2p::data::SIGNING_KEY_TYPE_EDDSA_SHA512_ED25519; // EdDSA by default
	if (argc > 3)
	{
		std::string str(argv[3]);
		type = NameToSigType(str);
	}

	int days = 365; // 1 year by default
	if (argc > 4)
		days = std::stoi (argv[4]);
	uint32_t expires = i2p::util::GetSecondsSinceEpoch () + days*24*60*60;

	auto offlineKeys = keys.CreateOfflineKeys (type, expires);
	std::ofstream f (argv[1], std::ofstream::binary | std::ofstream::out);
	if (f)
	{
		size_t len = offlineKeys.GetFullLen ();
		uint8_t * buf = new uint8_t[len];
		len = offlineKeys.ToBuffer (buf, len);
		f.write ((char *)buf, len);
		delete[] buf;
		std::cout << "Offline keys for destination " << offlineKeys.GetPublic ()->GetIdentHash ().ToBase32 () << " created" << std::endl;
	}
	else
		std::cout << "Can't create file " << argv[1] << std::endl;

	i2p::crypto::TerminateCrypto ();

	return 0;
}

<commit_msg>user friendly mode<commit_after>#include <iostream>
#include <fstream>
#include <stdlib.h>
#include <inttypes.h>
#include "Crypto.h"
#include "Identity.h"
#include "Timestamp.h"
#include "common/key.hpp"

int main (int argc, char * argv[])
{
	if (argc < 3)
	{
		std::cout << "Usage: offlinekeys <output file> <keys file> <signature type> <days>" << std::endl;
		return -1;
	}
	i2p::crypto::InitCrypto (false, true, true, false);

	std::string fname(argv[2]);
	i2p::data::PrivateKeys keys;
	{
		std::vector<uint8_t> buff;
		std::ifstream inf;
		inf.open(fname);
		if (!inf.is_open()) {
			std::cout << "cannot open keys file " << fname << std::endl;
			return 2;
		}
		inf.seekg(0, std::ios::end);
		const std::size_t len = inf.tellg();
		inf.seekg(0, std::ios::beg);
		buff.resize(len);
		inf.read((char*)buff.data(), buff.size());
		if (!keys.FromBuffer(buff.data(), buff.size())) {
			std::cout << "bad keys file format" << std::endl;
			return 3;
		}
	}

	i2p::data::SigningKeyType type = i2p::data::SIGNING_KEY_TYPE_EDDSA_SHA512_ED25519; // EdDSA by default
	if (argc > 3)
	{
		std::string str(argv[3]);
		type = NameToSigType(str);
		if (SigTypeToName(type).find("unknown") != std::string::npos) { std::cerr << "Incorrect signature type" << std::endl; return -2; }
	}

	int days = 365; // 1 year by default
	if (argc > 4)
		days = std::stoi (argv[4]);
	uint32_t expires = i2p::util::GetSecondsSinceEpoch () + days*24*60*60;

	auto offlineKeys = keys.CreateOfflineKeys (type, expires);
	std::ofstream f (argv[1], std::ofstream::binary | std::ofstream::out);
	if (f)
	{
		size_t len = offlineKeys.GetFullLen ();
		uint8_t * buf = new uint8_t[len];
		len = offlineKeys.ToBuffer (buf, len);
		f.write ((char *)buf, len);
		delete[] buf;
		std::cout << "Offline keys for destination " << offlineKeys.GetPublic ()->GetIdentHash ().ToBase32 () << " created" << std::endl
			<< "Signature type: " << SigTypeToName(type) << " (" << type << ")" << std::endl
			<< "Days: " << days << std::endl;
	}
	else
		std::cout << "Can't create file " << argv[1] << std::endl;

	i2p::crypto::TerminateCrypto ();

	return 0;
}
<|endoftext|>


<commit_before>#include <iostream>
#include <sstream>
#include <sys/ioctl.h>
#include <stdio.h>
#include <signal.h>
#include "au/string.h"
#include "au/list.h"
#include "au/Cronometer.h"
#include "au/utils.h"

#include "au/TokenTaker.h"

#include "au/ConsoleAutoComplete.h"
#include "au/ConsoleCommand.h"
#include "au/ConsoleCommandHistory.h"
#include "au/ConsoleCode.h"
#include "au/ConsoleEscapeSequence.h"

#include "au/Console.h"  // Own interface

namespace au {

    Console *current_console=NULL;
    void handle_winch(int sig);

    Console::Console() : token_pending_messages("token_pending_messages")
    {
        command_history = new ConsoleCommandHistory();
    }

    Console::~Console()
    {
        delete command_history;
    }

    void Console::init()
    {
        /* get the terminal settings for stdin */
        tcgetattr(STDIN_FILENO,&old_tio);

        /* we want to keep the old setting to restore them a the end */
        new_tio=old_tio;

        /* disable canonical mode (buffered i/o) and local echo */
        new_tio.c_lflag &=(~ICANON & ~ECHO);

        /* set the new settings immediately */
        tcsetattr(STDIN_FILENO,TCSANOW,&new_tio);

    }

    void Console::finish()
    {
        /* set the new settings immediately */
        tcsetattr(STDIN_FILENO,TCSANOW,&old_tio);
    }

    void Console::print_command()
    {
        std::string _command = command_history->current()->getCommand();
        int _pos = command_history->current()->getPos();

        clear_line();
        printf("%s %s", getPrompt().c_str() , _command.c_str());

        if( _pos != (int)_command.length() )
        {
            printf("\r");

            printf("%s " ,  getPrompt().c_str() );
            for (int i = 0 ; i < _pos ; i++ )
                printf("%c",_command[i]);
        }

        fflush(stdout);
        // At the moment we are not considering pos_command
    }

    bool Console::isImputReady()
    {
        struct timeval  timeVal;
        timeVal.tv_sec  = 0;
        timeVal.tv_usec = 0;

        fd_set          rFds;
        FD_ZERO(&rFds);
        FD_SET(0, &rFds);

        int s = select( 1, &rFds, NULL, NULL, &timeVal);

        return (s==1);
    }

    void Console::process_auto_complete( ConsoleAutoComplete * info )
    {
        // Do something with autocomplete information provided by thrid party
        print_command();
        printf("\n");
        info->print_last_words_alternatives();

        // Add necessary stuff...
        command_history->current()->add( info->stringToAppend() );
        print_command();

    }

    void Console::process_char( char c )
    {
        command_history->current()->add(c);

        if ( command_history->current()->isCursorAtEnd() )
        {
            printf("%c",c);
            fflush(stdout);
        }
        else
            print_command();
    }

    void Console::process_code( ConsoleCode code )
    {
        switch (code) {
            case ret:
            {
                std::string _command = command_history->current()->getCommand();

                //Print the command on screen...
                print_command();
                printf("\n");

                // Eval the command....
                evalCommand( _command );

                //New command in history
                command_history->new_command();
                print_command();
            }
                break;
            case tab:
            {
                ConsoleAutoComplete* info = new ConsoleAutoComplete( command_history->current()->getCommandUntilPointer() );
                autoComplete( info );
                process_auto_complete( info );
                delete info;

            }
                break;
            case del:
                command_history->current()->delete_char();
                print_command();
                break;
            case del_word:
                command_history->current()->delete_word();
                print_command();
                break;
            case move_forward:
                command_history->current()->move_cursor(1);
                print_command();
                break;
            case move_backward:
                command_history->current()->move_cursor(-1);
                print_command();
                break;
            case move_up:
                command_history->move_up();
                print_command();
                break;
            case move_down:
                command_history->move_down();
                print_command();
                break;
            case move_home:
                command_history->current()->move_home();
                print_command();
                break;
            case move_end:
                command_history->current()->move_end();
                print_command();
                break;
            case del_rest_line:
                command_history->current()->delete_rest_line();
                print_command();
                break;

            default:
                break;
        }
        /*
         printf("[%s]", getConsoleCodeName(code) );
         fflush(stdout);
         */
    }

    void Console::process_background()
    {
        if ( pending_messages.size() != 0 )
        {
            au::TokenTaker tt(&token_pending_messages);

            clear_line();

            while (pending_messages.size() != 0)
            {
                std::string txt = pending_messages.front();
                pending_messages.pop_front();
                printf("%s", txt.c_str() );
            }

            printf("\n\n");
            print_command();
            fflush(stdout);

        }
    }

    bool Console::isNormalChar( char c )
    {
        if( ( c>= 32) && ( c<=126))
            return true;
        return false;
    }

    void Console::runConsole()
    {
        // Init console
        init();

        t_running = pthread_self();

        // Signal to handle terminal changes...
        current_console = this;
        signal(SIGWINCH, handle_winch);

        // First version with just the promtp
        print_command();

        // Escape sequence...
        ConsoleEscapeSequence escape_sequence;
        bool escaping = false;   // Fag to indicate if we are inside a scape sequence...

        quit_console = false;
        au::Cronometer cronometer;
        while( !quit_console )
        {

            while ( !isImputReady() )
            {
                // Review background messages
                process_background();

                if( cronometer.diffTimeInSeconds() > 1 )
                {
                    cronometer.reset();
                    print_command();  // Refresh prompt
                }

                usleep(1000);
            }

            char c;
            read( 0 , &c , 1 );

            if( escaping )
            {
                escape_sequence.add( c );

                if ( escape_sequence.isFinished() )
                {
                    ConsoleCode code = escape_sequence.getCode();

                    if ( code == unknown )
                        writeWarningOnConsole( au::str( " Unknown escape sequence (%s)" , escape_sequence.description().c_str() ).c_str() );
                    else
                        process_code(code);
                    escaping = false;
                }
            }
            else
            {
                if ( isNormalChar(c) )
                    process_char(c);
                else if ( c == '\n' )
                    process_code( ret );
                else if ( c == '\t' )
                    process_code( tab );
                else if ( c == 127 )
                    process_code( del );
                else if ( c == 11 )
                    process_code( del_rest_line );
                else if ( c == 1 )
                    process_code( move_home );
                else if ( c == 5 )
                    process_code( move_end );
                else if ( c == 7 ) // bell
                    printf("%c",c);
                else if ( c == 27 )
                {
                    escaping = true;  // Start a scape mode
                    escape_sequence.init();
                }
                else
                {
                    writeWarningOnConsole( au::str( " Ignoring char (%d)" , c ) );
                }

            }

        }

        finish();

        // Remove the signal handler...
        signal(SIGWINCH, SIG_IGN);

    }

    void Console::quitConsole()
    {
        quit_console = true;
    }

    /* Methods to write things on screen */
    void Console::writeWarningOnConsole( std::string message )
    {
        std::ostringstream output;
        output << "\033[1;35m"<< message << "\033[0m";
        write( output.str() );
    }

    void Console::writeErrorOnConsole( std::string message )
    {
        std::ostringstream output;
        output << "\033[1;31m"<< message << "\033[0m";
        write( output.str() );
    }

    void Console::writeOnConsole( std::string message )
    {
        Console::write( message );
    }

    void Console::write( std::string message )
    {
        if( pthread_self() != t_running )
        {
            // Accumulate message
            au::TokenTaker tt(&token_pending_messages);
            pending_messages.push_back( message );
            return;
        }

        clear_line();
        printf("%s\n", message.c_str() );
        print_command();
        fflush( stdout );
    }

    std::string Console::getPrompt()
    {
        return "> ";
    }

    void Console::evalCommand( std::string command )
    {
        writeWarningOnConsole("Console::evalCommand not implemented");

        // Simple quit function...
        if ( command == "quit" )
            quit_console = true;
    }

    void Console::autoComplete( ConsoleAutoComplete* info )
    {
        writeWarningOnConsole("Console::auto_complete not implemented");
    }

    void Console::refresh()
    {
        print_command();
    }

    void handle_winch(int sig)
    {
        if( !current_console )
            return;

        // Rewrite current command
        current_console->refresh();
    }

}<commit_msg>Minor stuff to make hudson compile<commit_after>#include <iostream>
#include <sstream>
#include <sys/ioctl.h>
#include <stdio.h>
#include <signal.h>
#include "au/string.h"
#include "au/list.h"
#include "au/Cronometer.h"
#include "au/utils.h"

#include "au/TokenTaker.h"

#include "au/ConsoleAutoComplete.h"
#include "au/ConsoleCommand.h"
#include "au/ConsoleCommandHistory.h"
#include "au/ConsoleCode.h"
#include "au/ConsoleEscapeSequence.h"

#include "au/Console.h"  // Own interface

namespace au {

    Console *current_console=NULL;
    void handle_winch(int sig);

    Console::Console() : token_pending_messages("token_pending_messages")
    {
        command_history = new ConsoleCommandHistory();
    }

    Console::~Console()
    {
        delete command_history;
    }

    void Console::init()
    {
        /* get the terminal settings for stdin */
        tcgetattr(STDIN_FILENO,&old_tio);

        /* we want to keep the old setting to restore them a the end */
        new_tio=old_tio;

        /* disable canonical mode (buffered i/o) and local echo */
        new_tio.c_lflag &=(~ICANON & ~ECHO);

        /* set the new settings immediately */
        tcsetattr(STDIN_FILENO,TCSANOW,&new_tio);

    }

    void Console::finish()
    {
        /* set the new settings immediately */
        tcsetattr(STDIN_FILENO,TCSANOW,&old_tio);
    }

    void Console::print_command()
    {
        std::string _command = command_history->current()->getCommand();
        int _pos = command_history->current()->getPos();

        clear_line();
        printf("%s %s", getPrompt().c_str() , _command.c_str());

        if( _pos != (int)_command.length() )
        {
            printf("\r");

            printf("%s " ,  getPrompt().c_str() );
            for (int i = 0 ; i < _pos ; i++ )
                printf("%c",_command[i]);
        }

        fflush(stdout);
        // At the moment we are not considering pos_command
    }

    bool Console::isImputReady()
    {
        struct timeval  timeVal;
        timeVal.tv_sec  = 0;
        timeVal.tv_usec = 0;

        fd_set          rFds;
        FD_ZERO(&rFds);
        FD_SET(0, &rFds);

        int s = select( 1, &rFds, NULL, NULL, &timeVal);

        return (s==1);
    }

    void Console::process_auto_complete( ConsoleAutoComplete * info )
    {
        // Do something with autocomplete information provided by thrid party
        print_command();
        printf("\n");
        info->print_last_words_alternatives();

        // Add necessary stuff...
        command_history->current()->add( info->stringToAppend() );
        print_command();

    }

    void Console::process_char( char c )
    {
        command_history->current()->add(c);

        if ( command_history->current()->isCursorAtEnd() )
        {
            printf("%c",c);
            fflush(stdout);
        }
        else
            print_command();
    }

    void Console::process_code( ConsoleCode code )
    {
        switch (code) {
            case ret:
            {
                std::string _command = command_history->current()->getCommand();

                //Print the command on screen...
                print_command();
                printf("\n");

                // Eval the command....
                evalCommand( _command );

                //New command in history
                command_history->new_command();
                print_command();
            }
                break;
            case tab:
            {
                ConsoleAutoComplete* info = new ConsoleAutoComplete( command_history->current()->getCommandUntilPointer() );
                autoComplete( info );
                process_auto_complete( info );
                delete info;

            }
                break;
            case del:
                command_history->current()->delete_char();
                print_command();
                break;
            case del_word:
                command_history->current()->delete_word();
                print_command();
                break;
            case move_forward:
                command_history->current()->move_cursor(1);
                print_command();
                break;
            case move_backward:
                command_history->current()->move_cursor(-1);
                print_command();
                break;
            case move_up:
                command_history->move_up();
                print_command();
                break;
            case move_down:
                command_history->move_down();
                print_command();
                break;
            case move_home:
                command_history->current()->move_home();
                print_command();
                break;
            case move_end:
                command_history->current()->move_end();
                print_command();
                break;
            case del_rest_line:
                command_history->current()->delete_rest_line();
                print_command();
                break;

            default:
                break;
        }
        /*
         printf("[%s]", getConsoleCodeName(code) );
         fflush(stdout);
         */
    }

    void Console::process_background()
    {
        if ( pending_messages.size() != 0 )
        {
            au::TokenTaker tt(&token_pending_messages);

            clear_line();

            while (pending_messages.size() != 0)
            {
                std::string txt = pending_messages.front();
                pending_messages.pop_front();
                printf("%s", txt.c_str() );
            }

            printf("\n\n");
            print_command();
            fflush(stdout);

        }
    }

    bool Console::isNormalChar( char c )
    {
        if( ( c>= 32) && ( c<=126))
            return true;
        return false;
    }

    void Console::runConsole()
    {
        // Init console
        init();

        t_running = pthread_self();

        // Signal to handle terminal changes...
        current_console = this;
        signal(SIGWINCH, handle_winch);

        // First version with just the promtp
        print_command();

        // Escape sequence...
        ConsoleEscapeSequence escape_sequence;
        bool escaping = false;   // Fag to indicate if we are inside a scape sequence...

        quit_console = false;
        au::Cronometer cronometer;
        while( !quit_console )
        {

            while ( !isImputReady() )
            {
                // Review background messages
                process_background();

                if( cronometer.diffTimeInSeconds() > 1 )
                {
                    cronometer.reset();
                    print_command();  // Refresh prompt
                }

                usleep(1000);
            }

            char c;
            if( read( 0 , &c , 1 ) )
                LM_X(1, ("reading from stdin failed"));

            if( escaping )
            {
                escape_sequence.add( c );

                if ( escape_sequence.isFinished() )
                {
                    ConsoleCode code = escape_sequence.getCode();

                    if ( code == unknown )
                        writeWarningOnConsole( au::str( " Unknown escape sequence (%s)" , escape_sequence.description().c_str() ).c_str() );
                    else
                        process_code(code);
                    escaping = false;
                }
            }
            else
            {
                if ( isNormalChar(c) )
                    process_char(c);
                else if ( c == '\n' )
                    process_code( ret );
                else if ( c == '\t' )
                    process_code( tab );
                else if ( c == 127 )
                    process_code( del );
                else if ( c == 11 )
                    process_code( del_rest_line );
                else if ( c == 1 )
                    process_code( move_home );
                else if ( c == 5 )
                    process_code( move_end );
                else if ( c == 7 ) // bell
                    printf("%c",c);
                else if ( c == 27 )
                {
                    escaping = true;  // Start a scape mode
                    escape_sequence.init();
                }
                else
                {
                    writeWarningOnConsole( au::str( " Ignoring char (%d)" , c ) );
                }

            }

        }

        finish();

        // Remove the signal handler...
        signal(SIGWINCH, SIG_IGN);

    }

    void Console::quitConsole()
    {
        quit_console = true;
    }

    /* Methods to write things on screen */
    void Console::writeWarningOnConsole( std::string message )
    {
        std::ostringstream output;
        output << "\033[1;35m"<< message << "\033[0m";
        write( output.str() );
    }

    void Console::writeErrorOnConsole( std::string message )
    {
        std::ostringstream output;
        output << "\033[1;31m"<< message << "\033[0m";
        write( output.str() );
    }

    void Console::writeOnConsole( std::string message )
    {
        Console::write( message );
    }

    void Console::write( std::string message )
    {
        if( pthread_self() != t_running )
        {
            // Accumulate message
            au::TokenTaker tt(&token_pending_messages);
            pending_messages.push_back( message );
            return;
        }

        clear_line();
        printf("%s\n", message.c_str() );
        print_command();
        fflush( stdout );
    }

    std::string Console::getPrompt()
    {
        return "> ";
    }

    void Console::evalCommand( std::string command )
    {
        writeWarningOnConsole("Console::evalCommand not implemented");

        // Simple quit function...
        if ( command == "quit" )
            quit_console = true;
    }

    void Console::autoComplete( ConsoleAutoComplete* info )
    {
        writeWarningOnConsole("Console::auto_complete not implemented");
    }

    void Console::refresh()
    {
        print_command();
    }

    void handle_winch(int sig)
    {
        if( !current_console )
            return;

        // Rewrite current command
        current_console->refresh();
    }

}<|endoftext|>


<commit_before>/**
 * @brief Test cases for insert node.
 *
 * Copyright(c) 2015, CMU
 */

#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "gtest/gtest.h"
#include "gmock/gmock.h"

#include "backend/catalog/schema.h"
#include "backend/common/value_factory.h"

#include "backend/executor/delete_executor.h"
#include "backend/executor/insert_executor.h"
#include "backend/executor/seq_scan_executor.h"
#include "backend/executor/update_executor.h"
#include "backend/executor/logical_tile_factory.h"
#include "backend/expression/expression_util.h"
#include "backend/expression/abstract_expression.h"
#include "backend/expression/expression.h"
#include "backend/planner/delete_node.h"
#include "backend/planner/insert_node.h"
#include "backend/planner/seq_scan_node.h"
#include "backend/planner/update_node.h"
#include "backend/storage/tile_group.h"
#include "backend/storage/table_factory.h"

#include "executor_tests_util.h"
#include "executor/mock_executor.h"
#include "harness.h"

#include <atomic>

using ::testing::NotNull;
using ::testing::Return;

namespace peloton {
namespace test {

//===------------------------------===//
// Utility
//===------------------------------===//

/**
 * Cook a ProjectInfo object from a tuple.
 * Simply use a ConstantValueExpression for each attribute.
 */
planner::ProjectInfo* MakeProjectInfoFromTuple(const storage::Tuple* tuple){
  planner::ProjectInfo::TargetList target_list;
  for(oid_t col_id = START_OID; col_id < tuple->GetColumnCount(); col_id++) {
    auto value = tuple->GetValue(col_id);
    auto expression = expression::ConstantValueFactory(value);
    target_list.emplace_back(col_id, expression);
  }

  return new planner::ProjectInfo(std::move(target_list),
                                  planner::ProjectInfo::DirectMapList());
}

//===--------------------------------------------------------------------===//
// Mutator Tests
//===--------------------------------------------------------------------===//

std::atomic<int> tuple_id;
std::atomic<int> delete_tuple_id;

void InsertTuple(storage::DataTable *table){

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  auto tuple = ExecutorTestsUtil::GetTuple(table, ++tuple_id);

  auto project_info = MakeProjectInfoFromTuple(tuple);

  planner::InsertNode node(table, project_info);
  executor::InsertExecutor executor(&node, context.get());
  executor.Execute();

  tuple->FreeUninlinedData();
  delete tuple;

  txn_manager.CommitTransaction(txn);
}

void UpdateTuple(storage::DataTable *table){

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  std::vector<storage::Tuple *> tuples;

  // Update

  std::vector<oid_t> update_column_ids = { 2 };
  std::vector<Value> values;
  Value update_val = ValueFactory::GetDoubleValue(23.5);
  values.push_back(update_val);

  planner::UpdateNode update_node(table, update_column_ids, values);
  executor::UpdateExecutor update_executor(&update_node, context.get());

  // Predicate

  // WHERE ATTR_0 < 60
  expression::TupleValueExpression *tup_val_exp =
      new expression::TupleValueExpression(0, 0, std::string("tablename"), std::string("colname"));
  expression::ConstantValueExpression *const_val_exp =
      new expression::ConstantValueExpression(ValueFactory::GetIntegerValue(60));
  auto predicate =
      new expression::ComparisonExpression<expression::CmpLt>(EXPRESSION_TYPE_COMPARE_LT, tup_val_exp, const_val_exp);

  // Seq scan
  std::vector<oid_t> column_ids = { 0 };
  planner::SeqScanNode seq_scan_node(
      table,
      predicate,
      column_ids);
  executor::SeqScanExecutor seq_scan_executor(&seq_scan_node, context.get());

  // Parent-Child relationship
  update_node.AddChild(&seq_scan_node);
  update_executor.AddChild(&seq_scan_executor);

  EXPECT_TRUE(update_executor.Init());
  EXPECT_TRUE(update_executor.Execute());

  txn_manager.CommitTransaction(txn);
}

void DeleteTuple(storage::DataTable *table){

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  std::vector<storage::Tuple *> tuples;

  // Delete
  planner::DeleteNode delete_node(table, false);
  executor::DeleteExecutor delete_executor(&delete_node, context.get());

  // Predicate

  // WHERE ATTR_0 < 90
  expression::TupleValueExpression *tup_val_exp =
      new expression::TupleValueExpression(0, 0, std::string("tablename"), std::string("colname"));
  expression::ConstantValueExpression *const_val_exp =
      new expression::ConstantValueExpression(ValueFactory::GetIntegerValue(90));
  auto predicate =
      new expression::ComparisonExpression<expression::CmpLt>(EXPRESSION_TYPE_COMPARE_LT, tup_val_exp, const_val_exp);

  // Seq scan
  std::vector<oid_t> column_ids = { 0 };
  planner::SeqScanNode seq_scan_node(
      table,
      predicate,
      column_ids);
  executor::SeqScanExecutor seq_scan_executor(&seq_scan_node, context.get());

  // Parent-Child relationship
  delete_node.AddChild(&seq_scan_node);
  delete_executor.AddChild(&seq_scan_executor);

  EXPECT_TRUE(delete_executor.Init());
  EXPECT_TRUE(delete_executor.Execute());

  txn_manager.CommitTransaction(txn);
}

TEST(MutateTests, StressTests) {

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();

  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  // Create insert node for this test.
  storage::DataTable *table = ExecutorTestsUtil::CreateTable();

  // Pass through insert executor.
  storage::Tuple *tuple;
  tuple = ExecutorTestsUtil::GetNullTuple(table);

  auto project_info = MakeProjectInfoFromTuple(tuple);

  planner::InsertNode node(table, project_info);
  executor::InsertExecutor executor(&node, context.get());

  try{
    executor.Execute();
  }
  catch(ConstraintException& ce){
    std::cout << ce.what();
  }

  tuple->FreeUninlinedData();
  delete tuple;

  tuple = ExecutorTestsUtil::GetTuple(table, ++tuple_id);
  project_info = MakeProjectInfoFromTuple(tuple);
  planner::InsertNode node2(table, project_info);
  executor::InsertExecutor executor2(&node2, context.get());
  executor2.Execute();

  try{
    executor2.Execute();
  }
  catch(ConstraintException& ce){
    std::cout << ce.what();
  }

  tuple->FreeUninlinedData();
  delete tuple;

  txn_manager.CommitTransaction(txn);

  std::cout << "Start tests \n";

  LaunchParallelTest(4, InsertTuple, table);
  //std::cout << (*table);

  LaunchParallelTest(4, UpdateTuple, table);
  //std::cout << (*table);

  LaunchParallelTest(4, DeleteTuple, table);
  //std::cout << (*table);

  // PRIMARY KEY
  auto pkey_index = table->GetIndex(0);
  std::vector<catalog::Column> columns;

  columns.push_back(ExecutorTestsUtil::GetColumnInfo(0));
  catalog::Schema *key_schema = new catalog::Schema(columns);
  storage::Tuple *key1 = new storage::Tuple(key_schema, true);
  storage::Tuple *key2 = new storage::Tuple(key_schema, true);

  key1->SetValue(0, ValueFactory::GetIntegerValue(10));
  key2->SetValue(0, ValueFactory::GetIntegerValue(100));

  auto pkey_list = pkey_index->GetLocationsForKeyBetween(key1, key2);
  std::cout << "PKEY INDEX :: Entries : " << pkey_list.size() << "\n";

  delete key1;
  delete key2;
  delete key_schema;

  // SEC KEY
  auto sec_index = table->GetIndex(1);

  columns.clear();
  columns.push_back(ExecutorTestsUtil::GetColumnInfo(0));
  columns.push_back(ExecutorTestsUtil::GetColumnInfo(1));
  key_schema = new catalog::Schema(columns);

  storage::Tuple *key3 = new storage::Tuple(key_schema, true);
  storage::Tuple *key4 = new storage::Tuple(key_schema, true);

  key3->SetValue(0, ValueFactory::GetIntegerValue(10));
  key3->SetValue(1, ValueFactory::GetIntegerValue(11));
  key4->SetValue(0, ValueFactory::GetIntegerValue(100));
  key4->SetValue(1, ValueFactory::GetIntegerValue(101));

  auto sec_list = sec_index->GetLocationsForKeyBetween(key3, key4);
  std::cout << "SEC INDEX :: Entries : " << sec_list.size() << "\n";

  delete key3;
  delete key4;
  delete key_schema;

  delete table;
}

// Insert a logical tile into a table
TEST(MutateTests, InsertTest) {

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  // We are going to insert a tile group into a table in this test
  std::unique_ptr<storage::DataTable> source_data_table(ExecutorTestsUtil::CreateAndPopulateTable());
  std::unique_ptr<storage::DataTable> dest_data_table(ExecutorTestsUtil::CreateTable());
  const std::vector<storage::Tuple *> tuples;

  EXPECT_EQ(source_data_table->GetTileGroupCount(), 3);
  EXPECT_EQ(dest_data_table->GetTileGroupCount(), 1);

  planner::InsertNode node(dest_data_table.get(), nullptr);
  executor::InsertExecutor executor(&node, context.get());

  MockExecutor child_executor;
  executor.AddChild(&child_executor);

  // Uneventful init...
  EXPECT_CALL(child_executor, DInit())
  .WillOnce(Return(true));

  // Will return one tile.
  EXPECT_CALL(child_executor, DExecute())
  .WillOnce(Return(true))
  .WillOnce(Return(false));

  auto physical_tile = source_data_table->GetTileGroup(0)->GetTile(0);
  std::vector<storage::Tile *> physical_tiles;
  physical_tiles.push_back(physical_tile);

  std::unique_ptr<executor::LogicalTile> source_logical_tile1(
      executor::LogicalTileFactory::WrapTiles(physical_tiles, false));

  EXPECT_CALL(child_executor, GetOutput())
  .WillOnce(Return(source_logical_tile1.release()));

  EXPECT_TRUE(executor.Init());

  EXPECT_TRUE(executor.Execute());
  EXPECT_FALSE(executor.Execute());

  txn_manager.CommitTransaction(txn);

  // We have inserted all the tuples in this logical tile
  EXPECT_EQ(dest_data_table->GetTileGroupCount(), 1);
}

} // namespace test
} // namespace peloton
<commit_msg>temporarily fix double free problem in mutate test. Need to look into this case and define the free behaviors for freeing unlined data. The mutate test still fails due to other problems when parallel test is enabled<commit_after>/**
 * @brief Test cases for insert node.
 *
 * Copyright(c) 2015, CMU
 */

#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "gtest/gtest.h"
#include "gmock/gmock.h"

#include "backend/catalog/schema.h"
#include "backend/common/value_factory.h"

#include "backend/executor/delete_executor.h"
#include "backend/executor/insert_executor.h"
#include "backend/executor/seq_scan_executor.h"
#include "backend/executor/update_executor.h"
#include "backend/executor/logical_tile_factory.h"
#include "backend/expression/expression_util.h"
#include "backend/expression/abstract_expression.h"
#include "backend/expression/expression.h"
#include "backend/planner/delete_node.h"
#include "backend/planner/insert_node.h"
#include "backend/planner/seq_scan_node.h"
#include "backend/planner/update_node.h"
#include "backend/storage/tile_group.h"
#include "backend/storage/table_factory.h"

#include "executor_tests_util.h"
#include "executor/mock_executor.h"
#include "harness.h"

#include <atomic>

using ::testing::NotNull;
using ::testing::Return;

namespace peloton {
namespace test {

//===------------------------------===//
// Utility
//===------------------------------===//

/**
 * Cook a ProjectInfo object from a tuple.
 * Simply use a ConstantValueExpression for each attribute.
 */
planner::ProjectInfo* MakeProjectInfoFromTuple(const storage::Tuple* tuple){
  planner::ProjectInfo::TargetList target_list;
  planner::ProjectInfo::DirectMapList direct_map_list;
  for(oid_t col_id = START_OID; col_id < tuple->GetColumnCount(); col_id++) {
    auto value = tuple->GetValue(col_id);
    auto expression = expression::ConstantValueFactory(value);
    target_list.emplace_back(col_id, expression);
  }

  return new planner::ProjectInfo(target_list, direct_map_list);
}

//===--------------------------------------------------------------------===//
// Mutator Tests
//===--------------------------------------------------------------------===//

std::atomic<int> tuple_id;
std::atomic<int> delete_tuple_id;

void InsertTuple(storage::DataTable *table){

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  auto tuple = ExecutorTestsUtil::GetTuple(table, ++tuple_id);

  auto project_info = MakeProjectInfoFromTuple(tuple);

  planner::InsertNode node(table, project_info);
  executor::InsertExecutor executor(&node, context.get());
  executor.Execute();

  //tuple->FreeUninlinedData();  // double freeed
  delete tuple;

  txn_manager.CommitTransaction(txn);
}

void UpdateTuple(storage::DataTable *table){

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  // Update
  Value update_val = ValueFactory::GetDoubleValue(23.5);

  planner::ProjectInfo::TargetList target_list;
  planner::ProjectInfo::DirectMapList direct_map_list;
  target_list.emplace_back(2, expression::ConstantValueFactory(update_val));
  std::cout << target_list.at(0).first << std::endl;

  planner::UpdateNode update_node(table, new planner::ProjectInfo(target_list, direct_map_list));
  executor::UpdateExecutor update_executor(&update_node, context.get());

  // Predicate

  // WHERE ATTR_0 < 60
  expression::TupleValueExpression *tup_val_exp =
      new expression::TupleValueExpression(0, 0, std::string("tablename"), std::string("colname"));
  expression::ConstantValueExpression *const_val_exp =
      new expression::ConstantValueExpression(ValueFactory::GetIntegerValue(60));
  auto predicate =
      new expression::ComparisonExpression<expression::CmpLt>(EXPRESSION_TYPE_COMPARE_LT, tup_val_exp, const_val_exp);

  // Seq scan
  std::vector<oid_t> column_ids = { 0 };
  planner::SeqScanNode seq_scan_node(
      table,
      predicate,
      column_ids);
  executor::SeqScanExecutor seq_scan_executor(&seq_scan_node, context.get());

  // Parent-Child relationship
  update_node.AddChild(&seq_scan_node);
  update_executor.AddChild(&seq_scan_executor);

  EXPECT_TRUE(update_executor.Init());
  EXPECT_TRUE(update_executor.Execute());

  txn_manager.CommitTransaction(txn);
}

void DeleteTuple(storage::DataTable *table){

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  std::vector<storage::Tuple *> tuples;

  // Delete
  planner::DeleteNode delete_node(table, false);
  executor::DeleteExecutor delete_executor(&delete_node, context.get());

  // Predicate

  // WHERE ATTR_0 < 90
  expression::TupleValueExpression *tup_val_exp =
      new expression::TupleValueExpression(0, 0, std::string("tablename"), std::string("colname"));
  expression::ConstantValueExpression *const_val_exp =
      new expression::ConstantValueExpression(ValueFactory::GetIntegerValue(90));
  auto predicate =
      new expression::ComparisonExpression<expression::CmpLt>(EXPRESSION_TYPE_COMPARE_LT, tup_val_exp, const_val_exp);

  // Seq scan
  std::vector<oid_t> column_ids = { 0 };
  planner::SeqScanNode seq_scan_node(
      table,
      predicate,
      column_ids);
  executor::SeqScanExecutor seq_scan_executor(&seq_scan_node, context.get());

  // Parent-Child relationship
  delete_node.AddChild(&seq_scan_node);
  delete_executor.AddChild(&seq_scan_executor);

  EXPECT_TRUE(delete_executor.Init());
  EXPECT_TRUE(delete_executor.Execute());

  txn_manager.CommitTransaction(txn);
}

TEST(MutateTests, StressTests) {

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();

  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  // Create insert node for this test.
  storage::DataTable *table = ExecutorTestsUtil::CreateTable();

  // Pass through insert executor.
  storage::Tuple *tuple;
  tuple = ExecutorTestsUtil::GetNullTuple(table);

  auto project_info = MakeProjectInfoFromTuple(tuple);

  planner::InsertNode node(table, project_info);
  executor::InsertExecutor executor(&node, context.get());

  try{
    executor.Execute();
  }
  catch(ConstraintException& ce){
    std::cout << ce.what();
  }

  tuple->FreeUninlinedData();
  delete tuple;

  tuple = ExecutorTestsUtil::GetTuple(table, ++tuple_id);
  project_info = MakeProjectInfoFromTuple(tuple);
  planner::InsertNode node2(table, project_info);
  executor::InsertExecutor executor2(&node2, context.get());
  executor2.Execute();

  try{
    executor2.Execute();
  }
  catch(ConstraintException& ce){
    std::cout << ce.what();
  }

  //tuple->FreeUninlinedData(); // Double freed
  delete tuple;

  txn_manager.CommitTransaction(txn);

  std::cout << "Start tests \n";

  LaunchParallelTest(4, InsertTuple, table);
  //std::cout << (*table);

  LaunchParallelTest(4, UpdateTuple, table);
  //std::cout << (*table);

  LaunchParallelTest(4, DeleteTuple, table);
  //std::cout << (*table);

  // PRIMARY KEY
  auto pkey_index = table->GetIndex(0);
  std::vector<catalog::Column> columns;

  columns.push_back(ExecutorTestsUtil::GetColumnInfo(0));
  catalog::Schema *key_schema = new catalog::Schema(columns);
  storage::Tuple *key1 = new storage::Tuple(key_schema, true);
  storage::Tuple *key2 = new storage::Tuple(key_schema, true);

  key1->SetValue(0, ValueFactory::GetIntegerValue(10));
  key2->SetValue(0, ValueFactory::GetIntegerValue(100));

  auto pkey_list = pkey_index->GetLocationsForKeyBetween(key1, key2);
  std::cout << "PKEY INDEX :: Entries : " << pkey_list.size() << "\n";

  delete key1;
  delete key2;
  delete key_schema;

  // SEC KEY
  auto sec_index = table->GetIndex(1);

  columns.clear();
  columns.push_back(ExecutorTestsUtil::GetColumnInfo(0));
  columns.push_back(ExecutorTestsUtil::GetColumnInfo(1));
  key_schema = new catalog::Schema(columns);

  storage::Tuple *key3 = new storage::Tuple(key_schema, true);
  storage::Tuple *key4 = new storage::Tuple(key_schema, true);

  key3->SetValue(0, ValueFactory::GetIntegerValue(10));
  key3->SetValue(1, ValueFactory::GetIntegerValue(11));
  key4->SetValue(0, ValueFactory::GetIntegerValue(100));
  key4->SetValue(1, ValueFactory::GetIntegerValue(101));

  auto sec_list = sec_index->GetLocationsForKeyBetween(key3, key4);
  std::cout << "SEC INDEX :: Entries : " << sec_list.size() << "\n";

  delete key3;
  delete key4;
  delete key_schema;

  delete table;
}

// Insert a logical tile into a table
TEST(MutateTests, InsertTest) {

  auto& txn_manager = concurrency::TransactionManager::GetInstance();
  auto txn = txn_manager.BeginTransaction();
  std::unique_ptr<executor::ExecutorContext> context(
      new executor::ExecutorContext(txn));

  // We are going to insert a tile group into a table in this test
  std::unique_ptr<storage::DataTable> source_data_table(ExecutorTestsUtil::CreateAndPopulateTable());
  std::unique_ptr<storage::DataTable> dest_data_table(ExecutorTestsUtil::CreateTable());
  const std::vector<storage::Tuple *> tuples;

  EXPECT_EQ(source_data_table->GetTileGroupCount(), 3);
  EXPECT_EQ(dest_data_table->GetTileGroupCount(), 1);

  planner::InsertNode node(dest_data_table.get(), nullptr);
  executor::InsertExecutor executor(&node, context.get());

  MockExecutor child_executor;
  executor.AddChild(&child_executor);

  // Uneventful init...
  EXPECT_CALL(child_executor, DInit())
  .WillOnce(Return(true));

  // Will return one tile.
  EXPECT_CALL(child_executor, DExecute())
  .WillOnce(Return(true))
  .WillOnce(Return(false));

  auto physical_tile = source_data_table->GetTileGroup(0)->GetTile(0);
  std::vector<storage::Tile *> physical_tiles;
  physical_tiles.push_back(physical_tile);

  std::unique_ptr<executor::LogicalTile> source_logical_tile1(
      executor::LogicalTileFactory::WrapTiles(physical_tiles, false));

  EXPECT_CALL(child_executor, GetOutput())
  .WillOnce(Return(source_logical_tile1.release()));

  EXPECT_TRUE(executor.Init());

  EXPECT_TRUE(executor.Execute());
  EXPECT_FALSE(executor.Execute());

  txn_manager.CommitTransaction(txn);

  // We have inserted all the tuples in this logical tile
  EXPECT_EQ(dest_data_table->GetTileGroupCount(), 1);
}

} // namespace test
} // namespace peloton
<|endoftext|>