<commit_before>/******************************************************************************
 * Copyright (C) 2016 Felix Rohrbach <kde@fxrh.de>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "syncjob.h"

#include <QtCore/QElapsedTimer>

using namespace QMatrixClient;

static size_t jobId = 0;

SyncJob::SyncJob(const ConnectionData* connection, const QString& since,
                 const QString& filter, int timeout, const QString& presence)
    : BaseJob(connection, HttpVerb::Get, QString("SyncJob-%1").arg(++jobId),
              "_matrix/client/r0/sync")
{
    setLoggingCategory(SYNCJOB);
    QUrlQuery query;
    if( !filter.isEmpty() )
        query.addQueryItem("filter", filter);
    if( !presence.isEmpty() )
        query.addQueryItem("set_presence", presence);
    if( timeout >= 0 )
        query.addQueryItem("timeout", QString::number(timeout));
    if( !since.isEmpty() )
        query.addQueryItem("since", since);
    setRequestQuery(query);

    setMaxRetries(std::numeric_limits<int>::max());
}

QString SyncData::nextBatch() const
{
    return nextBatch_;
}

SyncDataList&& SyncData::takeRoomData()
{
    return std::move(roomData);
}

BaseJob::Status SyncJob::parseJson(const QJsonDocument& data)
{
    return d.parseJson(data);
}

BaseJob::Status SyncData::parseJson(const QJsonDocument &data) {
    QElapsedTimer et; et.start();
    QJsonObject json = data.object();
    nextBatch_ = json.value("next_batch").toString();
    // TODO: presence
    // TODO: account_data
    QJsonObject rooms = json.value("rooms").toObject();

    const struct { QString jsonKey; JoinState enumVal; } roomStates[]
    {
        { "join", JoinState::Join },
        { "invite", JoinState::Invite },
        { "leave", JoinState::Leave }
    };
    for (auto roomState: roomStates)
    {
        const QJsonObject rs = rooms.value(roomState.jsonKey).toObject();
        // We have a Qt container on the right and an STL one on the left
        roomData.reserve(static_cast<size_t>(rs.size()));
        for( auto rkey: rs.keys() )
            roomData.emplace_back(rkey, roomState.enumVal, rs[rkey].toObject());
    }
    qCDebug(PROFILER) << "*** SyncData::parseJson():" << et.elapsed() << "ms";
    return BaseJob::Success;
}

SyncRoomData::SyncRoomData(const QString& roomId_, JoinState joinState_,
                           const QJsonObject& room_)
    : roomId(roomId_)
    , joinState(joinState_)
    , state("state")
    , timeline("timeline")
    , ephemeral("ephemeral")
    , accountData("account_data")
    , inviteState("invite_state")
{
    switch (joinState) {
        case JoinState::Invite:
            inviteState.fromJson(room_);
            break;
        case JoinState::Join:
            state.fromJson(room_);
            timeline.fromJson(room_);
            ephemeral.fromJson(room_);
            accountData.fromJson(room_);
            break;
        case JoinState::Leave:
            state.fromJson(room_);
            timeline.fromJson(room_);
            break;
    default:
        qCWarning(SYNCJOB) << "SyncRoomData: Unknown JoinState value, ignoring:" << int(joinState);
    }

    QJsonObject timeline = room_.value("timeline").toObject();
    timelineLimited = timeline.value("limited").toBool();
    timelinePrevBatch = timeline.value("prev_batch").toString();

    QJsonObject unread = room_.value("unread_notifications").toObject();
    highlightCount = unread.value("highlight_count").toInt();
    notificationCount = unread.value("notification_count").toInt();
    qCDebug(SYNCJOB) << "Highlights: " << highlightCount << " Notifications:" << notificationCount;
}
<commit_msg>Minor optimisations in sync data parsing<commit_after>/******************************************************************************
 * Copyright (C) 2016 Felix Rohrbach <kde@fxrh.de>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "syncjob.h"

#include <QtCore/QElapsedTimer>

using namespace QMatrixClient;

static size_t jobId = 0;

SyncJob::SyncJob(const ConnectionData* connection, const QString& since,
                 const QString& filter, int timeout, const QString& presence)
    : BaseJob(connection, HttpVerb::Get, QString("SyncJob-%1").arg(++jobId),
              "_matrix/client/r0/sync")
{
    setLoggingCategory(SYNCJOB);
    QUrlQuery query;
    if( !filter.isEmpty() )
        query.addQueryItem("filter", filter);
    if( !presence.isEmpty() )
        query.addQueryItem("set_presence", presence);
    if( timeout >= 0 )
        query.addQueryItem("timeout", QString::number(timeout));
    if( !since.isEmpty() )
        query.addQueryItem("since", since);
    setRequestQuery(query);

    setMaxRetries(std::numeric_limits<int>::max());
}

QString SyncData::nextBatch() const
{
    return nextBatch_;
}

SyncDataList&& SyncData::takeRoomData()
{
    return std::move(roomData);
}

BaseJob::Status SyncJob::parseJson(const QJsonDocument& data)
{
    return d.parseJson(data);
}

BaseJob::Status SyncData::parseJson(const QJsonDocument &data)
{
    QElapsedTimer et; et.start();

    QJsonObject json = data.object();
    nextBatch_ = json.value("next_batch").toString();
    // TODO: presence
    // TODO: account_data
    QJsonObject rooms = json.value("rooms").toObject();

    static const struct { QString jsonKey; JoinState enumVal; } roomStates[]
    {
        { "join", JoinState::Join },
        { "invite", JoinState::Invite },
        { "leave", JoinState::Leave }
    };
    for (const auto& roomState: roomStates)
    {
        const QJsonObject rs = rooms.value(roomState.jsonKey).toObject();
        // We have a Qt container on the right and an STL one on the left
        roomData.reserve(static_cast<size_t>(rs.size()));
        for(auto roomIt = rs.begin(); roomIt != rs.end(); ++roomIt)
            roomData.emplace_back(roomIt.key(), roomState.enumVal,
                                  roomIt.value().toObject());
    }
    qCDebug(PROFILER) << "*** SyncData::parseJson():" << et.elapsed() << "ms";
    return BaseJob::Success;
}

SyncRoomData::SyncRoomData(const QString& roomId_, JoinState joinState_,
                           const QJsonObject& room_)
    : roomId(roomId_)
    , joinState(joinState_)
    , state("state")
    , timeline("timeline")
    , ephemeral("ephemeral")
    , accountData("account_data")
    , inviteState("invite_state")
{
    switch (joinState) {
        case JoinState::Invite:
            inviteState.fromJson(room_);
            break;
        case JoinState::Join:
            state.fromJson(room_);
            timeline.fromJson(room_);
            ephemeral.fromJson(room_);
            accountData.fromJson(room_);
            break;
        case JoinState::Leave:
            state.fromJson(room_);
            timeline.fromJson(room_);
            break;
    default:
        qCWarning(SYNCJOB) << "SyncRoomData: Unknown JoinState value, ignoring:" << int(joinState);
    }

    QJsonObject timeline = room_.value("timeline").toObject();
    timelineLimited = timeline.value("limited").toBool();
    timelinePrevBatch = timeline.value("prev_batch").toString();

    QJsonObject unread = room_.value("unread_notifications").toObject();
    highlightCount = unread.value("highlight_count").toInt();
    notificationCount = unread.value("notification_count").toInt();
    qCDebug(SYNCJOB) << "Highlights: " << highlightCount << " Notifications:" << notificationCount;
}
<|endoftext|>


<commit_before>#include <iostream>

#include "filelib.h"
#include "decoder.h"
#include "ff_register.h"
#include "verbose.h"

using namespace std;

int main(int argc, char** argv) {
  register_feature_functions();
  Decoder decoder(argc, argv);

  const string input = decoder.GetConf()["input"].as<string>();
  const bool show_feature_dictionary = decoder.GetConf().count("show_feature_dictionary");
  if (!SILENT) cerr << "Reading input from " << ((input == "-") ? "STDIN" : input.c_str()) << endl;
  ReadFile in_read(input);
  istream *in = in_read.stream();
  assert(*in);

  string buf;
#ifdef CP_TIME
    clock_t time_cp(0);//, end_cp;
#endif
  while(*in) {
    getline(*in, buf);
    if (buf.empty()) continue;
    decoder.Decode(buf);
  }
#ifdef CP_TIME
    cerr << "Time required for Cube Pruning execution: "
    << CpTime::Get()
    << " seconds." << "\n\n";
#endif
  if (show_feature_dictionary) {
    int num = FD::NumFeats();
    for (int i = 1; i < num; ++i) {
      cout << FD::Convert(i) << endl;
    }
  }
  return 0;
}

<commit_msg>Add memory size logging<commit_after>#include <iostream>

#include "filelib.h"
#include "decoder.h"
#include "ff_register.h"
#include "verbose.h"
#include "util/usage.hh"

using namespace std;

int main(int argc, char** argv) {
  register_feature_functions();
  Decoder decoder(argc, argv);

  const string input = decoder.GetConf()["input"].as<string>();
  const bool show_feature_dictionary = decoder.GetConf().count("show_feature_dictionary");
  if (!SILENT) cerr << "Reading input from " << ((input == "-") ? "STDIN" : input.c_str()) << endl;
  ReadFile in_read(input);
  istream *in = in_read.stream();
  assert(*in);

  string buf;
#ifdef CP_TIME
    clock_t time_cp(0);//, end_cp;
#endif
  while(*in) {
    getline(*in, buf);
    if (buf.empty()) continue;
    decoder.Decode(buf);
  }
#ifdef CP_TIME
    cerr << "Time required for Cube Pruning execution: "
    << CpTime::Get()
    << " seconds." << "\n\n";
#endif
  if (show_feature_dictionary) {
    int num = FD::NumFeats();
    for (int i = 1; i < num; ++i) {
      cout << FD::Convert(i) << endl;
    }
  }
  util::PrintUsage(std::cerr);
  return 0;
}

<|endoftext|>


<commit_before>#include <iostream>
#include <string>
#include <vector>
#include <tuple>

#include "acmacs-base/argv.hh"
#include "acmacs-base/named-type.hh"
#include "acmacs-base/string-join.hh"
#include "acmacs-base/string-split.hh"
#include "acmacs-base/filesystem.hh"
#include "acmacs-base/read-file.hh"
#include "acmacs-chart-2/factory-import.hh"
#include "acmacs-chart-2/factory-export.hh"
#include "acmacs-chart-2/chart.hh"
#include "acmacs-chart-2/chart-modify.hh"

// ----------------------------------------------------------------------

inline bool is_acmacs_file(const fs::path& path)
{
    if (path.extension() == ".ace")
        return true;
    if (path.extension() == ".bz2" && path.stem().extension() == ".acd1")
        return true;
    return false;
}

// ----------------------------------------------------------------------

class SerumIds
{
  public:
    using Name = acmacs::virus::name_t;
    using Passage = acmacs::virus::Passage;
    using Reassortant = acmacs::virus::Reassortant;
    using Annotations = acmacs::chart::Annotations;
    using SerumId = acmacs::chart::SerumId;
    using SerumIdRoot = acmacs::named_string_t<struct SerumIdRootTag>;
    using SerumEntry = std::tuple<Name, Reassortant, Annotations, SerumId, Passage>;
    using TableEntry = std::tuple<acmacs::virus::type_subtype_t, acmacs::chart::Lab, acmacs::chart::Assay, acmacs::chart::RbcSpecies, acmacs::chart::TableDate>;
    using Entry = std::tuple<SerumIdRoot, SerumId, Name, Reassortant, Annotations, acmacs::virus::type_subtype_t, acmacs::chart::Lab, acmacs::chart::Assay, acmacs::chart::RbcSpecies,
                             acmacs::chart::TableDate, Passage>;
    using Entries = std::vector<Entry>;
    using EntryPtr = typename Entries::const_iterator;
    using PerSerumIdEntry = std::tuple<EntryPtr, EntryPtr>;
    using PerSerumIdRootEntry = std::tuple<EntryPtr, EntryPtr, std::vector<PerSerumIdEntry>>;
    using PerSerumIdRootEntries = std::vector<PerSerumIdRootEntry>;

    SerumIds() = default;

    size_t size() const { return data_.size(); }

    // returns number of files processed
    size_t scan_directory(std::string dirname)
    {
        size_t charts_processed = 0;
        for (auto& entry : fs::directory_iterator(dirname)) {
            if (const auto pathname = entry.path(); entry.is_regular_file() && is_acmacs_file(pathname)) {
                auto chart = acmacs::chart::import_from_file(pathname);
                std::tuple table(chart->info()->virus_type(), chart->info()->lab(), chart->info()->assay(), chart->info()->rbc_species(), chart->info()->date());
                auto sera = chart->sera();
                for (auto serum : *sera)
                    add({serum->name(), serum->reassortant(), serum->annotations(), serum->serum_id(), serum->passage()}, table);
                ++charts_processed;
            }
        }
        sort();
        scan();
        return charts_processed;
    }

    void print(bool print_good) const
    {
        const bool show_assay = std::get<acmacs::virus::type_subtype_t>(*std::get<0>(per_root_.front())) == acmacs::virus::type_subtype_t{"A(H3N2)"};
        const bool show_rbc = show_assay;
        for (const auto& per_root_entry : per_root_) {
            const auto name = make_name(std::get<0>(per_root_entry)), name_last = make_name(std::get<1>(per_root_entry) - 1);
            if (const bool good = std::get<2>(per_root_entry).size() == 1; !good || print_good) {
                std::cout << fmt::format("{} {}\n", std::get<SerumIdRoot>(*std::get<0>(per_root_entry)), std::get<1>(per_root_entry) - std::get<0>(per_root_entry));
                for (const auto& per_serum_id_entry : std::get<2>(per_root_entry)) {
                    const auto tabs = tables(std::get<0>(per_serum_id_entry), std::get<1>(per_serum_id_entry), show_assay, show_rbc);
                    std::cout << fmt::format("    {} {} [{}]", std::get<SerumId>(*std::get<0>(per_serum_id_entry)), tabs.size(), make_name(std::get<0>(per_serum_id_entry)));
                    for (const auto& table : tabs)
                        std::cout << ' ' << table;
                    std::cout << '\n';
                }
                if (!good) {
                    const auto& sids = std::get<2>(per_root_entry);
                    if (std::get<SerumId>(*std::get<0>(sids.front())) != std::get<SerumId>(*std::get<0>(sids.back()))) {
                        const auto sid = std::max_element(sids.begin(), sids.end(),
                                                          [](const auto& e1, const auto& e2) -> bool { return (std::get<1>(e1) - std::get<0>(e1)) < (std::get<1>(e2) - std::get<0>(e2)); });
                        for (auto ep = sids.begin(); ep != sids.end(); ++ep) {
                            if (ep != sid) {
                                std::cout << "  --fix ";
                                const auto sid1 = std::get<SerumId>(*std::get<0>(*ep)), sid2 = std::get<SerumId>(*std::get<0>(*sid));
                                if ((std::get<1>(*ep) - std::get<0>(*ep)) == (std::get<1>(*sid) - std::get<0>(*sid)) && sid2.size() < sid1.size())
                                    std::cout << *sid2 << '^' << *sid1;
                                else
                                    std::cout << *sid1 << '^' << *sid2;
                                std::cout << '\n';
                            }
                        }
                    }
                }
            }
        }
    }

  private:
    Entries data_;
    PerSerumIdRootEntries per_root_;

    void sort() { std::sort(data_.begin(), data_.end()); /* data_.erase(std::unique(data_.begin(), data_.end()), data_.end()); */ }

    void add(const SerumEntry& serum, const TableEntry& table)
    {
        data_.emplace_back(serum_id_root(serum, table), std::get<SerumId>(serum), std::get<Name>(serum), std::get<Reassortant>(serum), std::get<Annotations>(serum),
                           std::get<acmacs::virus::type_subtype_t>(table), std::get<acmacs::chart::Lab>(table), std::get<acmacs::chart::Assay>(table), std::get<acmacs::chart::RbcSpecies>(table),
                           std::get<acmacs::chart::TableDate>(table), std::get<Passage>(serum));
    }

    void scan()
    {
        for (EntryPtr entry_ptr = data_.begin(); entry_ptr != data_.end(); ++entry_ptr) {
            if (per_root_.empty() || std::get<SerumIdRoot>(*entry_ptr) != std::get<SerumIdRoot>(*std::get<0>(per_root_.back()))) {
                per_root_.emplace_back(entry_ptr, entry_ptr + 1, std::vector<PerSerumIdEntry>{{entry_ptr, entry_ptr + 1}});
            }
            else {
                std::get<1>(per_root_.back()) = entry_ptr + 1;
                const auto last = std::get<0>(std::get<2>(per_root_.back()).back());
                const auto name = make_name(entry_ptr), name_last = make_name(last);
                if (std::get<SerumId>(*entry_ptr) != std::get<SerumId>(*last) || name != name_last) {
                    std::get<2>(per_root_.back()).emplace_back(entry_ptr, entry_ptr + 1);
                }
                else {
                    std::get<1>(std::get<2>(per_root_.back()).back()) = entry_ptr + 1;
                }
            }
        }
        std::cout << "per_root_ " << per_root_.size() << '\n';
    }

    SerumIdRoot serum_id_root(const SerumEntry& serum, const TableEntry& table) const
    {
        const auto& serum_id = std::get<SerumId>(serum);
        if (std::get<acmacs::chart::Lab>(table) == acmacs::chart::Lab{"MELB"}) {
            if (serum_id.size() > 6 && (serum_id[0] == 'F' || serum_id[0] == 'R' || serum_id[0] == 'A') && serum_id[5] == '-' && serum_id->back() == 'D')
                return SerumIdRoot(serum_id->substr(0, 5));
            else
                return SerumIdRoot(serum_id);
        }
        else
            return SerumIdRoot(serum_id);
    }

    static inline std::string make_name(EntryPtr ptr) { return acmacs::string::join(" ", std::get<Name>(*ptr), std::get<Reassortant>(*ptr), acmacs::string::join(" ", std::get<Annotations>(*ptr))); }

    static inline std::vector<std::string> tables(EntryPtr first, EntryPtr last, bool assay, bool rbc)
    {
        std::vector<std::string> tables(static_cast<size_t>(last - first));
        std::transform(first, last, tables.begin(), [assay, rbc](const auto& entry) {
            std::vector<std::string> fields;
            if (assay)
                fields.push_back(*std::get<acmacs::chart::Assay>(entry));
            if (rbc)
                fields.push_back(*std::get<acmacs::chart::RbcSpecies>(entry));
            fields.push_back(*std::get<acmacs::chart::TableDate>(entry));
            return acmacs::string::join(":", fields);
        });
        std::reverse(tables.begin(), tables.end());
        return tables;
    }

}; // class SerumIds

// ======================================================================

class FixSerumIds
{
  public:
    FixSerumIds(std::string_view program_name) : program_name_(program_name) {}
    void add_fix(std::string_view fix_entry)
    {
        const auto fields = acmacs::string::split(fix_entry, "^");
        data_.emplace_back(fields[0], fields[1]);
    }

    // returns number of files updated
    size_t fix_charts_in_directory(std::string dirname)
    {
        size_t fixed_charts = 0;
        for (auto& entry : fs::directory_iterator(dirname)) {
            if (const auto pathname = entry.path(); entry.is_regular_file() && is_acmacs_file(pathname)) {
                acmacs::chart::ChartModify chart(acmacs::chart::import_from_file(pathname));
                if (fix_in_chart(chart)) {
                    acmacs::file::backup(pathname.native(), acmacs::file::backup_move::yes);
                    acmacs::chart::export_factory(chart, make_export_filename(pathname), program_name_, report_time::no);
                    ++fixed_charts;
                }
            }
        }
        return fixed_charts;
    }

  private:
    std::string_view program_name_;
    std::vector<std::pair<std::string, std::string>> data_;

    bool fix_in_chart(acmacs::chart::ChartModify& chart)
    {
        auto sera = chart.sera_modify();
        bool modified = false;
        for (size_t serum_no = 0; serum_no < sera->size(); ++ serum_no) {
            auto& serum = sera->at(serum_no);
            const auto sid = serum.serum_id();
            for (const auto& fix : data_) {
                if (sid == acmacs::chart::SerumId{fix.first}) {
                    std::cout << fmt::format("{} FIX {} {} {} {} --> {}\n", chart.info()->make_name(), serum.name(), serum.reassortant(), acmacs::string::join(" ", serum.annotations()), serum.serum_id(), fix.second);
                    serum.serum_id(acmacs::chart::SerumId{fix.second});
                    modified = true;
                    break;
                }
            }
        }
        return modified;
    }

    std::string make_export_filename(const fs::path& pathname) const
    {
        if (pathname.extension() == ".ace")
            return pathname;
        if (pathname.extension() == ".bz2" && pathname.stem().extension() == ".acd1")
            return pathname.stem().stem() += ".ace";
        throw std::runtime_error("cannot figure out export filename for: " + pathname.string());
    }

}; // class FixSerumIds

// ======================================================================

using namespace acmacs::argv;

struct Options : public argv
{
    Options(int a_argc, const char* const a_argv[], on_error on_err = on_error::exit) : argv() { parse(a_argc, a_argv, on_err); }

    option<str_array> fix{*this, "fix"};
    // argument<str> period{*this, arg_name{"monthly|yearly|weekly"}, mandatory};
    // argument<Date> start{*this, arg_name{"start-date"}, mandatory};
    // argument<Date> end{*this, arg_name{"end-date"}, mandatory};
    // argument<str> output{*this, arg_name{"output.json"}, dflt{""}};
};

// ----------------------------------------------------------------------

int main(int argc, const char* argv[])
{
    int exit_code = 0;
    try {
        Options opt(argc, argv);
        std::vector<std::pair<std::string, std::string>> fix_data;
        if (opt.fix.has_value()) {
            FixSerumIds fix_serum_ids(opt.program_name());
            for (const auto& fix_entry : opt.fix.get())
                fix_serum_ids.add_fix(fix_entry);
            const auto charts_modified = fix_serum_ids.fix_charts_in_directory(".");
            std::cerr << "INFO: charts modified: " << charts_modified << '\n';
        }
        else {
            SerumIds serum_ids;
            const auto charts_processed = serum_ids.scan_directory(".");
            std::cout << charts_processed << " charts processed\n";
            serum_ids.print(false);
        }
    }
    catch (std::exception& err) {
        std::cerr << "ERROR: " << err.what() << '\n';
        exit_code = 2;
    }
    return exit_code;
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<commit_msg>acmacs::string::join interface improvement<commit_after>#include <iostream>
#include <string>
#include <vector>
#include <tuple>

#include "acmacs-base/argv.hh"
#include "acmacs-base/named-type.hh"
#include "acmacs-base/string-join.hh"
#include "acmacs-base/string-split.hh"
#include "acmacs-base/filesystem.hh"
#include "acmacs-base/read-file.hh"
#include "acmacs-chart-2/factory-import.hh"
#include "acmacs-chart-2/factory-export.hh"
#include "acmacs-chart-2/chart.hh"
#include "acmacs-chart-2/chart-modify.hh"

// ----------------------------------------------------------------------

inline bool is_acmacs_file(const fs::path& path)
{
    if (path.extension() == ".ace")
        return true;
    if (path.extension() == ".bz2" && path.stem().extension() == ".acd1")
        return true;
    return false;
}

// ----------------------------------------------------------------------

class SerumIds
{
  public:
    using Name = acmacs::virus::name_t;
    using Passage = acmacs::virus::Passage;
    using Reassortant = acmacs::virus::Reassortant;
    using Annotations = acmacs::chart::Annotations;
    using SerumId = acmacs::chart::SerumId;
    using SerumIdRoot = acmacs::named_string_t<struct SerumIdRootTag>;
    using SerumEntry = std::tuple<Name, Reassortant, Annotations, SerumId, Passage>;
    using TableEntry = std::tuple<acmacs::virus::type_subtype_t, acmacs::chart::Lab, acmacs::chart::Assay, acmacs::chart::RbcSpecies, acmacs::chart::TableDate>;
    using Entry = std::tuple<SerumIdRoot, SerumId, Name, Reassortant, Annotations, acmacs::virus::type_subtype_t, acmacs::chart::Lab, acmacs::chart::Assay, acmacs::chart::RbcSpecies,
                             acmacs::chart::TableDate, Passage>;
    using Entries = std::vector<Entry>;
    using EntryPtr = typename Entries::const_iterator;
    using PerSerumIdEntry = std::tuple<EntryPtr, EntryPtr>;
    using PerSerumIdRootEntry = std::tuple<EntryPtr, EntryPtr, std::vector<PerSerumIdEntry>>;
    using PerSerumIdRootEntries = std::vector<PerSerumIdRootEntry>;

    SerumIds() = default;

    size_t size() const { return data_.size(); }

    // returns number of files processed
    size_t scan_directory(std::string dirname)
    {
        size_t charts_processed = 0;
        for (auto& entry : fs::directory_iterator(dirname)) {
            if (const auto pathname = entry.path(); entry.is_regular_file() && is_acmacs_file(pathname)) {
                auto chart = acmacs::chart::import_from_file(pathname);
                std::tuple table(chart->info()->virus_type(), chart->info()->lab(), chart->info()->assay(), chart->info()->rbc_species(), chart->info()->date());
                auto sera = chart->sera();
                for (auto serum : *sera)
                    add({serum->name(), serum->reassortant(), serum->annotations(), serum->serum_id(), serum->passage()}, table);
                ++charts_processed;
            }
        }
        sort();
        scan();
        return charts_processed;
    }

    void print(bool print_good) const
    {
        const bool show_assay = std::get<acmacs::virus::type_subtype_t>(*std::get<0>(per_root_.front())) == acmacs::virus::type_subtype_t{"A(H3N2)"};
        const bool show_rbc = show_assay;
        for (const auto& per_root_entry : per_root_) {
            const auto name = make_name(std::get<0>(per_root_entry)), name_last = make_name(std::get<1>(per_root_entry) - 1);
            if (const bool good = std::get<2>(per_root_entry).size() == 1; !good || print_good) {
                std::cout << fmt::format("{} {}\n", std::get<SerumIdRoot>(*std::get<0>(per_root_entry)), std::get<1>(per_root_entry) - std::get<0>(per_root_entry));
                for (const auto& per_serum_id_entry : std::get<2>(per_root_entry)) {
                    const auto tabs = tables(std::get<0>(per_serum_id_entry), std::get<1>(per_serum_id_entry), show_assay, show_rbc);
                    std::cout << fmt::format("    {} {} [{}]", std::get<SerumId>(*std::get<0>(per_serum_id_entry)), tabs.size(), make_name(std::get<0>(per_serum_id_entry)));
                    for (const auto& table : tabs)
                        std::cout << ' ' << table;
                    std::cout << '\n';
                }
                if (!good) {
                    const auto& sids = std::get<2>(per_root_entry);
                    if (std::get<SerumId>(*std::get<0>(sids.front())) != std::get<SerumId>(*std::get<0>(sids.back()))) {
                        const auto sid = std::max_element(sids.begin(), sids.end(),
                                                          [](const auto& e1, const auto& e2) -> bool { return (std::get<1>(e1) - std::get<0>(e1)) < (std::get<1>(e2) - std::get<0>(e2)); });
                        for (auto ep = sids.begin(); ep != sids.end(); ++ep) {
                            if (ep != sid) {
                                std::cout << "  --fix ";
                                const auto sid1 = std::get<SerumId>(*std::get<0>(*ep)), sid2 = std::get<SerumId>(*std::get<0>(*sid));
                                if ((std::get<1>(*ep) - std::get<0>(*ep)) == (std::get<1>(*sid) - std::get<0>(*sid)) && sid2.size() < sid1.size())
                                    std::cout << *sid2 << '^' << *sid1;
                                else
                                    std::cout << *sid1 << '^' << *sid2;
                                std::cout << '\n';
                            }
                        }
                    }
                }
            }
        }
    }

  private:
    Entries data_;
    PerSerumIdRootEntries per_root_;

    void sort() { std::sort(data_.begin(), data_.end()); /* data_.erase(std::unique(data_.begin(), data_.end()), data_.end()); */ }

    void add(const SerumEntry& serum, const TableEntry& table)
    {
        data_.emplace_back(serum_id_root(serum, table), std::get<SerumId>(serum), std::get<Name>(serum), std::get<Reassortant>(serum), std::get<Annotations>(serum),
                           std::get<acmacs::virus::type_subtype_t>(table), std::get<acmacs::chart::Lab>(table), std::get<acmacs::chart::Assay>(table), std::get<acmacs::chart::RbcSpecies>(table),
                           std::get<acmacs::chart::TableDate>(table), std::get<Passage>(serum));
    }

    void scan()
    {
        for (EntryPtr entry_ptr = data_.begin(); entry_ptr != data_.end(); ++entry_ptr) {
            if (per_root_.empty() || std::get<SerumIdRoot>(*entry_ptr) != std::get<SerumIdRoot>(*std::get<0>(per_root_.back()))) {
                per_root_.emplace_back(entry_ptr, entry_ptr + 1, std::vector<PerSerumIdEntry>{{entry_ptr, entry_ptr + 1}});
            }
            else {
                std::get<1>(per_root_.back()) = entry_ptr + 1;
                const auto last = std::get<0>(std::get<2>(per_root_.back()).back());
                const auto name = make_name(entry_ptr), name_last = make_name(last);
                if (std::get<SerumId>(*entry_ptr) != std::get<SerumId>(*last) || name != name_last) {
                    std::get<2>(per_root_.back()).emplace_back(entry_ptr, entry_ptr + 1);
                }
                else {
                    std::get<1>(std::get<2>(per_root_.back()).back()) = entry_ptr + 1;
                }
            }
        }
        std::cout << "per_root_ " << per_root_.size() << '\n';
    }

    SerumIdRoot serum_id_root(const SerumEntry& serum, const TableEntry& table) const
    {
        const auto& serum_id = std::get<SerumId>(serum);
        if (std::get<acmacs::chart::Lab>(table) == acmacs::chart::Lab{"MELB"}) {
            if (serum_id.size() > 6 && (serum_id[0] == 'F' || serum_id[0] == 'R' || serum_id[0] == 'A') && serum_id[5] == '-' && serum_id->back() == 'D')
                return SerumIdRoot(serum_id->substr(0, 5));
            else
                return SerumIdRoot(serum_id);
        }
        else
            return SerumIdRoot(serum_id);
    }

    static inline std::string make_name(EntryPtr ptr) { return acmacs::string::join(acmacs::string::join_space, std::get<Name>(*ptr), std::get<Reassortant>(*ptr), acmacs::string::join(acmacs::string::join_space, std::get<Annotations>(*ptr))); }

    static inline std::vector<std::string> tables(EntryPtr first, EntryPtr last, bool assay, bool rbc)
    {
        std::vector<std::string> tables(static_cast<size_t>(last - first));
        std::transform(first, last, tables.begin(), [assay, rbc](const auto& entry) {
            std::vector<std::string> fields;
            if (assay)
                fields.push_back(*std::get<acmacs::chart::Assay>(entry));
            if (rbc)
                fields.push_back(*std::get<acmacs::chart::RbcSpecies>(entry));
            fields.push_back(*std::get<acmacs::chart::TableDate>(entry));
            return acmacs::string::join(acmacs::string::join_colon, fields);
        });
        std::reverse(tables.begin(), tables.end());
        return tables;
    }

}; // class SerumIds

// ======================================================================

class FixSerumIds
{
  public:
    FixSerumIds(std::string_view program_name) : program_name_(program_name) {}
    void add_fix(std::string_view fix_entry)
    {
        const auto fields = acmacs::string::split(fix_entry, "^");
        data_.emplace_back(fields[0], fields[1]);
    }

    // returns number of files updated
    size_t fix_charts_in_directory(std::string dirname)
    {
        size_t fixed_charts = 0;
        for (auto& entry : fs::directory_iterator(dirname)) {
            if (const auto pathname = entry.path(); entry.is_regular_file() && is_acmacs_file(pathname)) {
                acmacs::chart::ChartModify chart(acmacs::chart::import_from_file(pathname));
                if (fix_in_chart(chart)) {
                    acmacs::file::backup(pathname.native(), acmacs::file::backup_move::yes);
                    acmacs::chart::export_factory(chart, make_export_filename(pathname), program_name_, report_time::no);
                    ++fixed_charts;
                }
            }
        }
        return fixed_charts;
    }

  private:
    std::string_view program_name_;
    std::vector<std::pair<std::string, std::string>> data_;

    bool fix_in_chart(acmacs::chart::ChartModify& chart)
    {
        auto sera = chart.sera_modify();
        bool modified = false;
        for (size_t serum_no = 0; serum_no < sera->size(); ++ serum_no) {
            auto& serum = sera->at(serum_no);
            const auto sid = serum.serum_id();
            for (const auto& fix : data_) {
                if (sid == acmacs::chart::SerumId{fix.first}) {
                    std::cout << fmt::format("{} FIX {} {} {} {} --> {}\n", chart.info()->make_name(), serum.name(), serum.reassortant(), acmacs::string::join(acmacs::string::join_space, serum.annotations()), serum.serum_id(), fix.second);
                    serum.serum_id(acmacs::chart::SerumId{fix.second});
                    modified = true;
                    break;
                }
            }
        }
        return modified;
    }

    std::string make_export_filename(const fs::path& pathname) const
    {
        if (pathname.extension() == ".ace")
            return pathname;
        if (pathname.extension() == ".bz2" && pathname.stem().extension() == ".acd1")
            return pathname.stem().stem() += ".ace";
        throw std::runtime_error("cannot figure out export filename for: " + pathname.string());
    }

}; // class FixSerumIds

// ======================================================================

using namespace acmacs::argv;

struct Options : public argv
{
    Options(int a_argc, const char* const a_argv[], on_error on_err = on_error::exit) : argv() { parse(a_argc, a_argv, on_err); }

    option<str_array> fix{*this, "fix"};
    // argument<str> period{*this, arg_name{"monthly|yearly|weekly"}, mandatory};
    // argument<Date> start{*this, arg_name{"start-date"}, mandatory};
    // argument<Date> end{*this, arg_name{"end-date"}, mandatory};
    // argument<str> output{*this, arg_name{"output.json"}, dflt{""}};
};

// ----------------------------------------------------------------------

int main(int argc, const char* argv[])
{
    int exit_code = 0;
    try {
        Options opt(argc, argv);
        std::vector<std::pair<std::string, std::string>> fix_data;
        if (opt.fix.has_value()) {
            FixSerumIds fix_serum_ids(opt.program_name());
            for (const auto& fix_entry : opt.fix.get())
                fix_serum_ids.add_fix(fix_entry);
            const auto charts_modified = fix_serum_ids.fix_charts_in_directory(".");
            std::cerr << "INFO: charts modified: " << charts_modified << '\n';
        }
        else {
            SerumIds serum_ids;
            const auto charts_processed = serum_ids.scan_directory(".");
            std::cout << charts_processed << " charts processed\n";
            serum_ids.print(false);
        }
    }
    catch (std::exception& err) {
        std::cerr << "ERROR: " << err.what() << '\n';
        exit_code = 2;
    }
    return exit_code;
}

// ----------------------------------------------------------------------
/// Local Variables:
/// eval: (if (fboundp 'eu-rename-buffer) (eu-rename-buffer))
/// End:
<|endoftext|>


<commit_before>#include <aerial_autonomy/controller_hardware_connectors/position_controller_drone_connector.h>
#include <aerial_autonomy/controllers/builtin_position_controller.h>
#include <gtest/gtest.h>
#include <parsernode/common.h>
#include <parsernode/parser.h>
#include <pluginlib/class_loader.h>
#include <ros/ros.h>

class TestQuad : public parsernode::Parser {
public:
  virtual bool takeoff() { return true; } // Take off the quadcopter
  virtual bool land() { return true; }    // Land the quadcopter
  virtual bool disarm() { return true; }  // Disarm the quadcopter
  virtual bool flowControl(bool) {
    return true;
  } // Enable or disable control of quadcopter
  virtual bool calibrateimubias() {
    return true;
  } // Calibrate imu of the quadcopter based on sample data

  // Command the euler angles of the quad and the thrust. the msg format is
  // (x,y,z,w) -> (roll, pitch, yaw, thrust). The thrust value will be adjusted
  // based on whethere there is thrust bias or not.
  virtual bool cmdrpythrust(geometry_msgs::Quaternion &rpytmsg,
                            bool sendyaw = false) {
    return true;
  }
  virtual bool cmdvelguided(geometry_msgs::Vector3 &vel_cmd, double &yaw_ang) {
    return true;
  }
  virtual bool cmdwaypoint(geometry_msgs::Vector3 &desired_pos,
                           double desired_yaw = 0) {
    quad_data.localpos = desired_pos;
    quad_data.rpydata.z = desired_yaw;
    return true;
  }
  virtual void grip(int state) {}
  virtual void reset_attitude(double roll, double pitch, double yaw) {}
  virtual void setmode(std::string mode) {}

  // PluginLib initialization function
  virtual void initialize(ros::NodeHandle &nh_) {}
  virtual void getquaddata(parsernode::common::quaddata &d1) { d1 = quad_data; }
  virtual ~TestQuad() {}
  virtual void
  setaltitude(double altitude_){};         // Set the altitude value in the data
  virtual void setlogdir(string logdir){}; // Set whether to log data or not
  virtual void controllog(bool logswitch) {}

private:
  parsernode::common::quaddata quad_data;
};

/// \brief Test BuiltInPositionController
TEST(PositionControllerDroneConnectorTests, Constructor) {
  TestQuad drone_hardware;

  BuiltInPositionController position_controller;

  ASSERT_NO_THROW(new PositionControllerDroneConnector(drone_hardware,
                                                       position_controller));
}

TEST(PositionControllerDroneConnectorTests, SetGoal) {
  TestQuad drone_hardware;

  // Create controller and connector
  BuiltInPositionController position_controller;
  PositionControllerDroneConnector pos_controller_connector(drone_hardware, position_controller);

  // Test set goal
  PositionYaw goal(10, 10, 10, 0.1);
  pos_controller_connector.setGoal(goal);
  PositionYaw goal_get = pos_controller_connector.getGoal();
  ASSERT_EQ(goal_get.x, goal.x);
  ASSERT_EQ(goal_get.y, goal.y);
  ASSERT_EQ(goal_get.z, goal.z);
  pos_controller_connector.run();

  parsernode::common::quaddata sensor_data;
  drone_hardware.getquaddata(sensor_data);

  ASSERT_EQ(sensor_data.localpos.x, goal.x);
  ASSERT_EQ(sensor_data.localpos.y, goal.y);
  ASSERT_EQ(sensor_data.localpos.z, goal.z);
  ASSERT_EQ(sensor_data.rpydata.z, goal.yaw);
}
///

int main(int argc, char **argv) {
  testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<commit_msg>Write out variable name in test<commit_after>#include <aerial_autonomy/controller_hardware_connectors/position_controller_drone_connector.h>
#include <aerial_autonomy/controllers/builtin_position_controller.h>
#include <gtest/gtest.h>
#include <parsernode/common.h>
#include <parsernode/parser.h>
#include <pluginlib/class_loader.h>
#include <ros/ros.h>

class TestQuad : public parsernode::Parser {
public:
  virtual bool takeoff() { return true; } // Take off the quadcopter
  virtual bool land() { return true; }    // Land the quadcopter
  virtual bool disarm() { return true; }  // Disarm the quadcopter
  virtual bool flowControl(bool) {
    return true;
  } // Enable or disable control of quadcopter
  virtual bool calibrateimubias() {
    return true;
  } // Calibrate imu of the quadcopter based on sample data

  // Command the euler angles of the quad and the thrust. the msg format is
  // (x,y,z,w) -> (roll, pitch, yaw, thrust). The thrust value will be adjusted
  // based on whethere there is thrust bias or not.
  virtual bool cmdrpythrust(geometry_msgs::Quaternion &rpytmsg,
                            bool sendyaw = false) {
    return true;
  }
  virtual bool cmdvelguided(geometry_msgs::Vector3 &vel_cmd, double &yaw_ang) {
    return true;
  }
  virtual bool cmdwaypoint(geometry_msgs::Vector3 &desired_pos,
                           double desired_yaw = 0) {
    quad_data.localpos = desired_pos;
    quad_data.rpydata.z = desired_yaw;
    return true;
  }
  virtual void grip(int state) {}
  virtual void reset_attitude(double roll, double pitch, double yaw) {}
  virtual void setmode(std::string mode) {}

  // PluginLib initialization function
  virtual void initialize(ros::NodeHandle &nh_) {}
  virtual void getquaddata(parsernode::common::quaddata &d1) { d1 = quad_data; }
  virtual ~TestQuad() {}
  virtual void
  setaltitude(double altitude_){};         // Set the altitude value in the data
  virtual void setlogdir(string logdir){}; // Set whether to log data or not
  virtual void controllog(bool logswitch) {}

private:
  parsernode::common::quaddata quad_data;
};

/// \brief Test BuiltInPositionController
TEST(PositionControllerDroneConnectorTests, Constructor) {
  TestQuad drone_hardware;

  BuiltInPositionController position_controller;

  ASSERT_NO_THROW(new PositionControllerDroneConnector(drone_hardware,
                                                       position_controller));
}

TEST(PositionControllerDroneConnectorTests, SetGoal) {
  TestQuad drone_hardware;

  // Create controller and connector
  BuiltInPositionController position_controller;
  PositionControllerDroneConnector position_controller_connector(
      drone_hardware, position_controller);

  // Test set goal
  PositionYaw goal(10, 10, 10, 0.1);
  position_controller_connector.setGoal(goal);
  PositionYaw goal_get = position_controller_connector.getGoal();
  ASSERT_EQ(goal_get.x, goal.x);
  ASSERT_EQ(goal_get.y, goal.y);
  ASSERT_EQ(goal_get.z, goal.z);
  position_controller_connector.run();

  parsernode::common::quaddata sensor_data;
  drone_hardware.getquaddata(sensor_data);

  ASSERT_EQ(sensor_data.localpos.x, goal.x);
  ASSERT_EQ(sensor_data.localpos.y, goal.y);
  ASSERT_EQ(sensor_data.localpos.z, goal.z);
  ASSERT_EQ(sensor_data.rpydata.z, goal.yaw);
}
///

int main(int argc, char **argv) {
  testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<|endoftext|>


<commit_before>#include "RapidIPSmearGauss.h"

#include "TMath.h"
#include "TRandom.h"

std::pair<double,double> RapidIPSmearGauss::smearIP(double ip, double pt) {

	const double sigma_ = gRandom->Gaus(0.,intercept_ + slope_/pt);
	const double smear_ = ip+sigma_;
	return std::pair<double,double>(smear_,std::fabs(sigma_));

}

<commit_msg>avoid negative IPs<commit_after>#include "RapidIPSmearGauss.h"

#include "TMath.h"
#include "TRandom.h"

std::pair<double,double> RapidIPSmearGauss::smearIP(double ip, double pt) {

	const double sigma_ = gRandom->Gaus(0.,intercept_ + slope_/pt);
	const double smear_ = ip+sigma_;
	return std::pair<double,double>(std::fabs(smear_),std::fabs(sigma_));

}

<|endoftext|>


<commit_before>/*
Resembla: Word-based Japanese similar sentence search library
https://github.com/tuem/resembla

Copyright 2017 Takashi Uemura

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "text_classification_feature_extractor.hpp"

#include <fstream>
#include <sstream>
#include <vector>

namespace resembla {

TextClassificationFeatureExtractor::TextClassificationFeatureExtractor(
        const std::string& mecab_options, const std::string& dict_path, const std::string& model_path):
    tagger(MeCab::createTagger(mecab_options.c_str())),
    model(svm_load_model(model_path.c_str()))
{
    std::ifstream ifs(dict_path);
    if(ifs.fail()){
        throw std::runtime_error("input file is not available: " + dict_path);
    }

    while(ifs.good()){
        std::string line;
        std::getline(ifs, line);
        if(ifs.eof()){
            break;
        }

        auto i = line.find(column_delimiter<>());
        if(i != std::string::npos){
            dictionary[line.substr(0, i)] = std::stoi(line.substr(i + 1));
        }
    }
}

Feature::text_type TextClassificationFeatureExtractor::operator()(const string_type& text) const
{
    auto nodes = toNodes(text);
    double s;
    {
        std::lock_guard<std::mutex> lock(mutex_model);
        s = svm_predict(model, &nodes[0]);
    }
    std::stringstream ss;
    ss << s;
    return ss.str();
}

std::vector<svm_node> TextClassificationFeatureExtractor::toNodes(const string_type& text) const
{
    const auto text_string = cast_string<std::string>(text);
    BoW bow;
    {
        std::lock_guard<std::mutex> lock(mutex_tagger);
        for(const MeCab::Node* node = tagger->parseToNode(text_string.c_str()); node; node = node->next){
            // skip BOS/EOS nodes
            if(node->stat == MECAB_BOS_NODE || node->stat == MECAB_EOS_NODE){
                continue;
            }

            auto i = dictionary.find(std::string(node->surface, node->surface + node->length));
            if(i != dictionary.end()){
                auto j = bow.find(i->second);
                if(j == bow.end()){
                    bow[i->second] = 1;
                }
                else{
                    ++j->second;
                }
            }
        }
    }

    std::vector<svm_node> nodes(bow.size() + 1);
    size_t i = 0;
    for(const auto j: bow){
        nodes[j.first].index = j.first;
        nodes[j.first].value = static_cast<double>(j.second);
        ++i;
    }
    nodes[i].index = -1; // end of features
    return nodes;
}

}
<commit_msg>use common utility<commit_after>/*
Resembla: Word-based Japanese similar sentence search library
https://github.com/tuem/resembla

Copyright 2017 Takashi Uemura

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "text_classification_feature_extractor.hpp"

#include <fstream>
#include <vector>

namespace resembla {

TextClassificationFeatureExtractor::TextClassificationFeatureExtractor(
        const std::string& mecab_options, const std::string& dict_path, const std::string& model_path):
    tagger(MeCab::createTagger(mecab_options.c_str())),
    model(svm_load_model(model_path.c_str()))
{
    std::ifstream ifs(dict_path);
    if(ifs.fail()){
        throw std::runtime_error("input file is not available: " + dict_path);
    }

    while(ifs.good()){
        std::string line;
        std::getline(ifs, line);
        if(ifs.eof()){
            break;
        }

        auto i = line.find(column_delimiter<>());
        if(i != std::string::npos){
            dictionary[line.substr(0, i)] = std::stoi(line.substr(i + 1));
        }
    }
}

Feature::text_type TextClassificationFeatureExtractor::operator()(const string_type& text) const
{
    auto nodes = toNodes(text);
    double s;
    {
        std::lock_guard<std::mutex> lock(mutex_model);
        s = svm_predict(model, &nodes[0]);
    }
    return Feature::toText(s);
}

std::vector<svm_node> TextClassificationFeatureExtractor::toNodes(const string_type& text) const
{
    const auto text_string = cast_string<std::string>(text);
    BoW bow;
    {
        std::lock_guard<std::mutex> lock(mutex_tagger);
        for(const MeCab::Node* node = tagger->parseToNode(text_string.c_str()); node; node = node->next){
            // skip BOS/EOS nodes
            if(node->stat == MECAB_BOS_NODE || node->stat == MECAB_EOS_NODE){
                continue;
            }

            auto i = dictionary.find(std::string(node->surface, node->surface + node->length));
            if(i != dictionary.end()){
                auto j = bow.find(i->second);
                if(j == bow.end()){
                    bow[i->second] = 1;
                }
                else{
                    ++j->second;
                }
            }
        }
    }

    std::vector<svm_node> nodes(bow.size() + 1);
    size_t i = 0;
    for(const auto j: bow){
        nodes[j.first].index = j.first;
        nodes[j.first].value = static_cast<double>(j.second);
        ++i;
    }
    nodes[i].index = -1; // end of features
    return nodes;
}

}
<|endoftext|>


<commit_before>/*
 * Copyright © 2012 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/** @file brw_fs_register_coalesce.cpp
 *
 * Implements register coalescing: Checks if the two registers involved in a
 * raw move don't interfere, in which case they can both be stored in the same
 * place and the MOV removed.
 *
 * To do this, all uses of the source of the MOV in the shader are replaced
 * with the destination of the MOV. For example:
 *
 * add vgrf3:F, vgrf1:F, vgrf2:F
 * mov vgrf4:F, vgrf3:F
 * mul vgrf5:F, vgrf5:F, vgrf4:F
 *
 * becomes
 *
 * add vgrf4:F, vgrf1:F, vgrf2:F
 * mul vgrf5:F, vgrf5:F, vgrf4:F
 */

#include "brw_fs.h"
#include "brw_fs_live_variables.h"

static bool
is_nop_mov(const fs_inst *inst)
{
   if (inst->opcode == BRW_OPCODE_MOV) {
      return inst->dst.equals(inst->src[0]);
   }

   return false;
}

static bool
is_coalesce_candidate(const fs_inst *inst, const int *virtual_grf_sizes)
{
   if (inst->opcode != BRW_OPCODE_MOV ||
       inst->is_partial_write() ||
       inst->saturate ||
       inst->src[0].file != GRF ||
       inst->src[0].negate ||
       inst->src[0].abs ||
       !inst->src[0].is_contiguous() ||
       inst->dst.file != GRF ||
       inst->dst.type != inst->src[0].type) {
      return false;
   }

   if (virtual_grf_sizes[inst->src[0].reg] >
       virtual_grf_sizes[inst->dst.reg])
      return false;

   return true;
}

static bool
can_coalesce_vars(brw::fs_live_variables *live_intervals,
                  const exec_list *instructions, const fs_inst *inst, int ip,
                  int var_to, int var_from)
{
   if (!live_intervals->vars_interfere(var_from, var_to))
      return true;

   /* We know that the live ranges of A (var_from) and B (var_to)
    * interfere because of the ->vars_interfere() call above. If the end
    * of B's live range is after the end of A's range, then we know two
    * things:
    *  - the start of B's live range must be in A's live range (since we
    *    already know the two ranges interfere, this is the only remaining
    *    possibility)
    *  - the interference isn't of the form we're looking for (where B is
    *    entirely inside A)
    */
   if (live_intervals->end[var_to] > live_intervals->end[var_from])
      return false;

   assert(ip >= live_intervals->start[var_to]);

   fs_inst *scan_inst;
   for (scan_inst = (fs_inst *)inst->next;
        !scan_inst->is_tail_sentinel() && ip <= live_intervals->end[var_to];
        scan_inst = (fs_inst *)scan_inst->next, ip++) {
      if (scan_inst->opcode == BRW_OPCODE_WHILE)
         return false;

      if (scan_inst->dst.equals(inst->dst) ||
          scan_inst->dst.equals(inst->src[0]))
         return false;
   }

   return true;
}

bool
fs_visitor::register_coalesce()
{
   bool progress = false;

   calculate_live_intervals();

   int src_size = 0;
   int channels_remaining = 0;
   int reg_from = -1, reg_to = -1;
   int reg_to_offset[MAX_SAMPLER_MESSAGE_SIZE];
   fs_inst *mov[MAX_SAMPLER_MESSAGE_SIZE];
   int var_to[MAX_SAMPLER_MESSAGE_SIZE];
   int var_from[MAX_SAMPLER_MESSAGE_SIZE];
   int ip = -1;

   foreach_list(node, &this->instructions) {
      fs_inst *inst = (fs_inst *)node;
      ip++;

      if (!is_coalesce_candidate(inst, virtual_grf_sizes))
         continue;

      if (is_nop_mov(inst)) {
         inst->opcode = BRW_OPCODE_NOP;
         progress = true;
         continue;
      }

      if (reg_from != inst->src[0].reg) {
         reg_from = inst->src[0].reg;

         src_size = virtual_grf_sizes[inst->src[0].reg];
         assert(src_size <= MAX_SAMPLER_MESSAGE_SIZE);

         channels_remaining = src_size;
         memset(mov, 0, sizeof(mov));

         reg_to = inst->dst.reg;
      }

      if (reg_to != inst->dst.reg)
         continue;

      const int offset = inst->src[0].reg_offset;
      reg_to_offset[offset] = inst->dst.reg_offset;
      mov[offset] = inst;
      channels_remaining--;

      if (channels_remaining)
         continue;

      bool can_coalesce = true;
      for (int i = 0; i < src_size; i++) {
         var_to[i] = live_intervals->var_from_vgrf[reg_to] + reg_to_offset[i];
         var_from[i] = live_intervals->var_from_vgrf[reg_from] + i;

         if (!can_coalesce_vars(live_intervals, &instructions, inst, ip,
                                var_to[i], var_from[i])) {
            can_coalesce = false;
            reg_from = -1;
            break;
         }
      }

      if (!can_coalesce)
         continue;

      progress = true;

      for (int i = 0; i < src_size; i++) {
         if (mov[i]) {
            mov[i]->opcode = BRW_OPCODE_NOP;
            mov[i]->conditional_mod = BRW_CONDITIONAL_NONE;
            mov[i]->dst = reg_undef;
            mov[i]->src[0] = reg_undef;
            mov[i]->src[1] = reg_undef;
            mov[i]->src[2] = reg_undef;
         }
      }

      foreach_list(node, &this->instructions) {
         fs_inst *scan_inst = (fs_inst *)node;

         for (int i = 0; i < src_size; i++) {
            if (mov[i]) {
               if (scan_inst->dst.file == GRF &&
                   scan_inst->dst.reg == reg_from &&
                   scan_inst->dst.reg_offset == i) {
                  scan_inst->dst.reg = reg_to;
                  scan_inst->dst.reg_offset = reg_to_offset[i];
               }
               for (int j = 0; j < 3; j++) {
                  if (scan_inst->src[j].file == GRF &&
                      scan_inst->src[j].reg == reg_from &&
                      scan_inst->src[j].reg_offset == i) {
                     scan_inst->src[j].reg = reg_to;
                     scan_inst->src[j].reg_offset = reg_to_offset[i];
                  }
               }
            }
         }
      }

      for (int i = 0; i < src_size; i++) {
         live_intervals->start[var_to[i]] =
            MIN2(live_intervals->start[var_to[i]],
                 live_intervals->start[var_from[i]]);
         live_intervals->end[var_to[i]] =
            MAX2(live_intervals->end[var_to[i]],
                 live_intervals->end[var_from[i]]);
      }
      reg_from = -1;
   }

   if (progress) {
      foreach_list_safe(node, &this->instructions) {
         fs_inst *inst = (fs_inst *)node;

         if (inst->opcode == BRW_OPCODE_NOP) {
            inst->remove();
         }
      }

      invalidate_live_intervals();
   }

   return progress;
}
<commit_msg>i965/fs: Reduce restrictions on interference in register coalescing.<commit_after>/*
 * Copyright © 2012 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/** @file brw_fs_register_coalesce.cpp
 *
 * Implements register coalescing: Checks if the two registers involved in a
 * raw move don't interfere, in which case they can both be stored in the same
 * place and the MOV removed.
 *
 * To do this, all uses of the source of the MOV in the shader are replaced
 * with the destination of the MOV. For example:
 *
 * add vgrf3:F, vgrf1:F, vgrf2:F
 * mov vgrf4:F, vgrf3:F
 * mul vgrf5:F, vgrf5:F, vgrf4:F
 *
 * becomes
 *
 * add vgrf4:F, vgrf1:F, vgrf2:F
 * mul vgrf5:F, vgrf5:F, vgrf4:F
 */

#include "brw_fs.h"
#include "brw_fs_live_variables.h"

static bool
is_nop_mov(const fs_inst *inst)
{
   if (inst->opcode == BRW_OPCODE_MOV) {
      return inst->dst.equals(inst->src[0]);
   }

   return false;
}

static bool
is_coalesce_candidate(const fs_inst *inst, const int *virtual_grf_sizes)
{
   if (inst->opcode != BRW_OPCODE_MOV ||
       inst->is_partial_write() ||
       inst->saturate ||
       inst->src[0].file != GRF ||
       inst->src[0].negate ||
       inst->src[0].abs ||
       !inst->src[0].is_contiguous() ||
       inst->dst.file != GRF ||
       inst->dst.type != inst->src[0].type) {
      return false;
   }

   if (virtual_grf_sizes[inst->src[0].reg] >
       virtual_grf_sizes[inst->dst.reg])
      return false;

   return true;
}

static bool
can_coalesce_vars(brw::fs_live_variables *live_intervals,
                  const exec_list *instructions, const fs_inst *inst, int ip,
                  int var_to, int var_from)
{
   if (!live_intervals->vars_interfere(var_from, var_to))
      return true;

   assert(ip >= live_intervals->start[var_to]);

   fs_inst *scan_inst;
   for (scan_inst = (fs_inst *)inst->next;
        !scan_inst->is_tail_sentinel() && ip <= live_intervals->end[var_to];
        scan_inst = (fs_inst *)scan_inst->next, ip++) {
      if (scan_inst->opcode == BRW_OPCODE_WHILE)
         return false;

      if (scan_inst->dst.equals(inst->dst) ||
          scan_inst->dst.equals(inst->src[0]))
         return false;
   }

   return true;
}

bool
fs_visitor::register_coalesce()
{
   bool progress = false;

   calculate_live_intervals();

   int src_size = 0;
   int channels_remaining = 0;
   int reg_from = -1, reg_to = -1;
   int reg_to_offset[MAX_SAMPLER_MESSAGE_SIZE];
   fs_inst *mov[MAX_SAMPLER_MESSAGE_SIZE];
   int var_to[MAX_SAMPLER_MESSAGE_SIZE];
   int var_from[MAX_SAMPLER_MESSAGE_SIZE];
   int ip = -1;

   foreach_list(node, &this->instructions) {
      fs_inst *inst = (fs_inst *)node;
      ip++;

      if (!is_coalesce_candidate(inst, virtual_grf_sizes))
         continue;

      if (is_nop_mov(inst)) {
         inst->opcode = BRW_OPCODE_NOP;
         progress = true;
         continue;
      }

      if (reg_from != inst->src[0].reg) {
         reg_from = inst->src[0].reg;

         src_size = virtual_grf_sizes[inst->src[0].reg];
         assert(src_size <= MAX_SAMPLER_MESSAGE_SIZE);

         channels_remaining = src_size;
         memset(mov, 0, sizeof(mov));

         reg_to = inst->dst.reg;
      }

      if (reg_to != inst->dst.reg)
         continue;

      const int offset = inst->src[0].reg_offset;
      reg_to_offset[offset] = inst->dst.reg_offset;
      mov[offset] = inst;
      channels_remaining--;

      if (channels_remaining)
         continue;

      bool can_coalesce = true;
      for (int i = 0; i < src_size; i++) {
         var_to[i] = live_intervals->var_from_vgrf[reg_to] + reg_to_offset[i];
         var_from[i] = live_intervals->var_from_vgrf[reg_from] + i;

         if (!can_coalesce_vars(live_intervals, &instructions, inst, ip,
                                var_to[i], var_from[i])) {
            can_coalesce = false;
            reg_from = -1;
            break;
         }
      }

      if (!can_coalesce)
         continue;

      progress = true;

      for (int i = 0; i < src_size; i++) {
         if (mov[i]) {
            mov[i]->opcode = BRW_OPCODE_NOP;
            mov[i]->conditional_mod = BRW_CONDITIONAL_NONE;
            mov[i]->dst = reg_undef;
            mov[i]->src[0] = reg_undef;
            mov[i]->src[1] = reg_undef;
            mov[i]->src[2] = reg_undef;
         }
      }

      foreach_list(node, &this->instructions) {
         fs_inst *scan_inst = (fs_inst *)node;

         for (int i = 0; i < src_size; i++) {
            if (mov[i]) {
               if (scan_inst->dst.file == GRF &&
                   scan_inst->dst.reg == reg_from &&
                   scan_inst->dst.reg_offset == i) {
                  scan_inst->dst.reg = reg_to;
                  scan_inst->dst.reg_offset = reg_to_offset[i];
               }
               for (int j = 0; j < 3; j++) {
                  if (scan_inst->src[j].file == GRF &&
                      scan_inst->src[j].reg == reg_from &&
                      scan_inst->src[j].reg_offset == i) {
                     scan_inst->src[j].reg = reg_to;
                     scan_inst->src[j].reg_offset = reg_to_offset[i];
                  }
               }
            }
         }
      }

      for (int i = 0; i < src_size; i++) {
         live_intervals->start[var_to[i]] =
            MIN2(live_intervals->start[var_to[i]],
                 live_intervals->start[var_from[i]]);
         live_intervals->end[var_to[i]] =
            MAX2(live_intervals->end[var_to[i]],
                 live_intervals->end[var_from[i]]);
      }
      reg_from = -1;
   }

   if (progress) {
      foreach_list_safe(node, &this->instructions) {
         fs_inst *inst = (fs_inst *)node;

         if (inst->opcode == BRW_OPCODE_NOP) {
            inst->remove();
         }
      }

      invalidate_live_intervals();
   }

   return progress;
}
<|endoftext|>


<commit_before>#include "SemanticAnalyzer.h"
#include "Symbol.h"
#include "PrintFunction.h"

SemanticAnalyzer::SemanticAnalyzer() {
    this->currentStructure = NULL;
    this->lastStatement = NULL;
    this->symbolTable.newScope();
}

SemanticAnalyzer::~SemanticAnalyzer() {
}

void SemanticAnalyzer::analyzeRelationalOperationCasting(BinaryOperation* binaryop) {
}

void SemanticAnalyzer::newScope() {
    this->analyzeScopeCreation();
    this->symbolTable.setCurrentStructure(this->currentStructure);
    this->symbolTable.setLastStatement(this->lastStatement);
    this->currentStructure = NULL;
    this->symbolTable.newScope();
}

void SemanticAnalyzer::returnScope() {
    this->symbolTable.returnScope();
    this->lastStatement = this->symbolTable.getLastStatement();
}

void SemanticAnalyzer::setScope(float indentation) {
    int indINT = (int) indentation;

    if(indINT != indentation) {
        ERROR_LOGGER->log(ErrorLogger::SYNTAX, "INDENTATION 2 SPACES ERROR"); // TODO
        return;
    }

    if(indINT - this->getCurrentIndentation() > 1)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "INDENTATION ERROR"); // TODO
    else if(indINT > this->getCurrentIndentation()) {
        this->newScope();
    } else {
        int ind = this->getCurrentIndentation();
        while(indINT < ind) {
            this->returnScope();
            ind--;
        }
    }
}

void SemanticAnalyzer::pushLineScope(TreeNode* line) {
    this->symbolTable.pushLineScope(line);
}

CodeBlock* SemanticAnalyzer::getCurrentBody() {
    return this->symbolTable.getCurrentCodeBlock();
}

int SemanticAnalyzer::getCurrentIndentation() {
    return this->symbolTable.getCurrentIndentation();
}

void SemanticAnalyzer::setUnknownTypes(Data::Type type, CodeBlock* codeBlock) {
    this->symbolTable.setUnknownTypes(type);

    for(int i = 0; i < codeBlock->numberOfLines(); i++){
      codeBlock->getLine(i)->setType(type);
    }
}

void SemanticAnalyzer::analyzeProgram() {
    // Verifica a existência da função toc()
    Symbol tocFunction = this->symbolTable.getSymbol("toc");

    if(tocFunction.getType() != Symbol::FUNCTION
       || tocFunction.getDataType() != Data::VOID)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Main function toc() not found.");
}

void SemanticAnalyzer::analyzeScopeCreation() {
    if(this->currentStructure == NULL)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "INDENTATION ERROR"); // TODO
}

void SemanticAnalyzer::analyzeCasting(BinaryOperation* binaryOp) {
  TreeNode* left = binaryOp->left;
  TreeNode* right = binaryOp->right;
  std::string text = "";

  if (left->dataType() != right->dataType()) {
    if(right->dataType() == Data::STR) {

      switch (left->dataType()) {
        case Data::BOO:
          if (((String*)right)->isBoolean() || this->symbolTable.getSymbol(((Variable*)right)->getId()).getData()->dataType()) {
            binaryOp->right = new TypeCasting(left->dataType(), right);
            right = binaryOp->right;
          } else {
            ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "String value is not false or true.");
          }
          break;
        case Data::FLT:
        case Data::INT:
          if (((String*)right)->isNumber() || this->symbolTable.getSymbol(((Variable*)right)->getId()).getData()->dataType()) {
            binaryOp->right = new TypeCasting(left->dataType(), right);
            right = binaryOp->right;
          } else {
            ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "String value is not a number.");
          }
          break;
        default:
          break;
      }

    }else{
        binaryOp->right = new TypeCasting(left->dataType(), right);
        right = binaryOp->right;
    }
  }
}

void SemanticAnalyzer::analyzeLoop(std::string id) {
  if(this->symbolTable.getSymbol(id, true).getData()->classType() != TreeNode::ARRAY)
    ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Expecting an array type " + id + ".");
}

void SemanticAnalyzer::analyzeArray(std::string id, int size, TreeNode* attribuition){
  BinaryOperation* node = (BinaryOperation*)attribuition;
  int c = 0;
  if(attribuition->classType() != TreeNode::BINARY_OPERATION && size > 1)
    ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Illegal array assignment.");
  else if(attribuition->classType() == TreeNode::BINARY_OPERATION){
    while(node->right->classType() == TreeNode::BINARY_OPERATION){
      c ++;
      if(node->right->classType() != TreeNode::BINARY_OPERATION)
        break;
      node = (BinaryOperation*)(node->right);
    }
    c = c + 2;
    if(c > size)
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Number of elements in assignment differs from array length.");
  }
}

void SemanticAnalyzer::analyzeAssignArray(std::string id, TreeNode* size){
  Integer* i = (Integer*) size;
  TreeNode* t = this->symbolTable.getSymbol(id, true).getData();
  if (t->classType() == TreeNode::ARRAY)
    std::cout << "a" << std::endl;
  // TreeNode* s = array->getSize();
  // if(i->getValue() > s->getValue())

}

bool SemanticAnalyzer::checkIdentifier(std::string id) const {
    if(this->symbolTable.existsSymbol(id, false)) {
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Identifier " + id + " already used for declaration.");
        return false;
    }

    return true;
}

TreeNode* SemanticAnalyzer::cast(Data::Type type, TreeNode* node) const {
    if(type != node->dataType())
        return new TypeCasting(type, node);

    return node;
}

TreeNode* SemanticAnalyzer::declareVariable(std::string id, Data::Type dataType, int size) {
    if(this->lastStatement->classType() == TreeNode::OBJECT && this->currentStructure != NULL){
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Declaration expected encapsulation.");
      std::cout << this->lastStatement->classType() << std::endl;
    }
    if(this->checkIdentifier(id)) {
        if(size > 0) {
            Array* array = new Array(id, dataType, new Integer(size));
            this->symbolTable.addSymbol(id, Symbol(dataType, Symbol::VARIABLE, false, array)); // Adds variable to symbol table and save array size
            return array;
        } else {
            this->symbolTable.addSymbol(id, Symbol(dataType, Symbol::VARIABLE, false)); // Adds variable to symbol table
            Variable* v = new Variable(id, dataType);
            VariableDeclaration* vD = new VariableDeclaration(dataType, v);
            v->setSymbolTable(this->symbolTable);
            vD->setSymbolTable(this->symbolTable);

            return vD;
        }
    }

    // return a random variable if id is in use
    return new VariableDeclaration(dataType, new Variable(id, dataType));
}

TreeNode* SemanticAnalyzer::declareBinaryOperation(TreeNode* left, BinaryOperation::Type op, TreeNode* right) {
    // TODO
}

TreeNode* SemanticAnalyzer::declareObject(std::string id, CodeBlock* param, CodeBlock* body){
    if(this->checkIdentifier(id)) {
        Object* obj = new Object(id,param,NULL);
        this->symbolTable.addSymbol(id, Symbol(Data::OBJ, Symbol::OBJECT, true, obj));
        this->currentStructure = obj;
        return obj;
    }
    return NULL;
}

TreeNode* SemanticAnalyzer::declareAttribute(std::string id, Data::Type type, int encapsulation, int size) {
    if(this->lastStatement->classType() != TreeNode::OBJECT)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Encapsulation out of object body");

    Object* obj = (Object*) this->lastStatement;

    if(size > 0){
        Array* a = new Array(id, type, new Integer(size));
        a->setEncapsulation(encapsulation);
        this->symbolTable.addSymbol(id, Symbol(type, Symbol::VARIABLE, true, a));
        if(encapsulation == 1)
          this->symbolTable.getSymbol(id).setEncapsulation(true);
        return a;
    }else{
      VariableDeclaration* vd = new VariableDeclaration(type, new Variable(id, type));
      vd->setEncapsulation(encapsulation);
      this->symbolTable.addSymbol(id, Symbol(type, Symbol::VARIABLE, true, vd));
      if(encapsulation == 1)
        this->symbolTable.getSymbol(id).setEncapsulation(true);
      return vd;
    }
  return NULL;
}

TreeNode* SemanticAnalyzer::declareAssignAttribute(std::string id, Data::Type type, int encapsulation, TreeNode* value, int size) {
  if(this->lastStatement->classType() != TreeNode::OBJECT)
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Encapsulation out of object body");

    if(this->checkIdentifier(id)) {
        if(size > 0) {
            Array* array = new Array(id, type, new Integer(size));
            array->setEncapsulation(encapsulation);
            this->symbolTable.addSymbol(id, Symbol(type, Symbol::VARIABLE, false, array)); // Adds variable to symbol table and save array size
            this->symbolTable.setInitializedSymbol(id, array);
            if(encapsulation == 1)
              this->symbolTable.getSymbol(id).setEncapsulation(true);
            return array;
        }

        this->symbolTable.addSymbol(id, Symbol(type, Symbol::VARIABLE, false)); // Adds variable to symbol table
        this->symbolTable.setInitializedSymbol(id, value);
        Variable* v = new Variable(id, type);
        VariableDeclaration* vD = new VariableDeclaration(type, v);
        vD->setEncapsulation(encapsulation);
        if(encapsulation == 1)
          this->symbolTable.getSymbol(id).setEncapsulation(true);
        v->setSymbolTable(this->symbolTable);
        vD->setSymbolTable(this->symbolTable);
        return vD;
    }

  return NULL;
}

TreeNode* SemanticAnalyzer::declareMethod(std::string id, CodeBlock* params, CodeBlock* body, Data::Type returnType, int encapsulation) {
    if(this->lastStatement->classType() != TreeNode::OBJECT){
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Encapsulation method out of object body");
        std::cout << "method" + this->lastStatement->classType() << std::endl;
      }

    Function* function;
    if(this->checkIdentifier(id)) {
        function = new Function(id, params, body, NULL);
        function->setType(returnType);
        function->setEncapsulation(encapsulation);

        this->symbolTable.addSymbol(id, Symbol(returnType, Symbol::FUNCTION, true, function));
        this->currentStructure = function;
        return function;
  }

  return NULL;
}

TreeNode* SemanticAnalyzer::declareFunction(std::string id, CodeBlock* params, CodeBlock* body, Data::Type returnType) {
    if(this->checkIdentifier(id)) {
        TreeNode* function;

        if(!id.compare("toc")) { // Se é a função toc, cria a mesma
            function = new TocFunction(body);
        } else { // Outra função qualquer
            function = new Function(id, params, body, NULL);
            function->setType(returnType);
        }

        this->symbolTable.addSymbol(id, Symbol(returnType, Symbol::FUNCTION, true, function));
        this->currentStructure = function;
        this->lastStatement = function;
        return function;
    }

    return NULL;
}

TreeNode* SemanticAnalyzer::declareFunctionReturn(TreeNode* ret) {
    Function* f = (Function*) this->symbolTable.getParentStructure();
    if(f->classType() != TreeNode::FUNCTION){
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Return value for function " + f->getId() + "is different from expected;" );
    }

    f->setReturn(ret);
    return NULL; // TODO
}

TreeNode* SemanticAnalyzer::declareCondition(TreeNode* expression, CodeBlock* body) {
    Conditional* cond = new Conditional(expression, body, NULL);
    this->currentStructure = cond;
    this->lastStatement = cond;
    return cond;
}

TreeNode* SemanticAnalyzer::declareElseCondition(CodeBlock* body) {
    std::string elseError = "Else without a if.";

    if(this->lastStatement->classType() != TreeNode::CONDITIONAL)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, elseError);

    Conditional* cond = (Conditional*) this->lastStatement;

    if(cond->hasElse())
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, elseError);

    cond->setElse(true);
    this->currentStructure = this->lastStatement;
    return this->currentStructure;
}

TreeNode* SemanticAnalyzer::declareLoop(TreeNode* init, TreeNode* test, TreeNode* attribuition) {
  Loop* loop;

  if(test == NULL) {
    Variable* v = (Variable*) attribuition;
    std::string id = v->getId();
    Array* array = (Array*)this->symbolTable.getSymbol(id, true).getData();
    loop = new Loop(init, test, array);
  }else{
    loop = new Loop(init, test, attribuition);
  }

  this->currentStructure = loop;
  return loop;
}

TreeNode* SemanticAnalyzer::declarePrint(TreeNode* param) {
    TreeNode* paramPrint = param;

    if(param->dataType() != Data::STR)
        paramPrint = new TypeCasting(Data::STR, paramPrint);

    return new PrintFunction(paramPrint);
}

TreeNode* SemanticAnalyzer::assignVariable(std::string id, TreeNode* value, TreeNode* index) {
  if(!this->symbolTable.existsSymbol(id, true)) {
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Undeclared variable " + id + ".");
      return new Variable(id, Data::UNKNOWN); //Creates variable node anyway
  } else if (index != NULL) {
        this->symbolTable.setInitializedSymbol(id);
        Array* a = (Array*)this->symbolTable.getSymbol(id).getData();
        return new Array(id, this->symbolTable.getSymbol(id).getDataType(), index, value);
  }  else {
      this->symbolTable.setInitializedSymbol(id, value);
      Variable* v = new Variable(id, this->symbolTable.getSymbol(id, true).getDataType());
      v->setSymbolTable(this->symbolTable);
      return v;
  }
}

TreeNode* SemanticAnalyzer::declareAssignVariable(std::string id, Data::Type dataType, TreeNode* value, int size) {
<<<<<<< HEAD
    if(this->lastStructure != NULL && this->lastStructure->classType() == TreeNode::OBJECT){
=======
    if(this->lastStatement->classType() == TreeNode::OBJECT){
>>>>>>> 948d2b3b3fe2119923a70b3982c5e6b70b6369a9
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Declaration expected encapsulation.");
    }

    if(this->checkIdentifier(id)) {
        // sempre que size é maior do que zero, trata-se de uma declaração de array
        if(size > 0) {
            Array* array = new Array(id, dataType, new Integer(size), value);
            this->symbolTable.addSymbol(id, Symbol(dataType, Symbol::VARIABLE, false, array)); // Adds variable to symbol table and save array size
            this->symbolTable.setInitializedSymbol(id, array);
            return array;
        }

        this->symbolTable.addSymbol(id, Symbol(dataType, Symbol::VARIABLE, false, value)); // Adds variable to symbol table
        this->symbolTable.setInitializedSymbol(id, value);

        Variable* v = new Variable(id, dataType);
        VariableDeclaration* vD = new VariableDeclaration(dataType, v);
        v->setSymbolTable(this->symbolTable);
        vD->setSymbolTable(this->symbolTable);
        return vD;
    }

  // return a random variable case id is in use
  return new VariableDeclaration(dataType, new Variable(id, dataType));
}

TreeNode* SemanticAnalyzer::useVariable(std::string id, TreeNode* index) {
  if(!this->symbolTable.existsSymbol(id, true)) {
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Undeclared variable " + id + ".");
      return new Variable(id, Data::UNKNOWN); //Creates variable node anyway
  } else if(!this->symbolTable.isSymbolInitialized(id, true)) {
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Variable " + id + " used but not initialized.");
  }

  TreeNode* t = this->symbolTable.getSymbol(id, true).getData();
  if (index != NULL and t->classType() == TreeNode::ARRAY) {
    Array* array = (Array*)t;
    if (array != NULL and array->getSize()->classType() == TreeNode::INTEGER) {
      if(((Integer*)index)->getValue() > ((Integer*)(array->getSize()))->getValue() || ((Integer*)index)->getValue() < 0)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Index out of bounds " + id + ".");
    }
    return new Array(id, this->symbolTable.getSymbol(id,true).getDataType(), index);
  }
  Variable* v = new Variable(id, this->symbolTable.getSymbol(id,true).getDataType());
  v->setSymbolTable(this->symbolTable);
  return v;
}
<commit_msg>arquivos organizados<commit_after>#include "SemanticAnalyzer.h"
#include "Symbol.h"
#include "PrintFunction.h"

SemanticAnalyzer::SemanticAnalyzer() {
    this->currentStructure = NULL;
    this->lastStatement = NULL;
    this->symbolTable.newScope();
}

SemanticAnalyzer::~SemanticAnalyzer() {
}

void SemanticAnalyzer::analyzeRelationalOperationCasting(BinaryOperation* binaryop) {
}

void SemanticAnalyzer::newScope() {
    this->analyzeScopeCreation();
    this->symbolTable.setCurrentStructure(this->currentStructure);
    this->symbolTable.setLastStatement(this->lastStatement);
    this->currentStructure = NULL;
    this->symbolTable.newScope();
}

void SemanticAnalyzer::returnScope() {
    this->symbolTable.returnScope();
    this->lastStatement = this->symbolTable.getLastStatement();
}

void SemanticAnalyzer::setScope(float indentation) {
    int indINT = (int) indentation;

    if(indINT != indentation) {
        ERROR_LOGGER->log(ErrorLogger::SYNTAX, "INDENTATION 2 SPACES ERROR"); // TODO
        return;
    }

    if(indINT - this->getCurrentIndentation() > 1)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "INDENTATION ERROR"); // TODO
    else if(indINT > this->getCurrentIndentation()) {
        this->newScope();
    } else {
        int ind = this->getCurrentIndentation();
        while(indINT < ind) {
            this->returnScope();
            ind--;
        }
    }
}

void SemanticAnalyzer::pushLineScope(TreeNode* line) {
    this->symbolTable.pushLineScope(line);
}

CodeBlock* SemanticAnalyzer::getCurrentBody() {
    return this->symbolTable.getCurrentCodeBlock();
}

int SemanticAnalyzer::getCurrentIndentation() {
    return this->symbolTable.getCurrentIndentation();
}

void SemanticAnalyzer::setUnknownTypes(Data::Type type, CodeBlock* codeBlock) {
    this->symbolTable.setUnknownTypes(type);

    for(int i = 0; i < codeBlock->numberOfLines(); i++){
      codeBlock->getLine(i)->setType(type);
    }
}

void SemanticAnalyzer::analyzeProgram() {
    // Verifica a existência da função toc()
    Symbol tocFunction = this->symbolTable.getSymbol("toc");

    if(tocFunction.getType() != Symbol::FUNCTION
       || tocFunction.getDataType() != Data::VOID)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Main function toc() not found.");
}

void SemanticAnalyzer::analyzeScopeCreation() {
    if(this->currentStructure == NULL)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "INDENTATION ERROR"); // TODO
}

void SemanticAnalyzer::analyzeCasting(BinaryOperation* binaryOp) {
  TreeNode* left = binaryOp->left;
  TreeNode* right = binaryOp->right;
  std::string text = "";

  if (left->dataType() != right->dataType()) {
    if(right->dataType() == Data::STR) {

      switch (left->dataType()) {
        case Data::BOO:
          if (((String*)right)->isBoolean() || this->symbolTable.getSymbol(((Variable*)right)->getId()).getData()->dataType()) {
            binaryOp->right = new TypeCasting(left->dataType(), right);
            right = binaryOp->right;
          } else {
            ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "String value is not false or true.");
          }
          break;
        case Data::FLT:
        case Data::INT:
          if (((String*)right)->isNumber() || this->symbolTable.getSymbol(((Variable*)right)->getId()).getData()->dataType()) {
            binaryOp->right = new TypeCasting(left->dataType(), right);
            right = binaryOp->right;
          } else {
            ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "String value is not a number.");
          }
          break;
        default:
          break;
      }

    }else{
        binaryOp->right = new TypeCasting(left->dataType(), right);
        right = binaryOp->right;
    }
  }
}

void SemanticAnalyzer::analyzeLoop(std::string id) {
  if(this->symbolTable.getSymbol(id, true).getData()->classType() != TreeNode::ARRAY)
    ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Expecting an array type " + id + ".");
}

void SemanticAnalyzer::analyzeArray(std::string id, int size, TreeNode* attribuition){
  BinaryOperation* node = (BinaryOperation*)attribuition;
  int c = 0;
  if(attribuition->classType() != TreeNode::BINARY_OPERATION && size > 1)
    ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Illegal array assignment.");
  else if(attribuition->classType() == TreeNode::BINARY_OPERATION){
    while(node->right->classType() == TreeNode::BINARY_OPERATION){
      c ++;
      if(node->right->classType() != TreeNode::BINARY_OPERATION)
        break;
      node = (BinaryOperation*)(node->right);
    }
    c = c + 2;
    if(c > size)
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Number of elements in assignment differs from array length.");
  }
}

void SemanticAnalyzer::analyzeAssignArray(std::string id, TreeNode* size){
  Integer* i = (Integer*) size;
  TreeNode* t = this->symbolTable.getSymbol(id, true).getData();
  if (t->classType() == TreeNode::ARRAY)
    std::cout << "a" << std::endl;
  // TreeNode* s = array->getSize();
  // if(i->getValue() > s->getValue())

}

bool SemanticAnalyzer::checkIdentifier(std::string id) const {
    if(this->symbolTable.existsSymbol(id, false)) {
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Identifier " + id + " already used for declaration.");
        return false;
    }

    return true;
}

TreeNode* SemanticAnalyzer::cast(Data::Type type, TreeNode* node) const {
    if(type != node->dataType())
        return new TypeCasting(type, node);

    return node;
}

TreeNode* SemanticAnalyzer::declareVariable(std::string id, Data::Type dataType, int size) {
    if(this->checkIdentifier(id)) {
        if(size > 0) {
            Array* array = new Array(id, dataType, new Integer(size));
            this->symbolTable.addSymbol(id, Symbol(dataType, Symbol::VARIABLE, false, array)); // Adds variable to symbol table and save array size
            return array;
        } else {
            this->symbolTable.addSymbol(id, Symbol(dataType, Symbol::VARIABLE, false)); // Adds variable to symbol table
            Variable* v = new Variable(id, dataType);
            VariableDeclaration* vD = new VariableDeclaration(dataType, v);
            v->setSymbolTable(this->symbolTable);
            vD->setSymbolTable(this->symbolTable);

            return vD;
        }
    }

    // return a random variable if id is in use
    return new VariableDeclaration(dataType, new Variable(id, dataType));
}

TreeNode* SemanticAnalyzer::declareBinaryOperation(TreeNode* left, BinaryOperation::Type op, TreeNode* right) {
    // TODO
}

TreeNode* SemanticAnalyzer::declareObject(std::string id, CodeBlock* param, CodeBlock* body){
    if(this->checkIdentifier(id)) {
        Object* obj = new Object(id,param,NULL);
        this->symbolTable.addSymbol(id, Symbol(Data::OBJ, Symbol::OBJECT, true, obj));
        this->currentStructure = obj;
        return obj;
    }
    return NULL;
}

TreeNode* SemanticAnalyzer::declareAttribute(std::string id, Data::Type type, int encapsulation, int size) {
    Object* obj = (Object*) this->lastStatement;

    if(size > 0){
        Array* a = new Array(id, type, new Integer(size));
        a->setEncapsulation(encapsulation);
        this->symbolTable.addSymbol(id, Symbol(type, Symbol::VARIABLE, true, a));
        if(encapsulation == 1)
          this->symbolTable.getSymbol(id).setEncapsulation(true);
        return a;
    }else{
      VariableDeclaration* vd = new VariableDeclaration(type, new Variable(id, type));
      vd->setEncapsulation(encapsulation);
      this->symbolTable.addSymbol(id, Symbol(type, Symbol::VARIABLE, true, vd));
      if(encapsulation == 1)
        this->symbolTable.getSymbol(id).setEncapsulation(true);
      return vd;
    }
  return NULL;
}

TreeNode* SemanticAnalyzer::declareAssignAttribute(std::string id, Data::Type type, int encapsulation, TreeNode* value, int size) {
    if(this->checkIdentifier(id)) {
        if(size > 0) {
            Array* array = new Array(id, type, new Integer(size));
            array->setEncapsulation(encapsulation);
            this->symbolTable.addSymbol(id, Symbol(type, Symbol::VARIABLE, false, array)); // Adds variable to symbol table and save array size
            this->symbolTable.setInitializedSymbol(id, array);
            if(encapsulation == 1)
              this->symbolTable.getSymbol(id).setEncapsulation(true);
            return array;
        }

        this->symbolTable.addSymbol(id, Symbol(type, Symbol::VARIABLE, false)); // Adds variable to symbol table
        this->symbolTable.setInitializedSymbol(id, value);
        Variable* v = new Variable(id, type);
        VariableDeclaration* vD = new VariableDeclaration(type, v);
        vD->setEncapsulation(encapsulation);
        if(encapsulation == 1)
          this->symbolTable.getSymbol(id).setEncapsulation(true);
        v->setSymbolTable(this->symbolTable);
        vD->setSymbolTable(this->symbolTable);
        return vD;
    }

  return NULL;
}

TreeNode* SemanticAnalyzer::declareMethod(std::string id, CodeBlock* params, CodeBlock* body, Data::Type returnType, int encapsulation) {
    Function* function;
    if(this->checkIdentifier(id)) {
        function = new Function(id, params, body, NULL);
        function->setType(returnType);
        function->setEncapsulation(encapsulation);

        this->symbolTable.addSymbol(id, Symbol(returnType, Symbol::FUNCTION, true, function));
        this->currentStructure = function;
        return function;
  }

  return NULL;
}

TreeNode* SemanticAnalyzer::declareFunction(std::string id, CodeBlock* params, CodeBlock* body, Data::Type returnType) {
    if(this->checkIdentifier(id)) {
        TreeNode* function;

        if(!id.compare("toc")) { // Se é a função toc, cria a mesma
            function = new TocFunction(body);
        } else { // Outra função qualquer
            function = new Function(id, params, body, NULL);
            function->setType(returnType);
        }

        this->symbolTable.addSymbol(id, Symbol(returnType, Symbol::FUNCTION, true, function));
        this->currentStructure = function;
        this->lastStatement = function;
        return function;
    }

    return NULL;
}

TreeNode* SemanticAnalyzer::declareFunctionReturn(TreeNode* ret) {
    Function* f = (Function*) this->symbolTable.getParentStructure();
    if(f->classType() != TreeNode::FUNCTION){
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Return value for function " + f->getId() + "is different from expected;" );
    }

    f->setReturn(ret);
    return NULL; // TODO
}

TreeNode* SemanticAnalyzer::declareCondition(TreeNode* expression, CodeBlock* body) {
    Conditional* cond = new Conditional(expression, body, NULL);
    this->currentStructure = cond;
    this->lastStatement = cond;
    return cond;
}

TreeNode* SemanticAnalyzer::declareElseCondition(CodeBlock* body) {
    std::string elseError = "Else without a if.";

    if(this->lastStatement->classType() != TreeNode::CONDITIONAL)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, elseError);

    Conditional* cond = (Conditional*) this->lastStatement;

    if(cond->hasElse())
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, elseError);

    cond->setElse(true);
    this->currentStructure = this->lastStatement;
    return this->currentStructure;
}

TreeNode* SemanticAnalyzer::declareLoop(TreeNode* init, TreeNode* test, TreeNode* attribuition) {
  Loop* loop;

  if(test == NULL) {
    Variable* v = (Variable*) attribuition;
    std::string id = v->getId();
    Array* array = (Array*)this->symbolTable.getSymbol(id, true).getData();
    loop = new Loop(init, test, array);
  }else{
    loop = new Loop(init, test, attribuition);
  }

  this->currentStructure = loop;
  return loop;
}

TreeNode* SemanticAnalyzer::declarePrint(TreeNode* param) {
    TreeNode* paramPrint = param;

    if(param->dataType() != Data::STR)
        paramPrint = new TypeCasting(Data::STR, paramPrint);

    return new PrintFunction(paramPrint);
}

TreeNode* SemanticAnalyzer::assignVariable(std::string id, TreeNode* value, TreeNode* index) {
  if(!this->symbolTable.existsSymbol(id, true)) {
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Undeclared variable " + id + ".");
      return new Variable(id, Data::UNKNOWN); //Creates variable node anyway
  } else if (index != NULL) {
        this->symbolTable.setInitializedSymbol(id);
        Array* a = (Array*)this->symbolTable.getSymbol(id).getData();
        return new Array(id, this->symbolTable.getSymbol(id).getDataType(), index, value);
  }  else {
      this->symbolTable.setInitializedSymbol(id, value);
      Variable* v = new Variable(id, this->symbolTable.getSymbol(id, true).getDataType());
      v->setSymbolTable(this->symbolTable);
      return v;
  }
}

TreeNode* SemanticAnalyzer::declareAssignVariable(std::string id, Data::Type dataType, TreeNode* value, int size) {
    if(this->checkIdentifier(id)) {
        // sempre que size é maior do que zero, trata-se de uma declaração de array
        if(size > 0) {
            Array* array = new Array(id, dataType, new Integer(size), value);
            this->symbolTable.addSymbol(id, Symbol(dataType, Symbol::VARIABLE, false, array)); // Adds variable to symbol table and save array size
            this->symbolTable.setInitializedSymbol(id, array);
            return array;
        }

        this->symbolTable.addSymbol(id, Symbol(dataType, Symbol::VARIABLE, false, value)); // Adds variable to symbol table
        this->symbolTable.setInitializedSymbol(id, value);

        Variable* v = new Variable(id, dataType);
        VariableDeclaration* vD = new VariableDeclaration(dataType, v);
        v->setSymbolTable(this->symbolTable);
        vD->setSymbolTable(this->symbolTable);
        return vD;
    }

  // return a random variable case id is in use
  return new VariableDeclaration(dataType, new Variable(id, dataType));
}

TreeNode* SemanticAnalyzer::useVariable(std::string id, TreeNode* index) {
  if(!this->symbolTable.existsSymbol(id, true)) {
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Undeclared variable " + id + ".");
      return new Variable(id, Data::UNKNOWN); //Creates variable node anyway
  } else if(!this->symbolTable.isSymbolInitialized(id, true)) {
      ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Variable " + id + " used but not initialized.");
  }

  TreeNode* t = this->symbolTable.getSymbol(id, true).getData();
  if (index != NULL and t->classType() == TreeNode::ARRAY) {
    Array* array = (Array*)t;
    if (array != NULL and array->getSize()->classType() == TreeNode::INTEGER) {
      if(((Integer*)index)->getValue() > ((Integer*)(array->getSize()))->getValue() || ((Integer*)index)->getValue() < 0)
        ERROR_LOGGER->log(ErrorLogger::SEMANTIC, "Index out of bounds " + id + ".");
    }
    return new Array(id, this->symbolTable.getSymbol(id,true).getDataType(), index);
  }
  Variable* v = new Variable(id, this->symbolTable.getSymbol(id,true).getDataType());
  v->setSymbolTable(this->symbolTable);
  return v;
}
<|endoftext|>