<commit_before>/*
 * Copyright (C) 2017 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <boost/test/unit_test.hpp>
#include "flat_mutation_reader.hh"

// Intended to be called in a seastar thread
class flat_reader_assertions {
    flat_mutation_reader _reader;
    dht::partition_range _pr;
private:
    mutation_fragment_opt read_next() {
        return _reader().get0();
    }
public:
    flat_reader_assertions(flat_mutation_reader reader)
        : _reader(std::move(reader))
    { }

    flat_reader_assertions& produces_partition_start(const dht::decorated_key& dk,
                                                     stdx::optional<tombstone> tomb = stdx::nullopt) {
        BOOST_TEST_MESSAGE(sprint("Expecting partition start with key %s", dk));
        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL(sprint("Expected: partition start with key %s, got end of stream", dk));
        }
        if (!mfopt->is_partition_start()) {
            BOOST_FAIL(sprint("Expected: partition start with key %s, got: %s", dk, *mfopt));
        }
        if (!mfopt->as_partition_start().key().equal(*_reader.schema(), dk)) {
            BOOST_FAIL(sprint("Expected: partition start with key %s, got: %s", dk, *mfopt));
        }
        if (tomb && mfopt->as_partition_start().partition_tombstone() != *tomb) {
            BOOST_FAIL(sprint("Expected: partition start with tombstone %s, got: %s", *tomb, *mfopt));
        }
        return *this;
    }

    flat_reader_assertions& produces_static_row() {
        BOOST_TEST_MESSAGE(sprint("Expecting static row"));
        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL("Expected static row, got end of stream");
        }
        if (!mfopt->is_static_row()) {
            BOOST_FAIL(sprint("Expected static row, got: %s", *mfopt));
        }
        return *this;
    }

    flat_reader_assertions& produces_row_with_key(const clustering_key& ck) {
        BOOST_TEST_MESSAGE(sprint("Expect %s", ck));
        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL(sprint("Expected row with key %s, but got end of stream", ck));
        }
        if (!mfopt->is_clustering_row()) {
            BOOST_FAIL(sprint("Expected row with key %s, but got %s", ck, *mfopt));
        }
        auto& actual = mfopt->as_clustering_row().key();
        if (!actual.equal(*_reader.schema(), ck)) {
            BOOST_FAIL(sprint("Expected row with key %s, but key is %s", ck, actual));
        }
        return *this;
    }

    flat_reader_assertions& produces_partition_end() {
        BOOST_TEST_MESSAGE("Expecting partition end");
        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL(sprint("Expected partition end but got end of stream"));
        }
        if (!mfopt->is_end_of_partition()) {
            BOOST_FAIL(sprint("Expected partition end but got %s", *mfopt));
        }
        return *this;
    }

    flat_reader_assertions& produces_end_of_stream() {
        BOOST_TEST_MESSAGE("Expecting end of stream");
        auto mfopt = read_next();
        if (bool(mfopt)) {
            BOOST_FAIL(sprint("Expected end of stream, got %s", *mfopt));
        }
        return *this;
    }

    flat_reader_assertions& produces(mutation_fragment::kind k, std::vector<int> ck_elements) {
        std::vector<bytes> ck_bytes;
        for (auto&& e : ck_elements) {
            ck_bytes.emplace_back(int32_type->decompose(e));
        }
        auto ck = clustering_key_prefix::from_exploded(*_reader.schema(), std::move(ck_bytes));

        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL(sprint("Expected mutation fragment %s, got end of stream", ck));
        }
        if (mfopt->mutation_fragment_kind() != k) {
            BOOST_FAIL(sprint("Expected mutation fragment kind %s, got: %s", k, mfopt->mutation_fragment_kind()));
        }
        clustering_key::equality ck_eq(*_reader.schema());
        if (!ck_eq(mfopt->key(), ck)) {
            BOOST_FAIL(sprint("Expected key %s, got: %s", ck, mfopt->key()));
        }
        return *this;
    }

    flat_reader_assertions& produces_partition(const mutation& m) {
        return produces(m);
    }

    flat_reader_assertions& produces(const mutation& m) {
        auto mo = read_mutation_from_flat_mutation_reader(_reader).get0();
        BOOST_REQUIRE(bool(mo));
        memory::disable_failure_guard dfg;
        assert_that(*mo).is_equal_to(m);
        return *this;
    }

    flat_reader_assertions& produces(const dht::decorated_key& dk) {
        produces_partition_start(dk);
        next_partition();
        return *this;
    }

    template<typename Range>
    flat_reader_assertions& produces(const Range& range) {
        for (auto&& m : range) {
            produces(m);
        }
        return *this;
    }

    void has_monotonic_positions() {
        position_in_partition::less_compare less(*_reader.schema());
        mutation_fragment_opt previous_fragment;
        mutation_fragment_opt previous_partition;
        bool inside_partition = false;
        for (;;) {
            auto mfo = read_next();
            if (!mfo) {
                break;
            }
            if (mfo->is_partition_start()) {
                BOOST_REQUIRE(!inside_partition);
                auto& dk = mfo->as_partition_start().key();
                if (previous_partition && !previous_partition->as_partition_start().key().less_compare(*_reader.schema(), dk)) {
                    BOOST_FAIL(sprint("previous partition had greater key: prev=%s, current=%s", *previous_partition, *mfo));
                }
                previous_partition = std::move(mfo);
                previous_fragment = stdx::nullopt;
                inside_partition = true;
            } else if (mfo->is_end_of_partition()) {
                BOOST_REQUIRE(inside_partition);
                inside_partition = false;
            } else {
                BOOST_REQUIRE(inside_partition);
                if (previous_fragment) {
                    if (!less(previous_fragment->position(), mfo->position())) {
                        BOOST_FAIL(sprint("previous fragment has greater position: prev=%s, current=%s", *previous_fragment, *mfo));
                    }
                }
                previous_fragment = std::move(mfo);
            }
        }
        BOOST_REQUIRE(!inside_partition);
    }

    flat_reader_assertions& fast_forward_to(const dht::partition_range& pr) {
        _pr = pr;
        _reader.fast_forward_to(_pr).get();
        return *this;
    }

    flat_reader_assertions& next_partition() {
        _reader.next_partition();
        return *this;
    }

    flat_reader_assertions& fast_forward_to(position_range pr) {
        _reader.fast_forward_to(std::move(pr)).get();
        return *this;
    }

    mutation_assertion next_mutation() {
        auto mo = read_mutation_from_flat_mutation_reader(_reader).get0();
        BOOST_REQUIRE(bool(mo));
        return mutation_assertion(std::move(*mo));
    }

    future<> fill_buffer() {
        return _reader.fill_buffer();
    }

    void set_max_buffer_size(size_t size) {
        _reader.set_max_buffer_size(size);
    }
};

inline
flat_reader_assertions assert_that(flat_mutation_reader r) {
    return { std::move(r) };
}
<commit_msg>Add flat_mutation_reader_assertions::produces_range_tombstone<commit_after>/*
 * Copyright (C) 2017 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <boost/test/unit_test.hpp>
#include "flat_mutation_reader.hh"

// Intended to be called in a seastar thread
class flat_reader_assertions {
    flat_mutation_reader _reader;
    dht::partition_range _pr;
private:
    mutation_fragment_opt read_next() {
        return _reader().get0();
    }
public:
    flat_reader_assertions(flat_mutation_reader reader)
        : _reader(std::move(reader))
    { }

    flat_reader_assertions& produces_partition_start(const dht::decorated_key& dk,
                                                     stdx::optional<tombstone> tomb = stdx::nullopt) {
        BOOST_TEST_MESSAGE(sprint("Expecting partition start with key %s", dk));
        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL(sprint("Expected: partition start with key %s, got end of stream", dk));
        }
        if (!mfopt->is_partition_start()) {
            BOOST_FAIL(sprint("Expected: partition start with key %s, got: %s", dk, *mfopt));
        }
        if (!mfopt->as_partition_start().key().equal(*_reader.schema(), dk)) {
            BOOST_FAIL(sprint("Expected: partition start with key %s, got: %s", dk, *mfopt));
        }
        if (tomb && mfopt->as_partition_start().partition_tombstone() != *tomb) {
            BOOST_FAIL(sprint("Expected: partition start with tombstone %s, got: %s", *tomb, *mfopt));
        }
        return *this;
    }

    flat_reader_assertions& produces_static_row() {
        BOOST_TEST_MESSAGE(sprint("Expecting static row"));
        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL("Expected static row, got end of stream");
        }
        if (!mfopt->is_static_row()) {
            BOOST_FAIL(sprint("Expected static row, got: %s", *mfopt));
        }
        return *this;
    }

    flat_reader_assertions& produces_row_with_key(const clustering_key& ck) {
        BOOST_TEST_MESSAGE(sprint("Expect %s", ck));
        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL(sprint("Expected row with key %s, but got end of stream", ck));
        }
        if (!mfopt->is_clustering_row()) {
            BOOST_FAIL(sprint("Expected row with key %s, but got %s", ck, *mfopt));
        }
        auto& actual = mfopt->as_clustering_row().key();
        if (!actual.equal(*_reader.schema(), ck)) {
            BOOST_FAIL(sprint("Expected row with key %s, but key is %s", ck, actual));
        }
        return *this;
    }

    // If ck_ranges is passed, verifies only that information relevant for ck_ranges matches.
    flat_reader_assertions& produces_range_tombstone(const range_tombstone& rt, const query::clustering_row_ranges& ck_ranges = {}) {
        BOOST_TEST_MESSAGE(sprint("Expect %s", rt));
        auto mfo = read_next();
        if (!mfo) {
            BOOST_FAIL(sprint("Expected range tombstone %s, but got end of stream", rt));
        }
        if (!mfo->is_range_tombstone()) {
            BOOST_FAIL(sprint("Expected range tombstone %s, but got %s", rt, *mfo));
        }
        const schema& s = *_reader.schema();
        range_tombstone_list actual_list(s);
        position_in_partition::equal_compare eq(s);
        while (mutation_fragment* next = _reader.peek().get0()) {
            if (!next->is_range_tombstone() || !eq(next->position(), mfo->position())) {
                break;
            }
            actual_list.apply(s, _reader().get0()->as_range_tombstone());
        }
        actual_list.apply(s, mfo->as_range_tombstone());
        {
            range_tombstone_list expected_list(s);
            expected_list.apply(s, rt);
            actual_list.trim(s, ck_ranges);
            expected_list.trim(s, ck_ranges);
            if (!actual_list.equal(s, expected_list)) {
                BOOST_FAIL(sprint("Expected %s, but got %s", expected_list, actual_list));
            }
        }
        return *this;
    }

    flat_reader_assertions& produces_partition_end() {
        BOOST_TEST_MESSAGE("Expecting partition end");
        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL(sprint("Expected partition end but got end of stream"));
        }
        if (!mfopt->is_end_of_partition()) {
            BOOST_FAIL(sprint("Expected partition end but got %s", *mfopt));
        }
        return *this;
    }

    flat_reader_assertions& produces_end_of_stream() {
        BOOST_TEST_MESSAGE("Expecting end of stream");
        auto mfopt = read_next();
        if (bool(mfopt)) {
            BOOST_FAIL(sprint("Expected end of stream, got %s", *mfopt));
        }
        return *this;
    }

    flat_reader_assertions& produces(mutation_fragment::kind k, std::vector<int> ck_elements) {
        std::vector<bytes> ck_bytes;
        for (auto&& e : ck_elements) {
            ck_bytes.emplace_back(int32_type->decompose(e));
        }
        auto ck = clustering_key_prefix::from_exploded(*_reader.schema(), std::move(ck_bytes));

        auto mfopt = read_next();
        if (!mfopt) {
            BOOST_FAIL(sprint("Expected mutation fragment %s, got end of stream", ck));
        }
        if (mfopt->mutation_fragment_kind() != k) {
            BOOST_FAIL(sprint("Expected mutation fragment kind %s, got: %s", k, mfopt->mutation_fragment_kind()));
        }
        clustering_key::equality ck_eq(*_reader.schema());
        if (!ck_eq(mfopt->key(), ck)) {
            BOOST_FAIL(sprint("Expected key %s, got: %s", ck, mfopt->key()));
        }
        return *this;
    }

    flat_reader_assertions& produces_partition(const mutation& m) {
        return produces(m);
    }

    flat_reader_assertions& produces(const mutation& m) {
        auto mo = read_mutation_from_flat_mutation_reader(_reader).get0();
        BOOST_REQUIRE(bool(mo));
        memory::disable_failure_guard dfg;
        assert_that(*mo).is_equal_to(m);
        return *this;
    }

    flat_reader_assertions& produces(const dht::decorated_key& dk) {
        produces_partition_start(dk);
        next_partition();
        return *this;
    }

    template<typename Range>
    flat_reader_assertions& produces(const Range& range) {
        for (auto&& m : range) {
            produces(m);
        }
        return *this;
    }

    void has_monotonic_positions() {
        position_in_partition::less_compare less(*_reader.schema());
        mutation_fragment_opt previous_fragment;
        mutation_fragment_opt previous_partition;
        bool inside_partition = false;
        for (;;) {
            auto mfo = read_next();
            if (!mfo) {
                break;
            }
            if (mfo->is_partition_start()) {
                BOOST_REQUIRE(!inside_partition);
                auto& dk = mfo->as_partition_start().key();
                if (previous_partition && !previous_partition->as_partition_start().key().less_compare(*_reader.schema(), dk)) {
                    BOOST_FAIL(sprint("previous partition had greater key: prev=%s, current=%s", *previous_partition, *mfo));
                }
                previous_partition = std::move(mfo);
                previous_fragment = stdx::nullopt;
                inside_partition = true;
            } else if (mfo->is_end_of_partition()) {
                BOOST_REQUIRE(inside_partition);
                inside_partition = false;
            } else {
                BOOST_REQUIRE(inside_partition);
                if (previous_fragment) {
                    if (!less(previous_fragment->position(), mfo->position())) {
                        BOOST_FAIL(sprint("previous fragment has greater position: prev=%s, current=%s", *previous_fragment, *mfo));
                    }
                }
                previous_fragment = std::move(mfo);
            }
        }
        BOOST_REQUIRE(!inside_partition);
    }

    flat_reader_assertions& fast_forward_to(const dht::partition_range& pr) {
        _pr = pr;
        _reader.fast_forward_to(_pr).get();
        return *this;
    }

    flat_reader_assertions& next_partition() {
        _reader.next_partition();
        return *this;
    }

    flat_reader_assertions& fast_forward_to(position_range pr) {
        _reader.fast_forward_to(std::move(pr)).get();
        return *this;
    }

    mutation_assertion next_mutation() {
        auto mo = read_mutation_from_flat_mutation_reader(_reader).get0();
        BOOST_REQUIRE(bool(mo));
        return mutation_assertion(std::move(*mo));
    }

    future<> fill_buffer() {
        return _reader.fill_buffer();
    }

    void set_max_buffer_size(size_t size) {
        _reader.set_max_buffer_size(size);
    }
};

inline
flat_reader_assertions assert_that(flat_mutation_reader r) {
    return { std::move(r) };
}
<|endoftext|>


<commit_before>#pragma once

#include "host_interface.hpp"
#include "pel.hpp"
#include "repository.hpp"

#include <deque>
#include <sdeventplus/clock.hpp>
#include <sdeventplus/source/event.hpp>
#include <sdeventplus/utility/timer.hpp>

namespace openpower::pels
{

/**
 * @class HostNotifier
 *
 * This class handles notifying the host firmware of new PELs.
 */
class HostNotifier
{
  public:
    HostNotifier() = delete;
    HostNotifier(const HostNotifier&) = delete;
    HostNotifier& operator=(const HostNotifier&) = delete;
    HostNotifier(HostNotifier&&) = delete;
    HostNotifier& operator=(HostNotifier&&) = delete;

    /**
     * @brief Constructor
     *
     * @param[in] repo - The PEL repository object
     * @param[in] dataIface - The data interface object
     * @param[in] hostIface - The host interface object
     */
    HostNotifier(Repository& repo, DataInterfaceBase& dataIface,
                 std::unique_ptr<HostInterface> hostIface);

    /**
     * @brief Destructor
     */
    ~HostNotifier();

    /**
     * @brief Returns the PEL queue size.
     *
     * For testing.
     *
     * @return size_t - The queue size
     */
    size_t queueSize() const
    {
        return _pelQueue.size();
    }

    /**
     * @brief Specifies if the PEL needs to go onto the queue to be
     *        set to the host.
     *
     * Only returns false if:
     *  - Already acked by the host (or they didn't like it)
     *  - Hidden and the HMC already got it
     *  - The 'do not report to host' bit is set
     *
     * @param[in] id - The PEL ID
     *
     * @return bool - If enqueue is required
     */
    bool enqueueRequired(uint32_t id) const;

    /**
     * @brief If the host still needs to be notified of the PEL
     *        at the time of the notification.
     *
     * Only returns false if:
     *  - Already acked by the host
     *  - It's hidden, and the HMC already got or will get it.
     *
     * @param[in] id - The PEL ID
     *
     * @return bool - If the notify is required
     */
    bool notifyRequired(uint32_t id) const;

    /**
     * @brief Called when the host sends the 'ack' PLDM command.
     *
     * This means the PEL never needs to be sent up again.
     *
     * If the host was previously full, it is also an indication
     * it no longer is.
     *
     * @param[in] id - The PEL ID
     */
    void ackPEL(uint32_t id);

    /**
     * @brief Called when the host does not have room for more
     *        PELs at this time.
     *
     * This can happen when an OS isn't running yet, and the
     * staging area to hold the PELs before sending them up
     * to the OS is full.  This will stop future PEls from being
     * sent up, as explained below.
     *
     * The PEL with this ID will need to be sent again, so its
     * state is set back to 'new', and it is removed from the list
     * of already sent PELs.
     *
     * A timer will be started, if it isn't already running, to
     * issue another send in the hopes that space has been freed
     * up by then (Receiving an ackPEL response is also an
     * indication of this if there happened to have been other
     * PELs in flight).
     *
     * @param[in] id - The PEL ID
     */
    void setHostFull(uint32_t id);

    /**
     * @brief Called when the host receives a malformed PEL.
     *
     * Ideally this will never happen, as the Repository
     * class already purges malformed PELs.
     *
     * The PEL should never be sent up again.
     *
     * @param[in] id - The PEL ID
     */
    void setBadPEL(uint32_t id);

  private:
    /**
     * @brief This function gets called by the Repository class
     *        when a new PEL is added to it.
     *
     * This function puts the PEL on the queue to be sent up if it
     * needs it, and possibly dispatch the send if the conditions call
     * for it.
     *
     * @param[in] pel - The new PEL
     */
    void newLogCallback(const PEL& pel);

    /**
     * @brief This function runs on every existing PEL at startup
     *        and puts the PEL on the queue to send if necessary.
     *
     * @param[in] pel - The PEL
     *
     * @return bool - This is an indicator to the Repository::for_each
     *                function to traverse every PEL.  Always false.
     */
    bool addPELToQueue(const PEL& pel);

    /**
     * @brief Takes the first PEL from the queue that needs to be
     *        sent, and issues the send if conditions are right.
     */
    void doNewLogNotify();

    /**
     * @brief Creates the event object to handle sending the PLDM
     *        command from the event loop.
     */
    void scheduleDispatch();

    /**
     * @brief Kicks off the PLDM send, but called from the event
     *        loop.
     *
     * @param[in] source - The event source object
     */
    void dispatch(sdeventplus::source::EventBase& source);

    /**
     * @brief Called when the host changes state.
     *
     * If the new state is host up and there are PELs to send, it
     * will trigger the first command.  If the new state is off, then
     * it will transfer any PELs that were sent but not acked yet back
     * to the queue to be sent again.
     *
     * @param[in] hostUp - The new host state
     */
    void hostStateChange(bool hostUp);

    /**
     * @brief The callback function invoked after the asynchronous
     *        PLDM receive function is complete.
     *
     * If the command was successful, the state of that PEL will
     * be set to 'sent', and the next send will be triggered.
     *
     * If the command failed, a retry timer will be started so it
     * can be sent again.
     *
     * @param[in] status - The response status
     */
    void commandResponse(ResponseStatus status);

    /**
     * @brief The function called when the command failure retry
     *        time is up.
     *
     * It will issue a send of the previous PEL and increment the
     * retry count.
     */
    void retryTimerExpired();

    /**
     * @brief The function called when the 'host full' retry timer
     *        expires.
     *
     * This will re-issue a command to try again with the PEL at
     * the front of the queue.
     */
    void hostFullTimerExpired();

    /**
     * @brief Stops an in progress command
     *
     * In progress meaning after the send but before the response.
     */
    void stopCommand();

    /**
     * @brief The PEL repository object
     */
    Repository& _repo;

    /**
     * @brief The data interface object
     */
    DataInterfaceBase& _dataIface;

    /**
     * @brief Base class pointer for the host command interface
     */
    std::unique_ptr<HostInterface> _hostIface;

    /**
     * @brief The list of PEL IDs that need to be sent.
     */
    std::deque<uint32_t> _pelQueue;

    /**
     * @brief The list of IDs that were sent, but not acked yet.
     *
     * These move back to _pelQueue on a power off.
     */
    std::vector<uint32_t> _sentPELs;

    /**
     * @brief The ID the PEL where the notification has
     *        been kicked off but the asynchronous response
     *        hasn't been received yet.
     */
    uint32_t _inProgressPEL = 0;

    /**
     * @brief The command retry count
     */
    size_t _retryCount = 0;

    /**
     * @brief Indicates if the host has said it is full and does not
     *        currently have the space for more PELs.
     */
    bool _hostFull = false;

    /**
     * @brief The command retry timer.
     */
    sdeventplus::utility::Timer<sdeventplus::ClockId::Monotonic> _retryTimer;

    /**
     * @brief The host full timer, used to retry sending a PEL if the host
     *        said it is full.
     */
    sdeventplus::utility::Timer<sdeventplus::ClockId::Monotonic> _hostFullTimer;

    /**
     * @brief The object used to dispatch a new PEL send from the
     *        event loop, so the calling function can be returned from
     *        first.
     */
    std::unique_ptr<sdeventplus::source::Defer> _dispatcher;
};

} // namespace openpower::pels
<commit_msg>PEL: Add additional HostNotifier documentation<commit_after>#pragma once

#include "host_interface.hpp"
#include "pel.hpp"
#include "repository.hpp"

#include <deque>
#include <sdeventplus/clock.hpp>
#include <sdeventplus/source/event.hpp>
#include <sdeventplus/utility/timer.hpp>

namespace openpower::pels
{

/**
 * @class HostNotifier
 *
 * This class handles notifying the host firmware of new PELs.
 *
 * It uses the Repository class's subscription feature to be
 * notified about new PELs.
 *
 * Some PELs do not need to be sent - see enqueueRequired() and
 * notifyRequired().
 *
 * The high level good path flow for sending a single PEL is:
 *
 * 1) Send the ID and size of the new PEL to the host.
 *   - The command response is asynchronous.
 *
 * 2) The host reads the raw PEL data (outside of this class).
 *
 * 3) The host sends the PEL to the OS, and then sends an AckPEL
 *    PLDM command to the PLDM daemon, who makes a D-Bus method
 *    call to this daemon, which calls HostNotifer::ackPEL().
 *
 *    After this, a PEL never needs to be sent again, but if the
 *    host is rebooted before the ack comes it will.
 *
 * The host firmware has a finite amount of space to store PELs before
 * sending to the OS, and it's possible it will fill up.  In this case,
 * the AckPEL command will have a special response that will tell the
 * PLDM daemon to call  HostReject D-Bus method on this daemon instead
 * which will invoke HostNotifier::setHostFull(). This will stop new
 * PELs from being sent, and the first PEL that hits this will have
 * a timer set to retry again later.
 */
class HostNotifier
{
  public:
    HostNotifier() = delete;
    HostNotifier(const HostNotifier&) = delete;
    HostNotifier& operator=(const HostNotifier&) = delete;
    HostNotifier(HostNotifier&&) = delete;
    HostNotifier& operator=(HostNotifier&&) = delete;

    /**
     * @brief Constructor
     *
     * @param[in] repo - The PEL repository object
     * @param[in] dataIface - The data interface object
     * @param[in] hostIface - The host interface object
     */
    HostNotifier(Repository& repo, DataInterfaceBase& dataIface,
                 std::unique_ptr<HostInterface> hostIface);

    /**
     * @brief Destructor
     */
    ~HostNotifier();

    /**
     * @brief Returns the PEL queue size.
     *
     * For testing.
     *
     * @return size_t - The queue size
     */
    size_t queueSize() const
    {
        return _pelQueue.size();
    }

    /**
     * @brief Specifies if the PEL needs to go onto the queue to be
     *        set to the host.
     *
     * Only returns false if:
     *  - Already acked by the host (or they didn't like it)
     *  - Hidden and the HMC already got it
     *  - The 'do not report to host' bit is set
     *
     * @param[in] id - The PEL ID
     *
     * @return bool - If enqueue is required
     */
    bool enqueueRequired(uint32_t id) const;

    /**
     * @brief If the host still needs to be notified of the PEL
     *        at the time of the notification.
     *
     * Only returns false if:
     *  - Already acked by the host
     *  - It's hidden, and the HMC already got or will get it.
     *
     * @param[in] id - The PEL ID
     *
     * @return bool - If the notify is required
     */
    bool notifyRequired(uint32_t id) const;

    /**
     * @brief Called when the host sends the 'ack' PLDM command.
     *
     * This means the PEL never needs to be sent up again.
     *
     * If the host was previously full, it is also an indication
     * it no longer is.
     *
     * @param[in] id - The PEL ID
     */
    void ackPEL(uint32_t id);

    /**
     * @brief Called when the host does not have room for more
     *        PELs at this time.
     *
     * This can happen when an OS isn't running yet, and the
     * staging area to hold the PELs before sending them up
     * to the OS is full.  This will stop future PEls from being
     * sent up, as explained below.
     *
     * The PEL with this ID will need to be sent again, so its
     * state is set back to 'new', and it is removed from the list
     * of already sent PELs.
     *
     * A timer will be started, if it isn't already running, to
     * issue another send in the hopes that space has been freed
     * up by then (Receiving an ackPEL response is also an
     * indication of this if there happened to have been other
     * PELs in flight).
     *
     * @param[in] id - The PEL ID
     */
    void setHostFull(uint32_t id);

    /**
     * @brief Called when the host receives a malformed PEL.
     *
     * Ideally this will never happen, as the Repository
     * class already purges malformed PELs.
     *
     * The PEL should never be sent up again.
     *
     * @param[in] id - The PEL ID
     */
    void setBadPEL(uint32_t id);

  private:
    /**
     * @brief This function gets called by the Repository class
     *        when a new PEL is added to it.
     *
     * This function puts the PEL on the queue to be sent up if it
     * needs it, and possibly dispatch the send if the conditions call
     * for it.
     *
     * @param[in] pel - The new PEL
     */
    void newLogCallback(const PEL& pel);

    /**
     * @brief This function runs on every existing PEL at startup
     *        and puts the PEL on the queue to send if necessary.
     *
     * @param[in] pel - The PEL
     *
     * @return bool - This is an indicator to the Repository::for_each
     *                function to traverse every PEL.  Always false.
     */
    bool addPELToQueue(const PEL& pel);

    /**
     * @brief Takes the first PEL from the queue that needs to be
     *        sent, and issues the send if conditions are right.
     */
    void doNewLogNotify();

    /**
     * @brief Creates the event object to handle sending the PLDM
     *        command from the event loop.
     */
    void scheduleDispatch();

    /**
     * @brief Kicks off the PLDM send, but called from the event
     *        loop.
     *
     * @param[in] source - The event source object
     */
    void dispatch(sdeventplus::source::EventBase& source);

    /**
     * @brief Called when the host changes state.
     *
     * If the new state is host up and there are PELs to send, it
     * will trigger the first command.  If the new state is off, then
     * it will transfer any PELs that were sent but not acked yet back
     * to the queue to be sent again.
     *
     * @param[in] hostUp - The new host state
     */
    void hostStateChange(bool hostUp);

    /**
     * @brief The callback function invoked after the asynchronous
     *        PLDM receive function is complete.
     *
     * If the command was successful, the state of that PEL will
     * be set to 'sent', and the next send will be triggered.
     *
     * If the command failed, a retry timer will be started so it
     * can be sent again.
     *
     * @param[in] status - The response status
     */
    void commandResponse(ResponseStatus status);

    /**
     * @brief The function called when the command failure retry
     *        time is up.
     *
     * It will issue a send of the previous PEL and increment the
     * retry count.
     */
    void retryTimerExpired();

    /**
     * @brief The function called when the 'host full' retry timer
     *        expires.
     *
     * This will re-issue a command to try again with the PEL at
     * the front of the queue.
     */
    void hostFullTimerExpired();

    /**
     * @brief Stops an in progress command
     *
     * In progress meaning after the send but before the response.
     */
    void stopCommand();

    /**
     * @brief The PEL repository object
     */
    Repository& _repo;

    /**
     * @brief The data interface object
     */
    DataInterfaceBase& _dataIface;

    /**
     * @brief Base class pointer for the host command interface
     */
    std::unique_ptr<HostInterface> _hostIface;

    /**
     * @brief The list of PEL IDs that need to be sent.
     */
    std::deque<uint32_t> _pelQueue;

    /**
     * @brief The list of IDs that were sent, but not acked yet.
     *
     * These move back to _pelQueue on a power off.
     */
    std::vector<uint32_t> _sentPELs;

    /**
     * @brief The ID the PEL where the notification has
     *        been kicked off but the asynchronous response
     *        hasn't been received yet.
     */
    uint32_t _inProgressPEL = 0;

    /**
     * @brief The command retry count
     */
    size_t _retryCount = 0;

    /**
     * @brief Indicates if the host has said it is full and does not
     *        currently have the space for more PELs.
     */
    bool _hostFull = false;

    /**
     * @brief The command retry timer.
     */
    sdeventplus::utility::Timer<sdeventplus::ClockId::Monotonic> _retryTimer;

    /**
     * @brief The host full timer, used to retry sending a PEL if the host
     *        said it is full.
     */
    sdeventplus::utility::Timer<sdeventplus::ClockId::Monotonic> _hostFullTimer;

    /**
     * @brief The object used to dispatch a new PEL send from the
     *        event loop, so the calling function can be returned from
     *        first.
     */
    std::unique_ptr<sdeventplus::source::Defer> _dispatcher;
};

} // namespace openpower::pels
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include <fstream>

#include <sys/stat.h>

#include "cpp_utils/stop_watch.hpp"

#include "trainer/rbm_training_context.hpp"
#include "layer_traits.hpp"
#include "dbn_traits.hpp"

namespace dll {

template <typename R>
struct default_rbm_watcher {
    cpp::stop_watch<std::chrono::seconds> watch;

    template <typename RBM = R>
    void training_begin(const RBM& rbm) {
        using rbm_t = std::decay_t<RBM>;

        std::cout << "Train RBM with \"" << RBM::desc::template trainer_t<RBM, false>::name() << "\"" << std::endl;

        rbm.display();

        std::cout << "With parameters:" << std::endl;

        if(std::is_same<typename rbm_t::weight, float>::value){
            std::cout << "   single-precision" << std::endl;
        } else if(std::is_same<typename rbm_t::weight, double>::value){
            std::cout << "   double-precision" << std::endl;
        } else {
            std::cout << "   unknown-precision (something is wrong...)" << std::endl;
        }

        std::cout << "   learning_rate=" << rbm.learning_rate << std::endl;
        std::cout << "   batch_size=" << get_batch_size(rbm) << std::endl;

        if (rbm_layer_traits<RBM>::has_momentum()) {
            std::cout << "   momentum=" << rbm.momentum << std::endl;
        }

        if (rbm_layer_traits<RBM>::has_clip_gradients()) {
            std::cout << "   gradient clip=" << rbm.gradient_clip << std::endl;
        }

        if (w_decay(rbm_layer_traits<RBM>::decay()) == decay_type::L1 || w_decay(rbm_layer_traits<RBM>::decay()) == decay_type::L1L2) {
            std::cout << "   weight_cost(L1)=" << rbm.l1_weight_cost << std::endl;
        }

        if (w_decay(rbm_layer_traits<RBM>::decay()) == decay_type::L2 || w_decay(rbm_layer_traits<RBM>::decay()) == decay_type::L1L2) {
            std::cout << "   weight_cost(L2)=" << rbm.l2_weight_cost << std::endl;
        }

        if (rbm_layer_traits<RBM>::sparsity_method() == sparsity_method::LEE) {
            std::cout << "   Sparsity (Lee): pbias=" << rbm.pbias << std::endl;
            std::cout << "   Sparsity (Lee): pbias_lambda=" << rbm.pbias_lambda << std::endl;
        } else if (rbm_layer_traits<RBM>::sparsity_method() == sparsity_method::GLOBAL_TARGET) {
            std::cout << "   sparsity_target(Global)=" << rbm.sparsity_target << std::endl;
        } else if (rbm_layer_traits<RBM>::sparsity_method() == sparsity_method::LOCAL_TARGET) {
            std::cout << "   sparsity_target(Local)=" << rbm.sparsity_target << std::endl;
        }
    }

    template <typename RBM = R>
    void epoch_end(size_t epoch, const rbm_training_context& context, const RBM& /*rbm*/) {
        char formatted[1024];
        if (rbm_layer_traits<RBM>::free_energy()) {
            snprintf(formatted, 1024, "epoch %ld - Reconstruction error: %.5f - Free energy: %.3f - Sparsity: %.5f", epoch,
                     context.reconstruction_error, context.free_energy, context.sparsity);
        } else {
            snprintf(formatted, 1024, "epoch %ld - Reconstruction error: %.5f - Sparsity: %.5f", epoch, context.reconstruction_error, context.sparsity);
        }
        std::cout << formatted << std::endl;
    }

    template <typename RBM = R>
    void batch_end(const RBM& /* rbm */, const rbm_training_context& context, size_t batch, size_t batches) {
        char formatted[1024];
        sprintf(formatted, "Batch %ld/%ld - Reconstruction error: %.5f - Sparsity: %.5f", batch, batches,
                context.batch_error, context.batch_sparsity);
        std::cout << formatted << std::endl;
    }

    template <typename RBM = R>
    void training_end(const RBM&) {
        std::cout << "Training took " << watch.elapsed() << "s" << std::endl;
    }
};

template <typename DBN>
struct default_dbn_watcher {
    static constexpr bool ignore_sub  = false;
    static constexpr bool replace_sub = false;

    size_t ft_max_epochs = 0;
    dll::stop_timer ft_epoch_timer;
    dll::stop_timer ft_batch_timer;

    cpp::stop_watch<std::chrono::seconds> watch;

    void pretraining_begin(const DBN& /*dbn*/, size_t max_epochs) {
        std::cout << "DBN: Pretraining begin for " << max_epochs << " epochs" << std::endl;
    }

    template <typename RBM>
    void pretrain_layer(const DBN& /*dbn*/, size_t I, const RBM& rbm, size_t input_size) {
        if (input_size) {
            std::cout << "DBN: Pretrain layer " << I << " (" << rbm.to_short_string() << ") with " << input_size << " entries" << std::endl;
        } else {
            std::cout << "DBN: Pretrain layer " << I << " (" << rbm.to_short_string() << ")" << std::endl;
        }
    }

    void pretraining_end(const DBN& /*dbn*/) {
        std::cout << "DBN: Pretraining finished after " << watch.elapsed() << "s" << std::endl;
    }

    void pretraining_batch(const DBN& /*dbn*/, size_t batch) {
        std::cout << "DBN: Pretraining batch " << batch << std::endl;
    }

    /*!
     * \brief Fine-tuning of the given network just started
     * \param dbn The DBN that is being trained
     * \param max_epochs The maximum number of epochs to train the network
     */
    void fine_tuning_begin(const DBN& dbn, size_t max_epochs) {
        std::cout << "Train the network with \"" << DBN::desc::template trainer_t<DBN>::name() << "\"" << std::endl;
        std::cout << "With parameters:" << std::endl;
        std::cout << "          epochs=" << max_epochs << std::endl;
        std::cout << "      batch_size=" << DBN::batch_size << std::endl;
        std::cout << "   learning_rate=" << dbn.learning_rate << std::endl;

        if (dbn_traits<DBN>::has_momentum()) {
            std::cout << "   momentum=" << dbn.momentum << std::endl;
        }

        if (w_decay(dbn_traits<DBN>::decay()) == decay_type::L1 || w_decay(dbn_traits<DBN>::decay()) == decay_type::L1L2) {
            std::cout << "   weight_cost(L1)=" << dbn.l1_weight_cost << std::endl;
        }

        if (w_decay(dbn_traits<DBN>::decay()) == decay_type::L2 || w_decay(dbn_traits<DBN>::decay()) == decay_type::L1L2) {
            std::cout << "   weight_cost(L2)=" << dbn.l2_weight_cost << std::endl;
        }

        if (dbn_traits<DBN>::lr_driver() == lr_driver_type::BOLD) {
            std::cout << "   lr_driver(BOLD)=" << dbn.lr_bold_inc << ":" << dbn.lr_bold_dec << std::endl;
        }

        if (dbn_traits<DBN>::lr_driver() == lr_driver_type::STEP) {
            std::cout << "   lr_driver(STEP)=" << dbn.lr_step_size << ":" << dbn.lr_step_gamma << std::endl;
        }

        ft_max_epochs = max_epochs;
    }

    /*!
     * \brief One fine-tuning epoch is starting
     * \param epoch The current epoch
     * \param dbn The network being trained
     */
    void ft_epoch_start(size_t epoch, const DBN& dbn) {
        cpp_unused(epoch);
        cpp_unused(dbn);
        ft_epoch_timer.start();
    }

    /*!
     * \brief One fine-tuning epoch is over
     * \param epoch The current epoch
     * \param error The current error
     * \param loss The current loss
     * \param dbn The network being trained
     */
    void ft_epoch_end(size_t epoch, double error, double loss, const DBN& dbn) {
        cpp_unused(dbn);
        auto duration = ft_epoch_timer.stop();

        if /*constexpr*/ (dbn_traits<DBN>::error_on_epoch()){
            printf("Epoch %3ld/%ld - Classification error: %.5f Loss: %.5f Time %ldms \n", epoch, ft_max_epochs, error, loss, duration);
        } else {
            printf("Epoch %3ld/%ld - Loss: %.5f Time %ldms \n", epoch, ft_max_epochs, loss, duration);
        }

        std::cout.flush();
    }

    void ft_batch_start(size_t epoch, const DBN& dbn) {
        cpp_unused(epoch);
        cpp_unused(dbn);
        ft_batch_timer.start();
    }

    void ft_batch_end(size_t epoch, size_t batch, size_t batches, double batch_error, double batch_loss, const DBN&) {
        auto duration = ft_batch_timer.stop();
        printf("Epoch %3ld:%ld/%ld- B. Error: %.5f B. Loss: %.5f Time %ldms\n", epoch, batch, batches, batch_error, batch_loss, duration);
        std::cout.flush();
    }

    void ft_batch_end(size_t epoch, double batch_error, double batch_loss, const DBN&) {
        auto duration = ft_batch_timer.stop();
        printf("Epoch %3ld - B.Error: %.5f B.Loss: %.5f Time %ldms\n", epoch, batch_error, batch_loss, duration);
        std::cout.flush();
    }

    void lr_adapt(const DBN& dbn) {
        printf("driver: learning rate adapted to %.5f \n", dbn.learning_rate);
        std::cout.flush();
    }

    void fine_tuning_end(const DBN&) {
        std::cout << "Training took " << watch.elapsed() << "s" << std::endl;
    }
};

template <typename DBN>
struct silent_dbn_watcher : default_dbn_watcher<DBN> {
    static constexpr bool ignore_sub  = true;
    static constexpr bool replace_sub = false;
};

template <typename DBN>
struct mute_dbn_watcher {
    static constexpr bool ignore_sub  = true;
    static constexpr bool replace_sub = false;

    void pretraining_begin(const DBN& /*dbn*/, size_t /*max_epochs*/) {}

    template <typename RBM>
    void pretrain_layer(const DBN& /*dbn*/, size_t /*I*/, const RBM& /*rbm*/, size_t /*input_size*/) {}

    void pretraining_end(const DBN& /*dbn*/) {}

    void pretraining_batch(const DBN& /*dbn*/, size_t /*batch*/) {}

    void fine_tuning_begin(const DBN& /*dbn*/, size_t /*max_epochs*/) {}

    void ft_epoch_start(size_t /*epoch*/, const DBN& /*dbn*/) {}

    void ft_epoch_end(size_t /*epoch*/, double /*error*/, const DBN& /*dbn*/) {}

    void ft_batch_start(size_t /*epoch*/, const DBN&) {}
    void ft_batch_end(size_t /*epoch*/, size_t /*batch*/, size_t /*batches*/, double /*batch_error*/, const DBN& /*dbn*/) {}
    void ft_batch_end(size_t /*epoch*/, double /*batch_error*/, const DBN& /*dbn*/) {}

    void lr_adapt(const DBN& /*dbn*/) {}

    void fine_tuning_end(const DBN& /*dbn*/) {}
};

//TODO This is currently useless

template <typename R>
struct histogram_watcher {
    default_rbm_watcher<R> parent;

    template <typename RBM = R>
    void training_begin(const RBM& rbm) {
        parent.training_begin(rbm);
    }

    template <typename RBM = R>
    void epoch_end(size_t epoch, double error, double /*free_energy*/, const RBM& rbm) {
        parent.epoch_end(epoch, error, rbm);
    }

    template <typename RBM = R>
    void batch_end(const RBM& rbm, const rbm_training_context& context, size_t batch, size_t batches) {
        parent.batch_end(rbm, context, batch, batches);
    }

    template <typename RBM = R>
    void training_end(const RBM& rbm) {
        parent.training_end(rbm);
    }

    void generate_hidden_images(size_t epoch, const R& rbm) {
        mkdir("reports", 0777);

        auto folder = "reports/epoch_" + std::to_string(epoch);
        mkdir(folder.c_str(), 0777);

        for (size_t j = 0; j < R::num_hidden; ++j) {
            auto path = folder + "/h_" + std::to_string(j) + ".dat";
            std::ofstream file(path, std::ios::out);

            if (!file) {
                std::cout << "Could not open file " << path << std::endl;
            } else {
                size_t i = R::num_visible;
                while (i > 0) {
                    --i;

                    auto value = rbm.w(i, j);
                    file << static_cast<size_t>(value > 0 ? static_cast<size_t>(value * 255.0) << 8 : static_cast<size_t>(-value * 255.0) << 16) << " ";
                }

                file << std::endl;
                file.close();
            }
        }
    }

    void generate_histograms(size_t epoch, const R& rbm) {
        mkdir("reports", 0777);

        auto folder = "reports/epoch_" + std::to_string(epoch);
        mkdir(folder.c_str(), 0777);

        generate_histogram(folder + "/weights.dat", rbm.w);
        generate_histogram(folder + "/visibles.dat", rbm.a);
        generate_histogram(folder + "/hiddens.dat", rbm.b);
    }

    template <typename Container>
    void generate_histogram(const std::string& path, const Container& weights) {
        std::ofstream file(path, std::ios::out);

        if (!file) {
            std::cout << "Could not open file " << path << std::endl;
        } else {
            for (auto& weight : weights) {
                file << weight << std::endl;
            }

            file << std::endl;
            file.close();
        }
    }
};

} //end of dll namespace
<commit_msg>Display the loss when training<commit_after>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include <fstream>

#include <sys/stat.h>

#include "cpp_utils/stop_watch.hpp"

#include "trainer/rbm_training_context.hpp"
#include "layer_traits.hpp"
#include "dbn_traits.hpp"

namespace dll {

template <typename R>
struct default_rbm_watcher {
    cpp::stop_watch<std::chrono::seconds> watch;

    template <typename RBM = R>
    void training_begin(const RBM& rbm) {
        using rbm_t = std::decay_t<RBM>;

        std::cout << "Train RBM with \"" << RBM::desc::template trainer_t<RBM, false>::name() << "\"" << std::endl;

        rbm.display();

        std::cout << "With parameters:" << std::endl;

        if(std::is_same<typename rbm_t::weight, float>::value){
            std::cout << "   single-precision" << std::endl;
        } else if(std::is_same<typename rbm_t::weight, double>::value){
            std::cout << "   double-precision" << std::endl;
        } else {
            std::cout << "   unknown-precision (something is wrong...)" << std::endl;
        }

        std::cout << "   learning_rate=" << rbm.learning_rate << std::endl;
        std::cout << "   batch_size=" << get_batch_size(rbm) << std::endl;

        if (rbm_layer_traits<RBM>::has_momentum()) {
            std::cout << "   momentum=" << rbm.momentum << std::endl;
        }

        if (rbm_layer_traits<RBM>::has_clip_gradients()) {
            std::cout << "   gradient clip=" << rbm.gradient_clip << std::endl;
        }

        if (w_decay(rbm_layer_traits<RBM>::decay()) == decay_type::L1 || w_decay(rbm_layer_traits<RBM>::decay()) == decay_type::L1L2) {
            std::cout << "   weight_cost(L1)=" << rbm.l1_weight_cost << std::endl;
        }

        if (w_decay(rbm_layer_traits<RBM>::decay()) == decay_type::L2 || w_decay(rbm_layer_traits<RBM>::decay()) == decay_type::L1L2) {
            std::cout << "   weight_cost(L2)=" << rbm.l2_weight_cost << std::endl;
        }

        if (rbm_layer_traits<RBM>::sparsity_method() == sparsity_method::LEE) {
            std::cout << "   Sparsity (Lee): pbias=" << rbm.pbias << std::endl;
            std::cout << "   Sparsity (Lee): pbias_lambda=" << rbm.pbias_lambda << std::endl;
        } else if (rbm_layer_traits<RBM>::sparsity_method() == sparsity_method::GLOBAL_TARGET) {
            std::cout << "   sparsity_target(Global)=" << rbm.sparsity_target << std::endl;
        } else if (rbm_layer_traits<RBM>::sparsity_method() == sparsity_method::LOCAL_TARGET) {
            std::cout << "   sparsity_target(Local)=" << rbm.sparsity_target << std::endl;
        }
    }

    template <typename RBM = R>
    void epoch_end(size_t epoch, const rbm_training_context& context, const RBM& /*rbm*/) {
        char formatted[1024];
        if (rbm_layer_traits<RBM>::free_energy()) {
            snprintf(formatted, 1024, "epoch %ld - Reconstruction error: %.5f - Free energy: %.3f - Sparsity: %.5f", epoch,
                     context.reconstruction_error, context.free_energy, context.sparsity);
        } else {
            snprintf(formatted, 1024, "epoch %ld - Reconstruction error: %.5f - Sparsity: %.5f", epoch, context.reconstruction_error, context.sparsity);
        }
        std::cout << formatted << std::endl;
    }

    template <typename RBM = R>
    void batch_end(const RBM& /* rbm */, const rbm_training_context& context, size_t batch, size_t batches) {
        char formatted[1024];
        sprintf(formatted, "Batch %ld/%ld - Reconstruction error: %.5f - Sparsity: %.5f", batch, batches,
                context.batch_error, context.batch_sparsity);
        std::cout << formatted << std::endl;
    }

    template <typename RBM = R>
    void training_end(const RBM&) {
        std::cout << "Training took " << watch.elapsed() << "s" << std::endl;
    }
};

template <typename DBN>
struct default_dbn_watcher {
    static constexpr bool ignore_sub  = false;
    static constexpr bool replace_sub = false;

    size_t ft_max_epochs = 0;
    dll::stop_timer ft_epoch_timer;
    dll::stop_timer ft_batch_timer;

    cpp::stop_watch<std::chrono::seconds> watch;

    void pretraining_begin(const DBN& /*dbn*/, size_t max_epochs) {
        std::cout << "DBN: Pretraining begin for " << max_epochs << " epochs" << std::endl;
    }

    template <typename RBM>
    void pretrain_layer(const DBN& /*dbn*/, size_t I, const RBM& rbm, size_t input_size) {
        if (input_size) {
            std::cout << "DBN: Pretrain layer " << I << " (" << rbm.to_short_string() << ") with " << input_size << " entries" << std::endl;
        } else {
            std::cout << "DBN: Pretrain layer " << I << " (" << rbm.to_short_string() << ")" << std::endl;
        }
    }

    void pretraining_end(const DBN& /*dbn*/) {
        std::cout << "DBN: Pretraining finished after " << watch.elapsed() << "s" << std::endl;
    }

    void pretraining_batch(const DBN& /*dbn*/, size_t batch) {
        std::cout << "DBN: Pretraining batch " << batch << std::endl;
    }

    /*!
     * \brief Fine-tuning of the given network just started
     * \param dbn The DBN that is being trained
     * \param max_epochs The maximum number of epochs to train the network
     */
    void fine_tuning_begin(const DBN& dbn, size_t max_epochs) {
        std::cout << "Train the network with \"" << DBN::desc::template trainer_t<DBN>::name() << "\" (" << dll::to_string(DBN::loss) << ")" << std::endl;
        std::cout << "With parameters:" << std::endl;
        std::cout << "          epochs=" << max_epochs << std::endl;
        std::cout << "      batch_size=" << DBN::batch_size << std::endl;
        std::cout << "   learning_rate=" << dbn.learning_rate << std::endl;

        if (dbn_traits<DBN>::has_momentum()) {
            std::cout << "   momentum=" << dbn.momentum << std::endl;
        }

        if (w_decay(dbn_traits<DBN>::decay()) == decay_type::L1 || w_decay(dbn_traits<DBN>::decay()) == decay_type::L1L2) {
            std::cout << "   weight_cost(L1)=" << dbn.l1_weight_cost << std::endl;
        }

        if (w_decay(dbn_traits<DBN>::decay()) == decay_type::L2 || w_decay(dbn_traits<DBN>::decay()) == decay_type::L1L2) {
            std::cout << "   weight_cost(L2)=" << dbn.l2_weight_cost << std::endl;
        }

        if (dbn_traits<DBN>::lr_driver() == lr_driver_type::BOLD) {
            std::cout << "   lr_driver(BOLD)=" << dbn.lr_bold_inc << ":" << dbn.lr_bold_dec << std::endl;
        }

        if (dbn_traits<DBN>::lr_driver() == lr_driver_type::STEP) {
            std::cout << "   lr_driver(STEP)=" << dbn.lr_step_size << ":" << dbn.lr_step_gamma << std::endl;
        }

        ft_max_epochs = max_epochs;
    }

    /*!
     * \brief One fine-tuning epoch is starting
     * \param epoch The current epoch
     * \param dbn The network being trained
     */
    void ft_epoch_start(size_t epoch, const DBN& dbn) {
        cpp_unused(epoch);
        cpp_unused(dbn);
        ft_epoch_timer.start();
    }

    /*!
     * \brief One fine-tuning epoch is over
     * \param epoch The current epoch
     * \param error The current error
     * \param loss The current loss
     * \param dbn The network being trained
     */
    void ft_epoch_end(size_t epoch, double error, double loss, const DBN& dbn) {
        cpp_unused(dbn);
        auto duration = ft_epoch_timer.stop();

        if /*constexpr*/ (dbn_traits<DBN>::error_on_epoch()){
            printf("Epoch %3ld/%ld - Classification error: %.5f Loss: %.5f Time %ldms \n", epoch, ft_max_epochs, error, loss, duration);
        } else {
            printf("Epoch %3ld/%ld - Loss: %.5f Time %ldms \n", epoch, ft_max_epochs, loss, duration);
        }

        std::cout.flush();
    }

    void ft_batch_start(size_t epoch, const DBN& dbn) {
        cpp_unused(epoch);
        cpp_unused(dbn);
        ft_batch_timer.start();
    }

    void ft_batch_end(size_t epoch, size_t batch, size_t batches, double batch_error, double batch_loss, const DBN&) {
        auto duration = ft_batch_timer.stop();
        printf("Epoch %3ld:%ld/%ld- B. Error: %.5f B. Loss: %.5f Time %ldms\n", epoch, batch, batches, batch_error, batch_loss, duration);
        std::cout.flush();
    }

    void ft_batch_end(size_t epoch, double batch_error, double batch_loss, const DBN&) {
        auto duration = ft_batch_timer.stop();
        printf("Epoch %3ld - B.Error: %.5f B.Loss: %.5f Time %ldms\n", epoch, batch_error, batch_loss, duration);
        std::cout.flush();
    }

    void lr_adapt(const DBN& dbn) {
        printf("driver: learning rate adapted to %.5f \n", dbn.learning_rate);
        std::cout.flush();
    }

    void fine_tuning_end(const DBN&) {
        std::cout << "Training took " << watch.elapsed() << "s" << std::endl;
    }
};

template <typename DBN>
struct silent_dbn_watcher : default_dbn_watcher<DBN> {
    static constexpr bool ignore_sub  = true;
    static constexpr bool replace_sub = false;
};

template <typename DBN>
struct mute_dbn_watcher {
    static constexpr bool ignore_sub  = true;
    static constexpr bool replace_sub = false;

    void pretraining_begin(const DBN& /*dbn*/, size_t /*max_epochs*/) {}

    template <typename RBM>
    void pretrain_layer(const DBN& /*dbn*/, size_t /*I*/, const RBM& /*rbm*/, size_t /*input_size*/) {}

    void pretraining_end(const DBN& /*dbn*/) {}

    void pretraining_batch(const DBN& /*dbn*/, size_t /*batch*/) {}

    void fine_tuning_begin(const DBN& /*dbn*/, size_t /*max_epochs*/) {}

    void ft_epoch_start(size_t /*epoch*/, const DBN& /*dbn*/) {}

    void ft_epoch_end(size_t /*epoch*/, double /*error*/, const DBN& /*dbn*/) {}

    void ft_batch_start(size_t /*epoch*/, const DBN&) {}
    void ft_batch_end(size_t /*epoch*/, size_t /*batch*/, size_t /*batches*/, double /*batch_error*/, const DBN& /*dbn*/) {}
    void ft_batch_end(size_t /*epoch*/, double /*batch_error*/, const DBN& /*dbn*/) {}

    void lr_adapt(const DBN& /*dbn*/) {}

    void fine_tuning_end(const DBN& /*dbn*/) {}
};

//TODO This is currently useless

template <typename R>
struct histogram_watcher {
    default_rbm_watcher<R> parent;

    template <typename RBM = R>
    void training_begin(const RBM& rbm) {
        parent.training_begin(rbm);
    }

    template <typename RBM = R>
    void epoch_end(size_t epoch, double error, double /*free_energy*/, const RBM& rbm) {
        parent.epoch_end(epoch, error, rbm);
    }

    template <typename RBM = R>
    void batch_end(const RBM& rbm, const rbm_training_context& context, size_t batch, size_t batches) {
        parent.batch_end(rbm, context, batch, batches);
    }

    template <typename RBM = R>
    void training_end(const RBM& rbm) {
        parent.training_end(rbm);
    }

    void generate_hidden_images(size_t epoch, const R& rbm) {
        mkdir("reports", 0777);

        auto folder = "reports/epoch_" + std::to_string(epoch);
        mkdir(folder.c_str(), 0777);

        for (size_t j = 0; j < R::num_hidden; ++j) {
            auto path = folder + "/h_" + std::to_string(j) + ".dat";
            std::ofstream file(path, std::ios::out);

            if (!file) {
                std::cout << "Could not open file " << path << std::endl;
            } else {
                size_t i = R::num_visible;
                while (i > 0) {
                    --i;

                    auto value = rbm.w(i, j);
                    file << static_cast<size_t>(value > 0 ? static_cast<size_t>(value * 255.0) << 8 : static_cast<size_t>(-value * 255.0) << 16) << " ";
                }

                file << std::endl;
                file.close();
            }
        }
    }

    void generate_histograms(size_t epoch, const R& rbm) {
        mkdir("reports", 0777);

        auto folder = "reports/epoch_" + std::to_string(epoch);
        mkdir(folder.c_str(), 0777);

        generate_histogram(folder + "/weights.dat", rbm.w);
        generate_histogram(folder + "/visibles.dat", rbm.a);
        generate_histogram(folder + "/hiddens.dat", rbm.b);
    }

    template <typename Container>
    void generate_histogram(const std::string& path, const Container& weights) {
        std::ofstream file(path, std::ios::out);

        if (!file) {
            std::cout << "Could not open file " << path << std::endl;
        } else {
            for (auto& weight : weights) {
                file << weight << std::endl;
            }

            file << std::endl;
            file.close();
        }
    }
};

} //end of dll namespace
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2010
 * Written by:
 * Aly Hirani <alyhirani@gmail.com>
 * James Chou <uohcsemaj@gmail.com>
 *
 * All Rights Reserved
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 */

#include "precompile.h"

#include "HTTPRequest.h"
#include "Exception.h"
#include "Blob.h"

namespace Facebook
{

std::string Uri::GetUri() const
{
	if(base_uri.empty())
		throw InvalidArgument("base_uri");

	std::ostringstream oss;

	oss << base_uri;

	if(query_params.size() > 0)
	{
		oss << "?";

		// Due to the check above, we must have at least one element

		QueryParamMap::const_iterator it = query_params.begin();

		oss << curlpp::escape(it->first) << "=" << curlpp::escape(it->second);
		++it;

		while(it != query_params.end())
		{
			oss << '&';
			oss << curlpp::escape(it->first) << "=" << curlpp::escape(it->second);
			++it;
		}
	}

	return oss.str();
}

namespace HttpUtils
{

void DecomposeUri(const std::string& str, Uri& uri)
{
	uri.Clear();

	std::string::size_type lastPos = str.find('?');

	if(std::string::npos == lastPos)
	{
		uri.base_uri = str;
	}
	else
	{
		uri.base_uri = str.substr(0, lastPos);

		// XXX: This is an inefficient algorithm. Need to speed it up

		std::list<std::string> list;

		++lastPos;
		std::string::size_type pos = str.find_first_of("&=", lastPos);

		while (std::string::npos != pos || std::string::npos != lastPos)
		{
			list.push_back(str.substr(lastPos, pos - lastPos));
			lastPos = str.find_first_not_of("&=", pos);
			pos = str.find_first_of("&=", lastPos);
		}

		FACEBOOK_ASSERT(list.size() % 2 == 0);

		for(std::list<std::string>::const_iterator it = list.begin(); it != list.end();)
		{
			std::string str1 = *it++;
			std::string str2 = *it++;
			uri.query_params.insert(std::pair<std::string, std::string>(curlpp::unescape(str1), curlpp::unescape(str2)));
		}
	}
}

} // namespace HttpUtils

size_t HttpRequest::DebugFunction(curl_infotype type, char *data, size_t size)
{
	// Fix for issue #28: Only print to the debug log if we are text-based data
	//if(CURLINFO_TEXT == type)
	//	GetDebugLog().write(data, size);
	return 0;
}

size_t HttpRequest::HeaderFunction(char * /* data */, size_t size, size_t nmemb)
{
	FACEBOOK_ASSERT(blob_);

	// XXX: Implement content-type and content-length here

	return size * nmemb;
}

size_t HttpRequest::WriteFunction(char *data, size_t size, size_t nmemb)
{
	// TODO: This is a little inefficient. We should be allocating in larger chunks and reusing
	// the chunks

	FACEBOOK_ASSERT(blob_);
	FACEBOOK_ASSERT(data);

	size_t old_length = blob_->GetLength();
	blob_->Realloc(old_length + size * nmemb);
	memcpy((char*)blob_->GetData() + old_length, data, size * nmemb);
	return size * nmemb;
}

void HttpRequest::GetResponse(const Uri& uri, Blob *blob)
{
	FACEBOOK_ASSERT(blob);
	FACEBOOK_ASSERT(!blob_); // This object isn't thread-safe!

	blob_ = blob;

	// GetDebugLog() << uri.GetUri();
	curl_.setOpt(curlpp::options::Url(uri.GetUri()));
	curl_.perform();

	blob_ = NULL;
}

void HttpRequest::GetResponse(const Uri& uri, Json::Value *value)
{
	FACEBOOK_ASSERT(value);

	Blob blob;
	GetResponse(uri, &blob);

	Json::Reader reader;
	reader.parse((char*)blob.GetData(), (char*)blob.GetData() + blob.GetLength(), *value);
}

void HttpRequest::GetUri(Uri *uri) const
{
	FACEBOOK_ASSERT(uri);

	uri->query_params.insert(std::pair<std::string, std::string>("access_token", access_token_));
}

HttpRequest::HttpRequest(const std::string &access_token) : access_token_(access_token), blob_(NULL)
{
	curl_.setOpt(curlpp::Options::Verbose(true));
	curl_.setOpt(curlpp::options::DebugFunction(curlpp::types::DebugFunctionFunctor(this, &HttpRequest::DebugFunction)));
	curl_.setOpt(curlpp::Options::WriteFunction(curlpp::types::WriteFunctionFunctor(this, &HttpRequest::WriteFunction)));
	curl_.setOpt(curlpp::Options::HeaderFunction(curlpp::types::WriteFunctionFunctor(this, &HttpRequest::HeaderFunction)));
	curl_.setOpt(curlpp::Options::FollowLocation(true));
	// TODO: We shouldn't be disabling this. Instead, implementing our own Ctx
	curl_.setOpt(curlpp::options::SslVerifyPeer(false));
}
} // namespace Facebook
<commit_msg>Fixing compile errors<commit_after>/*
 * Copyright (C) 2010
 * Written by:
 * Aly Hirani <alyhirani@gmail.com>
 * James Chou <uohcsemaj@gmail.com>
 *
 * All Rights Reserved
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 */

#include "precompile.h"

#include "HTTPRequest.h"
#include "Exception.h"
#include "Blob.h"

namespace Facebook
{

std::string Uri::GetUri() const
{
	if(base_uri.empty())
		throw InvalidArgument("base_uri");

	std::ostringstream oss;

	oss << base_uri;

	if(query_params.size() > 0)
	{
		oss << "?";

		// Due to the check above, we must have at least one element

		QueryParamMap::const_iterator it = query_params.begin();

		oss << curlpp::escape(it->first) << "=" << curlpp::escape(it->second);
		++it;

		while(it != query_params.end())
		{
			oss << '&';
			oss << curlpp::escape(it->first) << "=" << curlpp::escape(it->second);
			++it;
		}
	}

	return oss.str();
}

namespace HttpUtils
{

void DecomposeUri(const std::string& str, Uri& uri)
{
	uri.Clear();

	std::string::size_type lastPos = str.find('?');

	if(std::string::npos == lastPos)
	{
		uri.base_uri = str;
	}
	else
	{
		uri.base_uri = str.substr(0, lastPos);

		// XXX: This is an inefficient algorithm. Need to speed it up

		std::list<std::string> list;

		++lastPos;
		std::string::size_type pos = str.find_first_of("&=", lastPos);

		while (std::string::npos != pos || std::string::npos != lastPos)
		{
			list.push_back(str.substr(lastPos, pos - lastPos));
			lastPos = str.find_first_not_of("&=", pos);
			pos = str.find_first_of("&=", lastPos);
		}

		FACEBOOK_ASSERT(list.size() % 2 == 0);

		for(std::list<std::string>::const_iterator it = list.begin(); it != list.end();)
		{
			std::string str1 = *it++;
			std::string str2 = *it++;
			uri.query_params.insert(std::pair<std::string, std::string>(curlpp::unescape(str1), curlpp::unescape(str2)));
		}
	}
}

} // namespace HttpUtils

size_t HttpRequest::DebugFunction(curl_infotype /* type */, char * /* data */, size_t /* size */)
{
	// Fix for issue #28: Only print to the debug log if we are text-based data
	//if(CURLINFO_TEXT == type)
	//	GetDebugLog().write(data, size);
	return 0;
}

size_t HttpRequest::HeaderFunction(char * /* data */, size_t size, size_t nmemb)
{
	FACEBOOK_ASSERT(blob_);

	// XXX: Implement content-type and content-length here

	return size * nmemb;
}

size_t HttpRequest::WriteFunction(char *data, size_t size, size_t nmemb)
{
	// TODO: This is a little inefficient. We should be allocating in larger chunks and reusing
	// the chunks

	FACEBOOK_ASSERT(blob_);
	FACEBOOK_ASSERT(data);

	size_t old_length = blob_->GetLength();
	blob_->Realloc(old_length + size * nmemb);
	memcpy((char*)blob_->GetData() + old_length, data, size * nmemb);
	return size * nmemb;
}

void HttpRequest::GetResponse(const Uri& uri, Blob *blob)
{
	FACEBOOK_ASSERT(blob);
	FACEBOOK_ASSERT(!blob_); // This object isn't thread-safe!

	blob_ = blob;

	// GetDebugLog() << uri.GetUri();
	curl_.setOpt(curlpp::options::Url(uri.GetUri()));
	curl_.perform();

	blob_ = NULL;
}

void HttpRequest::GetResponse(const Uri& uri, Json::Value *value)
{
	FACEBOOK_ASSERT(value);

	Blob blob;
	GetResponse(uri, &blob);

	Json::Reader reader;
	reader.parse((char*)blob.GetData(), (char*)blob.GetData() + blob.GetLength(), *value);
}

void HttpRequest::GetUri(Uri *uri) const
{
	FACEBOOK_ASSERT(uri);

	uri->query_params.insert(std::pair<std::string, std::string>("access_token", access_token_));
}

HttpRequest::HttpRequest(const std::string &access_token) : access_token_(access_token), blob_(NULL)
{
	curl_.setOpt(curlpp::Options::Verbose(true));
	curl_.setOpt(curlpp::options::DebugFunction(curlpp::types::DebugFunctionFunctor(this, &HttpRequest::DebugFunction)));
	curl_.setOpt(curlpp::Options::WriteFunction(curlpp::types::WriteFunctionFunctor(this, &HttpRequest::WriteFunction)));
	curl_.setOpt(curlpp::Options::HeaderFunction(curlpp::types::WriteFunctionFunctor(this, &HttpRequest::HeaderFunction)));
	curl_.setOpt(curlpp::Options::FollowLocation(true));
	// TODO: We shouldn't be disabling this. Instead, implementing our own Ctx
	curl_.setOpt(curlpp::options::SslVerifyPeer(false));
}
} // namespace Facebook
<|endoftext|>