<commit_before>/*
 * Copyright (c) 2013 Hideaki Ohno <hide.o.j55{at}gmail.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the 'Software'), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so.
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

/*
 * Copyright (C) 2011 Clearspring Technologies, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright 2015 Cloudius Systems
 *
 * Modified by Cloudius Systems
 */

#pragma once

/**
 * @file hyperloglog.hpp
 * @brief HyperLogLog cardinality estimator
 * @date Created 2013/3/20
 * @author Hideaki Ohno
 */

#include <vector>
#include <cmath>
#include <sstream>
#include <stdexcept>
#include <algorithm>
#if 0
#include "murmur3.h"
#endif

#define HLL_HASH_SEED 313

namespace hll {

static const double pow_2_32 = 4294967296.0; ///< 2^32
static const double neg_pow_2_32 = -4294967296.0; ///< -(2^32)

static inline size_t size_unsigned_var_int(unsigned int value) {
    size_t size = 0;
    while ((value & 0xFFFFFF80) != 0L) {
        size++;
        value >>= 7;
    }
    size++;
    return size;
}

static inline size_t write_unsigned_var_int(unsigned int value, uint8_t* to) {
    size_t size = 0;
    while ((value & 0xFFFFFF80) != 0L) {
        *to = (value & 0x7F) | 0x80;
        value >>= 7;
        to++;
        size++;
    }
    *to = value & 0x7F;
    size++;
    return size;
}

/** @class HyperLogLog
 *  @brief Implement of 'HyperLogLog' estimate cardinality algorithm
 */
class HyperLogLog {
public:

    /**
     * Constructor
     *
     * @param[in] b bit width (register size will be 2 to the b power).
     *            This value must be in the range[4,16].Default value is 4.
     *
     * @exception std::invalid_argument the argument is out of range.
     */
    HyperLogLog(uint8_t b = 4) throw (std::invalid_argument) :
            b_(b), m_(1 << b), M_(m_, 0) {

        if (b < 4 || 16 < b) {
            throw std::invalid_argument("bit width must be in the range [4,16]");
        }

        double alpha;
        switch (m_) {
            case 16:
                alpha = 0.673;
                break;
            case 32:
                alpha = 0.697;
                break;
            case 64:
                alpha = 0.709;
                break;
            default:
                alpha = 0.7213 / (1.0 + 1.079 / m_);
                break;
        }
        alphaMM_ = alpha * m_ * m_;
    }

    static HyperLogLog from_bytes(temporary_buffer<uint8_t> bytes) {
        // FIXME: implement class that creates a HyperLogLog from an array of bytes.
        // This will useful if we need to work with the cardinality data from the
        // compaction metadata.
        abort();
    }

    /**
     * Adds element to the estimator
     *
     * @param[in] str string to add
     * @param[in] len length of string
     */
#if 0
    void add(const char* str, uint32_t len) {
        uint32_t hash;
        MurmurHash3_x86_32(str, len, HLL_HASH_SEED, (void*) &hash);
        uint32_t index = hash >> (32 - b_);
        uint8_t rank = rho((hash << b_), 32 - b_);
        if (rank > M_[index]) {
            M_[index] = rank;
        }
    }
#endif
    void offer_hashed(uint64_t hash) {
        uint32_t index = hash >> (64 - b_);
        uint8_t rank = rho((hash << b_), 64 - b_);

        if (rank > M_[index]) {
            M_[index] = rank;
        }
    }

    /*
     * Calculate the size of buffer returned by get_bytes().
     */
    size_t get_bytes_size() {
        size_t size = 0;
        size += sizeof(int); // version
        size += size_unsigned_var_int(b_); // p; register width = b_.
        size += size_unsigned_var_int(0); // sp; // sparse set = 0.
        size += size_unsigned_var_int(0); // type;
        size += size_unsigned_var_int(M_.size()); // register size;
        size += M_.size();
        return size;
    }

    temporary_buffer<uint8_t> get_bytes() {
        // FIXME: add support to SPARSE format.
        static constexpr int version = 2;

        size_t s = get_bytes_size();
        temporary_buffer<uint8_t> bytes(s);
        size_t offset = 0;
        // write version
        *unaligned_cast<int*>(bytes.get_write() + offset) = htonl(-version);
        offset += sizeof(int);

        // write register width
        offset += write_unsigned_var_int(b_, bytes.get_write() + offset);
        // NOTE: write precision value for sparse set (not supported).
        offset += write_unsigned_var_int(0, bytes.get_write() + offset);
        // write type (NORMAL always!)
        offset += write_unsigned_var_int(0, bytes.get_write() + offset);
        // write register size
        offset += write_unsigned_var_int(M_.size(), bytes.get_write() + offset);
        // write register
        memcpy(bytes.get_write() + offset, M_.data(), M_.size());
        offset += M_.size();

        bytes.trim(offset);
        if (s != offset) {
            throw std::runtime_error("possible overflow while generating cardinality metadata");
        }
        return bytes;
    }

    /**
     * Estimates cardinality value.
     *
     * @return Estimated cardinality value.
     */
    double estimate() const {
        double estimate;
        double sum = 0.0;
        for (uint32_t i = 0; i < m_; i++) {
            sum += 1.0 / pow(2.0, M_[i]);
        }
        estimate = alphaMM_ / sum; // E in the original paper
        if (estimate <= 2.5 * m_) {
            uint32_t zeros = 0;
            for (uint32_t i = 0; i < m_; i++) {
                if (M_[i] == 0) {
                    zeros++;
                }
            }
            if (zeros != 0) {
                estimate = m_ * log(static_cast<double>(m_)/ zeros);
            }
        } else if (estimate > (1.0 / 30.0) * pow_2_32) {
            estimate = neg_pow_2_32 * log(1.0 - (estimate / pow_2_32));
        }
        return estimate;
    }

    /**
     * Merges the estimate from 'other' into this object, returning the estimate of their union.
     * The number of registers in each must be the same.
     *
     * @param[in] other HyperLogLog instance to be merged
     *
     * @exception std::invalid_argument number of registers doesn't match.
     */
    void merge(const HyperLogLog& other) throw (std::invalid_argument) {
        if (m_ != other.m_) {
            std::stringstream ss;
            ss << "number of registers doesn't match: " << m_ << " != " << other.m_;
            throw std::invalid_argument(ss.str().c_str());
        }
        for (uint32_t r = 0; r < m_; ++r) {
            if (M_[r] < other.M_[r]) {
                M_[r] = other.M_[r];
            }
        }
    }

    /**
     * Clears all internal registers.
     */
    void clear() {
        std::fill(M_.begin(), M_.end(), 0);
    }

    /**
     * Returns size of register.
     *
     * @return Register size
     */
    uint32_t registerSize() const {
        return m_;
    }

    /**
     * Exchanges the content of the instance
     *
     * @param[in,out] rhs Another HyperLogLog instance
     */
    void swap(HyperLogLog& rhs) {
        std::swap(b_, rhs.b_);
        std::swap(m_, rhs.m_);
        std::swap(alphaMM_, rhs.alphaMM_);
        M_.swap(rhs.M_);
    }

    /**
     * Dump the current status to a stream
     *
     * @param[out] os The output stream where the data is saved
     *
     * @exception std::runtime_error When failed to dump.
     */
    void dump(std::ostream& os) const throw(std::runtime_error){
        os.write((char*)&b_, sizeof(b_));
        os.write((char*)&M_[0], sizeof(M_[0]) * M_.size());
        if(os.fail()){
            throw std::runtime_error("Failed to dump");
        }
    }

    /**
     * Restore the status from a stream
     *
     * @param[in] is The input stream where the status is saved
     *
     * @exception std::runtime_error When failed to restore.
     */
    void restore(std::istream& is) throw(std::runtime_error){
        uint8_t b = 0;
        is.read((char*)&b, sizeof(b));
        HyperLogLog tempHLL(b);
        is.read((char*)&(tempHLL.M_[0]), sizeof(M_[0]) * tempHLL.m_);
        if(is.fail()){
           throw std::runtime_error("Failed to restore");
        }
        swap(tempHLL);
    }

private:
    uint8_t b_; ///< register bit width
    uint32_t m_; ///< register size
    double alphaMM_; ///< alpha * m^2
    std::vector<uint8_t> M_; ///< registers

    uint8_t rho(uint32_t x, uint8_t b) {
        uint8_t v = 1;
        while (v <= b && !(x & 0x80000000)) {
            v++;
            x <<= 1;
        }
        return v;
    }

};

} // namespace hll
<commit_msg>sstables: remove HyperLogLog throw() specifier<commit_after>/*
 * Copyright (c) 2013 Hideaki Ohno <hide.o.j55{at}gmail.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the 'Software'), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so.
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

/*
 * Copyright (C) 2011 Clearspring Technologies, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright 2015 Cloudius Systems
 *
 * Modified by Cloudius Systems
 */

#pragma once

/**
 * @file hyperloglog.hpp
 * @brief HyperLogLog cardinality estimator
 * @date Created 2013/3/20
 * @author Hideaki Ohno
 */

#include <vector>
#include <cmath>
#include <sstream>
#include <stdexcept>
#include <algorithm>
#if 0
#include "murmur3.h"
#endif

#define HLL_HASH_SEED 313

namespace hll {

static const double pow_2_32 = 4294967296.0; ///< 2^32
static const double neg_pow_2_32 = -4294967296.0; ///< -(2^32)

static inline size_t size_unsigned_var_int(unsigned int value) {
    size_t size = 0;
    while ((value & 0xFFFFFF80) != 0L) {
        size++;
        value >>= 7;
    }
    size++;
    return size;
}

static inline size_t write_unsigned_var_int(unsigned int value, uint8_t* to) {
    size_t size = 0;
    while ((value & 0xFFFFFF80) != 0L) {
        *to = (value & 0x7F) | 0x80;
        value >>= 7;
        to++;
        size++;
    }
    *to = value & 0x7F;
    size++;
    return size;
}

/** @class HyperLogLog
 *  @brief Implement of 'HyperLogLog' estimate cardinality algorithm
 */
class HyperLogLog {
public:

    /**
     * Constructor
     *
     * @param[in] b bit width (register size will be 2 to the b power).
     *            This value must be in the range[4,16].Default value is 4.
     *
     * @exception std::invalid_argument the argument is out of range.
     */
    HyperLogLog(uint8_t b = 4) :
            b_(b), m_(1 << b), M_(m_, 0) {

        if (b < 4 || 16 < b) {
            throw std::invalid_argument("bit width must be in the range [4,16]");
        }

        double alpha;
        switch (m_) {
            case 16:
                alpha = 0.673;
                break;
            case 32:
                alpha = 0.697;
                break;
            case 64:
                alpha = 0.709;
                break;
            default:
                alpha = 0.7213 / (1.0 + 1.079 / m_);
                break;
        }
        alphaMM_ = alpha * m_ * m_;
    }

    static HyperLogLog from_bytes(temporary_buffer<uint8_t> bytes) {
        // FIXME: implement class that creates a HyperLogLog from an array of bytes.
        // This will useful if we need to work with the cardinality data from the
        // compaction metadata.
        abort();
    }

    /**
     * Adds element to the estimator
     *
     * @param[in] str string to add
     * @param[in] len length of string
     */
#if 0
    void add(const char* str, uint32_t len) {
        uint32_t hash;
        MurmurHash3_x86_32(str, len, HLL_HASH_SEED, (void*) &hash);
        uint32_t index = hash >> (32 - b_);
        uint8_t rank = rho((hash << b_), 32 - b_);
        if (rank > M_[index]) {
            M_[index] = rank;
        }
    }
#endif
    void offer_hashed(uint64_t hash) {
        uint32_t index = hash >> (64 - b_);
        uint8_t rank = rho((hash << b_), 64 - b_);

        if (rank > M_[index]) {
            M_[index] = rank;
        }
    }

    /*
     * Calculate the size of buffer returned by get_bytes().
     */
    size_t get_bytes_size() {
        size_t size = 0;
        size += sizeof(int); // version
        size += size_unsigned_var_int(b_); // p; register width = b_.
        size += size_unsigned_var_int(0); // sp; // sparse set = 0.
        size += size_unsigned_var_int(0); // type;
        size += size_unsigned_var_int(M_.size()); // register size;
        size += M_.size();
        return size;
    }

    temporary_buffer<uint8_t> get_bytes() {
        // FIXME: add support to SPARSE format.
        static constexpr int version = 2;

        size_t s = get_bytes_size();
        temporary_buffer<uint8_t> bytes(s);
        size_t offset = 0;
        // write version
        *unaligned_cast<int*>(bytes.get_write() + offset) = htonl(-version);
        offset += sizeof(int);

        // write register width
        offset += write_unsigned_var_int(b_, bytes.get_write() + offset);
        // NOTE: write precision value for sparse set (not supported).
        offset += write_unsigned_var_int(0, bytes.get_write() + offset);
        // write type (NORMAL always!)
        offset += write_unsigned_var_int(0, bytes.get_write() + offset);
        // write register size
        offset += write_unsigned_var_int(M_.size(), bytes.get_write() + offset);
        // write register
        memcpy(bytes.get_write() + offset, M_.data(), M_.size());
        offset += M_.size();

        bytes.trim(offset);
        if (s != offset) {
            throw std::runtime_error("possible overflow while generating cardinality metadata");
        }
        return bytes;
    }

    /**
     * Estimates cardinality value.
     *
     * @return Estimated cardinality value.
     */
    double estimate() const {
        double estimate;
        double sum = 0.0;
        for (uint32_t i = 0; i < m_; i++) {
            sum += 1.0 / pow(2.0, M_[i]);
        }
        estimate = alphaMM_ / sum; // E in the original paper
        if (estimate <= 2.5 * m_) {
            uint32_t zeros = 0;
            for (uint32_t i = 0; i < m_; i++) {
                if (M_[i] == 0) {
                    zeros++;
                }
            }
            if (zeros != 0) {
                estimate = m_ * log(static_cast<double>(m_)/ zeros);
            }
        } else if (estimate > (1.0 / 30.0) * pow_2_32) {
            estimate = neg_pow_2_32 * log(1.0 - (estimate / pow_2_32));
        }
        return estimate;
    }

    /**
     * Merges the estimate from 'other' into this object, returning the estimate of their union.
     * The number of registers in each must be the same.
     *
     * @param[in] other HyperLogLog instance to be merged
     *
     * @exception std::invalid_argument number of registers doesn't match.
     */
    void merge(const HyperLogLog& other) throw (std::invalid_argument) {
        if (m_ != other.m_) {
            std::stringstream ss;
            ss << "number of registers doesn't match: " << m_ << " != " << other.m_;
            throw std::invalid_argument(ss.str().c_str());
        }
        for (uint32_t r = 0; r < m_; ++r) {
            if (M_[r] < other.M_[r]) {
                M_[r] = other.M_[r];
            }
        }
    }

    /**
     * Clears all internal registers.
     */
    void clear() {
        std::fill(M_.begin(), M_.end(), 0);
    }

    /**
     * Returns size of register.
     *
     * @return Register size
     */
    uint32_t registerSize() const {
        return m_;
    }

    /**
     * Exchanges the content of the instance
     *
     * @param[in,out] rhs Another HyperLogLog instance
     */
    void swap(HyperLogLog& rhs) {
        std::swap(b_, rhs.b_);
        std::swap(m_, rhs.m_);
        std::swap(alphaMM_, rhs.alphaMM_);
        M_.swap(rhs.M_);
    }

    /**
     * Dump the current status to a stream
     *
     * @param[out] os The output stream where the data is saved
     *
     * @exception std::runtime_error When failed to dump.
     */
    void dump(std::ostream& os) const throw(std::runtime_error){
        os.write((char*)&b_, sizeof(b_));
        os.write((char*)&M_[0], sizeof(M_[0]) * M_.size());
        if(os.fail()){
            throw std::runtime_error("Failed to dump");
        }
    }

    /**
     * Restore the status from a stream
     *
     * @param[in] is The input stream where the status is saved
     *
     * @exception std::runtime_error When failed to restore.
     */
    void restore(std::istream& is) throw(std::runtime_error){
        uint8_t b = 0;
        is.read((char*)&b, sizeof(b));
        HyperLogLog tempHLL(b);
        is.read((char*)&(tempHLL.M_[0]), sizeof(M_[0]) * tempHLL.m_);
        if(is.fail()){
           throw std::runtime_error("Failed to restore");
        }
        swap(tempHLL);
    }

private:
    uint8_t b_; ///< register bit width
    uint32_t m_; ///< register size
    double alphaMM_; ///< alpha * m^2
    std::vector<uint8_t> M_; ///< registers

    uint8_t rho(uint32_t x, uint8_t b) {
        uint8_t v = 1;
        while (v <= b && !(x & 0x80000000)) {
            v++;
            x <<= 1;
        }
        return v;
    }

};

} // namespace hll
<|endoftext|>


<commit_before>#ifndef CAMERA_PERSPECTIVE_H
#define CAMERA_PERSPECTIVE_H

#include "geometry/vec3.hpp"

class CameraPerspective
{
private:
    // class members
    int framePixelsX;
    int framePixelsY;
    Vec3f originPoint;
    Vec3f viewDirection;
    Vec3f upDirection;
    double focalLength;
    double frameSize;
    // private methods
public:
    // public methods
    CameraPerspective(int _framePixelsX, int _framePixelsY, Vec3f _originPoint, Vec3f _viewDirection, Vec3f _upDirection, double _focalLength, double _frameSize);

    friend std::ostream& operator << (std::ostream &os, CameraPerspective* cp)
    {
        os << "[" << "framePixelsX=" << cp->framePixelsX << ",framePixelsY=" << cp->framePixelsY << ",originPoint=" << cp->originPoint << ",viewDirection=" << cp->viewDirection << ",upDirection=" << cp->upDirection << ",focalLength=" << cp->focalLength << ",frameSize=" << cp->frameSize << "]";
        return os;
    }
};

#endif<commit_msg>added x/y pixels and frame size members added declarations for private and public member functions<commit_after>#ifndef CAMERA_PERSPECTIVE_H
#define CAMERA_PERSPECTIVE_H

#include <cmath>
#include <iostream>
#include "geometry/vec3.hpp"

class CameraPerspective
{
private:
    // class members
    double framePixelsX;
    double framePixelsY;
    Vec3f originPoint;
    Vec3f viewDirection;
    Vec3f upDirection;
    Vec3f rightDirection;
    double focalLength;
    double frameSize;
    double frameSize_h;
    double frameSize_v;
    // private methods
    Vec3f getPlaneVectorX(int xcoord);
    Vec3f getPlaneVectorY(int ycoord);
    Vec3f getPlaneVector(int xcoord, int ycoord);

public:
    // public methods
    CameraPerspective(int _framePixelsX, int _framePixelsY, Vec3f _originPoint, Vec3f _viewDirection, Vec3f _upDirection, double _focalLength, double _frameSize);

    friend std::ostream& operator << (std::ostream &os, CameraPerspective* cp)
    {
        os << "[" << "framePixelsX=" << cp->framePixelsX << ",framePixelsY=" << cp->framePixelsY << ",originPoint=" << cp->originPoint << ",viewDirection=" << cp->viewDirection << ",upDirection=" << cp->upDirection << ",focalLength=" << cp->focalLength << ",frameSize=" << cp->frameSize << "]";
        return os;
    }

    Vec3f getRay(int xcoord, int ycoord);
};

#endif<|endoftext|>


<commit_before>#include "template.hpp"
#include <iostream>
#include <fstream>
#include <gtest/gtest.h>

// The fixture for testing class Foo.
class CollectionsTest : public ::testing::Test
{
 protected:
    string result_string;
    string result_file;
    string template_string;
    string file;

    CollectionsTest()
    {
    }

    virtual ~CollectionsTest()
    {
    }

    virtual void SetUp()
    {
        template_string = "Hi I am {{me}}.\n";
        template_string += "{{# people}}";
        template_string += "Hi {{name}}!";
        template_string += "{{/ people}}";
        file = "collections.mustache";

        ofstream myfile;
        myfile.open (file.c_str());
        myfile << template_string;
        myfile.close();

        context ctx;
        ctx.add("me", "Daniel");
        buckets b;
        bucket b1;
        b1["name"] = "Tom";
        bucket b2;
        b2["name"] = "Jerry";
        b.push_back(b1);
        b.push_back(b2);
        ctx.add("people", b);

        template_t t;
        result_string = t.render(template_string, ctx);
        result_file = t.render(file, ctx);
    }

    virtual void TearDown()
    {
        remove(file.c_str());
    }

};

// Tests that a simple mustache tag is replaced
TEST_F(CollectionsTest, TestCollectionMustacheFromString)
{
    string expected = "Hi I am Daniel.\n";
          expected += "Hi Tom!";
          expected += "Hi Jerry!";
    EXPECT_EQ(expected, result_string);
}

TEST_F(CollectionsTest, TestCollectionMustacheFromFile)
{
    string expected = "Hi I am Daniel.\n";
          expected += "Hi Tom!";
          expected += "Hi Jerry!";
    EXPECT_EQ(expected, result_file);
}

TEST_F(CollectionsTest, TestCollectionMustacheWithSingleEntry)
{
    context ctx2;
    ctx2.add("me", "Daniel");
    template_t t;
    buckets bucks;
    bucket buck;
    buck["name"] = "Tom";
    bucks.push_back(buck);
    ctx2.add("people", bucks);
    string res;
    res = t.render(template_string, ctx2);
    string expected = "Hi I am Daniel.\n";
          expected += "Hi Tom!";
    EXPECT_EQ(expected, res);
}
<commit_msg>more collections test cases<commit_after>#include "template.hpp"
#include <iostream>
#include <fstream>
#include <gtest/gtest.h>

// The fixture for testing class Foo.
class CollectionsTest : public ::testing::Test
{
 protected:
    string result_single;
    string result_multiple;
    string result_multiple_fields;
    string template_string;
    string template_single_string;

    CollectionsTest()
    {
    }

    virtual ~CollectionsTest()
    {
    }

    virtual void SetUp()
    {
        template_string = "Hi I am {{me}}.\n";
        template_string += "{{# people}}";
        template_string += "Hi {{me}}, I am {{name}}, I do {{work}}.";
        template_string += "{{/ people}}";
        template_single_string = "Hi I am {{me}}.\n";
        template_single_string += "{{# people}}";
        template_single_string += "Hi {{name}}!";
        template_single_string += "{{/ people}}";

        // single entry
        bucket tom;
        bucket jerry;
        tom["name"] = "Tom";
        buckets b_single;
        b_single.push_back(tom);
        context ctx_single;
        ctx_single.add("me", "Daniel");
        ctx_single.add("people",b_single);

        // multiple entries
        jerry["name"] = "Jerry";
        buckets b_multiple;
        b_multiple.push_back(tom);
        b_multiple.push_back(jerry);
        context ctx_multiple;
        ctx_multiple.add("me", "Daniel");
        ctx_multiple.add("people", b_multiple);

        // multiple fields
        tom["work"] = "Accounting";
        jerry["work"] = "Magic";
        buckets b_multiple_fields;
        b_multiple_fields.push_back(tom);
        b_multiple_fields.push_back(jerry);
        context ctx;
        ctx.add("me", "Daniel");
        ctx.add("people", b_multiple_fields);

        template_t t;
        result_single = t.render(template_single_string, ctx_single);
        result_multiple = t.render(template_single_string, ctx_multiple);
        result_multiple_fields = t.render(template_string, ctx);
    }

    virtual void TearDown()
    {
    }

};

// Tests that a simple mustache tag is replaced
TEST_F(CollectionsTest, TestCollectionsSingle)
{
    string expected = "Hi I am Daniel.\n";
          expected += "Hi Tom!";
    EXPECT_EQ(expected, result_single);
}

TEST_F(CollectionsTest, TestCollectionsMultiple)
{
    string expected = "Hi I am Daniel.\n";
          expected += "Hi Tom!";
          expected += "Hi Jerry!";
    EXPECT_EQ(expected, result_multiple);
}

TEST_F(CollectionsTest, TestCollectionMultipleWithMultipleFields)
{
    string expected = "Hi I am Daniel.\n";
        expected += "Hi Daniel, I am Tom, I do Accounting.";
        expected += "Hi Daniel, I am Jerry, I do Magic.";
    EXPECT_EQ(expected, result_multiple_fields);
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015 PLUMgrid, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <map>
#include <string>
#include <algorithm>
#include <fcntl.h>
#include <ftw.h>
#include <map>
#include <stdio.h>
#include <string>
#include <sys/stat.h>
#include <sys/utsname.h>
#include <unistd.h>
#include <vector>
#include <linux/bpf.h>

#include <clang/Basic/FileManager.h>
#include <clang/Basic/TargetInfo.h>
#include <clang/CodeGen/BackendUtil.h>
#include <clang/CodeGen/CodeGenAction.h>
#include <clang/Driver/Compilation.h>
#include <clang/Driver/Driver.h>
#include <clang/Driver/Job.h>
#include <clang/Driver/Tool.h>
#include <clang/Frontend/CompilerInstance.h>
#include <clang/Frontend/CompilerInvocation.h>
#include <clang/Frontend/FrontendActions.h>
#include <clang/Frontend/FrontendDiagnostic.h>
#include <clang/Frontend/TextDiagnosticPrinter.h>
#include <clang/FrontendTool/Utils.h>

#include <llvm/IR/Module.h>

#include "common.h"
#include "exception.h"
#include "kbuild_helper.h"
#include "b_frontend_action.h"
#include "loader.h"

using std::map;
using std::string;
using std::unique_ptr;
using std::vector;

namespace ebpf {

ClangLoader::ClangLoader(llvm::LLVMContext *ctx, unsigned flags)
    : ctx_(ctx), flags_(flags)
{}

ClangLoader::~ClangLoader() {}

int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDesc>> *tables,
                       const string &file, bool in_memory) {
  using namespace clang;

  struct utsname un;
  uname(&un);
  char kdir[256];
  snprintf(kdir, sizeof(kdir), "%s/%s/build", KERNEL_MODULES_DIR, un.release);

  // clang needs to run inside the kernel dir
  DirStack dstack(kdir);
  if (!dstack.ok())
    return -1;

  string abs_file;
  if (in_memory) {
    abs_file = "<bcc-memory-buffer>";
  } else {
    if (file.substr(0, 1) == "/")
      abs_file = file;
    else
      abs_file = string(dstack.cwd()) + "/" + file;
  }

  vector<const char *> flags_cstr({"-O0", "-emit-llvm", "-I", dstack.cwd(),
                                   "-Wno-deprecated-declarations",
                                   "-x", "c", "-c", abs_file.c_str()});

  KBuildHelper kbuild_helper;
  vector<string> kflags;
  if (kbuild_helper.get_flags(un.release, &kflags))
    return -1;
  kflags.push_back("-include");
  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h");
  kflags.push_back("-I");
  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include");
  for (auto it = kflags.begin(); it != kflags.end(); ++it)
    flags_cstr.push_back(it->c_str());

  // set up the error reporting class
  IntrusiveRefCntPtr<DiagnosticOptions> diag_opts(new DiagnosticOptions());
  auto diag_client = new TextDiagnosticPrinter(llvm::errs(), &*diag_opts);

  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
  DiagnosticsEngine diags(DiagID, &*diag_opts, diag_client);

  // set up the command line argument wrapper
  driver::Driver drv("", "x86_64-unknown-linux-gnu", diags);
  drv.setTitle("bcc-clang-driver");
  drv.setCheckInputsExist(false);

  unique_ptr<driver::Compilation> compilation(drv.BuildCompilation(flags_cstr));
  if (!compilation)
    return -1;

  // expect exactly 1 job, otherwise error
  const driver::JobList &jobs = compilation->getJobs();
  if (jobs.size() != 1 || !isa<driver::Command>(*jobs.begin())) {
    SmallString<256> msg;
    llvm::raw_svector_ostream os(msg);
    jobs.Print(os, "; ", true);
    diags.Report(diag::err_fe_expected_compiler_job) << os.str();
    return -1;
  }

  const driver::Command &cmd = cast<driver::Command>(*jobs.begin());
  if (llvm::StringRef(cmd.getCreator().getName()) != "clang") {
    diags.Report(diag::err_fe_expected_clang_command);
    return -1;
  }

  // Initialize a compiler invocation object from the clang (-cc1) arguments.
  const driver::ArgStringList &ccargs = cmd.getArguments();

  // first pass
  auto invocation1 = make_unique<CompilerInvocation>();
  if (!CompilerInvocation::CreateFromArgs(*invocation1, const_cast<const char **>(ccargs.data()),
                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
    return -1;

  if (in_memory) {
    invocation1->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
                                                       llvm::MemoryBuffer::getMemBuffer(file).release());
    invocation1->getFrontendOpts().Inputs.clear();
    invocation1->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
  }
  invocation1->getFrontendOpts().DisableFree = false;

  CompilerInstance compiler1;
  compiler1.setInvocation(invocation1.release());
  compiler1.createDiagnostics();

  // capture the rewritten c file
  string out_str;
  llvm::raw_string_ostream os(out_str);
  BFrontendAction bact(os, flags_);
  if (!compiler1.ExecuteAction(bact))
    return -1;
  // this contains the open FDs
  *tables = bact.take_tables();

  // second pass, clear input and take rewrite buffer
  auto invocation2 = make_unique<CompilerInvocation>();
  if (!CompilerInvocation::CreateFromArgs(*invocation2, const_cast<const char **>(ccargs.data()),
                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
    return -1;
  CompilerInstance compiler2;
  invocation2->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
                                                     llvm::MemoryBuffer::getMemBuffer(out_str).release());
  invocation2->getFrontendOpts().Inputs.clear();
  invocation2->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
  invocation2->getFrontendOpts().DisableFree = false;
  // suppress warnings in the 2nd pass, but bail out on errors (our fault)
  invocation2->getDiagnosticOpts().IgnoreWarnings = true;
  compiler2.setInvocation(invocation2.release());
  compiler2.createDiagnostics();

  EmitLLVMOnlyAction ir_act(&*ctx_);
  if (!compiler2.ExecuteAction(ir_act))
    return -1;
  *mod = ir_act.takeModule();

  return 0;
}

}  // namespace ebpf
<commit_msg>Add clang command line invocation to debug=0x4<commit_after>/*
 * Copyright (c) 2015 PLUMgrid, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <map>
#include <string>
#include <algorithm>
#include <fcntl.h>
#include <ftw.h>
#include <map>
#include <stdio.h>
#include <string>
#include <sys/stat.h>
#include <sys/utsname.h>
#include <unistd.h>
#include <vector>
#include <linux/bpf.h>

#include <clang/Basic/FileManager.h>
#include <clang/Basic/TargetInfo.h>
#include <clang/CodeGen/BackendUtil.h>
#include <clang/CodeGen/CodeGenAction.h>
#include <clang/Driver/Compilation.h>
#include <clang/Driver/Driver.h>
#include <clang/Driver/Job.h>
#include <clang/Driver/Tool.h>
#include <clang/Frontend/CompilerInstance.h>
#include <clang/Frontend/CompilerInvocation.h>
#include <clang/Frontend/FrontendActions.h>
#include <clang/Frontend/FrontendDiagnostic.h>
#include <clang/Frontend/TextDiagnosticPrinter.h>
#include <clang/FrontendTool/Utils.h>

#include <llvm/IR/Module.h>

#include "common.h"
#include "exception.h"
#include "kbuild_helper.h"
#include "b_frontend_action.h"
#include "loader.h"

using std::map;
using std::string;
using std::unique_ptr;
using std::vector;

namespace ebpf {

ClangLoader::ClangLoader(llvm::LLVMContext *ctx, unsigned flags)
    : ctx_(ctx), flags_(flags)
{}

ClangLoader::~ClangLoader() {}

int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDesc>> *tables,
                       const string &file, bool in_memory) {
  using namespace clang;

  struct utsname un;
  uname(&un);
  char kdir[256];
  snprintf(kdir, sizeof(kdir), "%s/%s/build", KERNEL_MODULES_DIR, un.release);

  // clang needs to run inside the kernel dir
  DirStack dstack(kdir);
  if (!dstack.ok())
    return -1;

  string abs_file;
  if (in_memory) {
    abs_file = "<bcc-memory-buffer>";
  } else {
    if (file.substr(0, 1) == "/")
      abs_file = file;
    else
      abs_file = string(dstack.cwd()) + "/" + file;
  }

  vector<const char *> flags_cstr({"-O0", "-emit-llvm", "-I", dstack.cwd(),
                                   "-Wno-deprecated-declarations",
                                   "-x", "c", "-c", abs_file.c_str()});

  KBuildHelper kbuild_helper;
  vector<string> kflags;
  if (kbuild_helper.get_flags(un.release, &kflags))
    return -1;
  kflags.push_back("-include");
  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h");
  kflags.push_back("-I");
  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include");
  for (auto it = kflags.begin(); it != kflags.end(); ++it)
    flags_cstr.push_back(it->c_str());

  // set up the error reporting class
  IntrusiveRefCntPtr<DiagnosticOptions> diag_opts(new DiagnosticOptions());
  auto diag_client = new TextDiagnosticPrinter(llvm::errs(), &*diag_opts);

  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
  DiagnosticsEngine diags(DiagID, &*diag_opts, diag_client);

  // set up the command line argument wrapper
  driver::Driver drv("", "x86_64-unknown-linux-gnu", diags);
  drv.setTitle("bcc-clang-driver");
  drv.setCheckInputsExist(false);

  unique_ptr<driver::Compilation> compilation(drv.BuildCompilation(flags_cstr));
  if (!compilation)
    return -1;

  // expect exactly 1 job, otherwise error
  const driver::JobList &jobs = compilation->getJobs();
  if (jobs.size() != 1 || !isa<driver::Command>(*jobs.begin())) {
    SmallString<256> msg;
    llvm::raw_svector_ostream os(msg);
    jobs.Print(os, "; ", true);
    diags.Report(diag::err_fe_expected_compiler_job) << os.str();
    return -1;
  }

  const driver::Command &cmd = cast<driver::Command>(*jobs.begin());
  if (llvm::StringRef(cmd.getCreator().getName()) != "clang") {
    diags.Report(diag::err_fe_expected_clang_command);
    return -1;
  }

  // Initialize a compiler invocation object from the clang (-cc1) arguments.
  const driver::ArgStringList &ccargs = cmd.getArguments();

  if (flags_ & 0x4) {
    llvm::errs() << "clang";
    for (auto arg : ccargs)
      llvm::errs() << " " << arg;
    llvm::errs() << "\n";
  }

  // first pass
  auto invocation1 = make_unique<CompilerInvocation>();
  if (!CompilerInvocation::CreateFromArgs(*invocation1, const_cast<const char **>(ccargs.data()),
                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
    return -1;

  if (in_memory) {
    invocation1->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
                                                       llvm::MemoryBuffer::getMemBuffer(file).release());
    invocation1->getFrontendOpts().Inputs.clear();
    invocation1->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
  }
  invocation1->getFrontendOpts().DisableFree = false;

  CompilerInstance compiler1;
  compiler1.setInvocation(invocation1.release());
  compiler1.createDiagnostics();

  // capture the rewritten c file
  string out_str;
  llvm::raw_string_ostream os(out_str);
  BFrontendAction bact(os, flags_);
  if (!compiler1.ExecuteAction(bact))
    return -1;
  // this contains the open FDs
  *tables = bact.take_tables();

  // second pass, clear input and take rewrite buffer
  auto invocation2 = make_unique<CompilerInvocation>();
  if (!CompilerInvocation::CreateFromArgs(*invocation2, const_cast<const char **>(ccargs.data()),
                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
    return -1;
  CompilerInstance compiler2;
  invocation2->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
                                                     llvm::MemoryBuffer::getMemBuffer(out_str).release());
  invocation2->getFrontendOpts().Inputs.clear();
  invocation2->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
  invocation2->getFrontendOpts().DisableFree = false;
  // suppress warnings in the 2nd pass, but bail out on errors (our fault)
  invocation2->getDiagnosticOpts().IgnoreWarnings = true;
  compiler2.setInvocation(invocation2.release());
  compiler2.createDiagnostics();

  EmitLLVMOnlyAction ir_act(&*ctx_);
  if (!compiler2.ExecuteAction(ir_act))
    return -1;
  *mod = ir_act.takeModule();

  return 0;
}

}  // namespace ebpf
<|endoftext|>


<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "FactorizationMachineLayer.h"
#include <algorithm>
#include <vector>
#include "paddle/math/SparseMatrix.h"
#include "paddle/utils/Logging.h"
#include "paddle/utils/Stat.h"

namespace paddle {

REGISTER_LAYER(factorization_machine, FactorizationMachineLayer);

bool FactorizationMachineLayer::init(const LayerMap& layerMap,
                                     const ParameterMap& parameterMap) {
  /* Initialize the basic parent class */
  Layer::init(layerMap, parameterMap);

  factorSize_ = config_.factor_size();

  /* initialize the latentVectors_ */
  CHECK_EQ(inputLayers_.size(), 1UL);
  size_t height = inputLayers_[0]->getSize();
  CHECK_EQ(parameters_[0]->getSize(), height * factorSize_);
  latentVectors_ =
      std::unique_ptr<Weight>(new Weight(height, factorSize_, parameters_[0]));

  v2_ = Matrix::create(height, factorSize_, false, useGpu_);

  return true;
}

void FactorizationMachineLayer::forward(PassType passType) {
  Layer::forward(passType);

  const MatrixPtr& inputV = getInputValue(0);

  size_t batchSize = inputV->getHeight();
  size_t size = getSize();
  reserveOutput(batchSize, size);

  MatrixPtr outV = getOutputValue();

  Matrix::resizeOrCreate(tmpMul_, batchSize, factorSize_, false, useGpu_);
  Matrix::resizeOrCreate(tmpOut_, batchSize, factorSize_, false, useGpu_);

  REGISTER_TIMER_INFO("FwMulTimer", getName().c_str());
  tmpMul_->mul(*inputV, *latentVectors_->getW());
  tmpMul_->square2(*tmpOut_);
  outV->sumRows(*tmpOut_, 0.5, 0);

  x2_ = inputV->clone(0, 0, useGpu_);
  if (dynamic_cast<CpuSparseMatrix*>(x2_.get())) {
    x2_->copyFrom(*inputV);
    (dynamic_cast<CpuSparseMatrix*>(x2_.get()))->square2();
  } else {
    inputV->square2(*x2_);
  }
  latentVectors_->getW()->square2(*v2_);
  tmpOut_->mul(*x2_, *v2_);
  outV->sumRows(*tmpOut_, -0.5, 1.0);

  /* activation */ {
    REGISTER_TIMER_INFO("FwAtvTimer", getName().c_str());
    forwardActivation();
  }
}

void FactorizationMachineLayer::backward(const UpdateCallback& callback) {
  /* Do derivation */ {
    REGISTER_TIMER_INFO("BpAvtTimer", getName().c_str());
    backwardActivation();
  }

  const MatrixPtr& inputV = getInputValue(0);
  const MatrixPtr& oGrad = getOutputGrad();

  MatrixPtr tmpSum =
      Matrix::create(1, latentVectors_->getW()->getHeight(), false, useGpu_);
  MatrixPtr tmpSum_T = Matrix::create(tmpSum->getRowBuf(0),
                                      latentVectors_->getW()->getHeight(),
                                      1,
                                      false,
                                      useGpu_);

  /* Calculate the gradients of the latentVectors_ matrix */
  if (latentVectors_->getWGrad()) {
    MatrixPtr tmpIn = inputV->clone(0, 0, useGpu_);
    if (dynamic_cast<CpuSparseMatrix*>(inputV.get())) {
      CpuSparseMatrix* inputV_s = dynamic_cast<CpuSparseMatrix*>(inputV.get());
      CpuSparseMatrix* x2_s = dynamic_cast<CpuSparseMatrix*>(x2_.get());
      CpuSparseMatrix* tmpIn_s = dynamic_cast<CpuSparseMatrix*>(tmpIn.get());
      tmpIn_s->copyFrom(*inputV_s);
      tmpIn_s->rowScale(0, *inputV_s, *oGrad);
      latentVectors_->getWGrad()->mul(*tmpIn->getTranspose(), *tmpMul_, 1, 1);
      tmpIn_s->rowScale(0, *x2_s, *oGrad);
    } else {
      tmpIn->rowScale(0, *inputV, *oGrad);
      latentVectors_->getWGrad()->mul(*tmpIn->getTranspose(), *tmpMul_, 1, 1);
      tmpIn->rowScale(0, *x2_, *oGrad);
    }

    tmpSum->sumCols(*tmpIn, -1, 0);
    latentVectors_->getWGrad()->addRowScale(
        0, *latentVectors_->getW(), *tmpSum_T);

    /* Increasing the number of gradient */
    latentVectors_->getParameterPtr()->incUpdate(callback);
  }

  /* Calculate the input layers gradient */
  MatrixPtr inGrad = getInputGrad(0);
  if (inGrad != NULL) {
    MatrixPtr latentVectors_T = latentVectors_->getW()->getTranspose();
    inGrad->mul(*tmpMul_, *latentVectors_T, 1, 1);
    tmpSum_T->sumRows(*v2_, -1, 0);
    inGrad->addColScale(0, *inputV, *tmpSum);
    inGrad->rowScale(0, *inGrad, *oGrad);
  }
}

}  // namespace paddle
<commit_msg>Add sparse input support for factorization machine layer<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "FactorizationMachineLayer.h"
#include <algorithm>
#include <vector>
#include "paddle/math/SparseMatrix.h"
#include "paddle/utils/Logging.h"
#include "paddle/utils/Stat.h"

namespace paddle {

REGISTER_LAYER(factorization_machine, FactorizationMachineLayer);

bool FactorizationMachineLayer::init(const LayerMap& layerMap,
                                     const ParameterMap& parameterMap) {
  /* Initialize the basic parent class */
  Layer::init(layerMap, parameterMap);

  factorSize_ = config_.factor_size();

  /* initialize the latentVectors_ */
  CHECK_EQ(inputLayers_.size(), 1UL);
  size_t height = inputLayers_[0]->getSize();
  CHECK_EQ(parameters_[0]->getSize(), height * factorSize_);
  latentVectors_ =
      std::unique_ptr<Weight>(new Weight(height, factorSize_, parameters_[0]));

  v2_ = Matrix::create(height, factorSize_, false, useGpu_);

  return true;
}

void FactorizationMachineLayer::forward(PassType passType) {
  Layer::forward(passType);

  const MatrixPtr& inputV = getInputValue(0);

  size_t batchSize = inputV->getHeight();
  size_t size = getSize();
  reserveOutput(batchSize, size);

  MatrixPtr outV = getOutputValue();

  Matrix::resizeOrCreate(tmpMul_, batchSize, factorSize_, false, useGpu_);
  Matrix::resizeOrCreate(tmpOut_, batchSize, factorSize_, false, useGpu_);

  REGISTER_TIMER_INFO("FwMulTimer", getName().c_str());
  tmpMul_->mul(*inputV, *latentVectors_->getW());
  tmpMul_->square2(*tmpOut_);
  outV->sumRows(*tmpOut_, 0.5, 0);

  x2_ = inputV->clone(0, 0, useGpu_);
  if (dynamic_cast<CpuSparseMatrix*>(x2_.get())) {
    x2_->copyFrom(*inputV);
    (dynamic_cast<CpuSparseMatrix*>(x2_.get()))->square2();
  } else {
    inputV->square2(*x2_);
  }
  latentVectors_->getW()->square2(*v2_);
  tmpOut_->mul(*x2_, *v2_);
  outV->sumRows(*tmpOut_, -0.5, 1.0);

  /* activation */ {
    REGISTER_TIMER_INFO("FwAtvTimer", getName().c_str());
    forwardActivation();
  }
}

void FactorizationMachineLayer::backward(const UpdateCallback& callback) {
  /* Do derivation */ {
    REGISTER_TIMER_INFO("BpAvtTimer", getName().c_str());
    backwardActivation();
  }

  const MatrixPtr& inputV = getInputValue(0);
  const MatrixPtr& oGrad = getOutputGrad();

  MatrixPtr tmpSum =
      Matrix::create(1, latentVectors_->getW()->getHeight(), false, useGpu_);
  MatrixPtr tmpSum_T = Matrix::create(tmpSum->getRowBuf(0),
                                      latentVectors_->getW()->getHeight(),
                                      1,
                                      false,
                                      useGpu_);

  /* Calculate the gradients of the latentVectors_ matrix */
  if (latentVectors_->getWGrad()) {
    MatrixPtr tmpIn = inputV->clone(0, 0, useGpu_);
    if (dynamic_cast<CpuSparseMatrix*>(inputV.get())) {
      CpuSparseMatrix* inputV_s = dynamic_cast<CpuSparseMatrix*>(inputV.get());
      CpuSparseMatrix* x2_s = dynamic_cast<CpuSparseMatrix*>(x2_.get());
      CpuSparseMatrix* tmpIn_s = dynamic_cast<CpuSparseMatrix*>(tmpIn.get());
      tmpIn_s->copyFrom(*inputV_s);
      tmpIn_s->rowScale(0, *inputV_s, *oGrad);
      latentVectors_->getWGrad()->mul(*tmpIn_s->getTranspose(), *tmpMul_, 1, 1);
      tmpIn_s->rowScale(0, *x2_s, *oGrad);

      MatrixPtr ones = Matrix::create(1, inputV->getHeight(), false, useGpu_);
      ones->zeroMem();
      ones->add(-1);
      tmpSum->mul(*ones, *tmpIn_s, 1, 0);
    } else {
      tmpIn->rowScale(0, *inputV, *oGrad);
      latentVectors_->getWGrad()->mul(*tmpIn->getTranspose(), *tmpMul_, 1, 1);
      tmpIn->rowScale(0, *x2_, *oGrad);

      tmpSum->sumCols(*tmpIn, -1, 0);
    }

    latentVectors_->getWGrad()->addRowScale(
        0, *latentVectors_->getW(), *tmpSum_T);

    /* Increasing the number of gradient */
    latentVectors_->getParameterPtr()->incUpdate(callback);
  }

  /* Calculate the input layers gradient */
  MatrixPtr inGrad = getInputGrad(0);
  if (inGrad != NULL) {
    MatrixPtr latentVectors_T = latentVectors_->getW()->getTranspose();
    inGrad->mul(*tmpMul_, *latentVectors_T, 1, 1);
    tmpSum_T->sumRows(*v2_, -1, 0);
    inGrad->addColScale(0, *inputV, *tmpSum);
    inGrad->rowScale(0, *inGrad, *oGrad);
  }
}

}  // namespace paddle
<|endoftext|>


<commit_before>/* <plugins/director/DirectorPlugin.cpp>
 *
 * This file is part of the x0 web server project and is released under LGPL-3.
 * http://www.xzero.io/
 *
 * (c) 2009-2013 Christian Parpart <trapni@gmail.com>
 *
 * --------------------------------------------------------------------------
 *
 * plugin type: content generator
 *
 * description:
 *     Implements basic load balancing, ideally taking out the need of an HAproxy
 *     in front of us.
 *
 * setup API:
 *     function director.create(string director_name,
 *                              string backend_name_1 => string backend_url_1,
 *                              ...);
 *
 *     function director.load(string director_name_1 => string path_to_db,
 *                            ...);
 *
 *     function director.cache.ttl(timespan, [timespan])
 *     function director.cache.deliver_stale
 *     function director.cache.deliver_active
 *
 * request processing API:
 *     handler director.balance(string director, string bucket = "");
 *     handler director.pass(string director, string backend);
 *
 *     handler director.fcgi(socket_spec);
 *     handler director.http(socket_spec);
 *
 *     handler director.ondemand();
 *     function director.cache.ttl(timespan ttl);
 *     function director.cache.ttl(timespan ttl, timespan shadow_ttl);
 *     function director.cache.key(string pattern);
 *     function director.cache.bypass();
 */

#include "DirectorPlugin.h"
#include "Director.h"
#include "Backend.h"
#include "ApiRequest.h"
#include "RequestNotes.h"
#include "RoadWarrior.h"
#include "HaproxyApi.h"

#include <x0/http/HttpPlugin.h>
#include <x0/http/HttpServer.h>
#include <x0/http/HttpRequest.h>
#include <x0/SocketSpec.h>
#include <x0/Url.h>
#include <x0/Types.h>

using namespace x0;

DirectorPlugin::DirectorPlugin(HttpServer& srv, const std::string& name) :
	HttpPlugin(srv, name),
	directors_(),
	roadWarrior_(),
	haproxyApi_(new HaproxyApi(&directors_))
{
	registerSetupFunction<DirectorPlugin, &DirectorPlugin::director_create>("director.create", FlowValue::VOID);
	registerSetupFunction<DirectorPlugin, &DirectorPlugin::director_load>("director.load", FlowValue::VOID);
	registerFunction<DirectorPlugin, &DirectorPlugin::director_cache_bypass>("director.cache.bypass", FlowValue::VOID);
	registerHandler<DirectorPlugin, &DirectorPlugin::director_balance>("director.balance");
	registerHandler<DirectorPlugin, &DirectorPlugin::director_pass>("director.pass");
	registerHandler<DirectorPlugin, &DirectorPlugin::director_api>("director.api");
	registerHandler<DirectorPlugin, &DirectorPlugin::director_fcgi>("director.fcgi"); // "fastcgi"
	registerHandler<DirectorPlugin, &DirectorPlugin::director_http>("director.http"); // "proxy.reverse"
	registerHandler<DirectorPlugin, &DirectorPlugin::director_haproxy_stats>("director.haproxy_stats");
	registerHandler<DirectorPlugin, &DirectorPlugin::director_haproxy_monitor>("director.haproxy_monitor");
	roadWarrior_ = new RoadWarrior(srv.selectWorker());
}

DirectorPlugin::~DirectorPlugin()
{
	for (auto director: directors_)
		delete director.second;

	delete roadWarrior_;
	delete haproxyApi_;
}

RequestNotes* DirectorPlugin::requestNotes(HttpRequest* r)
{
	if (auto notes = r->customData<RequestNotes>(this))
		return notes;

	return r->setCustomData<RequestNotes>(this, r);
}

// {{{ setup_function director.load(...)
void DirectorPlugin::director_load(const FlowParams& args, FlowValue& result)
{
	for (auto& arg: args) {
		if (!arg.isArray())
			continue;

		const FlowArray& fa = arg.toArray();
		if (fa.size() != 2)
			continue;

		const FlowValue& directorName = fa[0];
		if (!directorName.isString())
			continue;

		const FlowValue& path = fa[1];
		if (!path.isString())
			continue;

		// TODO verify that given director name is still available.
		// TODO verify that no director has been instanciated already by given path.

		server().log(Severity::debug, "director: Loading director %s from %s.",
			directorName.toString(), path.toString());

		Director* director = new Director(server().nextWorker(), directorName.toString());
		if (!director)
			continue;

		director->load(path.toString());

		directors_[directorName.toString()] = director;
	}
}
// }}}
// {{{ setup_function director.create(...)
void DirectorPlugin::director_create(const FlowParams& args, FlowValue& result)
{
	const FlowValue& directorId = args[0];
	if (!directorId.isString())
		return;

	Director* director = createDirector(directorId.toString());
	if (!director)
		return;

	for (auto& arg: args.shift()) {
		if (!arg.isArray())
			continue;

		const FlowArray& fa = arg.toArray();
		if (fa.size() != 2)
			continue;

		const FlowValue& backendName = fa[0];
		if (!backendName.isString())
			continue;

		const FlowValue& backendUrl = fa[1];
		if (!backendUrl.isString())
			continue;

		registerBackend(director, backendName.toString(), backendUrl.toString());
	}

	directors_[director->name()] = director;
}

Director* DirectorPlugin::createDirector(const char* id)
{
	server().log(Severity::debug, "director: Creating director %s", id);
	Director* director = new Director(server().nextWorker(), id);
	return director;
}

Backend* DirectorPlugin::registerBackend(Director* director, const char* name, const char* url)
{
	server().log(Severity::debug, "director: %s, backend %s: %s",
			director->name().c_str(), name, url);

	return director->createBackend(name, Url::parse(url));
}
// }}}
// {{{ function director.cache.bypass()
void DirectorPlugin::director_cache_bypass(HttpRequest* r, const FlowParams& args, FlowValue& result)
{
	// FIXME we cannot do this currently, as requestNotes is director-dependant (which is wrong, a request can belong to only one director anyway)
	// TODO: then add `RequestNotes::manager`.
#if 0
	auto notes = requestNotes(r);
	notes->cacheIgnore = true;
#endif
}
// }}}
// {{{ handler director.balance(string director_id [, string segment_id ] );
/**
 * \returns always true.
 */
bool DirectorPlugin::internalServerError(HttpRequest* r)
{
	if (r->status != x0::HttpStatus::Undefined)
		r->status = x0::HttpStatus::InternalServerError;

	r->finish();
	return true;
}

// handler director.balance(string director_name, string bucket_name = '');
bool DirectorPlugin::director_balance(HttpRequest* r, const FlowParams& args)
{
	std::string directorName;
	Director* director = nullptr;

	std::string bucketName;
	RequestShaper::Node* bucket = nullptr;

	switch (args.size()) {
		case 2:
			if (!args[1].isString() && !args[1].isBuffer()) {
				r->log(Severity::error, "director.balance(): Invalid argument.");
				return internalServerError(r);
			}
			bucketName = args[1].asString();
			// fall through
		case 1: {
			if (!args[0].isString() && !args[0].isBuffer()) {
				r->log(Severity::error, "director.balance(): Invalid argument.");
				return internalServerError(r);
			}

			directorName = args[0].asString();
			auto i = directors_.find(directorName);
			if (i == directors_.end()) {
				r->log(Severity::error, "director.balance(): No director with name '%s' configured.", directorName.c_str());
				return internalServerError(r);
			}
			director = i->second;

			if (!bucketName.empty()) {
				bucket = director->findBucket(bucketName);
				if (!bucket) {
					// explicit bucket specified, but not found -> ignore.
					bucket = director->rootBucket();
					r->log(Severity::error, "director: Requested bucket '%s' not found in director '%s'. Assigning root bucket.",
						bucketName.c_str(), directorName.c_str());
				}
			} else {
				bucket = director->rootBucket();
			}
			break;
		}
		case 0:
			r->log(Severity::error, "director.balance(): No arguments passed.");
			return internalServerError(r);
		default:
			r->log(Severity::error, "director.balance(): Invalid argument count passed.");
			return internalServerError(r);
	}

	auto rn = requestNotes(r);
	rn->manager = director;

	server().log(Severity::debug, "director: passing request to %s [%s].", director->name().c_str(), bucket->name().c_str());
	director->schedule(rn, bucket);
	return true;
} // }}}
// {{{ handler director.pass(string director_id [, string backend_id ] );
bool DirectorPlugin::director_pass(HttpRequest* r, const FlowParams& args)
{
	std::string directorName;
	Director* director = nullptr;

	std::string backendName;
	Backend* backend = nullptr;

	switch (args.size()) {
		case 2:
			if (!args[1].isString() && !args[1].isBuffer()) {
				r->log(Severity::error, "director.pass(): Invalid argument.");
				break;
			}
			backendName = args[1].asString();
			// fall though
		case 1: {
			if (!args[0].isString() && !args[0].isBuffer()) {
				r->log(Severity::error, "director.pass(): Invalid argument.");
				break;
			}

			directorName = args[0].asString();
			auto i = directors_.find(directorName);
			if (i == directors_.end()) {
				r->log(Severity::error, "director.pass(): No director with name '%s' configured.", directorName.c_str());
				break;
			}

			director = i->second;

			// custom backend route
			if (!backendName.empty()) {
				backend = director->findBackend(backendName);

				if (!backend) {
					// explicit backend specified, but not found -> do not serve.
					r->log(Severity::error, "director: Requested backend '%s' not found.", backendName.c_str());
					return internalServerError(r);
				}
			}
			break;
		}
		case 0: {
			r->log(Severity::error, "director.pass(): No arguments passed.");
			break;
		}
		default: {
			r->log(Severity::error, "director.pass(): Too many arguments passed.");
			break;
		}
	}

	if (!director)
		return internalServerError(r);

	server().log(Severity::debug, "director: passing request to %s [backend %s].", director->name().c_str(), backend->name().c_str());

	auto rn = requestNotes(r);
	rn->manager = director;

	director->schedule(rn, backend);
	return true;
}
// }}}
// {{{ handler director.api(string prefix);
bool DirectorPlugin::director_api(HttpRequest* r, const FlowParams& args)
{
	const char* prefix = args[0].toString();
	if (!r->path.begins(prefix))
		return false;

	BufferRef path(r->path.ref(strlen(prefix)));

	return ApiRequest::process(&directors_, r, path);
}
// }}}
// {{{ handler director.fcgi(socketspec);
bool DirectorPlugin::director_fcgi(HttpRequest* r, const FlowParams& args)
{
	x0::SocketSpec spec;
	spec << args;
	roadWarrior_->handleRequest(requestNotes(r), spec, RoadWarrior::FCGI);
	return true;
}
// }}}
// {{{ handler director.http(socketspec);
bool DirectorPlugin::director_http(HttpRequest* r, const FlowParams& args)
{
	x0::SocketSpec spec;
	spec << args;
	roadWarrior_->handleRequest(requestNotes(r), spec, RoadWarrior::HTTP);
	return true;
}
// }}}
// {{{ haproxy compatibility API
bool DirectorPlugin::director_haproxy_monitor(x0::HttpRequest* r, const x0::FlowParams& args)
{
	std::string prefix("/");
	if (!args.empty()) {
		if (args.size() > 1) {
			r->log(Severity::error, "director.haproxy_stats: invalid argument count");
			r->status = HttpStatus::InternalServerError;
			r->finish();
			return true;
		}
		if (!args[0].isString()) {
			r->log(Severity::error, "director.haproxy_stats: invalid argument type.");
			r->status = HttpStatus::InternalServerError;
			r->finish();
			return true;
		}
		prefix = args[0].toString();
	}

	if (!r->path.begins(prefix) && !r->unparsedUri.begins(prefix))
		return false;

	haproxyApi_->monitor(r);
	return true;
}

bool DirectorPlugin::director_haproxy_stats(x0::HttpRequest* r, const x0::FlowParams& args)
{
	std::string prefix("/");

	if (!args.empty()) {
		if (args.size() > 1) {
			r->log(Severity::error, "director.haproxy_stats: invalid argument count");
			r->status = HttpStatus::InternalServerError;
			r->finish();
			return true;
		}
		if (!args[0].isString()) {
			r->log(Severity::error, "director.haproxy_stats: invalid argument type.");
			r->status = HttpStatus::InternalServerError;
			r->finish();
			return true;
		}
		prefix = args[0].toString();
	}

	if (!r->path.begins(prefix) && !r->unparsedUri.begins(prefix))
		return false;

	haproxyApi_->stats(r, prefix);
	return true;
}
// }}}
X0_EXPORT_PLUGIN_CLASS(DirectorPlugin)
<commit_msg>[plugins] director: enables director.cache.bypass() flow function<commit_after>/* <plugins/director/DirectorPlugin.cpp>
 *
 * This file is part of the x0 web server project and is released under LGPL-3.
 * http://www.xzero.io/
 *
 * (c) 2009-2013 Christian Parpart <trapni@gmail.com>
 *
 * --------------------------------------------------------------------------
 *
 * plugin type: content generator
 *
 * description:
 *     Implements basic load balancing, ideally taking out the need of an HAproxy
 *     in front of us.
 *
 * setup API:
 *     function director.create(string director_name,
 *                              string backend_name_1 => string backend_url_1,
 *                              ...);
 *
 *     function director.load(string director_name_1 => string path_to_db,
 *                            ...);
 *
 *     function director.cache.ttl(timespan, [timespan])
 *     function director.cache.deliver_stale
 *     function director.cache.deliver_active
 *
 * request processing API:
 *     handler director.balance(string director, string bucket = "");
 *     handler director.pass(string director, string backend);
 *
 *     handler director.fcgi(socket_spec);
 *     handler director.http(socket_spec);
 *
 *     handler director.ondemand();
 *     function director.cache.ttl(timespan ttl);
 *     function director.cache.ttl(timespan ttl, timespan shadow_ttl);
 *     function director.cache.key(string pattern);
 *     function director.cache.bypass();
 */

#include "DirectorPlugin.h"
#include "Director.h"
#include "Backend.h"
#include "ApiRequest.h"
#include "RequestNotes.h"
#include "RoadWarrior.h"
#include "HaproxyApi.h"

#include <x0/http/HttpPlugin.h>
#include <x0/http/HttpServer.h>
#include <x0/http/HttpRequest.h>
#include <x0/SocketSpec.h>
#include <x0/Url.h>
#include <x0/Types.h>

using namespace x0;

DirectorPlugin::DirectorPlugin(HttpServer& srv, const std::string& name) :
	HttpPlugin(srv, name),
	directors_(),
	roadWarrior_(),
	haproxyApi_(new HaproxyApi(&directors_))
{
	registerSetupFunction<DirectorPlugin, &DirectorPlugin::director_create>("director.create", FlowValue::VOID);
	registerSetupFunction<DirectorPlugin, &DirectorPlugin::director_load>("director.load", FlowValue::VOID);
#if defined(X0_DIRECTOR_CACHE)
	registerFunction<DirectorPlugin, &DirectorPlugin::director_cache_bypass>("director.cache.bypass", FlowValue::VOID);
#endif
	registerHandler<DirectorPlugin, &DirectorPlugin::director_balance>("director.balance");
	registerHandler<DirectorPlugin, &DirectorPlugin::director_pass>("director.pass");
	registerHandler<DirectorPlugin, &DirectorPlugin::director_api>("director.api");
	registerHandler<DirectorPlugin, &DirectorPlugin::director_fcgi>("director.fcgi"); // "fastcgi"
	registerHandler<DirectorPlugin, &DirectorPlugin::director_http>("director.http"); // "proxy.reverse"
	registerHandler<DirectorPlugin, &DirectorPlugin::director_haproxy_stats>("director.haproxy_stats");
	registerHandler<DirectorPlugin, &DirectorPlugin::director_haproxy_monitor>("director.haproxy_monitor");
	roadWarrior_ = new RoadWarrior(srv.selectWorker());
}

DirectorPlugin::~DirectorPlugin()
{
	for (auto director: directors_)
		delete director.second;

	delete roadWarrior_;
	delete haproxyApi_;
}

RequestNotes* DirectorPlugin::requestNotes(HttpRequest* r)
{
	if (auto notes = r->customData<RequestNotes>(this))
		return notes;

	return r->setCustomData<RequestNotes>(this, r);
}

// {{{ setup_function director.load(...)
void DirectorPlugin::director_load(const FlowParams& args, FlowValue& result)
{
	for (auto& arg: args) {
		if (!arg.isArray())
			continue;

		const FlowArray& fa = arg.toArray();
		if (fa.size() != 2)
			continue;

		const FlowValue& directorName = fa[0];
		if (!directorName.isString())
			continue;

		const FlowValue& path = fa[1];
		if (!path.isString())
			continue;

		// TODO verify that given director name is still available.
		// TODO verify that no director has been instanciated already by given path.

		server().log(Severity::debug, "director: Loading director %s from %s.",
			directorName.toString(), path.toString());

		Director* director = new Director(server().nextWorker(), directorName.toString());
		if (!director)
			continue;

		director->load(path.toString());

		directors_[directorName.toString()] = director;
	}
}
// }}}
// {{{ setup_function director.create(...)
void DirectorPlugin::director_create(const FlowParams& args, FlowValue& result)
{
	const FlowValue& directorId = args[0];
	if (!directorId.isString())
		return;

	Director* director = createDirector(directorId.toString());
	if (!director)
		return;

	for (auto& arg: args.shift()) {
		if (!arg.isArray())
			continue;

		const FlowArray& fa = arg.toArray();
		if (fa.size() != 2)
			continue;

		const FlowValue& backendName = fa[0];
		if (!backendName.isString())
			continue;

		const FlowValue& backendUrl = fa[1];
		if (!backendUrl.isString())
			continue;

		registerBackend(director, backendName.toString(), backendUrl.toString());
	}

	directors_[director->name()] = director;
}

Director* DirectorPlugin::createDirector(const char* id)
{
	server().log(Severity::debug, "director: Creating director %s", id);
	Director* director = new Director(server().nextWorker(), id);
	return director;
}

Backend* DirectorPlugin::registerBackend(Director* director, const char* name, const char* url)
{
	server().log(Severity::debug, "director: %s, backend %s: %s",
			director->name().c_str(), name, url);

	return director->createBackend(name, Url::parse(url));
}
// }}}
// {{{ function director.cache.bypass()
#if defined(X0_DIRECTOR_CACHE)
void DirectorPlugin::director_cache_bypass(HttpRequest* r, const FlowParams& args, FlowValue& result)
{
	auto notes = requestNotes(r);
	notes->cacheIgnore = true;
}
#endif
// }}}
// {{{ handler director.balance(string director_id [, string segment_id ] );
/**
 * \returns always true.
 */
bool DirectorPlugin::internalServerError(HttpRequest* r)
{
	if (r->status != x0::HttpStatus::Undefined)
		r->status = x0::HttpStatus::InternalServerError;

	r->finish();
	return true;
}

// handler director.balance(string director_name, string bucket_name = '');
bool DirectorPlugin::director_balance(HttpRequest* r, const FlowParams& args)
{
	std::string directorName;
	Director* director = nullptr;

	std::string bucketName;
	RequestShaper::Node* bucket = nullptr;

	switch (args.size()) {
		case 2:
			if (!args[1].isString() && !args[1].isBuffer()) {
				r->log(Severity::error, "director.balance(): Invalid argument.");
				return internalServerError(r);
			}
			bucketName = args[1].asString();
			// fall through
		case 1: {
			if (!args[0].isString() && !args[0].isBuffer()) {
				r->log(Severity::error, "director.balance(): Invalid argument.");
				return internalServerError(r);
			}

			directorName = args[0].asString();
			auto i = directors_.find(directorName);
			if (i == directors_.end()) {
				r->log(Severity::error, "director.balance(): No director with name '%s' configured.", directorName.c_str());
				return internalServerError(r);
			}
			director = i->second;

			if (!bucketName.empty()) {
				bucket = director->findBucket(bucketName);
				if (!bucket) {
					// explicit bucket specified, but not found -> ignore.
					bucket = director->rootBucket();
					r->log(Severity::error, "director: Requested bucket '%s' not found in director '%s'. Assigning root bucket.",
						bucketName.c_str(), directorName.c_str());
				}
			} else {
				bucket = director->rootBucket();
			}
			break;
		}
		case 0:
			r->log(Severity::error, "director.balance(): No arguments passed.");
			return internalServerError(r);
		default:
			r->log(Severity::error, "director.balance(): Invalid argument count passed.");
			return internalServerError(r);
	}

	auto rn = requestNotes(r);
	rn->manager = director;

	server().log(Severity::debug, "director: passing request to %s [%s].", director->name().c_str(), bucket->name().c_str());
	director->schedule(rn, bucket);
	return true;
} // }}}
// {{{ handler director.pass(string director_id [, string backend_id ] );
bool DirectorPlugin::director_pass(HttpRequest* r, const FlowParams& args)
{
	std::string directorName;
	Director* director = nullptr;

	std::string backendName;
	Backend* backend = nullptr;

	switch (args.size()) {
		case 2:
			if (!args[1].isString() && !args[1].isBuffer()) {
				r->log(Severity::error, "director.pass(): Invalid argument.");
				break;
			}
			backendName = args[1].asString();
			// fall though
		case 1: {
			if (!args[0].isString() && !args[0].isBuffer()) {
				r->log(Severity::error, "director.pass(): Invalid argument.");
				break;
			}

			directorName = args[0].asString();
			auto i = directors_.find(directorName);
			if (i == directors_.end()) {
				r->log(Severity::error, "director.pass(): No director with name '%s' configured.", directorName.c_str());
				break;
			}

			director = i->second;

			// custom backend route
			if (!backendName.empty()) {
				backend = director->findBackend(backendName);

				if (!backend) {
					// explicit backend specified, but not found -> do not serve.
					r->log(Severity::error, "director: Requested backend '%s' not found.", backendName.c_str());
					return internalServerError(r);
				}
			}
			break;
		}
		case 0: {
			r->log(Severity::error, "director.pass(): No arguments passed.");
			break;
		}
		default: {
			r->log(Severity::error, "director.pass(): Too many arguments passed.");
			break;
		}
	}

	if (!director)
		return internalServerError(r);

	server().log(Severity::debug, "director: passing request to %s [backend %s].", director->name().c_str(), backend->name().c_str());

	auto rn = requestNotes(r);
	rn->manager = director;

	director->schedule(rn, backend);
	return true;
}
// }}}
// {{{ handler director.api(string prefix);
bool DirectorPlugin::director_api(HttpRequest* r, const FlowParams& args)
{
	const char* prefix = args[0].toString();
	if (!r->path.begins(prefix))
		return false;

	BufferRef path(r->path.ref(strlen(prefix)));

	return ApiRequest::process(&directors_, r, path);
}
// }}}
// {{{ handler director.fcgi(socketspec);
bool DirectorPlugin::director_fcgi(HttpRequest* r, const FlowParams& args)
{
	x0::SocketSpec spec;
	spec << args;
	roadWarrior_->handleRequest(requestNotes(r), spec, RoadWarrior::FCGI);
	return true;
}
// }}}
// {{{ handler director.http(socketspec);
bool DirectorPlugin::director_http(HttpRequest* r, const FlowParams& args)
{
	x0::SocketSpec spec;
	spec << args;
	roadWarrior_->handleRequest(requestNotes(r), spec, RoadWarrior::HTTP);
	return true;
}
// }}}
// {{{ haproxy compatibility API
bool DirectorPlugin::director_haproxy_monitor(x0::HttpRequest* r, const x0::FlowParams& args)
{
	std::string prefix("/");
	if (!args.empty()) {
		if (args.size() > 1) {
			r->log(Severity::error, "director.haproxy_stats: invalid argument count");
			r->status = HttpStatus::InternalServerError;
			r->finish();
			return true;
		}
		if (!args[0].isString()) {
			r->log(Severity::error, "director.haproxy_stats: invalid argument type.");
			r->status = HttpStatus::InternalServerError;
			r->finish();
			return true;
		}
		prefix = args[0].toString();
	}

	if (!r->path.begins(prefix) && !r->unparsedUri.begins(prefix))
		return false;

	haproxyApi_->monitor(r);
	return true;
}

bool DirectorPlugin::director_haproxy_stats(x0::HttpRequest* r, const x0::FlowParams& args)
{
	std::string prefix("/");

	if (!args.empty()) {
		if (args.size() > 1) {
			r->log(Severity::error, "director.haproxy_stats: invalid argument count");
			r->status = HttpStatus::InternalServerError;
			r->finish();
			return true;
		}
		if (!args[0].isString()) {
			r->log(Severity::error, "director.haproxy_stats: invalid argument type.");
			r->status = HttpStatus::InternalServerError;
			r->finish();
			return true;
		}
		prefix = args[0].toString();
	}

	if (!r->path.begins(prefix) && !r->unparsedUri.begins(prefix))
		return false;

	haproxyApi_->stats(r, prefix);
	return true;
}
// }}}
X0_EXPORT_PLUGIN_CLASS(DirectorPlugin)
<|endoftext|>


<commit_before>#ifndef ALEPH_PERSISTENCE_DIAGRAMS_PERSISTENCE_INDICATOR_FUNCTION_HH__
#define ALEPH_PERSISTENCE_DIAGRAMS_PERSISTENCE_INDICATOR_FUNCTION_HH__

#include "math/StepFunction.hh"

#include "persistenceDiagrams/PersistenceDiagram.hh"

#include <algorithm>
#include <limits>
#include <utility>
#include <vector>

#include <cmath>

namespace aleph
{

namespace detail
{

template <class T> struct EventPoint
{
  T value;
  bool destroyer;

  bool operator<( const EventPoint& other ) const noexcept
  {
    // Sort event point by their corresponding values. In case of ties,
    // consider creators to come after destroyers. This makes sense, as
    // a creator may increase the number of active intervals again.
    return value < other.value || ( value == other.value && destroyer && !other.destroyer );
  }
};

template <class T> T next( T x )
{
  if( std::numeric_limits<T>::is_integer )
    return x+1;
  else
    return std::nextafter( x, std::numeric_limits<T>::max() );
}

} // namespace detail

/**
  Calculates the persistence indicator function of a persistence
  diagram. This function counts the number of 'active' intervals
  for every parameter value. It is a stable summary of a diagram
  and may be used to discern more information about the topology
  and its variation over time.
*/

template <class DataType> aleph::math::StepFunction<DataType> persistenceIndicatorFunction( const PersistenceDiagram<DataType>& D )
{
  using namespace detail;
  using namespace math;

  using EP = EventPoint<DataType>;

  std::vector<EP> eventPoints;
  eventPoints.reserve( 2 * D.size() );

  for( auto&& p : D )
  {
    eventPoints.push_back( { p.x(), false } );
    eventPoints.push_back( { p.y(), true  } );
  }

  std::sort( eventPoints.begin(), eventPoints.end() );

  unsigned numActiveFeatures = 0;

  // Lambda expression for counting duplicate event points that appear after a
  // certain index in the vector of event points. Duplicate event points occur
  // if the persistence diagram contains points with equal values.
  auto numDuplicateValues = [&eventPoints] ( std::size_t i )
  {
    auto eventPoint         = eventPoints.at(i);
    unsigned numOccurrences = 0;
    unsigned numDestroyers  = 0;
    do
    {
      numDestroyers += eventPoints.at(i).destroyer;

      ++numOccurrences;
      ++i;
    }
    while( i < eventPoints.size() && eventPoints.at(i).value == eventPoint.value );

    return std::make_pair( numOccurrences, numDestroyers );
  };

  StepFunction<DataType> f;

  // Previous interval end point. This is required in order to create proper
  // indicator functions later on.
  DataType previous = DataType();

  for( std::size_t i = 0; i < eventPoints.size(); )
  {
    auto pair        = numDuplicateValues(i);
    auto occurrences = pair.first;
    auto destroyers  = pair.second;
    auto creators    = occurrences - destroyers;

    bool useNextPoint = false;

    // Case 1: No duplicates or duplicates of the same type. In this case, the
    // number of active intervals changes and we add an interval. It comprises
    // the number of active intervals up to this point.
    if( occurrences == 1 || creators == occurrences || creators == 0 )
    {
      if( i != 0 )
        f.add( previous, eventPoints.at(i).value, static_cast<DataType>( numActiveFeatures ) );
    }

    // Case 2: There are duplicate creation & destruction values. This
    // necessitates the creation of two intervals: one interval at the
    // current even point, with the proper number of active intervals,
    // the other one *directly* afterwards to indicate the destruction
    // implied by the values.
    else
    {
      f.add( previous, eventPoints.at(i).value, static_cast<DataType>( numActiveFeatures + creators ) );
      useNextPoint = true;
    }

    numActiveFeatures += creators;
    numActiveFeatures -= destroyers;

    previous  = useNextPoint ? next( eventPoints.at(i).value ) : eventPoints.at(i).value;
    i        += occurrences;
  }

  return f;
}

} // namespace aleph

#endif
<commit_msg>Updated persistence indicator function calculation<commit_after>#ifndef ALEPH_PERSISTENCE_DIAGRAMS_PERSISTENCE_INDICATOR_FUNCTION_HH__
#define ALEPH_PERSISTENCE_DIAGRAMS_PERSISTENCE_INDICATOR_FUNCTION_HH__

#include "math/StepFunction.hh"

#include "persistenceDiagrams/PersistenceDiagram.hh"

#include <algorithm>
#include <limits>
#include <utility>
#include <vector>

#include <cmath>

namespace aleph
{

namespace detail
{

template <class T> struct EventPoint
{
  T value;
  bool destroyer;

  bool operator<( const EventPoint& other ) const noexcept
  {
    // Sort event point by their corresponding values. In case of ties,
    // consider creators to come after destroyers. This makes sense, as
    // a creator may increase the number of active intervals again.
    return value < other.value || ( value == other.value && destroyer && !other.destroyer );
  }
};

template <class T> T next( T x )
{
  if( std::numeric_limits<T>::is_integer )
    return x+1;
  else
    return std::nextafter( x, std::numeric_limits<T>::max() );
}

} // namespace detail

/**
  Calculates the persistence indicator function of a persistence
  diagram. This function counts the number of 'active' intervals
  for every parameter value. It is a stable summary of a diagram
  and may be used to discern more information about the topology
  and its variation over time.
*/

template <class DataType> aleph::math::StepFunction<DataType> persistenceIndicatorFunction( const PersistenceDiagram<DataType>& D )
{
  using namespace detail;
  using namespace math;

  using EP = EventPoint<DataType>;

  std::vector<EP> eventPoints;
  eventPoints.reserve( 2 * D.size() );

  for( auto&& p : D )
  {
    eventPoints.push_back( { p.x(), false } );
    eventPoints.push_back( { p.y(), true  } );
  }

  std::sort( eventPoints.begin(), eventPoints.end() );

  unsigned numActiveFeatures = 0;

  // Lambda expression for counting duplicate event points that appear after a
  // certain index in the vector of event points. Duplicate event points occur
  // if the persistence diagram contains points with equal values.
  auto numDuplicateValues = [&eventPoints] ( std::size_t i )
  {
    auto eventPoint         = eventPoints.at(i);
    unsigned numOccurrences = 0;
    unsigned numDestroyers  = 0;
    do
    {
      numDestroyers += eventPoints.at(i).destroyer;

      ++numOccurrences;
      ++i;
    }
    while( i < eventPoints.size() && eventPoints.at(i).value == eventPoint.value );

    return std::make_pair( numOccurrences, numDestroyers );
  };

  StepFunction<DataType> f;

  // Previous interval end point. This is required in order to create proper
  // indicator functions later on.
  DataType previous = DataType();

  for( std::size_t i = 0; i < eventPoints.size(); )
  {
    auto pair        = numDuplicateValues(i);
    auto occurrences = pair.first;
    auto destroyers  = pair.second;
    auto creators    = occurrences - destroyers;

    bool useNextPoint = false;

    // Case 1: No duplicates or duplicates of the same type. In this case, the
    // number of active intervals changes and we add an interval. It comprises
    // the number of active intervals up to this point.
    if( occurrences == 1 || creators == occurrences || creators == 0 )
    {
      if( i != 0 )
        f.add( previous, eventPoints.at(i).value, static_cast<DataType>( numActiveFeatures ) );
    }

    // Case 2: There are duplicate creation & destruction values. This
    // necessitates the creation of two intervals: one interval at the
    // current event point, with the proper number of active intervals,
    // the other one *directly* afterwards to indicate the destruction
    // implied by the values.
    else
    {
      f.add( previous, eventPoints.at(i).value, static_cast<DataType>( numActiveFeatures + creators ) );
      useNextPoint = true;
    }

    numActiveFeatures += creators;
    numActiveFeatures -= destroyers;

    previous  = useNextPoint ? next( eventPoints.at(i).value ) : eventPoints.at(i).value;
    i        += occurrences;
  }

  return f;
}

} // namespace aleph

#endif
<|endoftext|>


<commit_before>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2006 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

//$Id: postgisfs.cc 34 2005-04-04 13:27:23Z pavlenko $

#include "postgis.hpp"
#include <mapnik/global.hpp>
#include <mapnik/wkb.hpp>

using boost::lexical_cast;
using boost::bad_lexical_cast;
using std::string;

postgis_featureset::postgis_featureset(boost::shared_ptr<ResultSet> const& rs,
                                       unsigned num_attrs=0)
    : rs_(rs),
      num_attrs_(num_attrs),
      totalGeomSize_(0),
      count_(0)  {}

feature_ptr postgis_featureset::next()
{
    if (rs_->next())
    {
        feature_ptr feature(new Feature(count_));
        int size=rs_->getFieldLength(0);
        const char *data=rs_->getValue(0);
        geometry_ptr geom=geometry_utils::from_wkb(data,size,-1);
        totalGeomSize_+=size;

        if (geom)
        {
            feature->set_geometry(geom);

            for (unsigned pos=1;pos<num_attrs_;++pos)
            {
                std::string name = rs_->getFieldName(pos);
                const char* buf=rs_->getValue(pos);
                int oid = rs_->getTypeOID(pos);

                if (oid==23) //int4
                {
                    int val = int4net(buf);
                    boost::put(*feature,name,val);
                }
                else if (oid==21) //int2
                {
                    int val = int2net(buf);
                    boost::put(*feature,name,val);
                }
                else if (oid == 700) // float4
                {
                    float val;
                    float4net(val,buf);
                    boost::put(*feature,name,val);
                }
                else if (oid == 701) // float8
                {
                    double val;
                    float8net(val,buf);
                    boost::put(*feature,name,val);
                }
                else if (oid==1042 || oid==1043) //bpchar or varchar
                {
                    boost::put(*feature,name,buf);
                }
                else
                {
                    boost::put(*feature,name,0);
                }
            }
            ++count_;
        }
        return feature;
    }
    else
    {
        rs_->close();
        std::clog << "totalGeomSize="<<totalGeomSize_<<" bytes"<<std::endl;
        std::clog << "count="<<count_<<std::endl;
        return feature_ptr();
    }
}

postgis_featureset::~postgis_featureset()
{
    rs_->close();
}
<commit_msg>Bugfix for PostGIS plugin.<commit_after>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2006 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

//$Id: postgisfs.cc 34 2005-04-04 13:27:23Z pavlenko $

#include "postgis.hpp"
#include <mapnik/global.hpp>
#include <mapnik/wkb.hpp>

using boost::lexical_cast;
using boost::bad_lexical_cast;
using std::string;

postgis_featureset::postgis_featureset(boost::shared_ptr<ResultSet> const& rs,
                                       unsigned num_attrs=0)
    : rs_(rs),
      num_attrs_(num_attrs),
      totalGeomSize_(0),
      count_(0)  {}

feature_ptr postgis_featureset::next()
{
    if (rs_->next())
    {
        feature_ptr feature(new Feature(count_));
        int size=rs_->getFieldLength(0);
        const char *data=rs_->getValue(0);
        geometry_ptr geom=geometry_utils::from_wkb(data,size,-1);
        totalGeomSize_+=size;

        if (geom)
        {
            feature->set_geometry(geom);

            for (unsigned pos=1;pos<num_attrs_+1;++pos)
            {
                std::string name = rs_->getFieldName(pos);
                const char* buf=rs_->getValue(pos);
                int oid = rs_->getTypeOID(pos);

                if (oid==23) //int4
                {
                    int val = int4net(buf);
                    boost::put(*feature,name,val);
                }
                else if (oid==21) //int2
                {
                    int val = int2net(buf);
                    boost::put(*feature,name,val);
                }
                else if (oid == 700) // float4
                {
                    float val;
                    float4net(val,buf);
                    boost::put(*feature,name,val);
                }
                else if (oid == 701) // float8
                {
                    double val;
                    float8net(val,buf);
                    boost::put(*feature,name,val);
                }
                else if (oid==1042 || oid==1043) //bpchar or varchar
                {
                    boost::put(*feature,name,buf);
                }
                else
                {
                    boost::put(*feature,name,0);
                }
            }
            ++count_;
        }
        return feature;
    }
    else
    {
        rs_->close();
        std::clog << "totalGeomSize="<<totalGeomSize_<<" bytes"<<std::endl;
        std::clog << "count="<<count_<<std::endl;
        return feature_ptr();
    }
}

postgis_featureset::~postgis_featureset()
{
    rs_->close();
}
<|endoftext|>