<commit_before>/**
 * @brief A high-performance and multi-threaded trie implementation based on Hdb.
 *        Incremental update is also supported.
 *        See TR "A high-performance and multi-threaded trie based on HdbTrie" for details.
 * @author Wei Cao
 * @date 2009-12-1
 */

#ifndef _MT_TRIE_H_
#define _MT_TRIE_H_

#include <iostream>

#include "mt_trie_header.h"
#include "partition_trie.hpp"

NS_IZENELIB_AM_BEGIN

template <typename StringType>
class MtTrie
{
public:
    typedef uint64_t TrieNodeIDType;

public:
    /**
     *  @param partitionNum - how many partitions do we divide all input terms into.
     */
    MtTrie(const std::string& name, const int partitionNum)
    :   name_(name),
        writeCachePath_(name_+".writecache"),
        header_(name_ + ".header.xml"),
        trie_(name_ + ".trie")
    {
        if( partitionNum <= 0 || partitionNum > 256 ) {
            throw std::runtime_error( logHead() +
                "wrong partitionNum, should between 0 and 256");
        }
        if(!header_.exist()) {
            header_.partitionNum = partitionNum;
            header_.flush();
        } else {
            if(header_.partitionNum != partitionNum)
                throw std::runtime_error( logHead() +
                    "inconsistent partitionNum");
            boundaries_ = header_.boundaries;
        }

        needSample_ = false;
        if(boundaries_.size() != (size_t)(partitionNum-1) ) {
            boundaries_.clear();
            needSample_ = true;
            sampleStep_ = 1;
            sampleCounter_ = 0;
        }

        skipNodeID_.resize(partitionNum);
    }

    void open()
    {
        writeCache_.open(writeCachePath_.c_str(), std::ofstream::out|
            std::ofstream::binary );
        if(writeCache_.fail())
            std::cerr << logHead() << " fail to open write cache" << std::endl;

        writeCacheLine_ = 0;

        trie_.open();
    }

    void close()
    {
        writeCache_.close();
        trie_.close();
    }

    void flush()
    {
        writeCache_.flush();
        header_.flush();
        trie_.flush();
    }

    /**
     * @brief Append term into write cache of MtTrie.
     */
	void insert(const StringType & term)
	{
        try {
             int len = (int) term.size();
             writeCache_.write((char*) &len, sizeof(int));
             writeCache_.write((const char*)term.c_str(), term.size());

             if(needSample_) {
                 sampleCounter_ ++;
                 if(sampleCounter_ == sampleStep_) {
                    size_t measure = writeCacheLine_-lastSampleLine_;

                    // havn't collect enough samples from sequence
                    if(samples_.size() < sampleNum_) {
                        samples_.insert(std::make_pair(writeCacheLine_ , term));
                        associations_.insert(std::make_pair(writeCacheLine_ ,  mesures_.insert(
                            std::make_pair(measure, writeCacheLine_) )));
                        lastSampleLine_ = writeCacheLine_;
                        sampleCounter_ = 0;
                    } else {
                        //  replace existing samples with new one
                        size_t existingLeastMeasure = mesures_.begin()->first;

                        if( existingLeastMeasure < measure ) {
                            // step 1. erase replaced sample from samples_
                            size_t replacedLine = mesures_.begin()->second;
                            samples_.erase(replacedLine);
                            mesures_.erase(mesures_.begin());
                            associations_.erase(replacedLine);

                            // step 2. add measure of replaced line to the next line
                            size_t updatedLine = samples_.upper_bound(replacedLine)->first;
                            size_t updatedMeasure = existingLeastMeasure +
                                associations_[updatedLine]->first;
                            mesures_.erase(associations_[updatedLine]);
                            associations_.erase(updatedLine);
                            associations_.insert(std::make_pair( updatedLine, mesures_.insert(
                                std::make_pair(updatedMeasure, updatedLine) )));

                            // step 3. insert new line
                            samples_.insert(std::make_pair(writeCacheLine_, term));
                            associations_.insert(std::make_pair( writeCacheLine_, mesures_.insert(
                                std::make_pair(measure, writeCacheLine_))));
                            lastSampleLine_ = writeCacheLine_;
                        } else {
                            sampleStep_ *= 2;
                        }
                    }
                 }
             }

             writeCacheLine_ ++;
        } catch(const std::exception & e) {
            std::cerr << logHead() << " write cache locked" << std::endl;
        }
    }

    /**
     * @brief Main control flow, invoking multiple threads processing string collection.
     *        Called after all insert() operations.
     * @param   filename - input file that contains all terms.
     *          threadNum - how many threads do we start, each thread could process
     *                      one or more partitions.
     */
    void executeTask(const int threadNum) {
        threadNum_ = threadNum;
        if( threadNum_<= 0)
            throw std::runtime_error(logHead() + "wrong threadNum, should be larger than 0");
        if( header_.partitionNum < threadNum_ ) {
            std::cout << logHead() << " threadNum(" << threadNum << ") is less than partitionNum("
                << header_.partitionNum << ", shrink it" << std::endl;
            threadNum_ = header_.partitionNum;
        }

        writeCache_.flush();
        writeCache_.close();

        if( boundaries_.size() != header_.partitionNum-1 ) {
            std::cout << logHead() << "start computing boundaries" << std::endl;

            boundaries_.clear();
            computeBoundaries(1000);
            if(boundaries_.size() == header_.partitionNum-1 ) {
                header_.boundaries = boundaries_;
                header_.flush();
                needSample_ = false;
            } else {
                std::cout << logHead() << "number of terms isn't enough to be partitioned to "
                    << header_.partitionNum << " partitions" << std::endl;
                return;
            }
        } else {
            std::cout << logHead() << "skip computing boundareis, already exist"
                << std::endl;
        }

        std::cout << logHead() << "start spliting inputs" << std::endl;
        splitInput();

        std::cout << logHead() << "Start building tries with " << threadNum_
            << " threads on " << header_.partitionNum << " partitions" << std::endl;
        // multi-threaded building tries on partitions
        for( int i=0; i < threadNum_; i++ )
          workerThreads_.create_thread(boost::bind(&MtTrie::taskBody, this, i) );
        workerThreads_.join_all();

        // merging tries on partitions into one
        std::cout << logHead() << "Start merge tries " << std::endl;
        mergeTries();

        writeCacheLine_ = 0;
        writeCache_.open(writeCachePath_.c_str(), std::ofstream::out|
            std::ofstream::binary );
//            std::ofstream::trunc|std::ofstream::binary );
        if(writeCache_.fail())
            std::cerr << logHead() << "fail to reopen write cache" << std::endl;
        flush();
    }

    /**
     * @brief Get a list of values whose keys match the wildcard query. Only "*" and
     * "?" are supported currently, legal input looks like "ea?th", "her*", or "*ear?h".
     * @return true at leaset one result found.
     *         false nothing found.
     */
    bool findRegExp(const StringType& regexp,
        std::vector<StringType>& keyList,
        int maximumResultNumber = 100)
    {
        return trie_.findRegExp(regexp, keyList, maximumResultNumber);
    }

protected:

    std::string logHead() {
        return "MtTrie[" + name_ + "] ";
    }

    /**
     * @return starting point of NodeID in a specific partition.
     */
    TrieNodeIDType getPartitionStartNodeID(int partitionId) {
        TrieNodeIDType ret = 1;
        ret <<= (sizeof(TrieNodeIDType)*8 - 8);
        ret *= partitionId;
        return ret;
    }

    /**
     * @return prefix of files associated with a specific partition.
     */
    std::string getPartitionName(int partitionId) {
        return name_ + ".partition" + boost::lexical_cast<std::string>(partitionId);
    }

    /**
     * @brief Skip over all input terms and select some samples to get
     *        statistical information such as file's length,
     *        how many terms it contains, how to split this file
     *        to allow multiple threads rescan it at the same time.
     *        how to divide distinct terms into partitions, and so on.
     */
    void computeBoundaries(int sampleNumber) {

        std::cout << "MtTrie["<< name_ << "] preporcessing.. " << samples_.size()
            << " smaples are selected" << std::endl;

        std::set<StringType> sortedSamples;
        {
            for( typename std::map<size_t, StringType>::iterator it =
                samples_.begin(); it != samples_.end(); it++ ) {
                sortedSamples.insert(it->second);
            }
        }

        if(sortedSamples.size()) {
            // caculate boundaries between partitions
            size_t interval = sortedSamples.size()/header_.partitionNum;

            int count = 0;
            typename std::set<StringType>::iterator it =
                sortedSamples.begin();
            while(count < header_.partitionNum-1) {
                for(size_t i=0; i<interval; i++)
                    it++;
                boundaries_.push_back(*it);
                //std::cout << *it << std::endl;
                count ++;
            }
        }
    }

    /**
     * @brief Split orignal input files into multiple partitions.
     */
    void splitInput() {

        std::ifstream input;
        input.open(writeCachePath_.c_str(), std::ifstream::in|std::ifstream::binary );
        if(input.fail())
            throw std::runtime_error("failed open write cache " + writeCachePath_);

        std::ofstream* output = new std::ofstream[header_.partitionNum];
        for(int i = 0; i<header_.partitionNum; i++ ) {
            std::string outputName = getPartitionName(i) + ".input";
            output[i].open(outputName.c_str(), std::ofstream::out|std::ofstream::binary);
            if(output[i].fail())
                throw std::runtime_error("failed prepare input for " + getPartitionName(i) );
        }

        StringType term;
        int buffersize = 0;
        char charBuffer[256];

        while(!input.eof())
        {
            input.read((char*)&buffersize, sizeof(int));
            // skip too long term
            if(buffersize > 256) {
                input.seekg(buffersize, std::ifstream::cur);
                continue;
            }
            input.read( charBuffer, buffersize );
            term.assign( std::string(charBuffer,buffersize) );

            if(term.size() > 0) {
                if(header_.partitionNum == 1) {
                    output[0].write((char*)&buffersize, sizeof(int));
                    output[0].write(charBuffer, buffersize);
                } else {
                    if(term.compare(boundaries_[0]) < 0) {
                        output[0].write((char*)&buffersize, sizeof(int));
                        output[0].write(charBuffer, buffersize);
                    } else if(term.compare(boundaries_[boundaries_.size()-1]) >= 0) {
                        output[boundaries_.size()].write((char*)&buffersize, sizeof(int));
                        output[boundaries_.size()].write(charBuffer, buffersize);
                    } else {
                        for(size_t i=1; i<boundaries_.size(); i++) {
                            int c1 = term.compare(boundaries_[i-1]);
                            int c2 = term.compare(boundaries_[i]);
                            if(c1 >=0 && c2<0 ) {
                                output[i].write((char*)&buffersize, sizeof(int));
                                output[i].write(charBuffer, buffersize);
                                break;
                            }
                        }
                    }
                }
            }
        }
        input.close();
        for(int i = 0; i<header_.partitionNum; i++ ) {
            output[i].close();
        }
        delete[] output;
    }

    void taskBody(int threadId) {
        int partitionId = threadId;
        while(partitionId < header_.partitionNum) {
            std::string inputName = getPartitionName(partitionId) + ".input";
            std::string trieName = getPartitionName(partitionId) + ".trie";

            std::ifstream input;
            input.open(inputName.c_str(), std::ifstream::in|
                std::ifstream::binary );
            if(input.fail())
                throw std::runtime_error("failed open input for "
                        + getPartitionName(partitionId));

            PartitionTrie2<StringType> trie(trieName);
            trie.open();

            // Initialize trie at the first time.
            if(trie.num_items() == 0) {
                // Insert all boundaries into trie, to ensure NodeIDs are consistent
                // in all tries, the details of reasoning and proof see MtTrie TR.
                for(size_t i =0; i<boundaries_.size(); i++) {
                    trie.insert(boundaries_[i]);
                }
            }
            trie.setbase(getPartitionStartNodeID(partitionId));
            skipNodeID_[partitionId] = trie.nextNodeID();

            // Insert all terms in this partition
            StringType term;
            int buffersize = 0;
            char charBuffer[256];
            while(!input.eof())
            {
                input.read((char*)&buffersize, sizeof(int));
                input.read( charBuffer, buffersize );
                term.assign( std::string(charBuffer,buffersize) );
                trie.insert(term);
            }
            trie.optimize();
            trie.close();
            input.close();
            remove( inputName.c_str() );
            partitionId += threadNum_;
        }
    }

    void mergeTries() {

        // Initialize trie at the first time.
        if(trie_.num_items() == 0) {
            for(size_t i =0; i<boundaries_.size(); i++) {
                trie_.insert(boundaries_[i]);
            }
        }

        for(int i=0; i<header_.partitionNum; i++) {
            std::string p = getPartitionName(i) + ".trie";
            PartitionTrie2<StringType> pt(p);
            pt.open();
            trie_.concatenate(pt, skipNodeID_[i]);
            pt.close();
        }

        trie_.optimize();
        trie_.flush();
    }

private:

    /// @brief prefix of assoicated files' name.
    std::string name_;

    /// @brief path of write cache file.
    std::string writeCachePath_;

    /// @brief file acts as write cache.
    ///        all inputs are cached in this file first,
    ///        then inserted into trie in executeTask().
    std::ofstream writeCache_;

    /// @brief boundaries between partitions, there are partitionNum-1 elements.
    std::vector<StringType> boundaries_;

    /// @brief whether we need sample inputs.
    bool needSample_;

    /// @brief number of samples that need to collect.
    const static size_t sampleNum_ = 1000;

    /// @brief for implementing sampling at insert-time
    size_t sampleStep_;
    size_t sampleCounter_;
    size_t writeCacheLine_;
    size_t lastSampleLine_;
    std::map<size_t, StringType> samples_;
    std::multimap<size_t, size_t> mesures_;
    std::map<size_t, std::multimap<size_t, size_t>::iterator> associations_;

    /// @brief store persistent states of MtTrie.
    MtTrieHeader<StringType> header_;

    /// @brief next available NodeID of tries on each partition before insertions,
    ///        skip all previous NodeID when merging a partition trie into the final trie.
    ///        designed for supporting incremental updates. there are partitionNum_ elements.
    std::vector<TrieNodeIDType> skipNodeID_;

    /// @brief number of work threads.
    int threadNum_;

    /// @brief work thread pool.
    boost::thread_group workerThreads_;

    /// @brief the final(major) trie, findRegExp are operating this trie.
    PartitionTrie2<StringType, TrieNodeIDType> trie_;

};

NS_IZENELIB_AM_END

#endif
<commit_msg>optimize mttrie<commit_after>/**
 * @brief A high-performance and multi-threaded trie implementation based on Hdb.
 *        Incremental update is also supported.
 *        See TR "A high-performance and multi-threaded trie based on HdbTrie" for details.
 * @author Wei Cao
 * @date 2009-12-1
 */

#ifndef _MT_TRIE_H_
#define _MT_TRIE_H_

#include <iostream>

#include "mt_trie_header.h"
#include "partition_trie.hpp"

NS_IZENELIB_AM_BEGIN

template <typename StringType>
class MtTrie
{
public:
    typedef uint64_t TrieNodeIDType;

public:
    /**
     *  @param partitionNum - how many partitions do we divide all input terms into.
     */
    MtTrie(const std::string& name, const int partitionNum)
    :   name_(name),
        writeCachePath_(name_+".writecache"),
        header_(name_ + ".header.xml"),
        trie_(name_ + ".trie")
    {
        if( partitionNum <= 0 || partitionNum > 256 ) {
            throw std::runtime_error( logHead() +
                "wrong partitionNum, should between 0 and 256");
        }
        if(!header_.exist()) {
            header_.partitionNum = partitionNum;
            header_.flush();
        } else {
            if(header_.partitionNum != partitionNum)
                throw std::runtime_error( logHead() +
                    "inconsistent partitionNum");
            boundaries_ = header_.boundaries;
        }

        needSample_ = false;
        if(boundaries_.size() != (size_t)(partitionNum-1) ) {
            boundaries_.clear();
            needSample_ = true;
            sampleStep_ = 1;
            sampleCounter_ = 0;
        }

        skipNodeID_.resize(partitionNum);
    }

    void open()
    {
        writeCache_.open(writeCachePath_.c_str(), std::ofstream::out|
            std::ofstream::binary );
        if(writeCache_.fail())
            std::cerr << logHead() << " fail to open write cache" << std::endl;

        writeCacheLine_ = 0;

        trie_.open();
    }

    void close()
    {
        writeCache_.close();
        trie_.close();
    }

    void flush()
    {
        writeCache_.flush();
        header_.flush();
        trie_.flush();
    }

    /**
     * @brief Append term into write cache of MtTrie.
     */
	void insert(const StringType & term)
	{
        try {
             int len = (int) term.size();
             writeCache_.write((char*) &len, sizeof(int));
             writeCache_.write((const char*)term.c_str(), term.size());

             if(needSample_) {
                 sampleCounter_ ++;
                 if(sampleCounter_ == sampleStep_) {
                    size_t measure = writeCacheLine_-lastSampleLine_;

                    // havn't collect enough samples from sequence
                    if(samples_.size() < sampleNum_) {
                        samples_.insert(std::make_pair(writeCacheLine_ , term));
                        associations_.insert(std::make_pair(writeCacheLine_ ,  mesures_.insert(
                            std::make_pair(measure, writeCacheLine_) )));
                        lastSampleLine_ = writeCacheLine_;
                        sampleCounter_ = 0;
                    } else {
                        //  replace existing samples with new one
                        size_t existingLeastMeasure = mesures_.begin()->first;

                        if( existingLeastMeasure < measure ) {
                            // step 1. erase replaced sample from samples_
                            size_t replacedLine = mesures_.begin()->second;
                            samples_.erase(replacedLine);
                            mesures_.erase(mesures_.begin());
                            associations_.erase(replacedLine);

                            // step 2. add measure of replaced line to the next line
                            size_t updatedLine = samples_.upper_bound(replacedLine)->first;
                            size_t updatedMeasure = existingLeastMeasure +
                                associations_[updatedLine]->first;
                            mesures_.erase(associations_[updatedLine]);
                            associations_.erase(updatedLine);
                            associations_.insert(std::make_pair( updatedLine, mesures_.insert(
                                std::make_pair(updatedMeasure, updatedLine) )));

                            // step 3. insert new line
                            samples_.insert(std::make_pair(writeCacheLine_, term));
                            associations_.insert(std::make_pair( writeCacheLine_, mesures_.insert(
                                std::make_pair(measure, writeCacheLine_))));
                            lastSampleLine_ = writeCacheLine_;
                        } else {
                            sampleStep_ *= 2;
                        }
                    }
                 }
             }

             writeCacheLine_ ++;
        } catch(const std::exception & e) {
            std::cerr << logHead() << " write cache locked" << std::endl;
        }
    }

    /**
     * @brief Main control flow, invoking multiple threads processing string collection.
     *        Called after all insert() operations.
     * @param   filename - input file that contains all terms.
     *          threadNum - how many threads do we start, each thread could process
     *                      one or more partitions.
     */
    void executeTask(const int threadNum) {
        threadNum_ = threadNum;
        if( threadNum_<= 0)
            throw std::runtime_error(logHead() + "wrong threadNum, should be larger than 0");
        if( header_.partitionNum < threadNum_ ) {
            std::cout << logHead() << " threadNum(" << threadNum << ") is less than partitionNum("
                << header_.partitionNum << ", shrink it" << std::endl;
            threadNum_ = header_.partitionNum;
        }

        writeCache_.flush();
        writeCache_.close();

        if( boundaries_.size() != header_.partitionNum-1 ) {
            std::cout << logHead() << "start computing boundaries" << std::endl;

            boundaries_.clear();
            computeBoundaries(1000);
            if(boundaries_.size() == header_.partitionNum-1 ) {
                header_.boundaries = boundaries_;
                header_.flush();
                needSample_ = false;
            } else {
                std::cout << logHead() << "number of terms isn't enough to be partitioned to "
                    << header_.partitionNum << " partitions" << std::endl;
                return;
            }
        } else {
            std::cout << logHead() << "skip computing boundareis, already exist"
                << std::endl;
        }

        std::cout << logHead() << "start spliting inputs" << std::endl;
        splitInput();

        std::cout << logHead() << "Start building tries with " << threadNum_
            << " threads on " << header_.partitionNum << " partitions" << std::endl;
        // multi-threaded building tries on partitions
        for( int i=0; i < threadNum_; i++ )
          workerThreads_.create_thread(boost::bind(&MtTrie::taskBody, this, i) );
        workerThreads_.join_all();

        // merging tries on partitions into one
        std::cout << logHead() << "Start merge tries " << std::endl;
        mergeTries();

        writeCacheLine_ = 0;
        writeCache_.open(writeCachePath_.c_str(), std::ofstream::out|
            std::ofstream::binary );
        if(writeCache_.fail())
            std::cerr << logHead() << "fail to reopen write cache" << std::endl;
        flush();
    }

    /**
     * @brief Get a list of values whose keys match the wildcard query. Only "*" and
     * "?" are supported currently, legal input looks like "ea?th", "her*", or "*ear?h".
     * @return true at leaset one result found.
     *         false nothing found.
     */
    bool findRegExp(const StringType& regexp,
        std::vector<StringType>& keyList,
        int maximumResultNumber = 100)
    {
        return trie_.findRegExp(regexp, keyList, maximumResultNumber);
    }

protected:

    std::string logHead() {
        return "MtTrie[" + name_ + "] ";
    }

    /**
     * @return starting point of NodeID in a specific partition.
     */
    TrieNodeIDType getPartitionStartNodeID(int partitionId) {
        TrieNodeIDType ret = 1;
        ret <<= (sizeof(TrieNodeIDType)*8 - 8);
        ret *= partitionId;
        return ret;
    }

    /**
     * @return prefix of files associated with a specific partition.
     */
    std::string getPartitionName(int partitionId) {
        return name_ + ".partition" + boost::lexical_cast<std::string>(partitionId);
    }

    /**
     * @brief Skip over all input terms and select some samples to get
     *        statistical information such as file's length,
     *        how many terms it contains, how to split this file
     *        to allow multiple threads rescan it at the same time.
     *        how to divide distinct terms into partitions, and so on.
     */
    void computeBoundaries(int sampleNumber) {

        std::cout << "MtTrie["<< name_ << "] preporcessing.. " << samples_.size()
            << " smaples are selected" << std::endl;

        std::set<StringType> sortedSamples;
        {
            for( typename std::map<size_t, StringType>::iterator it =
                samples_.begin(); it != samples_.end(); it++ ) {
                sortedSamples.insert(it->second);
            }
        }

        if(sortedSamples.size()) {
            // caculate boundaries between partitions
            size_t interval = sortedSamples.size()/header_.partitionNum;

            int count = 0;
            typename std::set<StringType>::iterator it =
                sortedSamples.begin();
            while(count < header_.partitionNum-1) {
                for(size_t i=0; i<interval; i++)
                    it++;
                boundaries_.push_back(*it);
                //std::cout << *it << std::endl;
                count ++;
            }
        }
    }

    /**
     * @brief Split orignal input files into multiple partitions.
     */
    void splitInput() {

        std::ifstream input;
        input.open(writeCachePath_.c_str(), std::ifstream::in|std::ifstream::binary );
        if(input.fail())
            throw std::runtime_error("failed open write cache " + writeCachePath_);

        std::ofstream* output = new std::ofstream[header_.partitionNum];
        for(int i = 0; i<header_.partitionNum; i++ ) {
            std::string outputName = getPartitionName(i) + ".input";
            output[i].open(outputName.c_str(), std::ofstream::out|std::ofstream::binary);
            if(output[i].fail())
                throw std::runtime_error("failed prepare input for " + getPartitionName(i) );
        }

        StringType term;
        int buffersize = 0;
        char charBuffer[256];

        while(!input.eof())
        {
            input.read((char*)&buffersize, sizeof(int));
            // skip too long term
            if(buffersize > 256) {
                input.seekg(buffersize, std::ifstream::cur);
                continue;
            }
            input.read( charBuffer, buffersize );
            term.assign( std::string(charBuffer,buffersize) );

            if(term.size() > 0) {
                int pos = std::lower_bound(boundaries_.begin(), boundaries_.end(),
                        term) - boundaries_.begin();
                output[pos].write((char*)&buffersize, sizeof(int));
                output[pos].write(charBuffer, buffersize);
            }
        }
        input.close();
        for(int i = 0; i<header_.partitionNum; i++ ) {
            output[i].close();
        }
        delete[] output;
    }

    void taskBody(int threadId) {
        int partitionId = threadId;
        while(partitionId < header_.partitionNum) {
            std::string inputName = getPartitionName(partitionId) + ".input";
            std::string trieName = getPartitionName(partitionId) + ".trie";

            std::ifstream input;
            input.open(inputName.c_str(), std::ifstream::in|
                std::ifstream::binary );
            if(input.fail())
                throw std::runtime_error("failed open input for "
                        + getPartitionName(partitionId));

            PartitionTrie2<StringType> trie(trieName);
            trie.open();

            // Initialize trie at the first time.
            if(trie.num_items() == 0) {
                // Insert all boundaries into trie, to ensure NodeIDs are consistent
                // in all tries, the details of reasoning and proof see MtTrie TR.
                for(size_t i =0; i<boundaries_.size(); i++) {
                    trie.insert(boundaries_[i]);
                }
                trie.setbase(getPartitionStartNodeID(partitionId));
            }
            skipNodeID_[partitionId] = trie.nextNodeID();

            // Insert all terms in this partition
            StringType term;
            int buffersize = 0;
            char charBuffer[256];
            while(!input.eof())
            {
                input.read((char*)&buffersize, sizeof(int));
                input.read( charBuffer, buffersize );
                term.assign( std::string(charBuffer,buffersize) );
                trie.insert(term);
            }
            trie.optimize();
            trie.close();
            input.close();
            remove( inputName.c_str() );
            partitionId += threadNum_;
        }
    }

    void mergeTries() {

        // Initialize trie at the first time.
        if(trie_.num_items() == 0) {
            for(size_t i =0; i<boundaries_.size(); i++) {
                trie_.insert(boundaries_[i]);
            }
        }

        for(int i=0; i<header_.partitionNum; i++) {
            std::string p = getPartitionName(i) + ".trie";
            PartitionTrie2<StringType> pt(p);
            pt.open();
            trie_.concatenate(pt, skipNodeID_[i]);
            pt.close();
        }

        trie_.optimize();
        trie_.flush();
    }

private:

    /// @brief prefix of assoicated files' name.
    std::string name_;

    /// @brief path of write cache file.
    std::string writeCachePath_;

    /// @brief file acts as write cache.
    ///        all inputs are cached in this file first,
    ///        then inserted into trie in executeTask().
    std::ofstream writeCache_;

    /// @brief boundaries between partitions, there are partitionNum-1 elements.
    std::vector<StringType> boundaries_;

    /// @brief whether we need sample inputs.
    bool needSample_;

    /// @brief number of samples that need to collect.
    const static size_t sampleNum_ = 1000;

    /// @brief for implementing sampling at insert-time
    size_t sampleStep_;
    size_t sampleCounter_;
    size_t writeCacheLine_;
    size_t lastSampleLine_;
    std::map<size_t, StringType> samples_;
    std::multimap<size_t, size_t> mesures_;
    std::map<size_t, std::multimap<size_t, size_t>::iterator> associations_;

    /// @brief store persistent states of MtTrie.
    MtTrieHeader<StringType> header_;

    /// @brief next available NodeID of tries on each partition before insertions,
    ///        skip all previous NodeID when merging a partition trie into the final trie.
    ///        designed for supporting incremental updates. there are partitionNum_ elements.
    std::vector<TrieNodeIDType> skipNodeID_;

    /// @brief number of work threads.
    int threadNum_;

    /// @brief work thread pool.
    boost::thread_group workerThreads_;

    /// @brief the final(major) trie, findRegExp are operating this trie.
    PartitionTrie2<StringType, TrieNodeIDType> trie_;

};

NS_IZENELIB_AM_END

#endif
<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This file incorporates work covered by the following license notice:
 *
 *   Licensed to the Apache Software Foundation (ASF) under one or more
 *   contributor license agreements. See the NOTICE file distributed
 *   with this work for additional information regarding copyright
 *   ownership. The ASF licenses this file to you under the Apache
 *   License, Version 2.0 (the "License"); you may not use this file
 *   except in compliance with the License. You may obtain a copy of
 *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
 */

#ifndef INCLUDED_COMPHELPER_PROPAGG_HXX
#define INCLUDED_COMPHELPER_PROPAGG_HXX

#include <com/sun/star/uno/XAggregation.hpp>
#include <comphelper/propstate.hxx>
#include <comphelper/comphelperdllapi.h>

#include <map>

//= property helper classes

namespace comphelper
{

//= OPropertyAccessor
//= internal helper class for OPropertyArrayAggregationHelper

namespace internal
{
    struct OPropertyAccessor
    {
        sal_Int32   nOriginalHandle;
        sal_Int32   nPos;
        bool        bAggregate;

        OPropertyAccessor(sal_Int32 _nOriginalHandle, sal_Int32 _nPos, bool _bAggregate)
            :nOriginalHandle(_nOriginalHandle) ,nPos(_nPos) ,bAggregate(_bAggregate) { }
        OPropertyAccessor()
            :nOriginalHandle(-1) ,nPos(-1) ,bAggregate(false) { }

        bool operator==(const OPropertyAccessor& rOb) const { return nPos == rOb.nPos; }
        bool operator <(const OPropertyAccessor& rOb) const { return nPos < rOb.nPos; }
    };

    typedef std::map< sal_Int32, OPropertyAccessor, ::std::less< sal_Int32 > >  PropertyAccessorMap;
    typedef PropertyAccessorMap::iterator           PropertyAccessorMapIterator;
    typedef PropertyAccessorMap::const_iterator     ConstPropertyAccessorMapIterator;
}

/**
 * used as callback for a OPropertyArrayAggregationHelper
 */
class IPropertyInfoService
{
public:
    /** get the preferred handle for the given property
        @param      _rName      the property name
        @return                 the handle the property should be refered by, or -1 if there are no
                                preferences for the given property
    */
    virtual sal_Int32           getPreferredPropertyId(const OUString& _rName) = 0;

protected:
    ~IPropertyInfoService() {}
};

/**
 * used for implementing an cppu::IPropertyArrayHelper for classes
 * aggregating property sets
 */

#define DEFAULT_AGGREGATE_PROPERTY_ID   10000

class COMPHELPER_DLLPUBLIC OPropertyArrayAggregationHelper: public ::cppu::IPropertyArrayHelper
{
    friend class OPropertySetAggregationHelper;
protected:

    ::com::sun::star::uno::Sequence< ::com::sun::star::beans::Property> m_aProperties;
    internal::PropertyAccessorMap           m_aPropertyAccessors;

public:
    /** construct the object.
        @param  _rProperties    the properties of the object doing the aggregation. These properties
                                are used without any checks, so the caller has to ensure that the names and
                                handles are valid.
        @param  _rAggProperties the properties of the aggregate, usually got via an call to getProperties on the
                                XPropertySetInfo of the aggregate.
                                The names of the properties are used without any checks, so the caller has to ensure
                                that there are no doubles.
                                The handles are stored for later quick access, but the outside-handles the
                                aggregate properties get depend from the following two parameters.
        @param  _pInfoService
                                If not NULL, the object pointed to is used to calc handles which should be used
                                for refering the aggregate's properties from outside.
                                If one of the properties returned from the info service conflict with other handles
                                alread present (e.g. through _rProperties), the property is handled as if -1 was returned.
                                If NULL (or, for a special property, a call to getPreferredPropertyId returns -1),
                                the aggregate property(ies) get a new handle which they can be refered by from outside.
        @param  _nFirstAggregateId
                                if the object is about to create new handles for the aggregate properties, it uses
                                id's ascending from this given id.
                                No checks are made if the handle range determined by _nFirstAggregateId conflicts with other
                                handles within _rProperties.
    */
    OPropertyArrayAggregationHelper(const ::com::sun::star::uno::Sequence< ::com::sun::star::beans::Property>& _rProperties,
                                    const ::com::sun::star::uno::Sequence< ::com::sun::star::beans::Property>& _rAggProperties,
                                    IPropertyInfoService* _pInfoService = NULL,
                                    sal_Int32 _nFirstAggregateId = DEFAULT_AGGREGATE_PROPERTY_ID);

    /// inherited from IPropertyArrayHelper
    virtual sal_Bool SAL_CALL fillPropertyMembersByHandle( OUString* _pPropName, sal_Int16* _pAttributes,
                                            sal_Int32 _nHandle) ;

    /// inherited from IPropertyArrayHelper
    virtual ::com::sun::star::uno::Sequence< ::com::sun::star::beans::Property> SAL_CALL getProperties();
    /// inherited from IPropertyArrayHelper
    virtual ::com::sun::star::beans::Property SAL_CALL getPropertyByName(const OUString& _rPropertyName)
                                throw(::com::sun::star::beans::UnknownPropertyException);

    /// inherited from IPropertyArrayHelper
    virtual sal_Bool  SAL_CALL hasPropertyByName(const OUString& _rPropertyName) ;
    /// inherited from IPropertyArrayHelper
    virtual sal_Int32 SAL_CALL getHandleByName(const OUString & _rPropertyName);
    /// inherited from IPropertyArrayHelper
    virtual sal_Int32 SAL_CALL fillHandles( /*out*/sal_Int32* _pHandles, const ::com::sun::star::uno::Sequence< OUString >& _rPropNames );

    /** returns information about a property of the aggregate.
        @param  _pPropName          points to a string to receive the property name. No name is returned if this is NULL.
        @param  _pOriginalHandle    points to a sal_Int32 to receive the original property hande. No original handle is returned
                                    if this is NULL.
        @param  _nHandle            the handle of the property as got by, for instance, fillHandles

        @return sal_True, if _nHandle marks an aggregate property, otherwise sal_False
    */
    virtual bool SAL_CALL fillAggregatePropertyInfoByHandle(OUString* _pPropName, sal_Int32* _pOriginalHandle,
                                                   sal_Int32 _nHandle) const;

    /** returns information about a property given by handle
    */
    bool getPropertyByHandle( sal_Int32 _nHandle, ::com::sun::star::beans::Property& _rProperty ) const;

    enum PropertyOrigin
    {
        AGGREGATE_PROPERTY,
        DELEGATOR_PROPERTY,
        UNKNOWN_PROPERTY
    };
    /** prefer this one over the XPropertySetInfo of the aggregate!

The reason is that OPropertyArrayAggregationHelper is the only instance which really knows
        which properties of the aggregate are to be exposed.

        For instance, some derivee of OPropertySetAggregationHelper may decide to create an
        OPropertyArrayAggregationHelper which contains only a subset of the aggregate properties. This way,
        some of the aggregate properties may be hidded to the public.

        When using the XPropertySetInfo of the aggregate set to determine the existence of a property, then this
        would return false positives.

    */
    PropertyOrigin  classifyProperty( const OUString& _rName );

protected:
    const ::com::sun::star::beans::Property* findPropertyByName(const OUString& _rName) const;
};

namespace internal
{
    class PropertyForwarder;
}

/**
 * helper class for implementing the property-set-related interfaces
 * for an object doin' aggregation
 * supports at least XPropertySet and XMultiPropertySet
 *
 */
class COMPHELPER_DLLPUBLIC OPropertySetAggregationHelper    :public OPropertyStateHelper
                                    ,public ::com::sun::star::beans::XPropertiesChangeListener
                                    ,public ::com::sun::star::beans::XVetoableChangeListener
{
    friend class internal::PropertyForwarder;

protected:
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertyState>      m_xAggregateState;
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertySet>        m_xAggregateSet;
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XMultiPropertySet>   m_xAggregateMultiSet;
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XFastPropertySet>    m_xAggregateFastSet;

    internal::PropertyForwarder*    m_pForwarder;
    bool                        m_bListening : 1;

public:
    OPropertySetAggregationHelper( ::cppu::OBroadcastHelper& rBHelper );

    virtual ::com::sun::star::uno::Any SAL_CALL queryInterface(const ::com::sun::star::uno::Type& aType) throw(::com::sun::star::uno::RuntimeException, std::exception);

// XEventListener
    virtual void SAL_CALL disposing(const ::com::sun::star::lang::EventObject& Source) throw (::com::sun::star::uno::RuntimeException, std::exception);

// XFastPropertySet
    virtual void SAL_CALL setFastPropertyValue(sal_Int32 nHandle, const ::com::sun::star::uno::Any& aValue) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::beans::PropertyVetoException, ::com::sun::star::lang::IllegalArgumentException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual ::com::sun::star::uno::Any SAL_CALL getFastPropertyValue(sal_Int32 nHandle) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);

// XPropertySet
    virtual void SAL_CALL           addPropertyChangeListener(const OUString& aPropertyName, const ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertyChangeListener >& xListener) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual void SAL_CALL           addVetoableChangeListener(const OUString& PropertyName, const ::com::sun::star::uno::Reference< ::com::sun::star::beans::XVetoableChangeListener >& aListener) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);

// XPropertiesChangeListener
    virtual void SAL_CALL propertiesChange(const ::com::sun::star::uno::Sequence< ::com::sun::star::beans::PropertyChangeEvent >& evt) throw(::com::sun::star::uno::RuntimeException, std::exception);

// XVetoableChangeListener
    virtual void SAL_CALL vetoableChange(const ::com::sun::star::beans::PropertyChangeEvent& aEvent) throw(::com::sun::star::beans::PropertyVetoException, ::com::sun::star::uno::RuntimeException, std::exception);

// XMultiPropertySet
    virtual void SAL_CALL   setPropertyValues(const ::com::sun::star::uno::Sequence< OUString >& PropertyNames, const ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Any >& Values) throw(::com::sun::star::beans::PropertyVetoException, ::com::sun::star::lang::IllegalArgumentException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual void SAL_CALL   addPropertiesChangeListener(const ::com::sun::star::uno::Sequence< OUString >& aPropertyNames, const ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertiesChangeListener >& xListener) throw(::com::sun::star::uno::RuntimeException, std::exception);

// XPropertyState
    virtual ::com::sun::star::beans::PropertyState SAL_CALL getPropertyState(const OUString& PropertyName) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual void SAL_CALL                                   setPropertyToDefault(const OUString& PropertyName) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual ::com::sun::star::uno::Any SAL_CALL             getPropertyDefault(const OUString& aPropertyName) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);

// OPropertySetHelper
    /** still waiting to be overwritten ...
        you must use an OPropertyArrayAggregationHelper here, as the implementation strongly relies on this.
    */
    virtual ::cppu::IPropertyArrayHelper& SAL_CALL getInfoHelper() = 0;

    /** only implemented for "forwarded" properties, every other property must be handled
        in the derivee, and will assert if passed herein
    */
    virtual sal_Bool SAL_CALL convertFastPropertyValue( ::com::sun::star::uno::Any& _rConvertedValue, ::com::sun::star::uno::Any& _rOldValue, sal_Int32 _nHandle, const ::com::sun::star::uno::Any& _rValue ) throw(::com::sun::star::lang::IllegalArgumentException);

    /** only implemented for "forwarded" properties, every other property must be handled
        in the derivee, and will assert if passed herein
    */
    virtual void SAL_CALL setFastPropertyValue_NoBroadcast( sal_Int32 _nHandle, const ::com::sun::star::uno::Any& _rValue ) throw ( ::com::sun::star::uno::Exception, std::exception );

protected:
    ~OPropertySetAggregationHelper();

    virtual void SAL_CALL getFastPropertyValue(::com::sun::star::uno::Any& rValue, sal_Int32 nHandle) const;
    virtual void SAL_CALL disposing();

    sal_Int32       getOriginalHandle( sal_Int32 _nHandle ) const;
    OUString getPropertyName( sal_Int32 _nHandle ) const;

    /** declares the property with the given (public) handle as one to be forwarded to the aggregate

        Sometimes, you might want to overwrite properties at the aggregate. That is,
        though the aggregate implements this property, and still is to hold the property value,
        you want to do additional handling upon setting the property, but then forward the value
        to the aggregate.

        Use this method to declare such properties.

        When a "forwarded property" is set from outside, the class first calls
        <member>forwardingPropertyValue</member> for any preprocessing, then forwards the property
        value to the aggregate, and then calls <member>forwardedPropertyValue</member>.

        When you declare a property as "forwarded", the class takes care for some multi-threading
        issues, for instance, it won't fire any property change notifications which result from
        forwarding a property value, unless it's safe to do so (i.e. unless our mutex is
        released).

        @see forwardingPropertyValue
        @see forwardedPropertyValue
    */
    void declareForwardedProperty( sal_Int32 _nHandle );

    /** checks whether we're actually forwarding a property value to our aggregate

        @see declareForwardedProperty
        @see forwardingPropertyValue
        @see forwardedPropertyValue
    */
    bool    isCurrentlyForwardingProperty( sal_Int32 _nHandle ) const;

    /** called immediately before a property value which is overwritten in this instance
        is forwarded to the aggregate

        @see declareForwardedProperty
        @see forwardedPropertyValue
    */
    virtual void SAL_CALL forwardingPropertyValue( sal_Int32 _nHandle );

    /** called immediately after a property value which is overwritten in this instance
        has been forwarded to the aggregate

        @see declareForwardedProperty
        @see forwardingPropertyValue
    */
    virtual void SAL_CALL forwardedPropertyValue( sal_Int32 _nHandle, bool _bSuccess );

    /// must be called before aggregation, if aggregation is used
    void setAggregation(const ::com::sun::star::uno::Reference< ::com::sun::star::uno::XInterface >&) throw( ::com::sun::star::lang::IllegalArgumentException );
    void startListening();
};

}   // namespace comphelper

#endif // INCLUDED_COMPHELPER_PROPAGG_HXX

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<commit_msg>No need for fillAggregatePropertyInfoByHandle to be virtual<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This file incorporates work covered by the following license notice:
 *
 *   Licensed to the Apache Software Foundation (ASF) under one or more
 *   contributor license agreements. See the NOTICE file distributed
 *   with this work for additional information regarding copyright
 *   ownership. The ASF licenses this file to you under the Apache
 *   License, Version 2.0 (the "License"); you may not use this file
 *   except in compliance with the License. You may obtain a copy of
 *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
 */

#ifndef INCLUDED_COMPHELPER_PROPAGG_HXX
#define INCLUDED_COMPHELPER_PROPAGG_HXX

#include <com/sun/star/uno/XAggregation.hpp>
#include <comphelper/propstate.hxx>
#include <comphelper/comphelperdllapi.h>

#include <map>

//= property helper classes

namespace comphelper
{

//= OPropertyAccessor
//= internal helper class for OPropertyArrayAggregationHelper

namespace internal
{
    struct OPropertyAccessor
    {
        sal_Int32   nOriginalHandle;
        sal_Int32   nPos;
        bool        bAggregate;

        OPropertyAccessor(sal_Int32 _nOriginalHandle, sal_Int32 _nPos, bool _bAggregate)
            :nOriginalHandle(_nOriginalHandle) ,nPos(_nPos) ,bAggregate(_bAggregate) { }
        OPropertyAccessor()
            :nOriginalHandle(-1) ,nPos(-1) ,bAggregate(false) { }

        bool operator==(const OPropertyAccessor& rOb) const { return nPos == rOb.nPos; }
        bool operator <(const OPropertyAccessor& rOb) const { return nPos < rOb.nPos; }
    };

    typedef std::map< sal_Int32, OPropertyAccessor, ::std::less< sal_Int32 > >  PropertyAccessorMap;
    typedef PropertyAccessorMap::iterator           PropertyAccessorMapIterator;
    typedef PropertyAccessorMap::const_iterator     ConstPropertyAccessorMapIterator;
}

/**
 * used as callback for a OPropertyArrayAggregationHelper
 */
class IPropertyInfoService
{
public:
    /** get the preferred handle for the given property
        @param      _rName      the property name
        @return                 the handle the property should be refered by, or -1 if there are no
                                preferences for the given property
    */
    virtual sal_Int32           getPreferredPropertyId(const OUString& _rName) = 0;

protected:
    ~IPropertyInfoService() {}
};

/**
 * used for implementing an cppu::IPropertyArrayHelper for classes
 * aggregating property sets
 */

#define DEFAULT_AGGREGATE_PROPERTY_ID   10000

class COMPHELPER_DLLPUBLIC OPropertyArrayAggregationHelper: public ::cppu::IPropertyArrayHelper
{
    friend class OPropertySetAggregationHelper;
protected:

    ::com::sun::star::uno::Sequence< ::com::sun::star::beans::Property> m_aProperties;
    internal::PropertyAccessorMap           m_aPropertyAccessors;

public:
    /** construct the object.
        @param  _rProperties    the properties of the object doing the aggregation. These properties
                                are used without any checks, so the caller has to ensure that the names and
                                handles are valid.
        @param  _rAggProperties the properties of the aggregate, usually got via an call to getProperties on the
                                XPropertySetInfo of the aggregate.
                                The names of the properties are used without any checks, so the caller has to ensure
                                that there are no doubles.
                                The handles are stored for later quick access, but the outside-handles the
                                aggregate properties get depend from the following two parameters.
        @param  _pInfoService
                                If not NULL, the object pointed to is used to calc handles which should be used
                                for refering the aggregate's properties from outside.
                                If one of the properties returned from the info service conflict with other handles
                                alread present (e.g. through _rProperties), the property is handled as if -1 was returned.
                                If NULL (or, for a special property, a call to getPreferredPropertyId returns -1),
                                the aggregate property(ies) get a new handle which they can be refered by from outside.
        @param  _nFirstAggregateId
                                if the object is about to create new handles for the aggregate properties, it uses
                                id's ascending from this given id.
                                No checks are made if the handle range determined by _nFirstAggregateId conflicts with other
                                handles within _rProperties.
    */
    OPropertyArrayAggregationHelper(const ::com::sun::star::uno::Sequence< ::com::sun::star::beans::Property>& _rProperties,
                                    const ::com::sun::star::uno::Sequence< ::com::sun::star::beans::Property>& _rAggProperties,
                                    IPropertyInfoService* _pInfoService = NULL,
                                    sal_Int32 _nFirstAggregateId = DEFAULT_AGGREGATE_PROPERTY_ID);

    /// inherited from IPropertyArrayHelper
    virtual sal_Bool SAL_CALL fillPropertyMembersByHandle( OUString* _pPropName, sal_Int16* _pAttributes,
                                            sal_Int32 _nHandle) ;

    /// inherited from IPropertyArrayHelper
    virtual ::com::sun::star::uno::Sequence< ::com::sun::star::beans::Property> SAL_CALL getProperties();
    /// inherited from IPropertyArrayHelper
    virtual ::com::sun::star::beans::Property SAL_CALL getPropertyByName(const OUString& _rPropertyName)
                                throw(::com::sun::star::beans::UnknownPropertyException);

    /// inherited from IPropertyArrayHelper
    virtual sal_Bool  SAL_CALL hasPropertyByName(const OUString& _rPropertyName) ;
    /// inherited from IPropertyArrayHelper
    virtual sal_Int32 SAL_CALL getHandleByName(const OUString & _rPropertyName);
    /// inherited from IPropertyArrayHelper
    virtual sal_Int32 SAL_CALL fillHandles( /*out*/sal_Int32* _pHandles, const ::com::sun::star::uno::Sequence< OUString >& _rPropNames );

    /** returns information about a property of the aggregate.
        @param  _pPropName          points to a string to receive the property name. No name is returned if this is NULL.
        @param  _pOriginalHandle    points to a sal_Int32 to receive the original property hande. No original handle is returned
                                    if this is NULL.
        @param  _nHandle            the handle of the property as got by, for instance, fillHandles

        @return sal_True, if _nHandle marks an aggregate property, otherwise sal_False
    */
    bool fillAggregatePropertyInfoByHandle(OUString* _pPropName, sal_Int32* _pOriginalHandle,
                                                   sal_Int32 _nHandle) const;

    /** returns information about a property given by handle
    */
    bool getPropertyByHandle( sal_Int32 _nHandle, ::com::sun::star::beans::Property& _rProperty ) const;

    enum PropertyOrigin
    {
        AGGREGATE_PROPERTY,
        DELEGATOR_PROPERTY,
        UNKNOWN_PROPERTY
    };
    /** prefer this one over the XPropertySetInfo of the aggregate!

The reason is that OPropertyArrayAggregationHelper is the only instance which really knows
        which properties of the aggregate are to be exposed.

        For instance, some derivee of OPropertySetAggregationHelper may decide to create an
        OPropertyArrayAggregationHelper which contains only a subset of the aggregate properties. This way,
        some of the aggregate properties may be hidded to the public.

        When using the XPropertySetInfo of the aggregate set to determine the existence of a property, then this
        would return false positives.

    */
    PropertyOrigin  classifyProperty( const OUString& _rName );

protected:
    const ::com::sun::star::beans::Property* findPropertyByName(const OUString& _rName) const;
};

namespace internal
{
    class PropertyForwarder;
}

/**
 * helper class for implementing the property-set-related interfaces
 * for an object doin' aggregation
 * supports at least XPropertySet and XMultiPropertySet
 *
 */
class COMPHELPER_DLLPUBLIC OPropertySetAggregationHelper    :public OPropertyStateHelper
                                    ,public ::com::sun::star::beans::XPropertiesChangeListener
                                    ,public ::com::sun::star::beans::XVetoableChangeListener
{
    friend class internal::PropertyForwarder;

protected:
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertyState>      m_xAggregateState;
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertySet>        m_xAggregateSet;
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XMultiPropertySet>   m_xAggregateMultiSet;
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XFastPropertySet>    m_xAggregateFastSet;

    internal::PropertyForwarder*    m_pForwarder;
    bool                        m_bListening : 1;

public:
    OPropertySetAggregationHelper( ::cppu::OBroadcastHelper& rBHelper );

    virtual ::com::sun::star::uno::Any SAL_CALL queryInterface(const ::com::sun::star::uno::Type& aType) throw(::com::sun::star::uno::RuntimeException, std::exception);

// XEventListener
    virtual void SAL_CALL disposing(const ::com::sun::star::lang::EventObject& Source) throw (::com::sun::star::uno::RuntimeException, std::exception);

// XFastPropertySet
    virtual void SAL_CALL setFastPropertyValue(sal_Int32 nHandle, const ::com::sun::star::uno::Any& aValue) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::beans::PropertyVetoException, ::com::sun::star::lang::IllegalArgumentException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual ::com::sun::star::uno::Any SAL_CALL getFastPropertyValue(sal_Int32 nHandle) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);

// XPropertySet
    virtual void SAL_CALL           addPropertyChangeListener(const OUString& aPropertyName, const ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertyChangeListener >& xListener) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual void SAL_CALL           addVetoableChangeListener(const OUString& PropertyName, const ::com::sun::star::uno::Reference< ::com::sun::star::beans::XVetoableChangeListener >& aListener) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);

// XPropertiesChangeListener
    virtual void SAL_CALL propertiesChange(const ::com::sun::star::uno::Sequence< ::com::sun::star::beans::PropertyChangeEvent >& evt) throw(::com::sun::star::uno::RuntimeException, std::exception);

// XVetoableChangeListener
    virtual void SAL_CALL vetoableChange(const ::com::sun::star::beans::PropertyChangeEvent& aEvent) throw(::com::sun::star::beans::PropertyVetoException, ::com::sun::star::uno::RuntimeException, std::exception);

// XMultiPropertySet
    virtual void SAL_CALL   setPropertyValues(const ::com::sun::star::uno::Sequence< OUString >& PropertyNames, const ::com::sun::star::uno::Sequence< ::com::sun::star::uno::Any >& Values) throw(::com::sun::star::beans::PropertyVetoException, ::com::sun::star::lang::IllegalArgumentException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual void SAL_CALL   addPropertiesChangeListener(const ::com::sun::star::uno::Sequence< OUString >& aPropertyNames, const ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertiesChangeListener >& xListener) throw(::com::sun::star::uno::RuntimeException, std::exception);

// XPropertyState
    virtual ::com::sun::star::beans::PropertyState SAL_CALL getPropertyState(const OUString& PropertyName) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual void SAL_CALL                                   setPropertyToDefault(const OUString& PropertyName) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::uno::RuntimeException, std::exception);
    virtual ::com::sun::star::uno::Any SAL_CALL             getPropertyDefault(const OUString& aPropertyName) throw(::com::sun::star::beans::UnknownPropertyException, ::com::sun::star::lang::WrappedTargetException, ::com::sun::star::uno::RuntimeException, std::exception);

// OPropertySetHelper
    /** still waiting to be overwritten ...
        you must use an OPropertyArrayAggregationHelper here, as the implementation strongly relies on this.
    */
    virtual ::cppu::IPropertyArrayHelper& SAL_CALL getInfoHelper() = 0;

    /** only implemented for "forwarded" properties, every other property must be handled
        in the derivee, and will assert if passed herein
    */
    virtual sal_Bool SAL_CALL convertFastPropertyValue( ::com::sun::star::uno::Any& _rConvertedValue, ::com::sun::star::uno::Any& _rOldValue, sal_Int32 _nHandle, const ::com::sun::star::uno::Any& _rValue ) throw(::com::sun::star::lang::IllegalArgumentException);

    /** only implemented for "forwarded" properties, every other property must be handled
        in the derivee, and will assert if passed herein
    */
    virtual void SAL_CALL setFastPropertyValue_NoBroadcast( sal_Int32 _nHandle, const ::com::sun::star::uno::Any& _rValue ) throw ( ::com::sun::star::uno::Exception, std::exception );

protected:
    ~OPropertySetAggregationHelper();

    virtual void SAL_CALL getFastPropertyValue(::com::sun::star::uno::Any& rValue, sal_Int32 nHandle) const;
    virtual void SAL_CALL disposing();

    sal_Int32       getOriginalHandle( sal_Int32 _nHandle ) const;
    OUString getPropertyName( sal_Int32 _nHandle ) const;

    /** declares the property with the given (public) handle as one to be forwarded to the aggregate

        Sometimes, you might want to overwrite properties at the aggregate. That is,
        though the aggregate implements this property, and still is to hold the property value,
        you want to do additional handling upon setting the property, but then forward the value
        to the aggregate.

        Use this method to declare such properties.

        When a "forwarded property" is set from outside, the class first calls
        <member>forwardingPropertyValue</member> for any preprocessing, then forwards the property
        value to the aggregate, and then calls <member>forwardedPropertyValue</member>.

        When you declare a property as "forwarded", the class takes care for some multi-threading
        issues, for instance, it won't fire any property change notifications which result from
        forwarding a property value, unless it's safe to do so (i.e. unless our mutex is
        released).

        @see forwardingPropertyValue
        @see forwardedPropertyValue
    */
    void declareForwardedProperty( sal_Int32 _nHandle );

    /** checks whether we're actually forwarding a property value to our aggregate

        @see declareForwardedProperty
        @see forwardingPropertyValue
        @see forwardedPropertyValue
    */
    bool    isCurrentlyForwardingProperty( sal_Int32 _nHandle ) const;

    /** called immediately before a property value which is overwritten in this instance
        is forwarded to the aggregate

        @see declareForwardedProperty
        @see forwardedPropertyValue
    */
    virtual void SAL_CALL forwardingPropertyValue( sal_Int32 _nHandle );

    /** called immediately after a property value which is overwritten in this instance
        has been forwarded to the aggregate

        @see declareForwardedProperty
        @see forwardingPropertyValue
    */
    virtual void SAL_CALL forwardedPropertyValue( sal_Int32 _nHandle, bool _bSuccess );

    /// must be called before aggregation, if aggregation is used
    void setAggregation(const ::com::sun::star::uno::Reference< ::com::sun::star::uno::XInterface >&) throw( ::com::sun::star::lang::IllegalArgumentException );
    void startListening();
};

}   // namespace comphelper

#endif // INCLUDED_COMPHELPER_PROPAGG_HXX

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014-2016 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include "cpp_utils/assert.hpp" //Assertions

#include "etl/etl.hpp"

#include "neural_base.hpp"
#include "util/tmp.hpp"
#include "layer_traits.hpp"

namespace dll {

/*!
 * \brief Standard dynamic convolutional layer of neural network.
 */
template <typename Desc>
struct dyn_conv_layer final : neural_base<dyn_conv_layer<Desc>> {
    using desc      = Desc;                  ///< The descriptor type
    using weight    = typename desc::weight; ///< The weight type
    using this_type = dyn_conv_layer<desc>;  ///< This type

    static constexpr const bool dbn_only = layer_traits<this_type>::is_dbn_only();

    static constexpr const function activation_function = desc::activation_function;

    using input_one_t  = etl::dyn_matrix<weight, 3>; ///< The type for one input
    using output_one_t = etl::dyn_matrix<weight, 3>; ///< The type for one output
    using input_t      = std::vector<input_one_t>;   ///< The type for many input
    using output_t     = std::vector<output_one_t>;  ///< The type for many output

    //TODO CHECK
    template <std::size_t B>
    using input_batch_t = etl::fast_dyn_matrix<weight, B, 1>;

    using w_type = etl::dyn_matrix<weight, 4>;
    using b_type = etl::dyn_matrix<weight, 1>;

    //Weights and biases
    w_type w; //!< Weights
    b_type b; //!< Hidden biases

    //Backup weights and biases
    std::unique_ptr<w_type> bak_w; //!< Backup Weights
    std::unique_ptr<b_type> bak_b; //!< Backup Hidden biases

    //No copying
    dyn_conv_layer(const dyn_conv_layer& layer) = delete;
    dyn_conv_layer& operator=(const dyn_conv_layer& layer) = delete;

    //No moving
    dyn_conv_layer(dyn_conv_layer&& layer) = delete;
    dyn_conv_layer& operator=(dyn_conv_layer&& layer) = delete;

    size_t nv1; ///< The first visible dimension
    size_t nv2; ///< The second visible dimension
    size_t nh1; ///< The first output dimension
    size_t nh2; ///< The second output dimension
    size_t nc;  ///< The number of input channels
    size_t k;   ///< The number of filters

    size_t nw1; ///< The first dimension of the filters
    size_t nw2; ///< The second dimension of the filters

    size_t batch_size = 25;

    dyn_conv_layer(){
        // Nothing else to init
    }

    void init_layer(size_t nc, size_t nv1, size_t nv2, size_t k, size_t nh1, size_t nh2){
        this->nv1 = nv1;
        this->nv2 = nv2;
        this->nh1 = nh1;
        this->nh2 = nh2;
        this->nc = nc;
        this->k = k;

        this->nw1 = nv1 - nh1 + 1;
        this->nw2 = nv2 - nh2 + 1;

        w = etl::dyn_matrix<weight, 4>(nc, k, nw1, nw2);

        b = etl::dyn_vector<weight>(k);

        //Initialize the weights and biases following Lecun approach
        //to initialization [lecun-98b]

        w = etl::normal_generator<weight>() * std::sqrt(2.0 / double(nc * nv1 * nv2));

        if (activation_function == function::RELU) {
            b = 0.01;
        } else {
            b = etl::normal_generator<weight>() * std::sqrt(2.0 / double(nc * nv1 * nv2));
        }
    }

    std::size_t input_size() const noexcept {
        return nc * nv1 * nv2;
    }

    std::size_t output_size() const noexcept {
        return k * nh1 * nh2;
    }

    std::size_t parameters() const noexcept {
        return k * nw1 * nw2;
    }

    std::string to_short_string() const {
        char buffer[1024];
        snprintf(buffer, 1024, "Conv(dyn): %lux%lux%lu -> (%lux%lux%lu) -> %s -> %lux%lux%lu", nc, nv1, nv2, k, nw1, nw2, to_string(activation_function).c_str(), k, nh1, nh2);
        return {buffer};
    }

    void display() const {
        std::cout << to_short_string() << std::endl;
    }

    void backup_weights() {
        unique_safe_get(bak_w) = w;
        unique_safe_get(bak_b) = b;
    }

    void restore_weights() {
        w = *bak_w;
        b = *bak_b;
    }

    template <typename V>
    void activate_hidden(output_one_t& output, const V& v) const {
        etl::dyn_matrix<weight, 4> v_cv(2UL, k, nh1, nh2); //Temporary convolution

        auto w_f = etl::force_temporary(w);

        //flip all the kernels horizontally and vertically

        for (std::size_t channel = 0; channel < nc; ++channel) {
            for (size_t k = 0; k < k; ++k) {
                w_f(channel)(k).fflip_inplace();
            }
        }

        v_cv(1) = 0;

        for (std::size_t channel = 0; channel < nc; ++channel) {
            etl::conv_2d_valid_multi(v(channel), w_f(channel), v_cv(0));

            v_cv(1) += v_cv(0);
        }

        output = f_activate<activation_function>(etl::rep(b, nh1, nh2) + v_cv(1));
    }

    template <typename H1, typename V>
    void batch_activate_hidden(H1&& output, const V& v) const {
        etl::dyn_matrix<weight, 4> v_cv(2UL, k, nh1, nh2); //Temporary convolution

        const auto Batch = etl::dim<0>(v);

        auto w_f = force_temporary(w);

        //flip all the kernels horizontally and vertically

        for (std::size_t channel = 0; channel < nc; ++channel) {
            for (size_t k = 0; k < k; ++k) {
                w_f(channel)(k).fflip_inplace();
            }
        }

        for (std::size_t batch = 0; batch < Batch; ++batch) {
            v_cv(1) = 0;

            for (std::size_t channel = 0; channel < nc; ++channel) {
                etl::conv_2d_valid_multi(v(batch)(channel), w_f(channel), v_cv(0));

                v_cv(1) += v_cv(0);
            }

            output(batch) = f_activate<activation_function>(etl::rep(b, nh1, nh2) + v_cv(1));
        }
    }

    void prepare_input(input_one_t& input) const {
        input = input_one_t(nc, nv1, nv2);
    }

    template <typename Input>
    output_t prepare_output(std::size_t samples) const {
        output_t output;
        output.reserve(samples);
        for(size_t i = 0; i < samples; ++i){
            output.emplace_back(k, nh1, nh2);
        }
        return output;
    }

    template <typename Input>
    output_one_t prepare_one_output() const {
        return output_one_t(k, nh1, nh2);
    }

    template <std::size_t B>
    auto prepare_input_batch(){
        return etl::dyn_matrix<weight, 4>(B, nc, nv1, nv2);
    }

    template <std::size_t B>
    auto prepare_output_batch(){
        return etl::dyn_matrix<weight, 4>(B, k, nh1, nh2);
    }

    template <typename DBN>
    void init_sgd_context() {
        this->sgd_context_ptr = std::make_shared<sgd_context<DBN, this_type>>(nc, nv1, nv2, k, nh1, nh2);
    }
};

} //end of dll namespace
<commit_msg>Remove legacy field<commit_after>//=======================================================================
// Copyright (c) 2014-2016 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include "cpp_utils/assert.hpp" //Assertions

#include "etl/etl.hpp"

#include "neural_base.hpp"
#include "util/tmp.hpp"
#include "layer_traits.hpp"

namespace dll {

/*!
 * \brief Standard dynamic convolutional layer of neural network.
 */
template <typename Desc>
struct dyn_conv_layer final : neural_base<dyn_conv_layer<Desc>> {
    using desc      = Desc;                  ///< The descriptor type
    using weight    = typename desc::weight; ///< The weight type
    using this_type = dyn_conv_layer<desc>;  ///< This type

    static constexpr const bool dbn_only = layer_traits<this_type>::is_dbn_only();

    static constexpr const function activation_function = desc::activation_function;

    using input_one_t  = etl::dyn_matrix<weight, 3>; ///< The type for one input
    using output_one_t = etl::dyn_matrix<weight, 3>; ///< The type for one output
    using input_t      = std::vector<input_one_t>;   ///< The type for many input
    using output_t     = std::vector<output_one_t>;  ///< The type for many output

    //TODO CHECK
    template <std::size_t B>
    using input_batch_t = etl::fast_dyn_matrix<weight, B, 1>;

    using w_type = etl::dyn_matrix<weight, 4>;
    using b_type = etl::dyn_matrix<weight, 1>;

    //Weights and biases
    w_type w; //!< Weights
    b_type b; //!< Hidden biases

    //Backup weights and biases
    std::unique_ptr<w_type> bak_w; //!< Backup Weights
    std::unique_ptr<b_type> bak_b; //!< Backup Hidden biases

    //No copying
    dyn_conv_layer(const dyn_conv_layer& layer) = delete;
    dyn_conv_layer& operator=(const dyn_conv_layer& layer) = delete;

    //No moving
    dyn_conv_layer(dyn_conv_layer&& layer) = delete;
    dyn_conv_layer& operator=(dyn_conv_layer&& layer) = delete;

    size_t nv1; ///< The first visible dimension
    size_t nv2; ///< The second visible dimension
    size_t nh1; ///< The first output dimension
    size_t nh2; ///< The second output dimension
    size_t nc;  ///< The number of input channels
    size_t k;   ///< The number of filters

    size_t nw1; ///< The first dimension of the filters
    size_t nw2; ///< The second dimension of the filters

    dyn_conv_layer(){
        // Nothing else to init
    }

    void init_layer(size_t nc, size_t nv1, size_t nv2, size_t k, size_t nh1, size_t nh2){
        this->nv1 = nv1;
        this->nv2 = nv2;
        this->nh1 = nh1;
        this->nh2 = nh2;
        this->nc = nc;
        this->k = k;

        this->nw1 = nv1 - nh1 + 1;
        this->nw2 = nv2 - nh2 + 1;

        w = etl::dyn_matrix<weight, 4>(nc, k, nw1, nw2);

        b = etl::dyn_vector<weight>(k);

        //Initialize the weights and biases following Lecun approach
        //to initialization [lecun-98b]

        w = etl::normal_generator<weight>() * std::sqrt(2.0 / double(nc * nv1 * nv2));

        if (activation_function == function::RELU) {
            b = 0.01;
        } else {
            b = etl::normal_generator<weight>() * std::sqrt(2.0 / double(nc * nv1 * nv2));
        }
    }

    std::size_t input_size() const noexcept {
        return nc * nv1 * nv2;
    }

    std::size_t output_size() const noexcept {
        return k * nh1 * nh2;
    }

    std::size_t parameters() const noexcept {
        return k * nw1 * nw2;
    }

    std::string to_short_string() const {
        char buffer[1024];
        snprintf(buffer, 1024, "Conv(dyn): %lux%lux%lu -> (%lux%lux%lu) -> %s -> %lux%lux%lu", nc, nv1, nv2, k, nw1, nw2, to_string(activation_function).c_str(), k, nh1, nh2);
        return {buffer};
    }

    void display() const {
        std::cout << to_short_string() << std::endl;
    }

    void backup_weights() {
        unique_safe_get(bak_w) = w;
        unique_safe_get(bak_b) = b;
    }

    void restore_weights() {
        w = *bak_w;
        b = *bak_b;
    }

    template <typename V>
    void activate_hidden(output_one_t& output, const V& v) const {
        etl::dyn_matrix<weight, 4> v_cv(2UL, k, nh1, nh2); //Temporary convolution

        auto w_f = etl::force_temporary(w);

        //flip all the kernels horizontally and vertically

        for (std::size_t channel = 0; channel < nc; ++channel) {
            for (size_t k = 0; k < k; ++k) {
                w_f(channel)(k).fflip_inplace();
            }
        }

        v_cv(1) = 0;

        for (std::size_t channel = 0; channel < nc; ++channel) {
            etl::conv_2d_valid_multi(v(channel), w_f(channel), v_cv(0));

            v_cv(1) += v_cv(0);
        }

        output = f_activate<activation_function>(etl::rep(b, nh1, nh2) + v_cv(1));
    }

    template <typename H1, typename V>
    void batch_activate_hidden(H1&& output, const V& v) const {
        etl::dyn_matrix<weight, 4> v_cv(2UL, k, nh1, nh2); //Temporary convolution

        const auto Batch = etl::dim<0>(v);

        auto w_f = force_temporary(w);

        //flip all the kernels horizontally and vertically

        for (std::size_t channel = 0; channel < nc; ++channel) {
            for (size_t k = 0; k < k; ++k) {
                w_f(channel)(k).fflip_inplace();
            }
        }

        for (std::size_t batch = 0; batch < Batch; ++batch) {
            v_cv(1) = 0;

            for (std::size_t channel = 0; channel < nc; ++channel) {
                etl::conv_2d_valid_multi(v(batch)(channel), w_f(channel), v_cv(0));

                v_cv(1) += v_cv(0);
            }

            output(batch) = f_activate<activation_function>(etl::rep(b, nh1, nh2) + v_cv(1));
        }
    }

    void prepare_input(input_one_t& input) const {
        input = input_one_t(nc, nv1, nv2);
    }

    template <typename Input>
    output_t prepare_output(std::size_t samples) const {
        output_t output;
        output.reserve(samples);
        for(size_t i = 0; i < samples; ++i){
            output.emplace_back(k, nh1, nh2);
        }
        return output;
    }

    template <typename Input>
    output_one_t prepare_one_output() const {
        return output_one_t(k, nh1, nh2);
    }

    template <std::size_t B>
    auto prepare_input_batch(){
        return etl::dyn_matrix<weight, 4>(B, nc, nv1, nv2);
    }

    template <std::size_t B>
    auto prepare_output_batch(){
        return etl::dyn_matrix<weight, 4>(B, k, nh1, nh2);
    }

    template <typename DBN>
    void init_sgd_context() {
        this->sgd_context_ptr = std::make_shared<sgd_context<DBN, this_type>>(nc, nv1, nv2, k, nh1, nh2);
    }
};

} //end of dll namespace
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 CNRS
// Author: NMansard from Florent Lamiraux
//
//
// This file is part of hpp-pinocchio
// hpp-pinocchio is free software: you can redistribute it
// and/or modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation, either version
// 3 of the License, or (at your option) any later version.
//
// hpp-pinocchio is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Lesser Public License for more details.  You should have
// received a copy of the GNU Lesser General Public License along with
// hpp-pinocchio  If not, see
// <http://www.gnu.org/licenses/>.

#ifndef HPP_PINOCCHIO_JOINT_HH
# define HPP_PINOCCHIO_JOINT_HH

# include <cstddef>
# include <hpp/pinocchio/fwd.hh>
# include <hpp/pinocchio/config.hh>

namespace hpp {
  namespace pinocchio {
    /// Robot joint
    ///
    /// A joint maps an input vector to a transformation of SE(3) from the
    /// parent frame to the joint frame.
    ///
    /// The input vector is provided through the configuration vector of the
    /// robot the joint belongs to. The joint input vector is composed of the
    /// components of the robot configuration starting at
    /// Joint::rankInConfiguration.
    ///
    /// The joint input vector represents a element of a Lie group, either
    /// \li a vector space for JointTranslation, and bounded JointRotation,
    /// \li the unit circle for non-bounded JointRotation joints,
    /// \li an element of SO(3) for JointSO3, represented by a unit quaternion.
    ///
    /// Operations specific to joints (uniform sampling of input space, straight
    /// interpolation, distance, ...) are performed by a JointConfiguration
    /// instance that has the same class hierarchy as Joint.
    class HPP_PINOCCHIO_DLLAPI Joint {
    public:
      /// \name Construction and copy and destruction
      /// \{

      /// Create a new joint
      /// Returns a null pointer if indexInJointList is 0.
      static JointPtr_t create (DeviceWkPtr_t device, JointIndex indexInJointList );

      /// Constructor
      /// \param device pointer on the device the joint is belonging to.
      /// \param indexInJointList index of the joint, i.e. joint = device.model.joints[index]
      Joint (DeviceWkPtr_t device, JointIndex indexInJointList );

      ~Joint() {}
      /// \}
      // -----------------------------------------------------------------------
      /// \name Name
      /// \{

      /// Get name
      const std::string& name() const;

      /// \}
      // -----------------------------------------------------------------------
      /// \name Position
      /// \{

      /// Joint transformation
      const Transform3f& currentTransformation () const { return currentTransformation(data()); }

      /// Joint transformation
      const Transform3f& currentTransformation (const DeviceData& data) const;

      ///\}
      // -----------------------------------------------------------------------
      /// \name Size and rank
      ///\{

      /// Return number of degrees of freedom
      size_type numberDof () const;

      /// Return number of degrees of freedom
      size_type configSize () const;

      /// Return rank of the joint in the configuration vector
      size_type rankInConfiguration () const;

      /// Return rank of the joint in the velocity vector
      size_type rankInVelocity () const;

      ///\}
      // -----------------------------------------------------------------------
      /// \name Kinematic chain
      /// \{

      /// Get a pointer to the parent joint (if any).
      JointPtr_t parentJoint () const;

      /// Number of child joints
      std::size_t numberChildJoints () const;

      /// Get child joint
      JointPtr_t childJoint (std::size_t rank) const;

      /// Get (constant) placement of joint in parent frame, i.e. model.jointPlacement[idx]
      const Transform3f& positionInParentFrame () const;

      ///\}
      // -----------------------------------------------------------------------
      /// \name Bounds
      /// \{

      /// Set whether given degree of freedom is bounded
      /// \warning Joint always has bounds. When `bounded == false`,
      /// the bounds are `-infinity` and `infinity`.
      void isBounded (size_type rank, bool bounded);
      /// Get whether given degree of freedom is bounded
      bool isBounded (size_type rank) const;
      /// Get lower bound of given degree of freedom
      value_type lowerBound (size_type rank) const;
      /// Get upper bound of given degree of freedom
      value_type upperBound (size_type rank) const;
      /// Set lower bound of given degree of freedom
      void lowerBound (size_type rank, value_type lowerBound);
      /// Set upper bound of given degree of freedom
      void upperBound (size_type rank, value_type upperBound);
      /// Set lower bounds
      void lowerBounds (vectorIn_t lowerBounds);
      /// Set upper bounds
      void upperBounds (vectorIn_t upperBounds);

      /// Get upper bound on linear velocity of the joint frame
      /// \return coefficient \f$\lambda\f$ such that
      /// \f{equation*}
      /// \forall \mathbf{q}_{joint}\ \ \ \ \ \ \|\mathbf {v}\| \leq \lambda \|\mathbf{\dot{q}}_{joint}\|
      /// \f}
      /// where
      /// \li \f$\mathbf{q}_{joint}\f$ is any joint configuration,
      /// \li \f$\mathbf{\dot{q}}_{joint}\f$ is the joint velocity, and
      /// \li \f$\mathbf{v} = J(\mathbf{q})*\mathbf{\dot{q}} \f$ is the linear velocity of the joint frame.
      value_type upperBoundLinearVelocity () const;

      /// Get upper bound on angular velocity of the joint frame
      /// \return coefficient \f$\lambda\f$ such that
      /// \f{equation*}
      /// \forall \mathbf{q}_{joint}\ \ \ \ \ \ \|\omega\| \leq \lambda \|\mathbf{\dot{q}}_{joint}\|
      /// \f}
      /// where
      /// \li \f$\mathbf{q}_{joint}\f$ is any joint configuration,
      /// \li \f$\mathbf{\dot{q}}_{joint}\f$ is the joint velocity, and
      /// \li \f$\omega = J(\mathbf{q})*\mathbf{\dot{q}}\f$ is the angular velocity of the joint frame.
      value_type upperBoundAngularVelocity () const;

      /// Maximal distance of joint origin to parent origin
      const value_type& maximalDistanceToParent () const { return maximalDistanceToParent_; }

      /// \}
    protected:
      /// Compute the maximal distance. \sa maximalDistanceToParent
      void computeMaximalDistanceToParent ();
    public:
      // -----------------------------------------------------------------------
      /// \name Jacobian
      /// \{

      /// Get non const reference to Jacobian
      /// \param localFrame if true, compute the jacobian (6d) in the local frame,
      /// whose linear part corresponds to the velocity of the center of the frame.
      /// If false, the jacobian is expressed in the global frame and its linear part
      /// corresponds to the value of the velocity vector field at the center of the world.
      JointJacobian_t& jacobian (const bool localFrame=true) const { return jacobian (data(), localFrame); }

      /// Get reference to Jacobian
      /// \param data a DeviceData (see hpp::pinocchio::DeviceSync for details).
      /// \param localFrame if true, compute the jacobian (6d) in the local frame,
      /// whose linear part corresponds to the velocity of the center of the frame.
      /// If false, the jacobian is expressed in the global frame and its linear part
      /// corresponds to the value of the velocity vector field at the center of the world.
      JointJacobian_t& jacobian (DeviceData& data, const bool localFrame=true) const;

      /// \}
      // -----------------------------------------------------------------------

      /// Access robot owning the object
      DeviceConstPtr_t robot () const { return devicePtr.lock();}
      /// Access robot owning the object
      DevicePtr_t robot () { return devicePtr.lock();}

      /// \name Body linked to the joint
      /// \{

      /// Get linked body
      BodyPtr_t linkedBody () const;

      /// \}

      /// Display joint
      std::ostream& display (std::ostream& os) const;

      /// Get configuration space of joint
      LiegroupSpacePtr_t configurationSpace () const;

      /// Get configuration space of joint.
      /// Use R^n x SO(n) instead of SE(n).
      LiegroupSpacePtr_t RnxSOnConfigurationSpace () const;

      /// \name Pinocchio API
      /// \{

      const JointIndex& index () const
      {
        return jointIndex;
      }

      const JointModel& jointModel() const;

      /// \}

    protected:
      value_type maximalDistanceToParent_;
      DeviceWkPtr_t devicePtr;
      JointIndex jointIndex;
      std::vector<JointIndex> children;

      /// Store list of childrens.
      void setChildList();
      Model&        model() ;
      const Model&  model() const ;
      DeviceData& data() const;

      /// Assert that the members of the struct are valid (no null pointer, etc).
      void selfAssert() const;

      friend class Device;
    }; // class Joint

    inline std::ostream& operator<< (std::ostream& os, const Joint& joint) { return joint.display(os); }

  } // namespace pinocchio
} // namespace hpp

#endif // HPP_PINOCCHIO_JOINT_HH
<commit_msg>[Doc] document class Joint.<commit_after>//
// Copyright (c) 2016 CNRS
// Author: NMansard from Florent Lamiraux
//
//
// This file is part of hpp-pinocchio
// hpp-pinocchio is free software: you can redistribute it
// and/or modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation, either version
// 3 of the License, or (at your option) any later version.
//
// hpp-pinocchio is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Lesser Public License for more details.  You should have
// received a copy of the GNU Lesser General Public License along with
// hpp-pinocchio  If not, see
// <http://www.gnu.org/licenses/>.

#ifndef HPP_PINOCCHIO_JOINT_HH
# define HPP_PINOCCHIO_JOINT_HH

# include <cstddef>
# include <hpp/pinocchio/fwd.hh>
# include <hpp/pinocchio/config.hh>

namespace hpp {
  namespace pinocchio {
    /// Robot joint
    ///
    /// A joint maps an input vector to a transformation of SE(3) from the
    /// parent frame to the joint frame.
    ///
    /// The input vector is provided through the configuration vector of the
    /// robot the joint belongs to. The joint input vector is composed of the
    /// components of the robot configuration starting at
    /// Joint::rankInConfiguration.
    ///
    /// The joint input vector represents an element of a Lie group, either
    /// \li a vector space for JointTranslation, and bounded JointRotation,
    /// \li the unit circle for non-bounded JointRotation joints,
    /// \li an element of SO(3) for JointSO3, represented by a unit quaternion.
    ///
    /// This class is a wrapper to pinocchio::JointModelTpl.
    class HPP_PINOCCHIO_DLLAPI Joint {
    public:
      /// \name Construction and copy and destruction
      /// \{

      /// Create a new joint
      /// \param indexInJointList index in pinocchio vector of joints
      ///        (pinocchio::ModelTpl::joints)
      /// \return shared pointer to result if indexInJointList > 0, empty
      ///         shared pointer if indexInJointList == 0.
      /// \note indices of device start at 1 since 0 corresponds to "universe".
      static JointPtr_t create (DeviceWkPtr_t device, JointIndex indexInJointList );

      /// Constructor
      /// \param indexInJointList index in pinocchio vector of joints
      ///        (pinocchio::ModelTpl::joints). Should be > 0.
      /// \note indices of device start at 1 since 0 corresponds to "universe".
      Joint (DeviceWkPtr_t device, JointIndex indexInJointList );

      ~Joint() {}
      /// \}
      // -----------------------------------------------------------------------
      /// \name Name
      /// \{

      /// Get name
      const std::string& name() const;

      /// \}
      // -----------------------------------------------------------------------
      /// \name Position
      /// \{

      /// Joint transformation
      const Transform3f& currentTransformation () const { return currentTransformation(data()); }

      /// Joint transformation
      const Transform3f& currentTransformation (const DeviceData& data) const;

      ///\}
      // -----------------------------------------------------------------------
      /// \name Size and rank
      ///\{

      /// Return number of degrees of freedom
      size_type numberDof () const;

      /// Return number of degrees of freedom
      size_type configSize () const;

      /// Return rank of the joint in the configuration vector
      size_type rankInConfiguration () const;

      /// Return rank of the joint in the velocity vector
      size_type rankInVelocity () const;

      ///\}
      // -----------------------------------------------------------------------
      /// \name Kinematic chain
      /// \{

      /// Get a pointer to the parent joint (if any).
      JointPtr_t parentJoint () const;

      /// Number of child joints
      std::size_t numberChildJoints () const;

      /// Get child joint
      JointPtr_t childJoint (std::size_t rank) const;

      /// Get (constant) placement of joint in parent frame, i.e. model.jointPlacement[idx]
      const Transform3f& positionInParentFrame () const;

      ///\}
      // -----------------------------------------------------------------------
      /// \name Bounds
      /// \{

      /// Set whether given degree of freedom is bounded
      /// \warning Joint always has bounds. When `bounded == false`,
      /// the bounds are `-infinity` and `infinity`.
      void isBounded (size_type rank, bool bounded);
      /// Get whether given degree of freedom is bounded
      bool isBounded (size_type rank) const;
      /// Get lower bound of given degree of freedom
      value_type lowerBound (size_type rank) const;
      /// Get upper bound of given degree of freedom
      value_type upperBound (size_type rank) const;
      /// Set lower bound of given degree of freedom
      void lowerBound (size_type rank, value_type lowerBound);
      /// Set upper bound of given degree of freedom
      void upperBound (size_type rank, value_type upperBound);
      /// Set lower bounds
      void lowerBounds (vectorIn_t lowerBounds);
      /// Set upper bounds
      void upperBounds (vectorIn_t upperBounds);

      /// Get upper bound on linear velocity of the joint frame
      /// \return coefficient \f$\lambda\f$ such that
      /// \f{equation*}
      /// \forall \mathbf{q}_{joint}\ \ \ \ \ \ \|\mathbf {v}\| \leq \lambda \|\mathbf{\dot{q}}_{joint}\|
      /// \f}
      /// where
      /// \li \f$\mathbf{q}_{joint}\f$ is any joint configuration,
      /// \li \f$\mathbf{\dot{q}}_{joint}\f$ is the joint velocity, and
      /// \li \f$\mathbf{v} = J(\mathbf{q})*\mathbf{\dot{q}} \f$ is the linear velocity of the joint frame.
      value_type upperBoundLinearVelocity () const;

      /// Get upper bound on angular velocity of the joint frame
      /// \return coefficient \f$\lambda\f$ such that
      /// \f{equation*}
      /// \forall \mathbf{q}_{joint}\ \ \ \ \ \ \|\omega\| \leq \lambda \|\mathbf{\dot{q}}_{joint}\|
      /// \f}
      /// where
      /// \li \f$\mathbf{q}_{joint}\f$ is any joint configuration,
      /// \li \f$\mathbf{\dot{q}}_{joint}\f$ is the joint velocity, and
      /// \li \f$\omega = J(\mathbf{q})*\mathbf{\dot{q}}\f$ is the angular velocity of the joint frame.
      value_type upperBoundAngularVelocity () const;

      /// Maximal distance of joint origin to parent origin
      const value_type& maximalDistanceToParent () const { return maximalDistanceToParent_; }

      /// \}
    protected:
      /// Compute the maximal distance. \sa maximalDistanceToParent
      void computeMaximalDistanceToParent ();
    public:
      // -----------------------------------------------------------------------
      /// \name Jacobian
      /// \{

      /// Get non const reference to Jacobian
      /// \param localFrame if true, compute the jacobian (6d) in the local frame,
      /// whose linear part corresponds to the velocity of the center of the frame.
      /// If false, the jacobian is expressed in the global frame and its linear part
      /// corresponds to the value of the velocity vector field at the center of the world.
      JointJacobian_t& jacobian (const bool localFrame=true) const { return jacobian (data(), localFrame); }

      /// Get reference to Jacobian
      /// \param data a DeviceData (see hpp::pinocchio::DeviceSync for details).
      /// \param localFrame if true, compute the jacobian (6d) in the local frame,
      /// whose linear part corresponds to the velocity of the center of the frame.
      /// If false, the jacobian is expressed in the global frame and its linear part
      /// corresponds to the value of the velocity vector field at the center of the world.
      JointJacobian_t& jacobian (DeviceData& data, const bool localFrame=true) const;

      /// \}
      // -----------------------------------------------------------------------

      /// Access robot owning the object
      DeviceConstPtr_t robot () const { return devicePtr.lock();}
      /// Access robot owning the object
      DevicePtr_t robot () { return devicePtr.lock();}

      /// \name Body linked to the joint
      /// \{

      /// Get linked body
      BodyPtr_t linkedBody () const;

      /// \}

      /// Display joint
      std::ostream& display (std::ostream& os) const;

      /// Get configuration space of joint
      LiegroupSpacePtr_t configurationSpace () const;

      /// Get configuration space of joint.
      /// Use R^n x SO(n) instead of SE(n).
      LiegroupSpacePtr_t RnxSOnConfigurationSpace () const;

      /// \name Pinocchio API
      /// \{

      const JointIndex& index () const
      {
        return jointIndex;
      }

      const JointModel& jointModel() const;

      /// \}

    protected:
      value_type maximalDistanceToParent_;
      DeviceWkPtr_t devicePtr;
      JointIndex jointIndex;
      std::vector<JointIndex> children;

      /// Store list of childrens.
      void setChildList();
      Model&        model() ;
      const Model&  model() const ;
      DeviceData& data() const;

      /// Assert that the members of the struct are valid (no null pointer, etc).
      void selfAssert() const;

      friend class Device;
    }; // class Joint

    inline std::ostream& operator<< (std::ostream& os, const Joint& joint) { return joint.display(os); }

  } // namespace pinocchio
} // namespace hpp

#endif // HPP_PINOCCHIO_JOINT_HH
<|endoftext|>