<commit_before>////////////////////////////////////////////////////////////////////////////
//
// Copyright 2015 Realm Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////

#ifndef REALM_OBJECT_ACCESSOR_HPP
#define REALM_OBJECT_ACCESSOR_HPP

#include <string>
#include "shared_realm.hpp"
#include "list.hpp"

namespace realm {

    class Object {
    public:
        Object(SharedRealm &r, ObjectSchema &s, Row o) : realm(r), object_schema(s), row(o) {}
        // FIXME - all should be const
        SharedRealm realm;
        ObjectSchema &object_schema;
        Row row;

        // property getter/setter
        template<typename ValueType, typename ContextType>
        inline void set_property_value(ContextType ctx, std::string prop_name, ValueType value, bool try_update);

        template<typename ValueType, typename ContextType>
        inline ValueType get_property_value(ContextType ctx, std::string prop_name);

        // create an Object from a native representation
        template<typename ValueType, typename ContextType>
        static inline Object create(ContextType ctx, SharedRealm realm, ObjectSchema &object_schema, ValueType value, bool try_update);

    private:
        template<typename ValueType, typename ContextType>
        inline void set_property_value_impl(ContextType ctx, Property &property, ValueType value, bool try_update);
        template<typename ValueType, typename ContextType>
        inline ValueType get_property_value_impl(ContextType ctx, Property &property);
    };

    //
    // Value converters - template specializations must be implemented for each platform in order to call templated methods on Object
    //
    template<typename ValueType, typename ContextType>
    class NativeAccessor {
    public:
        static bool dict_has_value_for_key(ContextType ctx, ValueType dict, const std::string &prop_name);
        static ValueType dict_value_for_key(ContextType ctx, ValueType dict, const std::string &prop_name);

        static bool has_default_value_for_property(ContextType ctx, Realm *realm, const ObjectSchema &object_schema, const std::string &prop_name);
        static ValueType default_value_for_property(ContextType ctx, Realm *realm, const ObjectSchema &object_schema, const std::string &prop_name);

        static bool to_bool(ContextType, ValueType &);
        static ValueType from_bool(ContextType, bool);
        static long long to_long(ContextType, ValueType &);
        static ValueType from_long(ContextType, long long);
        static float to_float(ContextType, ValueType &);
        static ValueType from_float(ContextType, float);
        static double to_double(ContextType, ValueType &);
        static ValueType from_double(ContextType, double);
        static std::string to_string(ContextType, ValueType &);
        static ValueType from_string(ContextType, StringData);
        static DateTime to_datetime(ContextType, ValueType &);
        static ValueType from_datetime(ContextType, DateTime);

        static bool is_null(ContextType, ValueType &);
        static ValueType null_value(ContextType);

        // convert value to persisted object
        // for existing objects return the existing row index
        // for new/updated objects return the row index
        static size_t to_object_index(ContextType ctx, SharedRealm &realm, ValueType &val, std::string &type, bool try_update);
        static ValueType from_object(ContextType ctx, Object);

        // list value acessors
        static size_t list_size(ContextType ctx, ValueType &val);
        static ValueType list_value_at_index(ContextType ctx, ValueType &val, size_t index);
        static ValueType from_list(ContextType ctx, List);

        //
        // Deprecated
        //
        static Mixed to_mixed(ContextType ctx, ValueType &val) { throw std::runtime_error("'Any' type is unsupported"); }
    };

    //
    // template method implementations
    //
    template <typename ValueType, typename ContextType>
    inline void Object::set_property_value(ContextType ctx, std::string prop_name, ValueType value, bool try_update)
    {
        Property *prop = object_schema.property_for_name(prop_name);
        if (!prop) {
            throw std::runtime_error("Setting invalid property '" + prop_name + "' on object '" + object_schema.name + "'.");
        }
        set_property_value_impl(ctx, *prop, value, try_update);
    };

    template <typename ValueType, typename ContextType>
    inline ValueType Object::get_property_value(ContextType ctx, std::string prop_name)
    {
        Property *prop = object_schema.property_for_name(prop_name);
        if (!prop) {
            throw std::runtime_error("Setting invalid property '" + prop_name + "' on object '" + object_schema.name + "'.");
        }
        return get_property_value_impl<ValueType>(ctx, *prop);
    };

    template <typename ValueType, typename ContextType>
    inline void Object::set_property_value_impl(ContextType ctx, Property &property, ValueType value, bool try_update)
    {
        using Accessor = NativeAccessor<ValueType, ContextType>;

        if (!realm->is_in_transaction()) {
            throw std::runtime_error("Can only set property values within a transaction.");
        }

        size_t column = property.table_column;
        switch (property.type) {
            case PropertyTypeBool:
                row.set_bool(column, Accessor::to_bool(ctx, value));
                break;
            case PropertyTypeInt:
                row.set_int(column, Accessor::to_long(ctx, value));
                break;
            case PropertyTypeFloat:
                row.set_float(column, Accessor::to_float(ctx, value));
                break;
            case PropertyTypeDouble:
                row.set_double(column, Accessor::to_double(ctx, value));
                break;
            case PropertyTypeString:
                row.set_string(column, Accessor::to_string(ctx, value));
                break;
            case PropertyTypeData:
                row.set_binary(column, BinaryData(Accessor::to_string(ctx, value)));
                break;
            case PropertyTypeAny:
                row.set_mixed(column, Accessor::to_mixed(ctx, value));
                break;
            case PropertyTypeDate:
                row.set_datetime(column, Accessor::to_datetime(ctx, value));
                break;
            case PropertyTypeObject: {
                if (Accessor::is_null(ctx, value)) {
                    row.nullify_link(column);
                }
                else {
                    row.set_link(column, Accessor::to_object_index(ctx, realm, value, property.object_type, try_update));
                }
                break;
            }
            case PropertyTypeArray: {
                realm::LinkViewRef link_view = row.get_linklist(column);
                link_view->clear();
                size_t count = Accessor::list_size(ctx, value);
                for (size_t i = 0; i < count; i++) {
                    ValueType element = Accessor::list_value_at_index(ctx, value, i);
                    link_view->add(Accessor::to_object_index(ctx, realm, element, property.object_type, try_update));
                }
                break;
            }
        }
    }

    template <typename ValueType, typename ContextType>
    inline ValueType Object::get_property_value_impl(ContextType ctx, Property &property)
    {
        using Accessor = NativeAccessor<ValueType, ContextType>;

        size_t column = property.table_column;
        switch (property.type) {
            case PropertyTypeBool:
                return Accessor::from_bool(ctx, row.get_bool(column));
            case PropertyTypeInt:
                return Accessor::from_long(ctx, row.get_int(column));
            case PropertyTypeFloat:
                return Accessor::from_float(ctx, row.get_float(column));
            case PropertyTypeDouble:
                return Accessor::from_double(ctx, row.get_double(column));
            case PropertyTypeString:
                return Accessor::from_string(ctx, row.get_string(column));
            case PropertyTypeData:
                return Accessor::from_string(ctx, (std::string)row.get_binary(column));
            case PropertyTypeAny:
                throw "Any not supported";
            case PropertyTypeDate:
                return Accessor::from_datetime(ctx, row.get_datetime(column));
            case PropertyTypeObject: {
                auto linkObjectSchema = realm->config().schema->find(property.object_type);
                TableRef table = ObjectStore::table_for_object_type(realm->read_group(), linkObjectSchema->name);
                if (row.is_null_link(property.table_column)) {
                    return Accessor::null_value(ctx);
                }
                return Accessor::from_object(ctx, std::move(Object(realm, *linkObjectSchema, table->get(row.get_link(column)))));
            }
            case PropertyTypeArray: {
                auto arrayObjectSchema = realm->config().schema->find(property.object_type);
                return Accessor::from_list(ctx, std::move(List(realm, *arrayObjectSchema, static_cast<LinkViewRef>(row.get_linklist(column)))));
            }
        }
    }

    template<typename ValueType, typename ContextType>
    inline Object Object::create(ContextType ctx, SharedRealm realm, ObjectSchema &object_schema, ValueType value, bool try_update)
    {
        using Accessor = NativeAccessor<ValueType, ContextType>;

        if (!realm->is_in_transaction()) {
            throw std::runtime_error("Can only create objects within a transaction.");
        }

        // get or create our accessor
        bool created;

        // try to get existing row if updating
        size_t row_index = realm::not_found;
        realm::TableRef table = ObjectStore::table_for_object_type(realm->read_group(), object_schema.name);
        Property *primary_prop = object_schema.primary_key_property();
        if (primary_prop) {
            // search for existing object based on primary key type
            ValueType primary_value = Accessor::dict_value_for_key(ctx, value, object_schema.primary_key);
            if (primary_prop->type == PropertyTypeString) {
                row_index = table->find_first_string(primary_prop->table_column, Accessor::to_string(ctx, primary_value));
            }
            else {
                row_index = table->find_first_int(primary_prop->table_column, Accessor::to_long(ctx, primary_value));
            }

            if (!try_update && row_index != realm::not_found) {
                throw std::runtime_error("Attempting to create an object of type '" + object_schema.name + "' with an exising primary key value.");
            }
        }

        // if no existing, create row
        created = false;
        if (row_index == realm::not_found) {
            row_index = table->add_empty_row();
            created = true;
        }

        // populate
        Object object(realm, object_schema, table->get(row_index));
        for (Property &prop : object_schema.properties) {
            if (created || !prop.is_primary) {
                if (Accessor::dict_has_value_for_key(ctx, value, prop.name)) {
                    object.set_property_value_impl(ctx, prop, Accessor::dict_value_for_key(ctx, value, prop.name), try_update);
                }
                else if (created) {
                    if (Accessor::has_default_value_for_property(ctx, realm.get(), object_schema, prop.name)) {
                        object.set_property_value_impl(ctx, prop, Accessor::default_value_for_property(ctx, realm.get(), object_schema, prop.name), try_update);
                    }
                    else {
                        throw std::runtime_error("Missing property value for property " + prop.name);
                    }
                }
            }
        }
        return object;
    }
}

#endif /* defined(REALM_OBJECT_ACCESSOR_HPP) */
<commit_msg>make object members private<commit_after>////////////////////////////////////////////////////////////////////////////
//
// Copyright 2015 Realm Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////

#ifndef REALM_OBJECT_ACCESSOR_HPP
#define REALM_OBJECT_ACCESSOR_HPP

#include <string>
#include "shared_realm.hpp"
#include "list.hpp"

namespace realm {

    class Object {
    public:
        Object(SharedRealm &r, ObjectSchema &s, Row o) : m_realm(r), object_schema(s), m_row(o) {}

        // property getter/setter
        template<typename ValueType, typename ContextType>
        inline void set_property_value(ContextType ctx, std::string prop_name, ValueType value, bool try_update);

        template<typename ValueType, typename ContextType>
        inline ValueType get_property_value(ContextType ctx, std::string prop_name);

        // create an Object from a native representation
        template<typename ValueType, typename ContextType>
        static inline Object create(ContextType ctx, SharedRealm realm, ObjectSchema &object_schema, ValueType value, bool try_update);

        const ObjectSchema &object_schema;
        SharedRealm realm() { return m_realm; }
        Row row() { return m_row; }

    private:
        SharedRealm m_realm;
        Row m_row;

        template<typename ValueType, typename ContextType>
        inline void set_property_value_impl(ContextType ctx, const Property &property, ValueType value, bool try_update);
        template<typename ValueType, typename ContextType>
        inline ValueType get_property_value_impl(ContextType ctx, const Property &property);
    };

    //
    // Value converters - template specializations must be implemented for each platform in order to call templated methods on Object
    //
    template<typename ValueType, typename ContextType>
    class NativeAccessor {
    public:
        static bool dict_has_value_for_key(ContextType ctx, ValueType dict, const std::string &prop_name);
        static ValueType dict_value_for_key(ContextType ctx, ValueType dict, const std::string &prop_name);

        static bool has_default_value_for_property(ContextType ctx, Realm *realm, const ObjectSchema &object_schema, const std::string &prop_name);
        static ValueType default_value_for_property(ContextType ctx, Realm *realm, const ObjectSchema &object_schema, const std::string &prop_name);

        static bool to_bool(ContextType, ValueType &);
        static ValueType from_bool(ContextType, bool);
        static long long to_long(ContextType, ValueType &);
        static ValueType from_long(ContextType, long long);
        static float to_float(ContextType, ValueType &);
        static ValueType from_float(ContextType, float);
        static double to_double(ContextType, ValueType &);
        static ValueType from_double(ContextType, double);
        static std::string to_string(ContextType, ValueType &);
        static ValueType from_string(ContextType, StringData);
        static DateTime to_datetime(ContextType, ValueType &);
        static ValueType from_datetime(ContextType, DateTime);

        static bool is_null(ContextType, ValueType &);
        static ValueType null_value(ContextType);

        // convert value to persisted object
        // for existing objects return the existing row index
        // for new/updated objects return the row index
        static size_t to_object_index(ContextType ctx, SharedRealm &realm, ValueType &val, const std::string &type, bool try_update);
        static ValueType from_object(ContextType ctx, Object);

        // list value acessors
        static size_t list_size(ContextType ctx, ValueType &val);
        static ValueType list_value_at_index(ContextType ctx, ValueType &val, size_t index);
        static ValueType from_list(ContextType ctx, List);

        //
        // Deprecated
        //
        static Mixed to_mixed(ContextType ctx, ValueType &val) { throw std::runtime_error("'Any' type is unsupported"); }
    };

    //
    // template method implementations
    //
    template <typename ValueType, typename ContextType>
    inline void Object::set_property_value(ContextType ctx, std::string prop_name, ValueType value, bool try_update)
    {
        const Property *prop = object_schema.property_for_name(prop_name);
        if (!prop) {
            throw std::runtime_error("Setting invalid property '" + prop_name + "' on object '" + object_schema.name + "'.");
        }
        set_property_value_impl(ctx, *prop, value, try_update);
    };

    template <typename ValueType, typename ContextType>
    inline ValueType Object::get_property_value(ContextType ctx, std::string prop_name)
    {
        const Property *prop = object_schema.property_for_name(prop_name);
        if (!prop) {
            throw std::runtime_error("Setting invalid property '" + prop_name + "' on object '" + object_schema.name + "'.");
        }
        return get_property_value_impl<ValueType>(ctx, *prop);
    };

    template <typename ValueType, typename ContextType>
    inline void Object::set_property_value_impl(ContextType ctx, const Property &property, ValueType value, bool try_update)
    {
        using Accessor = NativeAccessor<ValueType, ContextType>;

        if (!m_realm->is_in_transaction()) {
            throw std::runtime_error("Can only set property values within a transaction.");
        }

        size_t column = property.table_column;
        switch (property.type) {
            case PropertyTypeBool:
                m_row.set_bool(column, Accessor::to_bool(ctx, value));
                break;
            case PropertyTypeInt:
                m_row.set_int(column, Accessor::to_long(ctx, value));
                break;
            case PropertyTypeFloat:
                m_row.set_float(column, Accessor::to_float(ctx, value));
                break;
            case PropertyTypeDouble:
                m_row.set_double(column, Accessor::to_double(ctx, value));
                break;
            case PropertyTypeString:
                m_row.set_string(column, Accessor::to_string(ctx, value));
                break;
            case PropertyTypeData:
                m_row.set_binary(column, BinaryData(Accessor::to_string(ctx, value)));
                break;
            case PropertyTypeAny:
                m_row.set_mixed(column, Accessor::to_mixed(ctx, value));
                break;
            case PropertyTypeDate:
                m_row.set_datetime(column, Accessor::to_datetime(ctx, value));
                break;
            case PropertyTypeObject: {
                if (Accessor::is_null(ctx, value)) {
                    m_row.nullify_link(column);
                }
                else {
                    m_row.set_link(column, Accessor::to_object_index(ctx, m_realm, value, property.object_type, try_update));
                }
                break;
            }
            case PropertyTypeArray: {
                realm::LinkViewRef link_view = m_row.get_linklist(column);
                link_view->clear();
                size_t count = Accessor::list_size(ctx, value);
                for (size_t i = 0; i < count; i++) {
                    ValueType element = Accessor::list_value_at_index(ctx, value, i);
                    link_view->add(Accessor::to_object_index(ctx, m_realm, element, property.object_type, try_update));
                }
                break;
            }
        }
    }

    template <typename ValueType, typename ContextType>
    inline ValueType Object::get_property_value_impl(ContextType ctx, const Property &property)
    {
        using Accessor = NativeAccessor<ValueType, ContextType>;

        size_t column = property.table_column;
        switch (property.type) {
            case PropertyTypeBool:
                return Accessor::from_bool(ctx, m_row.get_bool(column));
            case PropertyTypeInt:
                return Accessor::from_long(ctx, m_row.get_int(column));
            case PropertyTypeFloat:
                return Accessor::from_float(ctx, m_row.get_float(column));
            case PropertyTypeDouble:
                return Accessor::from_double(ctx, m_row.get_double(column));
            case PropertyTypeString:
                return Accessor::from_string(ctx, m_row.get_string(column));
            case PropertyTypeData:
                return Accessor::from_string(ctx, (std::string)m_row.get_binary(column));
            case PropertyTypeAny:
                throw "Any not supported";
            case PropertyTypeDate:
                return Accessor::from_datetime(ctx, m_row.get_datetime(column));
            case PropertyTypeObject: {
                auto linkObjectSchema = m_realm->config().schema->find(property.object_type);
                TableRef table = ObjectStore::table_for_object_type(m_realm->read_group(), linkObjectSchema->name);
                if (m_row.is_null_link(property.table_column)) {
                    return Accessor::null_value(ctx);
                }
                return Accessor::from_object(ctx, std::move(Object(m_realm, *linkObjectSchema, table->get(m_row.get_link(column)))));
            }
            case PropertyTypeArray: {
                auto arrayObjectSchema = m_realm->config().schema->find(property.object_type);
                return Accessor::from_list(ctx, std::move(List(m_realm, *arrayObjectSchema, static_cast<LinkViewRef>(m_row.get_linklist(column)))));
            }
        }
    }

    template<typename ValueType, typename ContextType>
    inline Object Object::create(ContextType ctx, SharedRealm realm, ObjectSchema &object_schema, ValueType value, bool try_update)
    {
        using Accessor = NativeAccessor<ValueType, ContextType>;

        if (!realm->is_in_transaction()) {
            throw std::runtime_error("Can only create objects within a transaction.");
        }

        // get or create our accessor
        bool created;

        // try to get existing row if updating
        size_t row_index = realm::not_found;
        realm::TableRef table = ObjectStore::table_for_object_type(realm->read_group(), object_schema.name);
        const Property *primary_prop = object_schema.primary_key_property();
        if (primary_prop) {
            // search for existing object based on primary key type
            ValueType primary_value = Accessor::dict_value_for_key(ctx, value, object_schema.primary_key);
            if (primary_prop->type == PropertyTypeString) {
                row_index = table->find_first_string(primary_prop->table_column, Accessor::to_string(ctx, primary_value));
            }
            else {
                row_index = table->find_first_int(primary_prop->table_column, Accessor::to_long(ctx, primary_value));
            }

            if (!try_update && row_index != realm::not_found) {
                throw std::runtime_error("Attempting to create an object of type '" + object_schema.name + "' with an exising primary key value.");
            }
        }

        // if no existing, create row
        created = false;
        if (row_index == realm::not_found) {
            row_index = table->add_empty_row();
            created = true;
        }

        // populate
        Object object(realm, object_schema, table->get(row_index));
        for (Property &prop : object_schema.properties) {
            if (created || !prop.is_primary) {
                if (Accessor::dict_has_value_for_key(ctx, value, prop.name)) {
                    object.set_property_value_impl(ctx, prop, Accessor::dict_value_for_key(ctx, value, prop.name), try_update);
                }
                else if (created) {
                    if (Accessor::has_default_value_for_property(ctx, realm.get(), object_schema, prop.name)) {
                        object.set_property_value_impl(ctx, prop, Accessor::default_value_for_property(ctx, realm.get(), object_schema, prop.name), try_update);
                    }
                    else {
                        throw std::runtime_error("Missing property value for property " + prop.name);
                    }
                }
            }
        }
        return object;
    }
}

#endif /* defined(REALM_OBJECT_ACCESSOR_HPP) */
<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 *     Copyright 2011 Couchbase, Inc.
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

#include "config.h"
#include "observe_registry.hh"
#include "command_ids.h"
#include "vbucket.hh"

protocol_binary_response_status ObserveRegistry::observeKey(const std::string &key,
                                                            const uint64_t cas,
                                                            const uint16_t vbucket,
                                                            const uint64_t expiration,
                                                            const std::string &obs_set_name) {
    LockHolder rl(registry_mutex);
    std::map<std::string, ObserveSet*>::iterator itr = registry.find(obs_set_name);
    ObserveSet* obs_set;
    if (itr == registry.end()) {
        obs_set = addObserveSet(obs_set_name, expiration);
        if (obs_set == NULL) {
            return PROTOCOL_BINARY_RESPONSE_ETMPFAIL;
        }
    } else if (itr->second->isExpired()) {
        removeObserveSet(itr);
        obs_set = addObserveSet(obs_set_name, expiration);
        if (obs_set == NULL) {
            return PROTOCOL_BINARY_RESPONSE_ETMPFAIL;
        }
    } else {
        obs_set = itr->second;
    }
    return obs_set->add(key, cas, vbucket);
}

void ObserveRegistry::unobserveKey(const std::string &key,
                                   const uint64_t cas,
                                   const uint16_t vbucket,
                                   const std::string &obs_set_name) {
    LockHolder rl(registry_mutex);
    std::map<std::string, ObserveSet*>::iterator itr = registry.find(obs_set_name);
    if (itr != registry.end()) {
        if (itr->second->isExpired()) {
            removeObserveSet(itr);
        } else {
            itr->second->remove(key, cas, vbucket);
        }
    }
}

void ObserveRegistry::removeExpired() {
    LockHolder lh(registry_mutex);
    std::map<std::string, ObserveSet*>::iterator itr;
    for (itr = registry.begin(); itr != registry.end(); itr++) {
        if (itr->second->isExpired()) {
            removeObserveSet(itr);
        }
    }
}

state_map* ObserveRegistry::getObserveSetState(const std::string &obs_set_name) {
    LockHolder rl(registry_mutex);
    std::map<std::string, ObserveSet*>::iterator obs_set = registry.find(obs_set_name);
    if (obs_set == registry.end()) {
        getLogger()->log(EXTENSION_LOG_DEBUG, NULL,
                         "Tryed to get state for non-existent observe set %s",
                         obs_set_name.c_str());
        return new state_map;
    }
    return obs_set->second->getState();
}

void ObserveRegistry::itemsPersisted(std::list<queued_item> &itemlist) {
    LockHolder lh(registry_mutex);
    std::list<queued_item>::iterator itr;
    for (itr = itemlist.begin(); itr != itemlist.end(); itr++) {
        std::map<std::string,ObserveSet*>::iterator obs_itr;
        for (obs_itr = registry.begin(); obs_itr != registry.end(); obs_itr++) {
            if (!obs_itr->second->isExpired()) {
                obs_itr->second->keyEvent((*itr)->getKey().c_str(),
                                          (*itr)->getCas(),
                                          (*itr)->getVBucketId(),
                                          OBS_PERSISTED_EVENT);
            } else {
                removeObserveSet(obs_itr);
            }
        }
    }
}

void ObserveRegistry::itemModified(const Item &itm) {
    LockHolder lh(registry_mutex);
    std::map<std::string,ObserveSet*>::iterator itr;
    for (itr = registry.begin(); itr != registry.end(); itr++) {
        if (!itr->second->isExpired()) {
            itr->second->keyEvent(itm.getKey().c_str(), itm.getCas(),
                                  itm.getVBucketId(), OBS_MODIFIED_EVENT);
        } else {
            removeObserveSet(itr);
        }
    }
}

void ObserveRegistry::itemDeleted(const std::string &key, const uint64_t cas,
                                  const uint16_t vbucket) {
    LockHolder lh(registry_mutex);
    std::map<std::string,ObserveSet*>::iterator itr;
    for (itr = registry.begin(); itr != registry.end(); itr++) {
        if (!itr->second->isExpired()) {
            itr->second->keyEvent(key, cas, vbucket, OBS_DELETED_EVENT);
        } else {
            removeObserveSet(itr);
        }
    }
}

void ObserveRegistry::itemReplicated(const Item &itm) {
    LockHolder lh(registry_mutex);
    std::map<std::string,ObserveSet*>::iterator itr;
    for (itr = registry.begin(); itr != registry.end(); itr++) {
        if (!itr->second->isExpired()) {
            itr->second->keyEvent(itm.getKey().c_str(), itm.getCas(),
                                  itm.getVBucketId(), OBS_REPLICATED_EVENT);
        } else {
            removeObserveSet(itr);
        }
    }
}

void ObserveRegistry::removeObserveSet(std::map<std::string,ObserveSet*>::iterator itr) {
    if (itr != registry.end()) {
        delete itr->second;
        registry.erase(itr);
    }
}

ObserveSet* ObserveRegistry::addObserveSet(const std::string &obs_set_name,
                                           const uint16_t expiration) {
    std::pair<std::map<std::string,ObserveSet*>::iterator,bool> res;
    res = registry.insert(std::pair<std::string,ObserveSet*>(obs_set_name,
                              new ObserveSet(epstore, stats, expiration)));
    if (!res.second) {
        stats->obsErrors++;
        return NULL;
    }
    stats->totalObserveSets++;
    getLogger()->log(EXTENSION_LOG_DEBUG, NULL, "Created new observe set: %s",
                     obs_set_name.c_str());
    return res.first->second;
}

const hrtime_t ObserveSet::ONE_SECOND = 1000000000;

protocol_binary_response_status ObserveSet::add(const std::string &key, uint64_t cas,
                                                const uint16_t vbucket) {
    if ((*epstore)->getVBucket(vbucket)->getState() != vbucket_state_dead) {
        std::map<int, VBObserveSet*>::iterator obs_set = observe_set.find(vbucket);
        if (obs_set == observe_set.end()) {
            std::pair<std::map<int,VBObserveSet*>::iterator,bool> res;
            res = observe_set.insert(std::pair<int,VBObserveSet*>(vbucket,
                                     new VBObserveSet(epstore, stats)));
            if (!res.second) {
                lastTouched = gethrtime();
                stats->obsErrors++;
                return PROTOCOL_BINARY_RESPONSE_ETMPFAIL;
            }
            obs_set = res.first;
        }
        lastTouched = gethrtime();
        if (size >= MAX_OBS_SET_SIZE) {
            stats->obsErrors++;
            return PROTOCOL_BINARY_RESPONSE_EBUSY;
        } else if (obs_set->second->add(key, cas, vbucket)) {
            size++;
            return PROTOCOL_BINARY_RESPONSE_SUCCESS;
        } else {
            stats->obsErrors++;
            return PROTOCOL_BINARY_RESPONSE_ETMPFAIL;
        }
    }
    return PROTOCOL_BINARY_RESPONSE_SUCCESS;
}

void ObserveSet::remove(const std::string &key, const uint64_t cas,
                        const uint16_t vbucket) {
    if (observe_set.find(vbucket) != observe_set.end()) {
        VBObserveSet *vb_observe_set = observe_set.find(vbucket)->second;
        if (vb_observe_set->remove(key, cas)) {
            size--;
        }
        lastTouched = gethrtime();
    }
}

void ObserveSet::keyEvent(const std::string &key, const uint64_t cas,
                          const uint16_t vbucket, int event) {
    std::map<int,VBObserveSet*>::iterator itr = observe_set.find(vbucket);
    if (itr != observe_set.end()) {
        itr->second->keyEvent(key, cas, event);
        lastTouched = gethrtime();
    }
}

bool ObserveSet::isExpired() {
    hrtime_t now = gethrtime();
    if ((now - lastTouched) > expiration) {
        return true;
    }
    return false;
}

state_map* ObserveSet::getState() {
    state_map *obs_state = new state_map();
    std::map<int, VBObserveSet* >::iterator itr;
    for (itr = observe_set.begin(); itr != observe_set.end(); itr++) {
        if ((*epstore)->getVBucket(itr->first)->getState() == vbucket_state_active) {
            VBObserveSet *vb_observe_set = itr->second;
            vb_observe_set->getState(obs_state);
        }
    }
    return obs_state;
}

ObserveSet::~ObserveSet() {
    std::map<int, VBObserveSet* >::iterator itr;
    for (itr = observe_set.begin(); itr != observe_set.end(); itr++) {
        size -= itr->second->size();
        delete itr->second;
        stats->totalObserveSets--;
    }
}

VBObserveSet::~VBObserveSet() {
    stats->obsRegSize -= keylist.size();
}

// Returns true if an item was added to the list
bool VBObserveSet::add(const std::string &key, const uint64_t cas,
                       const uint16_t vbucket) {
    observed_key_t obs_key(key, cas);
    std::list<observed_key_t>::iterator itr;
    for (itr = keylist.begin(); itr != keylist.end(); itr++) {
        if (itr->key.compare(key) == 0 && itr->cas == cas) {
            return true;
        }
    }
    if ((*epstore)->getVBucket(vbucket)->getState() == vbucket_state_replica) {
        obs_key.replicas = -1;
    }
    stats->obsRegSize++;
    keylist.push_back(obs_key);
    return true;
}

// Returns true if an item was removed from the list, returns false if
// the item didn't exist
bool VBObserveSet::remove(const std::string &key, const uint64_t cas) {
    std::list<observed_key_t>::iterator itr;
    for (itr = keylist.begin(); itr != keylist.end(); itr++) {
        observed_key_t obs_key = *itr;
        if (obs_key.key.compare(key) == 0 && obs_key.cas == cas) {
            stats->obsRegSize--;
            keylist.erase(itr);
            return true;
        }
    }
    return false;
}

void VBObserveSet::getState(state_map *sm) {
    std::list<observed_key_t>::iterator itr;
    for (itr = keylist.begin(); itr != keylist.end(); itr++) {
        std::stringstream state_key;
        std::stringstream state_value;
        state_key << itr->key << "," << itr->cas;
        state_value << (int)itr->replicas << ",";
        if (itr->deleted) {
            state_value << "deleted";
        }
        if (itr->mutated) {
            if (itr->deleted) {
                state_value << ",";
            }
            state_value << "mutated";
        }
        if (itr->persisted) {
            if (itr->deleted || itr->mutated) {
                state_value << ",";
            }
            state_value << "persisted";
        }
        if (!itr->persisted && !itr->mutated && !itr->deleted) {
            state_value << "none";
        }
        (*sm)[state_key.str()] = state_value.str();
    }
}

void VBObserveSet::keyEvent(const std::string &key, const uint64_t cas,
                            int event) {
    std::list<observed_key_t>::iterator itr;
    for (itr = keylist.begin(); itr != keylist.end(); itr++) {
        if (itr->key.compare(key) == 0 && event == OBS_DELETED_EVENT) {
            itr->deleted = true;
        } else if (itr->key.compare(key) == 0 && itr->cas != cas &&
                   event == OBS_MODIFIED_EVENT) {
            itr->mutated = true;
        } else if (itr->key.compare(key) == 0 && itr->cas == cas) {
            if (event == OBS_PERSISTED_EVENT) {
                itr->persisted = true;
            } else if (event == OBS_REPLICATED_EVENT) {
                itr->replicas++;
            }
        }
    }
}

bool ObserveRegistryCleaner::callback(Dispatcher &d, TaskId t) {
    ++stats.obsCleanerRuns;
    observeRegistry.removeExpired();
    d.snooze(t, sleepTime);
    return true;
}
<commit_msg>When a key is observed record its initial stat in the registry<commit_after>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 *     Copyright 2011 Couchbase, Inc.
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

#include "config.h"
#include "observe_registry.hh"
#include "command_ids.h"
#include "vbucket.hh"

protocol_binary_response_status ObserveRegistry::observeKey(const std::string &key,
                                                            const uint64_t cas,
                                                            const uint16_t vbucket,
                                                            const uint64_t expiration,
                                                            const std::string &obs_set_name) {
    LockHolder rl(registry_mutex);
    std::map<std::string, ObserveSet*>::iterator itr = registry.find(obs_set_name);
    ObserveSet* obs_set;
    if (itr == registry.end()) {
        obs_set = addObserveSet(obs_set_name, expiration);
        if (obs_set == NULL) {
            return PROTOCOL_BINARY_RESPONSE_ETMPFAIL;
        }
    } else if (itr->second->isExpired()) {
        removeObserveSet(itr);
        obs_set = addObserveSet(obs_set_name, expiration);
        if (obs_set == NULL) {
            return PROTOCOL_BINARY_RESPONSE_ETMPFAIL;
        }
    } else {
        obs_set = itr->second;
    }
    return obs_set->add(key, cas, vbucket);
}

void ObserveRegistry::unobserveKey(const std::string &key,
                                   const uint64_t cas,
                                   const uint16_t vbucket,
                                   const std::string &obs_set_name) {
    LockHolder rl(registry_mutex);
    std::map<std::string, ObserveSet*>::iterator itr = registry.find(obs_set_name);
    if (itr != registry.end()) {
        if (itr->second->isExpired()) {
            removeObserveSet(itr);
        } else {
            itr->second->remove(key, cas, vbucket);
        }
    }
}

void ObserveRegistry::removeExpired() {
    LockHolder lh(registry_mutex);
    std::map<std::string, ObserveSet*>::iterator itr;
    for (itr = registry.begin(); itr != registry.end(); itr++) {
        if (itr->second->isExpired()) {
            removeObserveSet(itr);
        }
    }
}

state_map* ObserveRegistry::getObserveSetState(const std::string &obs_set_name) {
    LockHolder rl(registry_mutex);
    std::map<std::string, ObserveSet*>::iterator obs_set = registry.find(obs_set_name);
    if (obs_set == registry.end()) {
        getLogger()->log(EXTENSION_LOG_DEBUG, NULL,
                         "Tryed to get state for non-existent observe set %s",
                         obs_set_name.c_str());
        return new state_map;
    }
    return obs_set->second->getState();
}

void ObserveRegistry::itemsPersisted(std::list<queued_item> &itemlist) {
    LockHolder lh(registry_mutex);
    std::list<queued_item>::iterator itr;
    for (itr = itemlist.begin(); itr != itemlist.end(); itr++) {
        std::map<std::string,ObserveSet*>::iterator obs_itr;
        for (obs_itr = registry.begin(); obs_itr != registry.end(); obs_itr++) {
            if (!obs_itr->second->isExpired()) {
                obs_itr->second->keyEvent((*itr)->getKey().c_str(),
                                          (*itr)->getCas(),
                                          (*itr)->getVBucketId(),
                                          OBS_PERSISTED_EVENT);
            } else {
                removeObserveSet(obs_itr);
            }
        }
    }
}

void ObserveRegistry::itemModified(const Item &itm) {
    LockHolder lh(registry_mutex);
    std::map<std::string,ObserveSet*>::iterator itr;
    for (itr = registry.begin(); itr != registry.end(); itr++) {
        if (!itr->second->isExpired()) {
            itr->second->keyEvent(itm.getKey().c_str(), itm.getCas(),
                                  itm.getVBucketId(), OBS_MODIFIED_EVENT);
        } else {
            removeObserveSet(itr);
        }
    }
}

void ObserveRegistry::itemDeleted(const std::string &key, const uint64_t cas,
                                  const uint16_t vbucket) {
    LockHolder lh(registry_mutex);
    std::map<std::string,ObserveSet*>::iterator itr;
    for (itr = registry.begin(); itr != registry.end(); itr++) {
        if (!itr->second->isExpired()) {
            itr->second->keyEvent(key, cas, vbucket, OBS_DELETED_EVENT);
        } else {
            removeObserveSet(itr);
        }
    }
}

void ObserveRegistry::itemReplicated(const Item &itm) {
    LockHolder lh(registry_mutex);
    std::map<std::string,ObserveSet*>::iterator itr;
    for (itr = registry.begin(); itr != registry.end(); itr++) {
        if (!itr->second->isExpired()) {
            itr->second->keyEvent(itm.getKey().c_str(), itm.getCas(),
                                  itm.getVBucketId(), OBS_REPLICATED_EVENT);
        } else {
            removeObserveSet(itr);
        }
    }
}

void ObserveRegistry::removeObserveSet(std::map<std::string,ObserveSet*>::iterator itr) {
    if (itr != registry.end()) {
        delete itr->second;
        registry.erase(itr);
    }
}

ObserveSet* ObserveRegistry::addObserveSet(const std::string &obs_set_name,
                                           const uint16_t expiration) {
    std::pair<std::map<std::string,ObserveSet*>::iterator,bool> res;
    res = registry.insert(std::pair<std::string,ObserveSet*>(obs_set_name,
                              new ObserveSet(epstore, stats, expiration)));
    if (!res.second) {
        stats->obsErrors++;
        return NULL;
    }
    stats->totalObserveSets++;
    getLogger()->log(EXTENSION_LOG_DEBUG, NULL, "Created new observe set: %s",
                     obs_set_name.c_str());
    return res.first->second;
}

const hrtime_t ObserveSet::ONE_SECOND = 1000000000;

protocol_binary_response_status ObserveSet::add(const std::string &key, uint64_t cas,
                                                const uint16_t vbucket) {
    if ((*epstore)->getVBucket(vbucket)->getState() != vbucket_state_dead) {
        std::map<int, VBObserveSet*>::iterator obs_set = observe_set.find(vbucket);
        if (obs_set == observe_set.end()) {
            std::pair<std::map<int,VBObserveSet*>::iterator,bool> res;
            res = observe_set.insert(std::pair<int,VBObserveSet*>(vbucket,
                                     new VBObserveSet(epstore, stats)));
            if (!res.second) {
                lastTouched = gethrtime();
                stats->obsErrors++;
                return PROTOCOL_BINARY_RESPONSE_ETMPFAIL;
            }
            obs_set = res.first;
        }
        lastTouched = gethrtime();
        if (size >= MAX_OBS_SET_SIZE) {
            stats->obsErrors++;
            return PROTOCOL_BINARY_RESPONSE_EBUSY;
        } else if (obs_set->second->add(key, cas, vbucket)) {
            size++;
            return PROTOCOL_BINARY_RESPONSE_SUCCESS;
        } else {
            stats->obsErrors++;
            return PROTOCOL_BINARY_RESPONSE_ETMPFAIL;
        }
    }
    return PROTOCOL_BINARY_RESPONSE_SUCCESS;
}

void ObserveSet::remove(const std::string &key, const uint64_t cas,
                        const uint16_t vbucket) {
    if (observe_set.find(vbucket) != observe_set.end()) {
        VBObserveSet *vb_observe_set = observe_set.find(vbucket)->second;
        if (vb_observe_set->remove(key, cas)) {
            size--;
        }
        lastTouched = gethrtime();
    }
}

void ObserveSet::keyEvent(const std::string &key, const uint64_t cas,
                          const uint16_t vbucket, int event) {
    std::map<int,VBObserveSet*>::iterator itr = observe_set.find(vbucket);
    if (itr != observe_set.end()) {
        itr->second->keyEvent(key, cas, event);
        lastTouched = gethrtime();
    }
}

bool ObserveSet::isExpired() {
    hrtime_t now = gethrtime();
    if ((now - lastTouched) > expiration) {
        return true;
    }
    return false;
}

state_map* ObserveSet::getState() {
    state_map *obs_state = new state_map();
    std::map<int, VBObserveSet* >::iterator itr;
    for (itr = observe_set.begin(); itr != observe_set.end(); itr++) {
        if ((*epstore)->getVBucket(itr->first)->getState() == vbucket_state_active) {
            VBObserveSet *vb_observe_set = itr->second;
            vb_observe_set->getState(obs_state);
        }
    }
    return obs_state;
}

ObserveSet::~ObserveSet() {
    std::map<int, VBObserveSet* >::iterator itr;
    for (itr = observe_set.begin(); itr != observe_set.end(); itr++) {
        size -= itr->second->size();
        delete itr->second;
        stats->totalObserveSets--;
    }
}

VBObserveSet::~VBObserveSet() {
    stats->obsRegSize -= keylist.size();
}

// Returns true if an item was added to the list
bool VBObserveSet::add(const std::string &key, const uint64_t cas,
                       const uint16_t vbucket) {
    observed_key_t obs_key(key, cas);
    std::list<observed_key_t>::iterator itr;
    for (itr = keylist.begin(); itr != keylist.end(); itr++) {
        if (itr->key.compare(key) == 0 && itr->cas == cas) {
            return true;
        }
    }
    StoredValue *sv = (*epstore)->getStoredValue(key, vbucket, false);
    if (sv == NULL) {
        obs_key.deleted = true;
    } else {
        obs_key.mutated = (sv->getCas() != cas);
        obs_key.persisted = (sv->getCas() == cas && sv->isClean());
    }
    if ((*epstore)->getVBucket(vbucket)->getState() == vbucket_state_replica) {
        obs_key.replicas = -1;
    }
    stats->obsRegSize++;
    keylist.push_back(obs_key);
    return true;
}

// Returns true if an item was removed from the list, returns false if
// the item didn't exist
bool VBObserveSet::remove(const std::string &key, const uint64_t cas) {
    std::list<observed_key_t>::iterator itr;
    for (itr = keylist.begin(); itr != keylist.end(); itr++) {
        observed_key_t obs_key = *itr;
        if (obs_key.key.compare(key) == 0 && obs_key.cas == cas) {
            stats->obsRegSize--;
            keylist.erase(itr);
            return true;
        }
    }
    return false;
}

void VBObserveSet::getState(state_map *sm) {
    std::list<observed_key_t>::iterator itr;
    for (itr = keylist.begin(); itr != keylist.end(); itr++) {
        std::stringstream state_key;
        std::stringstream state_value;
        state_key << itr->key << "," << itr->cas;
        state_value << (int)itr->replicas << ",";
        if (itr->deleted) {
            state_value << "deleted";
        }
        if (itr->mutated) {
            if (itr->deleted) {
                state_value << ",";
            }
            state_value << "mutated";
        }
        if (itr->persisted) {
            if (itr->deleted || itr->mutated) {
                state_value << ",";
            }
            state_value << "persisted";
        }
        if (!itr->persisted && !itr->mutated && !itr->deleted) {
            state_value << "none";
        }
        (*sm)[state_key.str()] = state_value.str();
    }
}

void VBObserveSet::keyEvent(const std::string &key, const uint64_t cas,
                            int event) {
    std::list<observed_key_t>::iterator itr;
    for (itr = keylist.begin(); itr != keylist.end(); itr++) {
        if (itr->key.compare(key) == 0 && event == OBS_DELETED_EVENT) {
            itr->deleted = true;
        } else if (itr->key.compare(key) == 0 && itr->cas != cas &&
                   event == OBS_MODIFIED_EVENT) {
            itr->mutated = true;
        } else if (itr->key.compare(key) == 0 && itr->cas == cas) {
            if (event == OBS_PERSISTED_EVENT) {
                itr->persisted = true;
            } else if (event == OBS_REPLICATED_EVENT) {
                itr->replicas++;
            }
        }
    }
}

bool ObserveRegistryCleaner::callback(Dispatcher &d, TaskId t) {
    ++stats.obsCleanerRuns;
    observeRegistry.removeExpired();
    d.snooze(t, sleepTime);
    return true;
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2016-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

#include "dictionary.h"
#include <assert.h>
#include <iostream>
#include <algorithm>
#include <iterator>
#include <unordered_map>
#include "args.h"

extern Args args;

const std::string Dictionary::EOS = "</s>";
const std::string Dictionary::BOW = "<";
const std::string Dictionary::EOW = ">";

Dictionary::Dictionary() {
  size_ = 0;
  nwords_ = 0;
  nlabels_ = 0;
  ntokens_ = 0;
  word2int_.resize(MAX_VOCAB_SIZE);
  for (int32_t i = 0; i < MAX_VOCAB_SIZE; i++) {
    word2int_[i] = -1;
  }
}

Dictionary::~Dictionary() {}

int32_t Dictionary::find(const std::string& w) {
  int32_t h = hashFn(w) % MAX_VOCAB_SIZE;
  while (word2int_[h] != -1 && words_[word2int_[h]].word != w) {
    h = (h + 1) % MAX_VOCAB_SIZE;
  }
  return h;
}

void Dictionary::add(const std::string& w) {
  int32_t h = find(w);
  ntokens_++;
  if (word2int_[h] == -1) {
    word2int_[h] = size_;
    entry we;
    we.word = w;
    we.id = size_;
    we.uf = 1;
    we.type = (w.find(args.label) == 0) ? 1 : 0;
    words_.push_back(we);
    size_++;
  } else {
    words_[word2int_[h]].uf++;
  }
}

int32_t Dictionary::nwords() {
  return nwords_;
}

int32_t Dictionary::nlabels() {
  return nlabels_;
}

int64_t Dictionary::ntokens() {
  return ntokens_;
}

const std::vector<int32_t>& Dictionary::getNgrams(int32_t i) {
  assert(i >= 0 && i < nwords_);
  return words_[i].subwords;
}

const std::vector<int32_t> Dictionary::getNgrams(const std::string& word) {
  std::vector<int32_t> ngrams;
  int32_t i = getId(word);
  if (i >= 0) {
    ngrams = words_[i].subwords;
  } else {
    computeNgrams(BOW + word + EOW, ngrams);
  }
  return ngrams;
}

bool Dictionary::discard(int32_t id, real rand) {
  assert(id >= 0 && id < nwords_);
  if (args.model == model_name::sup) return false;
  return rand > pdiscard_[id];
}

int32_t Dictionary::getId(const std::string& w) {
  int32_t h = find(w);
  return word2int_[h];
}

int8_t Dictionary::getType(int32_t id) {
  assert(id >= 0 && id < size_);
  return words_[id].type;
}

std::string Dictionary::getWord(int32_t id) {
  assert(id >= 0 && id < size_);
  return words_[id].word;
}

uint32_t Dictionary::hash(const std::string& str) {
  uint32_t h = 2166136261;
  for (size_t i = 0; i < str.size(); i++) {
    h = h ^ uint32_t(str[i]);
    h = h * 16777619;
  }
  return h;
}

void Dictionary::computeNgrams(const std::string& word,
                               std::vector<int32_t>& ngrams) {
  for (size_t i = 0; i < word.size(); i++) {
    std::string ngram;
    if ((word[i] & 0xC0) == 0x80) continue;
    for (size_t j = i, n = 1; j < word.size() && n <= args.maxn; n++) {
      ngram.push_back(word[j++]);
      while (j < word.size() && (word[j] & 0xC0) == 0x80) {
        ngram.push_back(word[j++]);
      }
      if (n >= args.minn) {
        int32_t h = hash(ngram) % args.bucket;
        ngrams.push_back(nwords_ + h);
      }
    }
  }
}

void Dictionary::initNgrams() {
  for (size_t i = 0; i < size_; i++) {
    std::string word = BOW + words_[i].word + EOW;
    words_[i].subwords.push_back(i);
    computeNgrams(word, words_[i].subwords);
  }
}

std::string Dictionary::readWord(std::ifstream& fin)
{
  char c;
  std::string word;
  while (!fin.eof()) {
    fin.get(c);
    if (iswspace(c)) {
      if (word.empty()) {
        if (c == '\n') return EOS;
        continue;
      } else {
        if (c == '\n') fin.unget();
        return word;
      }
    }
    word.push_back(c);
  }
  return word;
}

void Dictionary::readFromFile(std::ifstream& ifs) {
  std::string word;
  int64_t minThreshold = 1;
  while (!(word = readWord(ifs)).empty()) {
    add(word);
    if (ntokens_ % 1000000 == 0) {
      std::cout << "\rRead " << ntokens_  / 1000000 << "M words" << std::flush;
    }
    if (size_ > 0.75 * MAX_VOCAB_SIZE) threshold(minThreshold++);
  }
  std::cout << "\rRead " << ntokens_  / 1000000 << "M words" << std::endl;
  threshold(args.minCount);
  initTableDiscard();
  initNgrams();
}

void Dictionary::threshold(int64_t t) {
  sort(words_.begin(), words_.end(), [](const entry& e1, const entry& e2) {
      if (e1.type != e2.type) return e1.type < e2.type;
      return e1.uf > e2.uf;
    });
  words_.erase(remove_if(words_.begin(), words_.end(), [&](const entry& e) {
        return e.uf < t;
      }), words_.end());
  words_.shrink_to_fit();

  size_ = 0;
  nwords_ = 0;
  nlabels_ = 0;
  for (int32_t i = 0; i < MAX_VOCAB_SIZE; i++) {
    word2int_[i] = -1;
  }
  for (auto it = words_.begin(); it != words_.end(); ++it) {
    int32_t h = find(it->word);
    word2int_[h] = size_;
    it->id = size_;
    size_++;
    if (it->type == 0) nwords_++;
    if (it->type == 1) nlabels_++;
  }
}

void Dictionary::initTableDiscard() {
  pdiscard_.resize(size_);
  for (auto& w : words_) {
    real f = real(w.uf) / real(ntokens_);
    pdiscard_[w.id] = sqrt(args.t / f) + args.t / f;
  }
}

std::vector<int64_t> Dictionary::getWordFreq() {
  std::vector<int64_t> freq;
  for (auto& w : words_) {
    if (w.type == 0) {
      freq.push_back(w.uf);
    }
  }
  return freq;
}

std::vector<int64_t> Dictionary::getLabelFreq() {
  std::vector<int64_t> freq;
  for (auto& w : words_) {
    if (w.type == 1) {
      freq.push_back(w.uf);
    }
  }
  return freq;
}

void Dictionary::addNgrams(std::vector<int32_t>& line, int32_t n) {
  int32_t line_size = line.size();
  for (int32_t i = 0; i < line_size; i++) {
    uint64_t h = line[i];
    for (int32_t j = i + 1; j < line_size && j < i + n; j++) {
      h = h * 116049371 + line[j];
      line.push_back(nwords_ + (h % args.bucket));
    }
  }
}

int32_t Dictionary::getLine(std::ifstream& ifs,
                            std::vector<int32_t>& words,
                            std::vector<int32_t>& labels,
                            std::minstd_rand& rng) {
  std::uniform_real_distribution<> uniform(0, 1);
  std::string token;
  int32_t ntokens_ = 0;
  words.clear();
  labels.clear();
  if (ifs.eof()) {
    ifs.clear();
    ifs.seekg(std::streampos(0));
  }
  while (!(token = readWord(ifs)).empty()) {
    ntokens_++;
    if (token == EOS) break;
    int32_t wid = getId(token);
    if (wid < 0) continue;
    int8_t type = getType(wid);
    if (type == 0 && !discard(wid, uniform(rng))) {
      words.push_back(wid);
    }
    if (type == 1) {
      labels.push_back(wid - nwords_);
    }
    if (words.size() > MAX_LINE_SIZE && args.model != model_name::sup) break;
  }
  return ntokens_;
}

std::string Dictionary::getLabel(int32_t lid) {
  return words_[lid + nwords_].word;
}

void Dictionary::save(std::ofstream& ofs) {
  char ender = 0;
  int k = 0;
  if (ofs.is_open()) {
    ofs.write((char*) &size_, sizeof(int32_t));
    ofs.write((char*) &nwords_, sizeof(int32_t));
    ofs.write((char*) &nlabels_, sizeof(int32_t));
    ofs.write((char*) &ntokens_, sizeof(int64_t));
    for (int32_t i = 0; i < size_; i++) {
      entry e = words_[i];
      std::string w = e.word;
      for (int c = 0; c < w.size(); c++) {
        ofs.write(&w[c], sizeof(char));
      }
      ofs.write(&ender, 1);
      ofs.write((char*) &(e.id), sizeof(int32_t));
      ofs.write((char*) &(e.uf), sizeof(int64_t));
      ofs.write((char*) &(e.type), sizeof(int8_t));
      int32_t nSubwords = e.subwords.size();
      ofs.write((char*) &nSubwords, sizeof(int32_t));
      for (int32_t subword : e.subwords) {
        ofs.write((char*) &(subword), sizeof(int32_t));
      }
    }
  }
}

void Dictionary::load(std::ifstream& ifs) {
  words_.clear();
  if (ifs.is_open()) {
    ifs.read((char*) &size_, sizeof(int32_t));
    ifs.read((char*) &nwords_, sizeof(int32_t));
    ifs.read((char*) &nlabels_, sizeof(int32_t));
    ifs.read((char*) &ntokens_, sizeof(int64_t));
    for (int32_t j = 0; j < size_; j++) {
      char c;
      entry we;
      ifs.read(&c, sizeof(char));
      while (c != 0) {
        we.word.push_back(c);
        ifs.read(&c, sizeof(char));
      }
      ifs.read((char*) &we.id, sizeof(int32_t));
      ifs.read((char*) &we.uf, sizeof(int64_t));
      ifs.read((char*) &we.type, sizeof(int8_t));
      int32_t nSubwords = 0;
      ifs.read((char*) &nSubwords, sizeof(int32_t));
      int32_t subword = 0;
      for (int z = 0; z < nSubwords; z++) {
        ifs.read((char*) &subword, sizeof(int32_t));
        we.subwords.push_back(subword);
      }
      words_.push_back(we);
    }
  }
  for (int32_t i = 0; i < MAX_VOCAB_SIZE; i++) {
    word2int_[i] = -1;
  }
  for (auto it = words_.begin(); it != words_.end(); ++it) {
    int32_t h = find(it->word);
    word2int_[h] = it->id;
  }
  initTableDiscard();
}
<commit_msg>Fix local variable name in getLine<commit_after>/**
 * Copyright (c) 2016-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

#include "dictionary.h"
#include <assert.h>
#include <iostream>
#include <algorithm>
#include <iterator>
#include <unordered_map>
#include "args.h"

extern Args args;

const std::string Dictionary::EOS = "</s>";
const std::string Dictionary::BOW = "<";
const std::string Dictionary::EOW = ">";

Dictionary::Dictionary() {
  size_ = 0;
  nwords_ = 0;
  nlabels_ = 0;
  ntokens_ = 0;
  word2int_.resize(MAX_VOCAB_SIZE);
  for (int32_t i = 0; i < MAX_VOCAB_SIZE; i++) {
    word2int_[i] = -1;
  }
}

Dictionary::~Dictionary() {}

int32_t Dictionary::find(const std::string& w) {
  int32_t h = hashFn(w) % MAX_VOCAB_SIZE;
  while (word2int_[h] != -1 && words_[word2int_[h]].word != w) {
    h = (h + 1) % MAX_VOCAB_SIZE;
  }
  return h;
}

void Dictionary::add(const std::string& w) {
  int32_t h = find(w);
  ntokens_++;
  if (word2int_[h] == -1) {
    word2int_[h] = size_;
    entry we;
    we.word = w;
    we.id = size_;
    we.uf = 1;
    we.type = (w.find(args.label) == 0) ? 1 : 0;
    words_.push_back(we);
    size_++;
  } else {
    words_[word2int_[h]].uf++;
  }
}

int32_t Dictionary::nwords() {
  return nwords_;
}

int32_t Dictionary::nlabels() {
  return nlabels_;
}

int64_t Dictionary::ntokens() {
  return ntokens_;
}

const std::vector<int32_t>& Dictionary::getNgrams(int32_t i) {
  assert(i >= 0 && i < nwords_);
  return words_[i].subwords;
}

const std::vector<int32_t> Dictionary::getNgrams(const std::string& word) {
  std::vector<int32_t> ngrams;
  int32_t i = getId(word);
  if (i >= 0) {
    ngrams = words_[i].subwords;
  } else {
    computeNgrams(BOW + word + EOW, ngrams);
  }
  return ngrams;
}

bool Dictionary::discard(int32_t id, real rand) {
  assert(id >= 0 && id < nwords_);
  if (args.model == model_name::sup) return false;
  return rand > pdiscard_[id];
}

int32_t Dictionary::getId(const std::string& w) {
  int32_t h = find(w);
  return word2int_[h];
}

int8_t Dictionary::getType(int32_t id) {
  assert(id >= 0 && id < size_);
  return words_[id].type;
}

std::string Dictionary::getWord(int32_t id) {
  assert(id >= 0 && id < size_);
  return words_[id].word;
}

uint32_t Dictionary::hash(const std::string& str) {
  uint32_t h = 2166136261;
  for (size_t i = 0; i < str.size(); i++) {
    h = h ^ uint32_t(str[i]);
    h = h * 16777619;
  }
  return h;
}

void Dictionary::computeNgrams(const std::string& word,
                               std::vector<int32_t>& ngrams) {
  for (size_t i = 0; i < word.size(); i++) {
    std::string ngram;
    if ((word[i] & 0xC0) == 0x80) continue;
    for (size_t j = i, n = 1; j < word.size() && n <= args.maxn; n++) {
      ngram.push_back(word[j++]);
      while (j < word.size() && (word[j] & 0xC0) == 0x80) {
        ngram.push_back(word[j++]);
      }
      if (n >= args.minn) {
        int32_t h = hash(ngram) % args.bucket;
        ngrams.push_back(nwords_ + h);
      }
    }
  }
}

void Dictionary::initNgrams() {
  for (size_t i = 0; i < size_; i++) {
    std::string word = BOW + words_[i].word + EOW;
    words_[i].subwords.push_back(i);
    computeNgrams(word, words_[i].subwords);
  }
}

std::string Dictionary::readWord(std::ifstream& fin)
{
  char c;
  std::string word;
  while (!fin.eof()) {
    fin.get(c);
    if (iswspace(c)) {
      if (word.empty()) {
        if (c == '\n') return EOS;
        continue;
      } else {
        if (c == '\n') fin.unget();
        return word;
      }
    }
    word.push_back(c);
  }
  return word;
}

void Dictionary::readFromFile(std::ifstream& ifs) {
  std::string word;
  int64_t minThreshold = 1;
  while (!(word = readWord(ifs)).empty()) {
    add(word);
    if (ntokens_ % 1000000 == 0) {
      std::cout << "\rRead " << ntokens_  / 1000000 << "M words" << std::flush;
    }
    if (size_ > 0.75 * MAX_VOCAB_SIZE) threshold(minThreshold++);
  }
  std::cout << "\rRead " << ntokens_  / 1000000 << "M words" << std::endl;
  threshold(args.minCount);
  initTableDiscard();
  initNgrams();
}

void Dictionary::threshold(int64_t t) {
  sort(words_.begin(), words_.end(), [](const entry& e1, const entry& e2) {
      if (e1.type != e2.type) return e1.type < e2.type;
      return e1.uf > e2.uf;
    });
  words_.erase(remove_if(words_.begin(), words_.end(), [&](const entry& e) {
        return e.uf < t;
      }), words_.end());
  words_.shrink_to_fit();

  size_ = 0;
  nwords_ = 0;
  nlabels_ = 0;
  for (int32_t i = 0; i < MAX_VOCAB_SIZE; i++) {
    word2int_[i] = -1;
  }
  for (auto it = words_.begin(); it != words_.end(); ++it) {
    int32_t h = find(it->word);
    word2int_[h] = size_;
    it->id = size_;
    size_++;
    if (it->type == 0) nwords_++;
    if (it->type == 1) nlabels_++;
  }
}

void Dictionary::initTableDiscard() {
  pdiscard_.resize(size_);
  for (auto& w : words_) {
    real f = real(w.uf) / real(ntokens_);
    pdiscard_[w.id] = sqrt(args.t / f) + args.t / f;
  }
}

std::vector<int64_t> Dictionary::getWordFreq() {
  std::vector<int64_t> freq;
  for (auto& w : words_) {
    if (w.type == 0) {
      freq.push_back(w.uf);
    }
  }
  return freq;
}

std::vector<int64_t> Dictionary::getLabelFreq() {
  std::vector<int64_t> freq;
  for (auto& w : words_) {
    if (w.type == 1) {
      freq.push_back(w.uf);
    }
  }
  return freq;
}

void Dictionary::addNgrams(std::vector<int32_t>& line, int32_t n) {
  int32_t line_size = line.size();
  for (int32_t i = 0; i < line_size; i++) {
    uint64_t h = line[i];
    for (int32_t j = i + 1; j < line_size && j < i + n; j++) {
      h = h * 116049371 + line[j];
      line.push_back(nwords_ + (h % args.bucket));
    }
  }
}

int32_t Dictionary::getLine(std::ifstream& ifs,
                            std::vector<int32_t>& words,
                            std::vector<int32_t>& labels,
                            std::minstd_rand& rng) {
  std::uniform_real_distribution<> uniform(0, 1);
  std::string token;
  int32_t ntokens = 0;
  words.clear();
  labels.clear();
  if (ifs.eof()) {
    ifs.clear();
    ifs.seekg(std::streampos(0));
  }
  while (!(token = readWord(ifs)).empty()) {
    if (token == EOS) break;
    int32_t wid = getId(token);
    if (wid < 0) continue;
    int8_t type = getType(wid);
    ntokens++;
    if (type == 0 && !discard(wid, uniform(rng))) {
      words.push_back(wid);
    }
    if (type == 1) {
      labels.push_back(wid - nwords_);
    }
    if (words.size() > MAX_LINE_SIZE && args.model != model_name::sup) break;
  }
  return ntokens;
}

std::string Dictionary::getLabel(int32_t lid) {
  return words_[lid + nwords_].word;
}

void Dictionary::save(std::ofstream& ofs) {
  char ender = 0;
  int k = 0;
  if (ofs.is_open()) {
    ofs.write((char*) &size_, sizeof(int32_t));
    ofs.write((char*) &nwords_, sizeof(int32_t));
    ofs.write((char*) &nlabels_, sizeof(int32_t));
    ofs.write((char*) &ntokens_, sizeof(int64_t));
    for (int32_t i = 0; i < size_; i++) {
      entry e = words_[i];
      std::string w = e.word;
      for (int c = 0; c < w.size(); c++) {
        ofs.write(&w[c], sizeof(char));
      }
      ofs.write(&ender, 1);
      ofs.write((char*) &(e.id), sizeof(int32_t));
      ofs.write((char*) &(e.uf), sizeof(int64_t));
      ofs.write((char*) &(e.type), sizeof(int8_t));
      int32_t nSubwords = e.subwords.size();
      ofs.write((char*) &nSubwords, sizeof(int32_t));
      for (int32_t subword : e.subwords) {
        ofs.write((char*) &(subword), sizeof(int32_t));
      }
    }
  }
}

void Dictionary::load(std::ifstream& ifs) {
  words_.clear();
  if (ifs.is_open()) {
    ifs.read((char*) &size_, sizeof(int32_t));
    ifs.read((char*) &nwords_, sizeof(int32_t));
    ifs.read((char*) &nlabels_, sizeof(int32_t));
    ifs.read((char*) &ntokens_, sizeof(int64_t));
    for (int32_t j = 0; j < size_; j++) {
      char c;
      entry we;
      ifs.read(&c, sizeof(char));
      while (c != 0) {
        we.word.push_back(c);
        ifs.read(&c, sizeof(char));
      }
      ifs.read((char*) &we.id, sizeof(int32_t));
      ifs.read((char*) &we.uf, sizeof(int64_t));
      ifs.read((char*) &we.type, sizeof(int8_t));
      int32_t nSubwords = 0;
      ifs.read((char*) &nSubwords, sizeof(int32_t));
      int32_t subword = 0;
      for (int z = 0; z < nSubwords; z++) {
        ifs.read((char*) &subword, sizeof(int32_t));
        we.subwords.push_back(subword);
      }
      words_.push_back(we);
    }
  }
  for (int32_t i = 0; i < MAX_VOCAB_SIZE; i++) {
    word2int_[i] = -1;
  }
  for (auto it = words_.begin(); it != words_.end(); ++it) {
    int32_t h = find(it->word);
    word2int_[h] = it->id;
  }
  initTableDiscard();
}
<|endoftext|>


<commit_before>/* libwpd
 * Copyright (C) 2003 William Lachance (william.lachance@sympatico.ca)
 * Copyright (C) 2003 Marc Maurer (j.m.maurer@student.utwente.nl)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 *
 * For further information visit http://libwpd.sourceforge.net
 */

/* "This product is not manufactured, approved, or supported by
 * Corel Corporation or Corel Corporation Limited."
 */

#include "WP5VariableLengthGroup.h"
#include "WP5UnsupportedVariableLengthGroup.h"
#include "libwpd_internal.h"
#include "WP5FormatGroup.h"

WP5VariableLengthGroup::WP5VariableLengthGroup()
{
}

WP5VariableLengthGroup * WP5VariableLengthGroup::constructVariableLengthGroup(GsfInput *input, guint8 group)
{
	switch (group)
	{
		case WP5_TOP_FORMAT_GROUP:
			return new WP5FormatGroup(input);
		default:
			// this is an unhandled group, just skip it
			return new WP5UnsupportedVariableLengthGroup(input);
	}
}

void WP5VariableLengthGroup::_read(GsfInput *input)
{
	guint32 startPosition = gsf_input_tell(input);

	WPD_DEBUG_MSG(("WordPerfect: handling a variable length group (group id: 0x%x)\n", m_group));

	m_subGroup = gsf_le_read_guint8(input);
	m_size = gsf_le_read_guint16(input) + 4; // the length is the number of data bytes minus 4 (ie. the function codes)

	WPD_DEBUG_MSG(("WordPerfect: Read variable group header (start_position: %i, sub_group: 0x%x, size: %i)\n", startPosition, m_subGroup, m_size));

	_readContents(input);

	WPD_CHECK_FILE_SEEK_ERROR(gsf_input_seek(input, (startPosition + m_size - 1 - gsf_input_tell(input)), G_SEEK_CUR));

}
<commit_msg>Unbreak debug build<commit_after>/* libwpd
 * Copyright (C) 2003 William Lachance (william.lachance@sympatico.ca)
 * Copyright (C) 2003 Marc Maurer (j.m.maurer@student.utwente.nl)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 *
 * For further information visit http://libwpd.sourceforge.net
 */

/* "This product is not manufactured, approved, or supported by
 * Corel Corporation or Corel Corporation Limited."
 */

#include "WP5VariableLengthGroup.h"
#include "WP5UnsupportedVariableLengthGroup.h"
#include "libwpd_internal.h"
#include "WP5FormatGroup.h"

WP5VariableLengthGroup::WP5VariableLengthGroup()
{
}

WP5VariableLengthGroup * WP5VariableLengthGroup::constructVariableLengthGroup(GsfInput *input, guint8 group)
{
	switch (group)
	{
		case WP5_TOP_FORMAT_GROUP:
			return new WP5FormatGroup(input);
		default:
			// this is an unhandled group, just skip it
			return new WP5UnsupportedVariableLengthGroup(input);
	}
}

void WP5VariableLengthGroup::_read(GsfInput *input)
{
	guint32 startPosition = gsf_input_tell(input);

	WPD_DEBUG_MSG(("WordPerfect: handling a variable length group\n"));

	m_subGroup = gsf_le_read_guint8(input);
	m_size = gsf_le_read_guint16(input) + 4; // the length is the number of data bytes minus 4 (ie. the function codes)

	WPD_DEBUG_MSG(("WordPerfect: Read variable group header (start_position: %i, sub_group: 0x%x, size: %i)\n", startPosition, m_subGroup, m_size));

	_readContents(input);

	WPD_CHECK_FILE_SEEK_ERROR(gsf_input_seek(input, (startPosition + m_size - 1 - gsf_input_tell(input)), G_SEEK_CUR));

}
<|endoftext|>


<commit_before>/*
*
* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U
*
* This file is part of Orion Context Broker.
*
* Orion Context Broker is free software: you can redistribute it and/or
* modify it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Orion Context Broker is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
* General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
*
* For those usages not covered by this license please contact with
* iot_support at tid dot es
*
* Author: Ken Zangelin
*/
#include <string>

#include "ngsi/ContextElement.h"
#include "common/macroSubstitute.h"

/* ****************************************************************************
*
* attributeValue - return value of attribute as a string
*/
static void attributeValue(std::string* valueP, const std::vector<ContextAttribute*>& vec, char* attrName)
{
  for (unsigned int ix = 0; ix < vec.size(); ++ix)
  {
    if (vec[ix]->name != attrName)
    {
      continue;
    }

    if (vec[ix]->valueType == orion::ValueTypeString)
    {
      *valueP = vec[ix]->stringValue;
    }
    else if (vec[ix]->valueType == orion::ValueTypeNumber)
    {
      char i[STRING_SIZE_FOR_INT];

      snprintf(i, sizeof(i), "%f", vec[ix]->numberValue);
      *valueP = i;
    }
    else if (vec[ix]->valueType == orion::ValueTypeBoolean)
    {
      *valueP = (vec[ix]->boolValue == true)? "true" : "false";
    }
    else if (vec[ix]->valueType == orion::ValueTypeNone)
    {
      *valueP = "null";
    }
    else if (vec[ix]->valueType == orion::ValueTypeObject)  // meaning: Compound Value
    {
      if (vec[ix]->compoundValueP)
      {
        if (vec[ix]->compoundValueP->valueType == orion::ValueTypeVector)
        {
          *valueP = "[" + vec[ix]->compoundValueP->toJson(true) + "]";
        }
        else if (vec[ix]->compoundValueP->valueType == orion::ValueTypeObject)
        {
          *valueP = "{" + vec[ix]->compoundValueP->toJson(true) + "}";
        }
        else
        {
          *valueP = "";
        }
      }
      else
      {
        *valueP = "";
      }
    }
    else if (vec[ix]->valueType == orion::ValueTypeObject)
    {
      *valueP = "{" + vec[ix]->compoundValueP->toJson(true) + "}";
    }
    else
    {
      LM_E(("Runtime Error (unknown value type for attribute)"));
      *valueP = "";
    }

    return;
  }

  *valueP = "";
}

/* ****************************************************************************
*
* macroSubstitute -
*/
#define CHUNK_SIZE 1024
void macroSubstitute(std::string* to, const std::string& from, const ContextElement& ce)
{
  char*        toP     = (char*) calloc(1, CHUNK_SIZE);
  int          toIx    = 0;
  int          toLen   = CHUNK_SIZE;
  const char*  fromP   = (char*) from.c_str();

  if (toP == NULL)
  {
    LM_E(("Runtime Error (out of memory)"));
    *to = "";
    return;
  }

  while (*fromP != 0)
  {
    if ((*fromP == '$') && (fromP[1] == '{'))
    {
      char* varP = (char*) &fromP[2];

      fromP += 2;
      while ((*fromP != 0) && (*fromP != '}'))
      {
        ++fromP;
      }

      if (*fromP == 0)
      {
        break;  // ERROR: non-terminated variable
      }

      *((char*) fromP) = 0;
      ++fromP;

      // Variable found, get its value
      char* substitute    = NULL;
      int   substituteLen = 0;

      if (strcmp(varP, "id") == 0)
      {
        substitute = (char*) ce.entityId.id.c_str();
      }
      else if (strcmp(varP, "type") == 0)
      {
        substitute = (char*) ce.entityId.type.c_str();
      }
      else  // attribute
      {
        std::string value;

        attributeValue(&value, ce.contextAttributeVector.vec, varP);
        substitute = (char*) value.c_str();
      }

      if (substitute != NULL)
      {
        substituteLen = strlen(substitute);
      }

      // Room enough?  If not, realloc
      if (toIx + substituteLen >= toLen - 1)
      {
        toP    = (char*) realloc(toP, toLen + CHUNK_SIZE);
        toLen += CHUNK_SIZE;

        if (toP == NULL)
        {
          LM_E(("Runtime Error (out of memory)"));
          *to = "";
          return;
        }
      }

      if (substitute != NULL)
      {
        // Concatenate 'substitute' to toP
        if (substitute != NULL)
        {
          strcat(&toP[toIx], substitute);
          toIx += substituteLen;
        }
      }
    }
    else
    {
      toP[toIx++] = *fromP;
      ++fromP;
    }
  }

  *to = toP;
  free(toP);
}
<commit_msg>Lots of comments in the code for macro substitutions<commit_after>/*
*
* Copyright 2016 Telefonica Investigacion y Desarrollo, S.A.U
*
* This file is part of Orion Context Broker.
*
* Orion Context Broker is free software: you can redistribute it and/or
* modify it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Orion Context Broker is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
* General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
*
* For those usages not covered by this license please contact with
* iot_support at tid dot es
*
* Author: Ken Zangelin
*/
#include <string>

#include "ngsi/ContextElement.h"
#include "common/macroSubstitute.h"

/* ****************************************************************************
*
* attributeValue - return value of attribute as a string
*/
static void attributeValue(std::string* valueP, const std::vector<ContextAttribute*>& vec, char* attrName)
{
  for (unsigned int ix = 0; ix < vec.size(); ++ix)
  {
    if (vec[ix]->name != attrName)
    {
      continue;
    }

    if (vec[ix]->valueType == orion::ValueTypeString)
    {
      *valueP = vec[ix]->stringValue;
    }
    else if (vec[ix]->valueType == orion::ValueTypeNumber)
    {
      char i[STRING_SIZE_FOR_INT];

      snprintf(i, sizeof(i), "%f", vec[ix]->numberValue);
      *valueP = i;
    }
    else if (vec[ix]->valueType == orion::ValueTypeBoolean)
    {
      *valueP = (vec[ix]->boolValue == true)? "true" : "false";
    }
    else if (vec[ix]->valueType == orion::ValueTypeNone)
    {
      *valueP = "null";
    }
    else if ((vec[ix]->valueType == orion::ValueTypeObject) || (vec[ix]->valueType == orion::ValueTypeVector))
    {
      if (vec[ix]->compoundValueP)
      {
        if (vec[ix]->compoundValueP->valueType == orion::ValueTypeVector)
        {
          *valueP = "[" + vec[ix]->compoundValueP->toJson(true) + "]";
        }
        else if (vec[ix]->compoundValueP->valueType == orion::ValueTypeObject)
        {
          *valueP = "{" + vec[ix]->compoundValueP->toJson(true) + "}";
        }
        else
        {
          LM_E(("Runtime Error (attribute is of object type but its compound is of invalid type)"));
          *valueP = "";
        }
      }
      else
      {
        LM_E(("Runtime Error (attribute is of object type but has no compound)"));
        *valueP = "";
      }
    }
    else
    {
      LM_E(("Runtime Error (unknown value type for attribute)"));
      *valueP = "";
    }

    return;
  }

  *valueP = "";
}

/* ****************************************************************************
*
* macroSubstitute -
*
* An initial out-buffer of CHUNK_SIZE is allocated, and later on during the substitution if the
* out-buffer is not big enough, it is realloced with another CHUNK_SIZE.
*
* 1024 is set as CHUNK_SIZE and hopefully for most substitutions it will be enough with ONE allocation.
* Allocation is very costly operations ...
* This function could be a lot faster is we took the first 1024 bytes from the stack instead of using calloc.
* However, the function gets a little more complicated like that as the first reallos would have to be a normal malloc and a memcpy.
*
* Variable used:
*   - toP:     pointer to the beginning of the 'to string' where to add the new characters from the 'from string'
*   - toIx:    the index of the 'to string', always at the end of the string, where to append characters.
*              toIx also serves as the running size of the 'to string'
*   - toLen:   Total size of the 'to string'
*   - fromP:   pointer to the currently analysed character in the 'from string'
*
*/
#define CHUNK_SIZE 1024
void macroSubstitute(std::string* to, const std::string& from, const ContextElement& ce)
{
  char*        toP     = (char*) calloc(1, CHUNK_SIZE);
  int          toIx    = 0;
  int          toLen   = CHUNK_SIZE;
  const char*  fromP   = (char*) from.c_str();

  if (toP == NULL)
  {
    LM_E(("Runtime Error (out of memory)"));
    *to = "";
    return;
  }

  while (*fromP != 0)
  {
    //
    // If the current position in the 'from string' (fromP), points to the two characters
    // "${", then (if an ending '}' is found) a variable substitution is to be performed.
    //
    // Otherwise, just copy the character in 'from string' to 'to string'.
    //
    if ((fromP[0] == '$') && (fromP[1] == '{'))
    {
      //
      // varP points to the character after "${", i.e. the first character of the variable name.
      // E.g. if 'from string' is "${abc}", varP would point to the 'a'
      //
      char* varP = (char*) &fromP[2];

      // Set fromP to point to the chatactrer after "${" - the first character of the variable name
      fromP += 2;

      // Find the closing '}'
      while ((fromP[0] != 0) && (*fromP != '}'))
      {
        ++fromP;
      }

      // If the 'from string' ends without finding a closing '}', ERROR
      if (fromP[0] == 0)
      {
        break;  // ERROR: non-terminated variable
      }

      // Terminate the variable name, by replacing the '}' with a ZERO
      *((char*) fromP) = 0;

      // Then make fromP point to the character AFTER the closing '}'
      ++fromP;

      // Variable found, get its value
      char*        substitute    = NULL;
      int          substituteLen = 0;
      std::string  value;

      if (strcmp(varP, "id") == 0)
      {
        //
        // ${id} => substitute with ID of the entity
        //
        substitute = (char*) ce.entityId.id.c_str();
      }
      else if (strcmp(varP, "type") == 0)
      {
        //
        // ${type} => substitute with TYPE of the entity
        //
        substitute = (char*) ce.entityId.type.c_str();
      }
      else  // attribute
      {
        //
        // Neither ${id} nor ${type} => must be a name of an attribute ( ${ATTR_NAME} )
        // The value of the attribute ATTR_NAME is extracted by the function attributeValue
        // The variable 'substitute' is set to point to it
        //
        attributeValue(&value, ce.contextAttributeVector.vec, varP);
        substitute = (char*) value.c_str();
      }

      //
      // If substitution found, get its string length
      // If not found, substituteLen is ZERO already
      //
      if (substitute != NULL)
      {
        substituteLen = strlen(substitute);
      }

      //
      // Room enough?  If not, realloc
      //   toIx            is the current length of the 'to string'.
      //   substituteLen   is the length of the value to put in instead of ${XXX}
      //   toLen           is the TOTAL length of the 'to string'
      //
      // So, if toIx + substituteLen is bigger than THE TOTAL LENGTH of the 'to string'
      // we will have to get more memory before we insert the 'substitute' in the 'to string'
      // [ The '- 1' is for the zero-termination of the 'in string'.
      //
      if (toIx + substituteLen >= toLen - 1)
      {
        toP    = (char*) realloc(toP, toLen + CHUNK_SIZE);
        toLen += CHUNK_SIZE;

        if (toP == NULL)
        {
          LM_E(("Runtime Error (out of memory)"));
          *to = "";
          return;
        }
      }

      //
      // Now, if a substitute was found, append it to 'to string' and add the length of the
      // substitute to 'toIx', so that toP[toIx] point to the last non-used char of the 'to string'
      //
      // If no substitution was found (attribute with that name does not exist), then the ${XXX}
      // is substituted with NOTHING - it is silently consumed.
      //
      if (substitute != NULL)
      {
        // Concatenate 'substitute' to toP
        if (substitute != NULL)
        {
          strcat(&toP[toIx], substitute);
          toIx += substituteLen;
        }
      }
    }
    else
    {
      //
      // Room enough?  If not, realloc
      // See detaild description of the realloc step above
      //
      if (toIx >= toLen - 1)
      {
        toP    = (char*) realloc(toP, toLen + CHUNK_SIZE);
        toLen += CHUNK_SIZE;

        if (toP == NULL)
        {
          LM_E(("Runtime Error (out of memory)"));
          *to = "";
          return;
        }
      }

      //
      // Simply copy the char from 'from string' into 'to string'
      // and step over that char to prepare for the next one.
      //
      toP[toIx++] = fromP[0];
      ++fromP;
    }
  }

  //
  // Set the incoming std::string to the value of 'to string' and free the 'to string'
  //
  *to = toP;
  free(toP);
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "src/libfuzzer/libfuzzer_mutator.h"

#include <string.h>
#include <cassert>
#include <memory>
#include <string>

#include "port/protobuf.h"
#include "src/mutator.h"

#ifdef _MSC_VER

// see compiler-rt/lib/sanitizer-common/sanitizer_internal_defs.h
# if defined(_M_IX86) || defined(__i386__)
#  define WIN_SYM_PREFIX "_"
# else
#  define WIN_SYM_PREFIX
# endif

# define STRINGIFY_(A) #A
# define STRINGIFY(A) STRINGIFY_(A)

# define WIN_WEAK_ALIAS(Name, Default)                                         \
  __pragma(comment(linker, "/alternatename:" WIN_SYM_PREFIX STRINGIFY(Name) "="\
                                             WIN_SYM_PREFIX STRINGIFY(Default)))

WIN_WEAK_ALIAS(LLVMFuzzerMutate, LLVMFuzzerMutateDef)

extern "C" size_t LLVMFuzzerMutateDef(uint8_t*, size_t, size_t) { exit(1); }
extern "C" size_t LLVMFuzzerMutate(uint8_t*, size_t, size_t);

#else

extern "C" size_t LLVMFuzzerMutate(uint8_t*, size_t, size_t)
  __attribute__((weak));

#endif

namespace protobuf_mutator {
namespace libfuzzer {

namespace {

template <class T>
T MutateValue(T v) {
  size_t size =
      LLVMFuzzerMutate(reinterpret_cast<uint8_t*>(&v), sizeof(v), sizeof(v));
  memset(reinterpret_cast<uint8_t*>(&v) + size, 0, sizeof(v) - size);
  return v;
}

}  // namespace

int32_t Mutator::MutateInt32(int32_t value) { return MutateValue(value); }

int64_t Mutator::MutateInt64(int64_t value) { return MutateValue(value); }

uint32_t Mutator::MutateUInt32(uint32_t value) { return MutateValue(value); }

uint64_t Mutator::MutateUInt64(uint64_t value) { return MutateValue(value); }

float Mutator::MutateFloat(float value) { return MutateValue(value); }

double Mutator::MutateDouble(double value) { return MutateValue(value); }

std::string Mutator::MutateString(const std::string& value,
                                  size_t size_increase_hint) {
  // Randomly return empty strings as LLVMFuzzerMutate does not produce them.
  // Use uint16_t because on Windows, uniform_int_distribution does not support
  // any 8 bit types.
  if (!std::uniform_int_distribution<uint16_t>(0, 20)(*random())) return {};
  std::string result = value;
  result.resize(value.size() + size_increase_hint);
  if (result.empty()) result.push_back(0);
  result.resize(LLVMFuzzerMutate(reinterpret_cast<uint8_t*>(&result[0]),
                                 value.size(), result.size()));
  return result;
}

}  // namespace libfuzzer
}  // namespace protobuf_mutator
<commit_msg>Introduced SANITIZER_INTERFACE_WEAK_DEF<commit_after>// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "src/libfuzzer/libfuzzer_mutator.h"

#include <string.h>
#include <cassert>
#include <memory>
#include <string>

#include "port/protobuf.h"
#include "src/mutator.h"

// see compiler-rt/lib/sanitizer-common/sanitizer_internal_defs.h; usage of
// SANITIZER_INTERFACE_WEAK_DEF is the same with some functionality removed
#ifdef _MSC_VER
#if defined(_M_IX86) || defined(__i386__)
#define WIN_SYM_PREFIX "_"
#else
#define WIN_SYM_PREFIX
#endif

#define STRINGIFY_(A) #A
#define STRINGIFY(A) STRINGIFY_(A)

#define WEAK_DEFAULT_NAME(Name) Name##__def

// clang-format off
#define SANITIZER_INTERFACE_WEAK_DEF(ReturnType, Name, ...)   \
  __pragma(comment(linker, "/alternatename:"                  \
           WIN_SYM_PREFIX STRINGIFY(Name) "="                 \
           WIN_SYM_PREFIX STRINGIFY(WEAK_DEFAULT_NAME(Name))))\
  extern "C" ReturnType Name(__VA_ARGS__);                    \
  extern "C" ReturnType WEAK_DEFAULT_NAME(Name)(__VA_ARGS__)
// clang-format on
#else
#define SANITIZER_INTERFACE_WEAK_DEF(ReturnType, Name, ...) \
  extern "C" __attribute__((weak)) ReturnType Name(__VA_ARGS__)
#endif

SANITIZER_INTERFACE_WEAK_DEF(size_t, LLVMFuzzerMutate, uint8_t*, size_t,
                             size_t) {
  return 0;
}

namespace protobuf_mutator {
namespace libfuzzer {

namespace {

template <class T>
T MutateValue(T v) {
  size_t size =
      LLVMFuzzerMutate(reinterpret_cast<uint8_t*>(&v), sizeof(v), sizeof(v));
  memset(reinterpret_cast<uint8_t*>(&v) + size, 0, sizeof(v) - size);
  return v;
}

}  // namespace

int32_t Mutator::MutateInt32(int32_t value) { return MutateValue(value); }

int64_t Mutator::MutateInt64(int64_t value) { return MutateValue(value); }

uint32_t Mutator::MutateUInt32(uint32_t value) { return MutateValue(value); }

uint64_t Mutator::MutateUInt64(uint64_t value) { return MutateValue(value); }

float Mutator::MutateFloat(float value) { return MutateValue(value); }

double Mutator::MutateDouble(double value) { return MutateValue(value); }

std::string Mutator::MutateString(const std::string& value,
                                  size_t size_increase_hint) {
  // Randomly return empty strings as LLVMFuzzerMutate does not produce them.
  // Use uint16_t because on Windows, uniform_int_distribution does not support
  // any 8 bit types.
  if (!std::uniform_int_distribution<uint16_t>(0, 20)(*random())) return {};
  std::string result = value;
  result.resize(value.size() + size_increase_hint);
  if (result.empty()) result.push_back(0);
  result.resize(LLVMFuzzerMutate(reinterpret_cast<uint8_t*>(&result[0]),
                                 value.size(), result.size()));
  return result;
}

}  // namespace libfuzzer
}  // namespace protobuf_mutator
<|endoftext|>


<commit_before>/*
 * Copyright 2017 deepstreamHub GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <iostream>
#include <exception>

#include <deepstream.hpp>

int main(int argc, char *argv[])
{
    std::string uri = "ws://localhost:6020/deepstream";

    if (argc >= 2) {
	uri = argv[1];
    }

    try {
	deepstream::Client client(uri.c_str());

	if (client.login()) {
	    std::cout << "successfully logged in to " << uri << std::endl;
	    return EXIT_SUCCESS;
	} else {
	    std::cerr << "failed to login to " << uri << std::endl;
	    return EXIT_FAILURE;
	}
    } catch(std::exception& e) {
	std::cerr << "Caught exception \"" << e.what() << "\"" << std::endl;
	return EXIT_FAILURE;
    }
}
<commit_msg>Construct client URI with std::string<commit_after>/*
 * Copyright 2017 deepstreamHub GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <iostream>
#include <exception>

#include <deepstream.hpp>

int main(int argc, char *argv[])
{
    std::string uri = "ws://localhost:6020/deepstream";

    if (argc >= 2) {
	uri = argv[1];
    }

    try {
	deepstream::Client client(uri);

	if (client.login()) {
	    std::cout << "successfully logged in to " << uri << std::endl;
	    return EXIT_SUCCESS;
	} else {
	    std::cerr << "failed to login to " << uri << std::endl;
	    return EXIT_FAILURE;
	}
    } catch(std::exception& e) {
	std::cerr << "Caught exception \"" << e.what() << "\"" << std::endl;
	return EXIT_FAILURE;
    }
}
<|endoftext|>