<commit_before>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/system/table_index.hpp"

#include "vast/detail/overload.hpp"
#include "vast/expression_visitors.hpp"
#include "vast/load.hpp"
#include "vast/logger.hpp"
#include "vast/save.hpp"

namespace vast::system {

caf::expected<table_index> make_table_index(path base_dir,
                                                type event_type) {
  caf::error err;
  table_index result{event_type, base_dir};
  result.columns_.resize(table_index::meta_column_count
                         + flat_size(event_type));
  return result;
}
// -- constructors, destructors, and assignment operators ----------------------

table_index::~table_index() noexcept {
  // nop
}

// -- persistency --------------------------------------------------------------

caf::error table_index::flush_to_disk() {
  // Unless `add` was called at least once there's nothing to flush.
  if (!fully_initialized_)
    return caf::none;
  for (auto& col : columns_) {
    VAST_ASSERT(col != nullptr);
    auto err = col->flush_to_disk();
    if (err)
      return err;
  }
  return caf::none;
}

/// -- properties --------------------------------------------------------------

column_index& table_index::at(size_t column_index) {
  return *columns_[column_index];
}

column_index* table_index::by_name(std::string_view column_name) {
  // TODO: support string_view in path's operator /
  auto fname = base_dir_ / std::string{column_name};
  auto pred = [&](const column_index_ptr& ptr) {
    return ptr->filename() == fname;
  };
  auto i = std::find_if(columns_.begin(), columns_.end(), pred);
  return i != columns_.end() ? i->get() : nullptr;
}

caf::error table_index::add(const event& x) {
  if (fully_initialized_) {
    for (auto& col : columns_) {
      VAST_ASSERT(col != nullptr);
      col->add(x);
    }
    return caf::none;
  }
  auto fun = [&](column_index& col) -> caf::error {
    col.add(x);
    return caf::none;
  };
  auto mk_time = [&] { return make_time_index(meta_dir()); };
  auto mk_type = [&] { return make_type_index(meta_dir()); };
  return caf::error::eval(
    [&] {
      // Column 0 is our meta index for the time.
      return with_meta_column(0, mk_time, fun);
    },
    [&] {
      // Column 1 is our meta index for the event type.
      return with_meta_column(1, mk_type, fun);
    },
    [&]() -> caf::error {
      // Coluns 2-N are our data fields.
      auto r = get_if<record_type>(event_type_);
      if (!r) {
        auto fac = [&] {
          return make_flat_data_index(data_dir(), event_type_);
        };
        return with_data_column(0, fac, fun);
      }
      // Iterate all types of the record.
      size_t i = 0;
      for (auto&& f : record_type::each{*r}) {
        auto& value_type = f.trace.back()->type;
        if (!has_skip_attribute(event_type_)) {
          auto dir = data_dir();
          for (auto& k : f.key())
            dir /= k;
          auto fac = [&] {
            VAST_DEBUG("make field indexer at offset", f.offset, "with type",
                       value_type);
            return make_field_data_index(data_dir(), value_type, f.offset);
          };
          auto err = with_data_column(i++, fac, fun);
          if (err)
            return err;
        }
      }
      return caf::none;
    },
    [&]() -> caf::error {
      fully_initialized_ = true;
      return caf::none;
    });
}

path table_index::meta_dir() const {
  return base_dir_ / "meta";
}

path table_index::data_dir() const {
  return base_dir_ / "data";
}

caf::expected<bitmap> table_index::lookup(const predicate& pred) {
  // For now, we require that the predicate is part of a normalized expression,
  // i.e., LHS an extractor type and RHS of type data.
  auto rhs = get_if<data>(pred.rhs);
  if (!rhs)
    return ec::invalid_query;
  // Specialize the predicate for the type.
  auto resolved = type_resolver{event_type_}(pred);
  if (!resolved)
    return std::move(resolved.error());
  return lookup(*resolved);
}

caf::expected<bitmap> table_index::lookup(const expression& expr) {
  return visit(
    detail::overload(
      [&](const disjunction& dis) -> expected<bitmap> {
        bitmap result;
        for (auto& op : dis) {
          auto sub_result = lookup(op);
          if (!sub_result)
            return std::move(sub_result.error());
          result |= *sub_result;
        }
        return result;
      },
      [&](const predicate& p) {
        return visit(
          detail::overload(
            [&](const attribute_extractor& ex, const data& x) {
              return lookup(p, ex, x);
            },
            [&](const data_extractor& dx, const data& x) {
              return lookup(p, dx, x);
            },
            [&](const auto&, const auto&) -> expected<bitmap> {
              // Ignore unexpected lhs/rhs combinations.
              return bitmap{};
            }
          ),
          p.lhs, p.rhs);
      },
      [&](const auto&) -> expected<bitmap> {
        // Ignore unexpected expressions.
        return bitmap{};
      }
    ),
    expr);
}

caf::expected<bitmap> table_index::lookup(const predicate& pred,
                                          const attribute_extractor& ex,
                                          const data& x) {
  VAST_IGNORE_UNUSED(x);
  // We know that the columns vector contains two meta fields: time at index
  // 0 and type at index 1.
  static_assert(table_index::meta_column_count == 2);
  VAST_ASSERT(columns_.size() >= table_index::meta_column_count);
  if (ex.attr == "time") {
    VAST_ASSERT(is<timestamp>(x));
    auto fac = [&] { return make_time_index(meta_dir()); };
    return with_meta_column(0, fac, [&](column_index& col) {
      return col.lookup(pred);
    });
  } else if (ex.attr == "type") {
    VAST_ASSERT(is<std::string>(x));
    auto fac = [&] { return make_type_index(meta_dir()); };
    return with_meta_column(1, fac, [&](column_index& col) {
      return col.lookup(pred);
    });
  }
  VAST_WARNING("unsupported attribute:", ex.attr);
  return ec::invalid_query;
}

caf::expected<bitmap> table_index::lookup(const predicate& pred,
                                          const data_extractor& dx,
                                          const data& x) {
  VAST_IGNORE_UNUSED(x);
  if (dx.offset.empty()) {
    VAST_ASSERT(num_data_columns() == 1);
    auto fac = [&] { return make_flat_data_index(data_dir(), event_type_); };
    return with_data_column(0, fac, [&](column_index& col) {
      return col.lookup(pred);
    });
  } else {
    auto r = get<record_type>(dx.type);
    auto k = r.resolve(dx.offset);
    VAST_ASSERT(k);
    auto t = r.at(dx.offset);
    VAST_ASSERT(t);
    auto fac = [&] {
      auto p = data_dir();
      for (auto& x : *k)
        p /= x;
      return make_field_data_index(p, *t, dx.offset);
    };
    auto index = r.flat_index_at(dx.offset);
    if (!index) {
      VAST_DEBUG("invalid offset for record type", dx.type);
      return bitmap{};
    }
    return with_data_column(*index, fac, [&](column_index& col) {
      return col.lookup(pred);
    });
  }
  return bitmap{};
}

// -- constructors, destructors, and assignment operators ----------------------

table_index::table_index(type event_type, path base_dir)
  : event_type_(std::move(event_type)),
    base_dir_(std::move(base_dir)),
    fully_initialized_(false) {
  // nop
}

} // namespace vast::system
<commit_msg>Fix column base directory in factory<commit_after>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/system/table_index.hpp"

#include "vast/detail/overload.hpp"
#include "vast/expression_visitors.hpp"
#include "vast/load.hpp"
#include "vast/logger.hpp"
#include "vast/save.hpp"

namespace vast::system {

caf::expected<table_index> make_table_index(path base_dir,
                                                type event_type) {
  caf::error err;
  table_index result{event_type, base_dir};
  result.columns_.resize(table_index::meta_column_count
                         + flat_size(event_type));
  return result;
}
// -- constructors, destructors, and assignment operators ----------------------

table_index::~table_index() noexcept {
  // nop
}

// -- persistency --------------------------------------------------------------

caf::error table_index::flush_to_disk() {
  // Unless `add` was called at least once there's nothing to flush.
  if (!fully_initialized_)
    return caf::none;
  for (auto& col : columns_) {
    VAST_ASSERT(col != nullptr);
    auto err = col->flush_to_disk();
    if (err)
      return err;
  }
  return caf::none;
}

/// -- properties --------------------------------------------------------------

column_index& table_index::at(size_t column_index) {
  return *columns_[column_index];
}

column_index* table_index::by_name(std::string_view column_name) {
  // TODO: support string_view in path's operator /
  auto fname = base_dir_ / std::string{column_name};
  auto pred = [&](const column_index_ptr& ptr) {
    return ptr->filename() == fname;
  };
  auto i = std::find_if(columns_.begin(), columns_.end(), pred);
  return i != columns_.end() ? i->get() : nullptr;
}

caf::error table_index::add(const event& x) {
  if (fully_initialized_) {
    for (auto& col : columns_) {
      VAST_ASSERT(col != nullptr);
      col->add(x);
    }
    return caf::none;
  }
  auto fun = [&](column_index& col) -> caf::error {
    col.add(x);
    return caf::none;
  };
  auto mk_time = [&] { return make_time_index(meta_dir()); };
  auto mk_type = [&] { return make_type_index(meta_dir()); };
  return caf::error::eval(
    [&] {
      // Column 0 is our meta index for the time.
      return with_meta_column(0, mk_time, fun);
    },
    [&] {
      // Column 1 is our meta index for the event type.
      return with_meta_column(1, mk_type, fun);
    },
    [&]() -> caf::error {
      // Coluns 2-N are our data fields.
      auto r = get_if<record_type>(event_type_);
      if (!r) {
        auto fac = [&] {
          return make_flat_data_index(data_dir(), event_type_);
        };
        return with_data_column(0, fac, fun);
      }
      // Iterate all types of the record.
      size_t i = 0;
      for (auto&& f : record_type::each{*r}) {
        auto& value_type = f.trace.back()->type;
        if (!has_skip_attribute(event_type_)) {
          auto dir = data_dir();
          for (auto& k : f.key())
            dir /= k;
          auto fac = [&] {
            VAST_DEBUG("make field indexer at offset", f.offset, "with type",
                       value_type);
            return make_field_data_index(dir, value_type, f.offset);
          };
          auto err = with_data_column(i++, fac, fun);
          if (err)
            return err;
        }
      }
      return caf::none;
    },
    [&]() -> caf::error {
      fully_initialized_ = true;
      return caf::none;
    });
}

path table_index::meta_dir() const {
  return base_dir_ / "meta";
}

path table_index::data_dir() const {
  return base_dir_ / "data";
}

caf::expected<bitmap> table_index::lookup(const predicate& pred) {
  // For now, we require that the predicate is part of a normalized expression,
  // i.e., LHS an extractor type and RHS of type data.
  auto rhs = get_if<data>(pred.rhs);
  if (!rhs)
    return ec::invalid_query;
  // Specialize the predicate for the type.
  auto resolved = type_resolver{event_type_}(pred);
  if (!resolved)
    return std::move(resolved.error());
  return lookup(*resolved);
}

caf::expected<bitmap> table_index::lookup(const expression& expr) {
  return visit(
    detail::overload(
      [&](const disjunction& dis) -> expected<bitmap> {
        bitmap result;
        for (auto& op : dis) {
          auto sub_result = lookup(op);
          if (!sub_result)
            return std::move(sub_result.error());
          result |= *sub_result;
        }
        return result;
      },
      [&](const predicate& p) {
        return visit(
          detail::overload(
            [&](const attribute_extractor& ex, const data& x) {
              return lookup(p, ex, x);
            },
            [&](const data_extractor& dx, const data& x) {
              return lookup(p, dx, x);
            },
            [&](const auto&, const auto&) -> expected<bitmap> {
              // Ignore unexpected lhs/rhs combinations.
              return bitmap{};
            }
          ),
          p.lhs, p.rhs);
      },
      [&](const auto&) -> expected<bitmap> {
        // Ignore unexpected expressions.
        return bitmap{};
      }
    ),
    expr);
}

caf::expected<bitmap> table_index::lookup(const predicate& pred,
                                          const attribute_extractor& ex,
                                          const data& x) {
  VAST_IGNORE_UNUSED(x);
  // We know that the columns vector contains two meta fields: time at index
  // 0 and type at index 1.
  static_assert(table_index::meta_column_count == 2);
  VAST_ASSERT(columns_.size() >= table_index::meta_column_count);
  if (ex.attr == "time") {
    VAST_ASSERT(is<timestamp>(x));
    auto fac = [&] { return make_time_index(meta_dir()); };
    return with_meta_column(0, fac, [&](column_index& col) {
      return col.lookup(pred);
    });
  } else if (ex.attr == "type") {
    VAST_ASSERT(is<std::string>(x));
    auto fac = [&] { return make_type_index(meta_dir()); };
    return with_meta_column(1, fac, [&](column_index& col) {
      return col.lookup(pred);
    });
  }
  VAST_WARNING("unsupported attribute:", ex.attr);
  return ec::invalid_query;
}

caf::expected<bitmap> table_index::lookup(const predicate& pred,
                                          const data_extractor& dx,
                                          const data& x) {
  VAST_IGNORE_UNUSED(x);
  if (dx.offset.empty()) {
    VAST_ASSERT(num_data_columns() == 1);
    auto fac = [&] { return make_flat_data_index(data_dir(), event_type_); };
    return with_data_column(0, fac, [&](column_index& col) {
      return col.lookup(pred);
    });
  } else {
    auto r = get<record_type>(dx.type);
    auto k = r.resolve(dx.offset);
    VAST_ASSERT(k);
    auto t = r.at(dx.offset);
    VAST_ASSERT(t);
    auto fac = [&] {
      auto p = data_dir();
      for (auto& x : *k)
        p /= x;
      return make_field_data_index(p, *t, dx.offset);
    };
    auto index = r.flat_index_at(dx.offset);
    if (!index) {
      VAST_DEBUG("invalid offset for record type", dx.type);
      return bitmap{};
    }
    return with_data_column(*index, fac, [&](column_index& col) {
      return col.lookup(pred);
    });
  }
  return bitmap{};
}

// -- constructors, destructors, and assignment operators ----------------------

table_index::table_index(type event_type, path base_dir)
  : event_type_(std::move(event_type)),
    base_dir_(std::move(base_dir)),
    fully_initialized_(false) {
  // nop
}

} // namespace vast::system
<|endoftext|>


<commit_before>/*
* Copyright (C) 2007-2011 German Aerospace Center (DLR/SC)
*
* Created: 2010-08-13 Markus Litz <Markus.Litz@dlr.de>
* Changed: $Id$
*
* Version: $Revision$
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @file
* @brief  Implementation of CPACS fuselages handling routines.
*/

#include "CCPACSFuselages.h"
#include "CCPACSFuselage.h"
#include "CTiglError.h"
#include <iostream>
#include <sstream>

namespace tigl {

	// Constructor
	CCPACSFuselages::CCPACSFuselages(CCPACSConfiguration* config)
		: configuration(config)
	{
		Cleanup();
	}

	// Destructor
	CCPACSFuselages::~CCPACSFuselages(void)
	{
		Cleanup();
	}

	// Invalidates internal state
	void CCPACSFuselages::Invalidate(void)
	{
		profiles.Invalidate();
		for (int i = 1; i <= GetFuselageCount(); i++)
		{
			GetFuselage(i).Invalidate();
		}
	}

	// Cleanup routine
	void CCPACSFuselages::Cleanup(void)
	{
		for (CCPACSFuselageContainer::size_type i = 0; i < fuselages.size(); i++)
        {
			delete fuselages[i];
		}
		fuselages.clear();
	}

	// Read CPACS fuselages element
	void CCPACSFuselages::ReadCPACS(TixiDocumentHandle tixiHandle, char* configurationUID)
	{
		Cleanup();
		char *fuselagesXPathPrt = NULL;
		char *tmpString = NULL;

		if (tixiUIDGetXPath(tixiHandle, configurationUID, &tmpString) != SUCCESS)
            throw CTiglError("XML error: tixiUIDGetXPath failed in CCPACSFuselages::ReadCPACS", TIGL_XML_ERROR);

		fuselagesXPathPrt = (char *) malloc(sizeof(char) * (strlen(tmpString) + 50));
		strcpy(fuselagesXPathPrt, tmpString);
		strcat(fuselagesXPathPrt, "[@uID=\"");
		strcat(fuselagesXPathPrt, configurationUID);
		strcat(fuselagesXPathPrt, "\"]/fuselages");

		// Read fuselage profiles
		profiles.ReadCPACS(tixiHandle);

        if (tixiCheckElement(tixiHandle, fuselagesXPathPrt) != SUCCESS)
            return;

        /* Get fuselage element count */
		int fuselageCount;
		if (tixiGetNamedChildrenCount(tixiHandle, fuselagesXPathPrt, "fuselage", &fuselageCount) != SUCCESS)
			throw CTiglError("XML error: tixiGetNamedChildrenCount failed in CCPACSFuselages::ReadCPACS", TIGL_XML_ERROR);

		// Loop over all fuselages
		for (int i = 1; i <= fuselageCount; i++) {
			CCPACSFuselage* fuselage = new CCPACSFuselage(configuration);
			fuselages.push_back(fuselage);

			std::ostringstream xpath;
			xpath << fuselagesXPathPrt << "/fuselage[" << i << "]";
			fuselage->ReadCPACS(tixiHandle, xpath.str());
		}
		free(fuselagesXPathPrt);
	}

	// Returns the total count of fuselage profiles in this configuration
	int CCPACSFuselages::GetProfileCount(void) const
	{
		return profiles.GetProfileCount();
	}

	// Returns the fuselage profile for a given index.
	CCPACSFuselageProfile& CCPACSFuselages::GetProfile(int index) const
	{
		return profiles.GetProfile(index);
	}

	// Returns the fuselage profile for a given uid.
	CCPACSFuselageProfile& CCPACSFuselages::GetProfile(std::string uid) const
	{
		return profiles.GetProfile(uid);
	}

	// Returns the total count of fuselages in a configuration
	int CCPACSFuselages::GetFuselageCount(void) const
	{
		return (static_cast<int>(fuselages.size()));
	}

	// Returns the fuselage for a given index.
	CCPACSFuselage& CCPACSFuselages::GetFuselage(int index) const
	{
        index--;
		if (index < 0 || index >= GetFuselageCount())
			throw CTiglError("Error: Invalid index in CCPACSFuselages::GetFuselage", TIGL_INDEX_ERROR);
		return (*fuselages[index]);
	}

	// Returns the fuselage for a given index.
    CCPACSFuselage& CCPACSFuselages::GetFuselage(std::string UID) const
    {
        for(int i=0; i < GetFuselageCount(); i++)
        {
            const std::string tmpUID((*fuselages[i]).GetUIDPtr());
            if(tmpUID == UID) {
                return (*fuselages[i]);
            }
        }

        // UID not there
            throw CTiglError("Error: Invalid index in CCPACSFuselages::GetFuselage", TIGL_INDEX_ERROR);
    }

} // end namespace tigl
<commit_msg>Fixed memory leak in CCPACSFuselages<commit_after>/*
* Copyright (C) 2007-2011 German Aerospace Center (DLR/SC)
*
* Created: 2010-08-13 Markus Litz <Markus.Litz@dlr.de>
* Changed: $Id$
*
* Version: $Revision$
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @file
* @brief  Implementation of CPACS fuselages handling routines.
*/

#include "CCPACSFuselages.h"
#include "CCPACSFuselage.h"
#include "CTiglError.h"
#include <iostream>
#include <sstream>

namespace tigl {

	// Constructor
	CCPACSFuselages::CCPACSFuselages(CCPACSConfiguration* config)
		: configuration(config)
	{
		Cleanup();
	}

	// Destructor
	CCPACSFuselages::~CCPACSFuselages(void)
	{
		Cleanup();
	}

	// Invalidates internal state
	void CCPACSFuselages::Invalidate(void)
	{
		profiles.Invalidate();
		for (int i = 1; i <= GetFuselageCount(); i++)
		{
			GetFuselage(i).Invalidate();
		}
	}

	// Cleanup routine
	void CCPACSFuselages::Cleanup(void)
	{
		for (CCPACSFuselageContainer::size_type i = 0; i < fuselages.size(); i++)
        {
			delete fuselages[i];
		}
		fuselages.clear();
	}

	// Read CPACS fuselages element
	void CCPACSFuselages::ReadCPACS(TixiDocumentHandle tixiHandle, char* configurationUID)
	{
		Cleanup();
		char *fuselagesXPathPrt = NULL;
		char *tmpString = NULL;

		if (tixiUIDGetXPath(tixiHandle, configurationUID, &tmpString) != SUCCESS)
            throw CTiglError("XML error: tixiUIDGetXPath failed in CCPACSFuselages::ReadCPACS", TIGL_XML_ERROR);

		fuselagesXPathPrt = (char *) malloc(sizeof(char) * (strlen(tmpString) + 50));
		strcpy(fuselagesXPathPrt, tmpString);
		strcat(fuselagesXPathPrt, "[@uID=\"");
		strcat(fuselagesXPathPrt, configurationUID);
		strcat(fuselagesXPathPrt, "\"]/fuselages");

		// Read fuselage profiles
		profiles.ReadCPACS(tixiHandle);

		if (tixiCheckElement(tixiHandle, fuselagesXPathPrt) != SUCCESS){
			free(fuselagesXPathPrt);
			return;
		}
		/* Get fuselage element count */
		int fuselageCount;
		if (tixiGetNamedChildrenCount(tixiHandle, fuselagesXPathPrt, "fuselage", &fuselageCount) != SUCCESS)
			throw CTiglError("XML error: tixiGetNamedChildrenCount failed in CCPACSFuselages::ReadCPACS", TIGL_XML_ERROR);

		// Loop over all fuselages
		for (int i = 1; i <= fuselageCount; i++) {
			CCPACSFuselage* fuselage = new CCPACSFuselage(configuration);
			fuselages.push_back(fuselage);

			std::ostringstream xpath;
			xpath << fuselagesXPathPrt << "/fuselage[" << i << "]";
			fuselage->ReadCPACS(tixiHandle, xpath.str());
		}
		free(fuselagesXPathPrt);
	}

	// Returns the total count of fuselage profiles in this configuration
	int CCPACSFuselages::GetProfileCount(void) const
	{
		return profiles.GetProfileCount();
	}

	// Returns the fuselage profile for a given index.
	CCPACSFuselageProfile& CCPACSFuselages::GetProfile(int index) const
	{
		return profiles.GetProfile(index);
	}

	// Returns the fuselage profile for a given uid.
	CCPACSFuselageProfile& CCPACSFuselages::GetProfile(std::string uid) const
	{
		return profiles.GetProfile(uid);
	}

	// Returns the total count of fuselages in a configuration
	int CCPACSFuselages::GetFuselageCount(void) const
	{
		return (static_cast<int>(fuselages.size()));
	}

	// Returns the fuselage for a given index.
	CCPACSFuselage& CCPACSFuselages::GetFuselage(int index) const
	{
        index--;
		if (index < 0 || index >= GetFuselageCount())
			throw CTiglError("Error: Invalid index in CCPACSFuselages::GetFuselage", TIGL_INDEX_ERROR);
		return (*fuselages[index]);
	}

	// Returns the fuselage for a given index.
    CCPACSFuselage& CCPACSFuselages::GetFuselage(std::string UID) const
    {
        for(int i=0; i < GetFuselageCount(); i++)
        {
            const std::string tmpUID((*fuselages[i]).GetUIDPtr());
            if(tmpUID == UID) {
                return (*fuselages[i]);
            }
        }

        // UID not there
            throw CTiglError("Error: Invalid index in CCPACSFuselages::GetFuselage", TIGL_INDEX_ERROR);
    }

} // end namespace tigl
<|endoftext|>


<commit_before>#include <AlpinoCorpus/DbCorpusWriter.hh>
#include <AlpinoCorpus/Error.hh>
#include <QtGlobal>

namespace db = DbXml;

namespace alpinocorpus {
    DbCorpusWriter::DbCorpusWriter(QString const &qpath, bool overwrite)
     : mgr(), container()
    {
        try {
            db::XmlContainerConfig config;
            config.setReadOnly(false);

            std::string path(qpath.toLocal8Bit().data());

            if (overwrite)
                container = mgr.createContainer(path, config);
            else
                container = mgr.openContainer(path, config);
        } catch (XmlException const &e) {
            throw OpenError(e.what());
        }
    }

    db::XmlUpdateContext &DbCorpusWriter::mkUpdateContext(
                                            db::XmlUpdateContext &ctx)
    {
        // Note: this function may be unnecessary; no exceptions are listed
        // for XmlManager::createUpdateContext() or the copy constructor,
        // but the don't have no-throw guarantee either.
        try {
            return ctx = mgr.createUpdateContext();
        } catch (db::XmlException const &e) {
            std::ostringstream msg;
            msg << "cannot create XML database update context \"" << name
                << "\": " << e.what();
            throw Error(msg.str());
        }
    }

    void DbCorpusWriter::write(QString const &name, QString const &content)
    {
        write(name, content, mkUpdateContext());
    }

    void DbCorpusWriter::write(CorpusReader const &corpus)
    {
        QVector<QString> entry(corpus.entries);
        db::XmlUpdateContext ctx(mkUpdateContext());
        for (size_t i=0; i<entry.size(); i++)
            write(entry[i], corpus.read(entry[i]), ctx);
    }

    void DbCorpusWriter::write(QString const &name, QString const &content,
                               XmlUpdateContext &ctx)
    {
        try {
            std::string canonical(name.fromNativeSeparators().toUtf8().data());
            container.putDocument(name, content, ctx, db::WELL_FORMED_ONLY);
        } catch (XmlException const &e) {
            std::ostringstream msg;
            msg << "cannot write document \"" << name
                << "\": " << e.what();
            throw Error(msg.str());
        }
    }
}
<commit_msg>Use QDir properly<commit_after>#include <AlpinoCorpus/DbCorpusWriter.hh>
#include <AlpinoCorpus/Error.hh>
#include <QDir>

namespace db = DbXml;

namespace alpinocorpus {
    DbCorpusWriter::DbCorpusWriter(QString const &qpath, bool overwrite)
     : mgr(), container()
    {
        try {
            db::XmlContainerConfig config;
            config.setReadOnly(false);

            std::string path(qpath.toLocal8Bit().data());

            if (overwrite)
                container = mgr.createContainer(path, config);
            else
                container = mgr.openContainer(path, config);
        } catch (XmlException const &e) {
            throw OpenError(e.what());
        }
    }

    db::XmlUpdateContext &DbCorpusWriter::mkUpdateContext(
                                            db::XmlUpdateContext &ctx)
    {
        // Note: this function may be unnecessary; no exceptions are listed
        // for XmlManager::createUpdateContext() or the copy constructor,
        // but they don't have a no-throw guarantee either.
        try {
            return ctx = mgr.createUpdateContext();
        } catch (db::XmlException const &e) {
            std::ostringstream msg;
            msg << "cannot create XML database update context \"" << name
                << "\": " << e.what();
            throw Error(msg.str());
        }
    }

    void DbCorpusWriter::write(QString const &name, QString const &content)
    {
        write(name, content, mkUpdateContext());
    }

    void DbCorpusWriter::write(CorpusReader const &corpus)
    {
        QVector<QString> entry(corpus.entries);
        db::XmlUpdateContext ctx(mkUpdateContext());
        for (size_t i=0; i<entry.size(); i++)
            write(entry[i], corpus.read(entry[i]), ctx);
    }

    void DbCorpusWriter::write(QString const &name, QString const &content,
                               XmlUpdateContext &ctx)
    {
        try {
            std::string canonical(QDir::fromNativeSeparators(name)
                                  .toUtf8()
                                  .data());
            container.putDocument(name, content, ctx, db::WELL_FORMED_ONLY);
        } catch (XmlException const &e) {
            std::ostringstream msg;
            msg << "cannot write document \"" << name
                << "\": " << e.what();
            throw Error(msg.str());
        }
    }
}
<|endoftext|>


<commit_before>/* DepotsSettings.cpp
 * Copyright 2016 Brian Hill
 * All rights reserved. Distributed under the terms of the BSD License.
 */
#include "constants.h"
#include "DepotsSettings.h"

#include <FindDirectory.h>
#include <StringList.h>

#undef B_TRANSLATION_CONTEXT
#define B_TRANSLATION_CONTEXT "DepotsSettings"

const char* settingsFilename = "Depots_settings";

DepotsSettings::DepotsSettings()
{
	status_t status = find_directory(B_USER_SETTINGS_DIRECTORY, &fFilePath);
	if (status == B_OK) {
		status = fFilePath.Append(settingsFilename);
	}
	fInitStatus = status;
}

BRect
DepotsSettings::GetFrame()
{
	BMessage settings(_ReadFromFile());
	BRect frame;
	settings.FindRect(key_frame, &frame);
	return frame;
}

void
DepotsSettings::SetFrame(BRect frame)
{
	BMessage settings(_ReadFromFile());
	settings.RemoveData(key_frame);
	settings.AddRect(key_frame, frame);
	_SaveToFile(settings);
}

status_t
DepotsSettings::GetRepositories(int32 &repoCount, BStringList &nameList, BStringList &urlList)
{
	BMessage settings(_ReadFromFile());
	type_code type;
	int32 count;
	settings.GetInfo(key_name, &type, &count);

	status_t result = B_OK;
	int32 index, total=0;
	BString foundName, foundUrl;
	// get each repository and add to lists
	for(index = 0; index < count; index++)
	{
		status_t result1 = settings.FindString(key_name, index, &foundName);
		status_t result2 = settings.FindString(key_url, index, &foundUrl);
		if(result1 == B_OK && result2 == B_OK)
		{
			nameList.Add(foundName);
			urlList.Add(foundUrl);
			total++;
		}
		else
			result = B_ERROR;
	}
	repoCount = total;
	return result;
}

void
DepotsSettings::SetRepositories(BStringList &nameList, BStringList &urlList)
{
	BMessage settings(_ReadFromFile());
	settings.RemoveName(key_name);
	settings.RemoveName(key_url);

	int32 index, count = nameList.CountStrings();
	for(index=0; index < count; index++)
	{
		settings.AddString(key_name, nameList.StringAt(index));
		settings.AddString(key_url, urlList.StringAt(index));
	}
	_SaveToFile(settings);
}

BMessage
DepotsSettings::_ReadFromFile()
{
	BMessage settings;
	status_t status = fFile.SetTo(fFilePath.Path(), B_READ_ONLY);
	if (status == B_OK)
		status = settings.Unflatten(&fFile);
	fFile.Unset();
	return settings;
}

status_t
DepotsSettings::_SaveToFile(BMessage settings)
{
	status_t status = fFile.SetTo(fFilePath.Path(), B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE);
	if (status == B_OK)
		status = settings.Flatten(&fFile);
	fFile.Unset();
	return status;
}
<commit_msg>Set default window frame when no settings found<commit_after>/* DepotsSettings.cpp
 * Copyright 2016 Brian Hill
 * All rights reserved. Distributed under the terms of the BSD License.
 */
#include "constants.h"
#include "DepotsSettings.h"

#include <FindDirectory.h>
#include <StringList.h>

#undef B_TRANSLATION_CONTEXT
#define B_TRANSLATION_CONTEXT "DepotsSettings"

const char* settingsFilename = "Depots_settings";

DepotsSettings::DepotsSettings()
{
	status_t status = find_directory(B_USER_SETTINGS_DIRECTORY, &fFilePath);
	if (status == B_OK) {
		status = fFilePath.Append(settingsFilename);
	}
	fInitStatus = status;
}

BRect
DepotsSettings::GetFrame()
{
	BMessage settings(_ReadFromFile());
	BRect frame;
	status_t status = settings.FindRect(key_frame, &frame);
	// Set deafault off screen so it will center itself
	if(status != B_OK)
		frame.Set(-10, -10, 750, 300);
	return frame;
}

void
DepotsSettings::SetFrame(BRect frame)
{
	BMessage settings(_ReadFromFile());
	settings.RemoveData(key_frame);
	settings.AddRect(key_frame, frame);
	_SaveToFile(settings);
}

status_t
DepotsSettings::GetRepositories(int32 &repoCount, BStringList &nameList, BStringList &urlList)
{
	BMessage settings(_ReadFromFile());
	type_code type;
	int32 count;
	settings.GetInfo(key_name, &type, &count);

	status_t result = B_OK;
	int32 index, total=0;
	BString foundName, foundUrl;
	// get each repository and add to lists
	for(index = 0; index < count; index++)
	{
		status_t result1 = settings.FindString(key_name, index, &foundName);
		status_t result2 = settings.FindString(key_url, index, &foundUrl);
		if(result1 == B_OK && result2 == B_OK)
		{
			nameList.Add(foundName);
			urlList.Add(foundUrl);
			total++;
		}
		else
			result = B_ERROR;
	}
	repoCount = total;
	return result;
}

void
DepotsSettings::SetRepositories(BStringList &nameList, BStringList &urlList)
{
	BMessage settings(_ReadFromFile());
	settings.RemoveName(key_name);
	settings.RemoveName(key_url);

	int32 index, count = nameList.CountStrings();
	for(index=0; index < count; index++)
	{
		settings.AddString(key_name, nameList.StringAt(index));
		settings.AddString(key_url, urlList.StringAt(index));
	}
	_SaveToFile(settings);
}

BMessage
DepotsSettings::_ReadFromFile()
{
	BMessage settings;
	status_t status = fFile.SetTo(fFilePath.Path(), B_READ_ONLY);
	if (status == B_OK)
		status = settings.Unflatten(&fFile);
	fFile.Unset();
	return settings;
}

status_t
DepotsSettings::_SaveToFile(BMessage settings)
{
	status_t status = fFile.SetTo(fFilePath.Path(), B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE);
	if (status == B_OK)
		status = settings.Flatten(&fFile);
	fFile.Unset();
	return status;
}
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  $RCSfile: flags.hxx,v $
 *
 *  $Revision: 1.2 $
 *
 *  last change: $Author: jb $ $Date: 2002-02-15 14:34:33 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2002 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/

#ifndef INCLUDED_DATA_FLAGS_HXX
#define INCLUDED_DATA_FLAGS_HXX

#ifndef _SAL_TYPES_H_
#include <sal/types.h>
#endif

namespace configmgr
{
//-----------------------------------------------------------------------------
    namespace data
    {
    //-----------------------------------------------------------------------------
    // node type + value type selector - fits into a byte (bit 5 currently unassigned)
        namespace Type
        {
            enum Type
            {
            // Primitive Data Type is stored in bits 0-4 (bit 3 currently unused)
                // base data types
                value_any,       // = 0
                value_string,    // = 1
                value_boolean,   // = 2
                value_short,     // = 3
                value_int,       // = 4
                value_long,      // = 5
                value_double,    // = 6
                value_binary,    // = 7

                mask_basetype = 0x0F,

                // sequence data types
                flag_sequence = 0x10,

                value_invalid        = value_any | flag_sequence,

                value_string_list    = value_string   | flag_sequence,
                value_boolean_list   = value_boolean  | flag_sequence,
                value_short_list     = value_short    | flag_sequence,
                value_int_list       = value_int      | flag_sequence,
                value_long_list      = value_long     | flag_sequence,
                value_double_list    = value_double   | flag_sequence,
                value_binary_list    = value_binary   | flag_sequence,

                mask_valuetype = mask_basetype | flag_sequence, // 0x1F

            // Node Type is stored in bits 6+7
                nodetype_shift_ = 6,
                mask_nodetype  = 0x03 << nodetype_shift_, // 0xC0

                // maybe we should use 0 for node type 'value'
                // to make the complete type be the valuetype ?
                nodetype_invalid= 0x00,

                nodetype_value  = 0x01 << nodetype_shift_, // 0x40
                nodetype_group  = 0x02 << nodetype_shift_, // 0x80
                nodetype_set    = 0x03 << nodetype_shift_, // 0xC0

                flag_innernode = 0x02 << nodetype_shift_, // 0x80
                flag_setnode   = 0x01 << nodetype_shift_  // 0x40
            };
            typedef sal_uInt8 Field;
        }

    //-----------------------------------------------------------------------------
    // node attributes
        namespace Flags
        {
            enum Type
            {
                readonly    = 0x01,
                finalized   = 0x02,

                nullable    = 0x04,
                localized   = 0x08,

                valueAvailable   = 0x10, // only used for value nodes
                defaultAvailable = 0x20, // only used for value nodes

                defaulted   = 0x40, // somewhat redundant with State
                defaultable = 0x80  // redundant with State (merged || defaulted)
            };
            typedef sal_uInt8 Field;
        }
    //-----------------------------------------------------------------------------
        // set element or tree state
        namespace State
        {
            enum Type
            {
                merged,     // = 0, - result of merging defaults + changes
                defaulted,  // = 1, - result of copying defaults unchanged
                replaced,   // = 2, - result of copying new layer unchanged
                added,      // = 3, - same as 'replaced', but it is known,
                                  //        that there is no corresponding default

                mask_state = 0x0F, // leaves bits 3+4 reserved for states

                flag_mandatory      = 0x10, // marks this element as non-removable/replacable
                flag_readonly       = 0x20, // marks this element as read-only
                flag_default_avail  = 0x40  // marks this element as containing default values
            };
            typedef sal_uInt8 Field;
        }
    //-----------------------------------------------------------------------------
    }
//-----------------------------------------------------------------------------
}

#endif // INCLUDED_DATA_FLAGS_HXX
<commit_msg>INTEGRATION: CWS cfg01 (1.2.24); FILE MERGED 2003/02/28 16:33:15 ssmith 1.2.24.1: #107403# #107403# adding support for mandatory flag and changing dynamic properties semantics<commit_after>/*************************************************************************
 *
 *  $RCSfile: flags.hxx,v $
 *
 *  $Revision: 1.3 $
 *
 *  last change: $Author: vg $ $Date: 2003-04-01 13:32:49 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2002 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/

#ifndef INCLUDED_DATA_FLAGS_HXX
#define INCLUDED_DATA_FLAGS_HXX

#ifndef _SAL_TYPES_H_
#include <sal/types.h>
#endif

namespace configmgr
{
//-----------------------------------------------------------------------------
    namespace data
    {
    //-----------------------------------------------------------------------------
    // node type + value type selector - fits into a byte (bit 5 currently unassigned)
        namespace Type
        {
            enum Type
            {
            // Primitive Data Type is stored in bits 0-4 (bit 3 currently unused)
                // base data types
                value_any,       // = 0
                value_string,    // = 1
                value_boolean,   // = 2
                value_short,     // = 3
                value_int,       // = 4
                value_long,      // = 5
                value_double,    // = 6
                value_binary,    // = 7

                mask_basetype = 0x0F,

                // sequence data types
                flag_sequence = 0x10,

                value_invalid        = value_any | flag_sequence,

                value_string_list    = value_string   | flag_sequence,
                value_boolean_list   = value_boolean  | flag_sequence,
                value_short_list     = value_short    | flag_sequence,
                value_int_list       = value_int      | flag_sequence,
                value_long_list      = value_long     | flag_sequence,
                value_double_list    = value_double   | flag_sequence,
                value_binary_list    = value_binary   | flag_sequence,

                mask_valuetype = mask_basetype | flag_sequence, // 0x1F

            // Node Type is stored in bits 6+7
                nodetype_shift_ = 6,
                mask_nodetype  = 0x03 << nodetype_shift_, // 0xC0

                // maybe we should use 0 for node type 'value'
                // to make the complete type be the valuetype ?
                nodetype_invalid= 0x00,

                nodetype_value  = 0x01 << nodetype_shift_, // 0x40
                nodetype_group  = 0x02 << nodetype_shift_, // 0x80
                nodetype_set    = 0x03 << nodetype_shift_, // 0xC0

                flag_innernode = 0x02 << nodetype_shift_, // 0x80
                flag_setnode   = 0x01 << nodetype_shift_  // 0x40
            };
            typedef sal_uInt8 Field;
        }

    //-----------------------------------------------------------------------------
    // node attributes
        namespace Flags
        {
            enum Type
            {
                readonly    = 0x01,
                finalized   = 0x02,

                nullable    = 0x04,
                localized   = 0x08,

                valueAvailable   = 0x10, // only used for value nodes
                defaultAvailable = 0x20, // only used for value nodes

                defaulted   = 0x40, // somewhat redundant with State
                defaultable = 0x80  // redundant with State (merged || defaulted)
            };
            typedef sal_uInt8 Field;
        }
    //-----------------------------------------------------------------------------
        // set element or tree state
        namespace State
        {
            enum Type
            {
                merged,     // = 0, - result of merging defaults + changes
                defaulted,  // = 1, - result of copying defaults unchanged
                replaced,   // = 2, - result of copying new layer unchanged
                added,      // = 3, - same as 'replaced', but it is known,
                                  //        that there is no corresponding default

                mask_state = 0x0F, // leaves bits 3+4 reserved for states

                flag_mandatory      = 0x10, // marks this element as non-removable/replacable
                flag_readonly       = 0x20, // marks this element as read-only
                flag_default_avail  = 0x40, // marks this element as containing default values
                flag_removable      = 0x80  // marks this element as removable
            };
            typedef sal_uInt8 Field;
        }
    //-----------------------------------------------------------------------------
    }
//-----------------------------------------------------------------------------
}

#endif // INCLUDED_DATA_FLAGS_HXX
<|endoftext|>


<commit_before>#include "InterpolatedScorer.h"
#include "ScorerFactory.h"
#include "Util.h"

using namespace std;

// TODO: This is too long. Consider creating a function for
// initialization such as Init().
InterpolatedScorer::InterpolatedScorer(const string& name, const string& config)
    : Scorer(name,config)
{
  // name would be: HAMMING,BLEU or similar
  string scorers = name;
  while (scorers.length() > 0) {
    string scorertype = "";
    getNextPound(scorers, scorertype,",");
    Scorer *scorer = ScorerFactory::getScorer(scorertype,config);
    m_scorers.push_back(scorer);
  }
  if (m_scorers.size() == 0) {
    throw runtime_error("There are no scorers");
  }
  cerr << "Number of scorers: " << m_scorers.size() << endl;

  //TODO debug this
  string wtype = getConfig("weights","");
  //Default weights set to uniform ie. if two weights 0.5 each
  //weights should add to 1
  if (wtype.length() == 0) {
    float weight = 1.0 / m_scorers.size() ;
    //cout << " Default weights:" << weight << endl;
    for (size_t i = 0; i < m_scorers.size(); i ++) {
      m_scorer_weights.push_back(weight);
    }
  } else {
    float tot=0;
    //cout << "Defined weights:"  << endl;
    while (wtype.length() > 0) {
      string scoreweight = "";
      getNextPound(wtype,scoreweight,"+");
      float weight = atof(scoreweight.c_str());
      m_scorer_weights.push_back(weight);
      tot += weight;
      //cout << " :" << weight ;
    }
    //cout << endl;
    if (tot != float(1)) { // TODO: fix this checking in terms of readability.
      for (vector<float>::iterator it = m_scorer_weights.begin();
           it != m_scorer_weights.end(); ++it) {
        *it /= tot;
      }
    }

    if (m_scorers.size() != m_scorer_weights.size()) {
      throw runtime_error("The number of weights does not equal the number of scorers!");
    }
  }
  cerr << "The weights for the interpolated scorers are: " << endl;
  for (vector<float>::iterator it = m_scorer_weights.begin(); it < m_scorer_weights.end(); it++) {
    cerr << *it << " " ;
  }
  cerr <<endl;
}

void InterpolatedScorer::setScoreData(ScoreData* data)
{
  size_t last = 0;
  m_score_data = data;
  for (ScopedVector<Scorer>::iterator itsc = m_scorers.begin();
       itsc != m_scorers.end(); ++itsc) {
    int numScoresScorer = (*itsc)->NumberOfScores();
    ScoreData* newData =new ScoreData(*itsc);
    for (size_t i = 0; i < data->size(); i++) {
      ScoreArray scoreArray = data->get(i);
      ScoreArray newScoreArray;
      std::string istr;
      std::stringstream out;
      out << i;
      istr = out.str();
      size_t numNBest = scoreArray.size();
      //cout << " Datasize " << data->size() <<  " NumNBest " << numNBest << endl ;
      for (size_t j = 0; j < numNBest ; j++) {
        ScoreStats scoreStats = data->get(i, j);
        //cout << "Scorestats " << scoreStats << " i " << i << " j " << j << endl;
        ScoreStats newScoreStats;
        for (size_t k = last; k < size_t(numScoresScorer + last); k++) {
          ScoreStatsType score = scoreStats.get(k);
          newScoreStats.add(score);
        }
        //cout << " last " << last << " NumScores " << numScoresScorer << "newScorestats " << newScoreStats << endl;
        newScoreArray.add(newScoreStats);
      }
      newScoreArray.setIndex(istr);
      newData->add(newScoreArray);
    }
    //newData->dump();

    // NOTE: This class takes the ownership of the heap allocated
    // ScoreData objects to avoid the memory leak issues.
    m_scorers_score_data.push_back(newData);

    (*itsc)->setScoreData(newData);
    last += numScoresScorer;
  }
}

/** The interpolated scorer calls a vector of scorers and combines them with
    weights **/
void InterpolatedScorer::score(const candidates_t& candidates, const diffs_t& diffs,
                               statscores_t& scores) const
{
  //cout << "*******InterpolatedScorer::score" << endl;
  size_t scorerNum = 0;
  for (ScopedVector<Scorer>::const_iterator itsc = m_scorers.begin();
       itsc != m_scorers.end(); ++itsc) {
    //int numScores = (*itsc)->NumberOfScores();
    statscores_t tscores;
    (*itsc)->score(candidates,diffs,tscores);
    size_t inc = 0;
    for (statscores_t::iterator itstatsc = tscores.begin();
         itstatsc != tscores.end(); ++itstatsc) {
      //cout << "Scores " << (*itstatsc) << endl;
      float weight = m_scorer_weights[scorerNum];
      if (weight == 0) {
        stringstream msg;
        msg << "No weights for scorer" << scorerNum ;
        throw runtime_error(msg.str());
      }
      if (scorerNum == 0) {
        scores.push_back(weight * (*itstatsc));
      } else {
        scores[inc] +=  weight * (*itstatsc);
      }
      //cout << "Scorer:" << scorerNum <<  " scoreNum:" << inc << " score: " << (*itstatsc) << " weight:" << weight << endl;
      inc++;

    }
    scorerNum++;
  }

}

void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles)
{
  for (ScopedVector<Scorer>::iterator itsc = m_scorers.begin();
       itsc != m_scorers.end(); ++itsc) {
    (*itsc)->setReferenceFiles(referenceFiles);
  }
}

void InterpolatedScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
{
  stringstream buff;
  int i = 0;
  for (ScopedVector<Scorer>::iterator itsc = m_scorers.begin();
       itsc != m_scorers.end(); ++itsc) {
    ScoreStats tempEntry;
    (*itsc)->prepareStats(sid, text, tempEntry);
    if (i > 0) buff <<  " ";
    buff << tempEntry;
    i++;
  }
  //cout << " Scores for interpolated: " << buff << endl;
  string str = buff.str();
  entry.set(str);
}

void InterpolatedScorer::setFactors(const string& factors)
{
  if (factors.empty()) return;

  vector<string> fsplit;
  split(factors, ',', fsplit);

  if (fsplit.size() != m_scorers.size())
    throw runtime_error("Number of factor specifications does not equal number of interpolated scorers.");

  for (size_t i = 0; i < m_scorers.size(); ++i) {
    m_scorers[i]->setFactors(fsplit[i]);
  }
}

void InterpolatedScorer::setFilter(const string& filterCommand)
{
    for (size_t i = 0; i < m_scorers.size(); ++i) {
      m_scorers[i]->setFilter(filterCommand);
    }
}
<commit_msg>Fixed interpolated scorer<commit_after>#include "InterpolatedScorer.h"
#include "ScorerFactory.h"
#include "Util.h"

using namespace std;

// TODO: This is too long. Consider creating a function for
// initialization such as Init().
InterpolatedScorer::InterpolatedScorer(const string& name, const string& config)
    : Scorer(name,config)
{
  // name would be: HAMMING,BLEU or similar
  string scorers = name;
  while (scorers.length() > 0) {
    string scorertype = "";
    getNextPound(scorers, scorertype,",");
    Scorer *scorer = ScorerFactory::getScorer(scorertype,config);
    m_scorers.push_back(scorer);
  }
  if (m_scorers.size() == 0) {
    throw runtime_error("There are no scorers");
  }
  cerr << "Number of scorers: " << m_scorers.size() << endl;

  //TODO debug this
  string wtype = getConfig("weights","");
  //Default weights set to uniform ie. if two weights 0.5 each
  //weights should add to 1
  if (wtype.length() == 0) {
    float weight = 1.0 / m_scorers.size() ;
    //cout << " Default weights:" << weight << endl;
    for (size_t i = 0; i < m_scorers.size(); i ++) {
      m_scorer_weights.push_back(weight);
    }
  } else {
    float tot=0;
    //cout << "Defined weights:"  << endl;
    while (wtype.length() > 0) {
      string scoreweight = "";
      getNextPound(wtype,scoreweight,"+");
      float weight = atof(scoreweight.c_str());
      m_scorer_weights.push_back(weight);
      tot += weight;
      //cout << " :" << weight ;
    }
    //cout << endl;
    if (tot != float(1)) { // TODO: fix this checking in terms of readability.
      for (vector<float>::iterator it = m_scorer_weights.begin();
           it != m_scorer_weights.end(); ++it) {
        *it /= tot;
      }
    }

    if (m_scorers.size() != m_scorer_weights.size()) {
      throw runtime_error("The number of weights does not equal the number of scorers!");
    }
  }
  cerr << "The weights for the interpolated scorers are: " << endl;
  for (vector<float>::iterator it = m_scorer_weights.begin(); it < m_scorer_weights.end(); it++) {
    cerr << *it << " " ;
  }
  cerr <<endl;
}

void InterpolatedScorer::setScoreData(ScoreData* data)
{
  size_t last = 0;
  m_score_data = data;
  for (ScopedVector<Scorer>::iterator itsc = m_scorers.begin();
       itsc != m_scorers.end(); ++itsc) {
    int numScoresScorer = (*itsc)->NumberOfScores();
    ScoreData* newData =new ScoreData(*itsc);
    for (size_t i = 0; i < data->size(); i++) {
      ScoreArray scoreArray = data->get(i);
      ScoreArray newScoreArray;
      std::string istr;
      std::stringstream out;
      out << i;
      istr = out.str();
      size_t numNBest = scoreArray.size();
      //cout << " Datasize " << data->size() <<  " NumNBest " << numNBest << endl ;
      for (size_t j = 0; j < numNBest ; j++) {
        ScoreStats scoreStats = data->get(i, j);
        //cout << "Scorestats " << scoreStats << " i " << i << " j " << j << endl;
        ScoreStats newScoreStats;
        for (size_t k = last; k < size_t(numScoresScorer + last); k++) {
          ScoreStatsType score = scoreStats.get(k);
          newScoreStats.add(score);
        }
        //cout << " last " << last << " NumScores " << numScoresScorer << "newScorestats " << newScoreStats << endl;
        newScoreArray.add(newScoreStats);
      }
      newScoreArray.setIndex(istr);
      newData->add(newScoreArray);
    }
    //newData->dump();

    // NOTE: This class takes the ownership of the heap allocated
    // ScoreData objects to avoid the memory leak issues.
    m_scorers_score_data.push_back(newData);

    (*itsc)->setScoreData(newData);
    last += numScoresScorer;
  }
}

/** The interpolated scorer calls a vector of scorers and combines them with
    weights **/
void InterpolatedScorer::score(const candidates_t& candidates, const diffs_t& diffs,
                               statscores_t& scores) const
{
  //cout << "*******InterpolatedScorer::score" << endl;
  size_t scorerNum = 0;
  for (ScopedVector<Scorer>::const_iterator itsc = m_scorers.begin();
       itsc != m_scorers.end(); ++itsc) {
    //int numScores = (*itsc)->NumberOfScores();
    statscores_t tscores;
    (*itsc)->score(candidates,diffs,tscores);
    size_t inc = 0;
    for (statscores_t::iterator itstatsc = tscores.begin();
         itstatsc != tscores.end(); ++itstatsc) {
      //cout << "Scores " << (*itstatsc) << endl;
      float weight = m_scorer_weights[scorerNum];
      if (weight == 0) {
        stringstream msg;
        msg << "No weights for scorer" << scorerNum ;
        throw runtime_error(msg.str());
      }
      if (scorerNum == 0) {
        scores.push_back(weight * (*itstatsc));
      } else {
        scores[inc] +=  weight * (*itstatsc);
      }
      //cout << "Scorer:" << scorerNum <<  " scoreNum:" << inc << " score: " << (*itstatsc) << " weight:" << weight << endl;
      inc++;

    }
    scorerNum++;
  }

}

void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles)
{
  for (ScopedVector<Scorer>::iterator itsc = m_scorers.begin();
       itsc != m_scorers.end(); ++itsc) {
    (*itsc)->setReferenceFiles(referenceFiles);
  }
}

void InterpolatedScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
{
  stringstream buff;
  int i = 0;
  for (ScopedVector<Scorer>::iterator itsc = m_scorers.begin();
       itsc != m_scorers.end(); ++itsc) {
    ScoreStats tempEntry;
    (*itsc)->prepareStats(sid, text, tempEntry);
    if (i > 0) buff <<  " ";
    buff << tempEntry;
    i++;
  }
  //cout << " Scores for interpolated: " << buff << endl;
  string str = buff.str();
  entry.set(str);
}

void InterpolatedScorer::setFactors(const string& factors)
{
  if (factors.empty()) return;

  vector<string> fsplit;
  split(factors, ',', fsplit);

  if (fsplit.size() != m_scorers.size())
    throw runtime_error("Number of factor specifications does not equal number of interpolated scorers.");

  for (size_t i = 0; i < m_scorers.size(); ++i) {
    m_scorers[i]->setFactors(fsplit[i]);
  }
}

void InterpolatedScorer::setFilter(const string& filterCommand)
{
    if (filterCommand.empty()) return;

    vector<string> csplit;
    split(filterCommand, ',', csplit);

    if (csplit.size() != m_scorers.size())
      throw runtime_error("Number of command specifications does not equal number of interpolated scorers.");

    for (size_t i = 0; i < m_scorers.size(); ++i) {
      m_scorers[i]->setFilter(csplit[i]);
    }
}
<|endoftext|>


<commit_before>#include "../LessParser.h"
#include "gtest/gtest.h"

// At rules
TEST(LessParserTest, AtRule) {
  istringstream in("@x: 10; selector {key: @x}");

  LessTokenizer t(&in);
  LessParser p(&t);
  Stylesheet s;

  p.parseStylesheet(&s);

  Ruleset* set;
  Declaration* d;

  p.parseStylesheet(&s);

  ASSERT_EQ(1, s.getRulesets()->size());
  ASSERT_EQ(1, s.getRulesets()->at(0)->getDeclarations()->size());

  d = s.getRulesets()->at(0)->getDeclarations()->at(0);
  ASSERT_STREQ("key", d->getProperty()->c_str());
  ASSERT_STREQ("10", d->getValue()->toString()->c_str());
}

<commit_msg>Added tests for variables, mixins, nested rules and mixins with parameters.<commit_after>#include "../LessParser.h"
#include "gtest/gtest.h"

TEST(LessParserTest, Variable) {
  istringstream in("@x: 10; selector {key: @x}");

  LessTokenizer t(&in);
  LessParser p(&t);
  Stylesheet s;
  Declaration* d;

  p.parseStylesheet(&s);

  ASSERT_EQ((uint)1, s.getRulesets()->size());
  ASSERT_EQ((uint)1, s.getRulesets()->at(0)->getDeclarations()->size());

  d = s.getRulesets()->at(0)->getDeclarations()->at(0);
  EXPECT_STREQ("key", d->getProperty()->c_str());
  EXPECT_STREQ("10", d->getValue()->toString()->c_str());
}

// mixin
TEST(LessParserTest, Mixin) {
  istringstream in("mixin {key: 10} selector {mixin}");

  LessTokenizer t(&in);
  LessParser p(&t);
  Stylesheet s;

  Ruleset* set;
  Declaration* d;

  p.parseStylesheet(&s);

  ASSERT_EQ((uint)2, s.getRulesets()->size());
  set = s.getRulesets()->at(1);
  ASSERT_STREQ("selector", set->getSelector()->toString()->c_str());

  ASSERT_EQ((uint)1, set->getDeclarations()->size());
  d = set->getDeclarations()->at(0);

  EXPECT_STREQ("key", d->getProperty()->c_str());
  EXPECT_STREQ("10", d->getValue()->toString()->c_str());
}

// nested rule
TEST(LessParserTest, NestedRule) {
  istringstream in("selector { nested {key: value} }");

  LessTokenizer t(&in);
  LessParser p(&t);
  Stylesheet s;

  Ruleset* set;
  Declaration* d;

  p.parseStylesheet(&s);

  ASSERT_EQ((uint)2, s.getRulesets()->size());

  // Check for 'selector {}'
  set = s.getRulesets()->at(0);
  ASSERT_STREQ("selector", set->getSelector()->toString()->c_str());
  ASSERT_EQ((uint)0, set->getDeclarations()->size());

  // Check for 'selector nested {key: value}'
  set = s.getRulesets()->at(1);
  ASSERT_STREQ("selector nested", set->getSelector()->toString()->c_str());

  ASSERT_EQ((uint)1, set->getDeclarations()->size());
  d = set->getDeclarations()->at(0);

  EXPECT_STREQ("key", d->getProperty()->c_str());
  EXPECT_STREQ("value", d->getValue()->toString()->c_str());
}

// parameter mixin
TEST(LessParserTest, ParameterMixin) {
  istringstream in(".mixin(@x) {key: @x} selector {.mixin(10);}");

  LessTokenizer t(&in);
  LessParser p(&t);
  Stylesheet s;

  Ruleset* set;
  Declaration* d;

  p.parseStylesheet(&s);

  ASSERT_EQ((uint)1, s.getRulesets()->size());
  set = s.getRulesets()->at(0);
  ASSERT_STREQ("selector", set->getSelector()->toString()->c_str());

  ASSERT_EQ((uint)1, set->getDeclarations()->size());
  d = set->getDeclarations()->at(0);

  EXPECT_STREQ("key", d->getProperty()->c_str());
  EXPECT_STREQ("10", d->getValue()->toString()->c_str());
}
<|endoftext|>


<commit_before>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

// This pass forms `tf_executor.island` per replica from a single
// `tf_device.replicate` island.

#include <memory>
#include <utility>

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Casting.h"
#include "mlir/IR/Attributes.h"  // from @llvm-project
#include "mlir/IR/Block.h"  // from @llvm-project
#include "mlir/IR/BlockAndValueMapping.h"  // from @llvm-project
#include "mlir/IR/Builders.h"  // from @llvm-project
#include "mlir/IR/Diagnostics.h"  // from @llvm-project
#include "mlir/IR/Dialect.h"  // from @llvm-project
#include "mlir/IR/Visitors.h"  // from @llvm-project
#include "mlir/Pass/Pass.h"  // from @llvm-project
#include "mlir/Support/LogicalResult.h"  // from @llvm-project
#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h"
#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h"
#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h"
#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h"
#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h"

namespace mlir {
namespace TFDevice {
namespace {
constexpr char kDeviceAttr[] = "device";
constexpr char kReplicaIdAttr[] = "_xla_replica_id";
constexpr char kDeviceOrdinalAttr[] = "device_ordinal";
constexpr char kTPUCore0[] = "TPU_REPLICATED_CORE_0";

struct ReplicateToIslandPass
    : public PassWrapper<ReplicateToIslandPass, FunctionPass> {
  void runOnFunction() override;
};

// Returns whether op requires `_xla_replica_id` attribute.
bool RequiresReplicaIDAttribute(Operation* op) {
  return llvm::isa<TF::EnqueueTPUEmbeddingSparseTensorBatchOp,
                   TF::EnqueueTPUEmbeddingRaggedTensorBatchOp>(op);
}

// Collects TPU device ordinal for outside compilation communication ops. This
// currently assumes outside compilation only uses `TPU_REPLICATED_CORE_0`
// aliased device for the device computation.
llvm::Optional<int64_t> GetDeviceOrdinal(
    const llvm::Optional<DictionaryAttr>& devices, Location loc,
    unsigned replica_id) {
  int64_t device_ordinal = 0;
  if (devices.hasValue()) {
    if (auto tpu_replica_0 = devices.getValue().get(kTPUCore0)) {
      llvm::StringRef tpu_device = tpu_replica_0.cast<ArrayAttr>()[replica_id]
                                       .cast<StringAttr>()
                                       .getValue();
      if (succeeded(tensorflow::GetDeviceOrdinalFromDeviceString(
              loc, tpu_device, &device_ordinal))) {
        return llvm::Optional<int64_t>(device_ordinal);
      }
    }
  }
  return llvm::None;
}

// Updates replica variant ops in a region based on replica `replica_id`.
// TODO(b/157624749): Replace this with better abstraction to differentiate ops
// for different replicas. Some ops, such as XlaHostCompute op or TPU Embedding
// ops, require replica id to be added as an op attribute to be used during
// execution. Handle such ops separately and add an integer attribute that
// represents replica id.
LogicalResult UpdateRegionReplicateVariantOps(
    OpBuilder& builder, Location loc, Region& region, int replica_id,
    const llvm::Optional<DictionaryAttr>& devices) {
  llvm::Optional<int64_t> device_ordinal =
      GetDeviceOrdinal(devices, loc, replica_id);

  auto result = region.walk([&](Operation* op) -> WalkResult {
    if (RequiresReplicaIDAttribute(op)) {
      op->setAttr(kReplicaIdAttr, builder.getI32IntegerAttr(replica_id));
      return WalkResult::advance();
    }

    if (isa<TF::_TPUDeviceOrdinalPlaceholderOp>(op)) {
      if (!device_ordinal.hasValue())
        return op->emitOpError()
               << "requires device ordinal from device " << kTPUCore0
               << " to be present in 'tf.device.replicate' op";

      OpBuilder builder(op);
      auto const_op = builder.create<TF::ConstOp>(
          op->getLoc(), DenseIntElementsAttr::get(
                            RankedTensorType::get({}, builder.getI64Type()),
                            {device_ordinal.getValue()}));
      op->replaceAllUsesWith(const_op);
      op->erase();
      return WalkResult::advance();
    }

    if (!devices.hasValue()) return WalkResult::advance();

    // Map aliased devices to explicit devices based on replica.
    if (auto launch = dyn_cast<tf_device::LaunchOp>(op))
      if (auto device_by_replica = devices.getValue().get(launch.device()))
        launch.setAttr(
            kDeviceAttr,
            device_by_replica.cast<ArrayAttr>()[replica_id].cast<StringAttr>());

    return WalkResult::advance();
  });

  return failure(result.wasInterrupted());
}

// Creates islands per replica from `tf_device.replicate` region. If for a
// `tf_device.launch` op the device is an aliased device of the
// `tf_device.replicate`, the device will be remapped to an explicit device
// for the associated replica island.
LogicalResult ExpandReplicateIntoReplicas(
    const Dialect* tf_dialect, OpBuilder& builder,
    tf_executor::IslandOp island_op, tf_device::ReplicateOp replicate_op,
    int num_replicas, llvm::SmallVectorImpl<tf_executor::IslandOp>& replicas) {
  replicas.reserve(num_replicas);
  auto devices = replicate_op.devices();

  // Collect result types and operands.
  Operation& terminator = replicate_op.GetBody().back();
  llvm::SmallVector<Type, 8> output_types(terminator.getOperandTypes());
  auto control_type = tf_executor::ControlType::get(island_op.getContext());
  llvm::SmallVector<Value, 8> replica_inputs(island_op.controlInputs());

  // Replace replicate terminator with YieldOp.
  builder.setInsertionPoint(&terminator);
  builder.create<tf_executor::YieldOp>(terminator.getLoc(),
                                       terminator.getOperands());
  terminator.erase();

  builder.setInsertionPoint(island_op);
  BlockAndValueMapping mapping;
  for (int i : llvm::seq<int>(0, num_replicas)) {
    // Create new island for replica.
    auto replica = builder.create<tf_executor::IslandOp>(
        island_op.getLoc(), output_types, control_type, replica_inputs);

    // Map block arg to replica arg.
    mapping.clear();
    for (auto& block_arg : replicate_op.GetBody().getArguments())
      mapping.map(block_arg,
                  replicate_op.GetReplicaOperandForBlockArgument(block_arg, i));

    // Copy over replicate region into replica island.
    replicate_op.body().cloneInto(&replica.body(), mapping);

    if (failed(UpdateRegionReplicateVariantOps(builder, replicate_op.getLoc(),
                                               replica.body(),
                                               /*replica_id=*/i, devices)))
      return failure();

    replicas.push_back(replica);
  }

  return success();
}

// Creates islands per replica from `tf_device.replicate` region and remap
// replicate results with new island outputs. A single island is created to
// forward control dependencies if there is a control dependency output from the
// replicate island. Devices are remapped from aliased devices to explicit
// devices, for `tf_device.launch` ops.
//
// For example, the following:
//
// %0:2 = tf_executor.island(%control) {
//   %1:4 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<i1>)
//              {n = 2 : i32,
//               devices = {DEVICE_ALIAS_0 = ["/DEVICE:0", "/DEVICE:1"],
//                          DEVICE_ALIAS_1 = ["/DEVICE:2", "/DEVICE:3"]}} {
//     %a = "tf_device.launch"() ( {
//       %2 = "tf.opA"(%ri) : (tensor<i1>) -> tensor<i1>
//       tf_device.return %2 : tensor<i1>
//     }) {device = "DEVICE_ALIAS_0"} : () -> tensor<i1>
//     %b = "tf_device.launch"() ( {
//       %3 = "tf.opB"(%a) : (tensor<i1>) -> tensor<i1>
//       tf_device.return %3 : tensor<i1>
//     }) {device = "DEVICE_ALIAS_1"} : () -> tensor<i1>
//     tf_device.return %a, %b : tensor<i1>, tensor<i1>
//   }
//   tf_executor.yield %1#0 : tensor<i1>
// }
//
// gets lowered to:
//
// %0:3 = tf_executor.island(%control) {
//   %a0 = "tf_device.launch"() ( {
//     %1 = "tf.opA"(%arg0) : (tensor<i1>) -> tensor<i1>
//     tf_device.return %1 : tensor<i1>
//   }) {device = "/DEVICE:0"} : () -> tensor<i1>
//   %b0 = "tf_device.launch"() ( {
//     %2 = "tf.opB"(%a0) : (tensor<i1>) -> tensor<i1>
//     tf_device.return %2 : tensor<i1>
//   }) {device = "/DEVICE:2"} : () -> tensor<i1>
//   tf_executor.yield %a0, %b0 : tensor<i1>, tensor<i1>
// }
// %3:3 = tf_executor.island(%control) {
//   %a1 = "tf_device.launch"() ( {
//     %4 = "tf.opA"(%arg1) : (tensor<i1>) -> tensor<i1>
//     tf_device.return %4 : tensor<i1>
//   }) {device = "/DEVICE:1"} : () -> tensor<i1>
//   %b1 = "tf_device.launch"() ( {
//     %5 = "tf.opB"(%a1) : (tensor<i1>) -> tensor<i1>
//     tf_device.return %5 : tensor<i1>
//   }) {device = "/DEVICE:3"} : () -> tensor<i1>
//   tf_executor.yield %a1, %b1 : tensor<i1>, tensor<i1>
// }
LogicalResult CreateIslandsFromReplicate(const Dialect* tf_dialect,
                                         tf_executor::GraphOp graph_op,
                                         tf_executor::IslandOp island_op,
                                         tf_device::ReplicateOp replicate_op) {
  OpBuilder builder(island_op);
  const int num_replicas = replicate_op.n();

  // Create islands per replica.
  llvm::SmallVector<tf_executor::IslandOp, 8> replicas;
  if (failed(ExpandReplicateIntoReplicas(tf_dialect, builder, island_op,
                                         replicate_op, num_replicas, replicas)))
    return failure();

  // Collect all replica results.
  llvm::SmallVector<Value, 8> replicas_outputs(replicate_op.getNumResults(),
                                               nullptr);
  for (auto replica_and_idx : llvm::enumerate(replicas))
    for (auto replica_result_and_idx :
         llvm::enumerate(replica_and_idx.value().outputs()))
      replicas_outputs[num_replicas * replica_result_and_idx.index() +
                       replica_and_idx.index()] =
          replica_result_and_idx.value();

  // Remap replicate results to per replica result.
  for (auto result : llvm::zip(island_op.outputs(), replicas_outputs))
    std::get<0>(result).replaceAllUsesWith(std::get<1>(result));

  // Add sink island to pin all replicas as a control dependency if there is a
  // control dependency leading from the replicate originally.
  if (!island_op.control().use_empty()) {
    llvm::SmallVector<Value, 8> island_operands;
    for (auto& replica : replicas) island_operands.push_back(replica.control());

    builder.setInsertionPoint(island_op);
    auto island_sink = builder.create<tf_executor::IslandOp>(
        island_op.getLoc(), llvm::ArrayRef<Type>{},
        tf_executor::ControlType::get(island_op.getContext()), island_operands);
    island_sink.body().push_back(new Block);
    builder.setInsertionPointToEnd(&island_sink.GetBody());
    builder.create<tf_executor::YieldOp>(island_op.getLoc(),
                                         llvm::ArrayRef<Value>{});
    island_op.control().replaceAllUsesWith(island_sink.control());
  }

  // Replicas with no uses should be pinned to a graph fetch so they still
  // execute.
  llvm::SmallVector<Value, 8> unused_replica_controls;
  for (auto& replica : replicas)
    if (replica.use_empty())
      unused_replica_controls.push_back(replica.control());

  if (!unused_replica_controls.empty()) {
    tf_executor::FetchOp fetch = graph_op.GetFetch();
    auto fetches = llvm::to_vector<8>(fetch.getOperands());
    fetches.append(unused_replica_controls.begin(),
                   unused_replica_controls.end());
    builder.setInsertionPoint(fetch);
    builder.create<tf_executor::FetchOp>(fetch.getLoc(), fetches);
    fetch.erase();
  }

  island_op.erase();
  return success();
}

void ReplicateToIslandPass::runOnFunction() {
  const Dialect* tf_dialect = getContext().getLoadedDialect("tf");
  if (!tf_dialect) {
    getOperation().emitError() << "'tf' dialect is not registered";
    return signalPassFailure();
  }

  // Find islands with a single `tf_device.replicate` and create individual
  // islands per replica of the replicate.
  llvm::SmallVector<tf_executor::IslandOp, 4> replicate_op_islands;
  getOperation().walk([&](tf_executor::GraphOp graph_op) {
    for (auto island_op : graph_op.getOps<tf_executor::IslandOp>()) {
      if (!island_op.WrapsSingleOp()) continue;

      if (isa<tf_device::ReplicateOp>(&island_op.GetBody().front()))
        replicate_op_islands.push_back(island_op);
    }
  });

  for (tf_executor::IslandOp island_op : replicate_op_islands) {
    auto graph_op = island_op.getParentOfType<tf_executor::GraphOp>();
    auto replicate_op =
        cast<tf_device::ReplicateOp>(island_op.GetBody().front());
    if (failed(CreateIslandsFromReplicate(tf_dialect, graph_op, island_op,
                                          replicate_op)))
      return signalPassFailure();
  }
}
}  // anonymous namespace

std::unique_ptr<OperationPass<FuncOp>> CreateReplicateToIslandPass() {
  return std::make_unique<ReplicateToIslandPass>();
}

static PassRegistration<ReplicateToIslandPass> pass(
    "tf-replicate-to-island", "Lowers device replicate to executor islands");

}  // namespace TFDevice
}  // namespace mlir
<commit_msg>Set _xla_replica_id attribute as a i64 attribute instead of a i32 (NFC).<commit_after>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

// This pass forms `tf_executor.island` per replica from a single
// `tf_device.replicate` island.

#include <memory>
#include <utility>

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Casting.h"
#include "mlir/IR/Attributes.h"  // from @llvm-project
#include "mlir/IR/Block.h"  // from @llvm-project
#include "mlir/IR/BlockAndValueMapping.h"  // from @llvm-project
#include "mlir/IR/Builders.h"  // from @llvm-project
#include "mlir/IR/Diagnostics.h"  // from @llvm-project
#include "mlir/IR/Dialect.h"  // from @llvm-project
#include "mlir/IR/Visitors.h"  // from @llvm-project
#include "mlir/Pass/Pass.h"  // from @llvm-project
#include "mlir/Support/LogicalResult.h"  // from @llvm-project
#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h"
#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h"
#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h"
#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h"
#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h"

namespace mlir {
namespace TFDevice {
namespace {
constexpr char kDeviceAttr[] = "device";
constexpr char kReplicaIdAttr[] = "_xla_replica_id";
constexpr char kDeviceOrdinalAttr[] = "device_ordinal";
constexpr char kTPUCore0[] = "TPU_REPLICATED_CORE_0";

struct ReplicateToIslandPass
    : public PassWrapper<ReplicateToIslandPass, FunctionPass> {
  void runOnFunction() override;
};

// Returns whether op requires `_xla_replica_id` attribute.
bool RequiresReplicaIDAttribute(Operation* op) {
  return llvm::isa<TF::EnqueueTPUEmbeddingSparseTensorBatchOp,
                   TF::EnqueueTPUEmbeddingRaggedTensorBatchOp>(op);
}

// Collects TPU device ordinal for outside compilation communication ops. This
// currently assumes outside compilation only uses `TPU_REPLICATED_CORE_0`
// aliased device for the device computation.
llvm::Optional<int64_t> GetDeviceOrdinal(
    const llvm::Optional<DictionaryAttr>& devices, Location loc,
    unsigned replica_id) {
  int64_t device_ordinal = 0;
  if (devices.hasValue()) {
    if (auto tpu_replica_0 = devices.getValue().get(kTPUCore0)) {
      llvm::StringRef tpu_device = tpu_replica_0.cast<ArrayAttr>()[replica_id]
                                       .cast<StringAttr>()
                                       .getValue();
      if (succeeded(tensorflow::GetDeviceOrdinalFromDeviceString(
              loc, tpu_device, &device_ordinal))) {
        return llvm::Optional<int64_t>(device_ordinal);
      }
    }
  }
  return llvm::None;
}

// Updates replica variant ops in a region based on replica `replica_id`.
// TODO(b/157624749): Replace this with better abstraction to differentiate ops
// for different replicas. Some ops, such as XlaHostCompute op or TPU Embedding
// ops, require replica id to be added as an op attribute to be used during
// execution. Handle such ops separately and add an integer attribute that
// represents replica id.
LogicalResult UpdateRegionReplicateVariantOps(
    OpBuilder& builder, Location loc, Region& region, int replica_id,
    const llvm::Optional<DictionaryAttr>& devices) {
  llvm::Optional<int64_t> device_ordinal =
      GetDeviceOrdinal(devices, loc, replica_id);

  auto result = region.walk([&](Operation* op) -> WalkResult {
    if (RequiresReplicaIDAttribute(op)) {
      op->setAttr(kReplicaIdAttr, builder.getI64IntegerAttr(replica_id));
      return WalkResult::advance();
    }

    if (isa<TF::_TPUDeviceOrdinalPlaceholderOp>(op)) {
      if (!device_ordinal.hasValue())
        return op->emitOpError()
               << "requires device ordinal from device " << kTPUCore0
               << " to be present in 'tf.device.replicate' op";

      OpBuilder builder(op);
      auto const_op = builder.create<TF::ConstOp>(
          op->getLoc(), DenseIntElementsAttr::get(
                            RankedTensorType::get({}, builder.getI64Type()),
                            {device_ordinal.getValue()}));
      op->replaceAllUsesWith(const_op);
      op->erase();
      return WalkResult::advance();
    }

    if (!devices.hasValue()) return WalkResult::advance();

    // Map aliased devices to explicit devices based on replica.
    if (auto launch = dyn_cast<tf_device::LaunchOp>(op))
      if (auto device_by_replica = devices.getValue().get(launch.device()))
        launch.setAttr(
            kDeviceAttr,
            device_by_replica.cast<ArrayAttr>()[replica_id].cast<StringAttr>());

    return WalkResult::advance();
  });

  return failure(result.wasInterrupted());
}

// Creates islands per replica from `tf_device.replicate` region. If for a
// `tf_device.launch` op the device is an aliased device of the
// `tf_device.replicate`, the device will be remapped to an explicit device
// for the associated replica island.
LogicalResult ExpandReplicateIntoReplicas(
    const Dialect* tf_dialect, OpBuilder& builder,
    tf_executor::IslandOp island_op, tf_device::ReplicateOp replicate_op,
    int num_replicas, llvm::SmallVectorImpl<tf_executor::IslandOp>& replicas) {
  replicas.reserve(num_replicas);
  auto devices = replicate_op.devices();

  // Collect result types and operands.
  Operation& terminator = replicate_op.GetBody().back();
  llvm::SmallVector<Type, 8> output_types(terminator.getOperandTypes());
  auto control_type = tf_executor::ControlType::get(island_op.getContext());
  llvm::SmallVector<Value, 8> replica_inputs(island_op.controlInputs());

  // Replace replicate terminator with YieldOp.
  builder.setInsertionPoint(&terminator);
  builder.create<tf_executor::YieldOp>(terminator.getLoc(),
                                       terminator.getOperands());
  terminator.erase();

  builder.setInsertionPoint(island_op);
  BlockAndValueMapping mapping;
  for (int i : llvm::seq<int>(0, num_replicas)) {
    // Create new island for replica.
    auto replica = builder.create<tf_executor::IslandOp>(
        island_op.getLoc(), output_types, control_type, replica_inputs);

    // Map block arg to replica arg.
    mapping.clear();
    for (auto& block_arg : replicate_op.GetBody().getArguments())
      mapping.map(block_arg,
                  replicate_op.GetReplicaOperandForBlockArgument(block_arg, i));

    // Copy over replicate region into replica island.
    replicate_op.body().cloneInto(&replica.body(), mapping);

    if (failed(UpdateRegionReplicateVariantOps(builder, replicate_op.getLoc(),
                                               replica.body(),
                                               /*replica_id=*/i, devices)))
      return failure();

    replicas.push_back(replica);
  }

  return success();
}

// Creates islands per replica from `tf_device.replicate` region and remap
// replicate results with new island outputs. A single island is created to
// forward control dependencies if there is a control dependency output from the
// replicate island. Devices are remapped from aliased devices to explicit
// devices, for `tf_device.launch` ops.
//
// For example, the following:
//
// %0:2 = tf_executor.island(%control) {
//   %1:4 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<i1>)
//              {n = 2 : i32,
//               devices = {DEVICE_ALIAS_0 = ["/DEVICE:0", "/DEVICE:1"],
//                          DEVICE_ALIAS_1 = ["/DEVICE:2", "/DEVICE:3"]}} {
//     %a = "tf_device.launch"() ( {
//       %2 = "tf.opA"(%ri) : (tensor<i1>) -> tensor<i1>
//       tf_device.return %2 : tensor<i1>
//     }) {device = "DEVICE_ALIAS_0"} : () -> tensor<i1>
//     %b = "tf_device.launch"() ( {
//       %3 = "tf.opB"(%a) : (tensor<i1>) -> tensor<i1>
//       tf_device.return %3 : tensor<i1>
//     }) {device = "DEVICE_ALIAS_1"} : () -> tensor<i1>
//     tf_device.return %a, %b : tensor<i1>, tensor<i1>
//   }
//   tf_executor.yield %1#0 : tensor<i1>
// }
//
// gets lowered to:
//
// %0:3 = tf_executor.island(%control) {
//   %a0 = "tf_device.launch"() ( {
//     %1 = "tf.opA"(%arg0) : (tensor<i1>) -> tensor<i1>
//     tf_device.return %1 : tensor<i1>
//   }) {device = "/DEVICE:0"} : () -> tensor<i1>
//   %b0 = "tf_device.launch"() ( {
//     %2 = "tf.opB"(%a0) : (tensor<i1>) -> tensor<i1>
//     tf_device.return %2 : tensor<i1>
//   }) {device = "/DEVICE:2"} : () -> tensor<i1>
//   tf_executor.yield %a0, %b0 : tensor<i1>, tensor<i1>
// }
// %3:3 = tf_executor.island(%control) {
//   %a1 = "tf_device.launch"() ( {
//     %4 = "tf.opA"(%arg1) : (tensor<i1>) -> tensor<i1>
//     tf_device.return %4 : tensor<i1>
//   }) {device = "/DEVICE:1"} : () -> tensor<i1>
//   %b1 = "tf_device.launch"() ( {
//     %5 = "tf.opB"(%a1) : (tensor<i1>) -> tensor<i1>
//     tf_device.return %5 : tensor<i1>
//   }) {device = "/DEVICE:3"} : () -> tensor<i1>
//   tf_executor.yield %a1, %b1 : tensor<i1>, tensor<i1>
// }
LogicalResult CreateIslandsFromReplicate(const Dialect* tf_dialect,
                                         tf_executor::GraphOp graph_op,
                                         tf_executor::IslandOp island_op,
                                         tf_device::ReplicateOp replicate_op) {
  OpBuilder builder(island_op);
  const int num_replicas = replicate_op.n();

  // Create islands per replica.
  llvm::SmallVector<tf_executor::IslandOp, 8> replicas;
  if (failed(ExpandReplicateIntoReplicas(tf_dialect, builder, island_op,
                                         replicate_op, num_replicas, replicas)))
    return failure();

  // Collect all replica results.
  llvm::SmallVector<Value, 8> replicas_outputs(replicate_op.getNumResults(),
                                               nullptr);
  for (auto replica_and_idx : llvm::enumerate(replicas))
    for (auto replica_result_and_idx :
         llvm::enumerate(replica_and_idx.value().outputs()))
      replicas_outputs[num_replicas * replica_result_and_idx.index() +
                       replica_and_idx.index()] =
          replica_result_and_idx.value();

  // Remap replicate results to per replica result.
  for (auto result : llvm::zip(island_op.outputs(), replicas_outputs))
    std::get<0>(result).replaceAllUsesWith(std::get<1>(result));

  // Add sink island to pin all replicas as a control dependency if there is a
  // control dependency leading from the replicate originally.
  if (!island_op.control().use_empty()) {
    llvm::SmallVector<Value, 8> island_operands;
    for (auto& replica : replicas) island_operands.push_back(replica.control());

    builder.setInsertionPoint(island_op);
    auto island_sink = builder.create<tf_executor::IslandOp>(
        island_op.getLoc(), llvm::ArrayRef<Type>{},
        tf_executor::ControlType::get(island_op.getContext()), island_operands);
    island_sink.body().push_back(new Block);
    builder.setInsertionPointToEnd(&island_sink.GetBody());
    builder.create<tf_executor::YieldOp>(island_op.getLoc(),
                                         llvm::ArrayRef<Value>{});
    island_op.control().replaceAllUsesWith(island_sink.control());
  }

  // Replicas with no uses should be pinned to a graph fetch so they still
  // execute.
  llvm::SmallVector<Value, 8> unused_replica_controls;
  for (auto& replica : replicas)
    if (replica.use_empty())
      unused_replica_controls.push_back(replica.control());

  if (!unused_replica_controls.empty()) {
    tf_executor::FetchOp fetch = graph_op.GetFetch();
    auto fetches = llvm::to_vector<8>(fetch.getOperands());
    fetches.append(unused_replica_controls.begin(),
                   unused_replica_controls.end());
    builder.setInsertionPoint(fetch);
    builder.create<tf_executor::FetchOp>(fetch.getLoc(), fetches);
    fetch.erase();
  }

  island_op.erase();
  return success();
}

void ReplicateToIslandPass::runOnFunction() {
  const Dialect* tf_dialect = getContext().getLoadedDialect("tf");
  if (!tf_dialect) {
    getOperation().emitError() << "'tf' dialect is not registered";
    return signalPassFailure();
  }

  // Find islands with a single `tf_device.replicate` and create individual
  // islands per replica of the replicate.
  llvm::SmallVector<tf_executor::IslandOp, 4> replicate_op_islands;
  getOperation().walk([&](tf_executor::GraphOp graph_op) {
    for (auto island_op : graph_op.getOps<tf_executor::IslandOp>()) {
      if (!island_op.WrapsSingleOp()) continue;

      if (isa<tf_device::ReplicateOp>(&island_op.GetBody().front()))
        replicate_op_islands.push_back(island_op);
    }
  });

  for (tf_executor::IslandOp island_op : replicate_op_islands) {
    auto graph_op = island_op.getParentOfType<tf_executor::GraphOp>();
    auto replicate_op =
        cast<tf_device::ReplicateOp>(island_op.GetBody().front());
    if (failed(CreateIslandsFromReplicate(tf_dialect, graph_op, island_op,
                                          replicate_op)))
      return signalPassFailure();
  }
}
}  // anonymous namespace

std::unique_ptr<OperationPass<FuncOp>> CreateReplicateToIslandPass() {
  return std::make_unique<ReplicateToIslandPass>();
}

static PassRegistration<ReplicateToIslandPass> pass(
    "tf-replicate-to-island", "Lowers device replicate to executor islands");

}  // namespace TFDevice
}  // namespace mlir
<|endoftext|>


<commit_before>#include <boost/test/unit_test.hpp>

#include <vector>

#include "clotho/utility/bit_walker.hpp"

typedef unsigned long block_type;
typedef unsigned short sub_block_type;
typedef clotho::utility::block_walker< block_type, sub_block_type > walker_type;

struct set_bit_vector_op {

    set_bit_vector_op() {}

    void operator()( unsigned int idx ) {
        indices.push_back(idx);
    }

    void reset() { indices.clear(); }

    std::vector< unsigned int > indices;
};

BOOST_AUTO_TEST_SUITE( test_utility )

/**
 * Verify bits_per_block are initialized appropriately
 */
BOOST_AUTO_TEST_CASE( test_bit_walker_bit_sizes ) {

    const unsigned int bpb = walker_type::bits_per_block;
    const unsigned int bpsb = walker_type::bits_per_subblock;
    BOOST_REQUIRE_MESSAGE( bpb == sizeof(block_type) * 8, "Unexpected bits_per_block: " << bpb << " (" << (sizeof(block_type) * 8) << ")");
    BOOST_REQUIRE_MESSAGE( bpsb == sizeof(sub_block_type) * 8, "Unexpected bits_per_subblock: " << bpsb << " (" << (sizeof(sub_block_type) * 8) << ")");
}

BOOST_AUTO_TEST_CASE( test_bit_node_init ) {
    walker_type::bit_walker_type bw;

    for( unsigned int i = 1; i < walker_type::bit_walker_type::max_nodes; ++i ) {
        unsigned int s = (i & (1 << bw.getNode()[i].bit_index));
        BOOST_REQUIRE_MESSAGE( s != 0, "Unexpected bit state: " << i << " @ " << bw.getNode()[i].bit_index );

        if( bw.getNode()[i].next ) {
            unsigned int k = (i >> bw.getNode()[i].bit_shift_next);
            BOOST_REQUIRE_MESSAGE( k == bw.getNode()[i].next, "Unexpected next node: " << i << " -> " << k << " (" << bw.getNode()[i].next << ")");
        }
    }
}

/**
 * Initialize a block with every odd bit set, and initialize an expected index vector
 *
 * Apply the set_bit_vector_op
 *
 * Verify that the result vector matches the expected vector
 */
BOOST_AUTO_TEST_CASE( test_bit_walker_set_bits ) {
    std::vector< unsigned int > expected;

    block_type _bits = 0, mask = 1;
    for( unsigned int i = 0; i < walker_type::bits_per_block; ++i ) {
        if( i & 1 ) {
            expected.push_back( i );

            _bits |= mask;
        }
        mask <<= 1;
    }

    set_bit_vector_op sbv;
    walker_type::apply( _bits, sbv );

    BOOST_REQUIRE_MESSAGE( sbv.indices == expected, "Unexpected list of set bits was returned" );
}

BOOST_AUTO_TEST_SUITE_END()
<commit_msg>Testing type_trait specifications<commit_after>#include <boost/test/unit_test.hpp>

#include <vector>
#include <type_traits>

#include "clotho/utility/bit_walker.hpp"

typedef unsigned long block_type;
typedef unsigned short sub_block_type;
typedef clotho::utility::block_walker< block_type, sub_block_type > walker_type;

typedef clotho::utility::block_walker< block_type, sub_block_type, clotho::utility::bit_block_walker< sizeof(block_type) * 8> > other_walker_type;

struct set_bit_vector_op {

    set_bit_vector_op() {}

    void operator()( unsigned int idx ) {
        indices.push_back(idx);
    }

    void reset() { indices.clear(); }

    std::vector< unsigned int > indices;
};

BOOST_AUTO_TEST_SUITE( test_utility )

/**
 * Verify bits_per_block are initialized appropriately
 */
BOOST_AUTO_TEST_CASE( test_bit_walker_bit_sizes ) {

    const unsigned int bpb = walker_type::bits_per_block;
    const unsigned int bpsb = walker_type::bits_per_subblock;
    BOOST_REQUIRE_MESSAGE( bpb == sizeof(block_type) * 8, "Unexpected bits_per_block: " << bpb << " (" << (sizeof(block_type) * 8) << ")");
    BOOST_REQUIRE_MESSAGE( bpsb == sizeof(sub_block_type) * 8, "Unexpected bits_per_subblock: " << bpsb << " (" << (sizeof(sub_block_type) * 8) << ")");
}

BOOST_AUTO_TEST_CASE( test_bit_node_init ) {
    walker_type::bit_walker_type bw;

    for( unsigned int i = 1; i < walker_type::bit_walker_type::max_nodes; ++i ) {
        unsigned int s = (i & (1 << bw.getNode()[i].bit_index));
        BOOST_REQUIRE_MESSAGE( s != 0, "Unexpected bit state: " << i << " @ " << bw.getNode()[i].bit_index );

        if( bw.getNode()[i].next ) {
            unsigned int k = (i >> bw.getNode()[i].bit_shift_next);
            BOOST_REQUIRE_MESSAGE( k == bw.getNode()[i].next, "Unexpected next node: " << i << " -> " << k << " (" << bw.getNode()[i].next << ")");
        }
    }
}

/**
 * Initialize a block with every odd bit set, and initialize an expected index vector
 *
 * Apply the set_bit_vector_op
 *
 * Verify that the result vector matches the expected vector
 */
BOOST_AUTO_TEST_CASE( test_bit_walker_set_bits ) {
    std::vector< unsigned int > expected;

    block_type _bits = 0, mask = 1;
    for( unsigned int i = 0; i < walker_type::bits_per_block; ++i ) {
        if( i & 1 ) {
            expected.push_back( i );

            _bits |= mask;
        }
        mask <<= 1;
    }

    set_bit_vector_op sbv;
    walker_type::apply( _bits, sbv );

    BOOST_REQUIRE_MESSAGE( sbv.indices == expected, "Unexpected list of set bits was returned" );
}

/**
 * Initialize a block with every odd bit set, and initialize an expected index vector
 *
 * Apply the set_bit_vector_op
 *
 * Verify that the result vector matches the expected vector
 */
BOOST_AUTO_TEST_CASE( test_other_bit_walker_set_bits ) {
    BOOST_REQUIRE_MESSAGE( (std::is_same< other_walker_type::bit_walker_type, clotho::utility::bit_block_walker< sizeof(block_type) * 8 > >::value), "Unexpected bit block walker (test 1)" );
    BOOST_REQUIRE_MESSAGE( (!std::is_same< other_walker_type::bit_walker_type, clotho::utility::bit_block_walker< 16 > >::value), "Unexpected bit block walker (test 2)" );

    std::vector< unsigned int > expected;

    block_type _bits = 0, mask = 1;
    for( unsigned int i = 0; i < walker_type::bits_per_block; ++i ) {
        if( i & 1 ) {
            expected.push_back( i );

            _bits |= mask;
        }
        mask <<= 1;
    }

    set_bit_vector_op sbv;
    other_walker_type::apply( _bits, sbv );

    BOOST_REQUIRE_MESSAGE( sbv.indices == expected, "Unexpected list of set bits was returned" );
}

BOOST_AUTO_TEST_SUITE_END()
<|endoftext|>