<commit_before>////////////////////////////////////////////////////////////////////////////////
/// @brief index
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Dr. Frank Celler
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2011-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

#include "Index.h"
#include "Basics/Exceptions.h"
#include "Basics/json-utilities.h"
#include "VocBase/server.h"
#include "VocBase/VocShaper.h"

using namespace triagens::arango;

// -----------------------------------------------------------------------------
// --SECTION--                                                       class Index
// -----------------------------------------------------------------------------

// -----------------------------------------------------------------------------
// --SECTION--                                      constructors and destructors
// -----------------------------------------------------------------------------

Index::Index (TRI_idx_iid_t iid,
              TRI_document_collection_t* collection,
              std::vector<std::vector<triagens::basics::AttributeName>> const& fields)
  : _iid(iid),
    _collection(collection),
    _fields(fields) {
}

Index::~Index () {
}

// -----------------------------------------------------------------------------
// --SECTION--                                                    public methods
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief return the index type based on a type name
////////////////////////////////////////////////////////////////////////////////

Index::IndexType Index::type (char const* type) {
  if (::strcmp(type, "primary") == 0) {
    return TRI_IDX_TYPE_PRIMARY_INDEX;
  }
  if (::strcmp(type, "edge") == 0) {
    return TRI_IDX_TYPE_EDGE_INDEX;
  }
  if (::strcmp(type, "hash") == 0) {
    return TRI_IDX_TYPE_HASH_INDEX;
  }
  if (::strcmp(type, "skiplist") == 0) {
    return TRI_IDX_TYPE_SKIPLIST_INDEX;
  }
  if (::strcmp(type, "fulltext") == 0) {
    return TRI_IDX_TYPE_FULLTEXT_INDEX;
  }
  if (::strcmp(type, "cap") == 0) {
    return TRI_IDX_TYPE_CAP_CONSTRAINT;
  }
  if (::strcmp(type, "geo1") == 0) {
    return TRI_IDX_TYPE_GEO1_INDEX;
  }
  if (::strcmp(type, "geo2") == 0) {
    return TRI_IDX_TYPE_GEO2_INDEX;
  }

  return TRI_IDX_TYPE_UNKNOWN;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief return the name of an index type
////////////////////////////////////////////////////////////////////////////////

char const* Index::typeName (Index::IndexType type) {
  switch (type) {
    case TRI_IDX_TYPE_PRIMARY_INDEX:
      return "primary";
    case TRI_IDX_TYPE_EDGE_INDEX:
      return "edge";
    case TRI_IDX_TYPE_HASH_INDEX:
      return "hash";
    case TRI_IDX_TYPE_SKIPLIST_INDEX:
      return "skiplist";
    case TRI_IDX_TYPE_FULLTEXT_INDEX:
      return "fulltext";
    case TRI_IDX_TYPE_CAP_CONSTRAINT:
      return "cap";
    case TRI_IDX_TYPE_GEO1_INDEX:
      return "geo1";
    case TRI_IDX_TYPE_GEO2_INDEX:
      return "geo2";
    case TRI_IDX_TYPE_PRIORITY_QUEUE_INDEX:
    case TRI_IDX_TYPE_BITARRAY_INDEX:
    case TRI_IDX_TYPE_UNKNOWN: {
    }
  }

  return "";
}

////////////////////////////////////////////////////////////////////////////////
/// @brief validate an index id
////////////////////////////////////////////////////////////////////////////////

bool Index::validateId (char const* key) {
  char const* p = key;

  while (1) {
    const char c = *p;

    if (c == '\0') {
      return (p - key) > 0;
    }
    if (c >= '0' && c <= '9') {
      ++p;
      continue;
    }

    return false;
  }
}

////////////////////////////////////////////////////////////////////////////////
/// @brief validate an index handle (collection name + / + index id)
////////////////////////////////////////////////////////////////////////////////

bool Index::validateHandle (char const* key,
                            size_t* split) {
  char const* p = key;
  char c = *p;

  // extract collection name

  if (! (c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) {
    return false;
  }

  ++p;

  while (1) {
    c = *p;

    if ((c == '_') || (c == '-') || (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
      ++p;
      continue;
    }

    if (c == '/') {
      break;
    }

    return false;
  }

  if (p - key > TRI_COL_NAME_LENGTH) {
    return false;
  }

  // store split position
  *split = p - key;
  ++p;

  // validate index id
  return validateId(p);
}

////////////////////////////////////////////////////////////////////////////////
/// @brief generate a new index id
////////////////////////////////////////////////////////////////////////////////

TRI_idx_iid_t Index::generateId () {
  return TRI_NewTickServer();
}

////////////////////////////////////////////////////////////////////////////////
/// @brief index comparator, used by the coordinator to detect if two index
/// contents are the same
////////////////////////////////////////////////////////////////////////////////

bool Index::Compare (TRI_json_t const* lhs,
                     TRI_json_t const* rhs) {
  TRI_json_t* typeJson = TRI_LookupObjectJson(lhs, "type");
  TRI_ASSERT(TRI_IsStringJson(typeJson));

  // type must be identical
  if (! TRI_CheckSameValueJson(typeJson, TRI_LookupObjectJson(rhs, "type"))) {
    return false;
  }

  auto type = Index::type(typeJson->_value._string.data);

  // unique must be identical if present
  TRI_json_t* value = TRI_LookupObjectJson(lhs, "unique");
  if (TRI_IsBooleanJson(value)) {
    if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "unique"))) {
      return false;
    }
  }

  // sparse must be identical if present
  value = TRI_LookupObjectJson(lhs, "sparse");
  if (TRI_IsBooleanJson(value)) {
    if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "sparse"))) {
      return false;
    }
  }

  if (type == IndexType::TRI_IDX_TYPE_GEO1_INDEX) {
    // geoJson must be identical if present
    value = TRI_LookupObjectJson(lhs, "geoJson");
    if (TRI_IsBooleanJson(value)) {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "geoJson"))) {
        return false;
      }
    }
  }
  else if (type == IndexType::TRI_IDX_TYPE_FULLTEXT_INDEX) {
    // minLength
    value = TRI_LookupObjectJson(lhs, "minLength");
    if (TRI_IsNumberJson(value)) {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "minLength"))) {
        return false;
      }
    }
  }
  else if (type == IndexType::TRI_IDX_TYPE_CAP_CONSTRAINT) {
    // size, byteSize
    value = TRI_LookupObjectJson(lhs, "size");
    if (TRI_IsNumberJson(value)) {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "size"))) {
        return false;
      }
    }

    value = TRI_LookupObjectJson(lhs, "byteSize");
    if (TRI_IsNumberJson(value)) {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "byteSize"))) {
        return false;
      }
    }
  }

  // other index types: fields must be identical if present
  value = TRI_LookupObjectJson(lhs, "fields");

  if (TRI_IsArrayJson(value)) {
    if (type == IndexType::TRI_IDX_TYPE_HASH_INDEX) {
      size_t const nv = TRI_LengthArrayJson(value);

      // compare fields in arbitrary order
      TRI_json_t const* r = TRI_LookupObjectJson(rhs, "fields");

      if (! TRI_IsArrayJson(r) ||
          nv != TRI_LengthArrayJson(r)) {
        return false;
      }

      size_t const nr = TRI_LengthArrayJson(r);

      for (size_t i = 0; i < nv; ++i) {
        TRI_json_t const* v = TRI_LookupArrayJson(value, i);

        bool found = false;

        for (size_t j = 0; j < nr; ++j) {
          if (TRI_CheckSameValueJson(v, TRI_LookupArrayJson(r, j))) {
            found = true;
            break;
          }
        }

        if (! found) {
          return false;
        }
      }
    }
    else {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "fields"))) {
        return false;
      }
    }

  }

  return true;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief return a contextual string for logging
////////////////////////////////////////////////////////////////////////////////

std::string Index::context () const {
  std::ostringstream result;

  result << "index { id: " << id()
         << ", type: " << typeName()
         << ", collection: " << _collection->_vocbase->_name
         << "/" <<  _collection->_info._name << " }";

  return result.str();
}

////////////////////////////////////////////////////////////////////////////////
/// @brief create a JSON representation of the index
/// base functionality (called from derived classes)
////////////////////////////////////////////////////////////////////////////////

triagens::basics::Json Index::toJson (TRI_memory_zone_t* zone) const {
  triagens::basics::Json json(zone, triagens::basics::Json::Object, 4);

  json("id", triagens::basics::Json(zone, std::to_string(_iid)))
      ("type", triagens::basics::Json(zone, typeName()));

  if (dumpFields()) {
    triagens::basics::Json f(zone, triagens::basics::Json::Array, fields().size());

    for (auto const& field : fields()) {
      std::string fieldString;
      TRI_AttributeNamesToString(field, fieldString);
      f.add(triagens::basics::Json(zone, fieldString));
    }

    json("fields", f);
  }

  if (hasSelectivityEstimate()) {
    json("selectivityEstimate", triagens::basics::Json(selectivityEstimate()));
  }

  return json;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for selectivityEstimate
////////////////////////////////////////////////////////////////////////////////

double Index::selectivityEstimate () const {
  THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for selectivityEstimate
////////////////////////////////////////////////////////////////////////////////

int Index::batchInsert (std::vector<struct TRI_doc_mptr_t const*> const*, size_t) {
  THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for postInsert
////////////////////////////////////////////////////////////////////////////////

int Index::postInsert (struct TRI_transaction_collection_s*,
                       struct TRI_doc_mptr_t const*) {
  // do nothing
  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for cleanup
////////////////////////////////////////////////////////////////////////////////

int Index::cleanup () {
  // do nothing
  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for sizeHint
////////////////////////////////////////////////////////////////////////////////

int Index::sizeHint (size_t) {
  // do nothing
  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for hasBatchInsert
////////////////////////////////////////////////////////////////////////////////

bool Index::hasBatchInsert () const {
  return false;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief helper function to insert a document into any index type
////////////////////////////////////////////////////////////////////////////////

int Index::fillElement(std::function<TRI_index_element_t* ()> allocate,
                       std::vector<TRI_index_element_t*>& elements,
                       TRI_doc_mptr_t const* document,
                       std::vector<TRI_shape_pid_t> const& paths,
                       bool const sparse) {
  TRI_ASSERT(document != nullptr);
  TRI_ASSERT_EXPENSIVE(document->getDataPtr() != nullptr);   // ONLY IN INDEX, PROTECTED by RUNTIME

  TRI_shaped_json_t shapedJson;

  TRI_EXTRACT_SHAPED_JSON_MARKER(shapedJson, document->getDataPtr());  // ONLY IN INDEX, PROTECTED by RUNTIME

  if (shapedJson._sid == TRI_SHAPE_ILLEGAL) {
    LOG_WARNING("encountered invalid marker with shape id 0");

    return TRI_ERROR_INTERNAL;
  }

  char const* ptr = document->getShapedJsonPtr();  // ONLY IN INDEX, PROTECTED by RUNTIME

  size_t const n = paths.size();

  TRI_index_element_t* element = allocate();
  if (element == nullptr) {
    return TRI_ERROR_OUT_OF_MEMORY;
  }
  element->document(const_cast<TRI_doc_mptr_t*>(document));
  TRI_shaped_sub_t* subObjects = element->subObjects();
  for (size_t j = 0; j < n; ++j) {
    TRI_shape_pid_t path = paths[j];

    // ..........................................................................
    // Determine if document has that particular shape
    // ..........................................................................

    TRI_shape_access_t const* acc = _collection->getShaper()->findAccessor(shapedJson._sid, path);  // ONLY IN INDEX, PROTECTED by RUNTIME

    if (acc == nullptr || acc->_resultSid == TRI_SHAPE_ILLEGAL) {
      // OK, the document does not contain the attributed needed by
      // the index, are we sparse?
      subObjects[j]._sid = BasicShapes::TRI_SHAPE_SID_NULL;

      if (sparse) {
        // no need to continue
        // Free this element and return.
        TRI_index_element_t::free(element);
        return TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING;
      }
      continue;
    }

    // ..........................................................................
    // Extract the field
    // ..........................................................................

    TRI_shaped_json_t shapedObject;
    if (! TRI_ExecuteShapeAccessor(acc, &shapedJson, &shapedObject)) {
      return TRI_ERROR_INTERNAL;
    }

    if (shapedObject._sid == BasicShapes::TRI_SHAPE_SID_NULL) {
      if (sparse) {
        // no need to continue
        // Free this element and return.
        TRI_index_element_t::free(element);
        return TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING;
      }
    }

    // .........................................................................
    // Store the field
    // .........................................................................

    TRI_FillShapedSub(&subObjects[j], &shapedObject, ptr);
  }
  elements.push_back(element);

  return TRI_ERROR_NO_ERROR;
}

namespace triagens {
  namespace arango {

////////////////////////////////////////////////////////////////////////////////
/// @brief append the index description to an output stream
////////////////////////////////////////////////////////////////////////////////

    std::ostream& operator<< (std::ostream& stream,
                              Index const* index) {
      stream << index->context();
      return stream;
    }

////////////////////////////////////////////////////////////////////////////////
/// @brief append the index description to an output stream
////////////////////////////////////////////////////////////////////////////////

    std::ostream& operator<< (std::ostream& stream,
                              Index const& index) {
      stream << index.context();
      return stream;
    }

  }
}

// -----------------------------------------------------------------------------
// --SECTION--                                                       END-OF-FILE
// -----------------------------------------------------------------------------

// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:
<commit_msg>Attribtue missing will not be returned when inserting elements into the index any more.<commit_after>////////////////////////////////////////////////////////////////////////////////
/// @brief index
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Dr. Frank Celler
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2011-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

#include "Index.h"
#include "Basics/Exceptions.h"
#include "Basics/json-utilities.h"
#include "VocBase/server.h"
#include "VocBase/VocShaper.h"

using namespace triagens::arango;

// -----------------------------------------------------------------------------
// --SECTION--                                                       class Index
// -----------------------------------------------------------------------------

// -----------------------------------------------------------------------------
// --SECTION--                                      constructors and destructors
// -----------------------------------------------------------------------------

Index::Index (TRI_idx_iid_t iid,
              TRI_document_collection_t* collection,
              std::vector<std::vector<triagens::basics::AttributeName>> const& fields)
  : _iid(iid),
    _collection(collection),
    _fields(fields) {
}

Index::~Index () {
}

// -----------------------------------------------------------------------------
// --SECTION--                                                    public methods
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief return the index type based on a type name
////////////////////////////////////////////////////////////////////////////////

Index::IndexType Index::type (char const* type) {
  if (::strcmp(type, "primary") == 0) {
    return TRI_IDX_TYPE_PRIMARY_INDEX;
  }
  if (::strcmp(type, "edge") == 0) {
    return TRI_IDX_TYPE_EDGE_INDEX;
  }
  if (::strcmp(type, "hash") == 0) {
    return TRI_IDX_TYPE_HASH_INDEX;
  }
  if (::strcmp(type, "skiplist") == 0) {
    return TRI_IDX_TYPE_SKIPLIST_INDEX;
  }
  if (::strcmp(type, "fulltext") == 0) {
    return TRI_IDX_TYPE_FULLTEXT_INDEX;
  }
  if (::strcmp(type, "cap") == 0) {
    return TRI_IDX_TYPE_CAP_CONSTRAINT;
  }
  if (::strcmp(type, "geo1") == 0) {
    return TRI_IDX_TYPE_GEO1_INDEX;
  }
  if (::strcmp(type, "geo2") == 0) {
    return TRI_IDX_TYPE_GEO2_INDEX;
  }

  return TRI_IDX_TYPE_UNKNOWN;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief return the name of an index type
////////////////////////////////////////////////////////////////////////////////

char const* Index::typeName (Index::IndexType type) {
  switch (type) {
    case TRI_IDX_TYPE_PRIMARY_INDEX:
      return "primary";
    case TRI_IDX_TYPE_EDGE_INDEX:
      return "edge";
    case TRI_IDX_TYPE_HASH_INDEX:
      return "hash";
    case TRI_IDX_TYPE_SKIPLIST_INDEX:
      return "skiplist";
    case TRI_IDX_TYPE_FULLTEXT_INDEX:
      return "fulltext";
    case TRI_IDX_TYPE_CAP_CONSTRAINT:
      return "cap";
    case TRI_IDX_TYPE_GEO1_INDEX:
      return "geo1";
    case TRI_IDX_TYPE_GEO2_INDEX:
      return "geo2";
    case TRI_IDX_TYPE_PRIORITY_QUEUE_INDEX:
    case TRI_IDX_TYPE_BITARRAY_INDEX:
    case TRI_IDX_TYPE_UNKNOWN: {
    }
  }

  return "";
}

////////////////////////////////////////////////////////////////////////////////
/// @brief validate an index id
////////////////////////////////////////////////////////////////////////////////

bool Index::validateId (char const* key) {
  char const* p = key;

  while (1) {
    const char c = *p;

    if (c == '\0') {
      return (p - key) > 0;
    }
    if (c >= '0' && c <= '9') {
      ++p;
      continue;
    }

    return false;
  }
}

////////////////////////////////////////////////////////////////////////////////
/// @brief validate an index handle (collection name + / + index id)
////////////////////////////////////////////////////////////////////////////////

bool Index::validateHandle (char const* key,
                            size_t* split) {
  char const* p = key;
  char c = *p;

  // extract collection name

  if (! (c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) {
    return false;
  }

  ++p;

  while (1) {
    c = *p;

    if ((c == '_') || (c == '-') || (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
      ++p;
      continue;
    }

    if (c == '/') {
      break;
    }

    return false;
  }

  if (p - key > TRI_COL_NAME_LENGTH) {
    return false;
  }

  // store split position
  *split = p - key;
  ++p;

  // validate index id
  return validateId(p);
}

////////////////////////////////////////////////////////////////////////////////
/// @brief generate a new index id
////////////////////////////////////////////////////////////////////////////////

TRI_idx_iid_t Index::generateId () {
  return TRI_NewTickServer();
}

////////////////////////////////////////////////////////////////////////////////
/// @brief index comparator, used by the coordinator to detect if two index
/// contents are the same
////////////////////////////////////////////////////////////////////////////////

bool Index::Compare (TRI_json_t const* lhs,
                     TRI_json_t const* rhs) {
  TRI_json_t* typeJson = TRI_LookupObjectJson(lhs, "type");
  TRI_ASSERT(TRI_IsStringJson(typeJson));

  // type must be identical
  if (! TRI_CheckSameValueJson(typeJson, TRI_LookupObjectJson(rhs, "type"))) {
    return false;
  }

  auto type = Index::type(typeJson->_value._string.data);

  // unique must be identical if present
  TRI_json_t* value = TRI_LookupObjectJson(lhs, "unique");
  if (TRI_IsBooleanJson(value)) {
    if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "unique"))) {
      return false;
    }
  }

  // sparse must be identical if present
  value = TRI_LookupObjectJson(lhs, "sparse");
  if (TRI_IsBooleanJson(value)) {
    if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "sparse"))) {
      return false;
    }
  }

  if (type == IndexType::TRI_IDX_TYPE_GEO1_INDEX) {
    // geoJson must be identical if present
    value = TRI_LookupObjectJson(lhs, "geoJson");
    if (TRI_IsBooleanJson(value)) {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "geoJson"))) {
        return false;
      }
    }
  }
  else if (type == IndexType::TRI_IDX_TYPE_FULLTEXT_INDEX) {
    // minLength
    value = TRI_LookupObjectJson(lhs, "minLength");
    if (TRI_IsNumberJson(value)) {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "minLength"))) {
        return false;
      }
    }
  }
  else if (type == IndexType::TRI_IDX_TYPE_CAP_CONSTRAINT) {
    // size, byteSize
    value = TRI_LookupObjectJson(lhs, "size");
    if (TRI_IsNumberJson(value)) {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "size"))) {
        return false;
      }
    }

    value = TRI_LookupObjectJson(lhs, "byteSize");
    if (TRI_IsNumberJson(value)) {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "byteSize"))) {
        return false;
      }
    }
  }

  // other index types: fields must be identical if present
  value = TRI_LookupObjectJson(lhs, "fields");

  if (TRI_IsArrayJson(value)) {
    if (type == IndexType::TRI_IDX_TYPE_HASH_INDEX) {
      size_t const nv = TRI_LengthArrayJson(value);

      // compare fields in arbitrary order
      TRI_json_t const* r = TRI_LookupObjectJson(rhs, "fields");

      if (! TRI_IsArrayJson(r) ||
          nv != TRI_LengthArrayJson(r)) {
        return false;
      }

      size_t const nr = TRI_LengthArrayJson(r);

      for (size_t i = 0; i < nv; ++i) {
        TRI_json_t const* v = TRI_LookupArrayJson(value, i);

        bool found = false;

        for (size_t j = 0; j < nr; ++j) {
          if (TRI_CheckSameValueJson(v, TRI_LookupArrayJson(r, j))) {
            found = true;
            break;
          }
        }

        if (! found) {
          return false;
        }
      }
    }
    else {
      if (! TRI_CheckSameValueJson(value, TRI_LookupObjectJson(rhs, "fields"))) {
        return false;
      }
    }

  }

  return true;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief return a contextual string for logging
////////////////////////////////////////////////////////////////////////////////

std::string Index::context () const {
  std::ostringstream result;

  result << "index { id: " << id()
         << ", type: " << typeName()
         << ", collection: " << _collection->_vocbase->_name
         << "/" <<  _collection->_info._name << " }";

  return result.str();
}

////////////////////////////////////////////////////////////////////////////////
/// @brief create a JSON representation of the index
/// base functionality (called from derived classes)
////////////////////////////////////////////////////////////////////////////////

triagens::basics::Json Index::toJson (TRI_memory_zone_t* zone) const {
  triagens::basics::Json json(zone, triagens::basics::Json::Object, 4);

  json("id", triagens::basics::Json(zone, std::to_string(_iid)))
      ("type", triagens::basics::Json(zone, typeName()));

  if (dumpFields()) {
    triagens::basics::Json f(zone, triagens::basics::Json::Array, fields().size());

    for (auto const& field : fields()) {
      std::string fieldString;
      TRI_AttributeNamesToString(field, fieldString);
      f.add(triagens::basics::Json(zone, fieldString));
    }

    json("fields", f);
  }

  if (hasSelectivityEstimate()) {
    json("selectivityEstimate", triagens::basics::Json(selectivityEstimate()));
  }

  return json;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for selectivityEstimate
////////////////////////////////////////////////////////////////////////////////

double Index::selectivityEstimate () const {
  THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for selectivityEstimate
////////////////////////////////////////////////////////////////////////////////

int Index::batchInsert (std::vector<struct TRI_doc_mptr_t const*> const*, size_t) {
  THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for postInsert
////////////////////////////////////////////////////////////////////////////////

int Index::postInsert (struct TRI_transaction_collection_s*,
                       struct TRI_doc_mptr_t const*) {
  // do nothing
  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for cleanup
////////////////////////////////////////////////////////////////////////////////

int Index::cleanup () {
  // do nothing
  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for sizeHint
////////////////////////////////////////////////////////////////////////////////

int Index::sizeHint (size_t) {
  // do nothing
  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief default implementation for hasBatchInsert
////////////////////////////////////////////////////////////////////////////////

bool Index::hasBatchInsert () const {
  return false;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief helper function to insert a document into any index type
////////////////////////////////////////////////////////////////////////////////

int Index::fillElement(std::function<TRI_index_element_t* ()> allocate,
                       std::vector<TRI_index_element_t*>& elements,
                       TRI_doc_mptr_t const* document,
                       std::vector<TRI_shape_pid_t> const& paths,
                       bool const sparse) {
  TRI_ASSERT(document != nullptr);
  TRI_ASSERT_EXPENSIVE(document->getDataPtr() != nullptr);   // ONLY IN INDEX, PROTECTED by RUNTIME

  TRI_shaped_json_t shapedJson;

  TRI_EXTRACT_SHAPED_JSON_MARKER(shapedJson, document->getDataPtr());  // ONLY IN INDEX, PROTECTED by RUNTIME

  if (shapedJson._sid == TRI_SHAPE_ILLEGAL) {
    LOG_WARNING("encountered invalid marker with shape id 0");

    return TRI_ERROR_INTERNAL;
  }

  char const* ptr = document->getShapedJsonPtr();  // ONLY IN INDEX, PROTECTED by RUNTIME

  size_t const n = paths.size();

  TRI_index_element_t* element = allocate();
  if (element == nullptr) {
    return TRI_ERROR_OUT_OF_MEMORY;
  }
  element->document(const_cast<TRI_doc_mptr_t*>(document));
  TRI_shaped_sub_t* subObjects = element->subObjects();
  for (size_t j = 0; j < n; ++j) {
    TRI_shape_pid_t path = paths[j];

    // ..........................................................................
    // Determine if document has that particular shape
    // ..........................................................................

    TRI_shape_access_t const* acc = _collection->getShaper()->findAccessor(shapedJson._sid, path);  // ONLY IN INDEX, PROTECTED by RUNTIME

    if (acc == nullptr || acc->_resultSid == TRI_SHAPE_ILLEGAL) {
      // OK, the document does not contain the attributed needed by
      // the index, are we sparse?
      subObjects[j]._sid = BasicShapes::TRI_SHAPE_SID_NULL;

      if (sparse) {
        // no need to continue
        // Free this element and return.
        TRI_index_element_t::free(element);
        return TRI_ERROR_NO_ERROR;
        // return TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING;
      }
      continue;
    }

    // ..........................................................................
    // Extract the field
    // ..........................................................................

    TRI_shaped_json_t shapedObject;
    if (! TRI_ExecuteShapeAccessor(acc, &shapedJson, &shapedObject)) {
      return TRI_ERROR_INTERNAL;
    }

    if (shapedObject._sid == BasicShapes::TRI_SHAPE_SID_NULL) {
      if (sparse) {
        // no need to continue
        // Free this element and return.
        TRI_index_element_t::free(element);
        return TRI_ERROR_NO_ERROR;
        // return TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING;
      }
    }

    // .........................................................................
    // Store the field
    // .........................................................................

    TRI_FillShapedSub(&subObjects[j], &shapedObject, ptr);
  }
  elements.push_back(element);

  return TRI_ERROR_NO_ERROR;
}

namespace triagens {
  namespace arango {

////////////////////////////////////////////////////////////////////////////////
/// @brief append the index description to an output stream
////////////////////////////////////////////////////////////////////////////////

    std::ostream& operator<< (std::ostream& stream,
                              Index const* index) {
      stream << index->context();
      return stream;
    }

////////////////////////////////////////////////////////////////////////////////
/// @brief append the index description to an output stream
////////////////////////////////////////////////////////////////////////////////

    std::ostream& operator<< (std::ostream& stream,
                              Index const& index) {
      stream << index.context();
      return stream;
    }

  }
}

// -----------------------------------------------------------------------------
// --SECTION--                                                       END-OF-FILE
// -----------------------------------------------------------------------------

// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:
<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.h"

#include "tensorflow/compiler/xla/map_util.h"
#include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
#include "tensorflow/compiler/xla/service/cpu/shape_partition.h"
#include "tensorflow/compiler/xla/service/hlo_computation.h"
#include "tensorflow/compiler/xla/service/hlo_instruction.h"
#include "tensorflow/compiler/xla/service/hlo_opcode.h"
#include "tensorflow/compiler/xla/service/logical_buffer.h"
#include "tensorflow/compiler/xla/service/tuple_points_to_analysis.h"
#include "tensorflow/compiler/xla/types.h"
#include "tensorflow/compiler/xla/util.h"
#include "tensorflow/core/lib/strings/strcat.h"

namespace xla {
namespace cpu {

StatusOr<bool> ParallelizationPreparation::Run(HloModule* module) {
  XLA_VLOG_LINES(2, "ParallelizationPreparation ENTRY");
  XLA_VLOG_LINES(2, module->ToString());

  bool changed = false;
  TF_ASSIGN_OR_RETURN(changed, RunParallelTaskAssignment(module));

  HloComputation* entry_computation = module->entry_computation();
  std::unordered_set<HloInstruction*> outlined;
  std::vector<HloInstruction*> instructions_to_outline;
  for (HloInstruction* instruction :
       entry_computation->MakeInstructionPostOrder()) {
    // If the instruction has been outlined, it no longer exists and we must not
    // dereference it.
    if (outlined.count(instruction) > 0) {
      continue;
    }

    // Skip parameters and constants, there is nothing to parallelize.
    if (instruction->opcode() == HloOpcode::kParameter ||
        instruction->opcode() == HloOpcode::kConstant) {
      continue;
    }

    // Outline 'instruction' in isolation if it was assigned parallel tasks.
    if (OutlineParallelizableInstruction(instruction)) {
      outlined.insert(instruction);
      changed = true;
      continue;
    }

    instructions_to_outline.clear();
    HloInstruction* outline_candidate = instruction;
    instructions_to_outline.push_back(outline_candidate);
    bool all_bitcasts = outline_candidate->opcode() == HloOpcode::kBitcast;

    // Outline sole users with the current instruction.
    while (CanOutlineWithUser(outline_candidate)) {
      HloInstruction* prior_candidate = outline_candidate;
      outline_candidate = *outline_candidate->users().begin();
      all_bitcasts |= outline_candidate->opcode() == HloOpcode::kBitcast;
      if (std::any_of(outline_candidate->operands().begin(),
                      outline_candidate->operands().end(),
                      [&](const HloInstruction* operand) {
                        // Do not consider any candidates which have operands
                        // other than the prior candidate, constants or
                        // parameters. Otherwise, we'd increase the fan-in which
                        // would reduce parallelism.
                        return operand->opcode() != HloOpcode::kParameter &&
                               operand->opcode() != HloOpcode::kConstant &&
                               operand != prior_candidate;
                      })) {
        break;
      }
      instructions_to_outline.push_back(outline_candidate);
    }
    // If all instructions in the outline candidates are a bitcast, then create
    // a copy at the head of the bitcasts and include it in the outlined
    // instructions. The underlying problem is that a computation which forwards
    // a parameter buffer to the output is not properly handled by the backends
    // or analysis.
    //
    // This would be better handled by being smarter about choosing outline
    // candidates in the first place.
    if (all_bitcasts) {
      // 'head' is the first instruction in the chain of bitcasts.
      HloInstruction* head = instructions_to_outline[0];
      HloInstruction* head_operand = head->mutable_operand(0);
      HloInstruction* copy =
          entry_computation->AddInstruction(HloInstruction::CreateUnary(
              head_operand->shape(), HloOpcode::kCopy, head_operand));
      TF_RETURN_IF_ERROR(head->ReplaceOperandWith(0, copy));
      instructions_to_outline.insert(instructions_to_outline.begin(), copy);
    }

    outlined.insert(instructions_to_outline.begin(),
                    instructions_to_outline.end());

    // Optimization to avoid replacing a single existing kCall with another
    // kCall that just calls the first one.
    if (instructions_to_outline.size() == 1 &&
        instructions_to_outline[0]->opcode() == HloOpcode::kCall) {
      continue;
    }

    module->OutlineExpressionFromComputation(
        instructions_to_outline,
        tensorflow::strings::StrCat("pp_", instruction->name()),
        entry_computation);
    changed = true;
  }

  TF_ASSIGN_OR_RETURN(auto points_to_analysis,
                      TuplePointsToAnalysis::Run(module));
  for (auto& computation : module->computations()) {
    if (computation->IsFusionComputation()) {
      continue;
    }
    HloInstruction* root = computation->root_instruction();
    // Copy root instruction if it does not define its own top-level buffer.
    // TODO(b/32885001) Remove these copies (at least for the unambiguous case).
    // TODO(b/32885001) Perform shallow copy if root value is a tuple.
    if (!points_to_analysis->InstructionDefinesBufferAtIndex(root,
                                                             /*index=*/{})) {
      HloInstruction* copy = computation->AddInstruction(
          HloInstruction::CreateUnary(root->shape(), HloOpcode::kCopy, root));
      computation->set_root_instruction(copy);
      changed = true;
    }
  }

  XLA_VLOG_LINES(2, "ParallelizationPreparation EXIT");
  XLA_VLOG_LINES(2, module->ToString());
  return changed;
}

StatusOr<bool> ParallelizationPreparation::RunParallelTaskAssignment(
    HloModule* module) {
  VLOG(1) << "RunParallelTaskAssignment max_parallelism_: " << max_parallelism_;
  bool changed = false;
  // Run cost analysis on entry computation.
  HloCostAnalysis cost_analysis(shape_size_);
  HloComputation* computation = module->entry_computation();
  Status cost_status = computation->root_instruction()->Accept(&cost_analysis);
  for (auto& instruction : computation->instructions()) {
    // Currently, we do not assign parallel tasks to instructions with at least
    // one of the following properties:
    // *) Internal threading (library calls to kConv, kDot, and kCustomCall).
    // *) Emit custom loops (kSelectAndScatter, FusionKind::kTransposeDot).
    // *) Tuple-shaped.
    // TODO(b/27458679) Parallelize instructions which are skipped here.
    if (instruction->opcode() == HloOpcode::kParameter ||
        instruction->opcode() == HloOpcode::kConstant ||
        instruction->opcode() == HloOpcode::kCall ||
        instruction->opcode() == HloOpcode::kCustomCall ||
        instruction->opcode() == HloOpcode::kSelectAndScatter ||
        (instruction->opcode() == HloOpcode::kConvolution &&
         PotentiallyImplementedAsEigenConvolution(*instruction)) ||
        PotentiallyImplementedAsEigenDot(*instruction) ||
        (instruction->opcode() == HloOpcode::kFusion &&
         instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) ||
        ShapeUtil::IsTuple(instruction->shape())) {
      continue;
    }

    // Calculate target parallel task count in [1, max_parallelism_].
    const int64 target_parallel_task_count = GetTargetParallelTaskCount(
        cost_status.ok() ? &cost_analysis : nullptr, instruction.get());
    if (target_parallel_task_count == 1) {
      continue;
    }

    // Assign feasible dimension partitions (based on actual dimension sizes).
    auto dim_partition_counts = ShapePartitionAssigner(instruction->shape())
                                    .Run(target_parallel_task_count);
    const int64 total_partition_count =
        ShapePartitionAssigner::GetTotalPartitionCount(dim_partition_counts);
    if (total_partition_count <= 1) {
      // Feasible partition calculation resulting in no partitioning, so skip.
      continue;
    }
    VLOG(2) << "Assigning parallel task count: " << total_partition_count
            << " to instruction: " << instruction->name();
    // Map 'instruction' to assigned dimension partitioning.
    instruction->set_outer_dimension_partitions(dim_partition_counts);
  }

  return changed;
}

int64 ParallelizationPreparation::GetTargetParallelTaskCount(
    const HloCostAnalysis* cost_analysis, HloInstruction* instruction) {
  // Default to a simple cost model based on hlo size and typical L2 cache size.
  // Note that 'cost_analysis' can be 'nullptr' if HloCostAnalysis returns an
  // error status (likely because HLOs like CustomCall are not yet implemented
  // in the HloCostAnalysis).
  int64 instruction_cost = shape_size_(instruction->shape());
  int64 min_cost_per_thread = 256LL << 10;  // 256KB L2 Cache size.
  if (cost_analysis != nullptr) {
    // Calculate the instruction cost in cycles.
    // TODO(29630486) Improve on this linear cost model.
    // Consider making 'min_cost_per_thread' be a function of the target
    // bandwidth limit for instructions with low arithmetic complexity.
    instruction_cost = 1 * cost_analysis->flop_count(*instruction) +
                       2 * cost_analysis->transcendental_count(*instruction) +
                       10 * cost_analysis->bytes_accessed(*instruction);
    // Minimum per-thread cost is 100us of work on a 2GHz core.
    min_cost_per_thread = 100000;
  }
  // Return target parallel task count in [1, max_parallelism_].
  return std::min(max_parallelism_,
                  std::max(1LL, instruction_cost / min_cost_per_thread));
}

bool ParallelizationPreparation::OutlineParallelizableInstruction(
    HloInstruction* instruction) {
  if (instruction->outer_dimension_partitions().empty()) {
    return false;
  }
  // Store dimension partition counts before outlining (which clones
  // 'instruction').
  std::vector<int64> dim_partition_counts =
      instruction->outer_dimension_partitions();
  // Outline 'instruction' in its own sub-computation.
  HloModule* module = instruction->parent()->parent();
  auto* call = module->OutlineExpressionFromComputation(
      {instruction}, tensorflow::strings::StrCat("pp_", instruction->name()),
      module->entry_computation());
  // Map previously assigned 'dim_partition_counts' to cloned root instruction.
  VLOG(1) << "Outlining parallelizable"
          << " caller: " << call->name()
          << " callee: " << call->to_apply()->root_instruction()->name();
  call->to_apply()->root_instruction()->set_outer_dimension_partitions(
      dim_partition_counts);
  return true;
}

bool ParallelizationPreparation::CanOutlineWithUser(
    HloInstruction* instruction) {
  if (instruction->users().size() != 1) {
    // Do not outline 'instruction' with multiple users.
    return false;
  }
  if (AssignedParallelTasks(instruction) ||
      AssignedParallelTasks(*instruction->users().begin())) {
    // Do not outline if 'instruction' (or user) were assigned parallel tasks.
    return false;
  }
  return true;
}

bool ParallelizationPreparation::AssignedParallelTasks(
    HloInstruction* instruction) {
  return !instruction->outer_dimension_partitions().empty() ||
         (instruction->opcode() == HloOpcode::kCall &&
          !instruction->to_apply()
               ->root_instruction()
               ->outer_dimension_partitions()
               .empty());
}

}  // namespace cpu
}  // namespace xla
<commit_msg>[XLA:CPU] One character code review (Update 'all_bitcasts' variable correctly).<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.h"

#include "tensorflow/compiler/xla/map_util.h"
#include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
#include "tensorflow/compiler/xla/service/cpu/shape_partition.h"
#include "tensorflow/compiler/xla/service/hlo_computation.h"
#include "tensorflow/compiler/xla/service/hlo_instruction.h"
#include "tensorflow/compiler/xla/service/hlo_opcode.h"
#include "tensorflow/compiler/xla/service/logical_buffer.h"
#include "tensorflow/compiler/xla/service/tuple_points_to_analysis.h"
#include "tensorflow/compiler/xla/types.h"
#include "tensorflow/compiler/xla/util.h"
#include "tensorflow/core/lib/strings/strcat.h"

namespace xla {
namespace cpu {

StatusOr<bool> ParallelizationPreparation::Run(HloModule* module) {
  XLA_VLOG_LINES(2, "ParallelizationPreparation ENTRY");
  XLA_VLOG_LINES(2, module->ToString());

  bool changed = false;
  TF_ASSIGN_OR_RETURN(changed, RunParallelTaskAssignment(module));

  HloComputation* entry_computation = module->entry_computation();
  std::unordered_set<HloInstruction*> outlined;
  std::vector<HloInstruction*> instructions_to_outline;
  for (HloInstruction* instruction :
       entry_computation->MakeInstructionPostOrder()) {
    // If the instruction has been outlined, it no longer exists and we must not
    // dereference it.
    if (outlined.count(instruction) > 0) {
      continue;
    }

    // Skip parameters and constants, there is nothing to parallelize.
    if (instruction->opcode() == HloOpcode::kParameter ||
        instruction->opcode() == HloOpcode::kConstant) {
      continue;
    }

    // Outline 'instruction' in isolation if it was assigned parallel tasks.
    if (OutlineParallelizableInstruction(instruction)) {
      outlined.insert(instruction);
      changed = true;
      continue;
    }

    instructions_to_outline.clear();
    HloInstruction* outline_candidate = instruction;
    instructions_to_outline.push_back(outline_candidate);
    bool all_bitcasts = outline_candidate->opcode() == HloOpcode::kBitcast;

    // Outline sole users with the current instruction.
    while (CanOutlineWithUser(outline_candidate)) {
      HloInstruction* prior_candidate = outline_candidate;
      outline_candidate = *outline_candidate->users().begin();
      all_bitcasts &= outline_candidate->opcode() == HloOpcode::kBitcast;
      if (std::any_of(outline_candidate->operands().begin(),
                      outline_candidate->operands().end(),
                      [&](const HloInstruction* operand) {
                        // Do not consider any candidates which have operands
                        // other than the prior candidate, constants or
                        // parameters. Otherwise, we'd increase the fan-in which
                        // would reduce parallelism.
                        return operand->opcode() != HloOpcode::kParameter &&
                               operand->opcode() != HloOpcode::kConstant &&
                               operand != prior_candidate;
                      })) {
        break;
      }
      instructions_to_outline.push_back(outline_candidate);
    }
    // If all instructions in the outline candidates are a bitcast, then create
    // a copy at the head of the bitcasts and include it in the outlined
    // instructions. The underlying problem is that a computation which forwards
    // a parameter buffer to the output is not properly handled by the backends
    // or analysis.
    //
    // This would be better handled by being smarter about choosing outline
    // candidates in the first place.
    if (all_bitcasts) {
      // 'head' is the first instruction in the chain of bitcasts.
      HloInstruction* head = instructions_to_outline[0];
      HloInstruction* head_operand = head->mutable_operand(0);
      HloInstruction* copy =
          entry_computation->AddInstruction(HloInstruction::CreateUnary(
              head_operand->shape(), HloOpcode::kCopy, head_operand));
      TF_RETURN_IF_ERROR(head->ReplaceOperandWith(0, copy));
      instructions_to_outline.insert(instructions_to_outline.begin(), copy);
    }

    outlined.insert(instructions_to_outline.begin(),
                    instructions_to_outline.end());

    // Optimization to avoid replacing a single existing kCall with another
    // kCall that just calls the first one.
    if (instructions_to_outline.size() == 1 &&
        instructions_to_outline[0]->opcode() == HloOpcode::kCall) {
      continue;
    }

    module->OutlineExpressionFromComputation(
        instructions_to_outline,
        tensorflow::strings::StrCat("pp_", instruction->name()),
        entry_computation);
    changed = true;
  }

  TF_ASSIGN_OR_RETURN(auto points_to_analysis,
                      TuplePointsToAnalysis::Run(module));
  for (auto& computation : module->computations()) {
    if (computation->IsFusionComputation()) {
      continue;
    }
    HloInstruction* root = computation->root_instruction();
    // Copy root instruction if it does not define its own top-level buffer.
    // TODO(b/32885001) Remove these copies (at least for the unambiguous case).
    // TODO(b/32885001) Perform shallow copy if root value is a tuple.
    if (!points_to_analysis->InstructionDefinesBufferAtIndex(root,
                                                             /*index=*/{})) {
      HloInstruction* copy = computation->AddInstruction(
          HloInstruction::CreateUnary(root->shape(), HloOpcode::kCopy, root));
      computation->set_root_instruction(copy);
      changed = true;
    }
  }

  XLA_VLOG_LINES(2, "ParallelizationPreparation EXIT");
  XLA_VLOG_LINES(2, module->ToString());
  return changed;
}

StatusOr<bool> ParallelizationPreparation::RunParallelTaskAssignment(
    HloModule* module) {
  VLOG(1) << "RunParallelTaskAssignment max_parallelism_: " << max_parallelism_;
  bool changed = false;
  // Run cost analysis on entry computation.
  HloCostAnalysis cost_analysis(shape_size_);
  HloComputation* computation = module->entry_computation();
  Status cost_status = computation->root_instruction()->Accept(&cost_analysis);
  for (auto& instruction : computation->instructions()) {
    // Currently, we do not assign parallel tasks to instructions with at least
    // one of the following properties:
    // *) Internal threading (library calls to kConv, kDot, and kCustomCall).
    // *) Emit custom loops (kSelectAndScatter, FusionKind::kTransposeDot).
    // *) Tuple-shaped.
    // TODO(b/27458679) Parallelize instructions which are skipped here.
    if (instruction->opcode() == HloOpcode::kParameter ||
        instruction->opcode() == HloOpcode::kConstant ||
        instruction->opcode() == HloOpcode::kCall ||
        instruction->opcode() == HloOpcode::kCustomCall ||
        instruction->opcode() == HloOpcode::kSelectAndScatter ||
        (instruction->opcode() == HloOpcode::kConvolution &&
         PotentiallyImplementedAsEigenConvolution(*instruction)) ||
        PotentiallyImplementedAsEigenDot(*instruction) ||
        (instruction->opcode() == HloOpcode::kFusion &&
         instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) ||
        ShapeUtil::IsTuple(instruction->shape())) {
      continue;
    }

    // Calculate target parallel task count in [1, max_parallelism_].
    const int64 target_parallel_task_count = GetTargetParallelTaskCount(
        cost_status.ok() ? &cost_analysis : nullptr, instruction.get());
    if (target_parallel_task_count == 1) {
      continue;
    }

    // Assign feasible dimension partitions (based on actual dimension sizes).
    auto dim_partition_counts = ShapePartitionAssigner(instruction->shape())
                                    .Run(target_parallel_task_count);
    const int64 total_partition_count =
        ShapePartitionAssigner::GetTotalPartitionCount(dim_partition_counts);
    if (total_partition_count <= 1) {
      // Feasible partition calculation resulting in no partitioning, so skip.
      continue;
    }
    VLOG(2) << "Assigning parallel task count: " << total_partition_count
            << " to instruction: " << instruction->name();
    // Map 'instruction' to assigned dimension partitioning.
    instruction->set_outer_dimension_partitions(dim_partition_counts);
  }

  return changed;
}

int64 ParallelizationPreparation::GetTargetParallelTaskCount(
    const HloCostAnalysis* cost_analysis, HloInstruction* instruction) {
  // Default to a simple cost model based on hlo size and typical L2 cache size.
  // Note that 'cost_analysis' can be 'nullptr' if HloCostAnalysis returns an
  // error status (likely because HLOs like CustomCall are not yet implemented
  // in the HloCostAnalysis).
  int64 instruction_cost = shape_size_(instruction->shape());
  int64 min_cost_per_thread = 256LL << 10;  // 256KB L2 Cache size.
  if (cost_analysis != nullptr) {
    // Calculate the instruction cost in cycles.
    // TODO(29630486) Improve on this linear cost model.
    // Consider making 'min_cost_per_thread' be a function of the target
    // bandwidth limit for instructions with low arithmetic complexity.
    instruction_cost = 1 * cost_analysis->flop_count(*instruction) +
                       2 * cost_analysis->transcendental_count(*instruction) +
                       10 * cost_analysis->bytes_accessed(*instruction);
    // Minimum per-thread cost is 100us of work on a 2GHz core.
    min_cost_per_thread = 100000;
  }
  // Return target parallel task count in [1, max_parallelism_].
  return std::min(max_parallelism_,
                  std::max(1LL, instruction_cost / min_cost_per_thread));
}

bool ParallelizationPreparation::OutlineParallelizableInstruction(
    HloInstruction* instruction) {
  if (instruction->outer_dimension_partitions().empty()) {
    return false;
  }
  // Store dimension partition counts before outlining (which clones
  // 'instruction').
  std::vector<int64> dim_partition_counts =
      instruction->outer_dimension_partitions();
  // Outline 'instruction' in its own sub-computation.
  HloModule* module = instruction->parent()->parent();
  auto* call = module->OutlineExpressionFromComputation(
      {instruction}, tensorflow::strings::StrCat("pp_", instruction->name()),
      module->entry_computation());
  // Map previously assigned 'dim_partition_counts' to cloned root instruction.
  VLOG(1) << "Outlining parallelizable"
          << " caller: " << call->name()
          << " callee: " << call->to_apply()->root_instruction()->name();
  call->to_apply()->root_instruction()->set_outer_dimension_partitions(
      dim_partition_counts);
  return true;
}

bool ParallelizationPreparation::CanOutlineWithUser(
    HloInstruction* instruction) {
  if (instruction->users().size() != 1) {
    // Do not outline 'instruction' with multiple users.
    return false;
  }
  if (AssignedParallelTasks(instruction) ||
      AssignedParallelTasks(*instruction->users().begin())) {
    // Do not outline if 'instruction' (or user) were assigned parallel tasks.
    return false;
  }
  return true;
}

bool ParallelizationPreparation::AssignedParallelTasks(
    HloInstruction* instruction) {
  return !instruction->outer_dimension_partitions().empty() ||
         (instruction->opcode() == HloOpcode::kCall &&
          !instruction->to_apply()
               ->root_instruction()
               ->outer_dimension_partitions()
               .empty());
}

}  // namespace cpu
}  // namespace xla
<|endoftext|>


<commit_before>// RUN: %clangxx_tsan -O1 %s -o %t && %deflake %run %t | FileCheck %s
#include <pthread.h>
#include <stdlib.h>
#include <stdio.h>
#include "test.h"

void *Thread1(void *p) {
  *(int*)p = 42;
  return 0;
}

void *Thread2(void *p) {
  *(int*)p = 44;
  return 0;
}

void *alloc() {
  return malloc(99);
}

void *AllocThread(void* arg) {
  return alloc();
}

int main() {
  void *p = 0;
  pthread_t t[2];
  pthread_create(&t[0], 0, AllocThread, 0);
  pthread_join(t[0], &p);
  print_address("addr=", 1, p);
  pthread_create(&t[0], 0, Thread1, (char*)p + 16);
  pthread_create(&t[1], 0, Thread2, (char*)p + 16);
  pthread_join(t[0], 0);
  pthread_join(t[1], 0);
  return 0;
}

// CHECK: addr=[[ADDR:0x[0-9,a-f]+]]
// CHECK: WARNING: ThreadSanitizer: data race
// ...
// CHECK: Location is heap block of size 99 at [[ADDR]] allocated by thread T1:
// CHCEK:     #0 malloc
// CHECK:     #{{1|2}} alloc
// CHECK:     #{{2|3}} AllocThread
// ...
// CHECK:   Thread T1 (tid={{.*}}, finished) created by main thread at:
// CHECK:     #0 pthread_create
// CHECK:     #1 main
<commit_msg>Fix typos in tests. NFC.<commit_after>// RUN: %clangxx_tsan -O1 %s -o %t && %deflake %run %t | FileCheck %s
#include <pthread.h>
#include <stdlib.h>
#include <stdio.h>
#include "test.h"

void *Thread1(void *p) {
  *(int*)p = 42;
  return 0;
}

void *Thread2(void *p) {
  *(int*)p = 44;
  return 0;
}

void *alloc() {
  return malloc(99);
}

void *AllocThread(void* arg) {
  return alloc();
}

int main() {
  void *p = 0;
  pthread_t t[2];
  pthread_create(&t[0], 0, AllocThread, 0);
  pthread_join(t[0], &p);
  print_address("addr=", 1, p);
  pthread_create(&t[0], 0, Thread1, (char*)p + 16);
  pthread_create(&t[1], 0, Thread2, (char*)p + 16);
  pthread_join(t[0], 0);
  pthread_join(t[1], 0);
  return 0;
}

// CHECK: addr=[[ADDR:0x[0-9,a-f]+]]
// CHECK: WARNING: ThreadSanitizer: data race
// ...
// CHECK: Location is heap block of size 99 at [[ADDR]] allocated by thread T1:
// CHECK:     #0 malloc
// CHECK:     #{{1|2}} alloc
// CHECK:     #{{2|3}} AllocThread
// ...
// CHECK:   Thread T1 (tid={{.*}}, finished) created by main thread at:
// CHECK:     #0 pthread_create
// CHECK:     #1 main
<|endoftext|>


<commit_before>#include <babylon/actions/valueactions/interpolate_value_action.h>

#include <nlohmann/json.hpp>

#include <babylon/actions/action_manager.h>
#include <babylon/animations/animation.h>
#include <babylon/animations/animation_value.h>
#include <babylon/animations/ianimatable.h>
#include <babylon/animations/ianimation_key.h>
#include <babylon/audio/sound.h>
#include <babylon/core/logging.h>
#include <babylon/engines/scene.h>

namespace BABYLON {

InterpolateValueAction::InterpolateValueAction(unsigned int iTriggerOptions,
                                               const IAnimatablePtr& target,
                                               const std::string& iPropertyPath,
                                               AnimationValue* iValue, int iDuration,
                                               Condition* condition, bool iStopOtherAnimations,
                                               const std::function<void()>& iOnInterpolationDone)
    : Action(iTriggerOptions, condition)
    , propertyPath{iPropertyPath}
    , value{iValue}
    , duration{iDuration}
    , stopOtherAnimations{iStopOtherAnimations}
    , onInterpolationDone{iOnInterpolationDone}
    , _target{target}
    , _effectiveTarget{target}
{
}

InterpolateValueAction::~InterpolateValueAction() = default;

void InterpolateValueAction::_prepare()
{
  _effectiveTarget = _getEffectiveTarget(_effectiveTarget, propertyPath);
  _property        = _getProperty(propertyPath);
}

void InterpolateValueAction::execute(const IActionEventPtr& /*evt*/)
{
  auto scene = _actionManager->getScene();
  std::vector<IAnimationKey> keys{IAnimationKey(0, _effectiveTarget->getProperty({_property})), //
                                  IAnimationKey(100, *value)};

  auto dataType = value->animationType();
  if (!dataType.has_value()) {
    BABYLON_LOG_WARN("InterpolateValueAction", "InterpolateValueAction: Unsupported type")
  }

  auto animation = Animation::New("InterpolateValueAction", _property,
                                  static_cast<size_t>(100 * (1000.f / float(duration))),
                                  dataType.value(), Animation::ANIMATIONLOOPMODE_CONSTANT);

  animation->setKeys(keys);

  if (stopOtherAnimations) {
    scene->stopAnimation(_effectiveTarget);
  }

  const auto wrapper = [this]() {
    onInterpolationDoneObservable.notifyObservers(this);
    if (onInterpolationDone) {
      onInterpolationDone();
    }
  };

  scene->beginDirectAnimation(_effectiveTarget, {animation}, 0, 100, false, 1, wrapper);
}

json InterpolateValueAction::serialize(json& /*parent*/) const
{
  return nullptr;
}

} // end of namespace BABYLON
<commit_msg>const correctness<commit_after>#include <babylon/actions/valueactions/interpolate_value_action.h>

#include <nlohmann/json.hpp>

#include <babylon/actions/action_manager.h>
#include <babylon/animations/animation.h>
#include <babylon/animations/animation_value.h>
#include <babylon/animations/ianimatable.h>
#include <babylon/animations/ianimation_key.h>
#include <babylon/audio/sound.h>
#include <babylon/core/logging.h>
#include <babylon/engines/scene.h>

namespace BABYLON {

InterpolateValueAction::InterpolateValueAction(unsigned int iTriggerOptions,
                                               const IAnimatablePtr& target,
                                               const std::string& iPropertyPath,
                                               AnimationValue* iValue, int iDuration,
                                               Condition* condition, bool iStopOtherAnimations,
                                               const std::function<void()>& iOnInterpolationDone)
    : Action(iTriggerOptions, condition)
    , propertyPath{iPropertyPath}
    , value{iValue}
    , duration{iDuration}
    , stopOtherAnimations{iStopOtherAnimations}
    , onInterpolationDone{iOnInterpolationDone}
    , _target{target}
    , _effectiveTarget{target}
{
}

InterpolateValueAction::~InterpolateValueAction() = default;

void InterpolateValueAction::_prepare()
{
  _effectiveTarget = _getEffectiveTarget(_effectiveTarget, propertyPath);
  _property        = _getProperty(propertyPath);
}

void InterpolateValueAction::execute(const IActionEventPtr& /*evt*/)
{
  const auto scene = _actionManager->getScene();
  std::vector<IAnimationKey> keys{IAnimationKey(0, _effectiveTarget->getProperty({_property})), //
                                  IAnimationKey(100, *value)};

  const auto dataType = value->animationType();
  if (!dataType.has_value()) {
    BABYLON_LOG_WARN("InterpolateValueAction", "InterpolateValueAction: Unsupported type")
  }

  const auto animation = Animation::New("InterpolateValueAction", _property,
                                        static_cast<size_t>(100.f * (1000.f / float(duration))),
                                        dataType.value(), Animation::ANIMATIONLOOPMODE_CONSTANT);

  animation->setKeys(keys);

  if (stopOtherAnimations) {
    scene->stopAnimation(_effectiveTarget);
  }

  const auto wrapper = [this]() {
    onInterpolationDoneObservable.notifyObservers(this);
    if (onInterpolationDone) {
      onInterpolationDone();
    }
  };

  scene->beginDirectAnimation(_effectiveTarget, {animation}, 0, 100, false, 1, wrapper);
}

json InterpolateValueAction::serialize(json& /*parent*/) const
{
  return nullptr;
}

} // end of namespace BABYLON
<|endoftext|>


<commit_before>#include <agency/future.hpp>
#include <agency/new_executor_traits.hpp>
#include <iostream>
#include <cassert>

#include "test_executors.hpp"

template<class Executor>
void test()
{
  using executor_type = Executor;
  using traits = agency::new_executor_traits<executor_type>;

  {
    executor_type exec;

    auto void_future1 = traits::when_all(exec);
    auto void_future2 = traits::when_all(exec, void_future1);

    void_future2.get();
    assert(exec.valid());
  }

  {
    executor_type exec;

    auto int_ready   = traits::template make_ready_future<int>(exec, 13);
    auto float_ready = traits::template make_ready_future<float>(exec, 7.f);

    auto int_float_fut = traits::when_all(exec, int_ready, float_ready);

    auto int_float = int_float_fut.get();

    assert(std::get<0>(int_float) == 13);
    assert(std::get<1>(int_float) == 7.f);
    assert(exec.valid());
  }

  {
    executor_type exec;

    auto int_ready   = agency::detail::make_ready_future(13);
    auto void_ready  = agency::detail::make_ready_future();
    auto float_ready = agency::detail::make_ready_future(7.f);

    auto int_float_fut = traits::when_all(exec, int_ready, void_ready, float_ready);

    auto int_float = int_float_fut.get();

    assert(std::get<0>(int_float) == 13);
    assert(std::get<1>(int_float) == 7.f);
    assert(exec.valid());
  }
}

int main()
{
  using namespace test_executors;

  test<empty_executor>();
  test<single_agent_when_all_execute_and_select_executor>();
  test<multi_agent_when_all_execute_and_select_executor>();
  test<single_agent_then_execute_executor>();
  test<when_all_executor>();
  test<multi_agent_execute_returning_user_defined_container_executor>();
  test<multi_agent_execute_returning_default_container_executor>();

  std::cout << "OK" << std::endl;

  return 0;
}

<commit_msg>Test multi_agent_async_execute_executor_returning_user_specified_container_executor with test_when_all.cpp<commit_after>#include <agency/future.hpp>
#include <agency/new_executor_traits.hpp>
#include <iostream>
#include <cassert>

#include "test_executors.hpp"

template<class Executor>
void test()
{
  using executor_type = Executor;
  using traits = agency::new_executor_traits<executor_type>;

  {
    executor_type exec;

    auto void_future1 = traits::when_all(exec);
    auto void_future2 = traits::when_all(exec, void_future1);

    void_future2.get();
    assert(exec.valid());
  }

  {
    executor_type exec;

    auto int_ready   = traits::template make_ready_future<int>(exec, 13);
    auto float_ready = traits::template make_ready_future<float>(exec, 7.f);

    auto int_float_fut = traits::when_all(exec, int_ready, float_ready);

    auto int_float = int_float_fut.get();

    assert(std::get<0>(int_float) == 13);
    assert(std::get<1>(int_float) == 7.f);
    assert(exec.valid());
  }

  {
    executor_type exec;

    auto int_ready   = agency::detail::make_ready_future(13);
    auto void_ready  = agency::detail::make_ready_future();
    auto float_ready = agency::detail::make_ready_future(7.f);

    auto int_float_fut = traits::when_all(exec, int_ready, void_ready, float_ready);

    auto int_float = int_float_fut.get();

    assert(std::get<0>(int_float) == 13);
    assert(std::get<1>(int_float) == 7.f);
    assert(exec.valid());
  }
}

int main()
{
  using namespace test_executors;

  test<empty_executor>();
  test<single_agent_when_all_execute_and_select_executor>();
  test<multi_agent_when_all_execute_and_select_executor>();
  test<single_agent_then_execute_executor>();

  test<when_all_executor>();

  test<multi_agent_execute_returning_user_defined_container_executor>();
  test<multi_agent_execute_returning_default_container_executor>();
  test<multi_agent_execute_returning_void_executor>();

  test<multi_agent_async_execute_returning_user_defined_container_executor>();

  std::cout << "OK" << std::endl;

  return 0;
}

<|endoftext|>


<commit_before>// Filename: mayaToEgg.cxx
// Created by:  drose (15Feb00)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license.  You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////

#include "mayaToEgg.h"
#include "mayaToEggConverter.h"
#include "config_mayaegg.h"
#include "config_maya.h"  // for maya_cat

////////////////////////////////////////////////////////////////////
//     Function: MayaToEgg::Constructor
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
MayaToEgg::
MayaToEgg() :
  SomethingToEgg("Maya", ".mb")
{
  add_path_replace_options();
  add_path_store_options();
  add_animation_options();
  add_units_options();
  add_normals_options();
  add_transform_options();

  set_program_description
    ("This program converts Maya model files to egg.  Nothing fancy yet.");

  add_option
    ("p", "", 0,
     "Generate polygon output only.  Tesselate all NURBS surfaces to "
     "polygons via the built-in Maya tesselator.  The tesselation will "
     "be based on the tolerance factor given by -ptol.",
     &MayaToEgg::dispatch_none, &_polygon_output);

  add_option
    ("ptol", "tolerance", 0,
     "Specify the fit tolerance for Maya polygon tesselation.  The smaller "
     "the number, the more polygons will be generated.  The default is "
     "0.01.",
     &MayaToEgg::dispatch_double, NULL, &_polygon_tolerance);

  add_option
    ("notrans", "", 0,
     "Don't convert explicit DAG transformations given in the Maya file.  "
     "Instead, convert all vertices to world space and write the file as "
     "one big transform space.  Using this option doesn't change the "
     "position of objects in the scene, just the number of explicit "
     "transforms appearing in the resulting egg file.",
     &MayaToEgg::dispatch_none, &_ignore_transforms);

  add_option
    ("v", "", 0,
     "Increase verbosity.  More v's means more verbose.",
     &MayaToEgg::dispatch_count, NULL, &_verbose);
  _verbose = 0;
}

////////////////////////////////////////////////////////////////////
//     Function: MayaToEgg::run
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
void MayaToEgg::
run() {
  // Set the verbose level by using Notify.
  if (_verbose >= 3) {
    maya_cat->set_severity(NS_spam);
    mayaegg_cat->set_severity(NS_spam);
  } else if (_verbose >= 2) {
    maya_cat->set_severity(NS_debug);
    mayaegg_cat->set_severity(NS_debug);
  } else if (_verbose >= 1) {
    maya_cat->set_severity(NS_info);
    mayaegg_cat->set_severity(NS_info);
  }

  // Let's open the output file before we initialize Maya, since Maya
  // now has a nasty habit of changing the current directory.
  get_output();

  nout << "Initializing Maya.\n";
  MayaToEggConverter converter(_program_name);
  if (!converter.open_api()) {
    nout << "Unable to initialize Maya.\n";
    exit(1);
  }

  // Copy in the command-line parameters.
  converter._polygon_output = _polygon_output;
  converter._polygon_tolerance = _polygon_tolerance;
  converter._ignore_transforms = _ignore_transforms;

  // Copy in the path and animation parameters.
  apply_parameters(converter);

  // Set the coordinate system to match Maya's.
  if (!_got_coordinate_system) {
    _coordinate_system = converter._maya->get_coordinate_system();
  }
  _data.set_coordinate_system(_coordinate_system);

  // Get the units from the Maya file, if the user didn't override.
  if (_input_units == DU_invalid) {
    _input_units = converter._maya->get_units();
  }

  converter.set_egg_data(&_data, false);
  apply_parameters(converter);

  if (!converter.convert_file(_input_filename)) {
    nout << "Errors in conversion.\n";
    exit(1);
  }

  write_egg_file();
  nout << "\n";
}

int main(int argc, char *argv[]) {
  MayaToEgg prog;
  prog.parse_command_line(argc, argv);
  prog.run();
  return 0;
}
<commit_msg>update comment<commit_after>// Filename: mayaToEgg.cxx
// Created by:  drose (15Feb00)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license.  You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////

#include "mayaToEgg.h"
#include "mayaToEggConverter.h"
#include "config_mayaegg.h"
#include "config_maya.h"  // for maya_cat

////////////////////////////////////////////////////////////////////
//     Function: MayaToEgg::Constructor
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
MayaToEgg::
MayaToEgg() :
  SomethingToEgg("Maya", ".mb")
{
  add_path_replace_options();
  add_path_store_options();
  add_animation_options();
  add_units_options();
  add_normals_options();
  add_transform_options();

  set_program_description
    ("This program converts Maya model files to egg.  Nothing fancy yet.");

  add_option
    ("p", "", 0,
     "Generate polygon output only.  Tesselate all NURBS surfaces to "
     "polygons via the built-in Maya tesselator.  The tesselation will "
     "be based on the tolerance factor given by -ptol.",
     &MayaToEgg::dispatch_none, &_polygon_output);

  add_option
    ("ptol", "tolerance", 0,
     "Specify the fit tolerance for Maya polygon tesselation.  The smaller "
     "the number, the more polygons will be generated.  The default is "
     "0.01.",
     &MayaToEgg::dispatch_double, NULL, &_polygon_tolerance);

  add_option
    ("notrans", "", 0,
     "Don't convert explicit DAG transformations given in the Maya file.  "
     "Instead, convert all vertices to world space and write the file as "
     "one big transform space.  Using this option doesn't change the "
     "position of objects in the scene, just the number of explicit "
     "transforms appearing in the resulting egg file.",
     &MayaToEgg::dispatch_none, &_ignore_transforms);

  add_option
    ("v", "", 0,
     "Increase verbosity.  More v's means more verbose.",
     &MayaToEgg::dispatch_count, NULL, &_verbose);
  _verbose = 0;
}

////////////////////////////////////////////////////////////////////
//     Function: MayaToEgg::run
//       Access: Public
//  Description:
////////////////////////////////////////////////////////////////////
void MayaToEgg::
run() {
  // Set the verbose level by using Notify.
  if (_verbose >= 3) {
    maya_cat->set_severity(NS_spam);
    mayaegg_cat->set_severity(NS_spam);
  } else if (_verbose >= 2) {
    maya_cat->set_severity(NS_debug);
    mayaegg_cat->set_severity(NS_debug);
  } else if (_verbose >= 1) {
    maya_cat->set_severity(NS_info);
    mayaegg_cat->set_severity(NS_info);
  }

  // Let's open the output file before we initialize Maya, since Maya
  // now has a nasty habit of changing the current directory.
  get_output();

  nout << "Initializing Maya.\n";
  MayaToEggConverter converter(_program_name);
  if (!converter.open_api()) {
    nout << "Unable to initialize Maya.\n";
    exit(1);
  }

  // Copy in the command-line parameters.
  converter._polygon_output = _polygon_output;
  converter._polygon_tolerance = _polygon_tolerance;
  converter._ignore_transforms = _ignore_transforms;

  // Copy in the path and animation parameters.
  apply_parameters(converter);

  // Set the coordinate system to match Maya's.
  if (!_got_coordinate_system) {
    _coordinate_system = converter._maya->get_coordinate_system();
  }
  _data.set_coordinate_system(_coordinate_system);

  // Use the standard Maya units, if the user didn't specify
  // otherwise.  This always returns centimeters, which is the way all
  // Maya files are stored internally (and is the units returned by
  // all of the API functions called here).
  if (_input_units == DU_invalid) {
    _input_units = converter._maya->get_units();
  }

  converter.set_egg_data(&_data, false);
  apply_parameters(converter);

  if (!converter.convert_file(_input_filename)) {
    nout << "Errors in conversion.\n";
    exit(1);
  }

  write_egg_file();
  nout << "\n";
}

int main(int argc, char *argv[]) {
  MayaToEgg prog;
  prog.parse_command_line(argc, argv);
  prog.run();
  return 0;
}
<|endoftext|>


<commit_before>#ifdef COSMOLOGY

#include <iostream>
#include <fstream>
#include "cosmology.h"
#include "../io.h"

using namespace std;

void Cosmology::Load_Scale_Outputs( struct parameters *P ) {

  char filename_1[100];
  // create the filename to read from
  strcpy(filename_1, P->scale_outputs_file);
  chprintf( " Loading Scale_Factor Outpus: %s\n", filename_1);

  ifstream file_out ( filename_1 );
  string line;
  Real a_value;
  if (file_out.is_open()){
    while ( getline (file_out,line) ){
      a_value = atof( line.c_str() );
      scale_outputs.push_back( a_value );
      n_outputs += 1;
      // chprintf("%f\n", a_value);
    }
    file_out.close();
    n_outputs = scale_outputs.size();
    next_output_indx = 0;
    chprintf("  Loaded %d scale outputs \n", n_outputs);
  }
  else{
    chprintf("  Error: Unable to open cosmology outputs file\n");
    exit(1);
  }

  chprintf(" Setting next snapshot output\n");

  int scale_indx = next_output_indx;
  a_value = scale_outputs[scale_indx];

  while ( (current_a - a_value) > 1e-3  ){
    // chprintf( "%f   %f\n", a_value, current_a);
    scale_indx += 1;
    a_value = scale_outputs[scale_indx];
  }
  next_output_indx = scale_indx;
  next_output = a_value;
  chprintf("  Next output index: %d  \n", next_output_indx );
  chprintf("  Next output z value: %f  \n", 1./next_output - 1 );

  exit_now = false;

}

void Cosmology::Set_Scale_Outputs( struct parameters *P ){

  if ( P->scale_outputs_file[0] == '\0' ){
    chprintf( " Output every %d timesteps.\n", P->n_steps_output );
    Real scale_end = 1 / ( P->End_redshift + 1);
    scale_outputs.push_back( current_a );
    scale_outputs.push_back( scale_end );
    n_outputs = scale_outputs.size();
    next_output_indx = 0;
    next_output = current_a;
    chprintf("  Next output index: %d  \n", next_output_indx );
    chprintf("  Next output z value: %f  \n", 1./next_output - 1 );
  }
  else  Load_Scale_Outputs( P );

}

void Cosmology::Set_Next_Scale_Output(  ){

  chprintf("Setting next output index. Current index: %d    n_outputs: %d ", scale_indx, n_outputs);

  int scale_indx = next_output_indx;
  Real a_value = scale_outputs[scale_indx];
  if  ( ( scale_indx == 0 ) && ( abs(a_value - current_a )<1e-5 ) )scale_indx = 1;
  else scale_indx += 1;
  if ( scale_indx < n_outputs ){
    a_value = scale_outputs[scale_indx];
    next_output_indx = scale_indx;
    next_output = a_value;
  }
  else{
    exit_now = true;
  }
}

#endif<commit_msg>fixed<commit_after>#ifdef COSMOLOGY

#include <iostream>
#include <fstream>
#include "cosmology.h"
#include "../io.h"

using namespace std;

void Cosmology::Load_Scale_Outputs( struct parameters *P ) {

  char filename_1[100];
  // create the filename to read from
  strcpy(filename_1, P->scale_outputs_file);
  chprintf( " Loading Scale_Factor Outpus: %s\n", filename_1);

  ifstream file_out ( filename_1 );
  string line;
  Real a_value;
  if (file_out.is_open()){
    while ( getline (file_out,line) ){
      a_value = atof( line.c_str() );
      scale_outputs.push_back( a_value );
      n_outputs += 1;
      // chprintf("%f\n", a_value);
    }
    file_out.close();
    n_outputs = scale_outputs.size();
    next_output_indx = 0;
    chprintf("  Loaded %d scale outputs \n", n_outputs);
  }
  else{
    chprintf("  Error: Unable to open cosmology outputs file\n");
    exit(1);
  }

  chprintf(" Setting next snapshot output\n");

  int scale_indx = next_output_indx;
  a_value = scale_outputs[scale_indx];

  while ( (current_a - a_value) > 1e-3  ){
    // chprintf( "%f   %f\n", a_value, current_a);
    scale_indx += 1;
    a_value = scale_outputs[scale_indx];
  }
  next_output_indx = scale_indx;
  next_output = a_value;
  chprintf("  Next output index: %d  \n", next_output_indx );
  chprintf("  Next output z value: %f  \n", 1./next_output - 1 );

  exit_now = false;

}

void Cosmology::Set_Scale_Outputs( struct parameters *P ){

  if ( P->scale_outputs_file[0] == '\0' ){
    chprintf( " Output every %d timesteps.\n", P->n_steps_output );
    Real scale_end = 1 / ( P->End_redshift + 1);
    scale_outputs.push_back( current_a );
    scale_outputs.push_back( scale_end );
    n_outputs = scale_outputs.size();
    next_output_indx = 0;
    next_output = current_a;
    chprintf("  Next output index: %d  \n", next_output_indx );
    chprintf("  Next output z value: %f  \n", 1./next_output - 1 );
  }
  else  Load_Scale_Outputs( P );

}

void Cosmology::Set_Next_Scale_Output(  ){

  int scale_indx = next_output_indx;
  Real a_value = scale_outputs[scale_indx];
  chprintf("Setting next output index. Current index: %d    n_outputs: %d ", scale_indx, n_outputs);

  if  ( ( scale_indx == 0 ) && ( abs(a_value - current_a )<1e-5 ) )scale_indx = 1;
  else scale_indx += 1;
  if ( scale_indx < n_outputs ){
    a_value = scale_outputs[scale_indx];
    next_output_indx = scale_indx;
    next_output = a_value;
  }
  else{
    exit_now = true;
  }
}

#endif<|endoftext|>