<commit_before>//------------------------------------------------------------------------------
// Copyright (c) 2016 by contributors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//------------------------------------------------------------------------------

/*
Author: Chao Ma (mctt90@gmail.com)

This file is the implementation of the Model class.
*/

#include "src/data/model_parameters.h"

#include <string.h>
#include <pmmintrin.h>  // for SSE

#include "src/base/file_util.h"
#include "src/base/math.h"

namespace xLearn {

//------------------------------------------------------------------------------
// The Model class
//------------------------------------------------------------------------------

// Basic contributor.
void Model::Initialize(const std::string& score_func,
                  const std::string& loss_func,
                  index_t num_feature,
                  index_t num_field,
                  index_t num_K,
                  real_t scale) {
  CHECK(!score_func.empty());
  CHECK(!loss_func.empty());
  CHECK_GT(num_feature, 0);
  CHECK_GE(num_field, 0);
  CHECK_GE(num_K, 0);
  CHECK_GT(scale, 0);
  score_func_ = score_func;
  loss_func_ = loss_func;
  num_feat_ = num_feature;
  num_field_ = num_field;
  num_K_ = num_K;
  scale_ = scale;
  // Calculate the number of model parameters
  param_num_w_ = num_feature * 2;
  if (score_func == "linear") {
    param_num_v_ = 0;
  } else if (score_func == "fm") {
    param_num_v_ = num_feature *
                   get_aligned_k() * 2;
  } else if (score_func == "ffm") {
    param_num_v_ = num_feature *
                   get_aligned_k() *
                   num_field * 2;
  } else {
    LOG(FATAL) << "Unknow score function: " << score_func;
  }
  this->initial(true);
}

// To get the best performance for SSE, we need to
// allocate memory for the model parameters in aligned way.
// For SSE, the align number should be 16 byte.
void Model::initial(bool set_val) {
  try {
    // Conventional malloc for linear term and bias
    param_w_ = (real_t*)malloc(param_num_w_*sizeof(real_t));
    param_b_ = (real_t*)malloc(2*sizeof(real_t));
    if (score_func_.compare("fm") == 0 ||
        score_func_.compare("ffm") == 0) {
      // Aligned malloc for latent factor
#ifdef _WIN32
      param_v_ = _aligned_malloc(
          param_num_v_ * sizeof(real_t),
          kAlignByte);
#else
      posix_memalign(
          (void**)&param_v_,
          kAlignByte,
          param_num_v_ * sizeof(real_t));
#endif
    } else {
      param_v_ = nullptr;
    }
  } catch (std::bad_alloc&) {
    LOG(FATAL) << "Cannot allocate enough memory for current  \
                   model parameters. Parameter size: "
               << GetNumParameter();
  }
  // set value for model
  if (set_val) {
    set_value();
  }
}

// Set value for model
void Model::set_value() {
  // Use distribution to transform the random unsigned
  // int generated by gen into a float in [(0.0, 1.0) * coef]
  std::default_random_engine generator;
  std::uniform_real_distribution<real_t> dis(0.0, 1.0);
  /*********************************************************
   *  Initialize linear and bias term                      *
   *********************************************************/
  for (index_t i = 0; i < param_num_w_; i += 2) {
    param_w_[i] = 0;      /* model */
    param_w_[i+1] = 1.0;  /* gradient cache */
  }
  param_b_[0] = 0;    /* model */
  param_b_[1] = 1.0;  /* gradient cache */
  /*********************************************************
   *  Initialize latent factor for fm                      *
   *********************************************************/
  if (score_func_.compare("fm") == 0) {
    index_t k_aligned = get_aligned_k();
    real_t coef = 1.0f / sqrt(num_K_) * scale_;
    real_t* w = param_v_;
    for (index_t j = 0; j < num_feat_; ++j) {
      for(index_t d = 0; d < num_K_; d++, w++)
        *w = coef * dis(generator);
      for(index_t d = num_K_; d < k_aligned; d++, w++)
        *w = 0;
      for(index_t d = k_aligned; d < 2*k_aligned; d++, w++)
        *w = 1.0;
    }
  }
  /*********************************************************
   *  Initialize latent factor for ffm                     *
   *********************************************************/
  else if (score_func_.compare("ffm") == 0) {
    index_t k_aligned = get_aligned_k();
    real_t* w = param_v_;
    real_t coef = 1.0f / sqrt(num_K_) * scale_;
    for (index_t j = 0; j < num_feat_; ++j) {
      for (index_t f = 0; f < num_field_; ++f) {
        for (index_t d = 0; d < k_aligned; ) {
          for (index_t s = 0; s < kAlign; s++, w++, d++) {
            w[0] = (d < num_K_) ? coef * dis(generator) : 0.0;
            w[kAlign] = 1.0;
          }
          w += kAlign;
        }
      }
    }
  }
}

// Free the allocated memory
void Model::free_model() {
  free(param_w_);
  free(param_v_);
  free(param_b_);
}

// Initialize model from a checkpoint file
Model::Model(const std::string& filename) {
  CHECK_NE(filename.empty(), true);
  if (this->Deserialize(filename) == false) {
    printf("Cannot Load model from the file: %s\n",
           filename.c_str());
    exit(0);
  }
}

// Serialize current model to a disk file
void Model::Serialize(const std::string& filename) {
  CHECK_NE(filename.empty(), true);
  FILE* file = OpenFileOrDie(filename.c_str(), "w");
  // Write score function
  WriteStringToFile(file, score_func_);
  // Write loss function
  WriteStringToFile(file, loss_func_);
  // Write feature num
  WriteDataToDisk(file, (char*)&num_feat_, sizeof(num_feat_));
  // Write field num
  WriteDataToDisk(file, (char*)&num_field_, sizeof(num_field_));
  // Write K
  WriteDataToDisk(file, (char*)&num_K_, sizeof(num_K_));
  // Write w
  this->serialize_w_v_b(file);
  Close(file);
}

// Deserialize model from a checkpoint file
bool Model::Deserialize(const std::string& filename) {
  CHECK_NE(filename.empty(), true);
  FILE* file = OpenFileOrDie(filename.c_str(), "r");
  if (file == NULL) { return false; }
  // Read score function
  ReadStringFromFile(file, score_func_);
  // Read loss function
  ReadStringFromFile(file, loss_func_);
  // Read feature num
  ReadDataFromDisk(file, (char*)&num_feat_, sizeof(num_feat_));
  // Read field num
  ReadDataFromDisk(file, (char*)&num_field_, sizeof(num_field_));
  // Read K
  ReadDataFromDisk(file, (char*)&num_K_, sizeof(num_K_));
  // Read w
  this->deserialize_w_v_b(file);
  Close(file);
  return true;
}

// Take a record of the best model during training
void Model::SetBestModel() {
  if (param_best_w_ == nullptr) {
    param_best_w_ = (real_t*)malloc(
      param_num_w_*sizeof(real_t));
  }
  if (param_best_v_ == nullptr &&
      score_func_.compare("linear") != 0) {
#ifdef _WIN32
    param_best_v_ = _aligned_malloc(
      param_num_v_ * sizeof(real_t),
      kAlignByte);
#else
    posix_memalign(
      (void**)&param_best_v_,
      kAlignByte,
      param_num_v_ * sizeof(real_t));
#endif
  }
  if (param_best_b_ == nullptr) {
    param_best_b_ = (real_t*)malloc(
      2 * sizeof(real_t));
  }
  // Copy current model parameters
  memcpy(param_best_w_, param_w_, param_num_w_*sizeof(real_t));
  memcpy(param_best_v_, param_v_, param_num_v_*sizeof(real_t));
  memcpy(param_best_b_, param_b_, 2*sizeof(real_t));
}

// Shrink back for getting the best model
void Model::Shrink() {
  // Copy best model parameters
  if (param_best_w_ != nullptr) {
    memcpy(param_w_, param_best_w_, param_num_w_*sizeof(real_t));
  }
  if (param_best_v_ != nullptr) {
    memcpy(param_v_, param_best_v_, param_num_v_*sizeof(real_t));
  }
  if (param_best_b_ != nullptr) {
    memcpy(param_b_, param_best_b_, 2*sizeof(real_t));
  }
}

// Serialize w,v,b to disk file
void Model::serialize_w_v_b(FILE* file) {
  // Write size of w
  WriteDataToDisk(file, (char*)&param_num_w_, sizeof(param_num_w_));
  // Write size of v
  if (score_func_.compare("linear") != 0) {
    WriteDataToDisk(file, (char*)&param_num_v_, sizeof(param_num_v_));
  }
  // Write w
  WriteDataToDisk(file, (char*)param_w_, sizeof(real_t)*param_num_w_);
  // Write b
  WriteDataToDisk(file, (char*)param_b_, sizeof(real_t)*2);
  // Write v
  if (score_func_.compare("linear") != 0) {
    WriteDataToDisk(file, (char*)param_v_, sizeof(real_t)*param_num_v_);
  }
}

// Deserialize w,v,b from disk file
void Model::deserialize_w_v_b(FILE* file) {
  // Read size of w
  ReadDataFromDisk(file, (char*)&param_num_w_, sizeof(param_num_w_));
  // Read size of v
  if (score_func_.compare("linear") != 0) {
    ReadDataFromDisk(file, (char*)&param_num_v_, sizeof(param_num_v_));
  }
  // Allocate memory. Don't set value here
  this->initial(false);
  // Read w
  ReadDataFromDisk(file, (char*)param_w_, sizeof(real_t)*param_num_w_);
  // Read b
  ReadDataFromDisk(file, (char*)param_b_, sizeof(real_t)*2);
  // Read v
  if (score_func_.compare("linear") != 0) {
    ReadDataFromDisk(file, (char*)param_v_, sizeof(real_t)*param_num_v_);
  }
}

}  // namespace xLearn
<commit_msg>remove logger<commit_after>//------------------------------------------------------------------------------
// Copyright (c) 2016 by contributors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//------------------------------------------------------------------------------

/*
Author: Chao Ma (mctt90@gmail.com)

This file is the implementation of the Model class.
*/

#include "src/data/model_parameters.h"

#include <string.h>
#include <pmmintrin.h>  // for SSE

#include "src/base/file_util.h"
#include "src/base/math.h"

namespace xLearn {

//------------------------------------------------------------------------------
// The Model class
//------------------------------------------------------------------------------

// Basic contributor.
void Model::Initialize(const std::string& score_func,
                  const std::string& loss_func,
                  index_t num_feature,
                  index_t num_field,
                  index_t num_K,
                  real_t scale) {
  CHECK(!score_func.empty());
  CHECK(!loss_func.empty());
  CHECK_GT(num_feature, 0);
  CHECK_GE(num_field, 0);
  CHECK_GE(num_K, 0);
  CHECK_GT(scale, 0);
  score_func_ = score_func;
  loss_func_ = loss_func;
  num_feat_ = num_feature;
  num_field_ = num_field;
  num_K_ = num_K;
  scale_ = scale;
  // Calculate the number of model parameters
  param_num_w_ = num_feature * 2;
  if (score_func == "linear") {
    param_num_v_ = 0;
  } else if (score_func == "fm") {
    param_num_v_ = num_feature *
                   get_aligned_k() * 2;
  } else if (score_func == "ffm") {
    param_num_v_ = num_feature *
                   get_aligned_k() *
                   num_field * 2;
  } else {
    LOG(FATAL) << "Unknow score function: " << score_func;
  }
  this->initial(true);
}

// To get the best performance for SSE, we need to
// allocate memory for the model parameters in aligned way.
// For SSE, the align number should be 16 byte.
void Model::initial(bool set_val) {
  try {
    // Conventional malloc for linear term and bias
    param_w_ = (real_t*)malloc(param_num_w_*sizeof(real_t));
    param_b_ = (real_t*)malloc(2*sizeof(real_t));
    if (score_func_.compare("fm") == 0 ||
        score_func_.compare("ffm") == 0) {
      // Aligned malloc for latent factor
#ifdef _WIN32
      param_v_ = _aligned_malloc(
          param_num_v_ * sizeof(real_t),
          kAlignByte);
#else
      posix_memalign(
          (void**)&param_v_,
          kAlignByte,
          param_num_v_ * sizeof(real_t));
#endif
    } else {
      param_v_ = nullptr;
    }
  } catch (std::bad_alloc&) {
    //LOG(FATAL) << "Cannot allocate enough memory for current  \
    //               model parameters. Parameter size: "
    //           << GetNumParameter();
  }
  // set value for model
  if (set_val) {
    set_value();
  }
}

// Set value for model
void Model::set_value() {
  // Use distribution to transform the random unsigned
  // int generated by gen into a float in [(0.0, 1.0) * coef]
  std::default_random_engine generator;
  std::uniform_real_distribution<real_t> dis(0.0, 1.0);
  /*********************************************************
   *  Initialize linear and bias term                      *
   *********************************************************/
  for (index_t i = 0; i < param_num_w_; i += 2) {
    param_w_[i] = 0;      /* model */
    param_w_[i+1] = 1.0;  /* gradient cache */
  }
  param_b_[0] = 0;    /* model */
  param_b_[1] = 1.0;  /* gradient cache */
  /*********************************************************
   *  Initialize latent factor for fm                      *
   *********************************************************/
  if (score_func_.compare("fm") == 0) {
    index_t k_aligned = get_aligned_k();
    real_t coef = 1.0f / sqrt(num_K_) * scale_;
    real_t* w = param_v_;
    for (index_t j = 0; j < num_feat_; ++j) {
      for(index_t d = 0; d < num_K_; d++, w++)
        *w = coef * dis(generator);
      for(index_t d = num_K_; d < k_aligned; d++, w++)
        *w = 0;
      for(index_t d = k_aligned; d < 2*k_aligned; d++, w++)
        *w = 1.0;
    }
  }
  /*********************************************************
   *  Initialize latent factor for ffm                     *
   *********************************************************/
  else if (score_func_.compare("ffm") == 0) {
    index_t k_aligned = get_aligned_k();
    real_t* w = param_v_;
    real_t coef = 1.0f / sqrt(num_K_) * scale_;
    for (index_t j = 0; j < num_feat_; ++j) {
      for (index_t f = 0; f < num_field_; ++f) {
        for (index_t d = 0; d < k_aligned; ) {
          for (index_t s = 0; s < kAlign; s++, w++, d++) {
            w[0] = (d < num_K_) ? coef * dis(generator) : 0.0;
            w[kAlign] = 1.0;
          }
          w += kAlign;
        }
      }
    }
  }
}

// Free the allocated memory
void Model::free_model() {
  free(param_w_);
  free(param_v_);
  free(param_b_);
}

// Initialize model from a checkpoint file
Model::Model(const std::string& filename) {
  CHECK_NE(filename.empty(), true);
  if (this->Deserialize(filename) == false) {
    printf("Cannot Load model from the file: %s\n",
           filename.c_str());
    exit(0);
  }
}

// Serialize current model to a disk file
void Model::Serialize(const std::string& filename) {
  CHECK_NE(filename.empty(), true);
  FILE* file = OpenFileOrDie(filename.c_str(), "w");
  // Write score function
  WriteStringToFile(file, score_func_);
  // Write loss function
  WriteStringToFile(file, loss_func_);
  // Write feature num
  WriteDataToDisk(file, (char*)&num_feat_, sizeof(num_feat_));
  // Write field num
  WriteDataToDisk(file, (char*)&num_field_, sizeof(num_field_));
  // Write K
  WriteDataToDisk(file, (char*)&num_K_, sizeof(num_K_));
  // Write w
  this->serialize_w_v_b(file);
  Close(file);
}

// Deserialize model from a checkpoint file
bool Model::Deserialize(const std::string& filename) {
  CHECK_NE(filename.empty(), true);
  FILE* file = OpenFileOrDie(filename.c_str(), "r");
  if (file == NULL) { return false; }
  // Read score function
  ReadStringFromFile(file, score_func_);
  // Read loss function
  ReadStringFromFile(file, loss_func_);
  // Read feature num
  ReadDataFromDisk(file, (char*)&num_feat_, sizeof(num_feat_));
  // Read field num
  ReadDataFromDisk(file, (char*)&num_field_, sizeof(num_field_));
  // Read K
  ReadDataFromDisk(file, (char*)&num_K_, sizeof(num_K_));
  // Read w
  this->deserialize_w_v_b(file);
  Close(file);
  return true;
}

// Take a record of the best model during training
void Model::SetBestModel() {
  if (param_best_w_ == nullptr) {
    param_best_w_ = (real_t*)malloc(
      param_num_w_*sizeof(real_t));
  }
  if (param_best_v_ == nullptr &&
      score_func_.compare("linear") != 0) {
#ifdef _WIN32
    param_best_v_ = _aligned_malloc(
      param_num_v_ * sizeof(real_t),
      kAlignByte);
#else
    posix_memalign(
      (void**)&param_best_v_,
      kAlignByte,
      param_num_v_ * sizeof(real_t));
#endif
  }
  if (param_best_b_ == nullptr) {
    param_best_b_ = (real_t*)malloc(
      2 * sizeof(real_t));
  }
  // Copy current model parameters
  memcpy(param_best_w_, param_w_, param_num_w_*sizeof(real_t));
  memcpy(param_best_v_, param_v_, param_num_v_*sizeof(real_t));
  memcpy(param_best_b_, param_b_, 2*sizeof(real_t));
}

// Shrink back for getting the best model
void Model::Shrink() {
  // Copy best model parameters
  if (param_best_w_ != nullptr) {
    memcpy(param_w_, param_best_w_, param_num_w_*sizeof(real_t));
  }
  if (param_best_v_ != nullptr) {
    memcpy(param_v_, param_best_v_, param_num_v_*sizeof(real_t));
  }
  if (param_best_b_ != nullptr) {
    memcpy(param_b_, param_best_b_, 2*sizeof(real_t));
  }
}

// Serialize w,v,b to disk file
void Model::serialize_w_v_b(FILE* file) {
  // Write size of w
  WriteDataToDisk(file, (char*)&param_num_w_, sizeof(param_num_w_));
  // Write size of v
  if (score_func_.compare("linear") != 0) {
    WriteDataToDisk(file, (char*)&param_num_v_, sizeof(param_num_v_));
  }
  // Write w
  WriteDataToDisk(file, (char*)param_w_, sizeof(real_t)*param_num_w_);
  // Write b
  WriteDataToDisk(file, (char*)param_b_, sizeof(real_t)*2);
  // Write v
  if (score_func_.compare("linear") != 0) {
    WriteDataToDisk(file, (char*)param_v_, sizeof(real_t)*param_num_v_);
  }
}

// Deserialize w,v,b from disk file
void Model::deserialize_w_v_b(FILE* file) {
  // Read size of w
  ReadDataFromDisk(file, (char*)&param_num_w_, sizeof(param_num_w_));
  // Read size of v
  if (score_func_.compare("linear") != 0) {
    ReadDataFromDisk(file, (char*)&param_num_v_, sizeof(param_num_v_));
  }
  // Allocate memory. Don't set value here
  this->initial(false);
  // Read w
  ReadDataFromDisk(file, (char*)param_w_, sizeof(real_t)*param_num_w_);
  // Read b
  ReadDataFromDisk(file, (char*)param_b_, sizeof(real_t)*2);
  // Read v
  if (score_func_.compare("linear") != 0) {
    ReadDataFromDisk(file, (char*)param_v_, sizeof(real_t)*param_num_v_);
  }
}

}  // namespace xLearn
<|endoftext|>


<commit_before>/***************************Copyright-DO-NOT-REMOVE-THIS-LINE**
 * CONDOR Copyright Notice
 *
 * See LICENSE.TXT for additional notices and disclaimers.
 *
 * Copyright (c)1990-1998 CONDOR Team, Computer Sciences Department,
 * University of Wisconsin-Madison, Madison, WI.  All Rights Reserved.
 * No use of the CONDOR Software Program Source Code is authorized
 * without the express consent of the CONDOR Team.  For more information
 * contact: CONDOR Team, Attention: Professor Miron Livny,
 * 7367 Computer Sciences, 1210 W. Dayton St., Madison, WI 53706-1685,
 * (608) 262-0856 or miron@cs.wisc.edu.
 *
 * U.S. Government Rights Restrictions: Use, duplication, or disclosure
 * by the U.S. Government is subject to restrictions as set forth in
 * subparagraph (c)(1)(ii) of The Rights in Technical Data and Computer
 * Software clause at DFARS 252.227-7013 or subparagraphs (c)(1) and
 * (2) of Commercial Computer Software-Restricted Rights at 48 CFR
 * 52.227-19, as applicable, CONDOR Team, Attention: Professor Miron
 * Livny, 7367 Computer Sciences, 1210 W. Dayton St., Madison,
 * WI 53706-1685, (608) 262-0856 or miron@cs.wisc.edu.
****************************Copyright-DO-NOT-REMOVE-THIS-LINE**/

#include "condor_common.h"
#include "condor_classad.h"
#include "compress.h"

ClassAdBin::ClassAdBin( )
{
    count = 0;
    ad = new ClassAd( );
}

ClassAdBin::~ClassAdBin( )
{
    delete ad;
}

bool
MakeSignature( const ClassAd *ad, const References &refs, string &sig )
{
    References::iterator    itr;
    ExprTree                *expr;
    Value                   val;
    ClassAdUnParser         unp;

    sig = "";
    for( itr=refs.begin( ); itr!= refs.end( ); itr++ ) {
        if( ( expr = ad->Lookup( *itr ) ) && expr->GetKind( )==LITERAL_NODE ) {
            unp.Unparse( sig, expr );
        } else {
            return( false );
        }
        sig += ":";
    }

    return( true );
}

bool
Compress( ClassAdCollectionServer *server, LocalCollectionQuery *query,
    const References &refs, CompressedAds &comp, list<ClassAd*> &rest )
{
    string                      key, sig;
    References::const_iterator  ritr;
    ClassAd                     *ad;
    CompressedAds::iterator     citr;
    ClassAdBin                  *bin;

    query->ToFirst( );
    query->Current( key );
    while( !query->IsAfterLast( ) ) {
            // get ad
        if( !( ad = server->GetClassAd( key ) ) ) {
            return( false );
        }

            // get signature of current ad
        if( !MakeSignature( ad, refs, sig ) ) {
                // can't make signature --- can't compress
            rest.push_back( ad->Copy( ) );
        }

            // get bin
        if( ( citr = comp.find( sig ) ) == comp.end( ) ) {
                // no bin ... make one
            bin = new ClassAdBin;
            bin->count = 1;

                // make a projected classad
            for( ritr=refs.begin( ); ritr!=refs.end( ); ritr++ ) {
                bin->ad->Insert( *ritr, ad->Lookup( *ritr )->Copy( ) );
            }

				// insert bin into container
			comp[sig] = bin;
        } else {
                // increment membership in bin
            bin = citr->second;
            bin->count++;
        }

            // process next ad
        query->Next( key );
    }

    return( true );
}
<commit_msg>Added required scope resolution --- enums now live inside classes<commit_after>/***************************Copyright-DO-NOT-REMOVE-THIS-LINE**
 * CONDOR Copyright Notice
 *
 * See LICENSE.TXT for additional notices and disclaimers.
 *
 * Copyright (c)1990-1998 CONDOR Team, Computer Sciences Department,
 * University of Wisconsin-Madison, Madison, WI.  All Rights Reserved.
 * No use of the CONDOR Software Program Source Code is authorized
 * without the express consent of the CONDOR Team.  For more information
 * contact: CONDOR Team, Attention: Professor Miron Livny,
 * 7367 Computer Sciences, 1210 W. Dayton St., Madison, WI 53706-1685,
 * (608) 262-0856 or miron@cs.wisc.edu.
 *
 * U.S. Government Rights Restrictions: Use, duplication, or disclosure
 * by the U.S. Government is subject to restrictions as set forth in
 * subparagraph (c)(1)(ii) of The Rights in Technical Data and Computer
 * Software clause at DFARS 252.227-7013 or subparagraphs (c)(1) and
 * (2) of Commercial Computer Software-Restricted Rights at 48 CFR
 * 52.227-19, as applicable, CONDOR Team, Attention: Professor Miron
 * Livny, 7367 Computer Sciences, 1210 W. Dayton St., Madison,
 * WI 53706-1685, (608) 262-0856 or miron@cs.wisc.edu.
****************************Copyright-DO-NOT-REMOVE-THIS-LINE**/

#include "condor_common.h"
#include "condor_classad.h"
#include "compress.h"

ClassAdBin::ClassAdBin( )
{
    count = 0;
    ad = new ClassAd( );
}

ClassAdBin::~ClassAdBin( )
{
    delete ad;
}

bool
MakeSignature( const ClassAd *ad, const References &refs, string &sig )
{
    References::iterator    itr;
    ExprTree                *expr;
    Value                   val;
    ClassAdUnParser         unp;

    sig = "";
    for( itr=refs.begin( ); itr!= refs.end( ); itr++ ) {
        if((expr=ad->Lookup(*itr))&&expr->GetKind()==ExprTree::LITERAL_NODE){
            unp.Unparse( sig, expr );
        } else {
            return( false );
        }
        sig += ":";
    }

    return( true );
}

bool
Compress( ClassAdCollectionServer *server, LocalCollectionQuery *query,
    const References &refs, CompressedAds &comp, list<ClassAd*> &rest )
{
    string                      key, sig;
    References::const_iterator  ritr;
    ClassAd                     *ad;
    CompressedAds::iterator     citr;
    ClassAdBin                  *bin;

    query->ToFirst( );
    query->Current( key );
    while( !query->IsAfterLast( ) ) {
            // get ad
        if( !( ad = server->GetClassAd( key ) ) ) {
            return( false );
        }

            // get signature of current ad
        if( !MakeSignature( ad, refs, sig ) ) {
                // can't make signature --- can't compress
            rest.push_back( ad->Copy( ) );
        }

            // get bin
        if( ( citr = comp.find( sig ) ) == comp.end( ) ) {
                // no bin ... make one
            bin = new ClassAdBin;
            bin->count = 1;

                // make a projected classad
            for( ritr=refs.begin( ); ritr!=refs.end( ); ritr++ ) {
                bin->ad->Insert( *ritr, ad->Lookup( *ritr )->Copy( ) );
            }

				// insert bin into container
			comp[sig] = bin;
        } else {
                // increment membership in bin
            bin = citr->second;
            bin->count++;
        }

            // process next ad
        query->Next( key );
    }

    return( true );
}
<|endoftext|>


<commit_before>#include <iostream.h>
#include "exprTree.h"

int main( void )
{
	Source		src;
	Sink		snk;
	Value		val;
	ExprTree	*tree, *ftree;

	src.setSource( stdin );
	snk.setSink ( stdout );

	if( !src.parseExpression( tree, true ) ) {
		cerr << "Parse error" << endl;
		exit( 1 );
	}

	cout << endl << "---" << endl;
	tree->toSink( snk );
	snk.flushSink();
	cout << endl << "---" << endl;

	if( !tree->flatten( val, ftree ) ) {
		cerr << "Failed to flatten" << endl;
	}

	if( ftree ) {
		ftree->toSink( snk );
	} else {
		val.toSink( snk );
	}

	snk.flushSink();
	cout << endl << endl;

	val.clear();
	delete tree;
	delete ftree;

	return 0;
}

<commit_msg>Superceded by cxi, the Classad eXpresssion Interpreter<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/threading/platform_thread.h"

#include <errno.h>
#include <sched.h>

#include "base/lazy_instance.h"
#include "base/logging.h"
#include "base/memory/scoped_ptr.h"
#include "base/safe_strerror_posix.h"
#include "base/threading/thread_id_name_manager.h"
#include "base/threading/thread_restrictions.h"
#include "base/tracked_objects.h"

#if !defined(OS_NACL)
#include <sys/prctl.h>
#include <sys/resource.h>
#include <sys/syscall.h>
#include <sys/time.h>
#include <unistd.h>
#endif

namespace base {

namespace {

int ThreadNiceValue(ThreadPriority priority) {
  switch (priority) {
    case kThreadPriority_RealtimeAudio:
      return -10;
    case kThreadPriority_Background:
      return 10;
    case kThreadPriority_Normal:
      return 0;
    case kThreadPriority_Display:
      return -6;
    default:
      NOTREACHED() << "Unknown priority.";
      return 0;
  }
}

}  // namespace

// static
void PlatformThread::SetName(const char* name) {
  ThreadIdNameManager::GetInstance()->SetName(CurrentId(), name);
  tracked_objects::ThreadData::InitializeThreadContext(name);

#if !defined(OS_NACL)
  // On linux we can get the thread names to show up in the debugger by setting
  // the process name for the LWP.  We don't want to do this for the main
  // thread because that would rename the process, causing tools like killall
  // to stop working.
  if (PlatformThread::CurrentId() == getpid())
    return;

  // http://0pointer.de/blog/projects/name-your-threads.html
  // Set the name for the LWP (which gets truncated to 15 characters).
  // Note that glibc also has a 'pthread_setname_np' api, but it may not be
  // available everywhere and it's only benefit over using prctl directly is
  // that it can set the name of threads other than the current thread.
  int err = prctl(PR_SET_NAME, name);
  // We expect EPERM failures in sandboxed processes, just ignore those.
  if (err < 0 && errno != EPERM)
    DPLOG(ERROR) << "prctl(PR_SET_NAME)";
#endif  //  !defined(OS_NACL)
}

// static
void PlatformThread::SetThreadPriority(PlatformThreadHandle handle,
                                       ThreadPriority priority) {
#if !defined(OS_NACL)
  if (priority == kThreadPriority_RealtimeAudio) {
    const struct sched_param kRealTimePrio = {8};
    if (pthread_setschedparam(pthread_self(), SCHED_RR, &kRealTimePrio) == 0) {
      // Got real time priority, no need to set nice level.
      return;
    }
  }

  // setpriority(2) should change the whole thread group's (i.e. process)
  // priority. however, on linux it will only change the target thread's
  // priority. see the bugs section in
  // http://man7.org/linux/man-pages/man2/getpriority.2.html.
  // we prefer using 0 rather than the current thread id since they are
  // equivalent but it makes sandboxing easier (https://crbug.com/399473).
  DCHECK_NE(handle.id_, kInvalidThreadId);
  const int kNiceSetting = ThreadNiceValue(priority);
  const PlatformThreadId current_id = PlatformThread::CurrentId();
  if (setpriority(PRIO_PROCESS,
                  handle.id_ == current_id ? 0 : handle.id_,
                  kNiceSetting)) {
    DVPLOG(1) << "Failed to set nice value of thread (" << handle.id_ << ") to "
              << kNiceSetting;
  }
#endif  //  !defined(OS_NACL)
}

void InitThreading() {}

void InitOnThread() {}

void TerminateOnThread() {}

size_t GetDefaultThreadStackSize(const pthread_attr_t& attributes) {
#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER)
  return 0;
#else
  // ThreadSanitizer bloats the stack heavily. Evidence has been that the
  // default stack size isn't enough for some browser tests.
  // MemorySanitizer needs this as a temporary fix for http://crbug.com/353687
  return 2 * (1 << 23);  // 2 times 8192K (the default stack size on Linux).
#endif
}

}  // namespace base
<commit_msg>Revert r257779: "MemorySanitizer: adjust stack size to work around an MSan bug."<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/threading/platform_thread.h"

#include <errno.h>
#include <sched.h>

#include "base/lazy_instance.h"
#include "base/logging.h"
#include "base/memory/scoped_ptr.h"
#include "base/safe_strerror_posix.h"
#include "base/threading/thread_id_name_manager.h"
#include "base/threading/thread_restrictions.h"
#include "base/tracked_objects.h"

#if !defined(OS_NACL)
#include <sys/prctl.h>
#include <sys/resource.h>
#include <sys/syscall.h>
#include <sys/time.h>
#include <unistd.h>
#endif

namespace base {

namespace {

int ThreadNiceValue(ThreadPriority priority) {
  switch (priority) {
    case kThreadPriority_RealtimeAudio:
      return -10;
    case kThreadPriority_Background:
      return 10;
    case kThreadPriority_Normal:
      return 0;
    case kThreadPriority_Display:
      return -6;
    default:
      NOTREACHED() << "Unknown priority.";
      return 0;
  }
}

}  // namespace

// static
void PlatformThread::SetName(const char* name) {
  ThreadIdNameManager::GetInstance()->SetName(CurrentId(), name);
  tracked_objects::ThreadData::InitializeThreadContext(name);

#if !defined(OS_NACL)
  // On linux we can get the thread names to show up in the debugger by setting
  // the process name for the LWP.  We don't want to do this for the main
  // thread because that would rename the process, causing tools like killall
  // to stop working.
  if (PlatformThread::CurrentId() == getpid())
    return;

  // http://0pointer.de/blog/projects/name-your-threads.html
  // Set the name for the LWP (which gets truncated to 15 characters).
  // Note that glibc also has a 'pthread_setname_np' api, but it may not be
  // available everywhere and it's only benefit over using prctl directly is
  // that it can set the name of threads other than the current thread.
  int err = prctl(PR_SET_NAME, name);
  // We expect EPERM failures in sandboxed processes, just ignore those.
  if (err < 0 && errno != EPERM)
    DPLOG(ERROR) << "prctl(PR_SET_NAME)";
#endif  //  !defined(OS_NACL)
}

// static
void PlatformThread::SetThreadPriority(PlatformThreadHandle handle,
                                       ThreadPriority priority) {
#if !defined(OS_NACL)
  if (priority == kThreadPriority_RealtimeAudio) {
    const struct sched_param kRealTimePrio = {8};
    if (pthread_setschedparam(pthread_self(), SCHED_RR, &kRealTimePrio) == 0) {
      // Got real time priority, no need to set nice level.
      return;
    }
  }

  // setpriority(2) should change the whole thread group's (i.e. process)
  // priority. however, on linux it will only change the target thread's
  // priority. see the bugs section in
  // http://man7.org/linux/man-pages/man2/getpriority.2.html.
  // we prefer using 0 rather than the current thread id since they are
  // equivalent but it makes sandboxing easier (https://crbug.com/399473).
  DCHECK_NE(handle.id_, kInvalidThreadId);
  const int kNiceSetting = ThreadNiceValue(priority);
  const PlatformThreadId current_id = PlatformThread::CurrentId();
  if (setpriority(PRIO_PROCESS,
                  handle.id_ == current_id ? 0 : handle.id_,
                  kNiceSetting)) {
    DVPLOG(1) << "Failed to set nice value of thread (" << handle.id_ << ") to "
              << kNiceSetting;
  }
#endif  //  !defined(OS_NACL)
}

void InitThreading() {}

void InitOnThread() {}

void TerminateOnThread() {}

size_t GetDefaultThreadStackSize(const pthread_attr_t& attributes) {
#if !defined(THREAD_SANITIZER)
  return 0;
#else
  // ThreadSanitizer bloats the stack heavily. Evidence has been that the
  // default stack size isn't enough for some browser tests.
  return 2 * (1 << 23);  // 2 times 8192K (the default stack size on Linux).
#endif
}

}  // namespace base
<|endoftext|>


<commit_before>//
//  parser_tables.cpp
//  Parse
//
//  Created by Andrew Hunter on 07/05/2011.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#include <algorithm>

#include "parser_tables.h"

using namespace std;
using namespace contextfree;
using namespace lr;

/// \brief Ranks actions in the order they should appear in the final table
static inline int action_score(int action) {
    switch (action) {
        case lr_action::act_weakreduce:
            // Weak reduce actions come first as they should always be performed if their symbol will produce a shift
            return 0;

        case lr_action::act_shift:
            // Shift actions are preferred if there's a conflict
            return 1;

        case lr_action::act_reduce:
            // Reduce actions have the lowest priority
            return 2;

        case lr_action::act_goto:
            // Gotos never actually produce a clash
            return 3;

        default:
            return 4;
    }
}

static inline bool compare_actions(const parser_tables::action& a, const parser_tables::action& b) {
    // First, compare on symbol IDs
    if (a.m_SymbolId < b.m_SymbolId) return true;

    // Next, compare on action types
    if (action_score(a.m_Type) < action_score(b.m_Type)) return true;

    // Actions are equal
    return false;
}

/// \brief Creates a parser from the result of the specified builder class
parser_tables::parser_tables(const lalr_builder& builder) {
    // Allocate the tables
    m_NumStates             = builder.count_states();
    m_NonterminalActions    = new action*[m_NumStates];
    m_TerminalActions       = new action*[m_NumStates];
    m_Counts                = new action_count[m_NumStates];

    contextfree::end_of_input eoi;
    m_EndOfInput            = builder.gram().identifier_for_item(eoi);

    const grammar& gram = builder.gram();

    map<int, int> ruleIds;                          // Maps their rule IDs to our rule IDs

    // Build up the tables for each state
    for (int stateId = 0; stateId < m_NumStates; stateId++) {
        // Get the actions for this state
        const lr_action_set& actions = builder.actions_for_state(stateId);

        // Count the number of terminal and nonterminal actions
        int termCount       = 0;
        int nontermCount    = 0;

        for (lr_action_set::const_iterator nextAction = actions.begin(); nextAction != actions.end(); nextAction++) {
            if ((*nextAction)->item()->type() == item::terminal) {
                termCount++;
            } else {
                nontermCount++;
            }
        }

        // Allocate action tables of the appropriate size
        action* termActions     = new action[termCount];
        action* nontermActions  = new action[nontermCount];

        int     termPos         = 0;                    // Current item in the terminal table
        int     nontermPos      = 0;                    // Current item in the nonterminal table

        // Fill up the actions (not in order)
        for (lr_action_set::const_iterator nextAction = actions.begin(); nextAction != actions.end(); nextAction++) {
            // Get the next state
            int nextState   = (*nextAction)->next_state();
            int type        = (*nextAction)->type();

            // If the next action is a reduce action, then the next state should actually be the rule to reduce
            if (type == lr_action::act_reduce || type == lr_action::act_weakreduce) {
                // Look up the ID we assigned this rule
                int                     ruleId  = (*nextAction)->rule()->identifier(gram);
                map<int, int>::iterator found   = ruleIds.find(ruleId);

                if (found == ruleIds.end()) {
                    nextState       = (int)ruleIds.size();
                    ruleIds[ruleId] = nextState;
                } else {
                    nextState       = found->second;
                }
            }

            if ((*nextAction)->item()->type() == item::terminal) {
                // Add a new terminal action
                termActions[termPos].m_Type             = type;
                termActions[termPos].m_NextState        = nextState;
                termActions[termPos].m_SymbolId         = (*nextAction)->item()->symbol();

                termPos++;
            } else {
                // Add a new nonterminal action
                nontermActions[nontermPos].m_Type       = type;
                nontermActions[nontermPos].m_NextState  = nextState;
                nontermActions[nontermPos].m_SymbolId   = gram.identifier_for_item((*nextAction)->item());
            }
        }

        // Sort the actions for this state
        std::sort(termActions, termActions + termCount, compare_actions);
        std::sort(nontermActions, nontermActions + nontermCount, compare_actions);

        // Store the actions in the table
        m_TerminalActions[stateId]      = termActions;
        m_NonterminalActions[stateId]   = nontermActions;
        m_Counts[stateId].m_NumTerms    = termCount;
        m_Counts[stateId].m_NumNonterms = nontermCount;
    }

    // Fill in the rule table
    m_NumRules  = (int)ruleIds.size();
    m_Rules     = new reduce_rule[ruleIds.size()];

    for (map<int, int>::iterator ruleId = ruleIds.begin(); ruleId != ruleIds.end(); ruleId++) {
        m_Rules[ruleId->second].m_Identifier    = ruleId->first;
        m_Rules[ruleId->second].m_Length        = (int)gram.rule_with_identifier(ruleId->first)->items().size();
    }
}

/// \brief Copy constructor
parser_tables::parser_tables(const parser_tables& copyFrom)
: m_NumStates(copyFrom.m_NumStates)
, m_NumRules(copyFrom.m_NumRules)
, m_EndOfInput(copyFrom.m_EndOfInput) {
    // Allocate the action tables
    m_TerminalActions       = new action*[m_NumStates];
    m_NonterminalActions    = new action*[m_NumStates];
    m_Counts                = new action_count[m_NumStates];

    // Copy the states
    for (int stateId=0; stateId<m_NumStates; stateId++) {
        // Copy the counts
        m_Counts[stateId] = copyFrom.m_Counts[stateId];

        // Allocate the array for this state
        m_TerminalActions[stateId]      = new action[m_Counts[stateId].m_NumTerms];
        m_NonterminalActions[stateId]   = new action[m_Counts[stateId].m_NumNonterms];

        // Copy the terminals and nonterminals
        for (int x=0; x<m_Counts[stateId].m_NumTerms; x++) {
            m_TerminalActions[stateId][x] = copyFrom.m_TerminalActions[stateId][x];
        }
        for (int x=0; x<m_Counts[stateId].m_NumNonterms; x++) {
            m_NonterminalActions[stateId][x] = copyFrom.m_NonterminalActions[stateId][x];
        }
    }

    // Allocate the rule table
    m_Rules = new reduce_rule[m_NumRules];
    for (int ruleId=0; ruleId<m_NumRules; ruleId++) {
        m_Rules[ruleId] = copyFrom.m_Rules[ruleId];
    }
}

/// \brief Assignment
parser_tables& parser_tables::operator=(const parser_tables& copyFrom) {
    if (&copyFrom == this) return *this;

    // TODO

    return *this;
}

/// \brief Destructor
parser_tables::~parser_tables() {
    // Destroy each entry in the parser table
    for (int x=0; x<m_NumStates; x++) {
        delete[] m_NonterminalActions[x];
        delete[] m_TerminalActions[x];
    }

    // Destroy the tables themselves
    delete[] m_NonterminalActions;
    delete[] m_TerminalActions;
    delete[] m_Rules;
    delete[] m_Counts;
}
<commit_msg>Was writing out the rule ID instead of the nonterminal identifier to the rules Wasn't moving the position on in the nonterminal array<commit_after>//
//  parser_tables.cpp
//  Parse
//
//  Created by Andrew Hunter on 07/05/2011.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#include <algorithm>

#include "parser_tables.h"

using namespace std;
using namespace contextfree;
using namespace lr;

/// \brief Ranks actions in the order they should appear in the final table
static inline int action_score(int action) {
    switch (action) {
        case lr_action::act_weakreduce:
            // Weak reduce actions come first as they should always be performed if their symbol will produce a shift
            return 0;

        case lr_action::act_shift:
            // Shift actions are preferred if there's a conflict
            return 1;

        case lr_action::act_reduce:
            // Reduce actions have the lowest priority
            return 2;

        case lr_action::act_goto:
            // Gotos never actually produce a clash
            return 3;

        default:
            return 4;
    }
}

static inline bool compare_actions(const parser_tables::action& a, const parser_tables::action& b) {
    // First, compare on symbol IDs
    if (a.m_SymbolId < b.m_SymbolId) return true;

    // Next, compare on action types
    if (action_score(a.m_Type) < action_score(b.m_Type)) return true;

    // Actions are equal
    return false;
}

/// \brief Creates a parser from the result of the specified builder class
parser_tables::parser_tables(const lalr_builder& builder) {
    // Allocate the tables
    m_NumStates             = builder.count_states();
    m_NonterminalActions    = new action*[m_NumStates];
    m_TerminalActions       = new action*[m_NumStates];
    m_Counts                = new action_count[m_NumStates];

    contextfree::end_of_input eoi;
    m_EndOfInput            = builder.gram().identifier_for_item(eoi);

    const grammar& gram = builder.gram();

    map<int, int> ruleIds;                          // Maps their rule IDs to our rule IDs

    // Build up the tables for each state
    for (int stateId = 0; stateId < m_NumStates; stateId++) {
        // Get the actions for this state
        const lr_action_set& actions = builder.actions_for_state(stateId);

        // Count the number of terminal and nonterminal actions
        int termCount       = 0;
        int nontermCount    = 0;

        for (lr_action_set::const_iterator nextAction = actions.begin(); nextAction != actions.end(); nextAction++) {
            if ((*nextAction)->item()->type() == item::terminal) {
                termCount++;
            } else {
                nontermCount++;
            }
        }

        // Allocate action tables of the appropriate size
        action* termActions     = new action[termCount];
        action* nontermActions  = new action[nontermCount];

        int     termPos         = 0;                    // Current item in the terminal table
        int     nontermPos      = 0;                    // Current item in the nonterminal table

        // Fill up the actions (not in order)
        for (lr_action_set::const_iterator nextAction = actions.begin(); nextAction != actions.end(); nextAction++) {
            // Get the next state
            int nextState   = (*nextAction)->next_state();
            int type        = (*nextAction)->type();

            // If the next action is a reduce action, then the next state should actually be the rule to reduce
            if (type == lr_action::act_reduce || type == lr_action::act_weakreduce) {
                // Look up the ID we assigned this rule
                int                     ruleId  = (*nextAction)->rule()->identifier(gram);
                map<int, int>::iterator found   = ruleIds.find(ruleId);

                if (found == ruleIds.end()) {
                    nextState       = (int)ruleIds.size();
                    ruleIds[ruleId] = nextState;
                } else {
                    nextState       = found->second;
                }
            }

            if ((*nextAction)->item()->type() == item::terminal) {
                // Add a new terminal action
                termActions[termPos].m_Type             = type;
                termActions[termPos].m_NextState        = nextState;
                termActions[termPos].m_SymbolId         = (*nextAction)->item()->symbol();

                termPos++;
            } else {
                // Add a new nonterminal action
                nontermActions[nontermPos].m_Type       = type;
                nontermActions[nontermPos].m_NextState  = nextState;
                nontermActions[nontermPos].m_SymbolId   = gram.identifier_for_item((*nextAction)->item());

                nontermPos++;
            }
        }

        // Sort the actions for this state
        std::sort(termActions, termActions + termCount, compare_actions);
        std::sort(nontermActions, nontermActions + nontermCount, compare_actions);

        // Store the actions in the table
        m_TerminalActions[stateId]      = termActions;
        m_NonterminalActions[stateId]   = nontermActions;
        m_Counts[stateId].m_NumTerms    = termCount;
        m_Counts[stateId].m_NumNonterms = nontermCount;
    }

    // Fill in the rule table
    m_NumRules  = (int)ruleIds.size();
    m_Rules     = new reduce_rule[ruleIds.size()];

    for (map<int, int>::iterator ruleId = ruleIds.begin(); ruleId != ruleIds.end(); ruleId++) {
        const rule_container& rule = gram.rule_with_identifier(ruleId->first);

        m_Rules[ruleId->second].m_Identifier    = gram.identifier_for_item(rule->nonterminal());
        m_Rules[ruleId->second].m_Length        = (int)rule->items().size();
    }
}

/// \brief Copy constructor
parser_tables::parser_tables(const parser_tables& copyFrom)
: m_NumStates(copyFrom.m_NumStates)
, m_NumRules(copyFrom.m_NumRules)
, m_EndOfInput(copyFrom.m_EndOfInput) {
    // Allocate the action tables
    m_TerminalActions       = new action*[m_NumStates];
    m_NonterminalActions    = new action*[m_NumStates];
    m_Counts                = new action_count[m_NumStates];

    // Copy the states
    for (int stateId=0; stateId<m_NumStates; stateId++) {
        // Copy the counts
        m_Counts[stateId] = copyFrom.m_Counts[stateId];

        // Allocate the array for this state
        m_TerminalActions[stateId]      = new action[m_Counts[stateId].m_NumTerms];
        m_NonterminalActions[stateId]   = new action[m_Counts[stateId].m_NumNonterms];

        // Copy the terminals and nonterminals
        for (int x=0; x<m_Counts[stateId].m_NumTerms; x++) {
            m_TerminalActions[stateId][x] = copyFrom.m_TerminalActions[stateId][x];
        }
        for (int x=0; x<m_Counts[stateId].m_NumNonterms; x++) {
            m_NonterminalActions[stateId][x] = copyFrom.m_NonterminalActions[stateId][x];
        }
    }

    // Allocate the rule table
    m_Rules = new reduce_rule[m_NumRules];
    for (int ruleId=0; ruleId<m_NumRules; ruleId++) {
        m_Rules[ruleId] = copyFrom.m_Rules[ruleId];
    }
}

/// \brief Assignment
parser_tables& parser_tables::operator=(const parser_tables& copyFrom) {
    if (&copyFrom == this) return *this;

    // TODO

    return *this;
}

/// \brief Destructor
parser_tables::~parser_tables() {
    // Destroy each entry in the parser table
    for (int x=0; x<m_NumStates; x++) {
        delete[] m_NonterminalActions[x];
        delete[] m_TerminalActions[x];
    }

    // Destroy the tables themselves
    delete[] m_NonterminalActions;
    delete[] m_TerminalActions;
    delete[] m_Rules;
    delete[] m_Counts;
}
<|endoftext|>


<commit_before>//--------------------------------------------------------------------------------
// compile test code for g++
//--------------------------------------------------------------------------------

// [command sample]
// $ cd simplesp
// $ g++ compiletest.cpp -I ./ -std=c++11
// $ ./a.exe

#include "simplesp.h"
using namespace sp;

int main(){
	printf("compile test\n");

    return 0;
}
<commit_msg>refactoring<commit_after>//--------------------------------------------------------------------------------
// compile test code for g++
//--------------------------------------------------------------------------------

// [command sample]
// $ cd simplesp
// $ g++ compiletest.cpp -I ./ -std=c++11
// $ ./a.exe

#include "simplesp.h"
using namespace sp;

int main() {
    printf("compile test\n");

    return 0;
}
<|endoftext|>


<commit_before>#include <X11/Xlib.h>
#include <X11/keysym.h>
#include <irrlicht/Keycodes.h>
#include <stdio.h>
#include <unistd.h>

#define RETURN 13
#define BS 8
#define SPACE 32

XKeyEvent createKeyEvent(Display *display,
                         Window &win,
                         Window &winRoot,
                         bool press,
                         int keycode,
                         int modifiers)
{
  // Largely lifted from
  // http://www.doctort.org/adam/nerd-notes/x11-fake-keypress-event.html
  XKeyEvent event;
  event.display = display;
  event.window = win;
  event.root = winRoot;
  event.subwindow = None;
  event.time = CurrentTime;
  event.x = 1;
  event.y = 1;
  event.x_root = 1;
  event.y_root = 1;
  event.same_screen = True;
  event.keycode = XKeysymToKeycode(display, keycode);
  event.state = modifiers;

  if(press)
    event.type = KeyPress;
  else
    event.type = KeyRelease;

  return event;
}

class X11Display {
  public:
    Display *display;
    Window winRoot;
    X11Display(const char* dispName);
    ~X11Display();
    int sendKeyEvent(int keycode, bool keyDown, int mod);
};

// TODO: fail out if display doesn't initialize
X11Display::X11Display(const char* dispName) :
  display(XOpenDisplay(dispName)),
  winRoot(XDefaultRootWindow(display)) {}

X11Display::~X11Display() { XCloseDisplay(display); }

int X11Display::sendKeyEvent(int keycode, bool keyDown, int mod = 0)
{
  if(-1==keycode)
    return 0;
  printf("sendKeyEvent keycode: %d\n", keycode);
  Window winFocus;
  int revert;

  XGetInputFocus(display, &winFocus, &revert);
  XKeyEvent event = createKeyEvent(display, winFocus, winRoot, keyDown, keycode, mod);
  XSendEvent(event.display, event.window, True, KeyPressMask, (XEvent*)&event);
  return 0;
}

bool IsModKey(int keycode) {
  switch(keycode){
    //shift, control and menu (both left and right) are consecutive
    case irr::KEY_LSHIFT ... irr::KEY_RMENU :
      return true;
    default :
      return false;
  }
}

int mapKeyCode(int irrcode){
  switch(irrcode){
    //FIXME: only to get rid of arrow spam
    case 37 ... 40:
      return -1;
    case irr::KEY_BACK :
      return XK_BackSpace;
    case irr::KEY_TAB :
      return XK_Tab;
    //FIXME: Should escape be a modifier too?
    case irr::KEY_ESCAPE :
      return XK_Escape;
    case irr::KEY_RETURN :
      return XK_Return;
    case irr::KEY_MINUS :
      return XK_minus;
    case irr::KEY_OEM_1 :
      return XK_semicolon;
    case irr::KEY_OEM_2 :
      return XK_slash;
    case irr::KEY_OEM_7 :
      return XK_apostrophe;
    case irr::KEY_PERIOD :
      return XK_period;
    case irr::KEY_OEM_5 :
      return XK_backslash;
    case irr::KEY_LSHIFT :
      return XK_Shift_L;
    case irr::KEY_RSHIFT :
      return XK_Shift_R;
    case irr::KEY_LCONTROL :
      return XK_Control_L;
    case irr::KEY_RCONTROL :
      return XK_Control_R;
    default:
      return irrcode;
  }
}

<commit_msg>Added comma<commit_after>#include <X11/Xlib.h>
#include <X11/keysym.h>
#include <irrlicht/Keycodes.h>
#include <stdio.h>
#include <unistd.h>

#define RETURN 13
#define BS 8
#define SPACE 32

XKeyEvent createKeyEvent(Display *display,
                         Window &win,
                         Window &winRoot,
                         bool press,
                         int keycode,
                         int modifiers)
{
  // Largely lifted from
  // http://www.doctort.org/adam/nerd-notes/x11-fake-keypress-event.html
  XKeyEvent event;
  event.display = display;
  event.window = win;
  event.root = winRoot;
  event.subwindow = None;
  event.time = CurrentTime;
  event.x = 1;
  event.y = 1;
  event.x_root = 1;
  event.y_root = 1;
  event.same_screen = True;
  event.keycode = XKeysymToKeycode(display, keycode);
  event.state = modifiers;

  if(press)
    event.type = KeyPress;
  else
    event.type = KeyRelease;

  return event;
}

class X11Display {
  public:
    Display *display;
    Window winRoot;
    X11Display(const char* dispName);
    ~X11Display();
    int sendKeyEvent(int keycode, bool keyDown, int mod);
};

// TODO: fail out if display doesn't initialize
X11Display::X11Display(const char* dispName) :
  display(XOpenDisplay(dispName)),
  winRoot(XDefaultRootWindow(display)) {}

X11Display::~X11Display() { XCloseDisplay(display); }

int X11Display::sendKeyEvent(int keycode, bool keyDown, int mod = 0)
{
  if(-1==keycode)
    return 0;
  printf("sendKeyEvent keycode: %d\n", keycode);
  Window winFocus;
  int revert;

  XGetInputFocus(display, &winFocus, &revert);
  XKeyEvent event = createKeyEvent(display, winFocus, winRoot, keyDown, keycode, mod);
  XSendEvent(event.display, event.window, True, KeyPressMask, (XEvent*)&event);
  return 0;
}

bool IsModKey(int keycode) {
  switch(keycode){
    //shift, control and menu (both left and right) are consecutive
    case irr::KEY_LSHIFT ... irr::KEY_RMENU :
      return true;
    default :
      return false;
  }
}

int mapKeyCode(int irrcode){
  switch(irrcode){
    //FIXME: only to get rid of arrow spam
    case 37 ... 40:
      return -1;
    case irr::KEY_BACK :
      return XK_BackSpace;
    case irr::KEY_TAB :
      return XK_Tab;
    //FIXME: Should escape be a modifier too?
    case irr::KEY_ESCAPE :
      return XK_Escape;
    case irr::KEY_RETURN :
      return XK_Return;
    case irr::KEY_MINUS :
      return XK_minus;
    case irr::KEY_OEM_1 :
      return XK_semicolon;
    case irr::KEY_OEM_2 :
      return XK_slash;
    case irr::KEY_OEM_7 :
      return XK_apostrophe;
    case irr::KEY_PERIOD :
      return XK_period;
    case irr::KEY_COMMA :
      return XK_comma;
    case irr::KEY_OEM_5 :
      return XK_backslash;
    case irr::KEY_LSHIFT :
      return XK_Shift_L;
    case irr::KEY_RSHIFT :
      return XK_Shift_R;
    case irr::KEY_LCONTROL :
      return XK_Control_L;
    case irr::KEY_RCONTROL :
      return XK_Control_R;
    default:
      return irrcode;
  }
}

<|endoftext|>


<commit_before>#include "entity_groups.h"
#include "core/blob.h"
#include "core/string.h"
#include "universe/universe.h"

namespace Lumix
{
EntityGroups::EntityGroups(IAllocator& allocator)
	: m_groups(allocator)
	, m_group_infos(allocator)
	, m_entity_to_group_map(allocator)
	, m_allocator(allocator)
	, m_universe(nullptr)
{
	m_groups.emplace(allocator);
	auto& info = m_group_infos.emplace();
	copyString(info.name, "default");
}

int EntityGroups::getGroup(const char* name) const
{
	for (int i = 0, c = m_group_infos.size(); i < c; ++i)
	{
		if (compareString(m_group_infos[i].name, name) == 0) return i;
	}
	return -1;
}

void EntityGroups::freezeGroup(int idx, bool freeze)
{
	m_group_infos[idx].frozen = freeze;
}

bool EntityGroups::isGroupFrozen(int idx) const
{
	return m_group_infos[idx].frozen;
}

void EntityGroups::allEntitiesToDefault()
{
	ASSERT(m_groups.size() == 1);
	ASSERT(m_groups[0].empty());
	ASSERT(m_universe);

	for (int i = 0, c = m_universe->getEntityCount(); i < c; ++i)
	{
		Entity entity = m_universe->getEntityFromDenseIdx(i);
		m_groups[0].push(entity);
		m_entity_to_group_map[entity] = 0;
	}
}

void EntityGroups::deleteGroup(int idx)
{
	if (m_groups.size() == 1) return;
	int default_idx = idx == 0 ? 1 : 0;
	for (auto e : m_groups[idx])
	{
		m_groups[default_idx].push(e);
	}
	m_groups.eraseFast(idx);
	m_group_infos.eraseFast(idx);
}

void EntityGroups::createGroup(const char* name)
{
	if (name[0] == 0) return;
	for (auto& i : m_group_infos)
	{
		if (compareString(i.name, name) == 0) return;
	}

	m_groups.emplace(m_allocator);
	auto& group_info = m_group_infos.emplace();
	copyString(group_info.name, name);
	group_info.frozen = false;
}

void EntityGroups::setUniverse(Universe* universe)
{
	if (m_universe)
	{
		m_universe->entityCreated().unbind<EntityGroups, &EntityGroups::onEntityCreated>(this);
		m_universe->entityDestroyed().unbind<EntityGroups, &EntityGroups::onEntityDestroyed>(this);
	}

	m_universe = universe;
	m_group_infos.clear();
	m_groups.clear();
	m_groups.emplace(m_allocator);
	auto& info = m_group_infos.emplace();
	copyString(info.name, "default");

	if (m_universe)
	{
		m_universe->entityCreated().bind<EntityGroups, &EntityGroups::onEntityCreated>(this);
		m_universe->entityDestroyed().bind<EntityGroups, &EntityGroups::onEntityDestroyed>(this);
	}
}

int EntityGroups::getGroupEntitiesCount(int idx) const
{
	return m_groups[idx].size();
}

void EntityGroups::onEntityCreated(Entity entity)
{
	m_groups[0].push(entity);
	ASSERT(entity == m_entity_to_group_map.size());
	m_entity_to_group_map.push(0);
}

void EntityGroups::onEntityDestroyed(Entity entity)
{
	removeFromGroup(entity);
	m_entity_to_group_map[entity] = -1;
}

void EntityGroups::setGroup(Entity entity, int group)
{
	removeFromGroup(entity);
	m_groups[group].push(entity);
	m_entity_to_group_map[entity] = group;
}

void EntityGroups::removeFromGroup(Entity entity)
{
	m_groups[m_entity_to_group_map[entity]].eraseItemFast(entity);
	m_entity_to_group_map[entity] = -1;
}

void EntityGroups::serialize(OutputBlob& blob)
{
	ASSERT(sizeof(m_group_infos[0]) == 20);

	blob.write(m_group_infos.size());
	blob.write(&m_group_infos[0], m_group_infos.size() * sizeof(m_group_infos[0]));

	for(auto& i : m_groups)
	{
		blob.write(i.size());
		if(!i.empty()) blob.write(&i[0], i.size() * sizeof(i[0]));
	}
}

void EntityGroups::deserialize(InputBlob& blob)
{
	int count;
	blob.read(count);
	m_group_infos.resize(count);
	for (int i = 0; i < count; ++i)
	{
		blob.read(m_group_infos[i].name, sizeof(m_group_infos[i].name));
		m_group_infos[i].frozen = false;
	}

	m_groups.clear();
	int entity_count = 0;
	for(int i = 0; i < count; ++i)
	{
		int group_size;
		auto& group = m_groups.emplace(m_allocator);
		blob.read(group_size);
		entity_count += group_size;
		group.resize(group_size);
		if(group_size > 0) blob.read(&group[0], group_size * sizeof(group[0]));
	}
	m_entity_to_group_map.resize(entity_count);
	for (int i = 0; i < m_groups.size(); ++i)
	{
		for (auto e : m_groups[i])
		{
			m_entity_to_group_map[e] = i;
		}
	}
}

const Entity* EntityGroups::getGroupEntities(int idx) const
{
	return &m_groups[idx][0];
}

int EntityGroups::getGroupCount() const
{
	return m_groups.size();
}

const char* EntityGroups::getGroupName(int idx) const
{
	return m_group_infos[idx].name;
}

void EntityGroups::setGroupName(int idx, const char* name)
{
	copyString(m_group_infos[idx].name, name);
}
} // namespace Lumix
<commit_msg>fixed crash when there are spaces in the ids of entities<commit_after>#include "entity_groups.h"
#include "core/blob.h"
#include "core/string.h"
#include "universe/universe.h"

namespace Lumix
{
EntityGroups::EntityGroups(IAllocator& allocator)
	: m_groups(allocator)
	, m_group_infos(allocator)
	, m_entity_to_group_map(allocator)
	, m_allocator(allocator)
	, m_universe(nullptr)
{
	m_groups.emplace(allocator);
	auto& info = m_group_infos.emplace();
	copyString(info.name, "default");
}

int EntityGroups::getGroup(const char* name) const
{
	for (int i = 0, c = m_group_infos.size(); i < c; ++i)
	{
		if (compareString(m_group_infos[i].name, name) == 0) return i;
	}
	return -1;
}

void EntityGroups::freezeGroup(int idx, bool freeze)
{
	m_group_infos[idx].frozen = freeze;
}

bool EntityGroups::isGroupFrozen(int idx) const
{
	return m_group_infos[idx].frozen;
}

void EntityGroups::allEntitiesToDefault()
{
	ASSERT(m_groups.size() == 1);
	ASSERT(m_groups[0].empty());
	ASSERT(m_universe);

	for (int i = 0, c = m_universe->getEntityCount(); i < c; ++i)
	{
		Entity entity = m_universe->getEntityFromDenseIdx(i);
		m_groups[0].push(entity);
		m_entity_to_group_map[entity] = 0;
	}
}

void EntityGroups::deleteGroup(int idx)
{
	if (m_groups.size() == 1) return;
	int default_idx = idx == 0 ? 1 : 0;
	for (auto e : m_groups[idx])
	{
		m_groups[default_idx].push(e);
	}
	m_groups.eraseFast(idx);
	m_group_infos.eraseFast(idx);
}

void EntityGroups::createGroup(const char* name)
{
	if (name[0] == 0) return;
	for (auto& i : m_group_infos)
	{
		if (compareString(i.name, name) == 0) return;
	}

	m_groups.emplace(m_allocator);
	auto& group_info = m_group_infos.emplace();
	copyString(group_info.name, name);
	group_info.frozen = false;
}

void EntityGroups::setUniverse(Universe* universe)
{
	if (m_universe)
	{
		m_universe->entityCreated().unbind<EntityGroups, &EntityGroups::onEntityCreated>(this);
		m_universe->entityDestroyed().unbind<EntityGroups, &EntityGroups::onEntityDestroyed>(this);
	}

	m_universe = universe;
	m_group_infos.clear();
	m_groups.clear();
	m_groups.emplace(m_allocator);
	auto& info = m_group_infos.emplace();
	copyString(info.name, "default");

	if (m_universe)
	{
		m_universe->entityCreated().bind<EntityGroups, &EntityGroups::onEntityCreated>(this);
		m_universe->entityDestroyed().bind<EntityGroups, &EntityGroups::onEntityDestroyed>(this);
	}
}

int EntityGroups::getGroupEntitiesCount(int idx) const
{
	return m_groups[idx].size();
}

void EntityGroups::onEntityCreated(Entity entity)
{
	m_groups[0].push(entity);
	if (entity >= m_entity_to_group_map.size()) m_entity_to_group_map.resize(entity + 1);
	m_entity_to_group_map[entity] = 0;
}

void EntityGroups::onEntityDestroyed(Entity entity)
{
	removeFromGroup(entity);
	m_entity_to_group_map[entity] = -1;
}

void EntityGroups::setGroup(Entity entity, int group)
{
	removeFromGroup(entity);
	m_groups[group].push(entity);
	m_entity_to_group_map[entity] = group;
}

void EntityGroups::removeFromGroup(Entity entity)
{
	m_groups[m_entity_to_group_map[entity]].eraseItemFast(entity);
	m_entity_to_group_map[entity] = -1;
}

void EntityGroups::serialize(OutputBlob& blob)
{
	ASSERT(sizeof(m_group_infos[0]) == 20);

	blob.write(m_group_infos.size());
	blob.write(&m_group_infos[0], m_group_infos.size() * sizeof(m_group_infos[0]));

	for(auto& i : m_groups)
	{
		blob.write(i.size());
		if(!i.empty()) blob.write(&i[0], i.size() * sizeof(i[0]));
	}
}

void EntityGroups::deserialize(InputBlob& blob)
{
	int count;
	blob.read(count);
	m_group_infos.resize(count);
	for (int i = 0; i < count; ++i)
	{
		blob.read(m_group_infos[i].name, sizeof(m_group_infos[i].name));
		m_group_infos[i].frozen = false;
	}

	m_groups.clear();
	int entity_count = 0;
	for(int i = 0; i < count; ++i)
	{
		int group_size;
		auto& group = m_groups.emplace(m_allocator);
		blob.read(group_size);
		entity_count += group_size;
		group.resize(group_size);
		if(group_size > 0) blob.read(&group[0], group_size * sizeof(group[0]));
	}
	m_entity_to_group_map.resize(entity_count);
	for (int i = 0; i < m_groups.size(); ++i)
	{
		for (auto e : m_groups[i])
		{
			if (e >= m_entity_to_group_map.size()) m_entity_to_group_map.resize(e + 1);
			m_entity_to_group_map[e] = i;
		}
	}
}

const Entity* EntityGroups::getGroupEntities(int idx) const
{
	return &m_groups[idx][0];
}

int EntityGroups::getGroupCount() const
{
	return m_groups.size();
}

const char* EntityGroups::getGroupName(int idx) const
{
	return m_group_infos[idx].name;
}

void EntityGroups::setGroupName(int idx, const char* name)
{
	copyString(m_group_infos[idx].name, name);
}
} // namespace Lumix
<|endoftext|>


<commit_before>/*
Copyright (c) 2003-2010 Sony Pictures Imageworks Inc., et al.
All Rights Reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
  notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
  notice, this list of conditions and the following disclaimer in the
  documentation and/or other materials provided with the distribution.
* Neither the name of Sony Pictures Imageworks nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include <cstdio>
#include <iostream>
#include <iterator>

#include <OpenColorIO/OpenColorIO.h>

#include "FileTransform.h"
#include "Lut1DOp.h"
#include "Lut3DOp.h"
#include "ParseUtils.h"
#include "pystring/pystring.h"

OCIO_NAMESPACE_ENTER
{
    namespace
    {
        class LocalCachedFile : public CachedFile
        {
        public:
            LocalCachedFile () :
                has1D(false)
            {
                lut1D = OCIO_SHARED_PTR<Lut1D>(new Lut1D());
                lut3D = OCIO_SHARED_PTR<Lut3D>(new Lut3D());
            };
            ~LocalCachedFile() {};

            bool has1D;
            OCIO_SHARED_PTR<Lut1D> lut1D;
            OCIO_SHARED_PTR<Lut3D> lut3D;
        };

        typedef OCIO_SHARED_PTR<LocalCachedFile> LocalCachedFileRcPtr;

        class LocalFileFormat : public FileFormat
        {
        public:

            ~LocalFileFormat() {};

            virtual std::string GetExtension () const;

            virtual CachedFileRcPtr Load (std::istream & istream) const;

            virtual void BuildFileOps(OpRcPtrVec & ops,
                         CachedFileRcPtr untypedCachedFile,
                         const FileTransform& fileTransform,
                         TransformDirection dir) const;
        };

        // read the next non empty line
        // return whether it has succeeded or not.

        static bool
        nextline (std::istream &istream, std::string &line)
        {
            while ( istream.good() )
            {
                std::getline(istream, line);
                line = pystring::rstrip( line );
                if(!line.empty())
                {
                    return true;
                }
            }

            return false;
        }

        std::string
        LocalFileFormat::GetExtension() const { return "cub"; }

        CachedFileRcPtr
        LocalFileFormat::Load(std::istream & istream) const
        {
            // this shouldn't happen
            if(!istream)
            {
                throw Exception ("File stream empty when trying to read cub lut");
            }

            // Validate the file type
            std::string line;
            if(!nextline(istream, line) ||
               !pystring::startswith(pystring::lower(line), "# truelight cube"))
            {
                throw Exception("Lut doesn't seem to be a truelight cube lut.");
            }

            // Parse the file
            std::vector<float> raw1d;
            std::vector<float> raw3d;
            int size3d[] = { 0, 0, 0 };
            int size1d = 0;
            {
                std::vector<std::string> parts;
                std::vector<float> tmpfloats;

                bool in1d = false;
                bool in3d = false;

                while(nextline(istream, line))
                {
                    // Strip, lowercase, and split the line
                    pystring::split(pystring::lower(pystring::strip(line)), parts);

                    if(parts.empty()) continue;

                    if(pystring::startswith(parts[0],"#"))
                    {
                        if(parts.size() < 2) continue;

                        if(parts[1] == "width")
                        {
                            if(parts.size() != 5 ||
                               !StringToInt( &size3d[0], parts[2].c_str()) ||
                               !StringToInt( &size3d[1], parts[3].c_str()) ||
                               !StringToInt( &size3d[2], parts[4].c_str()))
                            {
                                throw Exception("Malformed width tag in truelight cube lut.");
                            }

                            raw3d.reserve(3*size3d[0]*size3d[1]*size3d[2]);
                        }
                        else if(parts[1] == "lutlength")
                        {
                            if(parts.size() != 3 ||
                               !StringToInt( &size1d, parts[2].c_str()))
                            {
                                throw Exception("Malformed lutlength tag in truelight cube lut.");
                            }
                            raw1d.reserve(3*size1d);
                        }
                        else if(parts[1] == "inputlut")
                        {
                            in1d = true;
                            in3d = false;
                        }
                        else if(parts[1] == "cube")
                        {
                            in3d = true;
                            in1d = false;
                        }
                    }

                    if(StringVecToFloatVec(tmpfloats, parts) && (tmpfloats.size() == 3))
                    {
                        if(in1d)
                        {
                            raw1d.push_back(tmpfloats[0]);
                            raw1d.push_back(tmpfloats[1]);
                            raw1d.push_back(tmpfloats[2]);
                        }
                        else if(in3d)
                        {
                            raw3d.push_back(tmpfloats[0]);
                            raw3d.push_back(tmpfloats[1]);
                            raw3d.push_back(tmpfloats[2]);
                        }
                        else
                        {
                            throw Exception("Parse error in truelight cube lut. Numbers outside of lut block.");
                        }
                    }
                }
            }

            // Interpret the parsed data, validate lut sizes

            if(size1d != static_cast<int>(raw1d.size()/3))
            {
                std::ostringstream os;
                os << "Parse error in truelight cube lut. ";
                os << "Incorrect number of lut1d entries. ";
                os << "Found " << raw1d.size()/3 << ", expected " << size1d << ".";
                throw Exception(os.str().c_str());
            }

            if(size3d[0]*size3d[1]*size3d[2] != static_cast<int>(raw3d.size()/3))
            {
                std::ostringstream os;
                os << "Parse error in truelight cube lut. ";
                os << "Incorrect number of lut3d entries. ";
                os << "Found " << raw3d.size()/3 << ", expected " << size3d[0]*size3d[1]*size3d[2] << ".";
                throw Exception(os.str().c_str());
            }

            if(size3d[0]*size3d[1]*size3d[2] == 0)
            {
                std::ostringstream os;
                os << "Parse error in truelight cube lut. ";
                os << "No 3D Lut entries found.";
                throw Exception(os.str().c_str());
            }

            LocalCachedFileRcPtr cachedFile = LocalCachedFileRcPtr(new LocalCachedFile());

            // Reformat 1D data
            if(size1d>0)
            {
                cachedFile->has1D = true;

                for(int channel=0; channel<3; ++channel)
                {
                    float descale = 1.0f / static_cast<float>(size3d[channel]);
                    cachedFile->lut1D->luts[channel].resize(size1d);
                    for(int i=0; i<size1d; ++i)
                    {
                        cachedFile->lut1D->luts[channel][i] = raw1d[3*i+channel] * descale;
                    }
                }

                // 1e-5 rel error is a good threshold when float numbers near 0
                // are written out with 6 decimal places of precision.  This is
                // a bit aggressive, I.e., changes in the 6th decimal place will
                // be considered roundoff error, but changes in the 5th decimal
                // will be considered lut 'intent'.
                // 1.0
                // 1.000005 equal to 1.0
                // 1.000007 equal to 1.0
                // 1.000010 not equal
                // 0.0
                // 0.000001 not equal

                cachedFile->lut1D->finalize(1e-5f);
            }

            // Reformat 3D data
            cachedFile->lut3D->size[0] = size3d[0];
            cachedFile->lut3D->size[1] = size3d[1];
            cachedFile->lut3D->size[2] = size3d[2];
            cachedFile->lut3D->lut = raw3d;
            cachedFile->lut3D->generateCacheID();

            return cachedFile;
        }

        void
        LocalFileFormat::BuildFileOps(OpRcPtrVec & ops,
                                      CachedFileRcPtr untypedCachedFile,
                                      const FileTransform& fileTransform,
                                      TransformDirection dir) const
        {
            LocalCachedFileRcPtr cachedFile = DynamicPtrCast<LocalCachedFile>(untypedCachedFile);

            // This should never happen.
            if(!cachedFile)
            {
                std::ostringstream os;
                os << "Cannot build CSP Op. Invalid cache type.";
                throw Exception(os.str().c_str());
            }

            TransformDirection newDir = CombineTransformDirections(dir,
                fileTransform.getDirection());
            if(newDir == TRANSFORM_DIR_UNKNOWN)
            {
                std::ostringstream os;
                os << "Cannot build file format transform,";
                os << " unspecified transform direction.";
                throw Exception(os.str().c_str());
            }

            // TODO: INTERP_LINEAR should not be hard-coded.
            // Instead query 'highest' interpolation?
            // (right now, it's linear). If cubic is added, consider
            // using it

            if(newDir == TRANSFORM_DIR_FORWARD)
            {
                if(cachedFile->has1D)
                {
                    CreateLut1DOp(ops, cachedFile->lut1D,
                                  INTERP_LINEAR, newDir);
                }

                CreateLut3DOp(ops, cachedFile->lut3D,
                              fileTransform.getInterpolation(), newDir);
            }
            else if(newDir == TRANSFORM_DIR_INVERSE)
            {
                CreateLut3DOp(ops, cachedFile->lut3D,
                              fileTransform.getInterpolation(), newDir);

                if(cachedFile->has1D)
                {
                    CreateLut1DOp(ops, cachedFile->lut1D,
                                  INTERP_LINEAR, newDir);
                }
            }
        }
    }

    struct AutoRegister
    {
        AutoRegister() { RegisterFileFormat(new LocalFileFormat); }
    };
    static AutoRegister registerIt;
}
OCIO_NAMESPACE_EXIT

///////////////////////////////////////////////////////////////////////////////

#ifdef OCIO_UNIT_TEST

namespace OCIO = OCIO_NAMESPACE;
#include "UnitTest.h"

BOOST_AUTO_TEST_SUITE( FileFormatTruelight_Unit_Tests )

BOOST_AUTO_TEST_CASE ( test_simple3D )
{
    // This lowers the red channel by 0.5, other channels are unaffected.
    const char * luttext = "# Truelight Cube v2.0\n"
       "# iDims 3\n"
       "# oDims 3\n"
       "# width 3 3 3\n"
       "# lutLength 4\n"
       "# InputLUT\n"
       "0.000000 0.000000 0.000000\n"
       "1.000000 1.000000 1.000000\n"
       "2.000000 2.000000 2.000000\n"
       "3.000000 3.000000 3.000000\n"
       "\n"
       "# Cube\n"
       "0.000000 0.000000 0.000000\n"
       "0.250000 0.000000 0.000000\n"
       "0.500000 0.000000 0.000000\n"
       "0.000000 0.500000 0.000000\n"
       "0.250000 0.500000 0.000000\n"
       "0.500000 0.500000 0.000000\n"
       "0.000000 1.000000 0.000000\n"
       "0.250000 1.000000 0.000000\n"
       "0.500000 1.000000 0.000000\n"
       "0.000000 0.000000 0.500000\n"
       "0.250000 0.000000 0.500000\n"
       "0.500000 0.000000 0.500000\n"
       "0.000000 0.500000 0.500000\n"
       "0.250000 0.500000 0.500000\n"
       "0.500000 0.500000 0.500000\n"
       "0.000000 1.000000 0.500000\n"
       "0.250000 1.000000 0.500000\n"
       "0.500000 1.000000 0.500000\n"
       "0.000000 0.000000 1.000000\n"
       "0.250000 0.000000 1.000000\n"
       "0.500000 0.000000 1.000000\n"
       "0.000000 0.500000 1.000000\n"
       "0.250000 0.500000 1.000000\n"
       "0.500000 0.500000 1.000000\n"
       "0.000000 1.000000 1.000000\n"
       "0.250000 1.000000 1.000000\n"
       "0.500000 1.000000 1.000000\n";

    std::istringstream simple3D;
    simple3D.str(luttext);

    // Load file
    OCIO::LocalFileFormat tester;
    OCIO::CachedFileRcPtr cachedFile = tester.Load(simple3D);
    OCIO::LocalCachedFileRcPtr csplut = OCIO::DynamicPtrCast<OCIO::LocalCachedFile>(cachedFile);

    // TODO: Test the results of truelight loading.
}

BOOST_AUTO_TEST_SUITE_END()

#endif // OCIO_UNIT_TEST
<commit_msg>updated Truelight fileformat error text<commit_after>/*
Copyright (c) 2003-2010 Sony Pictures Imageworks Inc., et al.
All Rights Reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
  notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
  notice, this list of conditions and the following disclaimer in the
  documentation and/or other materials provided with the distribution.
* Neither the name of Sony Pictures Imageworks nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include <cstdio>
#include <iostream>
#include <iterator>

#include <OpenColorIO/OpenColorIO.h>

#include "FileTransform.h"
#include "Lut1DOp.h"
#include "Lut3DOp.h"
#include "ParseUtils.h"
#include "pystring/pystring.h"

OCIO_NAMESPACE_ENTER
{
    namespace
    {
        class LocalCachedFile : public CachedFile
        {
        public:
            LocalCachedFile () :
                has1D(false)
            {
                lut1D = OCIO_SHARED_PTR<Lut1D>(new Lut1D());
                lut3D = OCIO_SHARED_PTR<Lut3D>(new Lut3D());
            };
            ~LocalCachedFile() {};

            bool has1D;
            OCIO_SHARED_PTR<Lut1D> lut1D;
            OCIO_SHARED_PTR<Lut3D> lut3D;
        };

        typedef OCIO_SHARED_PTR<LocalCachedFile> LocalCachedFileRcPtr;

        class LocalFileFormat : public FileFormat
        {
        public:

            ~LocalFileFormat() {};

            virtual std::string GetExtension () const;

            virtual CachedFileRcPtr Load (std::istream & istream) const;

            virtual void BuildFileOps(OpRcPtrVec & ops,
                         CachedFileRcPtr untypedCachedFile,
                         const FileTransform& fileTransform,
                         TransformDirection dir) const;
        };

        // read the next non empty line
        // return whether it has succeeded or not.

        static bool
        nextline (std::istream &istream, std::string &line)
        {
            while ( istream.good() )
            {
                std::getline(istream, line);
                line = pystring::rstrip( line );
                if(!line.empty())
                {
                    return true;
                }
            }

            return false;
        }

        std::string
        LocalFileFormat::GetExtension() const { return "cub"; }

        CachedFileRcPtr
        LocalFileFormat::Load(std::istream & istream) const
        {
            // this shouldn't happen
            if(!istream)
            {
                throw Exception ("File stream empty when trying to read Truelight .cub lut");
            }

            // Validate the file type
            std::string line;
            if(!nextline(istream, line) ||
               !pystring::startswith(pystring::lower(line), "# truelight cube"))
            {
                throw Exception("Lut doesn't seem to be a Truelight .cub lut.");
            }

            // Parse the file
            std::vector<float> raw1d;
            std::vector<float> raw3d;
            int size3d[] = { 0, 0, 0 };
            int size1d = 0;
            {
                std::vector<std::string> parts;
                std::vector<float> tmpfloats;

                bool in1d = false;
                bool in3d = false;

                while(nextline(istream, line))
                {
                    // Strip, lowercase, and split the line
                    pystring::split(pystring::lower(pystring::strip(line)), parts);

                    if(parts.empty()) continue;

                    if(pystring::startswith(parts[0],"#"))
                    {
                        if(parts.size() < 2) continue;

                        if(parts[1] == "width")
                        {
                            if(parts.size() != 5 ||
                               !StringToInt( &size3d[0], parts[2].c_str()) ||
                               !StringToInt( &size3d[1], parts[3].c_str()) ||
                               !StringToInt( &size3d[2], parts[4].c_str()))
                            {
                                throw Exception("Malformed width tag in Truelight .cub lut.");
                            }

                            raw3d.reserve(3*size3d[0]*size3d[1]*size3d[2]);
                        }
                        else if(parts[1] == "lutlength")
                        {
                            if(parts.size() != 3 ||
                               !StringToInt( &size1d, parts[2].c_str()))
                            {
                                throw Exception("Malformed lutlength tag in Truelight .cub lut.");
                            }
                            raw1d.reserve(3*size1d);
                        }
                        else if(parts[1] == "inputlut")
                        {
                            in1d = true;
                            in3d = false;
                        }
                        else if(parts[1] == "cube")
                        {
                            in3d = true;
                            in1d = false;
                        }
                    }

                    if(StringVecToFloatVec(tmpfloats, parts) && (tmpfloats.size() == 3))
                    {
                        if(in1d)
                        {
                            raw1d.push_back(tmpfloats[0]);
                            raw1d.push_back(tmpfloats[1]);
                            raw1d.push_back(tmpfloats[2]);
                        }
                        else if(in3d)
                        {
                            raw3d.push_back(tmpfloats[0]);
                            raw3d.push_back(tmpfloats[1]);
                            raw3d.push_back(tmpfloats[2]);
                        }
                        else
                        {
                            throw Exception("Parse error in Truelight .cub lut. Numbers outside of lut block.");
                        }
                    }
                }
            }

            // Interpret the parsed data, validate lut sizes

            if(size1d != static_cast<int>(raw1d.size()/3))
            {
                std::ostringstream os;
                os << "Parse error in Truelight .cub lut. ";
                os << "Incorrect number of lut1d entries. ";
                os << "Found " << raw1d.size()/3 << ", expected " << size1d << ".";
                throw Exception(os.str().c_str());
            }

            if(size3d[0]*size3d[1]*size3d[2] != static_cast<int>(raw3d.size()/3))
            {
                std::ostringstream os;
                os << "Parse error in Truelight .cub lut. ";
                os << "Incorrect number of lut3d entries. ";
                os << "Found " << raw3d.size()/3 << ", expected " << size3d[0]*size3d[1]*size3d[2] << ".";
                throw Exception(os.str().c_str());
            }

            if(size3d[0]*size3d[1]*size3d[2] == 0)
            {
                std::ostringstream os;
                os << "Parse error in Truelight .cub lut. ";
                os << "No 3D Lut entries found.";
                throw Exception(os.str().c_str());
            }

            LocalCachedFileRcPtr cachedFile = LocalCachedFileRcPtr(new LocalCachedFile());

            // Reformat 1D data
            if(size1d>0)
            {
                cachedFile->has1D = true;

                for(int channel=0; channel<3; ++channel)
                {
                    float descale = 1.0f / static_cast<float>(size3d[channel]);
                    cachedFile->lut1D->luts[channel].resize(size1d);
                    for(int i=0; i<size1d; ++i)
                    {
                        cachedFile->lut1D->luts[channel][i] = raw1d[3*i+channel] * descale;
                    }
                }

                // 1e-5 rel error is a good threshold when float numbers near 0
                // are written out with 6 decimal places of precision.  This is
                // a bit aggressive, I.e., changes in the 6th decimal place will
                // be considered roundoff error, but changes in the 5th decimal
                // will be considered lut 'intent'.
                // 1.0
                // 1.000005 equal to 1.0
                // 1.000007 equal to 1.0
                // 1.000010 not equal
                // 0.0
                // 0.000001 not equal

                cachedFile->lut1D->finalize(1e-5f);
            }

            // Reformat 3D data
            cachedFile->lut3D->size[0] = size3d[0];
            cachedFile->lut3D->size[1] = size3d[1];
            cachedFile->lut3D->size[2] = size3d[2];
            cachedFile->lut3D->lut = raw3d;
            cachedFile->lut3D->generateCacheID();

            return cachedFile;
        }

        void
        LocalFileFormat::BuildFileOps(OpRcPtrVec & ops,
                                      CachedFileRcPtr untypedCachedFile,
                                      const FileTransform& fileTransform,
                                      TransformDirection dir) const
        {
            LocalCachedFileRcPtr cachedFile = DynamicPtrCast<LocalCachedFile>(untypedCachedFile);

            // This should never happen.
            if(!cachedFile)
            {
                std::ostringstream os;
                os << "Cannot build Truelight .cub Op. Invalid cache type.";
                throw Exception(os.str().c_str());
            }

            TransformDirection newDir = CombineTransformDirections(dir,
                fileTransform.getDirection());
            if(newDir == TRANSFORM_DIR_UNKNOWN)
            {
                std::ostringstream os;
                os << "Cannot build file format transform,";
                os << " unspecified transform direction.";
                throw Exception(os.str().c_str());
            }

            // TODO: INTERP_LINEAR should not be hard-coded.
            // Instead query 'highest' interpolation?
            // (right now, it's linear). If cubic is added, consider
            // using it

            if(newDir == TRANSFORM_DIR_FORWARD)
            {
                if(cachedFile->has1D)
                {
                    CreateLut1DOp(ops, cachedFile->lut1D,
                                  INTERP_LINEAR, newDir);
                }

                CreateLut3DOp(ops, cachedFile->lut3D,
                              fileTransform.getInterpolation(), newDir);
            }
            else if(newDir == TRANSFORM_DIR_INVERSE)
            {
                CreateLut3DOp(ops, cachedFile->lut3D,
                              fileTransform.getInterpolation(), newDir);

                if(cachedFile->has1D)
                {
                    CreateLut1DOp(ops, cachedFile->lut1D,
                                  INTERP_LINEAR, newDir);
                }
            }
        }
    }

    struct AutoRegister
    {
        AutoRegister() { RegisterFileFormat(new LocalFileFormat); }
    };
    static AutoRegister registerIt;
}
OCIO_NAMESPACE_EXIT

///////////////////////////////////////////////////////////////////////////////

#ifdef OCIO_UNIT_TEST

namespace OCIO = OCIO_NAMESPACE;
#include "UnitTest.h"

BOOST_AUTO_TEST_SUITE( FileFormatTruelight_Unit_Tests )

BOOST_AUTO_TEST_CASE ( test_simple3D )
{
    // This lowers the red channel by 0.5, other channels are unaffected.
    const char * luttext = "# Truelight Cube v2.0\n"
       "# iDims 3\n"
       "# oDims 3\n"
       "# width 3 3 3\n"
       "# lutLength 4\n"
       "# InputLUT\n"
       "0.000000 0.000000 0.000000\n"
       "1.000000 1.000000 1.000000\n"
       "2.000000 2.000000 2.000000\n"
       "3.000000 3.000000 3.000000\n"
       "\n"
       "# Cube\n"
       "0.000000 0.000000 0.000000\n"
       "0.250000 0.000000 0.000000\n"
       "0.500000 0.000000 0.000000\n"
       "0.000000 0.500000 0.000000\n"
       "0.250000 0.500000 0.000000\n"
       "0.500000 0.500000 0.000000\n"
       "0.000000 1.000000 0.000000\n"
       "0.250000 1.000000 0.000000\n"
       "0.500000 1.000000 0.000000\n"
       "0.000000 0.000000 0.500000\n"
       "0.250000 0.000000 0.500000\n"
       "0.500000 0.000000 0.500000\n"
       "0.000000 0.500000 0.500000\n"
       "0.250000 0.500000 0.500000\n"
       "0.500000 0.500000 0.500000\n"
       "0.000000 1.000000 0.500000\n"
       "0.250000 1.000000 0.500000\n"
       "0.500000 1.000000 0.500000\n"
       "0.000000 0.000000 1.000000\n"
       "0.250000 0.000000 1.000000\n"
       "0.500000 0.000000 1.000000\n"
       "0.000000 0.500000 1.000000\n"
       "0.250000 0.500000 1.000000\n"
       "0.500000 0.500000 1.000000\n"
       "0.000000 1.000000 1.000000\n"
       "0.250000 1.000000 1.000000\n"
       "0.500000 1.000000 1.000000\n";

    std::istringstream simple3D;
    simple3D.str(luttext);

    // Load file
    OCIO::LocalFileFormat tester;
    OCIO::CachedFileRcPtr cachedFile = tester.Load(simple3D);
    OCIO::LocalCachedFileRcPtr csplut = OCIO::DynamicPtrCast<OCIO::LocalCachedFile>(cachedFile);

    // TODO: Test the results of truelight loading.
}

BOOST_AUTO_TEST_SUITE_END()

#endif // OCIO_UNIT_TEST
<|endoftext|>