<commit_before>#pragma once

#include <Grappa.hpp>
#include <GlobalCompletionEvent.hpp>
#include <Addressing.hpp>
#include <Collective.hpp>

#include <functional>
#include <cstdint>
#include <vector>
#include <unordered_map>

GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, mr_mapping_runtime);
GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, mr_combining_runtime);
GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, mr_reducing_runtime);
GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, mr_reallocation_runtime);

namespace MapReduce {

extern Grappa::GlobalCompletionEvent default_mr_gce;

template <Grappa::GlobalCompletionEvent * GCE, typename RandomAccess, typename AF, typename CF>
void forall_symmetric(GlobalAddress<RandomAccess> vs, AF accessor, CF f ) {

  Grappa::on_all_cores([=] {
      Grappa::forall_here<async,GCE>(0, accessor(vs).size(), [=](int64_t start, int64_t iters) {
        for (int i=start; i<start+iters; i++) {
           auto e = accessor(vs)[i];
           f(e);
        }
      }); // local task blocks for all iterations
     });
  GCE->wait(); // block until all tasks are done
}

template <typename K, typename V, typename OutType>
struct Reducer {
  std::unordered_map<K, std::vector<V>> * groups;
  std::vector<OutType> * result;

  Reducer() : groups(new std::unordered_map<K, std::vector<V>>()), result(new std::vector<OutType>()) {}

} GRAPPA_BLOCK_ALIGNED; // using pointers as members because of #157

template <typename K, typename V, typename OutType, Grappa::GlobalCompletionEvent * GCE = &default_mr_gce>
void reducer_append( GlobalAddress<Reducer<K,V,OutType>> r, K key, V val ) {
  Grappa::delegate::call<async, GCE>(r.core(), [=] {
    VLOG(5) << "add (" << key << ", " << val << ") at " << r.pointer();
    std::vector<V>& slot = (*(r->groups))[key];
    slot.push_back(val);
  });
}
// Overload for GCE specified
template <Grappa::GlobalCompletionEvent * GCE, typename K, typename V, typename OutType>
void reducer_append( GlobalAddress<Reducer<K,V,OutType>> r, K key, V val ) {
  reducer_append<K,V,OutType,GCE>(r, key, val);
}

template <typename K, typename V, typename OutType>
struct MapperContext {
  GlobalAddress<Reducer<K,V,OutType>> reducers;
  int64_t num_reducers;

  MapperContext(GlobalAddress<Reducer<K,V,OutType>> reducers, int64_t num_reducers)
    : reducers(reducers)
    , num_reducers(num_reducers) {}

  template < Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
  void emitIntermediate(K key, V val) const {
    auto index = std::hash<K>()(key) % num_reducers;
    auto target = reducers + index;
    VLOG(5) << "index = " << index;
    reducer_append<GCE>( target, key, val );
  }
};

template <typename K, typename V>
struct Combiner {
  std::unordered_map<K, std::vector<V>> * groups;

  Combiner() : groups(new std::unordered_map<K, std::vector<V>>()) {}
} GRAPPA_BLOCK_ALIGNED;

template <typename K, typename V, typename OutType>
struct CombiningMapperContext {
  GlobalAddress<Reducer<K,V,OutType>> reducers;
  int64_t num_reducers;
  GlobalAddress<Combiner<K,V>> combining;

  CombiningMapperContext(GlobalAddress<Reducer<K,V,OutType>> reducers, GlobalAddress<Combiner<K,V>> combining, int64_t num_reducers)
    : reducers(reducers)
    , combining(combining)
    , num_reducers(num_reducers) {}

  // called within user map
  void emitIntermediate(K key, V val) const {
    DVLOG(5) << "push key " << key;
    std::vector<V>& slot = (*(combining->groups))[key];
    slot.push_back(val);
  }

  // called within user combine
  template < Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
  void emitCombinedIntermediate(K key, V val) const {
    auto index = std::hash<K>()(key) % num_reducers;
    auto target = reducers + index;
    DVLOG(5) << "index = " << index;
    reducer_append<GCE>( target, key, val );
  }

  void combineDoneWithKey( K key ) const {
    DVLOG(4) << "clear key " << key;
    (*(combining->groups))[key].clear();
  }
};

template < typename K, typename V, typename OutType, typename CombineF >
void combineExecute(CombiningMapperContext<K,V,OutType> ctx, CombineF combinef) {
  Grappa::on_all_cores([=] {
      auto local = ctx.combining->groups;
      for (auto it=local->begin(); it!=local->end(); ++it) {
        combinef( ctx, it->first, it->second );
        ctx.combineDoneWithKey( it->first );
      }
  });
}

template < typename K, typename V, typename OutType >
void emit(Reducer<K,V,OutType>& ctx, OutType result) {
  ctx.result->push_back(result);
}

template < typename T, typename K, typename V, typename OutType, typename MapF, Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
void mapExecute(MapperContext<K, V, OutType> ctx, GlobalAddress<T> keyvals, size_t num, MapF mf) {
  Grappa::forall<GCE>(keyvals, num, [=]( T& kv ) {
     mf(ctx, kv);
  });
}

//TODO get rid of this code duplication
template < typename T, typename K, typename V, typename OutType, typename MapF, Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
void mapExecute(CombiningMapperContext<K, V, OutType> ctx, GlobalAddress<T> keyvals, size_t num, MapF mf) {
  Grappa::forall<GCE>(keyvals, num, [=]( T& kv ) {
     mf(ctx, kv);
  });
}

// takes a symmetric global address
template < typename T, typename K, typename V, typename OutType, typename MapF, typename RA, Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
void mapExecute(MapperContext<K, V, OutType> ctx, GlobalAddress<RA> keyvals_sym, MapF mf) {
  forall_symmetric<GCE>(keyvals_sym, [](GlobalAddress<RA> collection) { return collection->data; },
                                     [=]( T& kv ) {
     mf(ctx, kv);
  });
}

template < typename K, typename V, typename OutType, typename ReduceF, Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
void reduceExecute(GlobalAddress<Reducer<K,V,OutType>> reducers, size_t num, ReduceF rf) {
  Grappa::forall<GCE>(reducers, num, [=]( int64_t i, Reducer<K,V,OutType>& reducer) {
    for ( auto local_it = reducer.groups->begin(); local_it!= reducer.groups->end(); ++local_it ) {
      rf(reducer, local_it->first, local_it->second );
    }
    // deallocate the group
    reducer.groups->clear();
  });
}

template < typename K, typename V, typename OutType >
GlobalAddress<Reducer<K,V,OutType>> allocateReducers( size_t num_reducers ) {
  auto reducers = Grappa::global_alloc<Reducer<K, V, OutType>>( num_reducers );
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r = Reducer<K,V,OutType>();
      });
  return reducers;
}

template < typename K, typename V >
GlobalAddress<Combiner<K,V>> allocateCombiners() {
    auto combiners = Grappa::symmetric_global_alloc<Combiner<K,V>>();
    Grappa::on_all_cores([=] {
        *(combiners.localize()) = Combiner<K,V>();
        });
    return combiners;
}

// MapRedue with local combiner
// This signature takes pointers to existing reducer/combiner structures so they can be reused
// (global array) reducers
// (symmetric alloc) combiners
template < typename T, typename K, typename V, typename OutType, typename MapF, typename CombineF, typename ReduceF>
void CombiningMapReduceJobExecute(GlobalAddress<T> keyvals, size_t num, GlobalAddress<Reducer<K,V,OutType>> reducers, GlobalAddress<Combiner<K,V>> combiners, size_t num_reducers/*Grappa::cores()*/, MapF mf, CombineF cf, ReduceF rf) {
  auto start_realloc = Grappa::walltime();
  // clear all data structures from a previous usage
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r.result->clear();
      r.groups->clear();
      });
  Grappa::on_all_cores([=] {
      combiners->groups->clear();
  });
  auto stop_realloc = Grappa::walltime();
  mr_reallocation_runtime += stop_realloc - start_realloc;

  auto start_map = stop_realloc;

  CombiningMapperContext<K,V,OutType> ctx(reducers, combiners, num_reducers);
  VLOG(1) << "map";
  mapExecute<T,K,V,OutType,MapF>(ctx, keyvals, num, mf);
  auto stop_map = Grappa::walltime();
  mr_mapping_runtime += stop_map - start_map;

  auto start_combine = stop_map;
  VLOG(1) << "combine/send";
  combineExecute<K,V,OutType>(ctx, cf);
  auto stop_combine = Grappa::walltime();

  mr_combining_runtime += stop_combine - start_combine;

  auto start_reduce = stop_combine;
  VLOG(1) << "reduce";
  reduceExecute<K,V,OutType,ReduceF>(reducers, num_reducers, rf);
  auto stop_reduce = Grappa::walltime();

  mr_reducing_runtime += stop_reduce - start_reduce;

  VLOG(1) << "complete";
}
// (global array) reducers
// (symmetric alloc) combiners
template < typename T, typename K, typename V, typename OutType, typename MapF, typename ReduceF>
void MapReduceJobExecute(GlobalAddress<T> keyvals, size_t num, GlobalAddress<Reducer<K,V,OutType>> reducers, size_t num_reducers/*Grappa::cores()*/, MapF mf, ReduceF rf) {
  auto start_realloc = Grappa::walltime();
  // clear all data structures from a previous usage
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r.result->clear();
      r.groups->clear();
      });
  auto stop_realloc = Grappa::walltime();
  mr_reallocation_runtime += stop_realloc - start_realloc;

  auto start_map = stop_realloc;

  MapperContext<K,V,OutType> ctx(reducers, num_reducers);
  VLOG(1) << "map";
  mapExecute<T,K,V,OutType,MapF>(ctx, keyvals, num, mf);
  auto stop_map = Grappa::walltime();
  mr_mapping_runtime += stop_map - start_map;

  auto start_reduce = stop_map;
  VLOG(1) << "reduce";
  reduceExecute<K,V,OutType,ReduceF>(reducers, num_reducers, rf);
  auto stop_reduce = Grappa::walltime();

  mr_reducing_runtime += stop_reduce - start_reduce;

  VLOG(1) << "complete";
}

// push-based map reduce
template < typename T, typename K, typename V, typename OutType, typename MapF, typename ReduceF>
GlobalAddress<Reducer<K,V,OutType>> MapReduceJobExecute(GlobalAddress<T> keyvals, size_t num, size_t num_reducers/*Grappa::cores()*/, MapF mf, ReduceF rf) {
  auto reducers = allocateReducers<K,V,OutType>( num_reducers );
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r = Reducer<K,V,OutType>();
      });
  //TODO symmetric alloc of a MapReduceContext GCE to allow concurrent jobs

  mapExecute<T,K,V,OutType,MapF>(MapperContext<K,V,OutType>(reducers, num_reducers), keyvals, num, mf);

  reduceExecute<K,V,OutType,ReduceF>(reducers, num_reducers, rf);

  return reducers;
}

template < typename T, typename K, typename V, typename OutType, typename MapF, typename ReduceF, typename RA>
GlobalAddress<Reducer<K,V,OutType>> MapReduceJobExecute(GlobalAddress<RA> keyvals, size_t num_reducers/*Grappa::cores()*/, MapF mf, ReduceF rf) {
  auto reducers = allocateReducers<K,V,OutType>( num_reducers );
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r = Reducer<K,V,OutType>();
      });
  //TODO symmetric alloc of a MapReduceContext GCE to allow concurrent jobs

  mapExecute<T,K,V,OutType,MapF>(MapperContext<K,V,OutType>(reducers, num_reducers), keyvals, mf);

  reduceExecute<K,V,OutType,ReduceF>(reducers, num_reducers, rf);

  return reducers;
}

} // end namespace

        /// TODO: join: quite annoying might as well do parallel(map/map) reduce because otherwise need to marshal data from two tables into an forall() iterator anyway
<commit_msg>debugging in mapreduce<commit_after>#pragma once

#include <Grappa.hpp>
#include <GlobalCompletionEvent.hpp>
#include <Addressing.hpp>
#include <Collective.hpp>

#include <functional>
#include <cstdint>
#include <vector>
#include <unordered_map>

GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, mr_mapping_runtime);
GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, mr_combining_runtime);
GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, mr_reducing_runtime);
GRAPPA_DECLARE_METRIC(SummarizingMetric<double>, mr_reallocation_runtime);

namespace MapReduce {

extern Grappa::GlobalCompletionEvent default_mr_gce;

template <Grappa::GlobalCompletionEvent * GCE, typename RandomAccess, typename AF, typename CF>
void forall_symmetric(GlobalAddress<RandomAccess> vs, AF accessor, CF f ) {

  Grappa::on_all_cores([=] {
      //VLOG(1) << "size = " << accessor(vs).size();
      Grappa::forall_here<async,GCE>(0, accessor(vs).size(), [=](int64_t start, int64_t iters) {
        auto coll = accessor(vs);
        for (int i=start; i<start+iters; i++) {
          auto e = coll[i];
          f(e);
        }
      }); // local task blocks for all iterations
     // VLOG(1) << "finished mine";
     });
     // VLOG(1) << "calling wait on me only";
  GCE->wait(); // block until all tasks are done
      //VLOG(1) << "wait done";
}

template <typename K, typename V, typename OutType>
struct Reducer {
  std::unordered_map<K, std::vector<V>> * groups;
  std::vector<OutType> * result;

  Reducer() : groups(new std::unordered_map<K, std::vector<V>>()), result(new std::vector<OutType>()) {}

} GRAPPA_BLOCK_ALIGNED; // using pointers as members because of #157

template <typename K, typename V, typename OutType, Grappa::GlobalCompletionEvent * GCE = &default_mr_gce>
void reducer_append( GlobalAddress<Reducer<K,V,OutType>> r, K key, V val ) {
  Grappa::delegate::call<async, GCE>(r.core(), [=] {
    VLOG(5) << "add (" << key << ", " << val << ") at " << r.pointer();
    std::vector<V>& slot = (*(r->groups))[key];
    slot.push_back(val);
  });
}
// Overload for GCE specified
template <Grappa::GlobalCompletionEvent * GCE, typename K, typename V, typename OutType>
void reducer_append( GlobalAddress<Reducer<K,V,OutType>> r, K key, V val ) {
  reducer_append<K,V,OutType,GCE>(r, key, val);
}

template <typename K, typename V, typename OutType>
struct MapperContext {
  GlobalAddress<Reducer<K,V,OutType>> reducers;
  int64_t num_reducers;

  MapperContext(GlobalAddress<Reducer<K,V,OutType>> reducers, int64_t num_reducers)
    : reducers(reducers)
    , num_reducers(num_reducers) {}

  template < Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
  void emitIntermediate(K key, V val) const {
    auto index = std::hash<K>()(key) % num_reducers;
    auto target = reducers + index;
    VLOG(5) << "index = " << index;
    reducer_append<GCE>( target, key, val );
  }
};

template <typename K, typename V>
struct Combiner {
  std::unordered_map<K, std::vector<V>> * groups;

  Combiner() : groups(new std::unordered_map<K, std::vector<V>>()) {}
} GRAPPA_BLOCK_ALIGNED;

template <typename K, typename V, typename OutType>
struct CombiningMapperContext {
  GlobalAddress<Reducer<K,V,OutType>> reducers;
  int64_t num_reducers;
  GlobalAddress<Combiner<K,V>> combining;

  CombiningMapperContext(GlobalAddress<Reducer<K,V,OutType>> reducers, GlobalAddress<Combiner<K,V>> combining, int64_t num_reducers)
    : reducers(reducers)
    , combining(combining)
    , num_reducers(num_reducers) {}

  // called within user map
  void emitIntermediate(K key, V val) const {
    DVLOG(5) << "push key " << key;
    std::vector<V>& slot = (*(combining->groups))[key];
    slot.push_back(val);
  }

  // called within user combine
  template < Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
  void emitCombinedIntermediate(K key, V val) const {
    auto index = std::hash<K>()(key) % num_reducers;
    auto target = reducers + index;
    DVLOG(5) << "index = " << index;
    reducer_append<GCE>( target, key, val );
  }

  void combineDoneWithKey( K key ) const {
    DVLOG(4) << "clear key " << key;
    (*(combining->groups))[key].clear();
  }
};

template < typename K, typename V, typename OutType, typename CombineF >
void combineExecute(CombiningMapperContext<K,V,OutType> ctx, CombineF combinef) {
  Grappa::on_all_cores([=] {
      auto local = ctx.combining->groups;
      for (auto it=local->begin(); it!=local->end(); ++it) {
        combinef( ctx, it->first, it->second );
        ctx.combineDoneWithKey( it->first );
      }
  });
}

template < typename K, typename V, typename OutType >
void emit(Reducer<K,V,OutType>& ctx, OutType result) {
  ctx.result->push_back(result);
}

template < typename T, typename K, typename V, typename OutType, typename MapF, Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
void mapExecute(MapperContext<K, V, OutType> ctx, GlobalAddress<T> keyvals, size_t num, MapF mf) {
  Grappa::forall<GCE>(keyvals, num, [=]( T& kv ) {
     mf(ctx, kv);
  });
}

//TODO get rid of this code duplication
template < typename T, typename K, typename V, typename OutType, typename MapF, Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
void mapExecute(CombiningMapperContext<K, V, OutType> ctx, GlobalAddress<T> keyvals, size_t num, MapF mf) {
  Grappa::forall<GCE>(keyvals, num, [=]( T& kv ) {
     mf(ctx, kv);
  });
}

// takes a symmetric global address
template < typename T, typename K, typename V, typename OutType, typename MapF, typename RA, Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
void mapExecute(MapperContext<K, V, OutType> ctx, GlobalAddress<RA> keyvals_sym, MapF mf) {
  forall_symmetric<GCE>(keyvals_sym, [](GlobalAddress<RA> collection) { return collection->data; },
                                     [=]( T& kv ) {
     mf(ctx, kv);
  });
}

template < typename K, typename V, typename OutType, typename ReduceF, Grappa::GlobalCompletionEvent * GCE=&default_mr_gce >
void reduceExecute(GlobalAddress<Reducer<K,V,OutType>> reducers, size_t num, ReduceF rf) {
  Grappa::forall<GCE>(reducers, num, [=]( int64_t i, Reducer<K,V,OutType>& reducer) {
    for ( auto local_it = reducer.groups->begin(); local_it!= reducer.groups->end(); ++local_it ) {
      rf(reducer, local_it->first, local_it->second );
    }
    // deallocate the group
    reducer.groups->clear();
  });
}

template < typename K, typename V, typename OutType >
GlobalAddress<Reducer<K,V,OutType>> allocateReducers( size_t num_reducers ) {
  auto reducers = Grappa::global_alloc<Reducer<K, V, OutType>>( num_reducers );
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r = Reducer<K,V,OutType>();
      });
  return reducers;
}

template < typename K, typename V >
GlobalAddress<Combiner<K,V>> allocateCombiners() {
    auto combiners = Grappa::symmetric_global_alloc<Combiner<K,V>>();
    Grappa::on_all_cores([=] {
        *(combiners.localize()) = Combiner<K,V>();
        });
    return combiners;
}

// MapRedue with local combiner
// This signature takes pointers to existing reducer/combiner structures so they can be reused
// (global array) reducers
// (symmetric alloc) combiners
template < typename T, typename K, typename V, typename OutType, typename MapF, typename CombineF, typename ReduceF>
void CombiningMapReduceJobExecute(GlobalAddress<T> keyvals, size_t num, GlobalAddress<Reducer<K,V,OutType>> reducers, GlobalAddress<Combiner<K,V>> combiners, size_t num_reducers/*Grappa::cores()*/, MapF mf, CombineF cf, ReduceF rf) {
  auto start_realloc = Grappa::walltime();
  // clear all data structures from a previous usage
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r.result->clear();
      r.groups->clear();
      });
  Grappa::on_all_cores([=] {
      combiners->groups->clear();
  });
  auto stop_realloc = Grappa::walltime();
  mr_reallocation_runtime += stop_realloc - start_realloc;

  auto start_map = stop_realloc;

  CombiningMapperContext<K,V,OutType> ctx(reducers, combiners, num_reducers);
  VLOG(1) << "map";
  mapExecute<T,K,V,OutType,MapF>(ctx, keyvals, num, mf);
  auto stop_map = Grappa::walltime();
  mr_mapping_runtime += stop_map - start_map;

  auto start_combine = stop_map;
  VLOG(1) << "combine/send";
  combineExecute<K,V,OutType>(ctx, cf);
  auto stop_combine = Grappa::walltime();

  mr_combining_runtime += stop_combine - start_combine;

  auto start_reduce = stop_combine;
  VLOG(1) << "reduce";
  reduceExecute<K,V,OutType,ReduceF>(reducers, num_reducers, rf);
  auto stop_reduce = Grappa::walltime();

  mr_reducing_runtime += stop_reduce - start_reduce;

  VLOG(1) << "complete";
}
// (global array) reducers
// (symmetric alloc) combiners
template < typename T, typename K, typename V, typename OutType, typename MapF, typename ReduceF>
void MapReduceJobExecute(GlobalAddress<T> keyvals, size_t num, GlobalAddress<Reducer<K,V,OutType>> reducers, size_t num_reducers/*Grappa::cores()*/, MapF mf, ReduceF rf) {
  auto start_realloc = Grappa::walltime();
  // clear all data structures from a previous usage
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r.result->clear();
      r.groups->clear();
      });
  auto stop_realloc = Grappa::walltime();
  mr_reallocation_runtime += stop_realloc - start_realloc;

  auto start_map = stop_realloc;

  MapperContext<K,V,OutType> ctx(reducers, num_reducers);
  VLOG(1) << "map";
  mapExecute<T,K,V,OutType,MapF>(ctx, keyvals, num, mf);
  auto stop_map = Grappa::walltime();
  mr_mapping_runtime += stop_map - start_map;

  auto start_reduce = stop_map;
  VLOG(1) << "reduce";
  reduceExecute<K,V,OutType,ReduceF>(reducers, num_reducers, rf);
  auto stop_reduce = Grappa::walltime();

  mr_reducing_runtime += stop_reduce - start_reduce;

  VLOG(1) << "complete";
}

// push-based map reduce
template < typename T, typename K, typename V, typename OutType, typename MapF, typename ReduceF>
GlobalAddress<Reducer<K,V,OutType>> MapReduceJobExecute(GlobalAddress<T> keyvals, size_t num, size_t num_reducers/*Grappa::cores()*/, MapF mf, ReduceF rf) {
  auto reducers = allocateReducers<K,V,OutType>( num_reducers );
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r = Reducer<K,V,OutType>();
      });
  //TODO symmetric alloc of a MapReduceContext GCE to allow concurrent jobs

  mapExecute<T,K,V,OutType,MapF>(MapperContext<K,V,OutType>(reducers, num_reducers), keyvals, num, mf);

  reduceExecute<K,V,OutType,ReduceF>(reducers, num_reducers, rf);

  return reducers;
}

template < typename T, typename K, typename V, typename OutType, typename MapF, typename ReduceF, typename RA>
GlobalAddress<Reducer<K,V,OutType>> MapReduceJobExecute(GlobalAddress<RA> keyvals, size_t num_reducers/*Grappa::cores()*/, MapF mf, ReduceF rf) {
  auto reducers = allocateReducers<K,V,OutType>( num_reducers );
  Grappa::forall<&default_mr_gce>(reducers, num_reducers, [](Reducer<K,V,OutType>& r) {
      r = Reducer<K,V,OutType>();
      });
  //TODO symmetric alloc of a MapReduceContext GCE to allow concurrent jobs

  mapExecute<T,K,V,OutType,MapF>(MapperContext<K,V,OutType>(reducers, num_reducers), keyvals, mf);

  reduceExecute<K,V,OutType,ReduceF>(reducers, num_reducers, rf);

  return reducers;
}

} // end namespace

        /// TODO: join: quite annoying might as well do parallel(map/map) reduce because otherwise need to marshal data from two tables into an forall() iterator anyway
<|endoftext|>


<commit_before>#include "stdafx.h"

using namespace System;
using namespace System::Reflection;
using namespace System::Runtime::CompilerServices;
using namespace System::Runtime::InteropServices;
using namespace System::Security::Permissions;

//
// AZuɊւʏ͈ȉ̑ZbgƂĐ䂳܂B
// AZuɊ֘AtĂύXɂ́A
// ̑lύXĂB
//
[assembly:AssemblyTitleAttribute("treefrogsetup")];
[assembly:AssemblyDescriptionAttribute("")];
[assembly:AssemblyConfigurationAttribute("")];
[assembly:AssemblyCompanyAttribute("")];
[assembly:AssemblyProductAttribute("treefrogsetup")];
[assembly:AssemblyCopyrightAttribute("Copyright (c) 2010-2016")];
[assembly:AssemblyTrademarkAttribute("")];
[assembly:AssemblyCultureAttribute("")];

//
// AZũo[ẂAȉ 4 ̒lō\Ă܂:
//
//      Major Version
//      Minor Version
//      Build Number
//      Revision
//
// ׂĂ̒lw肷邩Â悤 '*' gărWуrhԍ
// lɂ邱Ƃł܂:

[assembly:AssemblyVersionAttribute("1.12.0")];

[assembly:ComVisible(false)];

[assembly:CLSCompliantAttribute(true)];

[assembly:SecurityPermission(SecurityAction::RequestMinimum, UnmanagedCode = true)];
<commit_msg>update version string for MSVC.<commit_after>#include "stdafx.h"

using namespace System;
using namespace System::Reflection;
using namespace System::Runtime::CompilerServices;
using namespace System::Runtime::InteropServices;
using namespace System::Security::Permissions;

//
// AZuɊւʏ͈ȉ̑ZbgƂĐ䂳܂B
// AZuɊ֘AtĂύXɂ́A
// ̑lύXĂB
//
[assembly:AssemblyTitleAttribute("treefrogsetup")];
[assembly:AssemblyDescriptionAttribute("")];
[assembly:AssemblyConfigurationAttribute("")];
[assembly:AssemblyCompanyAttribute("")];
[assembly:AssemblyProductAttribute("treefrogsetup")];
[assembly:AssemblyCopyrightAttribute("Copyright (c) 2010-2016")];
[assembly:AssemblyTrademarkAttribute("")];
[assembly:AssemblyCultureAttribute("")];

//
// AZũo[ẂAȉ 4 ̒lō\Ă܂:
//
//      Major Version
//      Minor Version
//      Build Number
//      Revision
//
// ׂĂ̒lw肷邩Â悤 '*' gărWуrhԍ
// lɂ邱Ƃł܂:

[assembly:AssemblyVersionAttribute("1.13.0")];

[assembly:ComVisible(false)];

[assembly:CLSCompliantAttribute(true)];

[assembly:SecurityPermission(SecurityAction::RequestMinimum, UnmanagedCode = true)];
<|endoftext|>


<commit_before>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.h"

#include <string>
#include <unordered_set>

#include "absl/types/span.h"
#include "llvm/Support/raw_ostream.h"
#include "mlir/IR/Attributes.h"  // from @llvm-project
#include "mlir/IR/BuiltinOps.h"  // from @llvm-project
#include "mlir/IR/Visitors.h"  // from @llvm-project
#include "mlir/Parser.h"  // from @llvm-project
#include "mlir/Pass/Pass.h"  // from @llvm-project
#include "mlir/Support/FileUtilities.h"  // from @llvm-project
#include "mlir/Support/LogicalResult.h"  // from @llvm-project
#include "mlir/Transforms/Passes.h"  // from @llvm-project
#include "tensorflow/compiler/mlir/lite/flatbuffer_export.h"
#include "tensorflow/compiler/mlir/lite/metrics/error_collector_inst.h"
#include "tensorflow/compiler/mlir/lite/quantization/quantization_config.h"
#include "tensorflow/compiler/mlir/lite/tf_tfl_passes.h"
#include "tensorflow/compiler/mlir/lite/transforms/passes.h"
#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h"
#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h"
#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.h"
#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h"
#include "tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.h"
#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_def.pb.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/lite/tools/optimize/quantize_weights.h"
#include "tensorflow/lite/tools/optimize/reduced_precision_support.h"
#include "tensorflow/stream_executor/lib/statusor.h"

namespace tensorflow {
namespace {
using mlir::MLIRContext;
using mlir::ModuleOp;
using mlir::Operation;
using mlir::OwningModuleRef;
using stream_executor::port::StatusOr;

bool IsControlFlowV1Op(Operation* op) {
  return mlir::isa<mlir::tf_executor::SwitchOp, mlir::tf_executor::MergeOp,
                   mlir::tf_executor::EnterOp, mlir::tf_executor::ExitOp,
                   mlir::tf_executor::NextIterationSinkOp,
                   mlir::tf_executor::NextIterationSourceOp>(op);
}

mlir::LogicalResult IsValidGraph(mlir::ModuleOp module) {
  auto result = module.walk([&](Operation* op) {
    return IsControlFlowV1Op(op) ? mlir::WalkResult::interrupt()
                                 : mlir::WalkResult::advance();
  });
  if (result.wasInterrupted()) {
    mlir::TFL::AttachErrorCode(
        module.emitError(
            "The graph has Control Flow V1 ops. TFLite converter doesn't "
            "support Control Flow V1 ops. Consider using Control Flow V2 ops "
            "instead. See https://www.tensorflow.org/api_docs/python/tf/compat/"
            "v1/enable_control_flow_v2."),
        tflite::metrics::ConverterErrorData::ERROR_UNSUPPORTED_CONTROL_FLOW_V1);
    return mlir::failure();
  }
  return mlir::success();
}

// Util that registers 'extra_tf_opdefs' to the TF global registry.
// Return OK on success, failure if registering failed.
Status RegisterExtraTfOpDefs(absl::Span<const std::string> extra_tf_opdefs) {
  for (const auto& tf_opdefs_string : extra_tf_opdefs) {
    tensorflow::OpDef opdef;
    if (!tensorflow::protobuf::TextFormat::ParseFromString(tf_opdefs_string,
                                                           &opdef)) {
      LOG(ERROR) << "OpDef parsing failed for: " << tf_opdefs_string;
      return errors::InvalidArgument("fail to parse extra OpDef");
    }
    // Register extra opdefs.
    // TODO(b/133770952): Support shape functions.
    tensorflow::OpRegistry::Global()->Register(
        [opdef](tensorflow::OpRegistrationData* op_reg_data) -> Status {
          *op_reg_data = tensorflow::OpRegistrationData(opdef);
          return Status::OK();
        });
  }
  return Status::OK();
}
}  // namespace

StatusOr<OwningModuleRef> LoadFromGraphdefOrMlirSource(
    const std::string& input_filename, bool input_mlir,
    bool use_splatted_constant, const std::vector<std::string>& extra_tf_opdefs,
    const GraphImportConfig& specs, absl::string_view debug_info_file,
    absl::string_view input_arrays, absl::string_view input_dtypes,
    absl::string_view input_shapes, absl::string_view output_arrays,
    absl::string_view control_output_arrays, llvm::SourceMgr* source_mgr,
    MLIRContext* context) {
  // Set up the input file.
  std::string error_message;
  auto file = mlir::openInputFile(input_filename, &error_message);
  if (!file) {
    llvm::errs() << error_message << "\n";
    return errors::InvalidArgument("fail to open input file");
  }

  if (input_mlir) {
    source_mgr->AddNewSourceBuffer(std::move(file), llvm::SMLoc());
    return OwningModuleRef(mlir::parseSourceFile(*source_mgr, context));
  }

  // Register extra TF ops passed as OpDef.
  auto extra_opdefs_status = RegisterExtraTfOpDefs(extra_tf_opdefs);
  if (!extra_opdefs_status.ok()) return extra_opdefs_status;

  if (use_splatted_constant) {
    return tensorflow::GraphdefToSplattedMlirTranslateFunction(
        file->getBuffer(), debug_info_file, input_arrays, input_dtypes,
        input_shapes, output_arrays, control_output_arrays,
        specs.prune_unused_nodes, /*convert_legacy_fed_inputs=*/true,
        /*graph_as_function=*/false, specs.upgrade_legacy,
        /*enable_shape_inference=*/false,
        /*unconditionally_use_set_output_shapes=*/false, context);
  }
  return tensorflow::GraphdefToMlirTranslateFunction(
      file->getBuffer(), debug_info_file, input_arrays, input_dtypes,
      input_shapes, output_arrays, control_output_arrays,
      specs.prune_unused_nodes, /*convert_legacy_fed_inputs=*/true,
      /*graph_as_function=*/false, specs.upgrade_legacy,
      /*enable_shape_inference=*/false,
      /*unconditionally_use_set_output_shapes=*/false, context);
}

// Applying post-training dynamic range quantization from the old TOCO quantizer
// on the translated_result using quant_specs and saving the final output in
// result.
Status ApplyDynamicRangeQuantizationFromOldQuantizer(
    const mlir::TFL::QuantizationSpecs& quant_specs,
    std::string translated_result, std::string* result) {
  flatbuffers::FlatBufferBuilder q_builder(/*initial_size=*/10240);
  const uint8_t* buffer =
      reinterpret_cast<const uint8_t*>(translated_result.c_str());
  const ::tflite::Model* input_model = ::tflite::GetModel(buffer);

  ::tflite::optimize::BufferType quantized_type;
  switch (quant_specs.inference_type) {
    case tensorflow::DT_QINT8:
      quantized_type = ::tflite::optimize::BufferType::QUANTIZED_INT8;
      break;
    case tensorflow::DT_HALF:
      quantized_type = ::tflite::optimize::BufferType::QUANTIZED_FLOAT16;
      break;
    default:
      return errors::InvalidArgument("Quantized type not supported");
      break;
  }

  bool use_updated_hybrid_scheme = !quant_specs.disable_per_channel;
  if (::tflite::optimize::QuantizeWeights(
          &q_builder, input_model, quantized_type, use_updated_hybrid_scheme) !=
      kTfLiteOk) {
    return errors::InvalidArgument("Quantize weights transformation failed.");
  }
  const uint8_t* q_buffer = q_builder.GetBufferPointer();
  *result =
      string(reinterpret_cast<const char*>(q_buffer), q_builder.GetSize());

  return Status::OK();
}

Status ConvertTFExecutorToTFLOrFlatbuffer(
    mlir::ModuleOp module, bool export_to_mlir,
    const toco::TocoFlags& toco_flags, const mlir::TFL::PassConfig& pass_config,
    const std::unordered_set<std::string>& saved_model_tags,
    llvm::StringRef saved_model_dir,
    llvm::Optional<tensorflow::Session*> session, std::string* result) {
  // Explicitly disable dumping Op details on failures.
  module.getContext()->printOpOnDiagnostic(false);

  // Register a warning handler only log to std out.
  mlir::ScopedDiagnosticHandler s(
      module.getContext(), [](mlir::Diagnostic& diag) {
        if (diag.getSeverity() == mlir::DiagnosticSeverity::Warning) {
          for (auto& note : diag.getNotes()) {
            std::cout << note.str() << "\n";
            LOG(WARNING) << note.str() << "\n";
          }
        }
        return mlir::failure();
      });

  mlir::StatusScopedDiagnosticHandler statusHandler(module.getContext(),
                                                    /*propagate=*/true);

  if (failed(IsValidGraph(module))) {
    return statusHandler.ConsumeStatus();
  }

  mlir::PassManager pass_manager(module.getContext());
  mlir::applyPassManagerCLOptions(pass_manager);
  pass_manager.addInstrumentation(
      std::make_unique<mlir::TFL::ErrorCollectorInstrumentation>(
          pass_manager.getContext()));

  tensorflow::AddPreVariableFreezingTFToTFLConversionPasses(
      saved_model_dir, toco_flags, pass_config, &pass_manager);
  if (failed(pass_manager.run(module))) {
    return statusHandler.ConsumeStatus();
  }

  // Freeze variables if a session is provided.
  if (session.hasValue()) {
    mlir::TFL::ErrorCollectorInstrumentation collector(module.getContext());
    if (failed(mlir::tf_saved_model::FreezeVariables(module,
                                                     session.getValue()))) {
      auto status = statusHandler.ConsumeStatus();
      mlir::TFL::ErrorCollector* collector =
          mlir::TFL::ErrorCollector::GetErrorCollector();
      if (!collector->CollectedErrors().empty()) {
        // LINT.IfChange
        return errors::InvalidArgument(
            "Variable constant folding is failed. Please consider using "
            "enabling `experimental_enable_resource_variables` flag in the "
            "TFLite converter object. For example, "
            "converter.experimental_enable_resource_variables = True");
        // LINT.ThenChange(//tensorflow/lite/python/lite_v2_test.py)
      }
      return status;
    }
  }
  pass_manager.clear();
  tensorflow::AddPostVariableFreezingTFToTFLConversionPasses(
      saved_model_dir, toco_flags, pass_config, &pass_manager);
  if (failed(pass_manager.run(module))) {
    auto status = statusHandler.ConsumeStatus();
    mlir::TFL::ErrorCollector* collector =
        mlir::TFL::ErrorCollector::GetErrorCollector();
    for (const auto& error_data : collector->CollectedErrors()) {
      if (error_data.subcomponent() == "FreezeGlobalTensorsPass") {
        // LINT.IfChange
        return errors::InvalidArgument(
            "Variable constant folding is failed. Please consider using "
            "enabling `experimental_enable_resource_variables` flag in the "
            "TFLite converter object. For example, "
            "converter.experimental_enable_resource_variables = True");
        // LINT.ThenChange(//tensorflow/lite/python/lite_v2_test.py)
      }
    }
    return status;
  }

  if (export_to_mlir) {
    llvm::raw_string_ostream os(*result);
    module.print(os);
    return statusHandler.ConsumeStatus();
  }

  // Write MLIR TFLite dialect into FlatBuffer
  const mlir::TFL::QuantizationSpecs& quant_specs = pass_config.quant_specs;
  OpOrArgLocNameMapper op_or_arg_name_mapper;
  tflite::FlatbufferExportOptions options;
  std::string translated_result;
  options.toco_flags = toco_flags;
  options.saved_model_tags = saved_model_tags;
  options.op_or_arg_name_mapper = &op_or_arg_name_mapper;
  if (quant_specs.support_mask !=
      tflite::optimize::ReducedPrecisionSupport::None) {
    options.metadata.insert(
        MetadataForReducedPrecisionSupport(quant_specs.support_mask));
  }
  if (!tflite::MlirToFlatBufferTranslateFunction(module, options,
                                                 &translated_result)) {
    return statusHandler.ConsumeStatus();
  }

  if (quant_specs.weight_quantization &&
      !quant_specs.RunAndRewriteDynamicRangeQuantizationPasses()) {
    // Apply post-training dynamic range quantization from the old TOCO
    // quantizer.Once MLIR has support for this, we can remove this if
    // statement.
    auto status = ApplyDynamicRangeQuantizationFromOldQuantizer(
        quant_specs, translated_result, result);
    if (!status.ok()) return status;
  } else {
    *result = translated_result;
  }

  if (mlir::failed(module.verify())) {
    return tensorflow::errors::Unknown("Final module is invalid");
  }
  return Status::OK();
}

StatusOr<mlir::OwningModuleRef> ImportSavedModel(
    const std::string& input_filename, const int saved_model_version,
    const std::unordered_set<std::string>& tags,
    absl::Span<const std::string> extra_tf_opdefs,
    absl::Span<std::string> exported_names, const GraphImportConfig& specs,
    bool enable_variable_lifting, mlir::MLIRContext* context,
    std::unique_ptr<tensorflow::SavedModelBundle>* saved_model_bundle) {
  // Register extra TF ops passed as OpDef.
  auto extra_opdefs_status = RegisterExtraTfOpDefs(extra_tf_opdefs);
  if (!extra_opdefs_status.ok()) return extra_opdefs_status;

  if (saved_model_version == 2) {
    auto module_or = tensorflow::SavedModelObjectGraphToMlirImport(
        input_filename, tags, exported_names, context);
    if (!module_or.status().ok()) return module_or.status();
    return module_or.ConsumeValueOrDie();
  } else if (saved_model_version == 1) {
    MLIRImportOptions options;
    options.upgrade_legacy = specs.upgrade_legacy;
    auto module_or = tensorflow::SavedModelSignatureDefsToMlirImport(
        input_filename, tags, exported_names, context, options,
        enable_variable_lifting, saved_model_bundle);

    if (!module_or.status().ok()) return module_or.status();
    return module_or.ConsumeValueOrDie();
  } else {
    return tensorflow::errors::InvalidArgument(
        "Should be either saved model v1 or v2");
  }
}

}  // namespace tensorflow
<commit_msg>Set `unconditionally_use_set_output_shapes=true` in GraphDef conversion path.<commit_after>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.h"

#include <string>
#include <unordered_set>

#include "absl/types/span.h"
#include "llvm/Support/raw_ostream.h"
#include "mlir/IR/Attributes.h"  // from @llvm-project
#include "mlir/IR/BuiltinOps.h"  // from @llvm-project
#include "mlir/IR/Visitors.h"  // from @llvm-project
#include "mlir/Parser.h"  // from @llvm-project
#include "mlir/Pass/Pass.h"  // from @llvm-project
#include "mlir/Support/FileUtilities.h"  // from @llvm-project
#include "mlir/Support/LogicalResult.h"  // from @llvm-project
#include "mlir/Transforms/Passes.h"  // from @llvm-project
#include "tensorflow/compiler/mlir/lite/flatbuffer_export.h"
#include "tensorflow/compiler/mlir/lite/metrics/error_collector_inst.h"
#include "tensorflow/compiler/mlir/lite/quantization/quantization_config.h"
#include "tensorflow/compiler/mlir/lite/tf_tfl_passes.h"
#include "tensorflow/compiler/mlir/lite/transforms/passes.h"
#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h"
#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h"
#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.h"
#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h"
#include "tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.h"
#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_def.pb.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/lite/tools/optimize/quantize_weights.h"
#include "tensorflow/lite/tools/optimize/reduced_precision_support.h"
#include "tensorflow/stream_executor/lib/statusor.h"

namespace tensorflow {
namespace {
using mlir::MLIRContext;
using mlir::ModuleOp;
using mlir::Operation;
using mlir::OwningModuleRef;
using stream_executor::port::StatusOr;

bool IsControlFlowV1Op(Operation* op) {
  return mlir::isa<mlir::tf_executor::SwitchOp, mlir::tf_executor::MergeOp,
                   mlir::tf_executor::EnterOp, mlir::tf_executor::ExitOp,
                   mlir::tf_executor::NextIterationSinkOp,
                   mlir::tf_executor::NextIterationSourceOp>(op);
}

mlir::LogicalResult IsValidGraph(mlir::ModuleOp module) {
  auto result = module.walk([&](Operation* op) {
    return IsControlFlowV1Op(op) ? mlir::WalkResult::interrupt()
                                 : mlir::WalkResult::advance();
  });
  if (result.wasInterrupted()) {
    mlir::TFL::AttachErrorCode(
        module.emitError(
            "The graph has Control Flow V1 ops. TFLite converter doesn't "
            "support Control Flow V1 ops. Consider using Control Flow V2 ops "
            "instead. See https://www.tensorflow.org/api_docs/python/tf/compat/"
            "v1/enable_control_flow_v2."),
        tflite::metrics::ConverterErrorData::ERROR_UNSUPPORTED_CONTROL_FLOW_V1);
    return mlir::failure();
  }
  return mlir::success();
}

// Util that registers 'extra_tf_opdefs' to the TF global registry.
// Return OK on success, failure if registering failed.
Status RegisterExtraTfOpDefs(absl::Span<const std::string> extra_tf_opdefs) {
  for (const auto& tf_opdefs_string : extra_tf_opdefs) {
    tensorflow::OpDef opdef;
    if (!tensorflow::protobuf::TextFormat::ParseFromString(tf_opdefs_string,
                                                           &opdef)) {
      LOG(ERROR) << "OpDef parsing failed for: " << tf_opdefs_string;
      return errors::InvalidArgument("fail to parse extra OpDef");
    }
    // Register extra opdefs.
    // TODO(b/133770952): Support shape functions.
    tensorflow::OpRegistry::Global()->Register(
        [opdef](tensorflow::OpRegistrationData* op_reg_data) -> Status {
          *op_reg_data = tensorflow::OpRegistrationData(opdef);
          return Status::OK();
        });
  }
  return Status::OK();
}
}  // namespace

StatusOr<OwningModuleRef> LoadFromGraphdefOrMlirSource(
    const std::string& input_filename, bool input_mlir,
    bool use_splatted_constant, const std::vector<std::string>& extra_tf_opdefs,
    const GraphImportConfig& specs, absl::string_view debug_info_file,
    absl::string_view input_arrays, absl::string_view input_dtypes,
    absl::string_view input_shapes, absl::string_view output_arrays,
    absl::string_view control_output_arrays, llvm::SourceMgr* source_mgr,
    MLIRContext* context) {
  // Set up the input file.
  std::string error_message;
  auto file = mlir::openInputFile(input_filename, &error_message);
  if (!file) {
    llvm::errs() << error_message << "\n";
    return errors::InvalidArgument("fail to open input file");
  }

  if (input_mlir) {
    source_mgr->AddNewSourceBuffer(std::move(file), llvm::SMLoc());
    return OwningModuleRef(mlir::parseSourceFile(*source_mgr, context));
  }

  // Register extra TF ops passed as OpDef.
  auto extra_opdefs_status = RegisterExtraTfOpDefs(extra_tf_opdefs);
  if (!extra_opdefs_status.ok()) return extra_opdefs_status;

  if (use_splatted_constant) {
    return tensorflow::GraphdefToSplattedMlirTranslateFunction(
        file->getBuffer(), debug_info_file, input_arrays, input_dtypes,
        input_shapes, output_arrays, control_output_arrays,
        specs.prune_unused_nodes, /*convert_legacy_fed_inputs=*/true,
        /*graph_as_function=*/false, specs.upgrade_legacy,
        /*enable_shape_inference=*/false,
        /*unconditionally_use_set_output_shapes=*/true, context);
  }
  return tensorflow::GraphdefToMlirTranslateFunction(
      file->getBuffer(), debug_info_file, input_arrays, input_dtypes,
      input_shapes, output_arrays, control_output_arrays,
      specs.prune_unused_nodes, /*convert_legacy_fed_inputs=*/true,
      /*graph_as_function=*/false, specs.upgrade_legacy,
      /*enable_shape_inference=*/false,
      /*unconditionally_use_set_output_shapes=*/true, context);
}

// Applying post-training dynamic range quantization from the old TOCO quantizer
// on the translated_result using quant_specs and saving the final output in
// result.
Status ApplyDynamicRangeQuantizationFromOldQuantizer(
    const mlir::TFL::QuantizationSpecs& quant_specs,
    std::string translated_result, std::string* result) {
  flatbuffers::FlatBufferBuilder q_builder(/*initial_size=*/10240);
  const uint8_t* buffer =
      reinterpret_cast<const uint8_t*>(translated_result.c_str());
  const ::tflite::Model* input_model = ::tflite::GetModel(buffer);

  ::tflite::optimize::BufferType quantized_type;
  switch (quant_specs.inference_type) {
    case tensorflow::DT_QINT8:
      quantized_type = ::tflite::optimize::BufferType::QUANTIZED_INT8;
      break;
    case tensorflow::DT_HALF:
      quantized_type = ::tflite::optimize::BufferType::QUANTIZED_FLOAT16;
      break;
    default:
      return errors::InvalidArgument("Quantized type not supported");
      break;
  }

  bool use_updated_hybrid_scheme = !quant_specs.disable_per_channel;
  if (::tflite::optimize::QuantizeWeights(
          &q_builder, input_model, quantized_type, use_updated_hybrid_scheme) !=
      kTfLiteOk) {
    return errors::InvalidArgument("Quantize weights transformation failed.");
  }
  const uint8_t* q_buffer = q_builder.GetBufferPointer();
  *result =
      string(reinterpret_cast<const char*>(q_buffer), q_builder.GetSize());

  return Status::OK();
}

Status ConvertTFExecutorToTFLOrFlatbuffer(
    mlir::ModuleOp module, bool export_to_mlir,
    const toco::TocoFlags& toco_flags, const mlir::TFL::PassConfig& pass_config,
    const std::unordered_set<std::string>& saved_model_tags,
    llvm::StringRef saved_model_dir,
    llvm::Optional<tensorflow::Session*> session, std::string* result) {
  // Explicitly disable dumping Op details on failures.
  module.getContext()->printOpOnDiagnostic(false);

  // Register a warning handler only log to std out.
  mlir::ScopedDiagnosticHandler s(
      module.getContext(), [](mlir::Diagnostic& diag) {
        if (diag.getSeverity() == mlir::DiagnosticSeverity::Warning) {
          for (auto& note : diag.getNotes()) {
            std::cout << note.str() << "\n";
            LOG(WARNING) << note.str() << "\n";
          }
        }
        return mlir::failure();
      });

  mlir::StatusScopedDiagnosticHandler statusHandler(module.getContext(),
                                                    /*propagate=*/true);

  if (failed(IsValidGraph(module))) {
    return statusHandler.ConsumeStatus();
  }

  mlir::PassManager pass_manager(module.getContext());
  mlir::applyPassManagerCLOptions(pass_manager);
  pass_manager.addInstrumentation(
      std::make_unique<mlir::TFL::ErrorCollectorInstrumentation>(
          pass_manager.getContext()));

  tensorflow::AddPreVariableFreezingTFToTFLConversionPasses(
      saved_model_dir, toco_flags, pass_config, &pass_manager);
  if (failed(pass_manager.run(module))) {
    return statusHandler.ConsumeStatus();
  }

  // Freeze variables if a session is provided.
  if (session.hasValue()) {
    mlir::TFL::ErrorCollectorInstrumentation collector(module.getContext());
    if (failed(mlir::tf_saved_model::FreezeVariables(module,
                                                     session.getValue()))) {
      auto status = statusHandler.ConsumeStatus();
      mlir::TFL::ErrorCollector* collector =
          mlir::TFL::ErrorCollector::GetErrorCollector();
      if (!collector->CollectedErrors().empty()) {
        // LINT.IfChange
        return errors::InvalidArgument(
            "Variable constant folding is failed. Please consider using "
            "enabling `experimental_enable_resource_variables` flag in the "
            "TFLite converter object. For example, "
            "converter.experimental_enable_resource_variables = True");
        // LINT.ThenChange(//tensorflow/lite/python/lite_v2_test.py)
      }
      return status;
    }
  }
  pass_manager.clear();
  tensorflow::AddPostVariableFreezingTFToTFLConversionPasses(
      saved_model_dir, toco_flags, pass_config, &pass_manager);
  if (failed(pass_manager.run(module))) {
    auto status = statusHandler.ConsumeStatus();
    mlir::TFL::ErrorCollector* collector =
        mlir::TFL::ErrorCollector::GetErrorCollector();
    for (const auto& error_data : collector->CollectedErrors()) {
      if (error_data.subcomponent() == "FreezeGlobalTensorsPass") {
        // LINT.IfChange
        return errors::InvalidArgument(
            "Variable constant folding is failed. Please consider using "
            "enabling `experimental_enable_resource_variables` flag in the "
            "TFLite converter object. For example, "
            "converter.experimental_enable_resource_variables = True");
        // LINT.ThenChange(//tensorflow/lite/python/lite_v2_test.py)
      }
    }
    return status;
  }

  if (export_to_mlir) {
    llvm::raw_string_ostream os(*result);
    module.print(os);
    return statusHandler.ConsumeStatus();
  }

  // Write MLIR TFLite dialect into FlatBuffer
  const mlir::TFL::QuantizationSpecs& quant_specs = pass_config.quant_specs;
  OpOrArgLocNameMapper op_or_arg_name_mapper;
  tflite::FlatbufferExportOptions options;
  std::string translated_result;
  options.toco_flags = toco_flags;
  options.saved_model_tags = saved_model_tags;
  options.op_or_arg_name_mapper = &op_or_arg_name_mapper;
  if (quant_specs.support_mask !=
      tflite::optimize::ReducedPrecisionSupport::None) {
    options.metadata.insert(
        MetadataForReducedPrecisionSupport(quant_specs.support_mask));
  }
  if (!tflite::MlirToFlatBufferTranslateFunction(module, options,
                                                 &translated_result)) {
    return statusHandler.ConsumeStatus();
  }

  if (quant_specs.weight_quantization &&
      !quant_specs.RunAndRewriteDynamicRangeQuantizationPasses()) {
    // Apply post-training dynamic range quantization from the old TOCO
    // quantizer.Once MLIR has support for this, we can remove this if
    // statement.
    auto status = ApplyDynamicRangeQuantizationFromOldQuantizer(
        quant_specs, translated_result, result);
    if (!status.ok()) return status;
  } else {
    *result = translated_result;
  }

  if (mlir::failed(module.verify())) {
    return tensorflow::errors::Unknown("Final module is invalid");
  }
  return Status::OK();
}

StatusOr<mlir::OwningModuleRef> ImportSavedModel(
    const std::string& input_filename, const int saved_model_version,
    const std::unordered_set<std::string>& tags,
    absl::Span<const std::string> extra_tf_opdefs,
    absl::Span<std::string> exported_names, const GraphImportConfig& specs,
    bool enable_variable_lifting, mlir::MLIRContext* context,
    std::unique_ptr<tensorflow::SavedModelBundle>* saved_model_bundle) {
  // Register extra TF ops passed as OpDef.
  auto extra_opdefs_status = RegisterExtraTfOpDefs(extra_tf_opdefs);
  if (!extra_opdefs_status.ok()) return extra_opdefs_status;

  if (saved_model_version == 2) {
    auto module_or = tensorflow::SavedModelObjectGraphToMlirImport(
        input_filename, tags, exported_names, context);
    if (!module_or.status().ok()) return module_or.status();
    return module_or.ConsumeValueOrDie();
  } else if (saved_model_version == 1) {
    MLIRImportOptions options;
    options.upgrade_legacy = specs.upgrade_legacy;
    auto module_or = tensorflow::SavedModelSignatureDefsToMlirImport(
        input_filename, tags, exported_names, context, options,
        enable_variable_lifting, saved_model_bundle);

    if (!module_or.status().ok()) return module_or.status();
    return module_or.ConsumeValueOrDie();
  } else {
    return tensorflow::errors::InvalidArgument(
        "Should be either saved model v1 or v2");
  }
}

}  // namespace tensorflow
<|endoftext|>


<commit_before>/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <memory>
#include <string>

#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/resource_mgr.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/kernels/boosted_trees/resources.h"
#include "tensorflow/core/lib/core/refcount.h"

namespace tensorflow {

REGISTER_RESOURCE_HANDLE_KERNEL(BoostedTreesEnsembleResource);

REGISTER_KERNEL_BUILDER(
    Name("IsBoostedTreesEnsembleInitialized").Device(DEVICE_CPU),
    IsResourceInitialized<BoostedTreesEnsembleResource>);

// Creates a tree ensemble resource.
class BoostedTreesCreateEnsembleOp : public OpKernel {
 public:
  explicit BoostedTreesCreateEnsembleOp(OpKernelConstruction* context)
      : OpKernel(context) {
    VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use "
            << "TensorFlow Decision Forests instead "
            << "(https://github.com/tensorflow/decision-forests).\n";
  }

  void Compute(OpKernelContext* context) override {
    // Get the stamp token.
    const Tensor* stamp_token_t;
    OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t));
    OP_REQUIRES(context, TensorShapeUtils::IsScalar(stamp_token_t->shape()),
                errors::InvalidArgument(
                    "stamp_token must be a scalar, got a tensor of shape ",
                    stamp_token_t->shape().DebugString()));
    int64_t stamp_token = stamp_token_t->scalar<int64_t>()();

    // Get the tree ensemble proto.
    const Tensor* tree_ensemble_serialized_t;
    OP_REQUIRES_OK(context, context->input("tree_ensemble_serialized",
                                           &tree_ensemble_serialized_t));
    std::unique_ptr<BoostedTreesEnsembleResource> result(
        new BoostedTreesEnsembleResource());
    if (!result->InitFromSerialized(
            tree_ensemble_serialized_t->scalar<tstring>()(), stamp_token)) {
      result->Unref();
      result.release();  // Needed due to the `->Unref` above, to prevent UAF
      OP_REQUIRES(
          context, false,
          errors::InvalidArgument("Unable to parse tree ensemble proto."));
    }

    // Only create one, if one does not exist already. Report status for all
    // other exceptions.
    auto status =
        CreateResource(context, HandleFromInput(context, 0), result.release());
    if (status.code() != tensorflow::error::ALREADY_EXISTS) {
      OP_REQUIRES_OK(context, status);
    }
  }
};

REGISTER_KERNEL_BUILDER(Name("BoostedTreesCreateEnsemble").Device(DEVICE_CPU),
                        BoostedTreesCreateEnsembleOp);

// Op for retrieving some model states (needed for training).
class BoostedTreesGetEnsembleStatesOp : public OpKernel {
 public:
  explicit BoostedTreesGetEnsembleStatesOp(OpKernelConstruction* context)
      : OpKernel(context) {
    VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use "
            << "TensorFlow Decision Forests instead "
            << "(https://github.com/tensorflow/decision-forests).\n";
  }

  void Compute(OpKernelContext* context) override {
    // Looks up the resource.
    core::RefCountPtr<BoostedTreesEnsembleResource> tree_ensemble_resource;
    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
                                           &tree_ensemble_resource));
    tf_shared_lock l(*tree_ensemble_resource->get_mutex());

    // Sets the outputs.
    const int num_trees = tree_ensemble_resource->num_trees();
    const int num_finalized_trees =
        (num_trees <= 0 ||
         tree_ensemble_resource->IsTreeFinalized(num_trees - 1))
            ? num_trees
            : num_trees - 1;
    const int num_attempted_layers =
        tree_ensemble_resource->GetNumLayersAttempted();

    // growing_metadata
    Tensor* output_stamp_token_t = nullptr;
    Tensor* output_num_trees_t = nullptr;
    Tensor* output_num_finalized_trees_t = nullptr;
    Tensor* output_num_attempted_layers_t = nullptr;
    Tensor* output_last_layer_nodes_range_t = nullptr;

    OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(),
                                                     &output_stamp_token_t));
    OP_REQUIRES_OK(context, context->allocate_output(1, TensorShape(),
                                                     &output_num_trees_t));
    OP_REQUIRES_OK(context,
                   context->allocate_output(2, TensorShape(),
                                            &output_num_finalized_trees_t));
    OP_REQUIRES_OK(context,
                   context->allocate_output(3, TensorShape(),
                                            &output_num_attempted_layers_t));
    OP_REQUIRES_OK(context, context->allocate_output(
                                4, {2}, &output_last_layer_nodes_range_t));

    output_stamp_token_t->scalar<int64_t>()() = tree_ensemble_resource->stamp();
    output_num_trees_t->scalar<int32>()() = num_trees;
    output_num_finalized_trees_t->scalar<int32>()() = num_finalized_trees;
    output_num_attempted_layers_t->scalar<int32>()() = num_attempted_layers;

    int32_t range_start;
    int32_t range_end;
    tree_ensemble_resource->GetLastLayerNodesRange(&range_start, &range_end);

    output_last_layer_nodes_range_t->vec<int32>()(0) = range_start;
    // For a completely empty ensemble, this will be 0. To make it a valid range
    // we add this max cond.
    output_last_layer_nodes_range_t->vec<int32>()(1) = std::max(1, range_end);
  }
};

REGISTER_KERNEL_BUILDER(
    Name("BoostedTreesGetEnsembleStates").Device(DEVICE_CPU),
    BoostedTreesGetEnsembleStatesOp);

// Op for serializing a model.
class BoostedTreesSerializeEnsembleOp : public OpKernel {
 public:
  explicit BoostedTreesSerializeEnsembleOp(OpKernelConstruction* context)
      : OpKernel(context) {
    VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use "
            << "TensorFlow Decision Forests instead "
            << "(https://github.com/tensorflow/decision-forests).\n";
  }

  void Compute(OpKernelContext* context) override {
    core::RefCountPtr<BoostedTreesEnsembleResource> tree_ensemble_resource;
    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
                                           &tree_ensemble_resource));
    tf_shared_lock l(*tree_ensemble_resource->get_mutex());
    Tensor* output_stamp_token_t = nullptr;
    OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(),
                                                     &output_stamp_token_t));
    output_stamp_token_t->scalar<int64_t>()() = tree_ensemble_resource->stamp();
    Tensor* output_proto_t = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(1, TensorShape(), &output_proto_t));
    output_proto_t->scalar<tstring>()() =
        tree_ensemble_resource->SerializeAsString();
  }
};

REGISTER_KERNEL_BUILDER(
    Name("BoostedTreesSerializeEnsemble").Device(DEVICE_CPU),
    BoostedTreesSerializeEnsembleOp);

// Op for deserializing a tree ensemble variable from a checkpoint.
class BoostedTreesDeserializeEnsembleOp : public OpKernel {
 public:
  explicit BoostedTreesDeserializeEnsembleOp(OpKernelConstruction* context)
      : OpKernel(context) {
    VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use "
            << "TensorFlow Decision Forests instead "
            << "(https://github.com/tensorflow/decision-forests).\n";
  }

  void Compute(OpKernelContext* context) override {
    core::RefCountPtr<BoostedTreesEnsembleResource> tree_ensemble_resource;
    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
                                           &tree_ensemble_resource));
    mutex_lock l(*tree_ensemble_resource->get_mutex());

    // Get the stamp token.
    const Tensor* stamp_token_t;
    OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t));
    int64_t stamp_token = stamp_token_t->scalar<int64_t>()();

    // Get the tree ensemble proto.
    const Tensor* tree_ensemble_serialized_t;
    OP_REQUIRES_OK(context, context->input("tree_ensemble_serialized",
                                           &tree_ensemble_serialized_t));
    // Deallocate all the previous objects on the resource.
    tree_ensemble_resource->Reset();
    OP_REQUIRES(
        context,
        tree_ensemble_resource->InitFromSerialized(
            tree_ensemble_serialized_t->scalar<tstring>()(), stamp_token),
        errors::InvalidArgument("Unable to parse tree ensemble proto."));
  }
};

REGISTER_KERNEL_BUILDER(
    Name("BoostedTreesDeserializeEnsemble").Device(DEVICE_CPU),
    BoostedTreesDeserializeEnsembleOp);

}  // namespace tensorflow
<commit_msg>Check tensor to be scalar before accessing it via `scalar<T>`<commit_after>/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <memory>
#include <string>

#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/resource_mgr.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/kernels/boosted_trees/resources.h"
#include "tensorflow/core/lib/core/refcount.h"

namespace tensorflow {

REGISTER_RESOURCE_HANDLE_KERNEL(BoostedTreesEnsembleResource);

REGISTER_KERNEL_BUILDER(
    Name("IsBoostedTreesEnsembleInitialized").Device(DEVICE_CPU),
    IsResourceInitialized<BoostedTreesEnsembleResource>);

// Creates a tree ensemble resource.
class BoostedTreesCreateEnsembleOp : public OpKernel {
 public:
  explicit BoostedTreesCreateEnsembleOp(OpKernelConstruction* context)
      : OpKernel(context) {
    VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use "
            << "TensorFlow Decision Forests instead "
            << "(https://github.com/tensorflow/decision-forests).\n";
  }

  void Compute(OpKernelContext* context) override {
    // Get the stamp token.
    const Tensor* stamp_token_t;
    OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t));
    OP_REQUIRES(context, TensorShapeUtils::IsScalar(stamp_token_t->shape()),
                errors::InvalidArgument(
                    "stamp_token must be a scalar, got a tensor of shape ",
                    stamp_token_t->shape().DebugString()));
    int64_t stamp_token = stamp_token_t->scalar<int64_t>()();

    // Get the tree ensemble proto.
    const Tensor* tree_ensemble_serialized_t;
    OP_REQUIRES_OK(context, context->input("tree_ensemble_serialized",
                                           &tree_ensemble_serialized_t));
    OP_REQUIRES(
        context,
        TensorShapeUtils::IsScalar(tree_ensemble_serialized_t->shape()),
        errors::InvalidArgument(
            "tree_ensemble_serialized must be a scalar, got a tensor of shape ",
            tree_ensemble_serialized_t->shape().DebugString()));
    std::unique_ptr<BoostedTreesEnsembleResource> result(
        new BoostedTreesEnsembleResource());
    if (!result->InitFromSerialized(
            tree_ensemble_serialized_t->scalar<tstring>()(), stamp_token)) {
      result->Unref();
      result.release();  // Needed due to the `->Unref` above, to prevent UAF
      OP_REQUIRES(
          context, false,
          errors::InvalidArgument("Unable to parse tree ensemble proto."));
    }

    // Only create one, if one does not exist already. Report status for all
    // other exceptions.
    auto status =
        CreateResource(context, HandleFromInput(context, 0), result.release());
    if (status.code() != tensorflow::error::ALREADY_EXISTS) {
      OP_REQUIRES_OK(context, status);
    }
  }
};

REGISTER_KERNEL_BUILDER(Name("BoostedTreesCreateEnsemble").Device(DEVICE_CPU),
                        BoostedTreesCreateEnsembleOp);

// Op for retrieving some model states (needed for training).
class BoostedTreesGetEnsembleStatesOp : public OpKernel {
 public:
  explicit BoostedTreesGetEnsembleStatesOp(OpKernelConstruction* context)
      : OpKernel(context) {
    VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use "
            << "TensorFlow Decision Forests instead "
            << "(https://github.com/tensorflow/decision-forests).\n";
  }

  void Compute(OpKernelContext* context) override {
    // Looks up the resource.
    core::RefCountPtr<BoostedTreesEnsembleResource> tree_ensemble_resource;
    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
                                           &tree_ensemble_resource));
    tf_shared_lock l(*tree_ensemble_resource->get_mutex());

    // Sets the outputs.
    const int num_trees = tree_ensemble_resource->num_trees();
    const int num_finalized_trees =
        (num_trees <= 0 ||
         tree_ensemble_resource->IsTreeFinalized(num_trees - 1))
            ? num_trees
            : num_trees - 1;
    const int num_attempted_layers =
        tree_ensemble_resource->GetNumLayersAttempted();

    // growing_metadata
    Tensor* output_stamp_token_t = nullptr;
    Tensor* output_num_trees_t = nullptr;
    Tensor* output_num_finalized_trees_t = nullptr;
    Tensor* output_num_attempted_layers_t = nullptr;
    Tensor* output_last_layer_nodes_range_t = nullptr;

    OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(),
                                                     &output_stamp_token_t));
    OP_REQUIRES_OK(context, context->allocate_output(1, TensorShape(),
                                                     &output_num_trees_t));
    OP_REQUIRES_OK(context,
                   context->allocate_output(2, TensorShape(),
                                            &output_num_finalized_trees_t));
    OP_REQUIRES_OK(context,
                   context->allocate_output(3, TensorShape(),
                                            &output_num_attempted_layers_t));
    OP_REQUIRES_OK(context, context->allocate_output(
                                4, {2}, &output_last_layer_nodes_range_t));

    output_stamp_token_t->scalar<int64_t>()() = tree_ensemble_resource->stamp();
    output_num_trees_t->scalar<int32>()() = num_trees;
    output_num_finalized_trees_t->scalar<int32>()() = num_finalized_trees;
    output_num_attempted_layers_t->scalar<int32>()() = num_attempted_layers;

    int32_t range_start;
    int32_t range_end;
    tree_ensemble_resource->GetLastLayerNodesRange(&range_start, &range_end);

    output_last_layer_nodes_range_t->vec<int32>()(0) = range_start;
    // For a completely empty ensemble, this will be 0. To make it a valid range
    // we add this max cond.
    output_last_layer_nodes_range_t->vec<int32>()(1) = std::max(1, range_end);
  }
};

REGISTER_KERNEL_BUILDER(
    Name("BoostedTreesGetEnsembleStates").Device(DEVICE_CPU),
    BoostedTreesGetEnsembleStatesOp);

// Op for serializing a model.
class BoostedTreesSerializeEnsembleOp : public OpKernel {
 public:
  explicit BoostedTreesSerializeEnsembleOp(OpKernelConstruction* context)
      : OpKernel(context) {
    VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use "
            << "TensorFlow Decision Forests instead "
            << "(https://github.com/tensorflow/decision-forests).\n";
  }

  void Compute(OpKernelContext* context) override {
    core::RefCountPtr<BoostedTreesEnsembleResource> tree_ensemble_resource;
    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
                                           &tree_ensemble_resource));
    tf_shared_lock l(*tree_ensemble_resource->get_mutex());
    Tensor* output_stamp_token_t = nullptr;
    OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(),
                                                     &output_stamp_token_t));
    output_stamp_token_t->scalar<int64_t>()() = tree_ensemble_resource->stamp();
    Tensor* output_proto_t = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(1, TensorShape(), &output_proto_t));
    output_proto_t->scalar<tstring>()() =
        tree_ensemble_resource->SerializeAsString();
  }
};

REGISTER_KERNEL_BUILDER(
    Name("BoostedTreesSerializeEnsemble").Device(DEVICE_CPU),
    BoostedTreesSerializeEnsembleOp);

// Op for deserializing a tree ensemble variable from a checkpoint.
class BoostedTreesDeserializeEnsembleOp : public OpKernel {
 public:
  explicit BoostedTreesDeserializeEnsembleOp(OpKernelConstruction* context)
      : OpKernel(context) {
    VLOG(1) << "Boosted Trees kernels in TF are deprecated. Please use "
            << "TensorFlow Decision Forests instead "
            << "(https://github.com/tensorflow/decision-forests).\n";
  }

  void Compute(OpKernelContext* context) override {
    core::RefCountPtr<BoostedTreesEnsembleResource> tree_ensemble_resource;
    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
                                           &tree_ensemble_resource));
    mutex_lock l(*tree_ensemble_resource->get_mutex());

    // Get the stamp token.
    const Tensor* stamp_token_t;
    OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t));
    int64_t stamp_token = stamp_token_t->scalar<int64_t>()();

    // Get the tree ensemble proto.
    const Tensor* tree_ensemble_serialized_t;
    OP_REQUIRES_OK(context, context->input("tree_ensemble_serialized",
                                           &tree_ensemble_serialized_t));
    // Deallocate all the previous objects on the resource.
    tree_ensemble_resource->Reset();
    OP_REQUIRES(
        context,
        tree_ensemble_resource->InitFromSerialized(
            tree_ensemble_serialized_t->scalar<tstring>()(), stamp_token),
        errors::InvalidArgument("Unable to parse tree ensemble proto."));
  }
};

REGISTER_KERNEL_BUILDER(
    Name("BoostedTreesDeserializeEnsemble").Device(DEVICE_CPU),
    BoostedTreesDeserializeEnsembleOp);

}  // namespace tensorflow
<|endoftext|>


<commit_before>#include <stdexcept>
#include "array.h"
#include "helpers.h"
#include "string.h"
#include "value.h"
#include "zendstring.h"

static inline zval* derefAndCheck(zval* z)
{
    if (Z_ISREF_P(z)) {
        z = Z_REFVAL_P(z);
    }

    if (UNEXPECTED(Z_TYPE_P(z) != IS_ARRAY)) {
        throw std::runtime_error("Not an array");
    }

    return z;
}

phpcxx::Array::Array()
{
    array_init(&this->m_z);
}

phpcxx::Array::Array(zval* z)
{
    if (Z_TYPE_P(z) == IS_ARRAY) {
        ZVAL_MAKE_REF(z);
        zend_reference* ref = Z_REF_P(z);
        ++GC_REFCOUNT(ref);
        ZVAL_REF(&this->m_z, ref);
    }
    else {
        ZVAL_COPY(&this->m_z, z);
        convert_to_array(&this->m_z);
    }
}

phpcxx::Array::Array(Value& v)
    : Array(&v.m_z)
{
}

phpcxx::Array::Array(Array& other)
{
    ZVAL_COPY(&this->m_z, &other.m_z);
}

phpcxx::Array::Array(Array&& other)
{
    ZVAL_UNDEF(&this->m_z);
    std::swap(this->m_z, other.m_z);
}

phpcxx::Array::~Array()
{
    i_zval_ptr_dtor(&this->m_z ZEND_FILE_LINE_CC);
#ifdef PHPCXX_DEBUG
    ZVAL_UNDEF(&this->m_z);
#endif
}

phpcxx::Array& phpcxx::Array::operator=(const Array& other)
{
    phpcxx::assign(&this->m_z, &other.m_z);
    return *this;
}

phpcxx::Value& phpcxx::Array::operator[](std::nullptr_t)
{
    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);

    zval* var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(z), &EG(uninitialized_zval));
    if (UNEXPECTED(!var_ptr)) {
        throw std::runtime_error("Cannot add element to the array as the next element is already occupied");
    }

    return *(new(var_ptr) Value(placement_construct));
}

phpcxx::Value& phpcxx::Array::operator[](zend_long idx)
{
    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);

    zend_ulong h = static_cast<zend_ulong>(idx);
    zval* retval = zend_hash_index_find(Z_ARRVAL_P(z), h);
    if (!retval) {
        retval = zend_hash_index_add_new(Z_ARRVAL_P(z), h, &EG(uninitialized_zval));
    }

    return *(new(retval) Value(placement_construct));
}

phpcxx::Value& phpcxx::Array::operator[](const Value& key)
{
    zval* z = &key.m_z;

    while (true) {
        switch (key.type()) {
            case Type::String:    return this->operator[](Z_STR(key.m_z));
            case Type::Integer:   return this->operator[](Z_LVAL(this->m_z));
            case Type::Double:    return this->operator[](zend_dval_to_lval(Z_DVAL(this->m_z)));
            case Type::True:      return this->operator[](static_cast<zend_long>(1l));
            case Type::False:     return this->operator[](static_cast<zend_long>(0l));
            case Type::Resource:  return this->operator[](Z_RES_HANDLE_P(z));
            case Type::Undefined: return this->operator[](ZSTR_EMPTY_ALLOC());
            case Type::Null:      return this->operator[](ZSTR_EMPTY_ALLOC());
            case Type::Reference:
                z = Z_REFVAL_P(z);
                break;

            default:
                throw std::runtime_error("Illegal offset type");
        }
    }
}

phpcxx::Value& phpcxx::Array::operator[](zend_string* key)
{
    zend_ulong hval;
    if (ZEND_HANDLE_NUMERIC(key, hval)) {
        return this->operator[](static_cast<zend_long>(hval));
    }

    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);

    zval* retval = zend_hash_find(Z_ARRVAL_P(z), key);
    if (retval) {
        if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) {
            retval = Z_INDIRECT_P(retval);
            if (UNEXPECTED(Z_TYPE_P(retval) == IS_UNDEF)) {
                ZVAL_NULL(retval);
            }
        }
    }
    else {
        retval = zend_hash_add_new(Z_ARRVAL_P(z), key, &EG(uninitialized_zval));
    }

    return *(new(retval) Value(placement_construct));
}

std::size_t phpcxx::Array::size() const
{
    zval* z = derefAndCheck(&this->m_z);
    return zend_hash_num_elements(Z_ARRVAL_P(z));
}

bool phpcxx::Array::contains(zend_long idx) const
{
    zval* z = derefAndCheck(&this->m_z);
    return zend_hash_index_exists(Z_ARRVAL_P(z), static_cast<zend_ulong>(idx));
}

bool phpcxx::Array::contains(const Value& key) const
{
    zval* z = &key.m_z;

    while (true) {
        switch (key.type()) {
            case Type::String:    return this->contains(Z_STR(key.m_z));
            case Type::Integer:   return this->contains(Z_LVAL(this->m_z));
            case Type::Double:    return this->contains(zend_dval_to_lval(Z_DVAL(this->m_z)));
            case Type::True:      return this->contains(static_cast<zend_long>(1));
            case Type::False:     return this->contains(static_cast<zend_long>(0));
            case Type::Resource:  return this->contains(Z_RES_HANDLE_P(z));
            case Type::Undefined: return this->contains(ZSTR_EMPTY_ALLOC());
            case Type::Null:      return this->contains(ZSTR_EMPTY_ALLOC());
            case Type::Reference:
                z = Z_REFVAL_P(z);
                break;

            default:
                throw std::runtime_error("Illegal offset type");
        }
    }
}

bool phpcxx::Array::contains(zend_string* key) const
{
    zend_ulong hval;
    if (ZEND_HANDLE_NUMERIC(key, hval)) {
        return this->contains(static_cast<zend_long>(hval));
    }

    zval* z = derefAndCheck(&this->m_z);
    return zend_hash_exists(Z_ARRVAL_P(z), key);
}

void phpcxx::Array::unset(zend_long idx)
{
    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);
    zend_hash_index_del(Z_ARRVAL_P(z), static_cast<zend_ulong>(idx));
}

void phpcxx::Array::unset(const Value& key)
{
    zval* z = &key.m_z;

    while (true) {
        switch (key.type()) {
            case Type::String:    return this->unset(Z_STR(key.m_z));
            case Type::Integer:   return this->unset(Z_LVAL(this->m_z));
            case Type::Double:    return this->unset(zend_dval_to_lval(Z_DVAL(this->m_z)));
            case Type::True:      return this->unset(static_cast<zend_long>(1));
            case Type::False:     return this->unset(static_cast<zend_long>(0));
            case Type::Resource:  return this->unset(Z_RES_HANDLE_P(z));
            case Type::Undefined: return this->unset(ZSTR_EMPTY_ALLOC());
            case Type::Null:      return this->unset(ZSTR_EMPTY_ALLOC());
            case Type::Reference:
                z = Z_REFVAL_P(z);
                break;

            default:
                throw std::runtime_error("Illegal offset type");
        }
    }
}

void phpcxx::Array::unset(zend_string* key)
{
    zend_ulong hval;
    if (ZEND_HANDLE_NUMERIC(key, hval)) {
        return this->unset(static_cast<zend_long>(hval));
    }

    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);

    HashTable* ht = Z_ARRVAL_P(z);
    if (UNEXPECTED(ht == &EG(symbol_table))) {
        zend_delete_global_variable(key);
    }
    else {
        zend_hash_del(ht, key);
    }
}

phpcxx::Value& phpcxx::Array::operator[](const char* key)
{
    return this->operator[](ZendString(key).release());
}

phpcxx::Value& phpcxx::Array::operator[](const string& key)
{
    return this->operator[](ZendString(key).release());
}

phpcxx::Value& phpcxx::Array::operator[](const ZendString& key)
{
    return this->operator[](key.get());
}

bool phpcxx::Array::contains(const char* key) const
{
    return this->contains(ZendString(key).release());
}

bool phpcxx::Array::contains(const string& key) const
{
    return this->contains(ZendString(key).release());
}

bool phpcxx::Array::contains(const ZendString& key) const
{
    return this->contains(key.get());
}

void phpcxx::Array::unset(const char* key)
{
    this->unset(ZendString(key).release());
}

void phpcxx::Array::unset(const string& key)
{
    this->unset(ZendString(key).release());
}

void phpcxx::Array::unset(const ZendString& key)
{
    this->unset(key.get());
}
<commit_msg>Fix memory leak<commit_after>#include <stdexcept>
#include "array.h"
#include "helpers.h"
#include "string.h"
#include "value.h"
#include "zendstring.h"

static inline zval* derefAndCheck(zval* z)
{
    if (Z_ISREF_P(z)) {
        z = Z_REFVAL_P(z);
    }

    if (UNEXPECTED(Z_TYPE_P(z) != IS_ARRAY)) {
        throw std::runtime_error("Not an array");
    }

    return z;
}

phpcxx::Array::Array()
{
    array_init(&this->m_z);
}

phpcxx::Array::Array(zval* z)
{
    if (Z_TYPE_P(z) == IS_ARRAY) {
        ZVAL_MAKE_REF(z);
        zend_reference* ref = Z_REF_P(z);
        ++GC_REFCOUNT(ref);
        ZVAL_REF(&this->m_z, ref);
    }
    else {
        ZVAL_COPY(&this->m_z, z);
        convert_to_array(&this->m_z);
    }
}

phpcxx::Array::Array(Value& v)
    : Array(&v.m_z)
{
}

phpcxx::Array::Array(Array& other)
{
    ZVAL_COPY(&this->m_z, &other.m_z);
}

phpcxx::Array::Array(Array&& other)
{
    ZVAL_UNDEF(&this->m_z);
    std::swap(this->m_z, other.m_z);
}

phpcxx::Array::~Array()
{
    i_zval_ptr_dtor(&this->m_z ZEND_FILE_LINE_CC);
#ifdef PHPCXX_DEBUG
    ZVAL_UNDEF(&this->m_z);
#endif
}

phpcxx::Array& phpcxx::Array::operator=(const Array& other)
{
    phpcxx::assign(&this->m_z, &other.m_z);
    return *this;
}

phpcxx::Value& phpcxx::Array::operator[](std::nullptr_t)
{
    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);

    zval* var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(z), &EG(uninitialized_zval));
    if (UNEXPECTED(!var_ptr)) {
        throw std::runtime_error("Cannot add element to the array as the next element is already occupied");
    }

    return *(new(var_ptr) Value(placement_construct));
}

phpcxx::Value& phpcxx::Array::operator[](zend_long idx)
{
    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);

    zend_ulong h = static_cast<zend_ulong>(idx);
    zval* retval = zend_hash_index_find(Z_ARRVAL_P(z), h);
    if (!retval) {
        retval = zend_hash_index_add_new(Z_ARRVAL_P(z), h, &EG(uninitialized_zval));
    }

    return *(new(retval) Value(placement_construct));
}

phpcxx::Value& phpcxx::Array::operator[](const Value& key)
{
    zval* z = &key.m_z;

    while (true) {
        switch (key.type()) {
            case Type::String:    return this->operator[](Z_STR(key.m_z));
            case Type::Integer:   return this->operator[](Z_LVAL(this->m_z));
            case Type::Double:    return this->operator[](zend_dval_to_lval(Z_DVAL(this->m_z)));
            case Type::True:      return this->operator[](static_cast<zend_long>(1l));
            case Type::False:     return this->operator[](static_cast<zend_long>(0l));
            case Type::Resource:  return this->operator[](Z_RES_HANDLE_P(z));
            case Type::Undefined: return this->operator[](ZSTR_EMPTY_ALLOC());
            case Type::Null:      return this->operator[](ZSTR_EMPTY_ALLOC());
            case Type::Reference:
                z = Z_REFVAL_P(z);
                break;

            default:
                throw std::runtime_error("Illegal offset type");
        }
    }
}

phpcxx::Value& phpcxx::Array::operator[](zend_string* key)
{
    zend_ulong hval;
    if (ZEND_HANDLE_NUMERIC(key, hval)) {
        return this->operator[](static_cast<zend_long>(hval));
    }

    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);

    zval* retval = zend_hash_find(Z_ARRVAL_P(z), key);
    if (retval) {
        if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) {
            retval = Z_INDIRECT_P(retval);
            if (UNEXPECTED(Z_TYPE_P(retval) == IS_UNDEF)) {
                ZVAL_NULL(retval);
            }
        }
    }
    else {
        retval = zend_hash_add_new(Z_ARRVAL_P(z), key, &EG(uninitialized_zval));
    }

    return *(new(retval) Value(placement_construct));
}

std::size_t phpcxx::Array::size() const
{
    zval* z = derefAndCheck(&this->m_z);
    return zend_hash_num_elements(Z_ARRVAL_P(z));
}

bool phpcxx::Array::contains(zend_long idx) const
{
    zval* z = derefAndCheck(&this->m_z);
    return zend_hash_index_exists(Z_ARRVAL_P(z), static_cast<zend_ulong>(idx));
}

bool phpcxx::Array::contains(const Value& key) const
{
    zval* z = &key.m_z;

    while (true) {
        switch (key.type()) {
            case Type::String:    return this->contains(Z_STR(key.m_z));
            case Type::Integer:   return this->contains(Z_LVAL(this->m_z));
            case Type::Double:    return this->contains(zend_dval_to_lval(Z_DVAL(this->m_z)));
            case Type::True:      return this->contains(static_cast<zend_long>(1));
            case Type::False:     return this->contains(static_cast<zend_long>(0));
            case Type::Resource:  return this->contains(Z_RES_HANDLE_P(z));
            case Type::Undefined: return this->contains(ZSTR_EMPTY_ALLOC());
            case Type::Null:      return this->contains(ZSTR_EMPTY_ALLOC());
            case Type::Reference:
                z = Z_REFVAL_P(z);
                break;

            default:
                throw std::runtime_error("Illegal offset type");
        }
    }
}

bool phpcxx::Array::contains(zend_string* key) const
{
    zend_ulong hval;
    if (ZEND_HANDLE_NUMERIC(key, hval)) {
        return this->contains(static_cast<zend_long>(hval));
    }

    zval* z = derefAndCheck(&this->m_z);
    return zend_hash_exists(Z_ARRVAL_P(z), key);
}

void phpcxx::Array::unset(zend_long idx)
{
    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);
    zend_hash_index_del(Z_ARRVAL_P(z), static_cast<zend_ulong>(idx));
}

void phpcxx::Array::unset(const Value& key)
{
    zval* z = &key.m_z;

    while (true) {
        switch (key.type()) {
            case Type::String:    return this->unset(Z_STR(key.m_z));
            case Type::Integer:   return this->unset(Z_LVAL(this->m_z));
            case Type::Double:    return this->unset(zend_dval_to_lval(Z_DVAL(this->m_z)));
            case Type::True:      return this->unset(static_cast<zend_long>(1));
            case Type::False:     return this->unset(static_cast<zend_long>(0));
            case Type::Resource:  return this->unset(Z_RES_HANDLE_P(z));
            case Type::Undefined: return this->unset(ZSTR_EMPTY_ALLOC());
            case Type::Null:      return this->unset(ZSTR_EMPTY_ALLOC());
            case Type::Reference:
                z = Z_REFVAL_P(z);
                break;

            default:
                throw std::runtime_error("Illegal offset type");
        }
    }
}

void phpcxx::Array::unset(zend_string* key)
{
    zend_ulong hval;
    if (ZEND_HANDLE_NUMERIC(key, hval)) {
        return this->unset(static_cast<zend_long>(hval));
    }

    zval* z = derefAndCheck(&this->m_z);
    SEPARATE_ARRAY(z);

    HashTable* ht = Z_ARRVAL_P(z);
    if (UNEXPECTED(ht == &EG(symbol_table))) {
        zend_delete_global_variable(key);
    }
    else {
        zend_hash_del(ht, key);
    }
}

phpcxx::Value& phpcxx::Array::operator[](const char* key)
{
    return this->operator[](ZendString(key).get());
}

phpcxx::Value& phpcxx::Array::operator[](const string& key)
{
    return this->operator[](ZendString(key).get());
}

phpcxx::Value& phpcxx::Array::operator[](const ZendString& key)
{
    return this->operator[](key.get());
}

bool phpcxx::Array::contains(const char* key) const
{
    return this->contains(ZendString(key).get());
}

bool phpcxx::Array::contains(const string& key) const
{
    return this->contains(ZendString(key).get());
}

bool phpcxx::Array::contains(const ZendString& key) const
{
    return this->contains(key.get());
}

void phpcxx::Array::unset(const char* key)
{
    this->unset(ZendString(key).get());
}

void phpcxx::Array::unset(const string& key)
{
    this->unset(ZendString(key).get());
}

void phpcxx::Array::unset(const ZendString& key)
{
    this->unset(key.get());
}
<|endoftext|>


<commit_before>#include "cssysdef.h"
#include "awssktst.h"
#include "csutil/scfstr.h"
#include "iaws/iawsparm.h"

#include <stdio.h>

static char *names[10] = { "Yellow", "Green", "Blue", "Orange", "Purple", "Red", "White", "Teal", "Black" };
static int   namec = 0;

awsTestSink::awsTestSink():wmgr(NULL), sink(NULL), user(NULL), pass(NULL), test(NULL)
{
}

awsTestSink::~awsTestSink()
{
  if (sink) sink->DecRef();
}

void
awsTestSink::SetSink(iAwsSink *s)
{
 sink=s;

 if (sink)
 {
   sink->IncRef();

   sink->RegisterTrigger("RedClicked", &RedClicked);
   sink->RegisterTrigger("BlueClicked", &BlueClicked);
   sink->RegisterTrigger("GreenClicked", &GreenClicked);
   sink->RegisterTrigger("SetUserName", &SetUser);
   sink->RegisterTrigger("SetPassword", &SetPass);
   sink->RegisterTrigger("Login", &Login);
   sink->RegisterTrigger("FillListBox", &FillListBox);
 }
}

void
awsTestSink::SetTestWin(iAwsWindow *testwin)
{
  test=testwin;
}

void
awsTestSink::SetWindowManager(iAws *_wmgr)
{
  wmgr=_wmgr;
}

void
awsTestSink::FillListBox(void *sk, iAwsSource *source)
{
  awsTestSink *sink = (awsTestSink *)sk;
  iAwsComponent *comp = source->GetComponent();

  iAwsParmList *pl=0;

  printf("awstest: Filling list box.\n");

  if (sink->wmgr)
    pl = sink->wmgr->CreateParmList();
  else
    printf("awstest: window manager is null.\n");

  if (pl==NULL)
  {
    printf("awstest: internal error, parameter list NULL.\n");
    return;
  }

  // Setup first row
  pl->AddString("text0", new scfString("Human"));

  pl->AddString("text1", new scfString("Enabled"));
  pl->AddBool("stateful1", true);

  pl->AddString("text2", new scfString("Shenobi"));

  // Add it into the list
  comp->Execute("InsertItem", *pl);

  pl->Clear();

  //////////////////////////
  // Setup second row
  pl->AddString("text0", new scfString("Android"));

  pl->AddString("text1", new scfString("Enabled"));
  pl->AddBool("stateful1", true);
  pl->AddBool("state1", true);

  pl->AddString("text2", new scfString("Tenalt"));

  // Add it into the list
  comp->Execute("InsertItem", *pl);

  pl->Clear();
  pl->DecRef();
}

void
awsTestSink::RedClicked(void *sink,   iAwsSource *source)
{
  printf("awstest: red button clicked, source: %p, owner: %p, component: %p\n", source, sink, source->GetComponent());

  namec++;
  if (namec > 8) namec=0;

  iAwsComponent *comp = source->GetComponent();
  comp->SetProperty("Caption", new scfString(names[namec]));
}

void
awsTestSink::BlueClicked(void *sink,  iAwsSource *source)
{
  printf("awstest: blue button clicked, source: %p, owner: %p\n", source, sink);
}

void
awsTestSink::GreenClicked(void *sink, iAwsSource *source)
{
  printf("awstest: green button clicked, source: %p, owner: %p\n", source, sink);
}

void
awsTestSink::SetPass(void *sk, iAwsSource *source)
{
  awsTestSink *sink = (awsTestSink *)sk;
  if (sink->pass) sink->pass->DecRef();

  iAwsComponent *comp = source->GetComponent();
  comp->GetProperty("Text", (void**)&sink->pass);
}

void
awsTestSink::SetUser(void *sk, iAwsSource *source)
{
  awsTestSink *sink = (awsTestSink *)sk;
  if (sink->user) sink->user->DecRef();

  iAwsComponent *comp = source->GetComponent();
  comp->GetProperty("Text", (void**)&sink->user);
}

void
awsTestSink::Login(void *sk, iAwsSource *source)
{
  awsTestSink *sink = (awsTestSink *)sk;
  if (sink->user==NULL || sink->pass==NULL)
    printf("awstest: You must enter a username AND password.\n");

  else {
    printf("awstest: Logging in as %s with password: %s  (not really.)\n", sink->user->GetData(), sink->pass->GetData());
    iAwsComponent *comp = source->GetComponent();
    iAwsParmList *pl;

    if (sink->wmgr) pl = sink->wmgr->CreateParmList();

    comp->Execute("HideWindow", *pl);
    if (sink->test) sink->test->Show();

    pl->DecRef();
  }
}<commit_msg>Added some test data for the listbox.<commit_after>#include "cssysdef.h"
#include "awssktst.h"
#include "csutil/scfstr.h"
#include "iaws/iawsparm.h"

#include <stdio.h>

static char *names[10] = { "Yellow", "Green", "Blue", "Orange", "Purple", "Red", "White", "Teal", "Black" };
static int   namec = 0;

awsTestSink::awsTestSink():wmgr(NULL), sink(NULL), user(NULL), pass(NULL), test(NULL)
{
}

awsTestSink::~awsTestSink()
{
  if (sink) sink->DecRef();
}

void
awsTestSink::SetSink(iAwsSink *s)
{
 sink=s;

 if (sink)
 {
   sink->IncRef();

   sink->RegisterTrigger("RedClicked", &RedClicked);
   sink->RegisterTrigger("BlueClicked", &BlueClicked);
   sink->RegisterTrigger("GreenClicked", &GreenClicked);
   sink->RegisterTrigger("SetUserName", &SetUser);
   sink->RegisterTrigger("SetPassword", &SetPass);
   sink->RegisterTrigger("Login", &Login);
   sink->RegisterTrigger("FillListBox", &FillListBox);
 }
}

void
awsTestSink::SetTestWin(iAwsWindow *testwin)
{
  test=testwin;
}

void
awsTestSink::SetWindowManager(iAws *_wmgr)
{
  wmgr=_wmgr;
}

void
awsTestSink::FillListBox(void *sk, iAwsSource *source)
{
  awsTestSink *sink = (awsTestSink *)sk;
  iAwsComponent *comp = source->GetComponent();

  iAwsParmList *pl=0;
  int parent;

  printf("awstest: Filling list box.\n");

  if (sink->wmgr)
    pl = sink->wmgr->CreateParmList();
  else
    printf("awstest: window manager is null.\n");

  if (pl==NULL)
  {
    printf("awstest: internal error, parameter list NULL.\n");
    return;
  }

  // Setup first row
  pl->AddString("text0", new scfString("Human"));

  pl->AddString("text1", new scfString("Enabled"));
  pl->AddBool("stateful1", true);

  pl->AddString("text2", new scfString("Shenobi"));

  // Add it into the list
  comp->Execute("InsertItem", *pl);

  pl->Clear();

  //////////////////////////
  // Setup second row
  pl->AddString("text0", new scfString("Android"));

  pl->AddString("text1", new scfString("Enabled"));
  pl->AddBool("stateful1", true);
  pl->AddBool("state1", true);

  pl->AddString("text2", new scfString("Tenalt"));

  // Add it into the list
  comp->Execute("InsertItem", *pl);

  // Get the id of the last item for hierarchical support.
  pl->GetInt("id", &parent);
  pl->Clear();

  //////////////////////////
  // Setup third row (hierarchical)
  pl->AddString("text0", new scfString("Ship"));

  pl->AddString("text1", new scfString("Active"));
  pl->AddBool("stateful1", true);
  pl->AddBool("state1", true);

  pl->AddString("text2", new scfString("Daedalus"));
  pl->AddInt("parent", parent);

  // Add it into the list
  comp->Execute("InsertItem", *pl);
  pl->Clear();

  //////////////////////////
  // Setup fourth row (hierarchical)
  pl->AddString("text0", new scfString("Ship"));

  pl->AddString("text1", new scfString("Active"));
  pl->AddBool("stateful1", true);
  pl->AddBool("state1", false);

  pl->AddString("text2", new scfString("Temtor"));
  pl->AddInt("parent", parent);

  // Add it into the list
  comp->Execute("InsertItem", *pl);

  pl->Clear();
  pl->DecRef();
}

void
awsTestSink::RedClicked(void *sink,   iAwsSource *source)
{
  printf("awstest: red button clicked, source: %p, owner: %p, component: %p\n", source, sink, source->GetComponent());

  namec++;
  if (namec > 8) namec=0;

  iAwsComponent *comp = source->GetComponent();
  comp->SetProperty("Caption", new scfString(names[namec]));
}

void
awsTestSink::BlueClicked(void *sink,  iAwsSource *source)
{
  printf("awstest: blue button clicked, source: %p, owner: %p\n", source, sink);
}

void
awsTestSink::GreenClicked(void *sink, iAwsSource *source)
{
  printf("awstest: green button clicked, source: %p, owner: %p\n", source, sink);
}

void
awsTestSink::SetPass(void *sk, iAwsSource *source)
{
  awsTestSink *sink = (awsTestSink *)sk;
  if (sink->pass) sink->pass->DecRef();

  iAwsComponent *comp = source->GetComponent();
  comp->GetProperty("Text", (void**)&sink->pass);
}

void
awsTestSink::SetUser(void *sk, iAwsSource *source)
{
  awsTestSink *sink = (awsTestSink *)sk;
  if (sink->user) sink->user->DecRef();

  iAwsComponent *comp = source->GetComponent();
  comp->GetProperty("Text", (void**)&sink->user);
}

void
awsTestSink::Login(void *sk, iAwsSource *source)
{
  awsTestSink *sink = (awsTestSink *)sk;
  if (sink->user==NULL || sink->pass==NULL)
    printf("awstest: You must enter a username AND password.\n");

  else {
    printf("awstest: Logging in as %s with password: %s  (not really.)\n", sink->user->GetData(), sink->pass->GetData());
    iAwsComponent *comp = source->GetComponent();
    iAwsParmList *pl;

    if (sink->wmgr) pl = sink->wmgr->CreateParmList();

    comp->Execute("HideWindow", *pl);
    if (sink->test) sink->test->Show();

    pl->DecRef();
  }
}<|endoftext|>