<commit_before>
// Copyright 2010-2012 University of Washington. All Rights Reserved.
// LICENSE_PLACEHOLDER
// This software was created with Government support under DE
// AC05-76RL01830 awarded by the United States Department of
// Energy. The Government has certain rights in the software.

#ifndef __INCOHERENT_ACQUIRER_HPP__
#define __INCOHERENT_ACQUIRER_HPP__

#include "Grappa.hpp"
#include "Addressing.hpp"
#include "Message.hpp"

#ifdef VTRACE
#include <vt_user.h>
#endif

// forward declare for active message templates
template< typename T >
class IncoherentAcquirer;

template< typename T >
static void incoherent_acquire_reply_am( typename IncoherentAcquirer< T >::ReplyArgs * args,
                                         size_t size,
                                         void * payload, size_t payload_size );

template< typename T >
static void incoherent_acquire_request_am( typename IncoherentAcquirer< T >::RequestArgs * args,
                                           size_t size,
                                           void * payload, size_t payload_size );
/// IncoherentAcquirer statistics
class IAStatistics {
  private:
  uint64_t acquire_ams;
  uint64_t acquire_ams_bytes;
  uint64_t acquire_blocked;
  uint64_t acquire_blocked_ticks_total;
  uint64_t acquire_network_ticks_total;
  uint64_t acquire_wakeup_ticks_total;
  uint64_t acquire_blocked_ticks_max;
  uint64_t acquire_blocked_ticks_min;
  uint64_t acquire_network_ticks_max;
  uint64_t acquire_network_ticks_min;
  uint64_t acquire_wakeup_ticks_max;
  uint64_t acquire_wakeup_ticks_min;
#ifdef VTRACE_SAMPLED
  unsigned ia_grp_vt;
  unsigned acquire_ams_ev_vt;
  unsigned acquire_ams_bytes_ev_vt;
  unsigned acquire_blocked_ev_vt;
  unsigned acquire_blocked_ticks_total_ev_vt;
  unsigned acquire_network_ticks_total_ev_vt;
  unsigned acquire_wakeup_ticks_total_ev_vt;
  unsigned acquire_blocked_ticks_max_ev_vt;
  unsigned acquire_blocked_ticks_min_ev_vt;
  unsigned acquire_wakeup_ticks_max_ev_vt;
  unsigned acquire_wakeup_ticks_min_ev_vt;
  unsigned acquire_network_ticks_max_ev_vt;
  unsigned acquire_network_ticks_min_ev_vt;
  unsigned average_latency_ev_vt;
  unsigned average_network_latency_ev_vt;
  unsigned average_wakeup_latency_ev_vt;
#endif

  public:
    IAStatistics();
    void reset();

    inline void count_acquire_ams( uint64_t bytes ) {
      acquire_ams++;
      acquire_ams_bytes+=bytes;
    }

  inline void record_wakeup_latency( int64_t start_time, int64_t network_time ) {
    acquire_blocked++;
    int64_t current_time = Grappa_get_timestamp();
    int64_t blocked_latency = current_time - start_time;
    int64_t wakeup_latency = current_time - network_time;
    acquire_blocked_ticks_total += blocked_latency;
    acquire_wakeup_ticks_total += wakeup_latency;
    if( blocked_latency > acquire_blocked_ticks_max )
      acquire_blocked_ticks_max = blocked_latency;
    if( blocked_latency < acquire_blocked_ticks_min )
      acquire_blocked_ticks_min = blocked_latency;
    if( wakeup_latency > acquire_wakeup_ticks_max )
      acquire_wakeup_ticks_max = wakeup_latency;
    if( wakeup_latency < acquire_wakeup_ticks_min )
      acquire_wakeup_ticks_min = wakeup_latency;
  }

  inline void record_network_latency( int64_t start_time ) {
    int64_t current_time = Grappa_get_timestamp();
    int64_t latency = current_time - start_time;
    acquire_network_ticks_total += latency;
    if( latency > acquire_network_ticks_max )
      acquire_network_ticks_max = latency;
    if( latency < acquire_network_ticks_min )
      acquire_network_ticks_min = latency;
  }

    void dump( std::ostream& o, const char * terminator );
    void sample();
    void profiling_sample();
    void merge(const IAStatistics * other);
};

extern IAStatistics incoherent_acquirer_stats;

/// IncoherentAcquirer behavior for Cache.
template< typename T >
class IncoherentAcquirer {
private:
  bool acquire_started_;
  bool acquired_;
  GlobalAddress< T > * request_address_;
  size_t * count_;
  T ** pointer_;
  Thread * thread_;
  int num_messages_;
  int response_count_;
  int total_reply_payload_;
  int expected_reply_payload_;
  int64_t start_time_;
  int64_t network_time_;

public:

  IncoherentAcquirer( GlobalAddress< T > * request_address, size_t * count, T ** pointer )
    : request_address_( request_address )
    , count_( count )
    , pointer_( pointer )
    , acquire_started_( false )
    , acquired_( false )
    , thread_(NULL)
    , num_messages_(0)
    , response_count_(0)
    , total_reply_payload_( 0 )
    , expected_reply_payload_( 0 )
    , start_time_(0)
    , network_time_(0)
  {
    reset( );
  }

  void reset( ) {
    CHECK( !acquire_started_ || acquired_ ) << "inconsistent state for reset";
    acquire_started_ = false;
    acquired_ = false;
    thread_ = NULL;
    num_messages_ = 0;
    response_count_ = 0;
    expected_reply_payload_ = sizeof( T ) * *count_;
    total_reply_payload_ = 0;
    start_time_ = 0;
    network_time_ = 0;
    if( *count_ == 0 ) {
      DVLOG(5) << "Zero-length acquire";
      *pointer_ = NULL;
      acquire_started_ = true;
      acquired_ = true;
    } else if( request_address_->is_2D() ) {
      num_messages_ = 1;
      if( request_address_->node() == Grappa_mynode() ) {
        DVLOG(5) << "Short-circuiting to address " << request_address_->pointer();
        *pointer_ = request_address_->pointer();
        acquire_started_ = true;
        acquired_ = true;
      }
    } else {
      DVLOG(5) << "Straddle: block_max is " << (*request_address_ + *count_).block_max() ;
      DVLOG(5) << ", request_address is " << *request_address_;
      DVLOG(5) << ", sizeof(T) is " << sizeof(T);
      DVLOG(5) << ", count is " << *count_;
      DVLOG(5) << ", block_min is " << request_address_->block_min();

      DVLOG(5) << "Straddle: address is " << *request_address_ ;
      DVLOG(5) << ", address + count is " << *request_address_ + *count_;

      ptrdiff_t byte_diff = ( (*request_address_ + *count_ - 1).last_byte().block_max() -
                               request_address_->first_byte().block_min() );

      DVLOG(5) << "Straddle: address block max is " << request_address_->block_max();
      DVLOG(5) << " address + count block max is " << (*request_address_ + *count_).block_max();
      DVLOG(5) << " address block min " << request_address_->block_min();
      DVLOG(5) << "Straddle: diff is " << byte_diff << " bs " << block_size;
      num_messages_ = byte_diff / block_size;
    }

    if( num_messages_ > 1 ) DVLOG(5) << "****************************** MULTI BLOCK CACHE REQUEST ******************************";

    DVLOG(5) << "New IncoherentAcquirer; detecting straddle for sizeof(T):" << sizeof(T)
             << " count:" << *count_
             << " num_messages_:" << num_messages_
             << " request_address:" << *request_address_;
  }

  void start_acquire() {
    if( !acquire_started_ ) {
#ifdef VTRACE_FULL
      VT_TRACER("incoherent start_acquire");
#endif
      DVLOG(5) << "Thread " << CURRENT_THREAD
              << " issuing acquire for " << *request_address_
              << " * " << *count_ ;
      acquire_started_ = true;
      RequestArgs args;
      args.request_address = *request_address_;
      DVLOG(5) << "Computing request_bytes from block_max " << request_address_->first_byte().block_max() << " and " << *request_address_;
      args.reply_address = make_global( this );
      args.offset = 0;
      size_t total_bytes = *count_ * sizeof(T);

      // allocate enough requests/messages that we don't run out
      size_t nmsg = total_bytes / block_size + 2;
      RequestArgs arg_array[nmsg];
      Grappa::ExternalMessage<RequestArgs> msg_array[nmsg];

      for(size_t i = 0;
           args.offset < total_bytes;
           args.offset += args.request_bytes, i++) {

        args.request_bytes = args.request_address.first_byte().block_max() - args.request_address.first_byte();

        if( args.request_bytes > total_bytes - args.offset ) {
          args.request_bytes = total_bytes - args.offset;
        }

        DVLOG(5) << "sending acquire request for " << args.request_bytes
                 << " of total bytes = " << *count_ * sizeof(T)
                 << " from " << args.request_address;

        arg_array[i] = args;
        new (msg_array+i) Grappa::ExternalMessage<RequestArgs>(arg_array[i].request_address.node(), &arg_array[i]);
        msg_array[i].enqueue();

        // TODO: change type so we don't screw with pointer like this
        args.request_address = GlobalAddress<T>::Raw( args.request_address.raw_bits() + args.request_bytes );
      }
      DVLOG(5) << "acquire started for " << args.request_address;

      // blocks here waiting for messages to be sent
    }
  }

  void block_until_acquired() {
    if( !acquired_ ) {
      start_acquire();
#ifdef VTRACE_FULL
      VT_TRACER("incoherent block_until_acquired");
#endif
      DVLOG(5) << "Thread " << CURRENT_THREAD
              << " ready to block on " << *request_address_
              << " * " << *count_ ;
      if( !acquired_ ) {
        start_time_ = Grappa_get_timestamp();
      } else {
        start_time_ = 0;
      }
      while( !acquired_ ) {
      DVLOG(5) << "Thread " << CURRENT_THREAD
              << " blocking on " << *request_address_
              << " * " << *count_ ;
        if( !acquired_ ) {
          thread_ = CURRENT_THREAD;
          Grappa_suspend();
          thread_ = NULL;
        }
        DVLOG(5) << "Thread " << CURRENT_THREAD
                 << " woke up for " << *request_address_
                 << " * " << *count_ ;
      }
      incoherent_acquirer_stats.record_wakeup_latency( start_time_, network_time_ );
    }
  }

  void acquire_reply( size_t offset, void * payload, size_t payload_size ) {
    DVLOG(5) << "Thread " << CURRENT_THREAD
             << " copying reply payload of " << payload_size
             << " and waking Thread " << thread_;
    memcpy( ((char*)(*pointer_)) + offset, payload, payload_size );
    ++response_count_;
    total_reply_payload_ += payload_size;
    if ( response_count_ == num_messages_ ) {
      DCHECK_EQ( total_reply_payload_, expected_reply_payload_ ) << "Got back the wrong amount of data "
                                                                 << "(with base = " << *request_address_
                                                                 << " and sizeof(T) = " << sizeof(T)
                                                                 << " and count = " << *count_;
      acquired_ = true;
      if( thread_ != NULL ) {
        Grappa_wake( thread_ );
      }
      if( start_time_ != 0 ) {
        network_time_ = Grappa_get_timestamp();
        incoherent_acquirer_stats.record_network_latency( start_time_ );
      }
    }
  }

  /// Has acquire completed?
  bool acquired() const { return acquired_; }

  /// Args for incoherent acquire reply
  struct ReplyArgs {
    GlobalAddress< IncoherentAcquirer > reply_address;
    int offset;

    void operator()(void * payload, size_t payload_size) {
      DVLOG(5) << "Thread " << CURRENT_THREAD
               << " received acquire reply to " << this->reply_address
               << " offset " << this->offset
               << " payload size " << payload_size;
      this->reply_address.pointer()->acquire_reply( this->offset, payload, payload_size );
    }
  };

  /// Args for incoherent acquire request
  struct RequestArgs {
    GlobalAddress< T > request_address;
    size_t request_bytes;
    GlobalAddress< IncoherentAcquirer > reply_address;
    int offset;

    void operator()() {
      incoherent_acquirer_stats.count_acquire_ams( this->request_bytes );
      DVLOG(5) << "Thread " << CURRENT_THREAD
               << " received acquire request to " << this->request_address
               << " size " << this->request_bytes
               << " offset " << this->offset
               << " reply to " << this->reply_address;

      typename IncoherentAcquirer<T>::ReplyArgs reply_args;
      reply_args.reply_address = this->reply_address;
      reply_args.offset = this->offset;

      DVLOG(5) << "Thread " << CURRENT_THREAD
               << " sending acquire reply to " << this->reply_address
               << " offset " << this->offset
               << " request address " << this->request_address
               << " payload address " << this->request_address.pointer()
               << " payload size " << this->request_bytes;

      // note: this will read the payload *later* when the message is copied into the actual send buffer,
      // should be okay because we're already assuming DRF, but something to watch out for
      Grappa::send_heap_message(this->reply_address.node(), reply_args,
                       this->request_address.pointer(), this->request_bytes );

      DVLOG(5) << "Thread " << CURRENT_THREAD
               << " sent acquire reply to " << this->reply_address
               << " offset " << this->offset
               << " request address " << this->request_address
               << " payload address " << this->request_address.pointer()
               << " payload size " << this->request_bytes;
    }
  };

};

#endif
<commit_msg>Use blocking MessagePool in IncoherentAcquirer as well.<commit_after>
// Copyright 2010-2012 University of Washington. All Rights Reserved.
// LICENSE_PLACEHOLDER
// This software was created with Government support under DE
// AC05-76RL01830 awarded by the United States Department of
// Energy. The Government has certain rights in the software.

#ifndef __INCOHERENT_ACQUIRER_HPP__
#define __INCOHERENT_ACQUIRER_HPP__

#include "Grappa.hpp"
#include "Addressing.hpp"
#include "Message.hpp"
#include "MessagePool.hpp"

#ifdef VTRACE
#include <vt_user.h>
#endif

// forward declare for active message templates
template< typename T >
class IncoherentAcquirer;

template< typename T >
static void incoherent_acquire_reply_am( typename IncoherentAcquirer< T >::ReplyArgs * args,
                                         size_t size,
                                         void * payload, size_t payload_size );

template< typename T >
static void incoherent_acquire_request_am( typename IncoherentAcquirer< T >::RequestArgs * args,
                                           size_t size,
                                           void * payload, size_t payload_size );
/// IncoherentAcquirer statistics
class IAStatistics {
  private:
  uint64_t acquire_ams;
  uint64_t acquire_ams_bytes;
  uint64_t acquire_blocked;
  uint64_t acquire_blocked_ticks_total;
  uint64_t acquire_network_ticks_total;
  uint64_t acquire_wakeup_ticks_total;
  uint64_t acquire_blocked_ticks_max;
  uint64_t acquire_blocked_ticks_min;
  uint64_t acquire_network_ticks_max;
  uint64_t acquire_network_ticks_min;
  uint64_t acquire_wakeup_ticks_max;
  uint64_t acquire_wakeup_ticks_min;
#ifdef VTRACE_SAMPLED
  unsigned ia_grp_vt;
  unsigned acquire_ams_ev_vt;
  unsigned acquire_ams_bytes_ev_vt;
  unsigned acquire_blocked_ev_vt;
  unsigned acquire_blocked_ticks_total_ev_vt;
  unsigned acquire_network_ticks_total_ev_vt;
  unsigned acquire_wakeup_ticks_total_ev_vt;
  unsigned acquire_blocked_ticks_max_ev_vt;
  unsigned acquire_blocked_ticks_min_ev_vt;
  unsigned acquire_wakeup_ticks_max_ev_vt;
  unsigned acquire_wakeup_ticks_min_ev_vt;
  unsigned acquire_network_ticks_max_ev_vt;
  unsigned acquire_network_ticks_min_ev_vt;
  unsigned average_latency_ev_vt;
  unsigned average_network_latency_ev_vt;
  unsigned average_wakeup_latency_ev_vt;
#endif

  public:
    IAStatistics();
    void reset();

    inline void count_acquire_ams( uint64_t bytes ) {
      acquire_ams++;
      acquire_ams_bytes+=bytes;
    }

  inline void record_wakeup_latency( int64_t start_time, int64_t network_time ) {
    acquire_blocked++;
    int64_t current_time = Grappa_get_timestamp();
    int64_t blocked_latency = current_time - start_time;
    int64_t wakeup_latency = current_time - network_time;
    acquire_blocked_ticks_total += blocked_latency;
    acquire_wakeup_ticks_total += wakeup_latency;
    if( blocked_latency > acquire_blocked_ticks_max )
      acquire_blocked_ticks_max = blocked_latency;
    if( blocked_latency < acquire_blocked_ticks_min )
      acquire_blocked_ticks_min = blocked_latency;
    if( wakeup_latency > acquire_wakeup_ticks_max )
      acquire_wakeup_ticks_max = wakeup_latency;
    if( wakeup_latency < acquire_wakeup_ticks_min )
      acquire_wakeup_ticks_min = wakeup_latency;
  }

  inline void record_network_latency( int64_t start_time ) {
    int64_t current_time = Grappa_get_timestamp();
    int64_t latency = current_time - start_time;
    acquire_network_ticks_total += latency;
    if( latency > acquire_network_ticks_max )
      acquire_network_ticks_max = latency;
    if( latency < acquire_network_ticks_min )
      acquire_network_ticks_min = latency;
  }

    void dump( std::ostream& o, const char * terminator );
    void sample();
    void profiling_sample();
    void merge(const IAStatistics * other);
};

extern IAStatistics incoherent_acquirer_stats;

/// IncoherentAcquirer behavior for Cache.
template< typename T >
class IncoherentAcquirer {
private:
  bool acquire_started_;
  bool acquired_;
  GlobalAddress< T > * request_address_;
  size_t * count_;
  T ** pointer_;
  Thread * thread_;
  int num_messages_;
  int response_count_;
  int total_reply_payload_;
  int expected_reply_payload_;
  int64_t start_time_;
  int64_t network_time_;

public:

  IncoherentAcquirer( GlobalAddress< T > * request_address, size_t * count, T ** pointer )
    : request_address_( request_address )
    , count_( count )
    , pointer_( pointer )
    , acquire_started_( false )
    , acquired_( false )
    , thread_(NULL)
    , num_messages_(0)
    , response_count_(0)
    , total_reply_payload_( 0 )
    , expected_reply_payload_( 0 )
    , start_time_(0)
    , network_time_(0)
  {
    reset( );
  }

  void reset( ) {
    CHECK( !acquire_started_ || acquired_ ) << "inconsistent state for reset";
    acquire_started_ = false;
    acquired_ = false;
    thread_ = NULL;
    num_messages_ = 0;
    response_count_ = 0;
    expected_reply_payload_ = sizeof( T ) * *count_;
    total_reply_payload_ = 0;
    start_time_ = 0;
    network_time_ = 0;
    if( *count_ == 0 ) {
      DVLOG(5) << "Zero-length acquire";
      *pointer_ = NULL;
      acquire_started_ = true;
      acquired_ = true;
    } else if( request_address_->is_2D() ) {
      num_messages_ = 1;
      if( request_address_->node() == Grappa_mynode() ) {
        DVLOG(5) << "Short-circuiting to address " << request_address_->pointer();
        *pointer_ = request_address_->pointer();
        acquire_started_ = true;
        acquired_ = true;
      }
    } else {
      DVLOG(5) << "Straddle: block_max is " << (*request_address_ + *count_).block_max() ;
      DVLOG(5) << ", request_address is " << *request_address_;
      DVLOG(5) << ", sizeof(T) is " << sizeof(T);
      DVLOG(5) << ", count is " << *count_;
      DVLOG(5) << ", block_min is " << request_address_->block_min();

      DVLOG(5) << "Straddle: address is " << *request_address_ ;
      DVLOG(5) << ", address + count is " << *request_address_ + *count_;

      ptrdiff_t byte_diff = ( (*request_address_ + *count_ - 1).last_byte().block_max() -
                               request_address_->first_byte().block_min() );

      DVLOG(5) << "Straddle: address block max is " << request_address_->block_max();
      DVLOG(5) << " address + count block max is " << (*request_address_ + *count_).block_max();
      DVLOG(5) << " address block min " << request_address_->block_min();
      DVLOG(5) << "Straddle: diff is " << byte_diff << " bs " << block_size;
      num_messages_ = byte_diff / block_size;
    }

    if( num_messages_ > 1 ) DVLOG(5) << "****************************** MULTI BLOCK CACHE REQUEST ******************************";

    DVLOG(5) << "New IncoherentAcquirer; detecting straddle for sizeof(T):" << sizeof(T)
             << " count:" << *count_
             << " num_messages_:" << num_messages_
             << " request_address:" << *request_address_;
  }

  void start_acquire() {
    if( !acquire_started_ ) {
#ifdef VTRACE_FULL
      VT_TRACER("incoherent start_acquire");
#endif
      DVLOG(5) << "Thread " << CURRENT_THREAD
              << " issuing acquire for " << *request_address_
              << " * " << *count_ ;
      acquire_started_ = true;
      RequestArgs args;
      args.request_address = *request_address_;
      DVLOG(5) << "Computing request_bytes from block_max " << request_address_->first_byte().block_max() << " and " << *request_address_;
      args.reply_address = make_global( this );
      args.offset = 0;
      size_t total_bytes = *count_ * sizeof(T);

      // allocate enough requests/messages that we don't run out
      size_t nmsg = total_bytes / block_size + 2;
      Grappa::MessagePoolStatic<STACK_SIZE/2> pool;

      for(size_t i = 0;
           args.offset < total_bytes;
           args.offset += args.request_bytes, i++) {

        args.request_bytes = args.request_address.first_byte().block_max() - args.request_address.first_byte();

        if( args.request_bytes > total_bytes - args.offset ) {
          args.request_bytes = total_bytes - args.offset;
        }

        DVLOG(5) << "sending acquire request for " << args.request_bytes
                 << " of total bytes = " << *count_ * sizeof(T)
                 << " from " << args.request_address;

        pool.send_message(args.request_address.core(), [args]{
          incoherent_acquirer_stats.count_acquire_ams( args.request_bytes );
          DVLOG(5) << "Thread " << CURRENT_THREAD
          << " received acquire request to " << args.request_address
          << " size " << args.request_bytes
          << " offset " << args.offset
          << " reply to " << args.reply_address;

          DVLOG(5) << "Thread " << CURRENT_THREAD
          << " sending acquire reply to " << args.reply_address
          << " offset " << args.offset
          << " request address " << args.request_address
          << " payload address " << args.request_address.pointer()
          << " payload size " << args.request_bytes;

          // note: this will read the payload *later* when the message is copied into the actual send buffer,
          // should be okay because we're already assuming DRF, but something to watch out for
          auto reply_address = args.reply_address;
          auto offset = args.offset;

          Grappa::send_heap_message(args.reply_address.node(),
            [reply_address, offset](void * payload, size_t payload_size) {
              DVLOG(5) << "Thread " << CURRENT_THREAD
              << " received acquire reply to " << reply_address
              << " offset " << offset
              << " payload size " << payload_size;
              reply_address.pointer()->acquire_reply( offset, payload, payload_size);
            },
            args.request_address.pointer(), args.request_bytes
          );

          DVLOG(5) << "Thread " << CURRENT_THREAD
          << " sent acquire reply to " << args.reply_address
          << " offset " << args.offset
          << " request address " << args.request_address
          << " payload address " << args.request_address.pointer()
          << " payload size " << args.request_bytes;
        });

        // TODO: change type so we don't screw with pointer like this
        args.request_address = GlobalAddress<T>::Raw( args.request_address.raw_bits() + args.request_bytes );
      }
      DVLOG(5) << "acquire started for " << args.request_address;

      // blocks here waiting for messages to be sent
    }
  }

  void block_until_acquired() {
    if( !acquired_ ) {
      start_acquire();
#ifdef VTRACE_FULL
      VT_TRACER("incoherent block_until_acquired");
#endif
      DVLOG(5) << "Thread " << CURRENT_THREAD
              << " ready to block on " << *request_address_
              << " * " << *count_ ;
      if( !acquired_ ) {
        start_time_ = Grappa_get_timestamp();
      } else {
        start_time_ = 0;
      }
      while( !acquired_ ) {
      DVLOG(5) << "Thread " << CURRENT_THREAD
              << " blocking on " << *request_address_
              << " * " << *count_ ;
        if( !acquired_ ) {
          thread_ = CURRENT_THREAD;
          Grappa_suspend();
          thread_ = NULL;
        }
        DVLOG(5) << "Thread " << CURRENT_THREAD
                 << " woke up for " << *request_address_
                 << " * " << *count_ ;
      }
      incoherent_acquirer_stats.record_wakeup_latency( start_time_, network_time_ );
    }
  }

  void acquire_reply( size_t offset, void * payload, size_t payload_size ) {
    DVLOG(5) << "Thread " << CURRENT_THREAD
             << " copying reply payload of " << payload_size
             << " and waking Thread " << thread_;
    memcpy( ((char*)(*pointer_)) + offset, payload, payload_size );
    ++response_count_;
    total_reply_payload_ += payload_size;
    if ( response_count_ == num_messages_ ) {
      DCHECK_EQ( total_reply_payload_, expected_reply_payload_ ) << "Got back the wrong amount of data "
                                                                 << "(with base = " << *request_address_
                                                                 << " and sizeof(T) = " << sizeof(T)
                                                                 << " and count = " << *count_;
      acquired_ = true;
      if( thread_ != NULL ) {
        Grappa_wake( thread_ );
      }
      if( start_time_ != 0 ) {
        network_time_ = Grappa_get_timestamp();
        incoherent_acquirer_stats.record_network_latency( start_time_ );
      }
    }
  }

  /// Has acquire completed?
  bool acquired() const { return acquired_; }

  /// Args for incoherent acquire reply
  struct ReplyArgs {
    GlobalAddress< IncoherentAcquirer > reply_address;
    int offset;

    void operator()(void * payload, size_t payload_size) {
      DVLOG(5) << "Thread " << CURRENT_THREAD
               << " received acquire reply to " << this->reply_address
               << " offset " << this->offset
               << " payload size " << payload_size;
      this->reply_address.pointer()->acquire_reply( this->offset, payload, payload_size );
    }
  };

  /// Args for incoherent acquire request
  struct RequestArgs {
    GlobalAddress< T > request_address;
    size_t request_bytes;
    GlobalAddress< IncoherentAcquirer > reply_address;
    int offset;

    void operator()() {
      incoherent_acquirer_stats.count_acquire_ams( this->request_bytes );
      DVLOG(5) << "Thread " << CURRENT_THREAD
               << " received acquire request to " << this->request_address
               << " size " << this->request_bytes
               << " offset " << this->offset
               << " reply to " << this->reply_address;

      typename IncoherentAcquirer<T>::ReplyArgs reply_args;
      reply_args.reply_address = this->reply_address;
      reply_args.offset = this->offset;

      DVLOG(5) << "Thread " << CURRENT_THREAD
               << " sending acquire reply to " << this->reply_address
               << " offset " << this->offset
               << " request address " << this->request_address
               << " payload address " << this->request_address.pointer()
               << " payload size " << this->request_bytes;

      // note: this will read the payload *later* when the message is copied into the actual send buffer,
      // should be okay because we're already assuming DRF, but something to watch out for
      Grappa::send_heap_message(this->reply_address.node(), reply_args,
                       this->request_address.pointer(), this->request_bytes );

      DVLOG(5) << "Thread " << CURRENT_THREAD
               << " sent acquire reply to " << this->reply_address
               << " offset " << this->offset
               << " request address " << this->request_address
               << " payload address " << this->request_address.pointer()
               << " payload size " << this->request_bytes;
    }
  };

};

#endif
<|endoftext|>


<commit_before>
//
// This source file is part of appleseed.
// Visit http://appleseedhq.net/ for additional information and resources.
//
// This software is released under the MIT license.
//
// Copyright (c) 2010-2012 Francois Beaune, Jupiter Jazz Limited
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

// Interface header.
#include "constantsurfaceshader.h"

// appleseed.renderer headers.
#include "renderer/global/globaltypes.h"
#include "renderer/kernel/shading/shadingcontext.h"
#include "renderer/kernel/shading/shadingpoint.h"
#include "renderer/kernel/shading/shadingresult.h"
#include "renderer/modeling/input/inputarray.h"
#include "renderer/modeling/surfaceshader/surfaceshader.h"
#include "renderer/utility/paramarray.h"

// appleseed.foundation headers.
#include "foundation/image/colorspace.h"
#include "foundation/utility/containers/dictionary.h"
#include "foundation/utility/containers/specializedarrays.h"

using namespace foundation;
using namespace std;

namespace renderer
{

namespace
{
    //
    // Constant color surface shader.
    //

    const char* Model = "constant_surface_shader";

    class ConstantSurfaceShader
      : public SurfaceShader
    {
      public:
        ConstantSurfaceShader(
            const char*             name,
            const ParamArray&       params)
          : SurfaceShader(name, params)
        {
            m_inputs.declare("color", InputFormatSpectrum);
            m_inputs.declare("color_multiplier", InputFormatScalar, "1.0");
            m_inputs.declare("alpha_multiplier", InputFormatScalar, "1.0");

            const string alpha_source = m_params.get_optional<string>("alpha_source", "color");
            if (alpha_source == "color")
                m_alpha_source = AlphaSourceColor;
            else if (alpha_source == "material")
                m_alpha_source = AlphaSourceMaterial;
            else
            {
                RENDERER_LOG_ERROR(
                    "invalid value \"%s\" for parameter \"alpha_source\", "
                    "using default value \"color\".",
                    alpha_source.c_str());
                m_alpha_source = AlphaSourceColor;
            }
        }

        virtual void release() override
        {
            delete this;
        }

        virtual const char* get_model() const override
        {
            return Model;
        }

        virtual void evaluate(
            SamplingContext&        sampling_context,
            const ShadingContext&   shading_context,
            const ShadingPoint&     shading_point,
            ShadingResult&          shading_result) const override
        {
            // Evaluate the shader inputs.
            InputValues values;
            m_inputs.evaluate(
                shading_context.get_texture_cache(),
                shading_point.get_uv(0),
                &values);

            shading_result.m_color_space = ColorSpaceSpectral;
            shading_result.m_color = values.m_color;

            // Handle alpha mapping.
            if (m_alpha_source == AlphaSourceColor)
                shading_result.m_alpha = values.m_alpha;
            else
            {
                const Material* material = shading_point.get_material();
                if (material && material->get_alpha_map())
                {
                    // Evaluate the alpha map at the shading point.
                    material->get_alpha_map()->evaluate(
                        shading_context.get_texture_cache(),
                        shading_point.get_uv(0),
                        shading_result.m_alpha);
                }
                else shading_result.m_alpha = Alpha(1.0f);
            }

            // Apply multipliers.
            shading_result.m_color *= static_cast<float>(values.m_color_multiplier);
            shading_result.m_alpha *= static_cast<float>(values.m_alpha_multiplier);
        }

      private:
        struct InputValues
        {
            Spectrum    m_color;
            Alpha       m_alpha;
            double      m_color_multiplier;
            double      m_alpha_multiplier;
        };

        enum AlphaSource
        {
            AlphaSourceColor,
            AlphaSourceMaterial
        };

        AlphaSource     m_alpha_source;
    };
}

//
// ConstantSurfaceShaderFactory class implementation.
//

const char* ConstantSurfaceShaderFactory::get_model() const
{
    return Model;
}

const char* ConstantSurfaceShaderFactory::get_human_readable_model() const
{
    return "Constant";
}

DictionaryArray ConstantSurfaceShaderFactory::get_widget_definitions() const
{
    DictionaryArray definitions;

    definitions.push_back(
        Dictionary()
            .insert("name", "color")
            .insert("label", "Color")
            .insert("widget", "entity_picker")
            .insert("entity_types",
                Dictionary()
                    .insert("color", "Colors")
                    .insert("texture_instance", "Textures"))
            .insert("use", "required")
            .insert("default", ""));

    definitions.push_back(
        Dictionary()
            .insert("name", "alpha_source")
            .insert("label", "Alpha Source")
            .insert("widget", "dropdown_list")
            .insert("dropdown_items",
                Dictionary()
                    .insert("Alpha channel of the color", "color")
                    .insert("Alpha map of the material", "material"))
            .insert("use", "optional")
            .insert("default", "color"));

    definitions.push_back(
        Dictionary()
            .insert("name", "color_multiplier")
            .insert("label", "Color Multiplier")
            .insert("widget", "entity_picker")
            .insert("entity_types",
                Dictionary().insert("texture_instance", "Textures"))
            .insert("default", "1.0")
            .insert("use", "optional"));

    definitions.push_back(
        Dictionary()
            .insert("name", "alpha_multiplier")
            .insert("label", "Alpha Multiplier")
            .insert("widget", "entity_picker")
            .insert("entity_types",
                Dictionary().insert("texture_instance", "Textures"))
            .insert("default", "1.0")
            .insert("use", "optional"));

    return definitions;
}

auto_release_ptr<SurfaceShader> ConstantSurfaceShaderFactory::create(
    const char*         name,
    const ParamArray&   params) const
{
    return
        auto_release_ptr<SurfaceShader>(
            new ConstantSurfaceShader(name, params));
}

}   // namespace renderer
<commit_msg>added missing #include statements.<commit_after>
//
// This source file is part of appleseed.
// Visit http://appleseedhq.net/ for additional information and resources.
//
// This software is released under the MIT license.
//
// Copyright (c) 2010-2012 Francois Beaune, Jupiter Jazz Limited
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

// Interface header.
#include "constantsurfaceshader.h"

// appleseed.renderer headers.
#include "renderer/global/globaltypes.h"
#include "renderer/kernel/shading/shadingcontext.h"
#include "renderer/kernel/shading/shadingpoint.h"
#include "renderer/kernel/shading/shadingresult.h"
#include "renderer/modeling/input/inputarray.h"
#include "renderer/modeling/input/source.h"
#include "renderer/modeling/material/material.h"
#include "renderer/modeling/surfaceshader/surfaceshader.h"
#include "renderer/utility/paramarray.h"

// appleseed.foundation headers.
#include "foundation/image/colorspace.h"
#include "foundation/utility/containers/dictionary.h"
#include "foundation/utility/containers/specializedarrays.h"

using namespace foundation;
using namespace std;

namespace renderer
{

namespace
{
    //
    // Constant color surface shader.
    //

    const char* Model = "constant_surface_shader";

    class ConstantSurfaceShader
      : public SurfaceShader
    {
      public:
        ConstantSurfaceShader(
            const char*             name,
            const ParamArray&       params)
          : SurfaceShader(name, params)
        {
            m_inputs.declare("color", InputFormatSpectrum);
            m_inputs.declare("color_multiplier", InputFormatScalar, "1.0");
            m_inputs.declare("alpha_multiplier", InputFormatScalar, "1.0");

            const string alpha_source = m_params.get_optional<string>("alpha_source", "color");
            if (alpha_source == "color")
                m_alpha_source = AlphaSourceColor;
            else if (alpha_source == "material")
                m_alpha_source = AlphaSourceMaterial;
            else
            {
                RENDERER_LOG_ERROR(
                    "invalid value \"%s\" for parameter \"alpha_source\", "
                    "using default value \"color\".",
                    alpha_source.c_str());
                m_alpha_source = AlphaSourceColor;
            }
        }

        virtual void release() override
        {
            delete this;
        }

        virtual const char* get_model() const override
        {
            return Model;
        }

        virtual void evaluate(
            SamplingContext&        sampling_context,
            const ShadingContext&   shading_context,
            const ShadingPoint&     shading_point,
            ShadingResult&          shading_result) const override
        {
            // Evaluate the shader inputs.
            InputValues values;
            m_inputs.evaluate(
                shading_context.get_texture_cache(),
                shading_point.get_uv(0),
                &values);

            shading_result.m_color_space = ColorSpaceSpectral;
            shading_result.m_color = values.m_color;

            // Handle alpha mapping.
            if (m_alpha_source == AlphaSourceColor)
                shading_result.m_alpha = values.m_alpha;
            else
            {
                const Material* material = shading_point.get_material();
                if (material && material->get_alpha_map())
                {
                    // Evaluate the alpha map at the shading point.
                    material->get_alpha_map()->evaluate(
                        shading_context.get_texture_cache(),
                        shading_point.get_uv(0),
                        shading_result.m_alpha);
                }
                else shading_result.m_alpha = Alpha(1.0f);
            }

            // Apply multipliers.
            shading_result.m_color *= static_cast<float>(values.m_color_multiplier);
            shading_result.m_alpha *= static_cast<float>(values.m_alpha_multiplier);
        }

      private:
        struct InputValues
        {
            Spectrum    m_color;
            Alpha       m_alpha;
            double      m_color_multiplier;
            double      m_alpha_multiplier;
        };

        enum AlphaSource
        {
            AlphaSourceColor,
            AlphaSourceMaterial
        };

        AlphaSource     m_alpha_source;
    };
}

//
// ConstantSurfaceShaderFactory class implementation.
//

const char* ConstantSurfaceShaderFactory::get_model() const
{
    return Model;
}

const char* ConstantSurfaceShaderFactory::get_human_readable_model() const
{
    return "Constant";
}

DictionaryArray ConstantSurfaceShaderFactory::get_widget_definitions() const
{
    DictionaryArray definitions;

    definitions.push_back(
        Dictionary()
            .insert("name", "color")
            .insert("label", "Color")
            .insert("widget", "entity_picker")
            .insert("entity_types",
                Dictionary()
                    .insert("color", "Colors")
                    .insert("texture_instance", "Textures"))
            .insert("use", "required")
            .insert("default", ""));

    definitions.push_back(
        Dictionary()
            .insert("name", "alpha_source")
            .insert("label", "Alpha Source")
            .insert("widget", "dropdown_list")
            .insert("dropdown_items",
                Dictionary()
                    .insert("Alpha channel of the color", "color")
                    .insert("Alpha map of the material", "material"))
            .insert("use", "optional")
            .insert("default", "color"));

    definitions.push_back(
        Dictionary()
            .insert("name", "color_multiplier")
            .insert("label", "Color Multiplier")
            .insert("widget", "entity_picker")
            .insert("entity_types",
                Dictionary().insert("texture_instance", "Textures"))
            .insert("default", "1.0")
            .insert("use", "optional"));

    definitions.push_back(
        Dictionary()
            .insert("name", "alpha_multiplier")
            .insert("label", "Alpha Multiplier")
            .insert("widget", "entity_picker")
            .insert("entity_types",
                Dictionary().insert("texture_instance", "Textures"))
            .insert("default", "1.0")
            .insert("use", "optional"));

    return definitions;
}

auto_release_ptr<SurfaceShader> ConstantSurfaceShaderFactory::create(
    const char*         name,
    const ParamArray&   params) const
{
    return
        auto_release_ptr<SurfaceShader>(
            new ConstantSurfaceShader(name, params));
}

}   // namespace renderer
<|endoftext|>


<commit_before>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "core/framework/graph_partitioner.h"

#include "core/framework/kernel_registry_manager.h"
#include "core/graph/function.h"
#include "core/graph/graph_viewer.h"
#include "core/framework/compute_capability.h"
#include "core/framework/kernel_registry_manager.h"
#include "core/framework/execution_providers.h"
#include "core/framework/kernel_registry.h"

// uncomment this line to count non-CUDA ops in ONNX domain
//#define COUNT_NON_CUDA_OPS

#ifdef COUNT_NON_CUDA_OPS
class NonCudaOps {
 public:
  ~NonCudaOps() {
    printf("Non-CUDA ops:\n");
    for (auto i : map_) {
      printf("%s: %d\n", i.first.c_str(), i.second);
    }
  }

  void AddOp(const std::string& name) {
    if (map_.count(name))
      map_.at(name)++;
    else
      map_.insert({name, 1});
  }

 private:
  std::map<std::string, int> map_;
};

NonCudaOps non_cuda;
#endif

using namespace ::onnxruntime::common;
namespace onnxruntime {

KernelDefBuilder& BuildFusedKernelDef(KernelDefBuilder& builder, const onnxruntime::Node& node) {
  auto schema = node.Op();
  builder.SetName(schema->Name())
      .SetDomain(schema->domain())
      .SinceVersion(schema->SinceVersion())
      .Provider(node.GetExecutionProviderType());
  auto& inputs = node.InputDefs();
  for (auto input : inputs) {
    builder.TypeConstraint(input->Name(), DataTypeImpl::TypeFromProto(*input->TypeAsProto()));
  }
  return builder;
}

Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const {
  if (providers_.Empty()) {
    return Status(ONNXRUNTIME, INVALID_ARGUMENT, "No provider specified.");
  }
  //fused_kernel_registry is prepareing the kernels created on the fly for fused sub graph.
  //It is only visiable for current session.
  std::shared_ptr<KernelRegistry> fused_kernel_registry = std::make_shared<KernelRegistry>();
  // Partitioning <graph> based on provider preference and their capabilities.
  auto kernel_registries = kernel_registry_mgr_.GetAllKernelRegistries();
  for (auto& provider : providers_) {
    auto capability_results = provider->GetCapability(GraphViewer(graph), kernel_registries);
    int count = 0;
    for (auto& capability : capability_results) {
      if (nullptr == capability || nullptr == capability->sub_graph) {
        continue;
      }
      if (nullptr == capability->sub_graph->GetMetaDef()) {
        // The <provider> can run a single node in the <graph> if not using meta-defs.
        // A fused kernel is not supported in this case.
        ORT_ENFORCE(1 == capability->sub_graph->nodes.size());
        ORT_ENFORCE(capability->fuse_kernel_function == nullptr);

        auto node = graph.GetNode(capability->sub_graph->nodes[0]);
        if (nullptr != node && node->GetExecutionProviderType().empty()) {
          node->SetExecutionProviderType(provider->Type());
        }
      } else {
        // The <provider> can run a fused <sub_graph> in the <graph>.
        //
        // Add fused node into <graph>
        ORT_ENFORCE(nullptr != capability->sub_graph->GetMetaDef());
        std::string node_name = provider->Type() + "_" + capability->sub_graph->GetMetaDef()->name + "_" + std::to_string(count++);
        auto& fused_node = graph.FuseSubGraph(std::move(capability->sub_graph), node_name);
        fused_node.SetExecutionProviderType(provider->Type());
        auto fused_kernel_func = capability->fuse_kernel_function;
        if (fused_kernel_func != nullptr) {
          // build the kernel definition on the fly, and register it to the fused_kernel_regisitry.
          KernelDefBuilder builder;
          BuildFusedKernelDef(builder, fused_node);
          fused_kernel_registry->Register(builder, fused_kernel_func);
        }
      }
    }
    // all done with this provider, resolve the graph before we move on to the next provider.
    // This is needed since we create a new GraphViewer() that we pass into the next provider's GetCapability().
    ORT_ENFORCE(graph.Resolve().IsOK());
  }

  // To see if the node with no provider can be inlined. If one such nodes can be
  // successfully inlined, we re-run the partitioner on the modified graph.
  bool inline_flag = false;
  for (auto& node : graph.Nodes()) {
    if (node.GetExecutionProviderType().empty()) {
      auto node_func = node.GetFunctionBody();
      if (nullptr == node_func) {
        continue;
      }
      Status inliner_status = graph.InlineFunction(node);
      // If the node has a functionbody with no kernel and cannot be inlined
      // it is a invalid function
      if (!inliner_status.IsOK()) return inliner_status;
      // Set the flag for re-run graph partition after successful inlining
      inline_flag = true;
      break;
    }
  }
  // Resolve and rerun graph partition
  if (inline_flag) {
    ORT_RETURN_IF_ERROR(graph.Resolve());
    this->Partition(graph);
  }

  //For some cases, like fp16 on cpu, right now we don't have any kernel support that.
  //But we will insert cast op to run the model, so skip the error checking here.
  //If after graph transform phase, the node still not assigned, we will report error
  //during kernel creation phase.
#ifdef COUNT_NON_CUDA_OPS
  for (auto& node : graph.Nodes()) {
    if (node.GetExecutionProviderType() != kCudaExecutionProvider &&
        node.Domain() != kMLDomain &&
        node.Domain() != kMSDomain)
      non_cuda.AddOp(node.OpType());
  }
#endif

  kernel_registry_mgr_.RegisterKernelRegistry(fused_kernel_registry, KernelRegistryPriority::HighPriority);

  return Status::OK();
}
}  // namespace onnxruntime
<commit_msg>Kezhan/partition logic update (#164)<commit_after>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "core/framework/graph_partitioner.h"

#include "core/framework/kernel_registry_manager.h"
#include "core/graph/function.h"
#include "core/graph/graph_viewer.h"
#include "core/framework/compute_capability.h"
#include "core/framework/kernel_registry_manager.h"
#include "core/framework/execution_providers.h"
#include "core/framework/kernel_registry.h"

// uncomment this line to count non-CUDA ops in ONNX domain
//#define COUNT_NON_CUDA_OPS

#ifdef COUNT_NON_CUDA_OPS
class NonCudaOps {
 public:
  ~NonCudaOps() {
    printf("Non-CUDA ops:\n");
    for (auto i : map_) {
      printf("%s: %d\n", i.first.c_str(), i.second);
    }
  }

  void AddOp(const std::string& name) {
    if (map_.count(name))
      map_.at(name)++;
    else
      map_.insert({name, 1});
  }

 private:
  std::map<std::string, int> map_;
};

NonCudaOps non_cuda;
#endif

using namespace ::onnxruntime::common;
namespace onnxruntime {

KernelDefBuilder& BuildFusedKernelDef(KernelDefBuilder& builder, const onnxruntime::Node& node) {
  auto schema = node.Op();
  builder.SetName(schema->Name())
      .SetDomain(schema->domain())
      .SinceVersion(schema->SinceVersion())
      .Provider(node.GetExecutionProviderType());
  auto& inputs = node.InputDefs();
  for (auto input : inputs) {
    builder.TypeConstraint(input->Name(), DataTypeImpl::TypeFromProto(*input->TypeAsProto()));
  }
  return builder;
}

Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const {
  // It is a greedy partitioning algorithm per provider preferences user provided when calling ONNX RUNTIME right now.
  // 1. Execution providers' capabilities are checked one by one.
  // 2. All sub-graphs that an execution provider returns will be assigned to it if it's not assigned yet.
  // 3. CPU execution provider is expected to be able to run any node and is the last one in execution provider preference.

  if (providers_.Empty()) {
    return Status(ONNXRUNTIME, INVALID_ARGUMENT, "No provider specified.");
  }
  //fused_kernel_registry is prepareing the kernels created on the fly for fused sub graph.
  //It is only visiable for current session.
  std::shared_ptr<KernelRegistry> fused_kernel_registry = std::make_shared<KernelRegistry>();
  // Partitioning <graph> based on provider preference and their capabilities.
  auto kernel_registries = kernel_registry_mgr_.GetAllKernelRegistries();

  std::vector<std::vector<std::unique_ptr<ComputeCapability>>> capabilities_of_all_providers;
  GraphViewer graph_viewer(graph);
  for (auto& provider : providers_) {
    capabilities_of_all_providers.push_back(provider->GetCapability(graph_viewer, kernel_registries));
  }

  int i = 0;
  for (auto& provider : providers_) {
    int count = 0;
    for (auto& capability : capabilities_of_all_providers[i++]) {
      if (nullptr == capability || nullptr == capability->sub_graph) {
        continue;
      }
      if (nullptr == capability->sub_graph->GetMetaDef()) {
        // The <provider> can run a single node in the <graph> if not using meta-defs.
        // A fused kernel is not supported in this case.
        ORT_ENFORCE(1 == capability->sub_graph->nodes.size());
        ORT_ENFORCE(capability->fuse_kernel_function == nullptr);

        auto node = graph.GetNode(capability->sub_graph->nodes[0]);
        if (nullptr != node && node->GetExecutionProviderType().empty()) {
          // The node was not fused or assigned. Assign it to this <provider>.
          node->SetExecutionProviderType(provider->Type());
        }
      } else {
        // The <provider> can run a fused <sub_graph> in the <graph>.
        ORT_ENFORCE(nullptr != capability->sub_graph->GetMetaDef());

        // Check whether any node in the <sub_graph> was already assigned.
        bool sub_graph_available_for_assignment = true;
        for (auto node_index : capability->sub_graph->nodes) {
          auto node = graph.GetNode(node_index);
          if (nullptr == node || !node->GetExecutionProviderType().empty()) {
            // The node was fused or assigned, so that the whole sub-graph will not be assigned to this <provider>
            // The assumption is that this <provider> can only run the sub-graph as a whole unit.
            sub_graph_available_for_assignment = false;
            break;
          }
        }

        if (sub_graph_available_for_assignment) {
          // Add fused node into <graph>
          std::string node_name = provider->Type() + "_" + capability->sub_graph->GetMetaDef()->name + "_" + std::to_string(count++);
          auto& fused_node = graph.FuseSubGraph(std::move(capability->sub_graph), node_name);
          fused_node.SetExecutionProviderType(provider->Type());
          auto fused_kernel_func = capability->fuse_kernel_function;
          if (fused_kernel_func != nullptr) {
            // build the kernel definition on the fly, and register it to the fused_kernel_regisitry.
            KernelDefBuilder builder;
            BuildFusedKernelDef(builder, fused_node);
            fused_kernel_registry->Register(builder, fused_kernel_func);
          }
        }
      }
    }
  }

  ORT_ENFORCE(graph.Resolve().IsOK());

  // To see if the node with no provider can be inlined. If one such nodes can be
  // successfully inlined, we re-run the partitioner on the modified graph.
  bool inline_flag = false;
  for (auto& node : graph.Nodes()) {
    if (node.GetExecutionProviderType().empty()) {
      auto node_func = node.GetFunctionBody();
      if (nullptr == node_func) {
        continue;
      }
      Status inliner_status = graph.InlineFunction(node);
      // If the node has a functionbody with no kernel and cannot be inlined
      // it is a invalid function
      if (!inliner_status.IsOK()) return inliner_status;
      // Set the flag for re-run graph partition after successful inlining
      inline_flag = true;
      break;
    }
  }
  // Resolve and rerun graph partition
  if (inline_flag) {
    ORT_RETURN_IF_ERROR(graph.Resolve());
    this->Partition(graph);
  }

    //For some cases, like fp16 on cpu, right now we don't have any kernel support that.
    //But we will insert cast op to run the model, so skip the error checking here.
    //If after graph transform phase, the node still not assigned, we will report error
    //during kernel creation phase.
#ifdef COUNT_NON_CUDA_OPS
  for (auto& node : graph.Nodes()) {
    if (node.GetExecutionProviderType() != kCudaExecutionProvider &&
        node.Domain() != kMLDomain &&
        node.Domain() != kMSDomain)
      non_cuda.AddOp(node.OpType());
  }
#endif

  kernel_registry_mgr_.RegisterKernelRegistry(fused_kernel_registry, KernelRegistryPriority::HighPriority);

  return Status::OK();
}
}  // namespace onnxruntime
<|endoftext|>


<commit_before>// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.

#include "core/providers/acl/tensor/concat.h"
#include "core/providers/common.h"
#include "core/framework/TensorSeq.h"
#include "arm_compute/core/utils/misc/ShapeCalculator.h"

#include "core/providers/acl/acl_common.h"
#include "core/providers/acl/acl_fwd.h"

#define PREF_DIM 4

namespace onnxruntime {
namespace acl {

template <typename T>
Status Concat<T>::Compute(OpKernelContext* ctx) const {
  if(axis_ > 3) {
    LOGS_DEFAULT(WARNING) << "ArmNN does not have support for tensors with 4 or more dimensions; defaulting to cpu implementation";
    return onnxruntime::Concat::Compute(ctx);
  }

  // Number of input tensors to concatenate
  auto input_count = Node().InputArgCount().front();

  // Hold pointers to the input tensors to be used in the PrepareForCompute() step
  std::vector<const Tensor*> input_tensors;
  input_tensors.reserve(input_count);
  for (int i = 0; i < input_count; ++i) {
    input_tensors.push_back(ctx->Input<Tensor>(i));
  }

  std::vector<int64_t> output_dims = input_tensors[0]->Shape().GetDims();

  // 'Concat' mode
  if (!is_stack_) {
    // While concating, the rank of the output is the same as the input rank(s)

    // Calculate the size of the concatenated axis
    size_t concat_axis_size = 0;
    for (int64_t index = 0; index < input_count; index++) {
      concat_axis_size += input_tensors[index]->Shape()[static_cast<int>(axis_)];
    }

    output_dims[axis_] = concat_axis_size;
  } else { // 'Stack' mode
    // While stacking, the rank of the output is one more than the input rank(s).
    // Stacking may be thought of as adding an unit dimension (of value 1) in the input tensors,
    // and concatenating them on thie new axis.
    // The value in the corresponding axis of the output will be the number of inputs that are being stacked.
    output_dims.insert(output_dims.begin() + axis_, static_cast<int64_t>(input_count));
  }

  if(output_dims.size() > 4 || axis_ > 3) {
    LOGS_DEFAULT(WARNING) << "ArmNN does not have support for tensors with 4 or more dimensions; defaulting to cpu implementation";
    return onnxruntime::Concat::Compute(ctx);
  }

  TensorShape output_shape(output_dims);
  Tensor* Y = ctx->Output(0, output_shape);

  LOGS_DEFAULT(VERBOSE) << "Concat ACL:";

  arm_compute::Tensor output;
  std::vector<arm_compute::ITensor*> inputs_vector;
  for (int i = 0; i < input_count; i++) {
    arm_compute::Tensor* input = new arm_compute::Tensor();
    auto X = input_tensors[i];
    LOGS_DEFAULT(VERBOSE) << "X[" << i << "]: " << X->Shape().ToString().c_str();
    input->allocator()->init(arm_compute::TensorInfo(ACLTensorShape(X->Shape(), PREF_DIM), arm_compute::Format::F32));

    inputs_vector.push_back(input);
  }

  arm_compute::NEConcatenateLayer layer;
  layer.configure(inputs_vector, &output, 3 - axis_);

  LOGS_DEFAULT(VERBOSE) << "axis: " << axis_;
  LOGS_DEFAULT(VERBOSE) << std::endl;

  for (int i = 0; i < input_count; i++) {
    auto X = input_tensors[i];
    const T* x_data = X->template Data<T>();
    arm_compute::Tensor* in = static_cast<arm_compute::Tensor*>(inputs_vector[i]);

    if (X->Shape().Size() != 0 && in->info()->has_padding() ){
      in->allocator()->allocate();
      importDataToTensor<T>(in, x_data);
    } else {
      ACLImportMemory(in->allocator(), (void*)x_data, X->Shape().Size() * 4);
    }
  }

  T* y_data = Y->template MutableData<T>();

  if(Y->Shape().Size() != 0 && output.info()->has_padding() ){
    output.allocator()->allocate();
  } else {
    ACLImportMemory(output.allocator(), (void*)y_data, Y->Shape().Size() * 4);
  }

  layer.run();

  if (Y->Shape().Size() != 0 && output.info()->has_padding() ){
    importDataFromTensor<T>(&output, y_data);
  }

  return Status::OK();
}

ONNX_OPERATOR_VERSIONED_KERNEL_EX(
    Concat,
    kOnnxDomain,
    4, 10,
    kAclExecutionProvider,
    KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
    Concat<float>);

ONNX_OPERATOR_KERNEL_EX(
    Concat,
    kOnnxDomain,
    11,
    kAclExecutionProvider,
    KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
    Concat<float>);

}  // namespace acl
}  // namespace onnxruntime
<commit_msg>Fix compile (#10667)<commit_after>// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.

#include "core/providers/acl/tensor/concat.h"
#include "core/providers/common.h"
#include "core/framework/TensorSeq.h"
#include "arm_compute/core/utils/misc/ShapeCalculator.h"

#include "core/providers/acl/acl_common.h"
#include "core/providers/acl/acl_fwd.h"

#define PREF_DIM 4

namespace onnxruntime {
namespace acl {

template <typename T>
Status Concat<T>::Compute(OpKernelContext* ctx) const {
  if(axis_ > 3) {
    LOGS_DEFAULT(WARNING) << "ArmNN does not have support for tensors with 4 or more dimensions; defaulting to cpu implementation";
    return onnxruntime::Concat::Compute(ctx);
  }

  // Number of input tensors to concatenate
  auto input_count = Node().InputArgCount().front();

  // Hold pointers to the input tensors to be used in the PrepareForCompute() step
  std::vector<const Tensor*> input_tensors;
  input_tensors.reserve(input_count);
  for (int i = 0; i < input_count; ++i) {
    input_tensors.push_back(ctx->Input<Tensor>(i));
  }

  auto output_dims = input_tensors[0]->Shape().AsShapeVector();

  // 'Concat' mode
  if (!is_stack_) {
    // While concating, the rank of the output is the same as the input rank(s)

    // Calculate the size of the concatenated axis
    size_t concat_axis_size = 0;
    for (int64_t index = 0; index < input_count; index++) {
      concat_axis_size += input_tensors[index]->Shape()[static_cast<int>(axis_)];
    }

    output_dims[axis_] = concat_axis_size;
  } else { // 'Stack' mode
    // While stacking, the rank of the output is one more than the input rank(s).
    // Stacking may be thought of as adding an unit dimension (of value 1) in the input tensors,
    // and concatenating them on thie new axis.
    // The value in the corresponding axis of the output will be the number of inputs that are being stacked.
    output_dims.insert(output_dims.begin() + axis_, static_cast<int64_t>(input_count));
  }

  if(output_dims.size() > 4 || axis_ > 3) {
    LOGS_DEFAULT(WARNING) << "ArmNN does not have support for tensors with 4 or more dimensions; defaulting to cpu implementation";
    return onnxruntime::Concat::Compute(ctx);
  }

  TensorShape output_shape(output_dims);
  Tensor* Y = ctx->Output(0, output_shape);

  LOGS_DEFAULT(VERBOSE) << "Concat ACL:";

  arm_compute::Tensor output;
  std::vector<arm_compute::ITensor*> inputs_vector;
  for (int i = 0; i < input_count; i++) {
    arm_compute::Tensor* input = new arm_compute::Tensor();
    auto X = input_tensors[i];
    LOGS_DEFAULT(VERBOSE) << "X[" << i << "]: " << X->Shape().ToString().c_str();
    input->allocator()->init(arm_compute::TensorInfo(ACLTensorShape(X->Shape(), PREF_DIM), arm_compute::Format::F32));

    inputs_vector.push_back(input);
  }

  arm_compute::NEConcatenateLayer layer;
  layer.configure(inputs_vector, &output, 3 - axis_);

  LOGS_DEFAULT(VERBOSE) << "axis: " << axis_;
  LOGS_DEFAULT(VERBOSE) << std::endl;

  for (int i = 0; i < input_count; i++) {
    auto X = input_tensors[i];
    const T* x_data = X->template Data<T>();
    arm_compute::Tensor* in = static_cast<arm_compute::Tensor*>(inputs_vector[i]);

    if (X->Shape().Size() != 0 && in->info()->has_padding() ){
      in->allocator()->allocate();
      importDataToTensor<T>(in, x_data);
    } else {
      ACLImportMemory(in->allocator(), (void*)x_data, X->Shape().Size() * 4);
    }
  }

  T* y_data = Y->template MutableData<T>();

  if(Y->Shape().Size() != 0 && output.info()->has_padding() ){
    output.allocator()->allocate();
  } else {
    ACLImportMemory(output.allocator(), (void*)y_data, Y->Shape().Size() * 4);
  }

  layer.run();

  if (Y->Shape().Size() != 0 && output.info()->has_padding() ){
    importDataFromTensor<T>(&output, y_data);
  }

  return Status::OK();
}

ONNX_OPERATOR_VERSIONED_KERNEL_EX(
    Concat,
    kOnnxDomain,
    4, 10,
    kAclExecutionProvider,
    KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
    Concat<float>);

ONNX_OPERATOR_KERNEL_EX(
    Concat,
    kOnnxDomain,
    11,
    kAclExecutionProvider,
    KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
    Concat<float>);

}  // namespace acl
}  // namespace onnxruntime
<|endoftext|>


<commit_before>//---------------------------------------------------------------------------
//    $Id$
//    Version: $Name$
//
//    Copyright (C) 2008, 2009 by the deal.II authors
//
//    This file is subject to QPL and may not be  distributed
//    without copyright and license information. Please refer
//    to the file deal.II/doc/license.html for the  text  and
//    further information on this license.
//
//---------------------------------------------------------------------------

#include <base/utilities.h>
#include <lac/trilinos_vector.h>

#include <cmath>

#ifdef DEAL_II_USE_TRILINOS

#include <Epetra_Import.h>

DEAL_II_NAMESPACE_OPEN

namespace TrilinosWrappers
{
  namespace MPI
  {

    Vector::Vector ()
		    :
		    communicator (Utilities::Trilinos::
				  duplicate_communicator(Utilities::Trilinos::comm_self())),
		    map (0, 0, *communicator)
    {
      last_action = Zero;
      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
    }

    Vector::Vector (const Epetra_Map &InputMap)
                    :
		    communicator (Utilities::Trilinos::
				  duplicate_communicator (InputMap.Comm())),
		    map (Utilities::Trilinos::
			 duplicate_map (InputMap, *communicator))
    {
      last_action = Zero;
      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
    }

    Vector::Vector (const Vector &v)
                    :
                    VectorBase(),
		    communicator (Utilities::Trilinos::
				  duplicate_communicator (*v.communicator)),
		    map (Utilities::Trilinos::duplicate_map (v.map,
							     *communicator))
    {
      last_action = Zero;
      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(*v.vector));
    }

    Vector::Vector (const Epetra_Map &InputMap,
		    const VectorBase &v)
                    :
                    VectorBase(),
		    communicator (Utilities::Trilinos::
				  duplicate_communicator (InputMap.Comm())),
		    map (Utilities::Trilinos::
			 duplicate_map (InputMap, *communicator))
    {
      AssertThrow (map.NumGlobalElements() == v.vector->Map().NumGlobalElements(),
		   ExcDimensionMismatch (map.NumGlobalElements(),
					 v.vector->Map().NumGlobalElements()));

      last_action = Zero;

      if (map.SameAs(v.vector->Map()) == true)
	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(*v.vector));
      else
	{
	  vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
	  reinit (v, false, true);
	}
    }

    void
    Vector::reinit (const Epetra_Map &input_map,
		    const bool        fast)
    {
      vector.reset();

      communicator.reset (Utilities::Trilinos::
			  duplicate_communicator(input_map.Comm()));
      map = Utilities::Trilinos::duplicate_map (input_map, *communicator);

      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));

      if (fast == false)
	{
	  const int ierr = vector->PutScalar(0.);
	  AssertThrow (ierr == 0, ExcTrilinosError(ierr));
	}

      last_action = Zero;
    }

    void
    Vector::reinit (const VectorBase &v,
		    const bool        fast,
		    const bool        allow_different_maps)
    {
					// In case we do not allow to
					// have different maps, this
					// call means that we have to
					// reset the vector. So clear
					// the vector, initialize our
					// map with the map in v, and
					// generate the vector.
      if (allow_different_maps == false)
        {
	  if (local_range() != v.local_range())
	    {
	      vector.reset();
	      communicator.reset (Utilities::Trilinos::
				  duplicate_communicator(v.trilinos_vector()
							 .Map().Comm()));
	      map = Utilities::Trilinos::
		    duplicate_map (v.trilinos_vector().Map(), *communicator);

	      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
	      last_action = Zero;
	    }
	  else if (fast == false)
	    {
	      int ierr;
	      Assert (vector->Map().SameAs(v.vector->Map()) == true,
		      ExcMessage ("The Epetra maps in the assignment operator ="
				  " do not match, even though the local_range "
				  " seems to be the same. Check vector setup!"));
	      ierr = vector->GlobalAssemble (last_action);
	      AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	      ierr = vector->PutScalar(0.0);
	      AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	      last_action = Zero;
	    }
	}

					// Otherwise, we have to check
					// that the two vectors are
					// already of the same size,
					// create an object for the data
					// exchange and then insert all
					// the data. The first assertion
					// is only a check whether the
					// user knows what she is doing.
      else
        {
	  Assert (fast == false,
		  ExcMessage ("It is not possible to exchange data with the "
			      "option fast set, which would not write "
			      "elements."));

	  AssertThrow (size() == v.size(),
		       ExcDimensionMismatch (size(), v.size()));

	  Epetra_Import data_exchange (vector->Map(), v.vector->Map());

	  const int ierr = vector->Import(*v.vector, data_exchange, Insert);
	  AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	  last_action = Insert;
	}

    }

    Vector &
    Vector::operator = (const Vector &v)
    {
      if (local_range() == v.local_range())
	{
	  Assert (vector->Map().SameAs(v.vector->Map()) == true,
		  ExcMessage ("The Epetra maps in the assignment operator ="
			      " do not match, even though the local_range "
			      " seems to be the same. Check vector setup!"));

	  const int ierr = vector->Update(1.0, *v.vector, 0.0);
	  AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	  last_action = Zero;
	}
      else
	{
	  vector.reset();

	  communicator.reset (Utilities::Trilinos::
			      duplicate_communicator(v.trilinos_vector().
						     Map().Comm()));
	  map = Utilities::Trilinos::
		duplicate_map (v.trilinos_vector().Map(),
			       *communicator);

	  vector = std::auto_ptr<Epetra_FEVector>
	                    (new Epetra_FEVector(*v.vector));
	  last_action = Zero;
	}

      return *this;
    }

    Vector &
    Vector::operator = (const TrilinosWrappers::Vector &v)
    {
      Assert (size() == v.size(), ExcDimensionMismatch(size(), v.size()));

      Epetra_Import data_exchange (vector->Map(), v.vector->Map());
      const int ierr = vector->Import(*v.vector, data_exchange, Insert);

      AssertThrow (ierr == 0, ExcTrilinosError(ierr));

      last_action = Insert;

      return *this;
    }

    void
    Vector::import_nonlocal_data_for_fe (const TrilinosWrappers::SparseMatrix &m,
					 const Vector                         &v)
    {
      Assert (m.trilinos_matrix().Filled() == true,
	      ExcMessage ("Matrix is not compressed. "
			  "Cannot find exchange information!"));
      Assert (v.vector->Map().UniqueGIDs() == true,
	      ExcMessage ("The input vector has overlapping data, "
			  "which is not allowed."));

      if (vector->Map().SameAs(m.col_partitioner()) == false)
	{
	  map = m.col_partitioner();
	  vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
	}

      Epetra_Import data_exchange (vector->Map(), v.vector->Map());
      const int ierr = vector->Import(*v.vector, data_exchange, Insert);

      AssertThrow (ierr == 0, ExcTrilinosError(ierr));

      last_action = Insert;
    }

  } /* end of namespace MPI */

  Vector::Vector ()
		  :
		  map (0, 0,
		       Utilities::Trilinos::comm_self())
  {
    last_action = Zero;
    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
  }

  Vector::Vector (const unsigned int n)
                 :
		  map ((int)n, 0,
		       Utilities::Trilinos::comm_self())
  {
    last_action = Zero;
    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector (map));
  }

  Vector::Vector (const Epetra_Map &InputMap)
                 :
                 map (InputMap.NumGlobalElements(), InputMap.IndexBase(),
		      InputMap.Comm())
  {
    last_action = Zero;
    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
  }

  Vector::Vector (const VectorBase &v)
                 :
                 map (v.vector->Map().NumGlobalElements(), 0, v.vector->Comm())
  {
    last_action = Zero;
    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));

    if (vector->Map().SameAs(v.vector->Map()) == true)
      {
	const int ierr = vector->Update(1.0, *v.vector, 0.0);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));
      }
    else
      reinit (v, false, true);

  }

  void
  Vector::reinit (const unsigned int n,
		  const bool         fast)
  {
    if (size() != n)
      {
	vector.reset();

	map = Epetra_LocalMap ((int)n, 0,
			       Utilities::Trilinos::comm_self());

	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector (map));
      }
    else if (fast == false)
      {
	int ierr;
	ierr = vector->GlobalAssemble(last_action);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	ierr = vector->PutScalar(0.0);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));
      }

    last_action = Zero;
  }

  void
  Vector::reinit (const Epetra_Map &input_map,
                  const bool        fast)
  {
    if (map.NumGlobalElements() != input_map.NumGlobalElements())
      {
	vector.reset();
	map = Epetra_LocalMap (input_map.NumGlobalElements(),
			       input_map.IndexBase(),
			       input_map.Comm());
	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector (map));
      }
    else if (fast == false)
      {
	int ierr;
	ierr = vector->GlobalAssemble(last_action);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	ierr = vector->PutScalar(0.0);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));
      }

    last_action = Zero;
  }

  void
  Vector::reinit (const VectorBase &v,
		  const bool        fast,
		  const bool        allow_different_maps)
  {
					// In case we do not allow to
					// have different maps, this
					// call means that we have to
					// reset the vector. So clear
					// the vector, initialize our
					// map with the map in v, and
					// generate the vector.
    if (allow_different_maps == false)
      {
	if (local_range() != v.local_range())
	  {
	    vector.reset();
	    map = Epetra_LocalMap (v.vector->GlobalLength(),
				   v.vector->Map().IndexBase(),
				   v.vector->Comm());

	    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
	  }
	else
	  {
	    int ierr;
	    Assert (vector->Map().SameAs(v.vector->Map()) == true,
		    ExcMessage ("The Epetra maps in the assignment operator ="
				" do not match, even though the local_range "
				" seems to be the same. Check vector setup!"));

	    ierr = vector->GlobalAssemble(last_action);
	    AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	    ierr = vector->PutScalar(0.0);
	    AssertThrow (ierr == 0, ExcTrilinosError(ierr));
	  }
	last_action = Zero;
      }

					// Otherwise, we have to check
					// that the two vectors are
					// already of the same size,
					// create an object for the data
					// exchange and then insert all
					// the data.
    else
      {
	Assert (fast == false,
		ExcMessage ("It is not possible to exchange data with the "
			    "option fast set, which would not write "
			    "elements."));

	AssertThrow (size() == v.size(),
		     ExcDimensionMismatch (size(), v.size()));

	Epetra_Import data_exchange (vector->Map(), v.vector->Map());

	const int ierr = vector->Import(*v.vector, data_exchange, Insert);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	last_action = Insert;
      }

  }

  Vector &
  Vector::operator = (const MPI::Vector &v)
  {
    if (size() != v.size())
      {
	vector.reset();
	map = Epetra_LocalMap (v.vector->Map().NumGlobalElements(),
			       v.vector->Map().IndexBase(),
			       v.vector->Comm());
	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
      }

    reinit (v, false, true);
    return *this;
  }

  Vector &
  Vector::operator = (const Vector &v)
  {
    if (size() != v.size())
      {
	map = Epetra_LocalMap (v.vector->Map().NumGlobalElements(),
			       v.vector->Map().IndexBase(),
			       v.vector->Comm());
	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
      }

    const int ierr = vector->Update(1.0, *v.vector, 0.0);
    AssertThrow (ierr == 0, ExcTrilinosError(ierr));

    return *this;
  }

}

DEAL_II_NAMESPACE_CLOSE

#endif // DEAL_II_USE_TRILINOS
<commit_msg>When doing reinit() on a distributed vector, it is not enough to ensure that the local range is the same, but we'd need to make sure that *all* local ranges are the same. We can do so by comparing the Trilinos maps.<commit_after>//---------------------------------------------------------------------------
//    $Id$
//    Version: $Name$
//
//    Copyright (C) 2008, 2009 by the deal.II authors
//
//    This file is subject to QPL and may not be  distributed
//    without copyright and license information. Please refer
//    to the file deal.II/doc/license.html for the  text  and
//    further information on this license.
//
//---------------------------------------------------------------------------

#include <base/utilities.h>
#include <lac/trilinos_vector.h>

#include <cmath>

#ifdef DEAL_II_USE_TRILINOS

#include <Epetra_Import.h>

DEAL_II_NAMESPACE_OPEN

namespace TrilinosWrappers
{
  namespace MPI
  {

    Vector::Vector ()
		    :
		    communicator (Utilities::Trilinos::
				  duplicate_communicator(Utilities::Trilinos::comm_self())),
		    map (0, 0, *communicator)
    {
      last_action = Zero;
      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
    }

    Vector::Vector (const Epetra_Map &InputMap)
                    :
		    communicator (Utilities::Trilinos::
				  duplicate_communicator (InputMap.Comm())),
		    map (Utilities::Trilinos::
			 duplicate_map (InputMap, *communicator))
    {
      last_action = Zero;
      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
    }

    Vector::Vector (const Vector &v)
                    :
                    VectorBase(),
		    communicator (Utilities::Trilinos::
				  duplicate_communicator (*v.communicator)),
		    map (Utilities::Trilinos::duplicate_map (v.map,
							     *communicator))
    {
      last_action = Zero;
      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(*v.vector));
    }

    Vector::Vector (const Epetra_Map &InputMap,
		    const VectorBase &v)
                    :
                    VectorBase(),
		    communicator (Utilities::Trilinos::
				  duplicate_communicator (InputMap.Comm())),
		    map (Utilities::Trilinos::
			 duplicate_map (InputMap, *communicator))
    {
      AssertThrow (map.NumGlobalElements() == v.vector->Map().NumGlobalElements(),
		   ExcDimensionMismatch (map.NumGlobalElements(),
					 v.vector->Map().NumGlobalElements()));

      last_action = Zero;

      if (map.SameAs(v.vector->Map()) == true)
	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(*v.vector));
      else
	{
	  vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
	  reinit (v, false, true);
	}
    }

    void
    Vector::reinit (const Epetra_Map &input_map,
		    const bool        fast)
    {
      vector.reset();

      communicator.reset (Utilities::Trilinos::
			  duplicate_communicator(input_map.Comm()));
      map = Utilities::Trilinos::duplicate_map (input_map, *communicator);

      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));

      if (fast == false)
	{
	  const int ierr = vector->PutScalar(0.);
	  AssertThrow (ierr == 0, ExcTrilinosError(ierr));
	}

      last_action = Zero;
    }

    void
    Vector::reinit (const VectorBase &v,
		    const bool        fast,
		    const bool        allow_different_maps)
    {
					// In case we do not allow to
					// have different maps, this
					// call means that we have to
					// reset the vector. So clear
					// the vector, initialize our
					// map with the map in v, and
					// generate the vector.
      if (allow_different_maps == false)
        {
	  if (vector->Map().SameAs(v.vector->Map()) == false)
	    {
	      vector.reset();
	      communicator.reset (Utilities::Trilinos::
				  duplicate_communicator(v.trilinos_vector()
							 .Map().Comm()));
	      map = Utilities::Trilinos::
		    duplicate_map (v.trilinos_vector().Map(), *communicator);

	      vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
	      last_action = Zero;
	    }
	  else if (fast == false)
	    {
					       // old and new vectors
					       // have exactly the
					       // same map, i.e. size
					       // and parallel
					       // distribution
	      int ierr;
	      ierr = vector->GlobalAssemble (last_action);
	      AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	      ierr = vector->PutScalar(0.0);
	      AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	      last_action = Zero;
	    }
	}

					// Otherwise, we have to check
					// that the two vectors are
					// already of the same size,
					// create an object for the data
					// exchange and then insert all
					// the data. The first assertion
					// is only a check whether the
					// user knows what she is doing.
      else
        {
	  Assert (fast == false,
		  ExcMessage ("It is not possible to exchange data with the "
			      "option fast set, which would not write "
			      "elements."));

	  AssertThrow (size() == v.size(),
		       ExcDimensionMismatch (size(), v.size()));

	  Epetra_Import data_exchange (vector->Map(), v.vector->Map());

	  const int ierr = vector->Import(*v.vector, data_exchange, Insert);
	  AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	  last_action = Insert;
	}

    }

    Vector &
    Vector::operator = (const Vector &v)
    {
      if (local_range() == v.local_range())
	{
	  Assert (vector->Map().SameAs(v.vector->Map()) == true,
		  ExcMessage ("The Epetra maps in the assignment operator ="
			      " do not match, even though the local_range "
			      " seems to be the same. Check vector setup!"));

	  const int ierr = vector->Update(1.0, *v.vector, 0.0);
	  AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	  last_action = Zero;
	}
      else
	{
	  vector.reset();

	  communicator.reset (Utilities::Trilinos::
			      duplicate_communicator(v.trilinos_vector().
						     Map().Comm()));
	  map = Utilities::Trilinos::
		duplicate_map (v.trilinos_vector().Map(),
			       *communicator);

	  vector = std::auto_ptr<Epetra_FEVector>
	                    (new Epetra_FEVector(*v.vector));
	  last_action = Zero;
	}

      return *this;
    }

    Vector &
    Vector::operator = (const TrilinosWrappers::Vector &v)
    {
      Assert (size() == v.size(), ExcDimensionMismatch(size(), v.size()));

      Epetra_Import data_exchange (vector->Map(), v.vector->Map());
      const int ierr = vector->Import(*v.vector, data_exchange, Insert);

      AssertThrow (ierr == 0, ExcTrilinosError(ierr));

      last_action = Insert;

      return *this;
    }

    void
    Vector::import_nonlocal_data_for_fe (const TrilinosWrappers::SparseMatrix &m,
					 const Vector                         &v)
    {
      Assert (m.trilinos_matrix().Filled() == true,
	      ExcMessage ("Matrix is not compressed. "
			  "Cannot find exchange information!"));
      Assert (v.vector->Map().UniqueGIDs() == true,
	      ExcMessage ("The input vector has overlapping data, "
			  "which is not allowed."));

      if (vector->Map().SameAs(m.col_partitioner()) == false)
	{
	  map = m.col_partitioner();
	  vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
	}

      Epetra_Import data_exchange (vector->Map(), v.vector->Map());
      const int ierr = vector->Import(*v.vector, data_exchange, Insert);

      AssertThrow (ierr == 0, ExcTrilinosError(ierr));

      last_action = Insert;
    }

  } /* end of namespace MPI */

  Vector::Vector ()
		  :
		  map (0, 0,
		       Utilities::Trilinos::comm_self())
  {
    last_action = Zero;
    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
  }

  Vector::Vector (const unsigned int n)
                 :
		  map ((int)n, 0,
		       Utilities::Trilinos::comm_self())
  {
    last_action = Zero;
    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector (map));
  }

  Vector::Vector (const Epetra_Map &InputMap)
                 :
                 map (InputMap.NumGlobalElements(), InputMap.IndexBase(),
		      InputMap.Comm())
  {
    last_action = Zero;
    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
  }

  Vector::Vector (const VectorBase &v)
                 :
                 map (v.vector->Map().NumGlobalElements(), 0, v.vector->Comm())
  {
    last_action = Zero;
    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));

    if (vector->Map().SameAs(v.vector->Map()) == true)
      {
	const int ierr = vector->Update(1.0, *v.vector, 0.0);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));
      }
    else
      reinit (v, false, true);

  }

  void
  Vector::reinit (const unsigned int n,
		  const bool         fast)
  {
    if (size() != n)
      {
	vector.reset();

	map = Epetra_LocalMap ((int)n, 0,
			       Utilities::Trilinos::comm_self());

	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector (map));
      }
    else if (fast == false)
      {
	int ierr;
	ierr = vector->GlobalAssemble(last_action);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	ierr = vector->PutScalar(0.0);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));
      }

    last_action = Zero;
  }

  void
  Vector::reinit (const Epetra_Map &input_map,
                  const bool        fast)
  {
    if (map.NumGlobalElements() != input_map.NumGlobalElements())
      {
	vector.reset();
	map = Epetra_LocalMap (input_map.NumGlobalElements(),
			       input_map.IndexBase(),
			       input_map.Comm());
	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector (map));
      }
    else if (fast == false)
      {
	int ierr;
	ierr = vector->GlobalAssemble(last_action);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	ierr = vector->PutScalar(0.0);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));
      }

    last_action = Zero;
  }

  void
  Vector::reinit (const VectorBase &v,
		  const bool        fast,
		  const bool        allow_different_maps)
  {
					// In case we do not allow to
					// have different maps, this
					// call means that we have to
					// reset the vector. So clear
					// the vector, initialize our
					// map with the map in v, and
					// generate the vector.
    if (allow_different_maps == false)
      {
	if (local_range() != v.local_range())
	  {
	    vector.reset();
	    map = Epetra_LocalMap (v.vector->GlobalLength(),
				   v.vector->Map().IndexBase(),
				   v.vector->Comm());

	    vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
	  }
	else
	  {
	    int ierr;
	    Assert (vector->Map().SameAs(v.vector->Map()) == true,
		    ExcMessage ("The Epetra maps in the assignment operator ="
				" do not match, even though the local_range "
				" seems to be the same. Check vector setup!"));

	    ierr = vector->GlobalAssemble(last_action);
	    AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	    ierr = vector->PutScalar(0.0);
	    AssertThrow (ierr == 0, ExcTrilinosError(ierr));
	  }
	last_action = Zero;
      }

					// Otherwise, we have to check
					// that the two vectors are
					// already of the same size,
					// create an object for the data
					// exchange and then insert all
					// the data.
    else
      {
	Assert (fast == false,
		ExcMessage ("It is not possible to exchange data with the "
			    "option fast set, which would not write "
			    "elements."));

	AssertThrow (size() == v.size(),
		     ExcDimensionMismatch (size(), v.size()));

	Epetra_Import data_exchange (vector->Map(), v.vector->Map());

	const int ierr = vector->Import(*v.vector, data_exchange, Insert);
	AssertThrow (ierr == 0, ExcTrilinosError(ierr));

	last_action = Insert;
      }

  }

  Vector &
  Vector::operator = (const MPI::Vector &v)
  {
    if (size() != v.size())
      {
	vector.reset();
	map = Epetra_LocalMap (v.vector->Map().NumGlobalElements(),
			       v.vector->Map().IndexBase(),
			       v.vector->Comm());
	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
      }

    reinit (v, false, true);
    return *this;
  }

  Vector &
  Vector::operator = (const Vector &v)
  {
    if (size() != v.size())
      {
	map = Epetra_LocalMap (v.vector->Map().NumGlobalElements(),
			       v.vector->Map().IndexBase(),
			       v.vector->Comm());
	vector = std::auto_ptr<Epetra_FEVector> (new Epetra_FEVector(map));
      }

    const int ierr = vector->Update(1.0, *v.vector, 0.0);
    AssertThrow (ierr == 0, ExcTrilinosError(ierr));

    return *this;
  }

}

DEAL_II_NAMESPACE_CLOSE

#endif // DEAL_II_USE_TRILINOS
<|endoftext|>


<commit_before><commit_msg>convert sd/source/ui/inc/MasterPageObserver.hxx from String to OUString<commit_after><|endoftext|>


<commit_before><commit_msg>trivial compile fixes<commit_after><|endoftext|>