src_stdpar/Communication.cpp
// Establish and allocate all receving/sending buffers
// for halo communication between MPI neighbours.
// To this end, it takes as an input 'rlist', the list
// of receiving halos. One ouput is 'slist', the halos
// to be sent.
/*
  @biref Get a list of the neighbours of the local MPI subdomain. Input is halo_fn
         that stores PIDs of the cells halo_fn(ivy,ivx,iy,ix). Ouput is hlist that
         includes a list of halo boxes. The union of all the hlist's boxes represents
         the actual halo of the local MPI subdomain.
         called in the initialization phase after neighboursList
                                     
  @param[in]  halo
    Identical to fn
  @param[in]  xrange[8] 
    face[8] = node.xmin[0], node.xmax[0] + 1, node.xmin[1], node.xmax[1] + 1, node.xmin[2], node.xmax[2] + 1, node.xmin[3], node.xmax[3] + 1}
    where node = ulist_[pid]
  @param[in]  lxmin[4]
    mynode.xmin, where mynode = ulist_[pid]
  @param[in]  lxmax[4]
    mynode.xmax, where mynode = ulist_[pid]
  @param[in]  count
    the total loop count of the 4D loop over halo regions ranging (-1:2) in each direction
  @param[inout] hlist
    Indentical to recv_list
 */
Establish and allocate all receving/sending buffers
for halo communication between MPI neighbours.
To this end, it takes as an input 'rlist', the list
of receiving halos. One ouput is 'slist', the halos
to be sent.

@biref Get a list of the neighbours of the local MPI subdomain. Input is halo_fn
that stores PIDs of the cells halo_fn(ivy,ivx,iy,ix). Ouput is hlist that
includes a list of halo boxes. The union of all the hlist's boxes represents
the actual halo of the local MPI subdomain.
called in the initialization phase after neighboursList


Establish and allocate all receving / sending buffers for halo communication between MPI neighbours . To this end it takes as an input ' rlist ' the list of receiving halos . One ouput is ' slist ' the halos to be sent . @biref Get a list of the neighbours of the local MPI subdomain . Input is halo_fn that stores PIDs of the cells halo_fn ( ivy ivx iy ix ) . Ouput is hlist that includes a list of halo boxes . The union of all the hlist ' s boxes represents the actual halo of the local MPI subdomain . called in the initialization phase after neighboursList
void Distrib::bookHalo(Config *conf) {
  int size_halo = sizeof(Halo);
  std::vector<unsigned short> vectp(nbp_, 0);
  std::vector<unsigned short> redp(nbp_, 0);
  std::vector<MPI_Request> req;
  std::vector<MPI_Status>  stat;
  int tag = 111;
  int nbreq = 0;
  for(auto it = recv_list_.begin(); it != recv_list_.end(); ++it) {
    vectp[(*it).pid_]++;
  }
  MPI_Allreduce(vectp.data(), redp.data(), nbp_, MPI_UNSIGNED_SHORT, MPI_SUM, MPI_COMM_WORLD);
  nbreq = redp[pid_];
  send_list_.resize(nbreq);
  req.resize(nbreq + recv_list_.size());
  stat.resize(nbreq + recv_list_.size());
  for(size_t i = 0; i < send_list_.size(); i++) {
    Halo *sender = &(send_list_[i]);
    MPI_Irecv(sender, size_halo, MPI_BYTE, MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &(req[i]));
  }
  for(size_t i = 0; i < recv_list_.size(); i++) {
    Halo *recver = &(recv_list_[i]);
    int dest = recver->pid_;
    MPI_Isend(recver, size_halo, MPI_BYTE, dest, tag, MPI_COMM_WORLD, &(req[nbreq]));
    nbreq++;
  }
  MPI_Waitall(nbreq, req.data(), stat.data());
  for(size_t i = 0; i < send_list_.size(); i++) {
    Halo *halo = &(send_list_[i]);
    int32 size = (halo->xmax_[0] - halo->xmin_[0] + 1) * (halo->xmax_[1] - halo->xmin_[1] + 1) * (halo->xmax_[2] - halo->xmin_[2] + 1)
               * (halo->xmax_[3] - halo->xmin_[3] + 1);
    halo->size_ = size;
  }
  for(size_t i = 0; i < recv_list_.size(); i++) {
    Halo *halo = &(recv_list_[i]);
    int32 size = (halo->xmax_[0] - halo->xmin_[0] + 1) * (halo->xmax_[1] - halo->xmin_[1] + 1) * (halo->xmax_[2] - halo->xmin_[2] + 1)
               * (halo->xmax_[3] - halo->xmin_[3] + 1);
    halo->size_ = size;
  }
  for(size_t i = 0; i < send_list_.size(); i++) {
    Halo *sender = &(send_list_[i]);
    sender->pid_ = stat[i].MPI_SOURCE;
  }
  send_buffers_ = new Halos();
  recv_buffers_ = new Halos();
  send_buffers_->set(conf, send_list_, "send", nbp_, pid_);
  recv_buffers_->set(conf, recv_list_, "recv", nbp_, pid_);
  fprintf(stderr, "[%d] Number of halo blocs = %lu\n", pid_, send_list_.size());
}
void Distrib :: bookHalo ( Config * conf ) { int size_halo = sizeof ( Halo ) ; std :: vector < unsigned short > vectp ( nbp_ , 0 ) ; std :: vector < unsigned short > redp ( nbp_ , 0 ) ; std :: vector < MPI_Request > req ; std :: vector < MPI_Status > stat ; int tag = 111 ; int nbreq = 0 ; for ( auto it = recv_list_ . begin ( ) ; it != recv_list_ . end ( ) ; ++ it ) { vectp [ ( * it ) . pid_ ] ++ ; } MPI_Allreduce ( vectp . data ( ) , redp . data ( ) , nbp_ , MPI_UNSIGNED_SHORT , MPI_SUM , MPI_COMM_WORLD ) ; nbreq = redp [ pid_ ] ; send_list_ . resize ( nbreq ) ; req . resize ( nbreq + recv_list_ . size ( ) ) ; stat . resize ( nbreq + recv_list_ . size ( ) ) ; for ( size_t i = 0 ; i < send_list_ . size ( ) ; i ++ ) { Halo * sender = & ( send_list_ [ i ] ) ; MPI_Irecv ( sender , size_halo , MPI_BYTE , MPI_ANY_SOURCE , tag , MPI_COMM_WORLD , & ( req [ i ] ) ) ; } for ( size_t i = 0 ; i < recv_list_ . size ( ) ; i ++ ) { Halo * recver = & ( recv_list_ [ i ] ) ; int dest = recver -> pid_ ; MPI_Isend ( recver , size_halo , MPI_BYTE , dest , tag , MPI_COMM_WORLD , & ( req [ nbreq ] ) ) ; nbreq ++ ; } MPI_Waitall ( nbreq , req . data ( ) , stat . data ( ) ) ; for ( size_t i = 0 ; i < send_list_ . size ( ) ; i ++ ) { Halo * halo = & ( send_list_ [ i ] ) ; int32 size = ( halo -> xmax_ [ 0 ] - halo -> xmin_ [ 0 ] + 1 ) * ( halo -> xmax_ [ 1 ] - halo -> xmin_ [ 1 ] + 1 ) * ( halo -> xmax_ [ 2 ] - halo -> xmin_ [ 2 ] + 1 ) * ( halo -> xmax_ [ 3 ] - halo -> xmin_ [ 3 ] + 1 ) ; halo -> size_ = size ; } for ( size_t i = 0 ; i < recv_list_ . size ( ) ; i ++ ) { Halo * halo = & ( recv_list_ [ i ] ) ; int32 size = ( halo -> xmax_ [ 0 ] - halo -> xmin_ [ 0 ] + 1 ) * ( halo -> xmax_ [ 1 ] - halo -> xmin_ [ 1 ] + 1 ) * ( halo -> xmax_ [ 2 ] - halo -> xmin_ [ 2 ] + 1 ) * ( halo -> xmax_ [ 3 ] - halo -> xmin_ [ 3 ] + 1 ) ; halo -> size_ = size ; } for ( size_t i = 0 ; i < send_list_ . size ( ) ; i ++ ) { Halo * sender = & ( send_list_ [ i ] ) ; sender -> pid_ = stat [ i ] . MPI_SOURCE ; } send_buffers_ = new Halos ( ) ; recv_buffers_ = new Halos ( ) ; send_buffers_ -> set ( conf , send_list_ , " " , nbp_ , pid_ ) ; recv_buffers_ -> set ( conf , recv_list_ , " " , nbp_ , pid_ ) ; fprintf ( stderr , " \n " , pid_ , send_list_ . size ( ) ) ; }
Establish and allocate all receving/sending buffers
for halo communication between MPI neighbours.
Establish and allocate all receving / sending buffers for halo communication between MPI neighbours .
// Compute the number of local halos that should be received // depending of the PID that should send it. // Reduce this 'vectp' array in order that each and ervey process // knows about the number of send/receive halos. // Initialize the vector 'slist' of halos to be sent // Receiving halo settings to be received to corresponding MPI processes // Send halo settings to be sent to corresponding MPI processes // One should wait the end of the exchange before allocating the halo // All local halos dimensions are now well known. // Let's allocate them. // Set the PID of each halo's sender in 'slist''s components // Copy into Kokkos suited structure // Halo size large enough to store 