caffe-distri/src/main/cpp/util/rdma.cpp
/**
 * Sends a buffer's memory region so that it can be mapped to it's remote end.
 */
Sends a buffer's memory region so that it can be mapped to it's remote end.
Sends a buffer ' s memory region so that it can be mapped to it ' s remote end .
void RDMAChannel::SendMR(ibv_mr* mr, int id) {
  ibv_mr* init = ibv_reg_mr(adapter_.pd_, mr, sizeof(ibv_mr),
                            IBV_ACCESS_LOCAL_WRITE);
  struct ibv_sge list;
  list.addr = (uint64_t) mr;
  list.length = sizeof(ibv_mr);
  list.lkey = init->lkey;
  struct ibv_send_wr wr;
  caffe_memset(sizeof(wr), 0, &wr);
  wr.wr_id = (uint64_t) init;
  wr.sg_list = &list;
  wr.num_sge = 1;
  wr.opcode = IBV_WR_SEND_WITH_IMM;
  wr.send_flags = IBV_SEND_SIGNALED;
  wr.imm_data = id;
  struct ibv_send_wr *bad_wr;
  CHECK(!ibv_post_send(qp_, &wr, &bad_wr));
  for (;;) {
    ibv_wc wc;
    int ne = ibv_poll_cq(write_cq_, 1, &wc);
    CHECK_GE(ne, 0);
    if (ne && wc.wr_id == (uint64_t) init) {
      break;
    }
  }
  CHECK(!ibv_dereg_mr(init));
}
void RDMAChannel :: SendMR ( ibv_mr * mr , int id ) { ibv_mr * init = ibv_reg_mr ( adapter_ . pd_ , mr , sizeof ( ibv_mr ) , IBV_ACCESS_LOCAL_WRITE ) ; struct ibv_sge list ; list . addr = ( uint64_t ) mr ; list . length = sizeof ( ibv_mr ) ; list . lkey = init -> lkey ; struct ibv_send_wr wr ; caffe_memset ( sizeof ( wr ) , 0 , & wr ) ; wr . wr_id = ( uint64_t ) init ; wr . sg_list = & list ; wr . num_sge = 1 ; wr . opcode = IBV_WR_SEND_WITH_IMM ; wr . send_flags = IBV_SEND_SIGNALED ; wr . imm_data = id ; struct ibv_send_wr * bad_wr ; CHECK ( ! ibv_post_send ( qp_ , & wr , & bad_wr ) ) ; for ( ; ; ) { ibv_wc wc ; int ne = ibv_poll_cq ( write_cq_ , 1 , & wc ) ; CHECK_GE ( ne , 0 ) ; if ( ne && wc . wr_id == ( uint64_t ) init ) { break ; } } CHECK ( ! ibv_dereg_mr ( init ) ) ; }
Sends a buffer's memory region so that it can be mapped to it's remote end.
Sends a buffer ' s memory region so that it can be mapped to it ' s remote end .
// Map the memory region itself so that it can be sent