
}

local WorkerField = {}
WorkerField.__index = WorkerField

--[[
params {
  bounds,
  ghost_width,
  type_size
}
--]]
local function NewFieldInstanceTable(params)
  local widths      = params.bounds:getwidths()
  local n_elems     = 1
  local ghost_width = {}
  local dim_size    = {}
  local strides     = {}
  for d = 1,#widths do
    strides[d]      = n_elems   -- column major?
    ghost_width[d]  = params.ghost_width[d] or 0
    dim_size[d]     = widths[d] + 1 -- adjust for hi/lo inclusive convention
    n_elems         = n_elems * (dim_size[d] + 2*ghost_width[d])
  end
  if #widths == 2 then
    strides[3]      = 0
    ghost_width[3]  = 0
    dim_size[3]     = 0
  end

  local elem_size   = params.type_size
  local elem_stride = pow2align(elem_size,4)
  assert(elem_stride == elem_size)
  local array = DynamicArray.New {
    size      = n_elems,
    type      = uint8[elem_size],
    processor = Pre.CPU,
  }
  return setmetatable ({
    _array        = array,
    _strides      = strides,
    _elem_size    = elem_size,
    _ghost_width  = ghost_width,
    _n_elems      = n_elems,
  }, FieldInstanceTable)
end

function FieldInstanceTable:DataPtr()
  return self._array:_raw_ptr()
end

function FieldInstanceTable:DataPtrGhostAdjusted()
  local ptr = terralib.cast(&uint8, self:DataPtr())
  local offset = 0
  for d = 1,#self._strides do
    offset = offset + self._ghost_width[d] * self._strides[d]
  end
  return ptr + offset * self._elem_size
end

function FieldInstanceTable:NElems() return self._n_elems end
function FieldInstanceTable:Strides() return self._strides end
function FieldInstanceTable:ElemSize() return self._elem_size end



-------------------------------------------------------------------------------
-- GHOSTS
-------------------------------------------------------------------------------


local struct GhostInstance  {
  union {
    src : &gaswrap.AsyncBufSrcChannel;
    dst : &gaswrap.AsyncBufDstChannel;
  }
  bounds    : BoundsStruct[3];
  ptr       : &uint8;
  strides   : uint64[3];  -- in element counts, matching field instances
  elem_size : uint64;
}
local terra printGI( g : &GhostInstance, prefix : rawstring )
  var prefix_skip = ""
  if prefix ~= nil then prefix_skip = "\n    "
                   else prefix = "" end
  C.printf(["[%d] %s%s"..
                 "chan, buf, elem_size:   %p %p %d\n"..
             "    bds ; strides:          %d,%d %d,%d %d,%d ; %d,%d,%d\n"
           ], THIS_NODE, prefix, prefix_skip, g.src, g.ptr, g.elem_size,
           g.bounds[0].lo, g.bounds[0].hi,
           g.bounds[1].lo, g.bounds[1].hi,
           g.bounds[2].lo, g.bounds[2].hi,
           g.strides[0], g.strides[1], g.strides[2]
           )
end


local controller_field_ghost_ready_semaphores = {}

local function InitFieldGhostSemaphore(f_id)
  assert(on_control_node(), 'only for controller')
  controller_field_ghost_ready_semaphores[f_id] = numComputeNodes()
end

local function MarkFieldGhostsReady(f_id, hid_base)
  f_id = tonumber(f_id)
  local semaphore_val = controller_field_ghost_ready_semaphores[f_id]
  controller_field_ghost_ready_semaphores[f_id] = semaphore_val - 1
  --print(THIS_NODE, 'DECREMENT f#'..f_id..': '..(semaphore_val-1))
end
gaswrap.registerLuaEvent('MarkFieldGhostsReady', MarkFieldGhostsReady)

local function WaitOnFieldGhostReady(f_id)
  assert(on_control_node(), 'only for controller')
  local semaphore_val = controller_field_ghost_ready_semaphores[f_id]
  assert(semaphore_val, 'tried to wait on uninitialized field '..f_id)
  while semaphore_val > 0 do
    C.usleep(POLL_USLEEP)
    gaswrap.pollLuaEvents(0, 0)
    semaphore_val = controller_field_ghost_ready_semaphores[f_id]
  end
  --print(THIS_NODE, 'DONE waiting on f#'..f_id)
end


-- Allocate incoming and outgoing buffers for data exchange, set up channels
-- with neighboring nodes, and send a confirmation to control node when all
-- channels are set up.
local ghost_listing_2d = newlist()
local ghost_listing_3d = newlist()
if USE_CONSERVATIVE_GHOSTS then
  for x = -1,1 do
    for y = -1,1 do
      if x ~= 0 or y ~= 0 then
        ghost_listing_2d:insert({x,y})
      end
      for z = -1,1 do
        if x ~= 0 or y ~= 0 or z ~= 0 then
          ghost_listing_3d:insert({x,y,z})
        end
      end
    end
  end
  assert(#ghost_listing_2d == 8)
  assert(#ghost_listing_3d == 26)
else
  for d = 1,2 do
    local neg, pos = {0,0}, {0,0}
    neg[d] = -1
    pos[d] =  1
    ghost_listing_2d:insertall({neg, pos})
  end
  for d = 1,3 do
    local neg, pos = {0,0,0}, {0,0,0}
    neg[d] = -1
    pos[d] =  1
    ghost_listing_3d:insertall({neg, pos})
  end
  assert(#ghost_listing_2d == 4)
  assert(#ghost_listing_3d == 6)
end

local worker_ghost_channel_semaphores = {} -- keyed by field id
local function ProcessAllGhostChannels(field, handshake_unq_id, ghost_width)
  local relation      = field.relation
  local ndims         = #relation.dims
  local off_patterns  = ndims == 2 and ghost_listing_2d or ghost_listing_3d
  worker_ghost_channel_semaphores[field.id] = 2 * #off_patterns
  local function dec_semaphore()
    local semaphore_val = worker_ghost_channel_semaphores[field.id] - 1
    worker_ghost_channel_semaphores[field.id] = semaphore_val
    if semaphore_val == 0 then
      gaswrap.sendLuaEvent(CONTROL_NODE, 'MarkFieldGhostsReady', field.id)
    end
  end

  local function gen_handshake_id(src_node, off, flip_off)
    local h_id  = handshake_unq_id
    h_id        = src_node + N_NODES * h_id
    local o     = { off[1], off[2], off[3] or 0 }
    if flip_off then for d=1,3 do o[d] = -o[d] end end
    h_id        = (3*3)*(o[1]+1) + 3*(o[2]+1) + (o[3]+1) + (3*3*3)*h_id