Transient search pipeline running on single node.
    Processes one or more segments of data (in which a single bgsub, (u,v,w), etc. can be used).
    Can search completely, independently, and saves candidates.
    If segments is a list of segments, then it will parallelize read/search processes.

    Stages:
    0) Take dictionary that defines metadata and search params
    -- This defines state of pipeline, including times, uv extent, pipeline search parameters, etc.
    1) Read data
    -- Overlapping reads needed to maintain sensitivity to all DMs at all times
    2) Prepare data
    -- Reads/applies telcal/CASA solutions, flags, bg time subtraction
    3) Search using all threads
    -- Option for plug-and-play detection algorithm and multiple filters
    4) Save candidate and noise info, if requested

def pipeline(d, segments):
    """ Transient search pipeline running on single node.
    Processes one or more segments of data (in which a single bgsub, (u,v,w), etc. can be used).
    Can search completely, independently, and saves candidates.
    If segments is a list of segments, then it will parallelize read/search processes.

    Stages:
    0) Take dictionary that defines metadata and search params
    -- This defines state of pipeline, including times, uv extent, pipeline search parameters, etc.
    1) Read data
    -- Overlapping reads needed to maintain sensitivity to all DMs at all times
    2) Prepare data
    -- Reads/applies telcal/CASA solutions, flags, bg time subtraction
    3) Search using all threads
    -- Option for plug-and-play detection algorithm and multiple filters
    4) Save candidate and noise info, if requested
    """

    if type(segments) == int:
        segments = [segments]

    logger.info('Starting search of %s, scan %d, segments %s' % (d['filename'], d['scan'], str(segments)))
    assert os.path.exists(d['gainfile']), 'Calibration file autodetection failed for gainfile {0}'.format(d['gainfile'])

    # seed the pseudo-random number generator # TJWL
    random.seed()    

    # set up shared arrays to fill
    data_read_mem = mps.Array(mps.ctypes.c_float, datasize(d)*2);  data_mem = mps.Array(mps.ctypes.c_float, datasize(d)*2)
    u_read_mem = mps.Array(mps.ctypes.c_float, d['nbl']);  u_mem = mps.Array(mps.ctypes.c_float, d['nbl'])
    v_read_mem = mps.Array(mps.ctypes.c_float, d['nbl']);  v_mem = mps.Array(mps.ctypes.c_float, d['nbl'])
    w_read_mem = mps.Array(mps.ctypes.c_float, d['nbl']);  w_mem = mps.Array(mps.ctypes.c_float, d['nbl'])

    # need these if debugging
    data = numpyview(data_mem, 'complex64', datashape(d)) # optional
    data_read = numpyview(data_read_mem, 'complex64', datashape(d)) # optional
    u = numpyview(u_mem, 'float32', d['nbl'], raw=False)
    v = numpyview(v_mem, 'float32', d['nbl'], raw=False)
    w = numpyview(w_mem, 'float32', d['nbl'], raw=False)
                
    # plan fft
    logger.debug('Planning FFT...')
    arr = pyfftw.empty_aligned((d['npixx'], d['npixy']), dtype='complex64', n=16)
    arr[:] = n.random.randn(*arr.shape) + 1j*n.random.randn(*arr.shape)
    fft_arr = pyfftw.interfaces.numpy_fft.ifft2(arr)

    results = {}
    # only one needed for parallel read/process. more would overwrite memory space
    with closing(mp.Pool(1, initializer=initread, initargs=(data_read_mem, u_read_mem, v_read_mem, w_read_mem, data_mem, u_mem, v_mem, w_mem))) as readpool:  

        try:
            # submit all segments to pool of 1. locking data should keep this from running away.
            for segment in segments:
                assert segment in range(d['nsegments']), 'Segment %d not in range of %d nsegments' % (segment, d['nsegments'])
                candsfile = getcandsfile(d, segment)
                if d['savecands'] and os.path.exists(candsfile):
                    logger.error('candsfile %s already exists. Ending processing...' % candsfile)
                else:
                    results[segment] = readpool.apply_async(pipeline_dataprep, (d, segment))   # no need for segment here? need to think through structure...

            # step through pool of jobs and pull data off as ready. this allows pool to continue to next segment.
            while results.keys():
                for segment in results.keys():
                    if results[segment].ready():
                        job = results.pop(segment)
                        d = job.get()
                    else:
                        continue
                    
                    with data_mem.get_lock():
                        cands = search(d, data_mem, u_mem, v_mem, w_mem)

                    # save candidate info
                    if d['savecands']:
                        logger.info('Saving %d candidates for segment %d...'
                                    % (len(cands), segment))
                        savecands(d, cands)

        except KeyboardInterrupt:
            logger.error('Caught Ctrl-C. Closing processing pool.')
            readpool.terminate()
            readpool.join()
            raise