Function defines pipeline state for search. Takes data/scan as input.
    fileroot is base name for associated products (cal files, noise, cands). if blank, it is set to filename.
    paramfile is name of file that defines all pipeline parameters (python-like syntax).
    kwargs used to overload paramfile definitions.
    Many parameters take 0 as default, which auto-defines ideal parameters. 
    This definition does not yet consider memory/cpu/time limitations.
    nsegments defines how to break jobs in time. nchunk defines how many jobs are sent to nthreads.

def set_pipeline(filename, scan, fileroot='', paramfile='', **kwargs):
    """ Function defines pipeline state for search. Takes data/scan as input.
    fileroot is base name for associated products (cal files, noise, cands). if blank, it is set to filename.
    paramfile is name of file that defines all pipeline parameters (python-like syntax).
    kwargs used to overload paramfile definitions.
    Many parameters take 0 as default, which auto-defines ideal parameters. 
    This definition does not yet consider memory/cpu/time limitations.
    nsegments defines how to break jobs in time. nchunk defines how many jobs are sent to nthreads.
    """

    
    workdir = os.path.dirname(os.path.abspath(filename))
    filename = filename.rstrip('/')
    assert os.path.exists(filename)

    # then get all metadata
    if os.path.exists(os.path.join(filename, 'Main.xml')):
        d = ps.get_metadata(filename, scan, paramfile=paramfile, **kwargs)   # can take file name or Params instance
        d['dataformat'] = 'sdm'
    else:
        d = pm.get_metadata(filename, scan, paramfile=paramfile, **kwargs)
        d['dataformat'] = 'ms'

    # set version
    d['rtpipe_version'] = __version__

    # define rootname for in/out cal/products
    if fileroot:
        d['fileroot'] = fileroot
    else:
        d['fileroot'] = os.path.basename(os.path.abspath(filename))

    # autodetect calibration products locally
    if not d['gainfile'] or not os.path.exists(d['gainfile']):
        # first try to get CASA gain file
        gainfilelist = glob.glob(os.path.join(d['workdir'], d['fileroot'] + '.g?'))
        bpfilelist = glob.glob(os.path.join(d['workdir'], d['fileroot'] + '.b?'))

        # if not in workdir, look locally
        if not gainfilelist or not bpfilelist:
            gainfilelist = glob.glob(d['fileroot'] + '.g?')
            bpfilelist = glob.glob(d['fileroot'] + '.b?')
            
        if gainfilelist and bpfilelist:
            gainfilelist.sort()
            d['gainfile'] = gainfilelist[-1]
            logger.info('Autodetected CASA gainfile %s' % d['gainfile'])
            bpfilelist.sort()
            d['bpfile'] = bpfilelist[-1]
            logger.info('Autodetected CASA bpfile %s' % d['bpfile'])

        # if that fails, look for telcal file
        filelist = glob.glob(os.path.join(d['workdir'], filename + '.GN'))
        if not filelist:
            filelist = glob.glob(filename + '.GN')

        if filelist:
            d['gainfile'] = filelist[0]
            logger.info('Autodetected telcal file %s' % d['gainfile'])

        if not os.path.exists(d['gainfile']):
            logger.warn('Calibration file autodetection failed for gainfile {0}'.format(d['gainfile']))

    # define features
    d['featureind'] = ['segment', 'int', 'dmind', 'dtind', 'beamnum']  # feature index. should be stable.
    if 'features' not in d:
        if d['searchtype'] == 'image1':
            d['features'] = ['snr1', 'immax1', 'l1', 'm1']   # features returned by image1
        elif d['searchtype'] == 'image1snip':
            d['features'] = ['snr1', 'immax1', 'l1', 'm1', 'im40', 'spec20']
        elif d['searchtype'] == 'image1stats':
            d['features'] = ['snr1', 'immax1', 'l1', 'm1', 'specstd', 'specskew', 'speckurtosis', 'imskew', 'imkurtosis']  # note: spec statistics are all or nothing.
        elif 'image2' in d['searchtype']:
            d['features'] = ['snr1', 'immax1', 'l1', 'm1', 'snr2', 'immax2', 'l2', 'm2']   # features returned by image1

    # set imaging parameters to use
    if d['uvres'] == 0:
        d['uvres'] = d['uvres_full']
    else:
        urange = d['urange'][scan]*(d['freq'].max()/d['freq_orig'][0])   # uvw from get_uvw already in lambda at ch0
        vrange = d['vrange'][scan]*(d['freq'].max()/d['freq_orig'][0])
        powers = n.fromfunction(lambda i,j: 2**i*3**j, (14,10), dtype='int')   # power array for 2**i * 3**j
        rangex = n.round(d['uvoversample']*urange).astype('int')
        rangey = n.round(d['uvoversample']*vrange).astype('int')
        largerx = n.where(powers-rangex/d['uvres'] > 0, powers, powers[-1,-1])
        p2x, p3x = n.where(largerx == largerx.min())
        largery = n.where(powers-rangey/d['uvres'] > 0, powers, powers[-1,-1])
        p2y, p3y = n.where(largery == largery.min())
        d['npixx_full'] = (2**p2x * 3**p3x)[0]
        d['npixy_full'] = (2**p2y * 3**p3y)[0]

    # set number of pixels to image
    d['npixx'] = d['npixx_full']
    d['npixy'] = d['npixy_full']
    if 'npix_max' in d:
        if d['npix_max']:
            d['npixx'] = min(d['npix_max'], d['npixx_full'])
            d['npixy'] = min(d['npix_max'], d['npixy_full'])
    if d['npix']:
        d['npixx'] = d['npix']
        d['npixy'] = d['npix']
    else:
        d['npix'] = max(d['npixx'], d['npixy'])   # this used to define fringe time

    # define dmarr, if not already
    if len(d['dmarr']) == 0:
        if d.has_key('dm_maxloss') and d.has_key('maxdm') and d.has_key('dm_pulsewidth'):
            d['dmarr'] = calc_dmgrid(d, maxloss=d['dm_maxloss'], maxdm=d['maxdm'], dt=d['dm_pulsewidth'])
            if d['maxdm'] > 0:
                logger.info('Calculated %d dms for max sensitivity loss %.2f, maxdm %d pc/cm3, and pulse width %d ms' % (len(d['dmarr']), d['dm_maxloss'], d['maxdm'], d['dm_pulsewidth']/1000))
        else:
            d['dmarr'] = [0]
            logger.info('Can\'t calculate dm grid without dm_maxloss, maxdm, and dm_pulsewidth defined. Setting to [0].')

    # define times for data to read
    d['t_overlap'] = rtlib.calc_delay(d['freq'], d['inttime'], max(d['dmarr'])).max()*d['inttime']   # time of overlap for total dm coverage at segment boundaries
    d['datadelay'] = [rtlib.calc_delay(d['freq'], d['inttime'],dm).max() for dm in d['dmarr']]
    d['nints'] = d['nints'] - d['nskip']

    # pols
    if d.has_key('selectpol'):
        d['pols'] = [pol for pol in d['pols_orig'] if pol in d['selectpol']]
    else:
        d['pols'] = d['pols_orig']
    d['npol'] = len(d['pols'])

    # split imaging into chunks. ideally one per thread, but can modify to fit available memory
    if d['nchunk'] == 0:
        d['nchunk'] = d['nthread']

    # if nsegments is 0, then auto-define within memory limit
    if not d['nsegments']:
        fringetime = calc_fringetime(d)
        d['nsegments'] = max(1, min(d['nints'], int(d['scale_nsegments']*d['inttime']*d['nints']/(fringetime-d['t_overlap']))))  # at least 1, at most nints
        calc_segment_times(d)

        # if auto nsegment definition makes segment too large, try to scale it down to fit in memory_limit (if provided)
        # limit defined for dm sweep time and max nchunk/nthread ratio
        if d.has_key('memory_limit'):
            (vismem0, immem0) = calc_memory_footprint(d, limit=True)
            assert vismem0+immem0 < d['memory_limit'], 'memory_limit of {0} is smaller than best solution of {1}. Try forcing nsegments/nchunk larger than {2}/{3} or reducing maxdm/npix'.format(d['memory_limit'], vismem0+immem0, d['nsegments'], max(d['dtarr'])/min(d['dtarr']))

            (vismem, immem) = calc_memory_footprint(d)
            if vismem+immem > d['memory_limit']:
                logger.info('Over memory limit of {4} when reading {0} segments with {1} chunks ({2}/{3} GB for visibilities/imaging). Searching for solution down to {5}/{6} GB...'.format(d['nsegments'], d['nchunk'], vismem, immem, d['memory_limit'], vismem0, immem0))

            while vismem+immem > d['memory_limit']:
                (vismem, immem) = calc_memory_footprint(d)
                logger.debug('Using {0} segments with {1} chunks ({2}/{3} GB for visibilities/imaging). Searching for better solution...'.format(d['nchunk'], vismem, immem, d['memory_limit']))
                d['scale_nsegments'] = d['scale_nsegments'] * (vismem+immem)/float(d['memory_limit'])
                d['nsegments'] = max(1, min(d['nints'], int(d['scale_nsegments']*d['inttime']*d['nints']/(fringetime-d['t_overlap']))))  # at least 1, at most nints
                calc_segment_times(d)

                (vismem, immem) = calc_memory_footprint(d)
                while vismem+immem > d['memory_limit']:
                    logger.debug('Doubling nchunk from %d to fit in %d GB memory limit.' % (d['nchunk'], d['memory_limit']))
                    d['nchunk'] = 2*d['nchunk']
                    (vismem, immem) = calc_memory_footprint(d)
                    if d['nchunk'] >= max(d['dtarr'])/min(d['dtarr'])*d['nthread']: # limit nchunk/nthread to at most the range in dt
                        d['nchunk'] = d['nthread']
                        break

                (vismem, immem) = calc_memory_footprint(d)

    # final set up of memory
    calc_segment_times(d)
    (vismem, immem) = calc_memory_footprint(d)

    # scaling of number of integrations beyond dt=1
    assert all(d['dtarr']) and (d['dtarr'] == sorted(d['dtarr'])), 'dtarr must be larger than 0 and in increasing order'

    # calculate number of thermal noise candidates per segment
    nfalse = calc_nfalse(d)

    logger.info('')
    logger.info('Pipeline summary:')
    if '.GN' in d['gainfile']:
        logger.info('\t Products saved with %s. telcal calibration with %s' % (d['fileroot'], os.path.basename(d['gainfile'])))
    else:
        logger.info('\t Products saved with %s. CASA calibration files (%s, %s)' % (d['fileroot'], os.path.basename(d['gainfile']), os.path.basename(d['bpfile'])))
    logger.info('\t Using %d segment%s of %d ints (%.1f s) with overlap of %.1f s' % (d['nsegments'], "s"[not d['nsegments']-1:], d['readints'], d['t_segment'], d['t_overlap']))
    if d['t_overlap'] > d['t_segment']/3.:
        logger.info('\t\t Lots of segments needed, since Max DM sweep (%.1f s) close to segment size (%.2f s)' % (d['t_overlap'], d['t_segment']))
    logger.info('\t Downsampling in time/freq by %d/%d and skipping %d ints from start of scan.' % (d['read_tdownsample'], d['read_fdownsample'], d['nskip']))
    logger.info('\t Excluding ants %s' % (d['excludeants']))
    logger.info('\t Using pols %s' % (d['pols']))
    logger.info('')

    logger.info('\t Search with %s and threshold %.1f.' % (d['searchtype'], d['sigma_image1']))
    logger.info('\t Using %d DMs from %.1f to %.1f and dts %s.' % (len(d['dmarr']), min(d['dmarr']), max(d['dmarr']), d['dtarr']))
    logger.info('\t Using uvgrid npix=(%d,%d) and res=%d.' % (d['npixx'], d['npixy'], d['uvres']))
    logger.info('\t Expect %d thermal false positives per segment.' % nfalse)

    logger.info('')
    logger.info('\t Visibility memory usage is %.1f GB/segment' % vismem)
    logger.info('\t Imaging in %d chunk%s using max of %.1f GB/segment' % (d['nchunk'], "s"[not d['nsegments']-1:], immem))
    logger.info('\t Grand total memory usage: %.1f GB/segment' % (vismem + immem))

    return d