def read_bdf_segment(d, segment=-1):
    """ Reads bdf (sdm) format data into numpy array for realtime pipeline.

    d defines pipeline state. assumes segmenttimes defined by RT.set_pipeline.
    d should have 'writebdfpkl' key to define
    boolean for writing to bdfpkls in ASDMBinary directory.
    """

    # define integration range
    if segment != -1:
        assert 'segmenttimes' in d
        assert segment < d['nsegments'], \
            'segment %d is too big for nsegments %d' % (segment,
                                                        d['nsegments'])
        readints = d['readints']
        nskip = (24*3600*(d['segmenttimes'][segment, 0]
                          - d['starttime_mjd'])
                 / d['inttime']).astype(int)
        logger.info('Reading scan %d, segment %d/%d, times %s to %s'
                    % (d['scan'], segment,
                       len(d['segmenttimes'])-1,
                       qa.time(qa.quantity(d['segmenttimes'][segment, 0], 'd'),
                               form=['hms'], prec=9)[0],
                       qa.time(qa.quantity(d['segmenttimes'][segment, 1], 'd'),
                               form=['hms'], prec=9)[0]))
    else:
        nskip = 0
        readints = 0

    # read (all) data
    if 'bdfdir' not in d:
        d['bdfdir'] = None

    data = read_bdf(d['filename'], d['scan'], nskip=nskip, readints=readints, bdfdir=d['bdfdir']).astype('complex64')

    # read Flag.xml and apply flags for given ant/time range
    # currently only implemented for segmented data
    if d['applyonlineflags'] and segment > -1:
        sdm = getsdm(d['filename'], bdfdir=d['bdfdir'])

        allantdict = dict(zip([str(ant.antennaId) for ant in sdm['Antenna']],
                              [int(str(ant.name).lstrip('ea'))
                               for ant in sdm['Antenna']]))
        antflags = [(allantdict[str(flag.antennaId).split(' ')[2]],
                     int(flag.startTime)/(1e9*24*3600),
                     int(flag.endTime)/(1e9*24*3600))
                    for flag in sdm['Flag']]  # assumes one flag per entry
        logger.info('Found online flags for %d antenna/time ranges.'
                    % (len(antflags)))
        blarr = calc_blarr(d)  # d may define different ants than in allantdict
        timearr = np.linspace(d['segmenttimes'][segment][0],
                              d['segmenttimes'][segment][1], d['readints'])
        badints_cum = []
        for antflag in antflags:
            antnum, time0, time1 = antflag
            badbls = np.where((blarr == antnum).any(axis=1))[0]
            badints = np.where((timearr >= time0) & (timearr <= time1))[0]
            for badint in badints:
                data[badint, badbls] = 0j
            badints_cum = badints_cum + list(badints)
        logger.info('Applied online flags to %d ints.'
                    % (len(set(badints_cum))))
    elif segment == -1:
        logger.warn('Online flags not yet supported without segments.')
    else:
        logger.info('Not applying online flags.')

    # test that spw are in freq sorted order
    # only one use case supported: rolled spw
    dfreq = np.array([d['spw_reffreq'][i+1] - d['spw_reffreq'][i]
                      for i in range(len(d['spw_reffreq'])-1)])
    dfreqneg = [df for df in dfreq if df < 0]
    # if spw are permuted, then roll them.
    # !! not a perfect test of permutability!!
    if len(dfreqneg) <= 1:
        if len(dfreqneg) == 1:
            logger.info('Rolling spw frequencies to increasing order: %s'
                        % str(d['spw_reffreq']))
            rollch = np.sum([d['spw_nchan'][ss]
                             for ss in range(np.where(dfreq < 0)[0][0]+1)])
            data = np.roll(data, rollch, axis=2)
    else:
        raise StandardError('SPW out of order and can\'t be permuted '
                            'to increasing order: %s'
                            % str(d['spw_reffreq']))

    # optionally integrate (downsample)
    if ((d['read_tdownsample'] > 1) or (d['read_fdownsample'] > 1)):
        sh = data.shape
        tsize = sh[0]/d['read_tdownsample']
        fsize = sh[2]/d['read_fdownsample']
        data2 = np.zeros((tsize, sh[1], fsize, sh[3]), dtype='complex64')
        if d['read_tdownsample'] > 1:
            logger.info('Downsampling in time by %d' % d['read_tdownsample'])
            for i in range(tsize):
                data2[i] = data[
                    i*d['read_tdownsample']:(i+1)*d['read_tdownsample']
                    ].mean(axis=0)
        if d['read_fdownsample'] > 1:
            logger.info('Downsampling in frequency by %d'
                        % d['read_fdownsample'])
            for i in range(fsize):
                data2[:, :, i, :] = data[
                    :, :, i * d['read_fdownsample']:(i+1)*d['read_fdownsample']
                    ].mean(axis=2)
        data = data2

    takepol = [d['pols_orig'].index(pol) for pol in d['pols']]
    logger.debug('Selecting pols %s' % d['pols'])

    return data.take(d['chans'], axis=2).take(takepol, axis=3)
def read_bdf_segment ( d , segment = - 1 ) : if segment != - 1 : assert 'segmenttimes' in d assert segment < d [ 'nsegments' ] , 'segment %d is too big for nsegments %d' % ( segment , d [ 'nsegments' ] ) readints = d [ 'readints' ] nskip = ( 24 * 3600 * ( d [ 'segmenttimes' ] [ segment , 0 ] - d [ 'starttime_mjd' ] ) / d [ 'inttime' ] ) . astype ( int ) logger . info ( 'Reading scan %d, segment %d/%d, times %s to %s' % ( d [ 'scan' ] , segment , len ( d [ 'segmenttimes' ] ) - 1 , qa . time ( qa . quantity ( d [ 'segmenttimes' ] [ segment , 0 ] , 'd' ) , form = [ 'hms' ] , prec = 9 ) [ 0 ] , qa . time ( qa . quantity ( d [ 'segmenttimes' ] [ segment , 1 ] , 'd' ) , form = [ 'hms' ] , prec = 9 ) [ 0 ] ) ) else : nskip = 0 readints = 0 if 'bdfdir' not in d : d [ 'bdfdir' ] = None data = read_bdf ( d [ 'filename' ] , d [ 'scan' ] , nskip = nskip , readints = readints , bdfdir = d [ 'bdfdir' ] ) . astype ( 'complex64' ) if d [ 'applyonlineflags' ] and segment > - 1 : sdm = getsdm ( d [ 'filename' ] , bdfdir = d [ 'bdfdir' ] ) allantdict = dict ( zip ( [ str ( ant . antennaId ) for ant in sdm [ 'Antenna' ] ] , [ int ( str ( ant . name ) . lstrip ( 'ea' ) ) for ant in sdm [ 'Antenna' ] ] ) ) antflags = [ ( allantdict [ str ( flag . antennaId ) . split ( ' ' ) [ 2 ] ] , int ( flag . startTime ) / ( 1e9 * 24 * 3600 ) , int ( flag . endTime ) / ( 1e9 * 24 * 3600 ) ) for flag in sdm [ 'Flag' ] ] logger . info ( 'Found online flags for %d antenna/time ranges.' % ( len ( antflags ) ) ) blarr = calc_blarr ( d ) timearr = np . linspace ( d [ 'segmenttimes' ] [ segment ] [ 0 ] , d [ 'segmenttimes' ] [ segment ] [ 1 ] , d [ 'readints' ] ) badints_cum = [ ] for antflag in antflags : antnum , time0 , time1 = antflag badbls = np . where ( ( blarr == antnum ) . any ( axis = 1 ) ) [ 0 ] badints = np . where ( ( timearr >= time0 ) & ( timearr <= time1 ) ) [ 0 ] for badint in badints : data [ badint , badbls ] = 0j badints_cum = badints_cum + list ( badints ) logger . info ( 'Applied online flags to %d ints.' % ( len ( set ( badints_cum ) ) ) ) elif segment == - 1 : logger . warn ( 'Online flags not yet supported without segments.' ) else : logger . info ( 'Not applying online flags.' ) dfreq = np . array ( [ d [ 'spw_reffreq' ] [ i + 1 ] - d [ 'spw_reffreq' ] [ i ] for i in range ( len ( d [ 'spw_reffreq' ] ) - 1 ) ] ) dfreqneg = [ df for df in dfreq if df < 0 ] if len ( dfreqneg ) <= 1 : if len ( dfreqneg ) == 1 : logger . info ( 'Rolling spw frequencies to increasing order: %s' % str ( d [ 'spw_reffreq' ] ) ) rollch = np . sum ( [ d [ 'spw_nchan' ] [ ss ] for ss in range ( np . where ( dfreq < 0 ) [ 0 ] [ 0 ] + 1 ) ] ) data = np . roll ( data , rollch , axis = 2 ) else : raise StandardError ( 'SPW out of order and can\'t be permuted ' 'to increasing order: %s' % str ( d [ 'spw_reffreq' ] ) ) if ( ( d [ 'read_tdownsample' ] > 1 ) or ( d [ 'read_fdownsample' ] > 1 ) ) : sh = data . shape tsize = sh [ 0 ] / d [ 'read_tdownsample' ] fsize = sh [ 2 ] / d [ 'read_fdownsample' ] data2 = np . zeros ( ( tsize , sh [ 1 ] , fsize , sh [ 3 ] ) , dtype = 'complex64' ) if d [ 'read_tdownsample' ] > 1 : logger . info ( 'Downsampling in time by %d' % d [ 'read_tdownsample' ] ) for i in range ( tsize ) : data2 [ i ] = data [ i * d [ 'read_tdownsample' ] : ( i + 1 ) * d [ 'read_tdownsample' ] ] . mean ( axis = 0 ) if d [ 'read_fdownsample' ] > 1 : logger . info ( 'Downsampling in frequency by %d' % d [ 'read_fdownsample' ] ) for i in range ( fsize ) : data2 [ : , : , i , : ] = data [ : , : , i * d [ 'read_fdownsample' ] : ( i + 1 ) * d [ 'read_fdownsample' ] ] . mean ( axis = 2 ) data = data2 takepol = [ d [ 'pols_orig' ] . index ( pol ) for pol in d [ 'pols' ] ] logger . debug ( 'Selecting pols %s' % d [ 'pols' ] ) return data . take ( d [ 'chans' ] , axis = 2 ) . take ( takepol , axis = 3 )
Reads bdf (sdm) format data into numpy array for realtime pipeline.

    d defines pipeline state. assumes segmenttimes defined by RT.set_pipeline.
    d should have 'writebdfpkl' key to define
    boolean for writing to bdfpkls in ASDMBinary directory.