def _parse (self):
        """Parse the BDF mime structure and record the locations of the binary
        blobs. Sets up various data fields in the BDFData object."""

        feedparser = FeedParser (Message)
        binarychunks = {}
        sizeinfo = None
        headxml = None
        self.fp.seek (0, 0)

        while True:
            data = self.fp.readline ()
            if not data:
                break

            feedparser.feed (data)

            skip = (data == '\n' and
                    len (feedparser._msgstack) == 3 and
                    feedparser._msgstack[-1].get_content_type () in ('application/octet-stream',
                                                                     'binary/octet-stream'))
            if skip:
                # We just finished reading the headers for a huge binary blob.
                # Time to remember where the data chunk is and pretend it doesn't
                # exist.
                msg = feedparser._msgstack[-1]
                ident = msg['Content-Location']
                assert ident.endswith ('.bin'), 'confusion #1 in hacky MIME parsing!'
                binarychunks[ident] = self.fp.tell ()
                if sizeinfo is None:
                    headxml, sizeinfo, tagpfx = _extract_size_info (feedparser)
                kind = ident.split ('/')[-1]
                assert kind in sizeinfo, 'no size info for binary chunk kind %s in MIME!' % kind
                self.fp.seek (sizeinfo[kind] + 1, 1) # skip ahead by data chunk size
                sample = self.fp.read (16)
                assert sample.startswith ('--MIME'), 'crap, unexpected chunk size in MIME parsing: %r' % sample
                self.fp.seek (-16, 1) # go back

            # check that two major kinds of data are read at least once
            if any([k.split('/')[3] == '3' for k in binarychunks.iterkeys()]):
                break

        if headxml is None:
            raise RuntimeError ('never found any binary data')

        self.mimemsg = feedparser.close ()
        self.headxml = headxml
        self.sizeinfo = sizeinfo
        self.binarychunks = binarychunks

        headsize, intsize = self.calc_intsize()

        # Compute some miscellaneous parameters that we'll need.
#        self.n_integrations = len (self.mimemsg.get_payload ()) - 1
        self.n_integrations = os.stat(self.fp.name).st_size/intsize
        self.n_antennas = int (headxml.find (tagpfx + nanttag).text)
        self.n_baselines = (self.n_antennas * (self.n_antennas - 1)) // 2

        ds = headxml.find (tagpfx + dstag)
        nbb = 0
        nspw = 0
        nchan = 0
        crosspolstr = None

        for bb in ds.findall (tagpfx + basebandtag):
            nbb += 1

            for spw in bb.getchildren ():
                nspw += 1
                nchan += int (spw.get ('numSpectralPoint'))

                if crosspolstr is None:
                    crosspolstr = spw.get ('crossPolProducts')
                elif spw.get ('crossPolProducts') != crosspolstr:
                    raise Exception ('can only handle spectral windows with identical cross pol products')

        self.n_basebands = nbb
        self.n_spws = nspw
        self.n_channels = nchan
        self.crosspols = crosspolstr.split ()
        self.n_pols = len(self.crosspols)

        # if bdf info pkl not present, write it
        if os.path.exists(os.path.dirname(self.pklname)) and self.pklname and (not os.path.exists(self.pklname)):
            logger.info('Writing bdf pkl info to %s...' % (self.pklname))
            with open(self.pklname,'wb') as pkl:
                # Compute some miscellaneous parameters that we'll need.
                pickle.dump( (self.mimemsg, self.headxml, self.sizeinfo, self.binarychunks, self.n_integrations, self.n_antennas, self.n_baselines, self.n_basebands, self.n_spws, self.n_channels, self.crosspols), pkl)

        return self
def _parse ( self ) : feedparser = FeedParser ( Message ) binarychunks = { } sizeinfo = None headxml = None self . fp . seek ( 0 , 0 ) while True : data = self . fp . readline ( ) if not data : break feedparser . feed ( data ) skip = ( data == '\n' and len ( feedparser . _msgstack ) == 3 and feedparser . _msgstack [ - 1 ] . get_content_type ( ) in ( 'application/octet-stream' , 'binary/octet-stream' ) ) if skip : msg = feedparser . _msgstack [ - 1 ] ident = msg [ 'Content-Location' ] assert ident . endswith ( '.bin' ) , 'confusion #1 in hacky MIME parsing!' binarychunks [ ident ] = self . fp . tell ( ) if sizeinfo is None : headxml , sizeinfo , tagpfx = _extract_size_info ( feedparser ) kind = ident . split ( '/' ) [ - 1 ] assert kind in sizeinfo , 'no size info for binary chunk kind %s in MIME!' % kind self . fp . seek ( sizeinfo [ kind ] + 1 , 1 ) sample = self . fp . read ( 16 ) assert sample . startswith ( '--MIME' ) , 'crap, unexpected chunk size in MIME parsing: %r' % sample self . fp . seek ( - 16 , 1 ) if any ( [ k . split ( '/' ) [ 3 ] == '3' for k in binarychunks . iterkeys ( ) ] ) : break if headxml is None : raise RuntimeError ( 'never found any binary data' ) self . mimemsg = feedparser . close ( ) self . headxml = headxml self . sizeinfo = sizeinfo self . binarychunks = binarychunks headsize , intsize = self . calc_intsize ( ) self . n_integrations = os . stat ( self . fp . name ) . st_size / intsize self . n_antennas = int ( headxml . find ( tagpfx + nanttag ) . text ) self . n_baselines = ( self . n_antennas * ( self . n_antennas - 1 ) ) // 2 ds = headxml . find ( tagpfx + dstag ) nbb = 0 nspw = 0 nchan = 0 crosspolstr = None for bb in ds . findall ( tagpfx + basebandtag ) : nbb += 1 for spw in bb . getchildren ( ) : nspw += 1 nchan += int ( spw . get ( 'numSpectralPoint' ) ) if crosspolstr is None : crosspolstr = spw . get ( 'crossPolProducts' ) elif spw . get ( 'crossPolProducts' ) != crosspolstr : raise Exception ( 'can only handle spectral windows with identical cross pol products' ) self . n_basebands = nbb self . n_spws = nspw self . n_channels = nchan self . crosspols = crosspolstr . split ( ) self . n_pols = len ( self . crosspols ) if os . path . exists ( os . path . dirname ( self . pklname ) ) and self . pklname and ( not os . path . exists ( self . pklname ) ) : logger . info ( 'Writing bdf pkl info to %s...' % ( self . pklname ) ) with open ( self . pklname , 'wb' ) as pkl : pickle . dump ( ( self . mimemsg , self . headxml , self . sizeinfo , self . binarychunks , self . n_integrations , self . n_antennas , self . n_baselines , self . n_basebands , self . n_spws , self . n_channels , self . crosspols ) , pkl ) return self
Parse the BDF mime structure and record the locations of the binary
        blobs. Sets up various data fields in the BDFData object.