<gh_stars>1-10
'''
Created on Aug 27, 2016

@author: <NAME>
datfile class

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that redistribution of source code include
the following disclaimer in the documentation and/or other materials provided
with the distribution.

THIS SOFTWARE IS PROVIDED BY ITS CREATOR "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE CREATOR OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''

import math
import traceback
from datcom.Payload import Payload
from datfile.FileEnd import FileEnd
from tickGroup import tickGroup
from AnalyzeDatResults import AnalyzeDatResults
from bitstring import Bits
from datfile.NotDatFile import NotDatFile
from datfile.Corrupted import Corrupted
from datfile.ResultCode import ResultCode

class DatFile(object):
    '''
    classdocs
    Class for reading byte data
    '''
 

    def __init__(self, fileName):
        '''
        Constructor
        fileName full path to data file to be read
        '''

        self.headerLength = 10
        self.memory = None
        self.filePos = 0
        self.file = None 
        self.inputStream = None 
        self._channel = None 
        self.fileLength = 0 
        self.buildStr = ""
        self.nextFilPos = 128 
        self.nextTickGroup = 0 
        self._fileEnd = FileEnd()
        self.numCorrupted = 0 
        self.results = None 
        self.startOfRecord = 0 
        self.lowestTickNo = -1L 
        self.highestTickNo = -1L
        self.motorStartTick = 0L
        self.motorStopTick = -1L 
        self.flightStartTick = -1L 
        self.gpsLockTick = -1L 
        self.tickGroups = [tickGroup(),tickGroup()]
        self.tgIndex = 1
        self.lastRecordTickNo = 0L
        self.alternateStructure = False 
        self.datfile(fileName)
        #     protected boolean alternateStructure = false;
    def datfile(self, _file):
        """ _file should be the full path to the data file """
        
        self.tickGroups[0] = tickGroup()
        self.tickGroups[1] = tickGroup()
        self.results = AnalyzeDatResults()
        self.file = open(_file,'rb')
        ## create a bits object with the content of the file
        dataAsBits = Bits(self.file)
        ## 8 bits per byte
        self.fileLength = (dataAsBits.length / 8)
        self.memory = dataAsBits.tobytes()
        try:
            if self.getByte(128) != 0x55:
                self.alternateStructure = True
        except FileEnd:
            self.close()
            raise NotDatFile()
        buildStr = self.getString(16)
        if buildStr.find("BUILD") < 0:
            self.close()
            raise NotDatFile()

    def reset(self):
        self.tickGroups[0].reset()
        self.tickGroups[1].reset()
        self.tgIndex = 1
        self.numCorrupted = 0
        self.results = AnalyzeDatResults()
        self.startOfRecord = 128
        self.setPosition(self.startOfRecord)

    def getTickGroup(self):
        thisTickGroup = None
        nextTickGroup = None
        if self.tgIndex == 1:
            self.tgIndex = 0
            thisTickGroup = self.tickGroups[0]
            nextTickGroup = self.tickGroups[1]
        else:
            self.tgIndex = 1
            thisTickGroup = self.tickGroups[1]
            nextTickGroup = self.tickGroups[0]
    
        thisGroupsTickNo = thisTickGroup.tickNo
        nextTickGroup.numElements = 0
        done = False
        length =0
        nextStartOfRecord = 0L
        while done is False:
            try:
                self.setPosition(self.startOfRecord)
                if self.getByte(self.startOfRecord) != 0X55:
                    raise Corrupted(thisGroupsTickNo, self.startOfRecord)
    
                length = (0xFF & self.getByte(self.startOfRecord + 1))
                # type in python is a reserved work so adding so type = mainType
                mainType = (0xFF & self.getByte(self.startOfRecord + 3))
                subType = (0xFF & self.getByte(self.startOfRecord + 4))
                msg = self.getByte(self.startOfRecord + 5)
                thisRecordTickNo = self.getUnsignedInt(self.startOfRecord + 6)
                if thisRecordTickNo < 0 or (self.alternateStructure is True \
                                            and thisRecordTickNo > 4500000)\
                                            or (self.alternateStructure is False\
                                            and thisRecordTickNo > 1500000):

                    raise Corrupted(self.lastRecordTickNo, self.startOfRecord + 1)
    
                self.lastRecordTickNo = thisRecordTickNo

                if length == 0:
                    raise Corrupted(thisGroupsTickNo, self.startOfRecord + 1)
    
                nextStartOfRecord = self.startOfRecord + length # the next 0x55 , we hope
                
                if nextStartOfRecord > self.fileLength:
                    raise self._fileEnd
    
                if self.getByte(nextStartOfRecord) != 0x55:
                    raise Corrupted(thisGroupsTickNo, nextStartOfRecord)
    
                if (0xff & msg) == 0x80:
                    mainType = 255 
                    subType = 1 

                if (0xff & msg) == 0xFF:
                    mainType = 255 
                    subType = 2 
    
                if thisGroupsTickNo == -1: # thisTickGroup doesn't yet have a tickNo
                    thisGroupsTickNo = thisRecordTickNo
                    thisTickGroup.tickNo = thisRecordTickNo
    
                if thisRecordTickNo > thisTickGroup.tickNo: # start next group 
                    nextTickGroup.reset()
                    nextTickGroup.add(self.startOfRecord + self.headerLength, 
                                      length - self.headerLength, mainType, subType, msg,thisRecordTickNo); 
                    done = True;
                    
                elif thisRecordTickNo == thisTickGroup.tickNo:
                    thisTickGroup.add(self.startOfRecord + self.headerLength, 
                                      length - self.headerLength, mainType, subType, msg)
                else:#tickNo < thisTickGroup.tickNo) in the last group
                    pass
                    ## for now, just ignore
                self.startOfRecord = nextStartOfRecord;

            except Corrupted as c:
                if self.getPos() > self.fileLength - 600: 
                    raise self._fileEnd

                self.numCorrupted += 1
                if self.numCorrupted > 25:
                    self.results.setResultCode(ResultCode.CORRUPTED);
                    raise Corrupted(thisGroupsTickNo, self.startOfRecord)
    
                try:
                    self.setPosition(c.filePos)
                    fiftyfive = self.readByte()
                    while fiftyfive != 0x55:
                        if self.getPos( ) > self.fileLength - 1000 :
                            raise self._fileEnd
    
                        fiftyfive = self.readByte()
    
                except Corrupted:
                    raise Corrupted(thisGroupsTickNo, self.startOfRecord)
    
                ##set position right before the next 0x55
                self.startOfRecord = self.getPos() -1
    
            except FileEnd:
                raise self._fileEnd
            except Corrupted:
                self.results.setResultCode(ResultCode.CORRUPTED)
                raise Corrupted(thisGroupsTickNo, self.startOfRecord)
    
        return thisTickGroup;

    def close(self): 
        self.file.close()
        self.memory = None

    def skipOver(self, num):
        newpos = self.filePos + num
        if newpos > self.fileLength:
            raise IOError
    
        self.filePos = newpos
    
    def toString(self):
        return self.file.name

    def bufferToString(self):
        rtstr = "%s:%0X : %s :Shrt %s :UShrt %s :I %s :UI %s :L %s :F %s :D %s " % (self.filePos, 
                    (0xff & self.getByte()), (0xff & self.getByte()), str(self.getShort()), 
                    str(self.getUnsignedShort()), str(self.getInt()), str(self.getUnsignedInt()),
                    str(self.getLong()), str(self.getFloat()),str(self.getDouble()))
        return rtstr

    def setPosition(self, pos):
        self.filePos = pos

        if self.filePos >= self.fileLength:
            raise FileEnd()

    def getPos(self):
        return self.filePos

    def getLength(self): 
        return self.fileLength

    def getByte(self, pos = None):
        if pos is None:
            pos = self.filePos
            self.filePos += 1
        if pos >= self.fileLength :
                raise FileEnd()
        ## get the string representation of the byte
        ## create a bits object encodes as hex and convert to little endian int    
        #rtv = Bits(hex=self.memory[pos].encode('hex')).intle
        rtv = int("0x%s" % self.memory[pos].encode('hex'),0)
        return rtv

    def getString(self, fp):
        length = 256
        rbytes = ""
        l = 0
        B = 0x00
        for i in range(length):
            try:
                B = self.getByte(fp + i)

            except FileEnd as e:
                print e
                traceback.print_stack()

            if B == 0x00 or B == "\r" or B == "\n":
                l = i
                break

            rbytes += chr(B)

        return rbytes[0:l]

    def readByte(self):
        rv = self.getByte()
        return rv;

    def  getShort(self):
        rtv = int("0x%s" % self.memory[self.filePos:self.filePos +1].encode('hex'),0)
        return rtv

    def getUnsignedShort(self, fp = None):
        if fp is None:
            fp = self.filePos
        
        if fp > self.fileLength - 2:
                raise FileEnd()
        ## get 2 bites create bites object encoded in hex
        ## return as unsigned int little endian
        #return Bits(hex=self.memory[fp:fp + 1].encode('hex')).uintle
        rtv = int("0x%s" % self.memory[fp:fp + 1].encode('hex'),0)
        return rtv

    def getInt(self):
        #return Bits(hex=self.memory[self.filePos:self.filePos +3].encode('hex')).intle
        rtv = int("0x%s" % self.memory[self.filePos:self.filePos +3].encode('hex'),0)
        return rtv

    def getUnsignedInt(self, fp = None):
        if fp is None:
            fp = self.filePos
        
        if fp > self.fileLength - 4:
                raise FileEnd()
        ## get 4 bites create bites object encoded in hex
        ## return as unsigned int little endian
        return Bits(hex=self.memory[fp:fp + 3].encode('hex')).uintle

    def getLong(self): 
        rtv = int("0x%s" % self.memory[self.filePos:self.filePos +7].encode('hex'),0)
        return rtv

    def getFloat(self):
        return float(Bits(hex=self.memory[self.filePos:self.filePos +3].encode('hex')).floatle)

    def getDouble(self):
        return float(Bits(hex=self.memory[self.filePos:self.filePos +7].encode('hex')).floatle)

    def getResults(self): 
        return self.results;

    def getFile(self):
        return self.file;

    def setStartOfRecord(self, sor):
        self.startOfRecord = sor;

    def fileName(self):
        return self.file.name

    def  findMarkers(self):
        self.findMotorStartEnd()
        self.findLowestTickNo()
        length = 0
        done = False
        nextStartOfRecord = 0
        localFilePos = long(self.fileLength - 200000)
        try:
            self.setPosition(localFilePos)
            fiftyfive = self.readByte()
            while fiftyfive != 0x55:
                if self.getPos() > self.fileLength - 1000:
                    raise self._fileEnd

                fiftyfive = self.readByte()

            ## bad form to use the same name for a local var that is already global
            ## change filePos to localFilePos
            localFilePos = self.getPos() -1
            while done is False:
                try:
                    self.setPosition(localFilePos)
                    length = (0xFF & self.getByte(localFilePos + 1));
                    if length == 0: 
                        raise Corrupted()

                    nextStartOfRecord = localFilePos + length
                    if nextStartOfRecord > self.fileLength:
                        raise self._fileEnd

                    if self.getByte(nextStartOfRecord) != 0x55:
                        raise Corrupted()

                    tickNo = long(self.getUnsignedInt(localFilePos + 6))
                    if self.lowestTickNo < 0:
                        self.lowestTickNo = tickNo

                    self.highestTickNo = tickNo
                    localFilePos = nextStartOfRecord

                except Corrupted:
                    self.setPosition(self.getPos() + 1)
                    fiftyfive = self.readByte()

                    while fiftyfive != 0x55:
                        if self.getPos() > self.fileLength - 1000:
                            raise self._fileEnd

                        fiftyfive = self.readByte()

                    localFilePos = self.getPos() - 1

        except Exception :
            pass

    def  findLowestTickNo(self):
        length = 0
        done = False
        nextStartOfRecord = 0
        localFilePos = 128;

        try:
            self.setPosition(localFilePos)
            fiftyfive = self.readByte()

            while fiftyfive != 0x55:
                if self.getPos() > self.fileLength - 1000 :
                    raise self._fileEnd
                fiftyfive = self.readByte()

            localFilePos = self.getFilePos() - 1

            while done is False:
                try:
                    self.setPosition(localFilePos)
                    length = (0xFF & self.getByte(localFilePos + 1))
                    if length == 0:
                        raise Corrupted()

                    nextStartOfRecord = localFilePos + length

                    if nextStartOfRecord > self.fileLength :
                        raise self._fileEnd

                    if self.getByte(nextStartOfRecord) != 0x55:
                        raise Corrupted()

                    tickNo = long(self.getUnsignedInt(localFilePos + 6))
                    self.lowestTickNo = tickNo
                    done = True

                except Corrupted:
                    self.setPosition(self.getPos() + 1)
                    fiftyfive = self.readByte()

                    while fiftyfive != 0x55:
                        if self.getPos() > self.fileLength - 1000 :
                            raise self._fileEnd

                        fiftyfive = self.readByte()

                    localFilePos = self.getPos() - 1

        except Exception:
            pass

    def findMotorStartEnd(self):
        try:
            self.reset()
            tickNo = 0L

            while True:
                if self.getPos() > self.fileLength - 8 :
                    raise self._fileEnd

                tG = self.getTickGroup()
                tickNo = tG.tickNo
                
                for tgIndex in range(tG.numElements):
                    payloadType =  tG.payloadType[tgIndex]
                    payloadStart = tG.start[tgIndex]
                    payloadLength = tG.length[tgIndex]
                    subType = tG.subType[tgIndex]

                    if payloadType == 255 and subType == 1:
                        xorBB =  Payload(self, payloadStart,payloadLength, payloadType, subType, tickNo)
                        payloadString = xorBB.getString()

                        if payloadString.find("M.Start") > 0 and self.motorStartTick == 0:
                            self.motorStartTick = tickNo

                        if payloadString.find("M. Stop") > 0:
                            self.motorStopTick = tickNo

                    elif self.gpsLockTick == -1  and payloadType == 42 and subType == 12:
                        payload = Payload(self, payloadStart,payloadLength, payloadType, subType, tickNo)
                        longitude = math.degrees(payload.getDouble(0))
                        latitude = math.degrees(payload.getDouble(8))

                        if longitude != 0.0 and latitude != 0.0\
                             and math.fabs(longitude) > 0.0175 and math.fabs(latitude) > 0.0175:
                            self.gpsLockTick = tickNo

                    elif self.flightStartTick == -1L and payloadType == 42 and subType == 12:
                        payload = Payload(self, payloadStart,payloadLength, payloadType, subType, tickNo)

                        if payload.getShort(42) > 0:
                            self.flightStartTick = tickNo - long(float(payload.getShort(42) * 100))
                            
        except FileEnd:
            pass

        except Corrupted: 
            pass

        except IOError:
            pass


if __name__ == "__main__":
    """ this code is intended for testing this module"""
    import time
    import cProfile
    st = time.time()
    # number of files from the list to test
    numtotest = 1
    # the list pf expected files
    testfiles = ['FLY000.DAT','FLY002.DAT','FLY013.DAT','FLY016.DAT']
    testfiles = ['FLY013.DAT','FLY016.DAT']
    # the full path to the data file directory
    datafile_dir = "C:/Users/isaac/workspace/DavidKovar/datcompy/src/data/"
    print "------- testing %d files --------" % numtotest
    for fname in testfiles[0:numtotest]:
        myd = DatFile('%s%s' %(datafile_dir,fname))
        print "processing file",myd.fileName()
        print "file length = ", myd.fileLength
        cProfile.run('myd.findMarkers()')
        print "lowest tick mark =", myd.lowestTickNo
        print "motor start tick =",myd.motorStartTick
        print "flight start tick =", myd.flightStartTick
        myd.close()
        print "-------- end file --------"
        
    print " test complete "
    print "runtime = ", time.time() - st
'''
Created on Aug 27, 2016

@author: <NAME>
datfile class

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that redistribution of source code include
the following disclaimer in the documentation and/or other materials provided
with the distribution.

THIS SOFTWARE IS PROVIDED BY ITS CREATOR "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE CREATOR OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''

import math
import traceback
from datcom.Payload import Payload
from datfile.FileEnd import FileEnd
from tickGroup import tickGroup
from AnalyzeDatResults import AnalyzeDatResults
from bitstring import Bits
from datfile.NotDatFile import NotDatFile
from datfile.Corrupted import Corrupted
from datfile.ResultCode import ResultCode

class DatFile(object):
    '''
    classdocs
    Class for reading byte data
    '''
 

    def __init__(self, fileName):
        '''
        Constructor
        fileName full path to data file to be read
        '''

        self.headerLength = 10
        self.memory = None
        self.filePos = 0
        self.file = None 
        self.inputStream = None 
        self._channel = None 
        self.fileLength = 0 
        self.buildStr = ""
        self.nextFilPos = 128 
        self.nextTickGroup = 0 
        self._fileEnd = FileEnd()
        self.numCorrupted = 0 
        self.results = None 
        self.startOfRecord = 0 
        self.lowestTickNo = -1L 
        self.highestTickNo = -1L
        self.motorStartTick = 0L
        self.motorStopTick = -1L 
        self.flightStartTick = -1L 
        self.gpsLockTick = -1L 
        self.tickGroups = [tickGroup(),tickGroup()]
        self.tgIndex = 1
        self.lastRecordTickNo = 0L
        self.alternateStructure = False 
        self.datfile(fileName)
        #     protected boolean alternateStructure = false;
    def datfile(self, _file):
        """ _file should be the full path to the data file """
        
        self.tickGroups[0] = tickGroup()
        self.tickGroups[1] = tickGroup()
        self.results = AnalyzeDatResults()
        self.file = open(_file,'rb')
        ## create a bits object with the content of the file
        dataAsBits = Bits(self.file)
        ## 8 bits per byte
        self.fileLength = (dataAsBits.length / 8)
        self.memory = dataAsBits.tobytes()
        try:
            if self.getByte(128) != 0x55:
                self.alternateStructure = True
        except FileEnd:
            self.close()
            raise NotDatFile()
        buildStr = self.getString(16)
        if buildStr.find("BUILD") < 0:
            self.close()
            raise NotDatFile()

    def reset(self):
        self.tickGroups[0].reset()
        self.tickGroups[1].reset()
        self.tgIndex = 1
        self.numCorrupted = 0
        self.results = AnalyzeDatResults()
        self.startOfRecord = 128
        self.setPosition(self.startOfRecord)

    def getTickGroup(self):
        thisTickGroup = None
        nextTickGroup = None
        if self.tgIndex == 1:
            self.tgIndex = 0
            thisTickGroup = self.tickGroups[0]
            nextTickGroup = self.tickGroups[1]
        else:
            self.tgIndex = 1
            thisTickGroup = self.tickGroups[1]
            nextTickGroup = self.tickGroups[0]
    
        thisGroupsTickNo = thisTickGroup.tickNo
        nextTickGroup.numElements = 0
        done = False
        length =0
        nextStartOfRecord = 0L
        while done is False:
            try:
                self.setPosition(self.startOfRecord)
                if self.getByte(self.startOfRecord) != 0X55:
                    raise Corrupted(thisGroupsTickNo, self.startOfRecord)
    
                length = (0xFF & self.getByte(self.startOfRecord + 1))
                # type in python is a reserved work so adding so type = mainType
                mainType = (0xFF & self.getByte(self.startOfRecord + 3))
                subType = (0xFF & self.getByte(self.startOfRecord + 4))
                msg = self.getByte(self.startOfRecord + 5)
                thisRecordTickNo = self.getUnsignedInt(self.startOfRecord + 6)
                if thisRecordTickNo < 0 or (self.alternateStructure is True \
                                            and thisRecordTickNo > 4500000)\
                                            or (self.alternateStructure is False\
                                            and thisRecordTickNo > 1500000):

                    raise Corrupted(self.lastRecordTickNo, self.startOfRecord + 1)
    
                self.lastRecordTickNo = thisRecordTickNo

                if length == 0:
                    raise Corrupted(thisGroupsTickNo, self.startOfRecord + 1)
    
                nextStartOfRecord = self.startOfRecord + length # the next 0x55 , we hope
                
                if nextStartOfRecord > self.fileLength:
                    raise self._fileEnd
    
                if self.getByte(nextStartOfRecord) != 0x55:
                    raise Corrupted(thisGroupsTickNo, nextStartOfRecord)
    
                if (0xff & msg) == 0x80:
                    mainType = 255 
                    subType = 1 

                if (0xff & msg) == 0xFF:
                    mainType = 255 
                    subType = 2 
    
                if thisGroupsTickNo == -1: # thisTickGroup doesn't yet have a tickNo
                    thisGroupsTickNo = thisRecordTickNo
                    thisTickGroup.tickNo = thisRecordTickNo
    
                if thisRecordTickNo > thisTickGroup.tickNo: # start next group 
                    nextTickGroup.reset()
                    nextTickGroup.add(self.startOfRecord + self.headerLength, 
                                      length - self.headerLength, mainType, subType, msg,thisRecordTickNo); 
                    done = True;
                    
                elif thisRecordTickNo == thisTickGroup.tickNo:
                    thisTickGroup.add(self.startOfRecord + self.headerLength, 
                                      length - self.headerLength, mainType, subType, msg)
                else:#tickNo < thisTickGroup.tickNo) in the last group
                    pass
                    ## for now, just ignore
                self.startOfRecord = nextStartOfRecord;

            except Corrupted as c:
                if self.getPos() > self.fileLength - 600: 
                    raise self._fileEnd

                self.numCorrupted += 1
                if self.numCorrupted > 25:
                    self.results.setResultCode(ResultCode.CORRUPTED);
                    raise Corrupted(thisGroupsTickNo, self.startOfRecord)
    
                try:
                    self.setPosition(c.filePos)
                    fiftyfive = self.readByte()
                    while fiftyfive != 0x55:
                        if self.getPos( ) > self.fileLength - 1000 :
                            raise self._fileEnd
    
                        fiftyfive = self.readByte()
    
                except Corrupted:
                    raise Corrupted(thisGroupsTickNo, self.startOfRecord)
    
                ##set position right before the next 0x55
                self.startOfRecord = self.getPos() -1
    
            except FileEnd:
                raise self._fileEnd
            except Corrupted:
                self.results.setResultCode(ResultCode.CORRUPTED)
                raise Corrupted(thisGroupsTickNo, self.startOfRecord)
    
        return thisTickGroup;

    def close(self): 
        self.file.close()
        self.memory = None

    def skipOver(self, num):
        newpos = self.filePos + num
        if newpos > self.fileLength:
            raise IOError
    
        self.filePos = newpos
    
    def toString(self):
        return self.file.name

    def bufferToString(self):
        rtstr = "%s:%0X : %s :Shrt %s :UShrt %s :I %s :UI %s :L %s :F %s :D %s " % (self.filePos, 
                    (0xff & self.getByte()), (0xff & self.getByte()), str(self.getShort()), 
                    str(self.getUnsignedShort()), str(self.getInt()), str(self.getUnsignedInt()),
                    str(self.getLong()), str(self.getFloat()),str(self.getDouble()))
        return rtstr

    def setPosition(self, pos):
        self.filePos = pos

        if self.filePos >= self.fileLength:
            raise FileEnd()

    def getPos(self):
        return self.filePos

    def getLength(self): 
        return self.fileLength

    def getByte(self, pos = None):
        if pos is None:
            pos = self.filePos
            self.filePos += 1
        if pos >= self.fileLength :
                raise FileEnd()
        ## get the string representation of the byte
        ## create a bits object encodes as hex and convert to little endian int    
        #rtv = Bits(hex=self.memory[pos].encode('hex')).intle
        rtv = int("0x%s" % self.memory[pos].encode('hex'),0)
        return rtv

    def getString(self, fp):
        length = 256
        rbytes = ""
        l = 0
        B = 0x00
        for i in range(length):
            try:
                B = self.getByte(fp + i)

            except FileEnd as e:
                print e
                traceback.print_stack()

            if B == 0x00 or B == "\r" or B == "\n":
                l = i
                break

            rbytes += chr(B)

        return rbytes[0:l]

    def readByte(self):
        rv = self.getByte()
        return rv;

    def  getShort(self):
        rtv = int("0x%s" % self.memory[self.filePos:self.filePos +1].encode('hex'),0)
        return rtv

    def getUnsignedShort(self, fp = None):
        if fp is None:
            fp = self.filePos
        
        if fp > self.fileLength - 2:
                raise FileEnd()
        ## get 2 bites create bites object encoded in hex
        ## return as unsigned int little endian
        #return Bits(hex=self.memory[fp:fp + 1].encode('hex')).uintle
        rtv = int("0x%s" % self.memory[fp:fp + 1].encode('hex'),0)
        return rtv

    def getInt(self):
        #return Bits(hex=self.memory[self.filePos:self.filePos +3].encode('hex')).intle
        rtv = int("0x%s" % self.memory[self.filePos:self.filePos +3].encode('hex'),0)
        return rtv

    def getUnsignedInt(self, fp = None):
        if fp is None:
            fp = self.filePos
        
        if fp > self.fileLength - 4:
                raise FileEnd()
        ## get 4 bites create bites object encoded in hex
        ## return as unsigned int little endian
        return Bits(hex=self.memory[fp:fp + 3].encode('hex')).uintle

    def getLong(self): 
        rtv = int("0x%s" % self.memory[self.filePos:self.filePos +7].encode('hex'),0)
        return rtv

    def getFloat(self):
        return float(Bits(hex=self.memory[self.filePos:self.filePos +3].encode('hex')).floatle)

    def getDouble(self):
        return float(Bits(hex=self.memory[self.filePos:self.filePos +7].encode('hex')).floatle)

    def getResults(self): 
        return self.results;

    def getFile(self):
        return self.file;

    def setStartOfRecord(self, sor):
        self.startOfRecord = sor;

    def fileName(self):
        return self.file.name

    def  findMarkers(self):
        self.findMotorStartEnd()
        self.findLowestTickNo()
        length = 0
        done = False
        nextStartOfRecord = 0
        localFilePos = long(self.fileLength - 200000)
        try:
            self.setPosition(localFilePos)
            fiftyfive = self.readByte()
            while fiftyfive != 0x55:
                if self.getPos() > self.fileLength - 1000:
                    raise self._fileEnd

                fiftyfive = self.readByte()

            ## bad form to use the same name for a local var that is already global
            ## change filePos to localFilePos
            localFilePos = self.getPos() -1
            while done is False:
                try:
                    self.setPosition(localFilePos)
                    length = (0xFF & self.getByte(localFilePos + 1));
                    if length == 0: 
                        raise Corrupted()

                    nextStartOfRecord = localFilePos + length
                    if nextStartOfRecord > self.fileLength:
                        raise self._fileEnd

                    if self.getByte(nextStartOfRecord) != 0x55:
                        raise Corrupted()

                    tickNo = long(self.getUnsignedInt(localFilePos + 6))
                    if self.lowestTickNo < 0:
                        self.lowestTickNo = tickNo

                    self.highestTickNo = tickNo
                    localFilePos = nextStartOfRecord

                except Corrupted:
                    self.setPosition(self.getPos() + 1)
                    fiftyfive = self.readByte()

                    while fiftyfive != 0x55:
                        if self.getPos() > self.fileLength - 1000:
                            raise self._fileEnd

                        fiftyfive = self.readByte()

                    localFilePos = self.getPos() - 1

        except Exception :
            pass

    def  findLowestTickNo(self):
        length = 0
        done = False
        nextStartOfRecord = 0
        localFilePos = 128;

        try:
            self.setPosition(localFilePos)
            fiftyfive = self.readByte()

            while fiftyfive != 0x55:
                if self.getPos() > self.fileLength - 1000 :
                    raise self._fileEnd
                fiftyfive = self.readByte()

            localFilePos = self.getFilePos() - 1

            while done is False:
                try:
                    self.setPosition(localFilePos)
                    length = (0xFF & self.getByte(localFilePos + 1))
                    if length == 0:
                        raise Corrupted()

                    nextStartOfRecord = localFilePos + length

                    if nextStartOfRecord > self.fileLength :
                        raise self._fileEnd

                    if self.getByte(nextStartOfRecord) != 0x55:
                        raise Corrupted()

                    tickNo = long(self.getUnsignedInt(localFilePos + 6))
                    self.lowestTickNo = tickNo
                    done = True

                except Corrupted:
                    self.setPosition(self.getPos() + 1)
                    fiftyfive = self.readByte()

                    while fiftyfive != 0x55:
                        if self.getPos() > self.fileLength - 1000 :
                            raise self._fileEnd

                        fiftyfive = self.readByte()

                    localFilePos = self.getPos() - 1

        except Exception:
            pass

    def findMotorStartEnd(self):
        try:
            self.reset()
            tickNo = 0L

            while True:
                if self.getPos() > self.fileLength - 8 :
                    raise self._fileEnd

                tG = self.getTickGroup()
                tickNo = tG.tickNo
                
                for tgIndex in range(tG.numElements):
                    payloadType =  tG.payloadType[tgIndex]
                    payloadStart = tG.start[tgIndex]
                    payloadLength = tG.length[tgIndex]
                    subType = tG.subType[tgIndex]

                    if payloadType == 255 and subType == 1:
                        xorBB =  Payload(self, payloadStart,payloadLength, payloadType, subType, tickNo)
                        payloadString = xorBB.getString()

                        if payloadString.find("M.Start") > 0 and self.motorStartTick == 0:
                            self.motorStartTick = tickNo

                        if payloadString.find("M. Stop") > 0:
                            self.motorStopTick = tickNo

                    elif self.gpsLockTick == -1  and payloadType == 42 and subType == 12:
                        payload = Payload(self, payloadStart,payloadLength, payloadType, subType, tickNo)
                        longitude = math.degrees(payload.getDouble(0))
                        latitude = math.degrees(payload.getDouble(8))

                        if longitude != 0.0 and latitude != 0.0\
                             and math.fabs(longitude) > 0.0175 and math.fabs(latitude) > 0.0175:
                            self.gpsLockTick = tickNo

                    elif self.flightStartTick == -1L and payloadType == 42 and subType == 12:
                        payload = Payload(self, payloadStart,payloadLength, payloadType, subType, tickNo)

                        if payload.getShort(42) > 0:
                            self.flightStartTick = tickNo - long(float(payload.getShort(42) * 100))
                            
        except FileEnd:
            pass

        except Corrupted: 
            pass

        except IOError:
            pass


if __name__ == "__main__":
    """ this code is intended for testing this module"""
    import time
    import cProfile
    st = time.time()
    # number of files from the list to test
    numtotest = 1
    # the list pf expected files
    testfiles = ['FLY000.DAT','FLY002.DAT','FLY013.DAT','FLY016.DAT']
    testfiles = ['FLY013.DAT','FLY016.DAT']
    # the full path to the data file directory
    datafile_dir = "C:/Users/isaac/workspace/DavidKovar/datcompy/src/data/"
    print "------- testing %d files --------" % numtotest
    for fname in testfiles[0:numtotest]:
        myd = DatFile('%s%s' %(datafile_dir,fname))
        print "processing file",myd.fileName()
        print "file length = ", myd.fileLength
        cProfile.run('myd.findMarkers()')
        print "lowest tick mark =", myd.lowestTickNo
        print "motor start tick =",myd.motorStartTick
        print "flight start tick =", myd.flightStartTick
        myd.close()
        print "-------- end file --------"
        
    print " test complete "
    print "runtime = ", time.time() - st
Created on Aug 27, 2016
 
 @author: <NAME>
 datfile class
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that redistribution of source code include
 the following disclaimer in the documentation and/or other materials provided
 with the distribution.
 
 THIS SOFTWARE IS PROVIDED BY ITS CREATOR "AS IS" AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 DISCLAIMED. IN NO EVENT SHALL THE CREATOR OR CONTRIBUTORS BE LIABLE FOR
 ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. classdocs
     Class for reading byte data Constructor
         fileName full path to data file to be read #     protected boolean alternateStructure = false; _file should be the full path to the data file ## create a bits object with the content of the file ## 8 bits per byte # type in python is a reserved work so adding so type = mainType # the next 0x55 , we hope # thisTickGroup doesn't yet have a tickNo # start next group #tickNo < thisTickGroup.tickNo) in the last group ## for now, just ignore ##set position right before the next 0x55 ## get the string representation of the byte ## create a bits object encodes as hex and convert to little endian int #rtv = Bits(hex=self.memory[pos].encode('hex')).intle ## get 2 bites create bites object encoded in hex ## return as unsigned int little endian #return Bits(hex=self.memory[fp:fp + 1].encode('hex')).uintle #return Bits(hex=self.memory[self.filePos:self.filePos +3].encode('hex')).intle ## get 4 bites create bites object encoded in hex ## return as unsigned int little endian ## bad form to use the same name for a local var that is already global ## change filePos to localFilePos this code is intended for testing this module # number of files from the list to test # the list pf expected files # the full path to the data file directory